├── .gitignore ├── CHANGES.txt ├── LICENSE.txt ├── README ├── build.xml ├── conf └── tables ├── framework ├── ruby │ ├── classes │ │ ├── Condition.rb │ │ ├── Initialize.rb │ │ ├── Query.rb │ │ ├── Util.rb │ │ ├── commands │ │ │ ├── Command.rb │ │ │ └── CommandHistory.rb │ │ ├── common │ │ │ └── String.rb │ │ ├── const │ │ │ ├── CommandConst.rb │ │ │ ├── ConstantsBase.rb │ │ │ ├── DateFormat.rb │ │ │ ├── EncodeConst.rb │ │ │ ├── OSConst.rb │ │ │ ├── OptionConst.rb │ │ │ ├── ProcessMode.rb │ │ │ └── ViewMode.rb │ │ ├── exception │ │ │ └── HBaseClientException.rb │ │ ├── tables │ │ │ └── TableBase.rb │ │ └── viewer │ │ │ └── Viewer.rb │ ├── doc │ │ └── help.txt │ ├── modules │ │ └── Out.rb │ ├── rdocgen.sh │ └── sculptor.rb └── src │ └── sculptor │ └── framework │ ├── FixedSplit.java │ ├── HClassDescriptor.java │ ├── HClient.java │ ├── HCompareOp.java │ ├── HEntity.java │ ├── HFieldDescriptor.java │ ├── HScanner.java │ ├── PutWrapper.java │ ├── Sculptor.java │ ├── annotation │ ├── Column.java │ ├── Rowkey.java │ └── Table.java │ └── util │ └── ByteArray.java ├── ivy.xml ├── sample ├── screenshot │ ├── sculptor-mapreduce.png │ └── sculptor-output.png ├── src │ └── sculptor │ │ └── sample │ │ ├── HItemData.java │ │ └── ItemData.java └── test │ ├── data │ └── sc_item_data.tsv │ └── sculptor │ └── sample │ └── HItemDataTest.java └── sculptor /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.jar 3 | *.class 4 | lib 5 | auxlib 6 | build 7 | framework/lib 8 | framework/.classpath 9 | framework/.project 10 | framework/.settings 11 | framework/javadoc 12 | framework/build 13 | framework/bin 14 | sample/lib 15 | sample/.classpath 16 | sample/.project 17 | sample/.settings 18 | sample/javadoc 19 | sample/build 20 | sample/bin 21 | -------------------------------------------------------------------------------- /CHANGES.txt: -------------------------------------------------------------------------------- 1 | Release 0.0.1 - 2012-03-07 2 | 3 | 1. The first release of Sculptor. 4 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Sculptor is a framework and command line interface that makes using Apache HBase much easier. 2 | 3 | == Features == 4 | - Human friendly output 5 | - Interactive command line interface 6 | - Run ad-hoc MapReduce over HBase 7 | - Customized client framework 8 | 9 | Please see the doc at the wiki page. 10 | 11 | 12 | == Pre-requirements == 13 | - Apache Ivy is required to build Sculptor 14 | - Apache HBase installed 15 | - Hadoop MapReduce (Sculptor MapReduce mode only) 16 | 17 | 18 | == Build == 19 | $ cd sculptor 20 | $ ant 21 | 22 | 23 | == Run Sculptor == 24 | 1. Copy sculptor folder to your HBase client 25 | 26 | 2. Edit $SCULPTOR_HOME/sculptor, change HBASE_HOME variable to fit your environment 27 | 28 | 3. Initialize the sample 29 | $ sh $SCULPTOR_HOME/sculptor sample 30 | 31 | 4. Start Sculptor 32 | $ sh $SCULPTOR_HOME/sculptor 33 | sculptor> get from sc_item_data item_id>100 limit=1 34 | sculptor> help 35 | 36 | 5. Use Sculptor for your own HBase tables 37 | Extend HEntity to create an entity class to represent your table 38 | Extend HClient, implement the abstract methods to handle your table access 39 | Package your classes, drop your jar under $SCULPTOR_HOME/auxlib directory 40 | Add one line to $SCULPTOR_HOME/conf/tables about your table 41 | See $SCULPTOR_HOME/sample for details 42 | 43 | 44 | == Sculptor MapReduce == 45 | 1. For Sculptor MapReduce mode only, get your environment ready to run customized MapReduce jobs over HBase. 46 | This usually requires the following steps: 47 | - Add HBase and ZooKeeper jars to Hadoop classpath by: 48 | $ echo "export HADOOP_CLASSPATH=$HBASE_HOME/hbase-0.9x.jar:$HBASE_HOME/lib/zookeeper-3.x.y.jar" >> hadoop-env.sh 49 | - Let Hadoop know where HBase is running 50 | $ ln -s $HBASE_HOME/conf/hbase-site.xml $HADOOP_HOME/conf/hbase-site.xml 51 | - Add your jar file (e.g. auxlib/sculptor-sample.jar) to Hadoop classpath. 52 | The easiest way to do this is to drop your jar file to $HADOOP_HOME/lib directory. 53 | - Sync the above changes across the cluster 54 | - Restart Hadoop MapReduce 55 | 56 | 2. Run ad-hoc MapReduce via Sculptor 57 | sculptor> set process mapreduce 58 | sculptor> get from sc_item_data item_id>100 59 | -------------------------------------------------------------------------------- /build.xml: -------------------------------------------------------------------------------- 1 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 28 | 29 | 30 | 31 | 32 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 63 | 64 | 65 | 66 | 67 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 86 | 87 | 88 | 89 | 90 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 109 | 110 | 111 | 112 | 113 | 114 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | -------------------------------------------------------------------------------- /conf/tables: -------------------------------------------------------------------------------- 1 | # Copyright 2010 The Apache Software Foundation 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # Set table, entity and hclient mapping here. 21 | # Format is: table_name : entity_class : client_class 22 | sc_item_data : sculptor.sample.ItemData : sculptor.sample.HItemData 23 | -------------------------------------------------------------------------------- /framework/ruby/classes/Condition.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | # 22 | #=検索条件を定義する 23 | # 24 | # 25 | class Condition 26 | #対象のカラム名 27 | attr_accessor :columnName; 28 | #関係演算子 29 | attr_accessor :sign; 30 | #値 31 | attr_accessor :value; 32 | # 33 | #===関係演算子をセットする 34 | # 35 | #==== args 36 | #signEnum :: sculptor.framework.HCompareOp 37 | def setSign(signEnum) 38 | @sign = nil; 39 | case signEnum 40 | when "<" 41 | @sign = HCompareOp::LESS; 42 | when "<=" 43 | @sign = HCompareOp::LESS_OR_EQUAL; 44 | when "=" 45 | @sign = HCompareOp::EQUAL; 46 | when ">=" 47 | @sign = HCompareOp::GREATER_OR_EQUAL; 48 | when ">" 49 | @sign = HCompareOp::GREATER; 50 | end 51 | end 52 | 53 | # 54 | #===文字列から関係演算子Enumを取得 55 | # 56 | #==== args 57 | #sign :: 関係演算子を表す文字列(<|<=|=|>=|>) 58 | #==== return 59 | #signEnum :: sculptor.framework.HCompareOp 60 | def self.getEnum(sign) 61 | sign = nil; 62 | case sign 63 | when "<" 64 | sign = HCompareOp::LESS; 65 | when "<=" 66 | sign = HCompareOp::LESS_OR_EQUAL; 67 | when "=" 68 | sign = HCompareOp::EQUAL; 69 | when ">=" 70 | sign = HCompareOp::GREATER_OR_EQUAL; 71 | when ">" 72 | sign = HCompareOp::GREATER; 73 | end 74 | 75 | return sign; 76 | end 77 | 78 | # 79 | #===2値を比較する 80 | # 81 | #==== args 82 | #value1 :: 値 83 | #value2 :: 比較したい値 84 | #signEnum :: sculptor.framework.HCompareOp 85 | #==== return 86 | #比較結果が正ならtrue 87 | def self.compare(value1, value2, sign) 88 | ret = false; 89 | 90 | case sign 91 | when HCompareOp::LESS 92 | ret = value1.to_i < value2.to_i; 93 | when HCompareOp::LESS_OR_EQUAL 94 | ret = value1.to_i <= value2.to_i; 95 | when HCompareOp::EQUAL 96 | ret = value1 == value2; 97 | when HCompareOp::GREATER_OR_EQUAL 98 | ret = value1.to_i >= value2.to_i; 99 | when HCompareOp::GREATER 100 | ret = value1.to_i < value2.to_i; 101 | end 102 | 103 | return ret; 104 | end 105 | end 106 | -------------------------------------------------------------------------------- /framework/ruby/classes/Initialize.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | require 'java' 22 | require 'pp' 23 | 24 | #libディレクトリのパス 25 | LIB_DIR = $SCULPTOR_ROOT + "/lib/"; 26 | #classesディレクトリのパス 27 | CLAASES_DIR = $basePath + "/classes/"; 28 | #constディレクトリのパス 29 | CONST_DIR = CLAASES_DIR + "const/"; 30 | #modulesディレクトリのパス 31 | MODULES_DIR = $basePath + "/modules/"; 32 | #docディレクトリのパス 33 | DOC_DIR = $basePath + "/doc/"; 34 | 35 | #apacheパッケージ 36 | APACHE_PACKAGE = "org.apache."; 37 | 38 | require CLAASES_DIR + 'common/String.rb' 39 | require CLAASES_DIR + 'commands/CommandHistory.rb'; 40 | 41 | require CLAASES_DIR + 'Util.rb' 42 | require CLAASES_DIR + 'Query.rb' 43 | require CONST_DIR + 'ViewMode.rb' 44 | require CONST_DIR + 'ProcessMode.rb' 45 | require CONST_DIR + 'CommandConst.rb' 46 | require CONST_DIR + 'OptionConst.rb' 47 | require CONST_DIR + 'EncodeConst.rb' 48 | require CONST_DIR + 'DateFormat.rb' 49 | require CLAASES_DIR + 'exception/HBaseClientException.rb' 50 | require CLAASES_DIR + 'commands/Command.rb' 51 | require CLAASES_DIR + 'tables/TableBase.rb' 52 | require MODULES_DIR + 'Out.rb' 53 | 54 | include Out; 55 | 56 | eputs $title; 57 | 58 | import 'sculptor.framework.HCompareOp' 59 | import 'sculptor.framework.Sculptor' 60 | 61 | #できるだけグローバルなんは避けたいけど、とりあえず 62 | # 表示モード。デフォルトはline 63 | $view_mode = ViewMode::LINE; 64 | # 処理モード。デフォルトはnomal 65 | $process_mode = ProcessMode::NORMAL; 66 | 67 | log_level = org.apache.log4j.Level::ERROR 68 | org.apache.log4j.Logger.getLogger("org.apache.zookeeper").setLevel(log_level) 69 | org.apache.log4j.Logger.getLogger("org.apache.hadoop").setLevel(log_level) 70 | # org.apache.log4j.Logger.getLogger("sculptor").setLevel(log_level) 71 | 72 | $tableNameList = []; 73 | $fieldNameList = []; 74 | $fieldNameHash = Hash.new; 75 | 76 | # Initialize table -> entity -> client mapping 77 | Sculptor.initialize($SCULPTOR_ROOT) 78 | 79 | # Initialize tables from entities 80 | for table in Sculptor.descriptors.keySet() 81 | eputs "Loading " + table + "..."; 82 | $tableNameList << table; 83 | descriptor = Sculptor.descriptors.get(table); 84 | tableClassFieldInfoList = descriptor.gethFieldDescriptors(); 85 | tableClassFieldInfoList.each do |tableClassFieldInfo| 86 | tableClassFieldName = tableClassFieldInfo.getQualifier(); 87 | $fieldNameList << tableClassFieldName; 88 | end 89 | $fieldNameHash[table] = $fieldNameList; 90 | end 91 | 92 | $tableNameList = $tableNameList.uniq(); 93 | -------------------------------------------------------------------------------- /framework/ruby/classes/Query.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | require CLAASES_DIR + "Condition.rb"; 22 | 23 | # 24 | #=HBaseClientのクエリを定義する 25 | # 26 | # 27 | class Query 28 | 29 | ERROR_MSG_TEMPLATE = "%s: %sが不正です"; 30 | ERROR_NOT_EXISTS_MSG_TEMPLATE = "%s: %sは存在しません"; 31 | 32 | #コマンド 33 | attr_accessor :operation; 34 | #表示カラム名一覧 35 | attr_accessor :showColumnArray; 36 | #from句 37 | attr_accessor :from; 38 | #テーブル名 39 | attr_accessor :tableName; 40 | #検索条件(Conditionオブジェクト)の配列 41 | attr_accessor :conditionArray; 42 | #第二コマンド 43 | attr_accessor :mOperation; 44 | #オフセット 45 | attr_accessor :offset; 46 | #リミット 47 | attr_accessor :limit; 48 | 49 | public 50 | # 51 | #===クエリをパースし、自分自身にセットする 52 | # 53 | #==== args 54 | #query :: クエリ文字列 55 | def parse(query) 56 | 57 | @showColumnArray = []; 58 | @conditionArray = []; 59 | 60 | # ぐーるぐる 61 | Query.splitRecord(query).each_with_index do |value, index| 62 | if (value != nil) 63 | if (index == 0) 64 | # 最初のトークン 65 | #operation 66 | @operation = value.downcase; 67 | else 68 | if (@operation == CommandConst::SET) 69 | if (index == 1) 70 | @mOperation = value.downcase; 71 | else 72 | if (@mOperation == "view") 73 | # viewMode切り替え 74 | if (ViewMode.isDefined(value.downcase)) 75 | $view_mode = value.downcase; 76 | eputs "view mode : " + $view_mode; 77 | else 78 | errMsg = sprintf(ERROR_MSG_TEMPLATE, "表示モード", value.downcase); 79 | raise HBaseClientException.new(errMsg); 80 | end 81 | elsif (@mOperation == "process") 82 | # processMode切り替え 83 | if (ProcessMode.isDefined(value.downcase)) 84 | $process_mode = value.downcase; 85 | eputs "process mode : " + $process_mode; 86 | else 87 | errMsg = sprintf(ERROR_MSG_TEMPLATE, "処理モード", value.downcase); 88 | raise HBaseClientException.new(errMsg); 89 | end 90 | end 91 | end 92 | elsif (@operation == CommandConst::PUT) 93 | # putコマンドの場合は表示カラム名とfrom句がない 94 | if (index == 1) 95 | # 2番目のトークン 96 | #tableName 97 | @tableName = value.downcase; 98 | else 99 | # テーブル名以降は「カラム名=値」 100 | #condition 101 | condition = parseCondition(value); 102 | unless (condition == nil) 103 | @conditionArray << condition; 104 | end 105 | end 106 | else 107 | if (@from == nil) 108 | if (value.downcase == "from") 109 | #from 110 | @from = value.downcase; 111 | else 112 | #showColumn 113 | @showColumnArray << value.downcase; 114 | end 115 | else 116 | # from句以降 117 | if (@tableName == nil) 118 | # from句の次はテーブル名 119 | #tableName 120 | @tableName = value.downcase; 121 | else 122 | # テーブル名以降は検索条件 123 | #condition 124 | condition = parseCondition(value); 125 | unless (condition == nil) 126 | @conditionArray << condition; 127 | end 128 | end 129 | end 130 | end 131 | end 132 | end 133 | end 134 | 135 | @offset = String::nvl(@offset, 0); 136 | @limit = String::nvl(@limit, -1); 137 | 138 | # カラム名チェック 139 | Query.checkParam(self); 140 | end 141 | 142 | public 143 | 144 | # 145 | #===検索条件をパースし、Conditionオブジェクトにセット 146 | # 147 | #==== args 148 | #query :: クエリ文字列 149 | #==== return 150 | #Conditionオブジェクト 151 | def parseCondition(query) 152 | 153 | condition = nil; 154 | arr = query.scan(/(\S+?)([=|<|>|\s]+)(.+)/);# TODO ここの正規表現なおす 155 | 156 | unless (arr == nil || arr == []) 157 | columnName = arr[0][0].strip(); 158 | sign = arr[0][1].strip(); 159 | value = arr[0][2].strip(); 160 | 161 | # p value; 162 | # if (value.startsWith("\"") && value.endsWith("\"")) 163 | # value = value.slice(1..value.length - 2); 164 | ## value = value.gsub("\"", "\\\\\\\""); 165 | # end 166 | # p value; 167 | if (columnName == OptionConst::OFFSET) 168 | @offset = value != nil ? value.to_i : nil; 169 | elsif (columnName == OptionConst::LIMIT) 170 | @limit = value != nil ? value.to_i : nil; 171 | else 172 | condition = Condition.new(); 173 | condition.columnName = columnName; 174 | condition.setSign(sign); 175 | condition.value = value; 176 | 177 | if (condition.columnName() == nil || condition.sign() == nil || condition.value() == nil) 178 | errMsg = sprintf(ERROR_MSG_TEMPLATE, "値", query); 179 | raise HBaseClientException.new(errMsg); 180 | end 181 | end 182 | else 183 | errMsg = sprintf(ERROR_MSG_TEMPLATE, "値", query); 184 | raise HBaseClientException.new(errMsg); 185 | end 186 | 187 | return condition; 188 | end 189 | 190 | private 191 | 192 | # 193 | #===クエリの各パラメータをチェックする 194 | # 195 | #==== args 196 | #query Queryオブジェクト 197 | #==== raise 198 | #HBaseClientException 199 | def self.checkParam(query) 200 | 201 | # オペレーションのチェック 202 | operation = query.operation(); 203 | self.checkCommand!(operation); 204 | 205 | # テーブル名のチェック 206 | tableName = query.tableName(); 207 | self.checkTableName(tableName); 208 | 209 | # 表示カラム名のチェック 210 | query.showColumnArray().each do |fieldName| 211 | self.checkColumnName(tableName, fieldName); 212 | end 213 | 214 | if (CommandConst.isIndispensableConditionCommand(operation) && query.conditionArray() == []) 215 | errMsg = operation + "コマンドには条件を必ず指定してください"; 216 | raise HBaseClientException.new(errMsg); 217 | end 218 | 219 | # 検索条件のカラム名のチェック 220 | query.conditionArray().each do |condition| 221 | self.checkColumnName(tableName, condition.columnName()); 222 | self.checkSyntax(condition.value()); 223 | end 224 | end 225 | 226 | private 227 | 228 | # 229 | #===コマンドが定義されたものかどうかチェックする 230 | #※破壊メソッド 231 | # 232 | #==== args 233 | #command :: コマンド文字列 234 | def self.checkCommand!(command) 235 | if (command != nil && !CommandConst.isDefined(command)) 236 | #raise HBaseClientException.new("コマンド: " + command + "が不正です"); 237 | command = CommandConst::HELP;; 238 | end 239 | end 240 | 241 | private 242 | 243 | # 244 | #===テーブルが存在するものかどうかチェックする 245 | # 246 | #==== args 247 | #tableName :: テーブル名 248 | #==== raise 249 | #HBaseClientException :: 指定されたテーブルが存在しない場合 250 | def self.checkTableName(tableName) 251 | if (tableName != nil && !$tableNameList.include?(tableName)) 252 | errMsg = sprintf(ERROR_NOT_EXISTS_MSG_TEMPLATE, "テーブル", tableName); 253 | raise HBaseClientException.new(errMsg); 254 | end 255 | end 256 | 257 | # 258 | #===カラムが指定されたテーブルに存在するものかどうかチェックする 259 | # 260 | #==== args 261 | #tableName :: テーブル名 262 | #fieldName :: フィールド名 263 | #==== raise 264 | #HBaseClientException :: 指定されたカラムがテーブルに存在しない場合 265 | def self.checkColumnName(tableName, fieldName) 266 | unless (fieldName == OptionConst::OFFSET || fieldName == OptionConst::LIMIT) 267 | if (tableName != nil && fieldName != nil && !$fieldNameHash[tableName].include?(fieldName)) 268 | errMsg = sprintf(ERROR_NOT_EXISTS_MSG_TEMPLATE, "カラム", tableName + "." + fieldName); 269 | raise HBaseClientException.new(errMsg); 270 | end 271 | end 272 | end 273 | 274 | # 275 | #===入力値のSyntaxをチェックする 276 | # 277 | #==== args 278 | #value :: 文字列 279 | #==== raise 280 | #HBaseClientException 281 | def self.checkSyntax(value) 282 | test = nil; 283 | begin 284 | # evalに食わせてSyntaxErrorがないかチェックする 285 | eval("test = \"" + value.to_s + "\";"); 286 | rescue SyntaxError => e 287 | begin 288 | eval("test = " + value.to_s + ";"); 289 | rescue SyntaxError => e 290 | errMsg = sprintf(ERROR_MSG_TEMPLATE, "値", value); 291 | raise HBaseClientException.new(errMsg); 292 | end 293 | end 294 | end 295 | 296 | # 297 | #===クエリ文字列を分解する 298 | # 299 | #==== args 300 | #message :: クエリ文字列 301 | #==== return 302 | #分解されたクエリ文字列の配列 303 | def self.splitRecord(message) 304 | 305 | records = []; 306 | 307 | quotStart = false; 308 | msg = ""; 309 | 310 | words = message.split(/\s/); 311 | 312 | words.each_with_index do |value, index| 313 | isPut = false; 314 | 315 | msg += value; 316 | 317 | if (quotStart)# ダブルクォートの中 318 | if (value.index("\"") != nil)# ダブルクォートが見つかった 319 | isPut = true; 320 | quotStart = false; 321 | else 322 | msg += " "; 323 | end 324 | else 325 | if (value.index("\"") != nil)# ダブルクォートが見つかった 326 | # ↓ダブルクォートのなか 327 | if (value.index("\"", value.index("\"") + 1) != nil)# 二つ目のダブルクォートがみつかった 328 | # ダブルクォートの外 329 | isPut = true; 330 | quotStart = false; 331 | else 332 | # ダブルクォートのなか 333 | # splitで半角スペース一個なくなっちゃってるから足す 334 | msg += " "; 335 | quotStart = true; 336 | end 337 | else 338 | if (msg != "")# ダブルクォートの外の半角スペースはひとつにまとめる 339 | isPut = true; 340 | else 341 | msg = ""; 342 | end 343 | end 344 | end 345 | 346 | # 最後の要素は無条件で追加 347 | if (isPut || index + 1 == words.length()) 348 | records << msg; 349 | msg = ""; 350 | end 351 | end 352 | 353 | return records; 354 | end 355 | end 356 | 357 | -------------------------------------------------------------------------------- /framework/ruby/classes/Util.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | # 22 | #=ユーティリティクラス 23 | #雑多に 24 | # 25 | # 26 | class Util 27 | # 28 | #===コンテンツアシストにに表示するワード一覧を取得 29 | # 30 | #====return 31 | #コンテンツアシストにに表示するワード一覧 32 | def self.getCompletionWords 33 | words = [ 34 | "view", 35 | "process", 36 | "from", 37 | "offset", 38 | "limit" 39 | ]; 40 | return words 41 | end 42 | 43 | # 44 | #===Javaのフィールド名をrubyにあわせて変換 45 | #キャメルでケースで定義されたフィールド名をスネークに変換 46 | #その際にイレギュラーにも対応する 47 | # 48 | #==== args 49 | #fName :: Javaのフィールド名 50 | #==== return 51 | #rubyのフィールド名 52 | def self.getFieldName(fName) 53 | fieldName = fName.toSnake(); 54 | fieldName = fieldName.sub("_i_d", "_id"); 55 | 56 | return fieldName; 57 | end 58 | 59 | # 60 | #===文字列からjava.util.Dateオブジェクトを生成 61 | # 62 | #==== args 63 | #yyyymmdd :: 日付文字列 64 | #==== return 65 | #指定された日付のjava.util.Dateオブジェクト 66 | def self.createJavaUtilDate(yyyymmdd) 67 | import 'java.util.Date'; 68 | import 'java.text.SimpleDateFormat'; 69 | 70 | date = nil; 71 | 72 | if (yyyymmdd != nil) 73 | yyyymmdd = yyyymmdd.to_s; 74 | 75 | msg = ""; 76 | count = 0; 77 | 78 | df = DateFormat.new; 79 | while (dateFormat = df.next()) 80 | 81 | if (count > 0) 82 | msg += ", "; 83 | end 84 | msg += dateFormat; 85 | 86 | begin 87 | sdf = SimpleDateFormat.new(dateFormat); 88 | date = sdf.parse(yyyymmdd); 89 | rescue NativeException 90 | else 91 | return date; 92 | end 93 | count = count + 1; 94 | end 95 | raise HBaseClientException.new("日付は以下のフォーマットで入力してください。" + msg); 96 | end 97 | end 98 | 99 | # 100 | #===java.util.Dateを指定したフォーマットの文字列に変換 101 | # 102 | #==== args 103 | #value :: java.util.Dateオブジェクト 104 | #format :: フォーマット文字列 105 | #==== return 106 | #指定したフォーマットの日付文字列 107 | def self.JavaDate2String(value, format) 108 | import 'java.util.Date'; 109 | import 'java.text.SimpleDateFormat'; 110 | 111 | date = SimpleDateFormat.new(format).format(value); 112 | 113 | return date; 114 | end 115 | 116 | # 117 | #===FixNum変換(nilチェックつき) 118 | # 119 | #==== args 120 | #value :: 数値文字列 121 | #==== return 122 | #FixNumに変換された文字列。文字列がnilの場合はnil 123 | def self.to_i_with_nil_check(value) 124 | ret = nil; 125 | if (value != nil) 126 | ret = value.to_i; 127 | end 128 | return ret; 129 | end 130 | 131 | # 132 | #===配列の要素の最大長を取得 133 | # 134 | #==== args 135 | #array :: 配列 136 | #==== return 137 | #配列内の要素の最大長 138 | def self.arrayMaxLength(array) 139 | maxLength = 0; 140 | array.each do |value| 141 | length = value.length; 142 | if (length > maxLength) 143 | maxLength = length; 144 | end 145 | end 146 | return maxLength; 147 | end 148 | 149 | # 150 | #===rubyオブジェクトをJavaオブジェクトに変換 151 | # 152 | #==== args 153 | #value :: 値 154 | #className :: Javaクラス名 155 | #==== return 156 | #Javaオブジェクトにキャストされた値 157 | def self.toJavaClass(value, className) 158 | ret = nil; 159 | 160 | case className 161 | when "java.util.Date" 162 | ret = self.createJavaUtilDate(value); 163 | when "int" 164 | ret = self.to_i_with_nil_check(value); 165 | when "java.lang.String" 166 | ret = value.to_s; 167 | when "byte" 168 | ret = self.to_i_with_nil_check(value); 169 | end 170 | return ret; 171 | end 172 | 173 | # 174 | #===フィールドの型を取得 175 | # 176 | #==== args 177 | #fieldInfoList :: 検索対象のフィールド一覧 178 | #fieldName :: フィールド名 179 | #==== return 180 | #指定したフィールドの型名 181 | def self.getType(fieldInfoList, fieldName) 182 | className = nil; 183 | fieldInfoList.each do |fieldInfo| 184 | if (fieldInfo.fieldName() == fieldName) 185 | className = fieldInfo.canonicalName(); 186 | end 187 | end 188 | return className; 189 | end 190 | end 191 | -------------------------------------------------------------------------------- /framework/ruby/classes/commands/Command.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | # 22 | #=HBaseClientのコマンドを定義する 23 | # 24 | # 25 | class Command 26 | # 27 | #===コマンドを実行する 28 | # 29 | #==== args 30 | #conf :: org.apache.hadoop.conf.Configuration 31 | #query :: Query 32 | #tableName :: テーブル名 33 | def execute(query, tableName) 34 | clazz = TableBase.new(tableName, query); 35 | 36 | operation = query.operation(); 37 | 38 | begin 39 | # oprationに応じたメソッドを呼ぶ 40 | eval("clazz." + operation + "();"); 41 | rescue NoMethodError => e 42 | # 不正なコマンドの場合、helpファイルを表示することにしたので、エラー表示はなし 43 | # raise HBaseClientException.new("コマンド: " + operation + "が不正です"); 44 | rescue SyntaxError => e 45 | # すでにチェックを通過してきているので実装のミスがないとありえない 46 | raise HBaseClientException.new("コマンド: " + operation + "が不正です"); 47 | end 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /framework/ruby/classes/commands/CommandHistory.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | require 'readline' 22 | require 'pathname' 23 | 24 | # 25 | #=コマンド履歴を定義する 26 | # 27 | # 28 | class CommandHistory 29 | 30 | #コマンド履歴を保存するファイル 31 | COMMAND_HISTORY_FILE = ".readline.history"; 32 | #コマンド履歴ファイルのディレクトリ 33 | COMMAND_HISTORY_DIR = ".sculptor/"; 34 | # 35 | #===Readlineを初期化する 36 | # 37 | #==== args 38 | #*list :: (可変長)コマンド補完リストに追加する文字列の配列 39 | def self.init(*lists) 40 | # コンソールの補完リスト 41 | words = Util.getCompletionWords(); 42 | # コマンド一覧 43 | words = words.concat(CommandConst.getValues()); 44 | # オプション一覧 45 | words = words.concat(OptionConst.getValues()); 46 | # 表示モード一覧 47 | words = words.concat(ViewMode.getValues()); 48 | # 処理モード一覧 49 | words = words.concat(ProcessMode.getValues()); 50 | 51 | lists.each do |list| 52 | words = words.concat(list); 53 | end 54 | 55 | words = words.uniq(); 56 | 57 | Readline.completion_proc = proc {|word| 58 | words.grep(/\A#{Regexp.quote word}/) 59 | } 60 | end 61 | 62 | # 63 | #===コンソールからユーザの入力を読み込む 64 | #==== return 65 | #ユーザの入力 66 | def self.readCommand() 67 | return Readline.readline("sculptor> ", true); 68 | end 69 | 70 | # 71 | #===コマンド履歴をファイルから読み込む 72 | # 73 | #==== args 74 | #userName :: ログインユーザ名 75 | def self.readHistoryFile(userName) 76 | 77 | userHomeDir = self.getHomeDir(userName); 78 | userCommandHistoryFile = userHomeDir + COMMAND_HISTORY_DIR + COMMAND_HISTORY_FILE; 79 | 80 | if (File.exist?(userCommandHistoryFile)) 81 | eputs "Loading command history..."; 82 | commandHistoryArray = []; 83 | Pathname.new(userCommandHistoryFile).open("rb") do |f| 84 | commandHistoryArray = Marshal.load(f); 85 | end 86 | 87 | commandHistoryArray.each do |commandHistory| 88 | Readline::HISTORY.push(commandHistory); 89 | end 90 | end 91 | end 92 | 93 | # 94 | #===コマンド履歴をファイルに書きこむ 95 | # 96 | #==== args 97 | #userName :: ログインユーザ名 98 | def self.writeCommandHistory(userName) 99 | 100 | userHomeDir = self.getHomeDir(userName); 101 | userCommandHitoryDir = userHomeDir + COMMAND_HISTORY_DIR; 102 | if (!FileTest::directory?(userCommandHitoryDir)) 103 | Dir::mkdir(userCommandHitoryDir, 0777); 104 | end 105 | 106 | userCommandHistoryFile = userCommandHitoryDir + COMMAND_HISTORY_FILE; 107 | 108 | Pathname.new(userCommandHistoryFile).open("wb") do |f| 109 | Marshal.dump(Readline::HISTORY.to_a, f, 100); 110 | end 111 | end 112 | 113 | # 114 | #===最後に入力されたコマンドを履歴から削除する 115 | # 116 | def self.ignore() 117 | Readline::HISTORY.pop; 118 | end 119 | 120 | private 121 | 122 | # 123 | #===ユーザのホームディレクトリを取得する 124 | # 125 | #==== args 126 | #userName :: ログインユーザ名 127 | def self.getHomeDir(userName) 128 | return "/home/" + userName + "/"; 129 | end 130 | end 131 | -------------------------------------------------------------------------------- /framework/ruby/classes/common/String.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | # 22 | #=Stringクラスの拡張クラス 23 | # 24 | # 25 | class String 26 | # 27 | #===文字列が指定した文字列から始まっているかチェック 28 | #要はJavaのstartsWithです 29 | # 30 | #==== args 31 | #args :: 文字列 32 | #==== return 33 | #文字列が指定した文字列から始まっていればtrue 34 | def startsWith(*args) 35 | retult = false 36 | for arg in args 37 | result |= self[0, arg.length] == arg 38 | break if result 39 | end 40 | result 41 | end 42 | 43 | # 44 | #===文字列が指定した文字列で終わっているかチェック 45 | #要はJavaのendsWithです 46 | # 47 | #==== args 48 | #args :: 文字列 49 | #==== return 50 | #文字列が指定した文字列で終わっていればtrue 51 | def endsWith(*args) 52 | retult = false 53 | for arg in args 54 | result |= self[-arg.length, arg.length] == arg 55 | break if result 56 | end 57 | result 58 | end 59 | 60 | # 61 | #===キャメルケースをスネークに変換 62 | # 63 | def toSnake() 64 | return self.split(/(?![a-z])(?=[A-Z])/).map{|s| s.downcase}.join('_') 65 | end 66 | 67 | # 68 | #===スネークをキャメルケースに変換 69 | # 70 | def toCamel() 71 | return self.split('_').map{|s| s.capitalize}.join('') 72 | end 73 | 74 | def encode(encode) 75 | require 'kconv' 76 | 77 | return self.kconv(encode, Kconv::UTF8); 78 | end 79 | 80 | def self.nvl(value, param) 81 | return value == nil ? param : value; 82 | end 83 | end 84 | -------------------------------------------------------------------------------- /framework/ruby/classes/const/CommandConst.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | # 21 | require CONST_DIR + "ConstantsBase.rb" 22 | 23 | # 24 | #=コマンドを定義する 25 | # 26 | # 27 | class CommandConst < ConstantsBase 28 | #コマンド :: set 29 | SET = "set"; 30 | #コマンド :: get 31 | GET = "get"; 32 | #コマンド :: put 33 | PUT = "put"; 34 | #コマンド :: delete 35 | DELETE = "delete"; 36 | #コマンド :: count 37 | COUNT = "count"; 38 | #コマンド :: help 39 | HELP = "help"; 40 | #コマンド :: exit 41 | EXIT = "exit"; 42 | # 43 | #===条件が必須なコマンド一覧を取得 44 | # 45 | #==== retuen 46 | #条件が必須なコマンド一覧 47 | def self.getIndispensableConditionCommand() 48 | return [self::PUT, self::DELETE]; 49 | end 50 | 51 | # 52 | #===指定したコマンドが条件必須かどうかを返す 53 | # 54 | #==== args 55 | #command :: コマンド 56 | #==== return 57 | #指定したコマンドが条件必須であればtrue、必須でなければfalse 58 | def self.isIndispensableConditionCommand(command) 59 | ret = false; 60 | self.getIndispensableConditionCommand().each do |iCommand| 61 | ret |= iCommand == command; 62 | end 63 | return ret; 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /framework/ruby/classes/const/ConstantsBase.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | # 21 | 22 | # 23 | #=定数のベースクラス 24 | # 25 | # 26 | class ConstantsBase 27 | # 28 | #===指定した値が定義されているかどうかを返す 29 | # 30 | #==== args 31 | #value :: 値 32 | #==== return 33 | #指定した値が定義されていればtrue 34 | def self.isDefined(value) 35 | ret = false; 36 | value.gsub!(/"/, "\\\""); 37 | self.constants.each do |constant| 38 | eval("ret |= \"" + value.to_s + "\" == " + self.to_s + "::" + constant + ".to_s"); 39 | end 40 | return ret; 41 | end 42 | 43 | # 44 | #===定数の値一覧を返す 45 | # 46 | #==== return 47 | #定数の値一覧 48 | def self.getValues() 49 | ret = []; 50 | 51 | self.constants.each do |constant| 52 | eval("ret << " + self.to_s + "::" + constant + ".to_s"); 53 | end 54 | 55 | return ret; 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /framework/ruby/classes/const/DateFormat.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | require CONST_DIR + "ConstantsBase.rb" 22 | 23 | # 24 | #=日付フォーマットを定義する 25 | # 26 | # 27 | class DateFormat < ConstantsBase 28 | FORMAT1 = "yyyy/MM/dd"; 29 | FORMAT2 = "yyyyMMdd"; 30 | FORMAT3 = "yyyy-MM-dd"; 31 | 32 | # 33 | #===コンストラクタ 34 | # 35 | def initialize() 36 | @index = 0; 37 | @formatArray = []; 38 | DateFormat.constants.each do |constant| 39 | value = nil; 40 | eval("value = DateFormat::" + constant + ";"); 41 | @formatArray << value; 42 | end 43 | end 44 | 45 | # 46 | #===次のindexの値を返す 47 | #indexの範囲を超えるとnil 48 | # 49 | #==== return 50 | #日付フォーマットを表す文字列 51 | def next() 52 | format = nil; 53 | 54 | begin 55 | format = @formatArray[@index]; 56 | @index = @index + 1; 57 | rescue 58 | format = nil; 59 | end 60 | 61 | return format; 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /framework/ruby/classes/const/EncodeConst.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | require CONST_DIR + "ConstantsBase.rb" 22 | require 'kconv' 23 | 24 | # 25 | #=オプションを定義する 26 | # 27 | # 28 | class EncodeConst < ConstantsBase 29 | #shift_jis 30 | SJIS = Kconv::SJIS; 31 | #utf-8 32 | UTF8 = Kconv::UTF8; 33 | end 34 | -------------------------------------------------------------------------------- /framework/ruby/classes/const/OSConst.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | require CONST_DIR + "ConstantsBase.rb" 22 | 23 | import APACHE_PACKAGE + 'commons.lang.SystemUtils'; 24 | 25 | # 26 | #=OSの種類を定義する 27 | # 28 | # 29 | class OSConst < ConstantsBase 30 | #windows XP 31 | WINDOWS_XP = "windows"; 32 | #Linux 33 | LINUX = "linux"; 34 | # 35 | #===実行環境のOS名を取得する 36 | # 37 | #==== return 38 | #OS名 39 | def self.getOsName() 40 | return SystemUtils::OS_NAME.downcase; 41 | end 42 | 43 | # 44 | #===Windowsかどうか 45 | # 46 | #==== args 47 | #os :: OSConst 48 | #==== return 49 | #windowsならtrue 50 | def self.isWindows(os) 51 | return os.downcase.startsWith(OSConst::WINDOWS_XP); 52 | end 53 | 54 | # 55 | #===Linuxかどうか 56 | # 57 | #==== args 58 | #os :: OSConst 59 | #==== return 60 | #linuxならtrue 61 | def self.isLunux(os) 62 | return os.downcase == OSConst::LINUX; 63 | end 64 | 65 | # 66 | #===OSのデフォルトエンコードを取得 67 | # TODO ちゃんとつくり直す 68 | # 69 | #==== args 70 | #os :: OSConst 71 | #==== return 72 | #エンコード 73 | def self.getDefaultEncode(os) 74 | encode = EncodeConst::UTF8; 75 | if (OSConst::isWindows(os)) 76 | encode = EncodeConst::SJIS; 77 | end 78 | return encode; 79 | end 80 | end 81 | -------------------------------------------------------------------------------- /framework/ruby/classes/const/OptionConst.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | require CONST_DIR + "ConstantsBase.rb" 22 | 23 | # 24 | #=オプションを定義する 25 | # 26 | # 27 | class OptionConst < ConstantsBase 28 | #offset 29 | OFFSET = "offset"; 30 | #limit 31 | LIMIT = "limit"; 32 | end 33 | -------------------------------------------------------------------------------- /framework/ruby/classes/const/ProcessMode.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | require CONST_DIR + "ConstantsBase.rb" 22 | 23 | # 24 | #=処理モードを定義する 25 | # 26 | # 27 | class ProcessMode < ConstantsBase 28 | #処理モード :: 通常 29 | NORMAL = "normal"; 30 | #処理モード :: MapReduce 31 | MAP_REDUCE = "mapreduce"; 32 | end 33 | -------------------------------------------------------------------------------- /framework/ruby/classes/const/ViewMode.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | require CONST_DIR + "ConstantsBase.rb" 22 | 23 | # 24 | #=表示モードを定義する 25 | # 26 | # 27 | class ViewMode < ConstantsBase 28 | #表示モード :: ライン 29 | LINE = "line"; 30 | #表示モード :: テーブル 31 | TABLE = "table"; 32 | end 33 | -------------------------------------------------------------------------------- /framework/ruby/classes/exception/HBaseClientException.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | # 22 | #=HBaseClient独自の例外を定義する 23 | # 24 | # 25 | class HBaseClientException < Exception 26 | #独自のメッセージ 27 | attr_accessor :message; 28 | 29 | # 30 | #===コンストラクタ 31 | # 32 | #==== args 33 | #エラーメッセージ 34 | def initialize(_message) 35 | @message=(_message); 36 | end 37 | 38 | # 39 | #===独自に設定したメッセージを取得する 40 | # 41 | #==== return 42 | #エラーメッセージ 43 | def getMessage() 44 | return @message; 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /framework/ruby/classes/tables/TableBase.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | #require 'singleton' 22 | require CLAASES_DIR + "viewer/Viewer.rb"; 23 | 24 | require 'pp' 25 | 26 | # 27 | #=データアクセスクラス 28 | # 29 | # 30 | class TableBase 31 | # include Singleton 32 | 33 | #テーブルのカラム名一覧 34 | @columnNameArray = []; 35 | @classInfo = nil; 36 | @conditionHash = nil; 37 | 38 | @query = nil; 39 | 40 | @tableName = ""; 41 | 42 | public 43 | # 44 | #===コンストラクタ 45 | # 46 | def initialize(tableName, query) 47 | @tableName = tableName; 48 | @query = query; 49 | @conditionHash = Hash.new 50 | 51 | # 結果データを保持するJavaクラス 52 | begin 53 | @dataCls = Sculptor.entities.get(tableName).newInstance(); 54 | 55 | clazz = @dataCls.getClassInfo(); 56 | 57 | @classInfo = ClassInfo.new(clazz); 58 | @columnNameArray = @classInfo.columnNameArray(); 59 | rescue => exception 60 | pp exception 61 | raise HBaseClientException.new("テーブル名が不正です"); 62 | end 63 | end 64 | 65 | # 66 | #===apiのデータアクセスクラスを呼び出す 67 | # 68 | #==== args 69 | #conf :: org.apache.hadoop.conf.Configuration 70 | #==== return 71 | #データアクセスクラス 72 | def getInstance() 73 | @client = Sculptor.clients.get(@tableName).newInstance(); 74 | return @client; 75 | end 76 | 77 | # 78 | #===テーブルのカラム名一覧を取得 79 | # 80 | #==== return 81 | #テーブルのカラム名一覧 82 | def TableBase.GetColumnNameArray() 83 | return @columnNameArray; 84 | end 85 | 86 | def get() 87 | @client = self.getInstance(); 88 | 89 | setGetCondition(); 90 | 91 | isScan = validRowKey(); 92 | 93 | object = executeGet(isScan); 94 | 95 | Viewer.getView(object, @query, isScan, @columnNameArray, @classInfo.fieldInfoList(), @client); 96 | 97 | self.clearCondition(); 98 | end 99 | 100 | def count() 101 | @client = self.getInstance(); 102 | 103 | setGetCondition(); 104 | 105 | isScan = validRowKey(); 106 | 107 | count = 0; 108 | 109 | if (isScan) 110 | # scanが呼ばれた場合 111 | if ($process_mode == ProcessMode::NORMAL) 112 | object = executeGet(isScan); 113 | if (object != nil) 114 | while (object.next()) 115 | count = count + 1; 116 | end 117 | end 118 | elsif ($process_mode == ProcessMode::MAP_REDUCE) 119 | # 検索条件セット 120 | setCondition(); 121 | count = @client.countMR(@dataCls, @conditionHash); 122 | else 123 | # モードおかしい 124 | # チェック済みなのでありえないけど一応例外投げる 125 | raise HBaseClientException.new("(´・ω・｀)"); 126 | end 127 | else 128 | # getが呼ばれた場合 129 | object = executeGet(isScan); 130 | if (object != nil) 131 | count = count + 1; 132 | end 133 | end 134 | 135 | Viewer.countView(count); 136 | 137 | self.clearCondition(); 138 | end 139 | 140 | # 141 | #===指定したレコードを削除する 142 | # 143 | #==== args 144 | #conf ::org.apache.hadoop.conf.Configuration 145 | def delete() 146 | 147 | @client = self.getInstance(); 148 | 149 | setGetCondition(); 150 | 151 | isScan = validRowKey(); 152 | # まずは条件で検索する 153 | object = executeGet(isScan); 154 | 155 | # 検索結果からrowKeyを取得し、そのrowKeyでレコードを削除する 156 | if (isScan) 157 | # scanの場合 158 | while (record = object.next()) 159 | rowKey = record.getRowkey(); 160 | @client.delete(rowKey); 161 | end 162 | else 163 | # getの場合 164 | rowKey = object.getRowkey(); 165 | @client.delete(rowKey); 166 | end 167 | 168 | self.clearCondition(); 169 | end 170 | 171 | # 172 | #===指定したレコードを挿入/更新する 173 | # 174 | #==== args 175 | #conf ::org.apache.hadoop.conf.Configuration 176 | #====raise 177 | #HBaseClientException :: rowKeyが指定されていない、または不正なクエリが入力された場合 178 | def put() 179 | 180 | setGetCondition(); 181 | 182 | if (validRowKey()) 183 | errMgs = ""; 184 | count = 0; 185 | @classInfo.fieldInfoList().each do |fieldInfo| 186 | if (fieldInfo.rowkey) 187 | condition = @conditionHash[fieldInfo.qualifier()]; 188 | if (condition == nil) 189 | if (count > 0) 190 | errMgs += ", "; 191 | end 192 | errMgs += fieldInfo.qualifier(); 193 | count = count + 1; 194 | end 195 | end 196 | end 197 | raise HBaseClientException.new("条件: " + errMgs + "は必須項目です"); 198 | end 199 | 200 | @client = self.getInstance(); 201 | 202 | @query.conditionArray().each do |condition| 203 | 204 | unless (condition.sign() == HCompareOp::EQUAL) 205 | raise HBaseClientException.new("検索条件に不正な文字列があります"); 206 | end 207 | 208 | fieldInfo = FieldInfo.getFieldInfoFromQualifier(@classInfo.fieldInfoList(), condition.columnName().downcase); 209 | fName = fieldInfo.javaFieldName(); 210 | 211 | 212 | value = condition.value(); 213 | if (value.startsWith("\"") && value.endsWith("\"")) 214 | value = value.slice(1..value.length - 2); 215 | 216 | end 217 | 218 | className = Util.getType(@classInfo.fieldInfoList(), condition.columnName().downcase); 219 | 220 | eval("@dataCls." + fName + " = Util.toJavaClass(value, className);"); 221 | end 222 | 223 | @client.put(@dataCls); 224 | 225 | self.clearCondition(); 226 | end 227 | 228 | private 229 | 230 | # 231 | #===getの検索条件をセットする 232 | # 233 | #==== raise 234 | #HBaseClientException 235 | def setGetCondition() 236 | 237 | @query.conditionArray().each do |condition| 238 | 239 | begin 240 | # 検索条件をセット 241 | eval("@" + condition.columnName().downcase + " = \"" + condition.value() + "\";"); 242 | rescue SyntaxError 243 | begin 244 | eval("@" + condition.columnName().downcase + " = " + condition.value() + ";"); 245 | rescue SyntaxError 246 | raise HBaseClientException.new("検索条件に不正な文字列があります"); 247 | end 248 | end 249 | @conditionHash[condition.columnName().downcase] = condition.sign(); 250 | end 251 | end 252 | 253 | def setCondition() 254 | # 検索条件セット 255 | @classInfo.fieldInfoList().each do |fieldInfo| 256 | className = fieldInfo.canonicalName(); 257 | javaFieldName = fieldInfo.javaFieldName(); 258 | fieldName = fieldInfo.fieldName(); 259 | unless (fieldName == nil) 260 | eval("@dataCls." + javaFieldName + " = Util.toJavaClass(@" + fieldName + ", className);"); 261 | end 262 | end 263 | end 264 | 265 | # 266 | #===検索を実行する 267 | # 268 | #==== args 269 | #isScan :: apiのscanメソッドを呼ぶがどうか 270 | def executeGet(isScan) 271 | # 検索条件セット 272 | setCondition(); 273 | 274 | return isScan ? scan() : getAndsearch(); 275 | end 276 | 277 | def scan() 278 | object = nil; 279 | if ($process_mode == ProcessMode::NORMAL) 280 | # nomalモードでは普通のscan 281 | object = @client.scan(@dataCls, @conditionHash); 282 | elsif ($process_mode == ProcessMode::MAP_REDUCE) 283 | # mapreduceモードではscanMR 284 | offset = @query.offset(); 285 | limit = @query.limit(); 286 | object = @client.scanMR(@dataCls, @conditionHash, offset, limit); 287 | else 288 | # モードおかしい 289 | # チェック済みなのでありえないけど一応例外投げる 290 | raise HBaseClientException.new("(´・ω・｀)"); 291 | end 292 | 293 | return object; 294 | end 295 | 296 | # 297 | #===getでレコードを取得し、それに対して他の検索条件を当てていく 298 | # 299 | #==== return 300 | #結果オブジェクト 301 | def getAndsearch() 302 | object = @client.get(@dataCls); 303 | 304 | unless (object == nil) 305 | # rowKey以外の項目を当てていく 306 | @conditionHash.each do |key, value| 307 | # 検索条件で指定したカラム名を元にJavaオブジェクトのフィールド名を取得 308 | fieldInfo = FieldInfo.getFieldInfoFromQualifier(@classInfo.fieldInfoList(), key); 309 | fieldName = fieldInfo.javaFieldName(); 310 | 311 | # 検索条件で指定したカラムに対応した検索結果 312 | resultValue = nil; 313 | eval("resultValue = object." + fieldName + ";"); 314 | 315 | # 検索条件で指定した値 316 | searchValue = nil; 317 | # Javaオブジェクトの型を取得 318 | className = fieldInfo.canonicalName(); 319 | 320 | eval("searchValue = Util.toJavaClass(@" + key + ", className);"); 321 | 322 | # 検索条件を照らし合わせ、条件に合わない項目があれば検索結果を削除 323 | unless (Condition.compare(resultValue, searchValue, value)) 324 | object = nil; 325 | # 1件でもあればおしまい 326 | break; 327 | end 328 | end 329 | end 330 | 331 | return object; 332 | end 333 | 334 | # 335 | #===検索条件をクリアする 336 | # 337 | def clearCondition() 338 | @classInfo.fieldInfoList().each do |fieldInfo| 339 | eval("@" + fieldInfo.fieldName() + " = nil"); 340 | end 341 | 342 | @dataCls = Sculptor.entities.get(@tableName).newInstance(); 343 | 344 | end 345 | 346 | # 347 | #===rowkey項目が全て揃っているかどうかチェック 348 | # 349 | #==== return 350 | #scanを呼ぶかどうか 351 | def validRowKey() 352 | 353 | isScan = false 354 | isChecked = false; 355 | @classInfo.fieldInfoList().each do |fieldInfo| 356 | if (fieldInfo.rowkey) 357 | isChecked = true; 358 | isScan |= @conditionHash[fieldInfo.qualifier()] != HCompareOp::EQUAL; 359 | end 360 | end 361 | 362 | if (!isChecked) 363 | isScan = true; 364 | end 365 | 366 | return isScan; 367 | end 368 | end 369 | 370 | # 371 | #===クラスの情報を保持する 372 | # 373 | # 374 | class ClassInfo 375 | #クラス名 376 | attr_accessor :className; 377 | #クラスが表すテーブル 378 | attr_accessor :table; 379 | #クラスのフィールド一覧 380 | attr_accessor :fieldInfoList; 381 | #クラスが表すテーブルのカラム名一覧 382 | attr_accessor :columnNameArray; 383 | # 384 | #===コンストラクタ 385 | # 386 | def initialize(hClassDescriptor) 387 | @className=(hClassDescriptor.getEntityClassName()); 388 | @table=(hClassDescriptor.getTable()); 389 | 390 | fields = hClassDescriptor.gethFieldDescriptors(); 391 | 392 | @columnNameArray = []; 393 | @fieldInfoList = []; 394 | fields.to_array.each_with_index do |hFieldDescriptor, index| 395 | 396 | javaFieldName = hFieldDescriptor.getFieldName(); 397 | fieldName = hFieldDescriptor.getQualifier(); 398 | family = hFieldDescriptor.getFamily(); 399 | qualifier = hFieldDescriptor.getQualifier(); 400 | isRowkey = hFieldDescriptor.isRowkey().to_s; 401 | canonicalName = hFieldDescriptor. getCanonicalName(); 402 | 403 | fieldInfo = FieldInfo.new(javaFieldName, fieldName, family, qualifier, isRowkey, canonicalName); 404 | 405 | unless (fieldInfo.fieldName() == nil) 406 | @columnNameArray << fieldName; 407 | @fieldInfoList << fieldInfo; 408 | end 409 | end 410 | end 411 | end 412 | 413 | # 414 | #===フィールドの情報を保持する 415 | # 416 | # 417 | class FieldInfo 418 | #Javaクラスのフィールド名 419 | attr_accessor :javaFieldName; 420 | #フィールド名 421 | attr_accessor :fieldName; 422 | #データファミリー 423 | attr_accessor :family; 424 | #このフィールドを表す 425 | attr_accessor :qualifier; 426 | #このフィールドがrowKeyかどうか 427 | attr_accessor :rowkey; 428 | #このフィールドのJava型名(フル) 429 | attr_accessor :canonicalName; 430 | # 431 | #===コンストラクタ 432 | # 433 | def initialize(_javaFieldName, _fieldName, _family, _qualifier, _isRowkey, _canonicalName) 434 | @javaFieldName=(_javaFieldName); 435 | @fieldName=(_fieldName); 436 | @family=(_family); 437 | @qualifier=(_qualifier); 438 | @rowkey=(_isRowkey.to_s == "true"); 439 | @canonicalName=(_canonicalName); 440 | end 441 | 442 | # 443 | #===qualifierから対応するフィールド名を取得する 444 | #==== args 445 | #fieldInfoList :: FieldInfoの配列 446 | #qualifier :: qualifier 447 | #==== return 448 | #フィールド名 449 | def self.getJavaFieldNameFromQualifier(fieldInfoList, qualifier) 450 | retName = ""; 451 | fieldInfoList.each do |fieldInfo| 452 | if (fieldInfo.qualifier() == qualifier) 453 | retName = fieldInfo.javaFieldName(); 454 | break; 455 | end 456 | end 457 | return retName; 458 | end 459 | 460 | # 461 | #===qualifierから対応するフィールドの型を取得する 462 | # 463 | #==== args 464 | #fieldInfoList :: FieldInfoの配列 465 | #qualifier :: qualifier 466 | #==== return 467 | #フィールドの型名 468 | def self.getCanonicalNameFromQualifier(fieldInfoList, qualifier) 469 | retName = ""; 470 | fieldInfoList.each do |fieldInfo| 471 | if (fieldInfo.qualifier() == qualifier) 472 | retName = fieldInfo.canonicalName(); 473 | break; 474 | end 475 | end 476 | return retName; 477 | end 478 | 479 | # 480 | #===qualifierからそのフィールドがrowkeyかどうかを取得する 481 | #==== args 482 | #fieldInfoList :: FieldInfoの配列 483 | #qualifier :: qualifier 484 | #==== return 485 | #rowkeyかどうか 486 | def self.getRowkeyFromQualifier(fieldInfoList, qualifier) 487 | ret = false; 488 | fieldInfoList.each do |fieldInfo| 489 | if (fieldInfo.qualifier() == qualifier) 490 | ret = fieldInfo.rowkey(); 491 | break; 492 | end 493 | end 494 | return ret; 495 | end 496 | 497 | # 498 | #===qualifierからフィールドオブジェクトを取得する 499 | #==== args 500 | #fieldInfoList :: FieldInfoの配列 501 | #qualifier :: qualifier 502 | #==== return 503 | #FieldInfo 504 | def self.getFieldInfoFromQualifier(fieldInfoList, qualifier) 505 | ret = false; 506 | fieldInfoList.each do |fieldInfo| 507 | if (fieldInfo.qualifier() == qualifier) 508 | ret = fieldInfo; 509 | break; 510 | end 511 | end 512 | return ret; 513 | end 514 | end 515 | -------------------------------------------------------------------------------- /framework/ruby/classes/viewer/Viewer.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | # 22 | #=検索結果表示を定義する 23 | # 24 | # 25 | class Viewer 26 | public 27 | # 28 | #===getコマンドの検索結果を表示する 29 | # 30 | #==== args 31 | #obj :: frameworkから返ってきた結果オブジェクト 32 | #query :: Queryオブジェクト 33 | #isScan :: 検索時にscanを呼び出したかどうか 34 | #columnNameArray :: 対象テーブルのカラム名一覧 35 | #fieldInfoList :: 対象テーブルクラスのフィールド名一覧 36 | #client :: 37 | def self.getView(obj, query, isScan, columnNameArray, fieldInfoList, client) 38 | 39 | count = 0; 40 | mrOutputPath = nil; 41 | 42 | eputs "-------- result --------"; 43 | if (obj == nil) 44 | eputs "no data"; 45 | else 46 | if (isScan) 47 | # scanが呼ばれた場合 48 | offset = query.offset(); 49 | limit = query.limit(); 50 | 51 | index = 1; 52 | if ($process_mode == ProcessMode::NORMAL) 53 | while (record = obj.next()) 54 | if (offset < index) 55 | if (limit == -1 || index < offset + limit + 1) 56 | # レコードの表示 57 | eputs "----------------------"; 58 | self.showRecord(record, query, columnNameArray, fieldInfoList); 59 | count = count + 1; 60 | eputs "----------------------"; 61 | else 62 | break; 63 | end 64 | end 65 | 66 | index = index + 1; 67 | end 68 | elsif ($process_mode == ProcessMode::MAP_REDUCE) 69 | if (obj.size() == 0) 70 | eputs "no data"; 71 | else 72 | count = obj.size(); 73 | if (count == 100) 74 | mrOutputPath = client.getMROutputPath(); 75 | end 76 | 77 | obj.each do |record| 78 | 79 | # レコードの表示 80 | eputs "----------------------"; 81 | self.showRecord(record, query, columnNameArray, fieldInfoList); 82 | # count = count + 1; 83 | eputs "----------------------"; 84 | 85 | index = index + 1; 86 | end 87 | 88 | end 89 | else 90 | # モードおかしい 91 | # チェック済みなのでありえないけど一応例外投げる 92 | raise HBaseClientException.new("(´・ω・｀)"); 93 | end 94 | else 95 | # getが呼ばれた場合 96 | # レコードの表示 97 | self.showRecord(obj, query, columnNameArray, fieldInfoList); 98 | count = count + 1; 99 | end 100 | end 101 | eputs "-------- /result --------"; 102 | eputs count.to_s + " result"; 103 | 104 | unless (mrOutputPath == nil) 105 | eputs "検索結果が100件を超えています。100件目以降のデータは以下をご確認ください"; 106 | eputs mrOutputPath; 107 | end 108 | end 109 | 110 | # 111 | #===countコマンドの検索結果を表示する 112 | # 113 | #==== args 114 | #count :: 結果のカウント 115 | def self.countView(count) 116 | 117 | eputs "-------- result --------"; 118 | self.dispLine(CommandConst::COUNT, count, CommandConst::COUNT.length); 119 | eputs "-------- /result --------"; 120 | eputs "1 result" 121 | 122 | end 123 | 124 | private 125 | 126 | # 127 | #===レコードの表示 128 | # 129 | #==== args 130 | #record :: frameworkから返ってきた結果オブジェクト 131 | #query :: Queryオブジェクト 132 | #columnNameArray :: 対象テーブルのカラム名一覧 133 | #fieldInfoList :: 対象テーブルクラスのフィールド名一覧 134 | def self.showRecord(record, query, columnNameArray, fieldInfoList) 135 | 136 | # ユーザが指定した表示カラム一覧 137 | showColumnArray = query.showColumnArray(); 138 | if (showColumnArray == []) 139 | # 表示カラムの指定がない場合は全カラム表示 140 | showColumnArray = columnNameArray; 141 | end 142 | # padding用にカラム名の最大長を取得 143 | maxColumnNameLength = Util.arrayMaxLength(showColumnArray); 144 | 145 | # カラム名のリストをぐーるぐる 146 | showColumnArray.each do |columnName| 147 | value = ""; 148 | fieldInfo = FieldInfo.getFieldInfoFromQualifier(fieldInfoList, columnName); 149 | fieldName = fieldInfo.javaFieldName(); 150 | 151 | begin 152 | # 動的にItemRankingの変数を呼び出して値にセット 153 | eval("value = record." + fieldName); 154 | if (fieldInfo.rowkey()) 155 | columnName = "*" + columnName; 156 | end 157 | rescue SyntaxError 158 | raise HBaseClientException.new("カラム名が不正です"); 159 | end 160 | if ($view_mode == ViewMode::LINE) 161 | # 表示モード：line 162 | # 1件表示 163 | self.dispLine(columnName, value, maxColumnNameLength); 164 | else 165 | # 1項目 166 | self.dispLineOnTableMode(columnName, value, true); 167 | end 168 | end 169 | end 170 | 171 | # 172 | #===[カラム名 => 値]を表示する 173 | #カラム名は最長のものに合わせてpaddingされる 174 | # 175 | #==== args 176 | #columnName :: カラム名 177 | #value :: 値 178 | #maxColumnNameLength :: カラム名の最大長 179 | def self.dispLine(columnName, value, maxColumnNameLength) 180 | if (value.kind_of?(Java::JavaUtil::Date)) 181 | # Date型の場合、yyyy/MM/ddに整形する 182 | value = Util.JavaDate2String(value, DateFormat::FORMAT1); 183 | elsif (value.kind_of?(Java::JavaUtil::List)) 184 | # List型の場合、展開し、カンマつなぎの文字列にする 185 | tmpValue = ""; 186 | 187 | value.each_with_index do |val, index| 188 | 189 | if (index == 0) 190 | tmpValue += "["; 191 | else 192 | tmpValue += "," 193 | end 194 | # Date型の場合、yyyy/MM/ddに整形する 195 | if (val.kind_of?(Java::JavaUtil::Date)) 196 | tmpValue += Util.JavaDate2String(val, DateFormat::FORMAT1); 197 | elsif (val.class.name == "") 198 | # FIXME これで大丈夫なのかな？おそらくjRubyのバグ。プリミティブ型のclassnameとれない？ 199 | tmpValue += org.apache.hadoop.hbase.util.Bytes.toString(val); 200 | else 201 | tmpValue += val.to_s; 202 | end 203 | 204 | if (index + 1 == value.size()) 205 | tmpValue += "]" 206 | end 207 | end 208 | value = tmpValue; 209 | end 210 | 211 | eputs columnName.ljust(maxColumnNameLength + 1) + " => " + value.to_s; 212 | end 213 | 214 | def self.dispLineOnTableMode(columnName, value, withHeader) 215 | print "|" + value.to_s.ljust(columnName.length); 216 | end 217 | end 218 | -------------------------------------------------------------------------------- /framework/ruby/doc/help.txt: -------------------------------------------------------------------------------- 1 | これヘルプファイル 2 | 3 | get: 4 | GET [column_name] FROM table_name [where_expression [where_expression]] [limit_expression] [offset_expression] 5 | count: 6 | COUNT [column_name] FROM table_name [where_expression [where_expression]] [limit_expression] [offset_expression] 7 | put: 8 | PUT tablename where_expression [where_expression] 9 | *It is indispensable to specify rowkey. 10 | delete: 11 | DELETE FROM table_name where_expression [where_expression] 12 | 13 | expression: 14 | where_expression: 15 | column_name(=|<|>|<=|>=)value 16 | limit_expression: 17 | LIMIT=fixnum 18 | offset_expression: 19 | OFFSET=fixnum 20 | 21 | set view: 22 | SET VIEW (line|table) 23 | set process: 24 | SET PROCESS (normal|mapreduce) 25 | 26 | exit: 27 | EXIT 28 | help: 29 | HELP -------------------------------------------------------------------------------- /framework/ruby/modules/Out.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | require CONST_DIR + 'OSConst.rb' 22 | 23 | # 24 | #===出力を定義するモジュール 25 | # 26 | module Out 27 | $os = OSConst::getOsName(); 28 | $encode = OSConst::getDefaultEncode($os); 29 | 30 | # 31 | #===putsのエンコード機能付き 32 | # 33 | #==== args 34 | #文字列 35 | def eputs(value) 36 | puts value.to_s.encode($encode); 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /framework/ruby/rdocgen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2010 The Apache Software Foundation 4 | # 5 | # Licensed to the Apache Software Foundation (ASF) under one 6 | # or more contributor license agreements. See the NOTICE file 7 | # distributed with this work for additional information 8 | # regarding copyright ownership. The ASF licenses this file 9 | # to you under the Apache License, Version 2.0 (the 10 | # "License"); you may not use this file except in compliance 11 | # with the License. You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | 22 | rdoc --main HBaseClient --op rdoc --charset utf-8 -d -S -U -N -a 23 | -------------------------------------------------------------------------------- /framework/ruby/sculptor.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2010 The Apache Software Foundation 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | #Sculptorのメインスクリプト 22 | # 23 | require 'etc' 24 | require 'pp' 25 | 26 | $userName = Etc.getlogin; 27 | 28 | $title = <<"EOS" 29 | +--------------------------------------------------+ 30 | | Sculptor | 31 | | var 0.0.1 |дﾟ) 32 | +--------------------------------------------------+ 33 | EOS 34 | 35 | #このスクリプトのベースディレクトリ 36 | $basePath = File::dirname(__FILE__); 37 | 38 | # sculptor root 39 | $SCULPTOR_ROOT = $basePath + '/../..' 40 | 41 | # 初期設定 42 | require $basePath + '/classes/Initialize.rb' 43 | 44 | CommandHistory::init($tableNameList, $fieldNameList) 45 | 46 | operation = ""; 47 | 48 | # コマンドヒストリーを読み込む 49 | CommandHistory::readHistoryFile($userName); 50 | 51 | while (true) 52 | begin 53 | # ユーザの入力を取得 54 | # -eの場合は引数から値をとる 55 | if(ARGV[0]) then 56 | readCommand = ARGV.join(" "); 57 | else 58 | readCommand = CommandHistory::readCommand(); 59 | end 60 | # クエリをパース 61 | query = Query.new(); 62 | query.parse(readCommand); 63 | 64 | operation = query.operation(); 65 | unless (operation == nil) 66 | if (operation == CommandConst::EXIT) 67 | # exitコマンドで終了 68 | # コマンドヒストリーをシリアライズ 69 | CommandHistory::writeCommandHistory($userName); 70 | break; 71 | elsif (operation== CommandConst::HELP || !CommandConst.isDefined(operation)) 72 | # 不正なコマンドはコマンド履歴に残さない 73 | if (!CommandConst.isDefined(operation)) 74 | CommandHistory::ignore(); 75 | end 76 | 77 | # helpファイル表示 78 | open(DOC_DIR + "help.txt") do |file| 79 | while (line = file.gets) 80 | eputs "\t" + line; 81 | end 82 | end 83 | end 84 | 85 | if (operation != CommandConst::SET && query.tableName() != nil) 86 | # 呼び出すコマンドクラスの名前を生成 87 | #className = query.operation().downcase.gsub(/\b\w/) { |word| word.upcase }; 88 | 89 | # 呼び出すテーブルクラスの名前を生成 90 | tableName = query.tableName().downcase; 91 | 92 | obj = Command.new(); 93 | obj.execute(query, tableName); 94 | end 95 | end 96 | rescue SyntaxError => e 97 | eputs "Command syntax error."; 98 | rescue => e 99 | pp e; 100 | pp e.backtrace 101 | end 102 | #-eならばループしない 103 | if(ARGV[0]) then 104 | break; 105 | end 106 | end 107 | 108 | eputs "|дﾟ)ﾉｼ < bye!"; 109 | -------------------------------------------------------------------------------- /framework/src/sculptor/framework/FixedSplit.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package sculptor.framework; 19 | 20 | import java.io.BufferedReader; 21 | import java.io.File; 22 | import java.io.FileNotFoundException; 23 | import java.io.FileReader; 24 | import java.io.IOException; 25 | import java.util.ArrayList; 26 | import java.util.List; 27 | 28 | import org.apache.commons.logging.Log; 29 | import org.apache.commons.logging.LogFactory; 30 | import org.apache.hadoop.hbase.util.Bytes; 31 | import org.apache.hadoop.hbase.util.RegionSplitter.SplitAlgorithm; 32 | 33 | /** 34 | * Split algorithm using a specified file.
35 | * Put your splitting keys in {@link #SPLIT_KEY_FILE}, one key per line. 36 | * 37 | */ 38 | public class FixedSplit implements SplitAlgorithm { 39 | static final Log LOG = LogFactory.getLog(FixedSplit.class); 40 | 41 | public static final String SPLIT_KEY_FILE = "/tmp/fixed-split-keys.txt"; 42 | private File _regions; 43 | 44 | public FixedSplit() { 45 | try { 46 | 47 | _regions = new File(SPLIT_KEY_FILE); 48 | if (!_regions.exists()) { 49 | throw new FileNotFoundException("split key file not found. " + SPLIT_KEY_FILE); 50 | } 51 | 52 | 53 | } catch (FileNotFoundException e) { 54 | LOG.error(String.format("put your splitting keys in [%s], one key per line", SPLIT_KEY_FILE), e); 55 | throw new RuntimeException("Aborted!.", e); 56 | } 57 | } 58 | 59 | @Override 60 | public byte[] firstRow() { 61 | // TODO Auto-generated method stub 62 | return null; 63 | } 64 | 65 | @Override 66 | public byte[] lastRow() { 67 | // TODO Auto-generated method stub 68 | return null; 69 | } 70 | 71 | @Override 72 | public String rowToStr(byte[] row) { 73 | // TODO Auto-generated method stub 74 | return null; 75 | } 76 | 77 | @Override 78 | public String separator() { 79 | // TODO Auto-generated method stub 80 | return null; 81 | } 82 | 83 | @Override 84 | public byte[] split(byte[] start, byte[] end) { 85 | // TODO Auto-generated method stub 86 | return null; 87 | } 88 | 89 | @Override 90 | public byte[][] split(int numberOfSplits) { 91 | try { 92 | 93 | List returnBytes = new ArrayList (); 94 | BufferedReader br = new BufferedReader(new FileReader(_regions)); 95 | String line; 96 | while ((line = br.readLine()) != null) { 97 | if (line.trim().length() > 0) { 98 | returnBytes.add(Bytes.toBytes(line)); 99 | } 100 | } 101 | return returnBytes.toArray(new byte[0][]); 102 | 103 | } catch (IOException e) { 104 | LOG.error("Error reading splitting keys from " + SPLIT_KEY_FILE, e); 105 | throw new RuntimeException("Aborted!.", e); 106 | } 107 | } 108 | 109 | @Override 110 | public byte[] strToRow(String input) { 111 | // TODO Auto-generated method stub 112 | return null; 113 | } 114 | 115 | } 116 | -------------------------------------------------------------------------------- /framework/src/sculptor/framework/HClassDescriptor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package sculptor.framework; 19 | 20 | import java.util.List; 21 | 22 | /** 23 | * HBase上のテーブルに紐づいたクラスを定義 24 | * 25 | */ 26 | public class HClassDescriptor { 27 | /** Client class name */ 28 | String clientClassName; 29 | /** Entity class name */ 30 | String entityClassName; 31 | /** 紐づいているテーブル名 */ 32 | String table; 33 | /** List of column family */ 34 | String[] columnFamilies; 35 | /** フィールドのリスト */ 36 | List hFieldDescriptors; 37 | 38 | /** 39 | * Get entity's canonical name. 40 | * 41 | * @return the entity's canonical name 42 | */ 43 | public String getEntityClassName() { 44 | return entityClassName; 45 | } 46 | 47 | /** 48 | * Get client's canonical name. 49 | * 50 | * @return the client's canonical name 51 | */ 52 | public String getClientClassName() { 53 | return clientClassName; 54 | } 55 | 56 | /** 57 | * Get the table name 58 | * 59 | * @return the table name 60 | */ 61 | public String getTable() { 62 | return table; 63 | } 64 | 65 | /** 66 | * Get all fields 67 | * 68 | * @return all fields 69 | */ 70 | public List gethFieldDescriptors() { 71 | return hFieldDescriptors; 72 | } 73 | 74 | public String[] getColumnFamilies() { 75 | return columnFamilies; 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /framework/src/sculptor/framework/HClient.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package sculptor.framework; 19 | 20 | import java.io.Closeable; 21 | import java.io.IOException; 22 | import java.io.ObjectInputStream; 23 | import java.io.ObjectOutputStream; 24 | import java.util.ArrayList; 25 | import java.util.Calendar; 26 | import java.util.List; 27 | import java.util.Map; 28 | 29 | import org.apache.commons.lang.math.RandomUtils; 30 | import org.apache.hadoop.conf.Configuration; 31 | import org.apache.hadoop.fs.FileSystem; 32 | import org.apache.hadoop.fs.Path; 33 | import org.apache.hadoop.hbase.HBaseConfiguration; 34 | import org.apache.hadoop.hbase.client.Delete; 35 | import org.apache.hadoop.hbase.client.Get; 36 | import org.apache.hadoop.hbase.client.HTable; 37 | import org.apache.hadoop.hbase.client.Put; 38 | import org.apache.hadoop.hbase.client.Result; 39 | import org.apache.hadoop.hbase.client.Scan; 40 | import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; 41 | import org.apache.hadoop.hbase.io.ImmutableBytesWritable; 42 | import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; 43 | import org.apache.hadoop.hbase.mapreduce.TableMapper; 44 | import org.apache.hadoop.hbase.util.Bytes; 45 | import org.apache.hadoop.io.Text; 46 | import org.apache.hadoop.mapreduce.Counter; 47 | import org.apache.hadoop.mapreduce.Counters; 48 | import org.apache.hadoop.mapreduce.Job; 49 | import org.apache.hadoop.mapreduce.Reducer; 50 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 51 | import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; 52 | import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; 53 | 54 | /** 55 | * HBase client 56 | * Note: this class is NOT thread safe 57 | * 58 | * @param data store 59 | */ 60 | public abstract class HClient implements Closeable { 61 | 62 | protected String tableName; 63 | protected HTable table; 64 | 65 | protected boolean closed = false; 66 | 67 | private static Configuration defaultConfig = HBaseConfiguration.create(); 68 | 69 | private Configuration thatConfig; 70 | 71 | // some key for MR 72 | public static final String SCAN_MR_OUTPUT = "sculptor.scanMR.output"; 73 | public static final String SCAN_MR_OFFSET = "sculptor.scanMR.offset"; 74 | public static final String SCAN_MR_LIMIT = "sculptor.scanMR.limit"; 75 | public static final String SCAN_MR_CLIENT = "sculptor.scanMR.client"; 76 | 77 | protected String MROutput; 78 | static final Text EMPTY_TEXT = new Text(""); 79 | 80 | /** home of sculptor on DFS */ 81 | public static final String DFS_HOME = "/tmp/sculptor"; 82 | 83 | /** default count return to client via Mapreduce scan */ 84 | public static final int MR_DEFAULT_RETURN_COUNT = 100; 85 | 86 | 87 | /** 88 | * Constructor 89 | * 90 | * @param tableName the table name 91 | * @param that specific configuration 92 | */ 93 | public HClient(String tableName, Configuration that) { 94 | this.tableName = tableName; 95 | try { 96 | 97 | if (that != null) { 98 | this.thatConfig = that; 99 | } 100 | 101 | this.table = new HTable(getConfig(), this.tableName); 102 | 103 | } catch (IOException e) { 104 | throw new RuntimeException("can not connect to HBase for table: " + this.tableName, e); 105 | } 106 | 107 | } 108 | 109 | /** 110 | * Constructor 111 | * 112 | * @param tableName the table name 113 | */ 114 | public HClient(String tableName) { 115 | this(tableName, null); 116 | } 117 | 118 | /** 119 | * この接続の設定情報を取得 120 | * 121 | * @return 設定情報 122 | */ 123 | public Configuration getConfig() { 124 | return thatConfig == null ? defaultConfig : thatConfig; 125 | } 126 | 127 | public void close() { 128 | if (table != null) { 129 | try { 130 | table.close(); 131 | } catch (IOException e) { 132 | // ingore 133 | } 134 | } 135 | closed = true; 136 | } 137 | 138 | public boolean isClosed() { 139 | return closed; 140 | } 141 | 142 | public String getTableName() { 143 | return tableName; 144 | } 145 | 146 | public HTable getTable() { 147 | return table; 148 | } 149 | 150 | /** 151 | * HBaseの一行を取得。1 familyのみ取得。 152 | * 153 | * @param rowkey row key 154 | * @param family family 155 | * @return 一行 156 | * @throws IOException 157 | */ 158 | public D get(byte[] rowkey, byte[] family) throws IOException { 159 | if (rowkey == null || rowkey.length == 0) { 160 | return null; 161 | } 162 | Get g = new Get(rowkey); 163 | g.addFamily(family); 164 | g.setMaxVersions(); 165 | Result r = table.get(g); 166 | return toEntity(r); 167 | } 168 | 169 | /** 170 | * Mapreduce jobの出力base path 171 | * 172 | * @return output bash path 173 | */ 174 | public String generateMROutput(String prefix) { 175 | String username = System.getProperty("user.name"); 176 | int random = RandomUtils.nextInt(9999); 177 | String path = String.format("/%1$s/%2$tY%2$tm%2$td%2$tH%2$tM_%3$s_%4$s_%5$d", username, Calendar.getInstance(), getTableName(), prefix, random); 178 | return DFS_HOME + path; 179 | } 180 | 181 | /** 182 | * Get the temporary path on DFS 183 | * 184 | * @param outputName the last part of output path 185 | * @return temporary path on DFS 186 | */ 187 | public static String getTemporaryPath(String outputName) { 188 | return String.format("%s/tmp/%s", DFS_HOME, outputName); 189 | } 190 | 191 | /** 192 | * Get the path of serialized object on DFS 193 | * 194 | * @param outputName the last part of output path 195 | * @return the path of serialized object on DFS 196 | */ 197 | public static String getSerializePath(String outputName) { 198 | return getTemporaryPath(outputName) + "/serialized"; 199 | } 200 | 201 | /** 202 | * HCompareOp to CompareOp 203 | * 204 | * @param hop the HCompareOp 205 | * @return CompareOp 206 | */ 207 | public static CompareOp toCompareOp(HCompareOp hop) { 208 | CompareOp op; 209 | if (hop == HCompareOp.LESS) { 210 | op = CompareOp.LESS; 211 | } else if (hop == HCompareOp.LESS_OR_EQUAL) { 212 | op = CompareOp.LESS_OR_EQUAL; 213 | } else if (hop == HCompareOp.EQUAL) { 214 | op = CompareOp.EQUAL; 215 | } else if (hop == HCompareOp.NOT_EQUAL) { 216 | op = CompareOp.NOT_EQUAL; 217 | } else if (hop == HCompareOp.GREATER_OR_EQUAL) { 218 | op = CompareOp.GREATER_OR_EQUAL; 219 | } else if (hop == HCompareOp.GREATER) { 220 | op = CompareOp.GREATER; 221 | } else { 222 | throw new RuntimeException("not implemented"); 223 | } 224 | return op; 225 | } 226 | 227 | public abstract byte[] toRowkey(D d); 228 | 229 | /** 230 | * get one row via D 231 | * 232 | * @param d data store entity 233 | * @return the row 234 | */ 235 | public D get(D d) throws IOException { 236 | byte[] rowkey = toRowkey(d); 237 | if (rowkey == null || rowkey.length == 0) { 238 | return null; 239 | } 240 | Get g = new Get(rowkey); 241 | String[] families = Sculptor.descriptors.get(tableName).columnFamilies; 242 | for (String family : families) { 243 | byte[] bFamily = Bytes.toBytes(family); 244 | g.addFamily(bFamily); 245 | } 246 | g.setMaxVersions(); 247 | Result r = table.get(g); 248 | return toEntity(r); 249 | } 250 | 251 | /** 252 | * Get the native scan. 253 | * This method maybe slow. Specify the row key items to gain speed. 254 | * 255 | * @param kv data store entity 256 | * @param ops 項目ごとの比較条件 257 | * @return the native scan 258 | */ 259 | public abstract Scan getRawScan(D kv, Map ops); 260 | 261 | /** 262 | * Scan the table.
This method maybe slow. Specify the row 263 | * key items to gain speed. 264 | * 265 | * @param kv entity 266 | * @param ops filter conditions 267 | * @return the scanner 268 | * @throws IOException 269 | */ 270 | public abstract HScanner scan(D kv, Map ops) throws IOException; 271 | 272 | /** 273 | * Scan using mapreduce. 274 | * 275 | * @param kv entity 276 | * @param ops filter conditions 277 | * @return Result of the scan.
Also write to HDFS. Use {@link #getMROutputPath()} to get the output path. 278 | * @throws Exception 279 | */ 280 | public List scanMR(D kv, Map ops) throws Exception { 281 | return scanMR(kv, ops, 0, -1); 282 | } 283 | 284 | /** 285 | * Scan using mapreduce. 286 | * 287 | * @param kv entity 288 | * @param ops filter conditions 289 | * @param offset result offset 290 | * @param limit result limit 291 | * @return Result of the scan.
Also write to HDFS. Use {@link #getMROutputPath()} to get the output path. 292 | * @throws Exception 293 | */ 294 | public List scanMR(D kv, Map ops, long offset, long limit) throws Exception { 295 | List result = new ArrayList(0); 296 | if (limit == 0) { 297 | return result; 298 | } 299 | 300 | String jobName = "Scan " + tableName; 301 | Job job = new Job(this.getConfig(), jobName); 302 | job.setJarByClass(HClient.class); 303 | 304 | // scan setting 305 | Scan scan = getRawScan(kv, ops); 306 | scan.setCacheBlocks(false); 307 | 308 | // initialize the mapper 309 | TableMapReduceUtil.initTableMapperJob(getTableName(), scan, ScanMapper.class, ImmutableBytesWritable.class, Result.class, job, false); 310 | 311 | // the reducer 312 | job.setReducerClass(ScanReducer.class); 313 | // must do global sort by the row key 314 | job.setNumReduceTasks(1); 315 | job.setOutputFormatClass(TextOutputFormat.class); 316 | 317 | // set output path 318 | MROutput = generateMROutput("scan"); 319 | Configuration conf = job.getConfiguration(); 320 | conf.set(SCAN_MR_OUTPUT, MROutput); 321 | Path output = new Path(MROutput); 322 | FileSystem fs = FileSystem.get(conf); 323 | if (fs.exists(output)) { 324 | fs.delete(output, true); 325 | } 326 | FileOutputFormat.setOutputPath(job, output); 327 | 328 | // set offset and limit 329 | conf.set(SCAN_MR_OFFSET, String.valueOf(offset)); 330 | conf.set(SCAN_MR_LIMIT, String.valueOf(limit)); 331 | 332 | // for reducer 333 | conf.set(SCAN_MR_CLIENT, Sculptor.descriptors.get(tableName).clientClassName); 334 | 335 | boolean jobResult = job.waitForCompletion(true); 336 | 337 | if (jobResult) { 338 | // deserialize some results from HDFS 339 | String outputName = MROutput.substring(MROutput.lastIndexOf("/")); 340 | String serializePath = HClient.getSerializePath(outputName); 341 | 342 | ObjectInputStream ois = new ObjectInputStream(fs.open(new Path(serializePath))); 343 | result = (List) ois.readObject(); 344 | 345 | // delete the temporary file 346 | String tempPath = HClient.getTemporaryPath(outputName); 347 | fs.delete(new Path(tempPath), true); 348 | } 349 | return result; 350 | 351 | } 352 | 353 | /** 354 | * Row count using mapreduce. 355 | * 356 | * @param kv entity 357 | * @param ops filter conditions 358 | * @return row count if job completed successfully, -1 if failed. 359 | * @throws Exception 360 | */ 361 | public long countMR(D kv, Map ops) throws Exception { 362 | String jobName = "Count " + tableName; 363 | Job job = new Job(this.getConfig(), jobName); 364 | job.setJarByClass(HClient.class); 365 | 366 | // scan setting 367 | Scan scan = getRawScan(kv, ops); 368 | scan.setCacheBlocks(false); 369 | 370 | // initialize the mapper 371 | TableMapReduceUtil.initTableMapperJob(getTableName(), scan, CountMapper.class, ImmutableBytesWritable.class, Result.class, job, false); 372 | job.setNumReduceTasks(0); 373 | job.setOutputFormatClass(NullOutputFormat.class); 374 | 375 | boolean jobResult = job.waitForCompletion(true); 376 | if (!jobResult) { 377 | return -1; 378 | } 379 | Counters counters = job.getCounters(); 380 | Counter rowCounter = counters.findCounter(CountMapper.Counters.ROWS); 381 | return rowCounter.getValue(); 382 | } 383 | 384 | /** 385 | * Get the output path of mapreduce job. 386 | * 387 | * @return Output path of mapreduce job 388 | */ 389 | public String getMROutputPath() { 390 | return MROutput; 391 | } 392 | 393 | /** 394 | * data store to Put 395 | * 396 | * @param d data store entity 397 | * @return the put 398 | */ 399 | public abstract Put toPut(D d); 400 | 401 | /** 402 | * row to entity, simple version 403 | * 404 | * @param r result 405 | * @return the entity 406 | */ 407 | public abstract D toEntity(Result r); 408 | 409 | /** 410 | * Delete one row. 411 | * 412 | * @param rowkey 413 | * row key 414 | * @throws IOException 415 | */ 416 | public void delete(byte[] rowkey) throws IOException { 417 | if (rowkey == null || rowkey.length == 0) { 418 | return; 419 | } 420 | Delete d = new Delete(rowkey); 421 | table.delete(d); 422 | } 423 | 424 | /** 425 | * Delete one row via data store 426 | * 427 | * @param d data store 428 | * @throws IOException 429 | */ 430 | public void delete(D d) throws IOException { 431 | byte[] rowkey = toRowkey(d); 432 | delete(rowkey); 433 | } 434 | 435 | /** 436 | * Put data into HBase table. 437 | * 438 | * @param d Data 439 | * @throws IOException 440 | */ 441 | public void put(D d) throws IOException { 442 | Put p = toPut(d); 443 | table.put(p); 444 | } 445 | 446 | //###### inner class 447 | static class CountMapper extends TableMapper { 448 | /** Counter enumeration to count the actual rows. */ 449 | public static enum Counters {ROWS} 450 | 451 | @Override 452 | protected void map(ImmutableBytesWritable key, Result value, 453 | Context context) throws IOException, InterruptedException { 454 | context.getCounter(Counters.ROWS).increment(1); 455 | } 456 | } 457 | 458 | /** 459 | * Scan mapper 460 | */ 461 | static class ScanMapper extends TableMapper { 462 | 463 | @Override 464 | protected void map(ImmutableBytesWritable key, Result value, 465 | Context context) throws IOException, InterruptedException { 466 | context.write(key, value); 467 | } 468 | 469 | } 470 | 471 | /** 472 | * Scan reducer 473 | */ 474 | static class ScanReducer extends Reducer { 475 | private long _recordCount = 0; 476 | private long _returnCount = 0; 477 | private HClient _hclient; 478 | private List _MRResult = new ArrayList(100); 479 | 480 | private long _offset; 481 | private long _limit; 482 | private long _ubound; 483 | private long _returnLimit; 484 | 485 | @Override 486 | protected void setup(Context context) throws IOException, 487 | InterruptedException { 488 | _MRResult.clear(); 489 | 490 | Configuration conf = context.getConfiguration(); 491 | _offset = Integer.parseInt(conf.get(SCAN_MR_OFFSET)); 492 | _limit = Integer.parseInt(conf.get(SCAN_MR_LIMIT)); 493 | _returnLimit = _limit; 494 | 495 | if (_offset < 0) { 496 | _offset = 0; 497 | } 498 | if (_limit < 0) { 499 | _limit = Long.MAX_VALUE; 500 | _returnLimit = MR_DEFAULT_RETURN_COUNT; 501 | } 502 | 503 | _ubound = _offset + _limit; 504 | 505 | // the client 506 | String client = conf.get(SCAN_MR_CLIENT); 507 | try { 508 | Class clientClass = Class.forName(client); 509 | _hclient = (HClient) clientClass.newInstance(); 510 | } catch (Exception e) { 511 | throw new IOException("Can not create HClient instance.", e); 512 | } 513 | } 514 | 515 | @Override 516 | protected void reduce(ImmutableBytesWritable key, Iterable value, Context context) 517 | throws IOException, InterruptedException { 518 | if (!value.iterator().hasNext()) { 519 | return; 520 | } 521 | if (_recordCount >= _offset && _recordCount < _ubound) { 522 | // record between the offset and limit 523 | Text outKey = new Text(Bytes.toString(key.get())); 524 | Text outValue = EMPTY_TEXT; 525 | Object d = _hclient.toEntity(value.iterator().next()); 526 | outValue = new Text(d.toString()); 527 | context.write(outKey, outValue); 528 | 529 | if (_returnCount < _returnLimit) { 530 | // return count in the return limit 531 | _MRResult.add(d); 532 | _returnCount++; 533 | } 534 | } 535 | _recordCount++; 536 | } 537 | 538 | @Override 539 | protected void cleanup(Context context) 540 | throws IOException, InterruptedException { 541 | // serialize some results to HDFS 542 | Configuration conf = context.getConfiguration(); 543 | FileSystem fs = FileSystem.get(conf); 544 | String output = conf.get("sculptor.scanMR.output"); 545 | String outputName = output.substring(output.lastIndexOf("/") + 1); 546 | String serializePath = HClient.getSerializePath(outputName); 547 | ObjectOutputStream oos = new ObjectOutputStream(fs.create(new Path(serializePath))); 548 | oos.writeObject(_MRResult); 549 | } 550 | } 551 | 552 | } 553 | -------------------------------------------------------------------------------- /framework/src/sculptor/framework/HCompareOp.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package sculptor.framework; 19 | 20 | public enum HCompareOp { 21 | /** less than */ 22 | LESS, 23 | /** less than or equal to */ 24 | LESS_OR_EQUAL, 25 | /** equals */ 26 | EQUAL, 27 | /** not equal */ 28 | NOT_EQUAL, 29 | /** greater than or equal to */ 30 | GREATER_OR_EQUAL, 31 | /** greater than */ 32 | GREATER, 33 | /** regular expression */ 34 | REGEX, 35 | /** exist */ 36 | EXIST, 37 | /** no operation */ 38 | NO_OP, 39 | 40 | } 41 | -------------------------------------------------------------------------------- /framework/src/sculptor/framework/HEntity.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package sculptor.framework; 19 | 20 | import java.io.Serializable; 21 | import java.lang.reflect.Field; 22 | import java.util.ArrayList; 23 | import java.util.HashSet; 24 | import java.util.List; 25 | import java.util.Set; 26 | 27 | import org.apache.commons.lang.StringUtils; 28 | import org.apache.hadoop.hbase.util.Bytes; 29 | 30 | import sculptor.framework.annotation.Column; 31 | import sculptor.framework.annotation.Rowkey; 32 | import sculptor.framework.annotation.Table; 33 | 34 | /** 35 | * HBase上のデータを表す 36 | * 37 | */ 38 | public class HEntity implements Serializable { 39 | 40 | private static final long serialVersionUID = -1195335302706430611L; 41 | 42 | /** 43 | * Get the class descriptor 44 | * 45 | * @param clazz the class 46 | * @return class descriptor 47 | */ 48 | public static HClassDescriptor getClassInfo(Class clazz) { 49 | HClassDescriptor hClassDescriptor = new HClassDescriptor(); 50 | hClassDescriptor.entityClassName = clazz.getCanonicalName(); 51 | hClassDescriptor.table = clazz.getAnnotation(Table.class).name(); 52 | hClassDescriptor.hFieldDescriptors = getFields(clazz); 53 | hClassDescriptor.columnFamilies = getColumnFamilies(clazz); 54 | 55 | return hClassDescriptor; 56 | } 57 | 58 | /** 59 | * Get all fields descriptor 60 | * 61 | * @param clazz the class 62 | * @return all fields descriptor 63 | */ 64 | public static List getFields(Class clazz) { 65 | Field[] fields = clazz.getFields(); 66 | List hFields = new ArrayList(); 67 | 68 | for (Field f : fields) { 69 | HFieldDescriptor hf = toHField(f); 70 | hFields.add(hf); 71 | } 72 | 73 | return hFields; 74 | } 75 | 76 | /** 77 | * Get all column families. 78 | * 79 | * @param clazz the entity class 80 | * @return Set contains all column families 81 | */ 82 | public static String[] getColumnFamilies(Class clazz) { 83 | List hfields = getFields(clazz); 84 | Set columnFamilies = new HashSet(); 85 | for (HFieldDescriptor hfield : hfields) { 86 | if (hfield.family != null) { 87 | columnFamilies.add(hfield.family); 88 | } 89 | } 90 | return columnFamilies.toArray(new String[0]); 91 | } 92 | 93 | /** 94 | * Get the maximum length of the qualifiers 95 | * 96 | * @param clazz the class 97 | * @return maximum qualifier 98 | */ 99 | public static int getMaxQulifierLen(Class clazz) { 100 | List fields = getFields(clazz); 101 | int maxLength = 0; 102 | for (HFieldDescriptor field : fields) { 103 | String qualifier = field.getQualifier(); 104 | if (qualifier == null) { 105 | continue; 106 | } 107 | int length = qualifier.length(); 108 | if (length > maxLength) { 109 | maxLength = length; 110 | } 111 | } 112 | return maxLength; 113 | } 114 | 115 | /** 116 | * class情報を取得 117 | * 118 | * @return class情報 119 | */ 120 | public HClassDescriptor getClassInfo() { 121 | return getClassInfo(this.getClass()); 122 | } 123 | 124 | /** 125 | * すべてのfield情報を取得 126 | * 127 | * @return すべてのfield情報 128 | */ 129 | public List getFields() throws SecurityException { 130 | return getFields(this.getClass()); 131 | } 132 | 133 | /** 134 | * 指定のfiledの情報を取得 135 | * 136 | * @param fieldName 137 | * filed name 138 | * @return field情報 139 | * @throws NoSuchFieldException 140 | * @throws SecurityException 141 | */ 142 | public HFieldDescriptor getField(String fieldName) 143 | throws SecurityException, NoSuchFieldException { 144 | Field f = this.getClass().getField(fieldName); 145 | return toHField(f); 146 | } 147 | 148 | private static HFieldDescriptor toHField(Field f) { 149 | HFieldDescriptor hf = new HFieldDescriptor(); 150 | hf.fieldName = f.getName(); 151 | hf.canonicalName = f.getType().getCanonicalName(); 152 | hf.isRowkey = f.getAnnotation(Rowkey.class) != null; 153 | Column column = f.getAnnotation(Column.class); 154 | if (column != null) { 155 | hf.family = column.family(); 156 | hf.qualifier = column.qulifier(); 157 | } 158 | return hf; 159 | } 160 | 161 | protected void appendln(StringBuilder sb, String msg) { 162 | if (msg == null) { 163 | sb.append("").append("\n"); 164 | } else { 165 | sb.append(msg).append("\n"); 166 | } 167 | } 168 | 169 | protected void appendln(StringBuilder sb, byte msg) { 170 | sb.append(msg).append("\n"); 171 | } 172 | 173 | protected void appendln(StringBuilder sb, byte[] msg) { 174 | sb.append(Bytes.toString(msg)).append("\n"); 175 | } 176 | 177 | protected void appendln(StringBuilder sb, long msg) { 178 | sb.append(String.valueOf(msg)).append("\n"); 179 | } 180 | 181 | protected void appendFieldName(StringBuilder sb, String fieldName, int length) { 182 | String rpad = StringUtils.rightPad(fieldName, length); 183 | sb.append(rpad).append("=> "); 184 | } 185 | 186 | } 187 | -------------------------------------------------------------------------------- /framework/src/sculptor/framework/HFieldDescriptor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package sculptor.framework; 19 | 20 | /** 21 | * HBaseのcolumnをJava classの filedで表す 22 | * 23 | */ 24 | public class HFieldDescriptor { 25 | /** java classのfiled名 */ 26 | String fieldName; 27 | 28 | /** hbase column family */ 29 | String family; 30 | 31 | /** hbase column */ 32 | String qualifier; 33 | 34 | /** hbaseのrowkey項目かどうか */ 35 | boolean isRowkey; 36 | 37 | /** java classのfieldの正規名 */ 38 | String canonicalName; 39 | 40 | public String getFieldName() { 41 | return fieldName; 42 | } 43 | 44 | public String getFamily() { 45 | return family; 46 | } 47 | 48 | public String getQualifier() { 49 | return qualifier; 50 | } 51 | 52 | public boolean isRowkey() { 53 | return isRowkey; 54 | } 55 | 56 | public String getCanonicalName() { 57 | return canonicalName; 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /framework/src/sculptor/framework/HScanner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package sculptor.framework; 19 | 20 | import java.io.Closeable; 21 | import java.io.IOException; 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | 25 | import org.apache.hadoop.hbase.client.Result; 26 | import org.apache.hadoop.hbase.client.ResultScanner; 27 | 28 | /** 29 | * Abstract scanner for table on HBase. 30 | * 31 | * @param the type returned with iteration 32 | */ 33 | public class HScanner implements Closeable { 34 | protected HClient client; 35 | protected ResultScanner rs; 36 | protected boolean closed = false; 37 | 38 | // The next RowResult, possibly pre-read 39 | private D next = null; 40 | 41 | public HScanner(HClient client, ResultScanner rs) { 42 | this.client = client; 43 | this.rs = rs; 44 | } 45 | 46 | public D next() throws IOException { 47 | 48 | // since hasNext() does the real advancing, we call this to 49 | // determine 50 | // if there is a next before proceeding. 51 | if (!hasNext()) { 52 | return null; 53 | } 54 | 55 | // if we get to here, then hasNext() has given us an item to 56 | // return. 57 | // we want to return the item and then null out the next 58 | // pointer, so 59 | // we use a temporary variable. 60 | D temp = next; 61 | next = null; 62 | return temp; 63 | 64 | } 65 | 66 | public List next(int nbRows) throws IOException { 67 | Result[] rows = rs.next(nbRows); 68 | List resultSets = new ArrayList(); 69 | for (Result r : rows) { 70 | D ds = client.toEntity(r); 71 | if (ds != null) { 72 | resultSets.add(ds); 73 | } 74 | } 75 | 76 | return resultSets; 77 | } 78 | 79 | public boolean hasNext() { 80 | if (closed) { 81 | return false; 82 | } 83 | 84 | if (next == null) { 85 | try { 86 | 87 | Result r = rs.next(); 88 | if (r == null) { 89 | return false; 90 | } 91 | next = client.toEntity(r); 92 | return next != null; 93 | 94 | } catch (IOException e) { 95 | throw new RuntimeException(e); 96 | } 97 | } 98 | return true; 99 | } 100 | 101 | public void close() { 102 | if (rs != null) { 103 | rs.close(); 104 | rs = null; 105 | } 106 | closed = true; 107 | } 108 | 109 | } 110 | -------------------------------------------------------------------------------- /framework/src/sculptor/framework/PutWrapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package sculptor.framework; 19 | 20 | import org.apache.hadoop.hbase.client.Put; 21 | import org.apache.hadoop.hbase.util.Bytes; 22 | 23 | /** 24 | * Putのラッパークラス 25 | * 26 | */ 27 | public class PutWrapper { 28 | 29 | /** putインスタンス */ 30 | private Put put = null; 31 | 32 | /** カラムファミリー名 */ 33 | private byte[] family = null; 34 | 35 | // for performance 36 | private static final byte[] dataFamily = Bytes.toBytes("data"); 37 | private static final byte[] metaFamily = Bytes.toBytes("meta"); 38 | 39 | /** 40 | * コンストラクタ 41 | */ 42 | public PutWrapper(Put put) { 43 | this.put = put; 44 | } 45 | 46 | /** 47 | * コンストラクタ 48 | */ 49 | public PutWrapper(Put put, String family) { 50 | this.put = put; 51 | 52 | // for performance 53 | byte[] theFamily; 54 | if ("data".equals(family)) { 55 | theFamily = dataFamily; 56 | } else if ("meta".equals(family)) { 57 | theFamily = metaFamily; 58 | } else { 59 | theFamily = Bytes.toBytes(family); 60 | } 61 | 62 | this.family = theFamily; 63 | } 64 | 65 | /** 66 | * コンストラクタ 67 | */ 68 | public PutWrapper(Put put, byte[] family) { 69 | this.put = put; 70 | this.family = family; 71 | } 72 | 73 | /** 74 | * putインスタンスを返す 75 | */ 76 | public Put getInstance() { 77 | return put; 78 | } 79 | 80 | 81 | // family指定タイプ 82 | public void add(String family, String qualifier, byte[] value) { 83 | put.add(Bytes.toBytes(family), Bytes.toBytes(qualifier), value); 84 | } 85 | 86 | public void add(String family, String qualifier, byte value) { 87 | // byte型変数をbyte配列に変換する 88 | byte[] array = new byte[1]; 89 | array[0] = value; 90 | put.add(Bytes.toBytes(family), Bytes.toBytes(qualifier), array); 91 | } 92 | 93 | public void add(String family, String qualifier, int value) { 94 | put.add(Bytes.toBytes(family), Bytes.toBytes(qualifier), Bytes.toBytes(value)); 95 | } 96 | 97 | public void add(String family, String qualifier, short value) { 98 | put.add(Bytes.toBytes(family), Bytes.toBytes(qualifier), Bytes.toBytes(value)); 99 | } 100 | 101 | public void add(String family, String qualifier, long value) { 102 | put.add(Bytes.toBytes(family), Bytes.toBytes(qualifier), Bytes.toBytes(value)); 103 | } 104 | 105 | public void add(String family, String qualifier, float value) { 106 | put.add(Bytes.toBytes(family), Bytes.toBytes(qualifier), Bytes.toBytes(value)); 107 | } 108 | 109 | public void add(String family, String qualifier, double value) { 110 | put.add(Bytes.toBytes(family), Bytes.toBytes(qualifier), Bytes.toBytes(value)); 111 | } 112 | 113 | public void add(String family, String qualifier, String value) { 114 | put.add(Bytes.toBytes(family), Bytes.toBytes(qualifier), Bytes.toBytes(value)); 115 | } 116 | 117 | public void add(String family, String qualifier, boolean value) { 118 | put.add(Bytes.toBytes(family), Bytes.toBytes(qualifier), Bytes.toBytes(value)); 119 | } 120 | 121 | 122 | // family省略タイプ 123 | public void add(String qualifier, byte[] value) { 124 | put.add(family, Bytes.toBytes(qualifier), value); 125 | } 126 | 127 | public void add(String qualifier, byte value) { 128 | // byte型変数をbyte配列に変換する 129 | byte[] array = new byte[1]; 130 | array[0] = value; 131 | put.add(family, Bytes.toBytes(qualifier), array); 132 | } 133 | 134 | public void add(String qualifier, int value) { 135 | put.add(family, Bytes.toBytes(qualifier), Bytes.toBytes(value)); 136 | } 137 | 138 | public void add(String qualifier, short value) { 139 | put.add(family, Bytes.toBytes(qualifier), Bytes.toBytes(value)); 140 | } 141 | 142 | public void add(String qualifier, long value) { 143 | put.add(family, Bytes.toBytes(qualifier), Bytes.toBytes(value)); 144 | } 145 | 146 | public void add(String qualifier, float value) { 147 | put.add(family, Bytes.toBytes(qualifier), Bytes.toBytes(value)); 148 | } 149 | 150 | public void add(String qualifier, double value) { 151 | put.add(family, Bytes.toBytes(qualifier), Bytes.toBytes(value)); 152 | } 153 | 154 | public void add(String qualifier, String value) { 155 | put.add(family, Bytes.toBytes(qualifier), Bytes.toBytes(value)); 156 | } 157 | 158 | public void add(String qualifier, boolean value) { 159 | put.add(family, Bytes.toBytes(qualifier), Bytes.toBytes(value)); 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /framework/src/sculptor/framework/Sculptor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package sculptor.framework; 19 | 20 | import java.io.BufferedReader; 21 | import java.io.File; 22 | import java.io.FileReader; 23 | import java.util.Map; 24 | import java.util.TreeMap; 25 | 26 | import org.apache.commons.logging.Log; 27 | import org.apache.commons.logging.LogFactory; 28 | 29 | import sculptor.framework.annotation.Table; 30 | 31 | /** 32 | * The bootstrap class. 33 | * 34 | */ 35 | public class Sculptor { 36 | 37 | private static Log log = LogFactory.getLog(Sculptor.class); 38 | public static File sculptorRoot; 39 | 40 | public static Map> entities; 41 | public static Map> clients; 42 | public static Map descriptors; 43 | 44 | /** 45 | * Initialize Sculptor. 46 | * 47 | * @param root 48 | * the root path of Sculptor 49 | * @throws Exception 50 | * Something wrong during the initialization. 51 | */ 52 | public static void initialize(String root) throws Exception { 53 | try { 54 | sculptorRoot = new File(root); 55 | 56 | entities = new TreeMap>(); 57 | clients = new TreeMap>(); 58 | descriptors = new TreeMap(); 59 | 60 | String tables = sculptorRoot.getAbsolutePath() + "/conf/tables"; 61 | String tableDefine; 62 | BufferedReader br = new BufferedReader(new FileReader(tables)); 63 | while ((tableDefine = br.readLine()) != null) { 64 | if ("".equals(tableDefine.trim()) 65 | || tableDefine.startsWith("#")) { 66 | // ignore empty or comment lines 67 | continue; 68 | } 69 | String[] mapping = tableDefine.split(":"); 70 | if (mapping.length < 3) { 71 | throw new Exception("Wrong table mapping: " + tableDefine); 72 | } 73 | 74 | // table name 75 | String tableName = mapping[0].trim(); 76 | log.info(String.format("Loading table %s...", tableName)); 77 | 78 | // load entity 79 | Class entityClass = Class.forName(mapping[1].trim()); 80 | HClassDescriptor descriptor = HEntity.getClassInfo(entityClass); 81 | if (!tableName.equals(descriptor.table)) { 82 | throw new Exception( 83 | String.format( 84 | "Wrong HEntity table annotation, expected: %s, actual: %s", 85 | tableName, descriptor.table)); 86 | } 87 | 88 | // load client 89 | String client = mapping[2].trim(); 90 | Class clientClass = Class.forName(client); 91 | String annotationTableName = ((Table) clientClass 92 | .getAnnotation(Table.class)).name(); 93 | if (!tableName.equals(annotationTableName)) { 94 | throw new Exception( 95 | String.format( 96 | "Wrong HClient table annotation, expected: %s, actual: %s", 97 | tableName, annotationTableName)); 98 | } 99 | descriptor.clientClassName = client; 100 | 101 | entities.put(tableName, entityClass); 102 | clients.put(tableName, clientClass); 103 | descriptors.put(tableName, descriptor); 104 | } 105 | 106 | } catch (Exception e) { 107 | log.error("Can not initialize Sculptor.", e); 108 | throw e; 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /framework/src/sculptor/framework/annotation/Column.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package sculptor.framework.annotation; 19 | 20 | import java.lang.annotation.ElementType; 21 | import java.lang.annotation.Retention; 22 | import java.lang.annotation.RetentionPolicy; 23 | import java.lang.annotation.Target; 24 | 25 | /** 26 | * HBaseのcolumn familyを示す 27 | * 28 | */ 29 | @Retention(RetentionPolicy.RUNTIME) 30 | @Target( { ElementType.FIELD }) 31 | public @interface Column { 32 | String family(); 33 | String qulifier(); 34 | } 35 | -------------------------------------------------------------------------------- /framework/src/sculptor/framework/annotation/Rowkey.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package sculptor.framework.annotation; 19 | 20 | import java.lang.annotation.ElementType; 21 | import java.lang.annotation.Retention; 22 | import java.lang.annotation.RetentionPolicy; 23 | import java.lang.annotation.Target; 24 | 25 | /** 26 | * HBaseのrowkey項目であることを示す 27 | * 28 | */ 29 | @Retention(RetentionPolicy.RUNTIME) 30 | @Target({ ElementType.FIELD }) 31 | public @interface Rowkey { 32 | 33 | } 34 | -------------------------------------------------------------------------------- /framework/src/sculptor/framework/annotation/Table.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package sculptor.framework.annotation; 19 | 20 | import java.lang.annotation.ElementType; 21 | import java.lang.annotation.Retention; 22 | import java.lang.annotation.RetentionPolicy; 23 | import java.lang.annotation.Target; 24 | 25 | /** 26 | * HBase上のテーブルを示す 27 | * 28 | */ 29 | @Retention(RetentionPolicy.RUNTIME) 30 | @Target({ ElementType.TYPE }) 31 | public @interface Table { 32 | String name(); 33 | } 34 | -------------------------------------------------------------------------------- /framework/src/sculptor/framework/util/ByteArray.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package sculptor.framework.util; 19 | 20 | import org.apache.hadoop.hbase.util.Bytes; 21 | 22 | /** 23 | * byte関連処理 24 | * 25 | */ 26 | public class ByteArray { 27 | 28 | public static final byte b0 = Bytes.toBytes("0")[0]; 29 | public static final byte b1 = Bytes.toBytes("1")[0]; 30 | public static final byte b2 = Bytes.toBytes("2")[0]; 31 | public static final byte b3 = Bytes.toBytes("3")[0]; 32 | public static final byte b4 = Bytes.toBytes("4")[0]; 33 | public static final byte b5 = Bytes.toBytes("5")[0]; 34 | public static final byte b6 = Bytes.toBytes("6")[0]; 35 | public static final byte b7 = Bytes.toBytes("7")[0]; 36 | public static final byte b8 = Bytes.toBytes("8")[0]; 37 | public static final byte b9 = Bytes.toBytes("9")[0]; 38 | public static final byte ba = Bytes.toBytes("a")[0]; 39 | public static final byte bb = Bytes.toBytes("b")[0]; 40 | public static final byte bc = Bytes.toBytes("c")[0]; 41 | public static final byte bd = Bytes.toBytes("d")[0]; 42 | public static final byte be = Bytes.toBytes("e")[0]; 43 | public static final byte bf = Bytes.toBytes("f")[0]; 44 | public static final byte bg = Bytes.toBytes("g")[0]; 45 | public static final byte bh = Bytes.toBytes("h")[0]; 46 | public static final byte bi = Bytes.toBytes("i")[0]; 47 | public static final byte bj = Bytes.toBytes("j")[0]; 48 | public static final byte bk = Bytes.toBytes("k")[0]; 49 | public static final byte bl = Bytes.toBytes("l")[0]; 50 | public static final byte bm = Bytes.toBytes("m")[0]; 51 | public static final byte bn = Bytes.toBytes("n")[0]; 52 | public static final byte bo = Bytes.toBytes("o")[0]; 53 | public static final byte bp = Bytes.toBytes("p")[0]; 54 | public static final byte bq = Bytes.toBytes("q")[0]; 55 | public static final byte br = Bytes.toBytes("r")[0]; 56 | public static final byte bs = Bytes.toBytes("s")[0]; 57 | public static final byte bt = Bytes.toBytes("t")[0]; 58 | public static final byte bu = Bytes.toBytes("u")[0]; 59 | public static final byte bv = Bytes.toBytes("v")[0]; 60 | 61 | 62 | /** 32進基数変換テーブル */ 63 | public static final byte[] RADIX_CONVERT_TABLE = new byte[] { 64 | b0 65 | ,b1 66 | ,b2 67 | ,b3 68 | ,b4 69 | ,b5 70 | ,b6 71 | ,b7 72 | ,b8 73 | ,b9 74 | ,ba 75 | ,bb 76 | ,bc 77 | ,bd 78 | ,be 79 | ,bf 80 | ,bg 81 | ,bh 82 | ,bi 83 | ,bj 84 | ,bk 85 | ,bl 86 | ,bm 87 | ,bn 88 | ,bo 89 | ,bp 90 | ,bq 91 | ,br 92 | ,bs 93 | ,bt 94 | ,bu 95 | ,bv 96 | }; 97 | 98 | /** 99 | * 10進数から32進数に基数変換を行います。 100 | * 101 | * @param decimal 10進数（long型なので19桁までの正数値しか扱えない） 102 | * @param length 基数変換後の桁数 103 | * @return 基数変換結果 104 | */ 105 | public static byte[] toByte32Radix(long decimal, int length) { 106 | 107 | // 基数変換結果が最大桁に達しないかもしれないので、とりあえず配列を0で埋めておく。 108 | byte[] bArray = new byte[length]; 109 | for (int i = 0; i < length; i++) { 110 | bArray[i] = RADIX_CONVERT_TABLE[0]; 111 | } 112 | 113 | // 商 114 | long q = decimal; 115 | // 剰余 116 | long rem = 0; 117 | // 配列インデックス 118 | // 基数変換結果は下桁から求まるので、配列の最後から結果を埋めていく。 119 | int i = length - 1; 120 | 121 | /* 基数変換 */ 122 | while (q >= 32) { 123 | // 剰余算 124 | rem = q & ~(Long.MAX_VALUE << 5); 125 | // 除算 126 | q = (q - rem) >> 5; 127 | // 基数変換テーブルから基数表現（0～9,a～v）を取得する。 128 | bArray[i] = RADIX_CONVERT_TABLE[(int)rem]; 129 | // 次の基数変換結果は、上桁にセットする。 130 | i--; 131 | } 132 | bArray[i] = RADIX_CONVERT_TABLE[(int)q]; 133 | 134 | return bArray; 135 | } 136 | 137 | /** 138 | * 32進数に変換した数字を10進に逆変換。 139 | * 140 | * @param byte32 32進数の数字 141 | * @return 10進数字 142 | */ 143 | public static long toDecimal(byte[] byte32) { 144 | int length = byte32.length; 145 | if (length == 0) { 146 | return -1L; 147 | } 148 | 149 | byte b; 150 | int decimal = 0; 151 | int place = 0; 152 | for (int i = length - 1; i >= 0; i--) { 153 | b = byte32[i]; 154 | decimal += (long)Math.pow(32, place) * toSingleDecimal(b); 155 | place++; 156 | } 157 | return decimal; 158 | } 159 | 160 | /** 161 | * 基数変換を行います。 162 | * 163 | * @param decimal 10進数（long型に変換するので19桁までの数値しか扱えない） 164 | * @param length 基数変換後の桁数 165 | * @return 基数変換結果 166 | */ 167 | public static byte[] toByte32Radix(String decimal, int length) { 168 | return toByte32Radix(Long.parseLong(decimal), length); 169 | } 170 | 171 | 172 | /** 173 | * byte配列の連結します。 174 | * 175 | * @param bArrays byte配列 176 | * @return byte配列の連結結果 177 | */ 178 | public static byte[] concatenate(byte[]... bArrays) { 179 | 180 | if (bArrays.length == 0) { 181 | return new byte[0]; 182 | } 183 | 184 | byte[] b = bArrays[0]; 185 | for (int i = 1; i < bArrays.length; i++) { 186 | b = Bytes.add(b, bArrays[i]); 187 | } 188 | return b; 189 | 190 | } 191 | 192 | /** 193 | * 0から31の10進数が対応した32進のbyte変換 194 | * 195 | * @param decimal 10進数(0~31) 196 | * @return 32進のbyte。 197 | * 0~31以外の10進数の場合、-0x01 198 | */ 199 | public static byte toSingleByte32(int decimal) { 200 | if (decimal < 0 || decimal > 31) { 201 | return -0x01; 202 | } 203 | return RADIX_CONVERT_TABLE[decimal]; 204 | } 205 | 206 | /** 207 | * 0~zの32進byteを10進の数字に変換 208 | * 209 | * @param byte32 0~zの32進byte 210 | * @return 10進の数字。 211 | * 0-zの32進byte以外の場合、-1 212 | */ 213 | public static int toSingleDecimal(byte byte32) { 214 | if (byte32 < b0 || byte32 > bv) { 215 | return -1; 216 | } 217 | 218 | int decimal = -1; 219 | if (byte32 == b0) { 220 | decimal = 0; 221 | } else if (byte32 == b1) { 222 | decimal = 1; 223 | } else if (byte32 == b2) { 224 | decimal = 2; 225 | } else if (byte32 == b3) { 226 | decimal = 3; 227 | } else if (byte32 == b4) { 228 | decimal = 4; 229 | } else if (byte32 == b5) { 230 | decimal = 5; 231 | } else if (byte32 == b6) { 232 | decimal = 6; 233 | } else if (byte32 == b7) { 234 | decimal = 7; 235 | } else if (byte32 == b8) { 236 | decimal = 8; 237 | } else if (byte32 == b9) { 238 | decimal = 9; 239 | } else if (byte32 == ba) { 240 | decimal = 10; 241 | } else if (byte32 == bb) { 242 | decimal = 11; 243 | } else if (byte32 == bc) { 244 | decimal = 12; 245 | } else if (byte32 == bd) { 246 | decimal = 13; 247 | } else if (byte32 == be) { 248 | decimal = 14; 249 | } else if (byte32 == bf) { 250 | decimal = 15; 251 | } else if (byte32 == bg) { 252 | decimal = 16; 253 | } else if (byte32 == bh) { 254 | decimal = 17; 255 | } else if (byte32 == bi) { 256 | decimal = 18; 257 | } else if (byte32 == bj) { 258 | decimal = 19; 259 | } else if (byte32 == bk) { 260 | decimal = 20; 261 | } else if (byte32 == bl) { 262 | decimal = 21; 263 | } else if (byte32 == bm) { 264 | decimal = 22; 265 | } else if (byte32 == bn) { 266 | decimal = 23; 267 | } else if (byte32 == bo) { 268 | decimal = 24; 269 | } else if (byte32 == bp) { 270 | decimal = 25; 271 | } else if (byte32 == bq) { 272 | decimal = 26; 273 | } else if (byte32 == br) { 274 | decimal = 27; 275 | } else if (byte32 == bs) { 276 | decimal = 28; 277 | } else if (byte32 == bt) { 278 | decimal = 29; 279 | } else if (byte32 == bu) { 280 | decimal = 30; 281 | } else if (byte32 == bv) { 282 | decimal = 31; 283 | } 284 | 285 | return decimal; 286 | } 287 | 288 | public static void main(String[] args) { 289 | // for unit test 290 | long decimal; 291 | long tmp; 292 | byte[] byte32; 293 | 294 | decimal = 0; 295 | byte32 = toByte32Radix(decimal, 3); 296 | assert ( Bytes.equals(byte32, new byte[] {b0, b0, b0}) ); 297 | 298 | tmp = toDecimal(byte32); 299 | assert ( decimal == tmp ); 300 | 301 | decimal = 48; 302 | byte32 = toByte32Radix(decimal, 4); 303 | assert ( Bytes.equals(byte32, new byte[] {b0, b0, b1, bg}) ); 304 | 305 | tmp = toDecimal(byte32); 306 | assert ( decimal == tmp ); 307 | 308 | System.out.println("unit test passed"); 309 | } 310 | } 311 | -------------------------------------------------------------------------------- /ivy.xml: -------------------------------------------------------------------------------- 1 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /sample/screenshot/sculptor-mapreduce.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hbase-sculptor/sculptor/305fd4505b87b6a5c9797f8aafbec6fd7a23b3d6/sample/screenshot/sculptor-mapreduce.png -------------------------------------------------------------------------------- /sample/screenshot/sculptor-output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hbase-sculptor/sculptor/305fd4505b87b6a5c9797f8aafbec6fd7a23b3d6/sample/screenshot/sculptor-output.png -------------------------------------------------------------------------------- /sample/src/sculptor/sample/HItemData.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package sculptor.sample; 19 | 20 | import java.io.IOException; 21 | import java.util.Calendar; 22 | import java.util.Map; 23 | import java.util.NavigableMap; 24 | 25 | import org.apache.commons.logging.Log; 26 | import org.apache.commons.logging.LogFactory; 27 | import org.apache.hadoop.conf.Configuration; 28 | import org.apache.hadoop.hbase.client.Put; 29 | import org.apache.hadoop.hbase.client.Result; 30 | import org.apache.hadoop.hbase.client.ResultScanner; 31 | import org.apache.hadoop.hbase.client.Scan; 32 | import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; 33 | import org.apache.hadoop.hbase.filter.FilterList; 34 | import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; 35 | import org.apache.hadoop.hbase.util.Bytes; 36 | 37 | import sculptor.framework.HClient; 38 | import sculptor.framework.HCompareOp; 39 | import sculptor.framework.HScanner; 40 | import sculptor.framework.PutWrapper; 41 | import sculptor.framework.annotation.Table; 42 | import sculptor.framework.util.ByteArray; 43 | 44 | /** 45 | * 46 | * The client class for sc_item_data table in HBase. 47 | * 48 | * Note: this class is NOT thread safe 49 | * 50 | */ 51 | @Table(name = "sc_item_data") 52 | public class HItemData extends HClient { 53 | /** 店舗IDの長さ（32進） */ 54 | static final int SIZE_OF_SHOP_ID_32 = 5; 55 | 56 | /** 商品IDの長さ（32進） */ 57 | static final int SIZE_OF_ITEM_ID_32 = 8; 58 | 59 | /** shop IDの最大値(10進) */ 60 | static final int MAX_SHOP_ID = 9999999; 61 | 62 | /** item IDの最大値(10進) */ 63 | static final int MAX_ITEM_ID = Integer.MAX_VALUE; 64 | 65 | /** sc_item_dataのrow keyの長さ */ 66 | static final int SIZE_OF_ROWKEY_ITEM_DATA_32 = 13; 67 | 68 | final byte[] DATA_FAMILY = Bytes.toBytes("data"); 69 | 70 | /** put bufferの初期値 */ 71 | static final int DEFAULT_PUT_BUFFER = 256; 72 | 73 | /** default # of row to cache when excuting a scan */ 74 | static final int DEFAULT_SCAN_CACHE = 256; 75 | 76 | /** default # of row to cache when excuting a scan for key only */ 77 | static final int DEFAULT_SCAN_KEY_ONLY_CACHE = 2048; 78 | 79 | static final byte[] META_FAMILY = Bytes.toBytes("meta"); 80 | 81 | private static Log log = LogFactory.getLog(HItemData.class); 82 | 83 | /** 84 | * the constructor 85 | * 86 | * @param countryCode 87 | * country code 88 | * @param that 89 | * specific configuration 90 | */ 91 | public HItemData(Configuration that) { 92 | super("sc_item_data", that); 93 | } 94 | 95 | /** 96 | * the constructor 97 | */ 98 | public HItemData() { 99 | super("sc_item_data"); 100 | } 101 | 102 | /** 103 | * hbaseのsc_item_dataのrow-keyを生成する 104 | * 105 | * @param shopID 106 | * 店舗ID 107 | * @param itemID 108 | * 商品ID 109 | * @return row-key 110 | */ 111 | public static byte[] encodeRowkey(int shopID, int itemID) { 112 | if (shopID == -1 || itemID == -1) { 113 | return new byte[0]; 114 | } 115 | 116 | // row key: shop_id + item_id 117 | byte[] bshopid = ByteArray.toByte32Radix(shopID, 118 | SIZE_OF_SHOP_ID_32); 119 | byte[] bitemid = ByteArray.toByte32Radix(itemID, 120 | SIZE_OF_ITEM_ID_32); 121 | byte[] rowKey = ByteArray.concatenate(bshopid, bitemid); 122 | return rowKey; 123 | } 124 | 125 | @Override 126 | public byte[] toRowkey(ItemData d) { 127 | return encodeRowkey(d.shopID, d.itemID); 128 | } 129 | 130 | /** 131 | * row keyを文字列にdecode 132 | * 133 | * @param rowkey 134 | * row key 135 | * @return 10進のshop_id-item_id 136 | */ 137 | public static String decodeRowkey(byte[] rowkey) { 138 | int srcOffset = 0; 139 | byte[] bShopID = new byte[SIZE_OF_SHOP_ID_32]; 140 | System.arraycopy(rowkey, srcOffset, bShopID, 0, 141 | SIZE_OF_SHOP_ID_32); 142 | srcOffset += SIZE_OF_SHOP_ID_32; 143 | 144 | byte[] bItemID = new byte[SIZE_OF_ITEM_ID_32]; 145 | System.arraycopy(rowkey, srcOffset, bItemID, 0, 146 | SIZE_OF_ITEM_ID_32); 147 | 148 | long shopID = ByteArray.toDecimal(bShopID); 149 | long itemID = ByteArray.toDecimal(bItemID); 150 | 151 | return String.format("%d-%d", shopID, itemID); 152 | } 153 | 154 | @Override 155 | public Scan getRawScan(ItemData d, Map ops) { 156 | int startShopID = 0; 157 | int startItemID = 0; 158 | 159 | int endShopID = MAX_SHOP_ID; 160 | int endItemID = MAX_ITEM_ID; 161 | 162 | // some performance improvement 163 | // shop_id指定 164 | HCompareOp shopIDOp = ops.get("shop_id"); 165 | if (shopIDOp == HCompareOp.EQUAL) { 166 | startShopID = d.shopID; 167 | endShopID = startShopID; 168 | } 169 | 170 | // item idも指定 171 | HCompareOp itemIDOp = ops.get("item_id"); 172 | if (itemIDOp == HCompareOp.EQUAL) { 173 | startItemID = d.itemID; 174 | endItemID = startItemID; 175 | } 176 | 177 | log.info(String 178 | .format("scan start row, shop_id=%d, item_id=%d", startShopID, startItemID)); 179 | log.info(String 180 | .format("scan stop row, shop_id=%d, item_id=%d", endShopID, endItemID)); 181 | 182 | byte[] startRow = encodeRowkey(startShopID, startItemID); 183 | byte[] endRow = encodeRowkey(endShopID, endItemID); 184 | Scan s = new Scan(startRow, endRow); 185 | s.addFamily(DATA_FAMILY); 186 | s.addFamily(META_FAMILY); 187 | s.setCacheBlocks(false); 188 | s.setMaxVersions(); 189 | s.setCaching(DEFAULT_SCAN_CACHE); 190 | 191 | FilterList fl = new FilterList(); 192 | for (String column : ops.keySet()) { 193 | byte[] value; 194 | byte[] family = DATA_FAMILY; 195 | 196 | if ("ctime".equals(column)) { 197 | value = Bytes.toBytes(d.ctime); 198 | family = META_FAMILY; 199 | 200 | } else if ("shop_id".equals(column)) { 201 | value = Bytes.toBytes(d.shopID); 202 | 203 | } else if ("item_id".equals(column)) { 204 | value = Bytes.toBytes(d.itemID); 205 | 206 | } else if ("genre_id".equals(column)) { 207 | value = Bytes.toBytes(d.genreID); 208 | 209 | } else if ("price".equals(column)) { 210 | value = Bytes.toBytes(d.price); 211 | 212 | } else if ("full_item_url".equals(column)) { 213 | value = Bytes.toBytes(d.fullItemUrl); 214 | 215 | } else if ("item_name".equals(column)) { 216 | value = Bytes.toBytes(d.itemName); 217 | 218 | } else { 219 | // ignore 220 | continue; 221 | } 222 | 223 | byte[] qualifier = Bytes.toBytes(column); 224 | HCompareOp hop = ops.get(column); 225 | CompareOp op = HClient.toCompareOp(hop); 226 | 227 | SingleColumnValueFilter filter = new SingleColumnValueFilter( 228 | family, qualifier, op, value); 229 | filter.setFilterIfMissing(true); 230 | fl.addFilter(filter); 231 | } 232 | 233 | s.setFilter(fl); 234 | return s; 235 | } 236 | 237 | /** 238 | * HBase更新用Putに変換 239 | * 240 | * @param item 241 | * itemdata 242 | * @return put 243 | */ 244 | public Put toPut(ItemData item) { 245 | 246 | byte[] rowKey = encodeRowkey(item.shopID, item.itemID); 247 | if (rowKey.length == 0) { 248 | return null; 249 | } 250 | 251 | Put p = new Put(rowKey); 252 | PutWrapper wrapper = new PutWrapper(p); 253 | 254 | // add column to family "meta" 255 | long ctime; 256 | if (item.ctime != -1l) { 257 | ctime = item.ctime; 258 | } else { 259 | ctime = Calendar.getInstance().getTimeInMillis(); 260 | } 261 | wrapper.add("meta", "ctime", ctime); 262 | 263 | // add column to family "data" 264 | if (item.shopID != -1) { 265 | wrapper.add("data", "shop_id", item.shopID); 266 | } 267 | 268 | if (item.itemID != -1) { 269 | wrapper.add("data", "item_id", item.itemID); 270 | } 271 | 272 | if (item.genreID != -1) { 273 | wrapper.add("data", "genre_id", item.genreID); 274 | } 275 | 276 | if (item.itemName != null) { 277 | wrapper.add("data", "item_name", item.itemName); 278 | } 279 | 280 | if (item.fullItemUrl != null) { 281 | wrapper.add("data", "full_item_url", item.fullItemUrl); 282 | } 283 | 284 | if (item.price != null) { 285 | wrapper.add("data", "price", item.price); 286 | } 287 | 288 | return wrapper.getInstance(); 289 | 290 | } 291 | 292 | /** 293 | * HBaseの一行をItemDataに変換 294 | * 295 | * @param r 296 | * row in HBase 297 | * @return 商品情報 298 | */ 299 | public ItemData toEntity(Result r) { 300 | if (r == null || r.isEmpty()) { 301 | return null; 302 | } 303 | 304 | ItemData item = null; 305 | 306 | // meta family 307 | NavigableMap map = r.getFamilyMap(META_FAMILY); 308 | if (map != null && !map.isEmpty()) { 309 | item = new ItemData(); 310 | 311 | // ctime 312 | byte[] bCtime = map.get(Bytes.toBytes("ctime")); 313 | if (bCtime != null) { 314 | item.ctime = Bytes.toLong(bCtime); 315 | } 316 | } 317 | 318 | // data family 319 | map = r.getFamilyMap(DATA_FAMILY); 320 | if (map != null && !map.isEmpty()) { 321 | if (item == null) { 322 | item = new ItemData(); 323 | } 324 | 325 | // shop ID 326 | byte[] bShopID = map.get(Bytes.toBytes("shop_id")); 327 | if (bShopID != null) { 328 | item.shopID = Bytes.toInt(bShopID); 329 | } 330 | 331 | // item ID 332 | byte[] bItemID = map.get(Bytes.toBytes("item_id")); 333 | if (bItemID != null) { 334 | item.itemID = Bytes.toInt(bItemID); 335 | } 336 | 337 | // genre ID 338 | byte[] bGenreID = map.get(Bytes.toBytes("genre_id")); 339 | if (bGenreID != null) { 340 | item.genreID = Bytes.toInt(bGenreID); 341 | } 342 | 343 | // item name 344 | byte[] bItemName = map.get(Bytes.toBytes("item_name")); 345 | if (bItemName != null) { 346 | item.itemName = Bytes.toString(bItemName); 347 | } 348 | 349 | // full item url 350 | byte[] bFullItemUrl = map.get(Bytes.toBytes("full_item_url")); 351 | if (bFullItemUrl != null) { 352 | item.fullItemUrl = Bytes.toString(bFullItemUrl); 353 | } 354 | 355 | // price 356 | byte[] bPrice = map.get(Bytes.toBytes("price")); 357 | if (bPrice != null) { 358 | item.price = Bytes.toString(bPrice); 359 | } 360 | } 361 | 362 | if (item == null) { 363 | item = new ItemData(); 364 | } 365 | item.rowkey = r.getRow(); 366 | 367 | return item; 368 | } 369 | 370 | @Override 371 | public HScanner scan(ItemData kv, Map ops) 372 | throws IOException { 373 | Scan s = getRawScan(kv, ops); 374 | ResultScanner rs = table.getScanner(s); 375 | return new ItemDataScanner(this, rs); 376 | } 377 | 378 | 379 | // ######## inner class ########## 380 | protected class ItemDataScanner extends HScanner { 381 | 382 | public ItemDataScanner(HClient client, ResultScanner rs) { 383 | super(client, rs); 384 | } 385 | } 386 | } 387 | -------------------------------------------------------------------------------- /sample/src/sculptor/sample/ItemData.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package sculptor.sample; 19 | 20 | import sculptor.framework.HEntity; 21 | import sculptor.framework.annotation.Column; 22 | import sculptor.framework.annotation.Rowkey; 23 | import sculptor.framework.annotation.Table; 24 | 25 | /** 26 | * The entity class for sc_item_data table in HBase. 27 | * 28 | */ 29 | @Table(name = "sc_item_data") 30 | public class ItemData extends HEntity { 31 | 32 | private static final long serialVersionUID = 7861331319218391209L; 33 | public static int maxQulifierLength; 34 | private static int disQulifierLangth; 35 | 36 | static { 37 | maxQulifierLength = HEntity.getMaxQulifierLen(ItemData.class); 38 | disQulifierLangth = maxQulifierLength + 2; 39 | } 40 | 41 | byte[] rowkey; 42 | 43 | public byte[] getRowkey() { 44 | return rowkey; 45 | } 46 | 47 | // data column family 48 | /** 店舗ID */ 49 | @Rowkey 50 | @Column(family = "data", qulifier = "shop_id") 51 | public int shopID = -1; 52 | 53 | /** 商品ID */ 54 | @Rowkey 55 | @Column(family = "data", qulifier = "item_id") 56 | public int itemID = -1; 57 | 58 | /** ジャンルID */ 59 | @Column(family = "data", qulifier = "genre_id") 60 | public int genreID = -1; 61 | 62 | /** 商品名 */ 63 | @Column(family = "data", qulifier = "item_name") 64 | public String itemName; 65 | 66 | /** 商品URL */ 67 | @Column(family = "data", qulifier = "full_item_url") 68 | public String fullItemUrl; 69 | 70 | /** 価格 */ 71 | @Column(family = "data", qulifier = "price") 72 | public String price; 73 | 74 | // meta column family 75 | /** 更新日時 */ 76 | @Column(family = "meta", qulifier = "ctime") 77 | public long ctime = -1l; 78 | 79 | private void appendName(StringBuilder sb, String fieldName) { 80 | super.appendFieldName(sb, fieldName, disQulifierLangth); 81 | } 82 | 83 | @Override 84 | public String toString() { 85 | StringBuilder sb = new StringBuilder(); 86 | 87 | appendln(sb, "----------------------"); 88 | 89 | appendName(sb, "rowkey"); 90 | appendln(sb, rowkey); 91 | 92 | appendName(sb, "shop_id"); 93 | appendln(sb, shopID); 94 | 95 | appendName(sb, "item_id"); 96 | appendln(sb, itemID); 97 | 98 | appendName(sb, "genre_id"); 99 | appendln(sb, genreID); 100 | 101 | appendName(sb, "item_name"); 102 | appendln(sb, itemName); 103 | 104 | appendName(sb, "full_item_url"); 105 | appendln(sb, fullItemUrl); 106 | 107 | appendName(sb, "price"); 108 | appendln(sb, price); 109 | 110 | appendName(sb, "ctime"); 111 | appendln(sb, ctime); 112 | 113 | appendln(sb, "----------------------"); 114 | return sb.toString(); 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /sample/test/data/sc_item_data.tsv: -------------------------------------------------------------------------------- 1 | 200000 1 1000 Item_1 101.99 2 | 200000 2 1000 Item_2 102.99 3 | 200000 3 1000 Item_3 103.99 4 | 200000 4 1000 Item_4 104.99 5 | 200000 5 1000 Item_5 105.99 6 | 200000 6 1000 Item_6 106.99 7 | 200000 7 1000 Item_7 107.99 8 | 200000 8 1000 Item_8 108.99 9 | 200000 9 1000 Item_9 109.99 10 | 200010 10 1000 Item_10 110.99 11 | 200010 11 1000 Item_11 111.99 12 | 200010 12 1000 Item_12 112.99 13 | 200010 13 1000 Item_13 113.99 14 | 200010 14 1000 Item_14 114.99 15 | 200010 15 1000 Item_15 115.99 16 | 200010 16 1000 Item_16 116.99 17 | 200010 17 1000 Item_17 117.99 18 | 200010 18 1000 Item_18 118.99 19 | 200010 19 1000 Item_19 119.99 20 | 200020 20 1000 Item_20 120.99 21 | 200020 21 1000 Item_21 121.99 22 | 200020 22 1000 Item_22 122.99 23 | 200020 23 1000 Item_23 123.99 24 | 200020 24 1000 Item_24 124.99 25 | 200020 25 1000 Item_25 125.99 26 | 200020 26 1000 Item_26 126.99 27 | 200020 27 1000 Item_27 127.99 28 | 200020 28 1000 Item_28 128.99 29 | 200020 29 1000 Item_29 129.99 30 | 200030 30 1000 Item_30 130.99 31 | 200030 31 1000 Item_31 131.99 32 | 200030 32 1000 Item_32 132.99 33 | 200030 33 1000 Item_33 133.99 34 | 200030 34 1000 Item_34 134.99 35 | 200030 35 1000 Item_35 135.99 36 | 200030 36 1000 Item_36 136.99 37 | 200030 37 1000 Item_37 137.99 38 | 200030 38 1000 Item_38 138.99 39 | 200030 39 1000 Item_39 139.99 40 | 200040 40 1000 Item_40 140.99 41 | 200040 41 1000 Item_41 141.99 42 | 200040 42 1000 Item_42 142.99 43 | 200040 43 1000 Item_43 143.99 44 | 200040 44 1000 Item_44 144.99 45 | 200040 45 1000 Item_45 145.99 46 | 200040 46 1000 Item_46 146.99 47 | 200040 47 1000 Item_47 147.99 48 | 200040 48 1000 Item_48 148.99 49 | 200040 49 1000 Item_49 149.99 50 | 200050 50 1000 Item_50 150.99 51 | 200050 51 1000 Item_51 151.99 52 | 200050 52 1000 Item_52 152.99 53 | 200050 53 1000 Item_53 153.99 54 | 200050 54 1000 Item_54 154.99 55 | 200050 55 1000 Item_55 155.99 56 | 200050 56 1000 Item_56 156.99 57 | 200050 57 1000 Item_57 157.99 58 | 200050 58 1000 Item_58 158.99 59 | 200050 59 1000 Item_59 159.99 60 | 200060 60 1000 Item_60 160.99 61 | 200060 61 1000 Item_61 161.99 62 | 200060 62 1000 Item_62 162.99 63 | 200060 63 1000 Item_63 163.99 64 | 200060 64 1000 Item_64 164.99 65 | 200060 65 1000 Item_65 165.99 66 | 200060 66 1000 Item_66 166.99 67 | 200060 67 1000 Item_67 167.99 68 | 200060 68 1000 Item_68 168.99 69 | 200060 69 1000 Item_69 169.99 70 | 200070 70 1000 Item_70 170.99 71 | 200070 71 1000 Item_71 171.99 72 | 200070 72 1000 Item_72 172.99 73 | 200070 73 1000 Item_73 173.99 74 | 200070 74 1000 Item_74 174.99 75 | 200070 75 1000 Item_75 175.99 76 | 200070 76 1000 Item_76 176.99 77 | 200070 77 1000 Item_77 177.99 78 | 200070 78 1000 Item_78 178.99 79 | 200070 79 1000 Item_79 179.99 80 | 200080 80 1000 Item_80 180.99 81 | 200080 81 1000 Item_81 181.99 82 | 200080 82 1000 Item_82 182.99 83 | 200080 83 1000 Item_83 183.99 84 | 200080 84 1000 Item_84 184.99 85 | 200080 85 1000 Item_85 185.99 86 | 200080 86 1000 Item_86 186.99 87 | 200080 87 1000 Item_87 187.99 88 | 200080 88 1000 Item_88 188.99 89 | 200080 89 1000 Item_89 189.99 90 | 200090 90 1000 Item_90 190.99 91 | 200090 91 1000 Item_91 191.99 92 | 200090 92 1000 Item_92 192.99 93 | 200090 93 1000 Item_93 193.99 94 | 200090 94 1000 Item_94 194.99 95 | 200090 95 1000 Item_95 195.99 96 | 200090 96 1000 Item_96 196.99 97 | 200090 97 1000 Item_97 197.99 98 | 200090 98 1000 Item_98 198.99 99 | 200090 99 1000 Item_99 199.99 100 | 200100 100 2000 Item_100 200.99 101 | 200100 101 2000 Item_101 201.99 102 | 200100 102 2000 Item_102 202.99 103 | 200100 103 2000 Item_103 203.99 104 | 200100 104 2000 Item_104 204.99 105 | 200100 105 2000 Item_105 205.99 106 | 200100 106 2000 Item_106 206.99 107 | 200100 107 2000 Item_107 207.99 108 | 200100 108 2000 Item_108 208.99 109 | 200100 109 2000 Item_109 209.99 110 | 200100 110 2000 Item_110 210.99 111 | 200100 111 2000 Item_111 211.99 112 | 200100 112 2000 Item_112 212.99 113 | 200100 113 2000 Item_113 213.99 114 | 200100 114 2000 Item_114 214.99 115 | 200100 115 2000 Item_115 215.99 116 | 200100 116 2000 Item_116 216.99 117 | 200100 117 2000 Item_117 217.99 118 | 200100 118 2000 Item_118 218.99 119 | 200100 119 2000 Item_119 219.99 120 | 200100 120 2000 Item_120 220.99 121 | 200100 121 2000 Item_121 221.99 122 | 200100 122 2000 Item_122 222.99 123 | 200100 123 2000 Item_123 223.99 124 | 200100 124 2000 Item_124 224.99 125 | 200100 125 2000 Item_125 225.99 126 | 200100 126 2000 Item_126 226.99 127 | 200100 127 2000 Item_127 227.99 128 | 200100 128 2000 Item_128 228.99 129 | 200100 129 2000 Item_129 229.99 130 | 200100 130 2000 Item_130 230.99 131 | 200100 131 2000 Item_131 231.99 132 | 200100 132 2000 Item_132 232.99 133 | 200100 133 2000 Item_133 233.99 134 | 200100 134 2000 Item_134 234.99 135 | 200100 135 2000 Item_135 235.99 136 | 200100 136 2000 Item_136 236.99 137 | 200100 137 2000 Item_137 237.99 138 | 200100 138 2000 Item_138 238.99 139 | 200100 139 2000 Item_139 239.99 140 | 200100 140 2000 Item_140 240.99 141 | 200100 141 2000 Item_141 241.99 142 | 200100 142 2000 Item_142 242.99 143 | 200100 143 2000 Item_143 243.99 144 | 200100 144 2000 Item_144 244.99 145 | 200100 145 2000 Item_145 245.99 146 | 200100 146 2000 Item_146 246.99 147 | 200100 147 2000 Item_147 247.99 148 | 200100 148 2000 Item_148 248.99 149 | 200100 149 2000 Item_149 249.99 150 | 200100 150 2000 Item_150 250.99 151 | 200100 151 2000 Item_151 251.99 152 | 200100 152 2000 Item_152 252.99 153 | 200100 153 2000 Item_153 253.99 154 | 200100 154 2000 Item_154 254.99 155 | 200100 155 2000 Item_155 255.99 156 | 200100 156 2000 Item_156 256.99 157 | 200100 157 2000 Item_157 257.99 158 | 200100 158 2000 Item_158 258.99 159 | 200100 159 2000 Item_159 259.99 160 | 200100 160 2000 Item_160 260.99 161 | 200100 161 2000 Item_161 261.99 162 | 200100 162 2000 Item_162 262.99 163 | 200100 163 2000 Item_163 263.99 164 | 200100 164 2000 Item_164 264.99 165 | 200100 165 2000 Item_165 265.99 166 | 200100 166 2000 Item_166 266.99 167 | 200100 167 2000 Item_167 267.99 168 | 200100 168 2000 Item_168 268.99 169 | 200100 169 2000 Item_169 269.99 170 | 200100 170 2000 Item_170 270.99 171 | 200100 171 2000 Item_171 271.99 172 | 200100 172 2000 Item_172 272.99 173 | 200100 173 2000 Item_173 273.99 174 | 200100 174 2000 Item_174 274.99 175 | 200100 175 2000 Item_175 275.99 176 | 200100 176 2000 Item_176 276.99 177 | 200100 177 2000 Item_177 277.99 178 | 200100 178 2000 Item_178 278.99 179 | 200100 179 2000 Item_179 279.99 180 | 200100 180 2000 Item_180 280.99 181 | 200100 181 2000 Item_181 281.99 182 | 200100 182 2000 Item_182 282.99 183 | 200100 183 2000 Item_183 283.99 184 | 200100 184 2000 Item_184 284.99 185 | 200100 185 2000 Item_185 285.99 186 | 200100 186 2000 Item_186 286.99 187 | 200100 187 2000 Item_187 287.99 188 | 200100 188 2000 Item_188 288.99 189 | 200100 189 2000 Item_189 289.99 190 | 200100 190 2000 Item_190 290.99 191 | 200100 191 2000 Item_191 291.99 192 | 200100 192 2000 Item_192 292.99 193 | 200100 193 2000 Item_193 293.99 194 | 200100 194 2000 Item_194 294.99 195 | 200100 195 2000 Item_195 295.99 196 | 200100 196 2000 Item_196 296.99 197 | 200100 197 2000 Item_197 297.99 198 | 200100 198 2000 Item_198 298.99 199 | 200100 199 2000 Item_199 299.99 200 | 200100 200 2000 Item_200 300.99 201 | 200100 201 2000 Item_201 301.99 202 | 200100 202 2000 Item_202 302.99 203 | 200100 203 2000 Item_203 303.99 204 | 200100 204 2000 Item_204 304.99 205 | 200100 205 2000 Item_205 305.99 206 | 200100 206 2000 Item_206 306.99 207 | 200100 207 2000 Item_207 307.99 208 | 200100 208 2000 Item_208 308.99 209 | 200100 209 2000 Item_209 309.99 210 | 200100 210 2000 Item_210 310.99 211 | 200100 211 2000 Item_211 311.99 212 | 200100 212 2000 Item_212 312.99 213 | 200100 213 2000 Item_213 313.99 214 | 200100 214 2000 Item_214 314.99 215 | 200100 215 2000 Item_215 315.99 216 | 200100 216 2000 Item_216 316.99 217 | 200100 217 2000 Item_217 317.99 218 | 200100 218 2000 Item_218 318.99 219 | 200100 219 2000 Item_219 319.99 220 | 200100 220 2000 Item_220 320.99 221 | 200100 221 2000 Item_221 321.99 222 | 200100 222 2000 Item_222 322.99 223 | 200100 223 2000 Item_223 323.99 224 | 200100 224 2000 Item_224 324.99 225 | 200100 225 2000 Item_225 325.99 226 | 200100 226 2000 Item_226 326.99 227 | 200100 227 2000 Item_227 327.99 228 | 200100 228 2000 Item_228 328.99 229 | 200100 229 2000 Item_229 329.99 230 | 200100 230 2000 Item_230 330.99 231 | 200100 231 2000 Item_231 331.99 232 | 200100 232 2000 Item_232 332.99 233 | 200100 233 2000 Item_233 333.99 234 | 200100 234 2000 Item_234 334.99 235 | 200100 235 2000 Item_235 335.99 236 | 200100 236 2000 Item_236 336.99 237 | 200100 237 2000 Item_237 337.99 238 | 200100 238 2000 Item_238 338.99 239 | 200100 239 2000 Item_239 339.99 240 | 200100 240 2000 Item_240 340.99 241 | 200100 241 2000 Item_241 341.99 242 | 200100 242 2000 Item_242 342.99 243 | 200100 243 2000 Item_243 343.99 244 | 200100 244 2000 Item_244 344.99 245 | 200100 245 2000 Item_245 345.99 246 | 200100 246 2000 Item_246 346.99 247 | 200100 247 2000 Item_247 347.99 248 | 200100 248 2000 Item_248 348.99 249 | 200100 249 2000 Item_249 349.99 250 | 200100 250 2000 Item_250 350.99 251 | 200100 251 2000 Item_251 351.99 252 | 200100 252 2000 Item_252 352.99 253 | 200100 253 2000 Item_253 353.99 254 | 200100 254 2000 Item_254 354.99 255 | 200100 255 2000 Item_255 355.99 256 | 200100 256 2000 Item_256 356.99 257 | 200100 257 2000 Item_257 357.99 258 | 200100 258 2000 Item_258 358.99 259 | 200100 259 2000 Item_259 359.99 260 | 200100 260 2000 Item_260 360.99 261 | 200100 261 2000 Item_261 361.99 262 | 200100 262 2000 Item_262 362.99 263 | 200100 263 2000 Item_263 363.99 264 | 200100 264 2000 Item_264 364.99 265 | 200100 265 2000 Item_265 365.99 266 | 200100 266 2000 Item_266 366.99 267 | 200100 267 2000 Item_267 367.99 268 | 200100 268 2000 Item_268 368.99 269 | 200100 269 2000 Item_269 369.99 270 | 200100 270 2000 Item_270 370.99 271 | 200100 271 2000 Item_271 371.99 272 | 200100 272 2000 Item_272 372.99 273 | 200100 273 2000 Item_273 373.99 274 | 200100 274 2000 Item_274 374.99 275 | 200100 275 2000 Item_275 375.99 276 | 200100 276 2000 Item_276 376.99 277 | 200100 277 2000 Item_277 377.99 278 | 200100 278 2000 Item_278 378.99 279 | 200100 279 2000 Item_279 379.99 280 | 200100 280 2000 Item_280 380.99 281 | 200100 281 2000 Item_281 381.99 282 | 200100 282 2000 Item_282 382.99 283 | 200100 283 2000 Item_283 383.99 284 | 200100 284 2000 Item_284 384.99 285 | 200100 285 2000 Item_285 385.99 286 | 200100 286 2000 Item_286 386.99 287 | 200100 287 2000 Item_287 387.99 288 | 200100 288 2000 Item_288 388.99 289 | 200100 289 2000 Item_289 389.99 290 | 200100 290 2000 Item_290 390.99 291 | 200100 291 2000 Item_291 391.99 292 | 200100 292 2000 Item_292 392.99 293 | 200100 293 2000 Item_293 393.99 294 | 200100 294 2000 Item_294 394.99 295 | 200100 295 2000 Item_295 395.99 296 | 200100 296 2000 Item_296 396.99 297 | 200100 297 2000 Item_297 397.99 298 | 200100 298 2000 Item_298 398.99 299 | 200100 299 2000 Item_299 399.99 300 | 200100 300 2000 Item_300 400.99 301 | -------------------------------------------------------------------------------- /sample/test/sculptor/sample/HItemDataTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package sculptor.sample; 19 | 20 | import java.io.BufferedReader; 21 | import java.io.FileReader; 22 | 23 | import junit.framework.TestCase; 24 | 25 | import org.apache.hadoop.hbase.util.Bytes; 26 | import org.junit.Test; 27 | 28 | import sculptor.framework.util.ByteArray; 29 | 30 | public class HItemDataTest extends TestCase { 31 | 32 | public static void main(String[] args) { 33 | if (args.length < 2) { 34 | usage(); 35 | System.exit(-1); 36 | } 37 | String action = args[0]; 38 | if ("loaddata".equals(action)) { 39 | String dataFile = args[1]; 40 | loadData(dataFile); 41 | } else { 42 | usage(); 43 | System.exit(-1); 44 | } 45 | } 46 | 47 | @Test 48 | public void testEncodeRowkey() { 49 | byte b0 = ByteArray.b0; 50 | 51 | int shopID = 48; 52 | int itemID = 51; 53 | byte[] rowkey = HItemData.encodeRowkey(shopID, itemID); 54 | 55 | byte[] expected = new byte[] { b0, b0, b0, ByteArray.b1, ByteArray.bg, 56 | b0, b0, b0, b0, b0, b0, ByteArray.b1, ByteArray.bj }; 57 | assertEquals(true, Bytes.equals(rowkey, expected)); 58 | } 59 | 60 | @Test 61 | public void testDecodeRowkey() { 62 | int shopID = 48; 63 | int itemID = 51; 64 | byte[] rowkey = HItemData.encodeRowkey(shopID, itemID); 65 | String decoded = HItemData.decodeRowkey(rowkey); 66 | assertEquals("48-51", decoded); 67 | } 68 | 69 | // TODO add test case 70 | 71 | private static void loadData(String dataFile) { 72 | BufferedReader br = null; 73 | HItemData client = null; 74 | try { 75 | 76 | br = new BufferedReader(new FileReader(dataFile)); 77 | client = new HItemData(); 78 | String line; 79 | while ((line = br.readLine()) != null) { 80 | String[] columns = line.split("\\t"); 81 | ItemData item = new ItemData(); 82 | item.shopID = Integer.valueOf(columns[0]).intValue(); 83 | item.itemID = Integer.valueOf(columns[1]).intValue(); 84 | item.genreID = Integer.valueOf(columns[2]).intValue(); 85 | item.itemName = columns[3]; 86 | item.price = columns[4]; 87 | 88 | client.put(item); 89 | } 90 | 91 | } catch (Exception e) { 92 | System.err.println("Error loading data from " + dataFile + " into HBase table."); 93 | e.printStackTrace(); 94 | } finally { 95 | if (br != null) { 96 | try { 97 | br.close(); 98 | } catch (Exception e) { 99 | // ignore 100 | } 101 | } 102 | if (client != null) { 103 | client.close(); 104 | } 105 | } 106 | } 107 | 108 | private static void usage() { 109 | System.err.println("Usage:"); 110 | System.err.println("HItemDataTest loaddata "); 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /sculptor: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2010 The Apache Software Foundation 4 | # 5 | # Licensed to the Apache Software Foundation (ASF) under one 6 | # or more contributor license agreements. See the NOTICE file 7 | # distributed with this work for additional information 8 | # regarding copyright ownership. The ASF licenses this file 9 | # to you under the Apache License, Version 2.0 (the 10 | # "License"); you may not use this file except in compliance 11 | # with the License. You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | 22 | bin=`dirname "$0"` 23 | bin=`cd "$bin"; pwd` 24 | 25 | HBASE_HOME=/usr/local/hbase/current 26 | 27 | # add sculptor.jar to HBase classpath 28 | for jar in $bin/sculptor-*.jar 29 | do 30 | HBASE_CLASSPATH=$HBASE_CLASSPATH:$jar 31 | done 32 | 33 | # add user defined libraries 34 | if [ -z "$SCULPTOR_AUXLIB_PATH" ]; then 35 | auxlib_path=$bin/auxlib 36 | else 37 | auxlib_path=$SCULPTOR_AUXLIB_PATH 38 | fi 39 | for jar in $auxlib_path/*.jar 40 | do 41 | HBASE_CLASSPATH=$HBASE_CLASSPATH:$jar 42 | done 43 | export HBASE_CLASSPATH=$HBASE_CLASSPATH 44 | 45 | function init_and_run_sample() 46 | { 47 | # create sample sc_item_data table 48 | echo "Creating sample table in HBase..." 49 | echo "create 'sc_item_data', {NAME => 'data', VERSIONS => '1'}, {NAME => 'meta', VERSIONS => '1'}" | $HBASE_HOME/bin/hbase shell 50 | echo "" 51 | echo "Created sample table: sc_item_data" 52 | 53 | # load data into the table 54 | echo "Loading test data into sample table..." 55 | $HBASE_HOME/bin/hbase sculptor.sample.HItemDataTest loaddata $bin/sample/test/data/sc_item_data.tsv 56 | 57 | # start sculptor 58 | ${HBASE_HOME}/bin/hbase org.jruby.Main ${bin}/framework/ruby/sculptor.rb 59 | } 60 | 61 | 62 | ## main routine 63 | case $1 in 64 | -e ) 65 | ${HBASE_HOME}/bin/hbase org.jruby.Main ${bin}/framework/ruby/sculptor.rb $2 66 | ;; 67 | sample ) 68 | init_and_run_sample 69 | ;; 70 | * ) 71 | ${HBASE_HOME}/bin/hbase org.jruby.Main ${bin}/framework/ruby/sculptor.rb 72 | esac 73 | 74 | --------------------------------------------------------------------------------