├── tests ├── lib │ ├── __init__.py │ ├── test_config.py │ └── restriction.py ├── enum_test.py ├── decoder_test.py ├── decoder_unittest.py ├── __init__.py ├── utils_test.py ├── client_unittest.py ├── auth_unittest.py ├── timeseries_flat_buffer_unittest.py ├── timeseries_flat_buffer_test.py ├── async_test.py └── sql_query_test.py ├── tablestore ├── flatbuffer │ ├── __init__.py │ ├── dataprotocol │ │ ├── __init__.py │ │ ├── DataType.py │ │ ├── BytesValue.py │ │ ├── SQLResponseColumn.py │ │ ├── SQLResponseColumns.py │ │ └── RLEStringValues.py │ ├── timeseries │ │ ├── __init__.py │ │ ├── DataType.py │ │ ├── Tag.py │ │ ├── BytesValue.py │ │ ├── FlatBufferRows.py │ │ ├── FlatBufferRowInGroup.py │ │ └── FlatBufferRowGroup.py │ ├── sql.fbs │ ├── timeseries.fbs │ └── flat_buffer_decoder.py ├── protobuf │ ├── __init__.py │ ├── table_store_filter.proto │ └── table_store_filter_pb2.py ├── plainbuffer │ ├── __init__.py │ ├── plain_buffer_consts.py │ ├── plain_buffer_crc8.py │ └── plain_buffer_stream.py ├── const.py ├── const_module.py ├── consts.py ├── credentials.py ├── types.py ├── error.py ├── utils.py ├── timeseries_condition.py ├── __init__.py ├── connection.py ├── aggregation.py ├── retry.py └── auth.py ├── MANIFEST.in ├── bin ├── flatc-mac-22.9.24 │ └── flatc-mac ├── protoc-25.0-linux-x86_64 │ └── protoc └── protoc-25.0-osx-universal_binary │ └── protoc ├── examples ├── example_config.py ├── set_tls.py ├── put_row.py ├── put_get_timeseries_data.py ├── create_timeseries_table.py ├── timeseries_meta_operations.py ├── delete_row.py ├── update_row.py ├── sql_query.py ├── secondary_index_operations.py ├── table_operations.py ├── batch_write_row.py ├── batch_get_row.py ├── pk_auto_incr.py ├── get_row.py ├── transaction_and_abort.py ├── transaction_and_commit.py ├── get_range.py ├── parallel_scan.py └── vector_search.py ├── tests_run.sh ├── flatc.sh ├── pyproject.toml ├── LICENSE ├── .gitignore ├── tests_setup.sh ├── setup.py ├── CHANGELOG.rst ├── protoc.sh └── README.rst /tests/lib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tablestore/flatbuffer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tablestore/protobuf/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tablestore/plainbuffer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tablestore/flatbuffer/dataprotocol/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tablestore/flatbuffer/timeseries/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE README.rst CHANGELOG.rst 2 | recursive-include tests *.py 3 | recursive-include examples *.py 4 | -------------------------------------------------------------------------------- /bin/flatc-mac-22.9.24/flatc-mac: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-tablestore-python-sdk/HEAD/bin/flatc-mac-22.9.24/flatc-mac -------------------------------------------------------------------------------- /bin/protoc-25.0-linux-x86_64/protoc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-tablestore-python-sdk/HEAD/bin/protoc-25.0-linux-x86_64/protoc -------------------------------------------------------------------------------- /bin/protoc-25.0-osx-universal_binary/protoc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-tablestore-python-sdk/HEAD/bin/protoc-25.0-osx-universal_binary/protoc -------------------------------------------------------------------------------- /tablestore/flatbuffer/timeseries/DataType.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: timeseries 4 | 5 | class DataType(object): 6 | NONE = 0 7 | LONG = 1 8 | BOOLEAN = 2 9 | DOUBLE = 3 10 | STRING = 4 11 | BINARY = 5 12 | -------------------------------------------------------------------------------- /tests/lib/test_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | OTS_ACCESS_KEY_ID = os.getenv("OTS_TEST_ACCESS_KEY_ID") 4 | OTS_ACCESS_KEY_SECRET = os.getenv("OTS_TEST_ACCESS_KEY_SECRET") 5 | OTS_ENDPOINT = os.getenv("OTS_TEST_ENDPOINT") 6 | OTS_INSTANCE = os.getenv("OTS_TEST_INSTANCE") 7 | OTS_REGION = os.getenv("OTS_TEST_REGION") 8 | -------------------------------------------------------------------------------- /tablestore/flatbuffer/dataprotocol/DataType.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: dataprotocol 4 | 5 | class DataType(object): 6 | NONE = 0 7 | LONG = 1 8 | BOOLEAN = 2 9 | DOUBLE = 3 10 | STRING = 4 11 | BINARY = 5 12 | STRING_RLE = 6 13 | -------------------------------------------------------------------------------- /examples/example_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | OTS_ACCESS_KEY_ID = os.getenv("OTS_TEST_ACCESS_KEY_ID") 4 | OTS_ACCESS_KEY_SECRET = os.getenv("OTS_TEST_ACCESS_KEY_SECRET") 5 | OTS_ENDPOINT = os.getenv("OTS_TEST_ENDPOINT") 6 | OTS_INSTANCE = os.getenv("OTS_TEST_INSTANCE") 7 | OTS_STS_TOKEN = os.getenv('OTS_TEST_STS_TOKEN') 8 | OTS_REGION = os.getenv('OTS_TEST_REGION') 9 | -------------------------------------------------------------------------------- /tests_run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR=$(cd $(dirname \$0); pwd) 4 | cd ${DIR} 5 | 6 | # 遍历所有.py文件并运行unittest 7 | for file in tests/*_test.py; do 8 | if [ -f "$file" ]; then 9 | echo "Running tests in $file..." 10 | python -m unittest "$file" 11 | else 12 | echo "No .py files found in tests directory." 13 | fi 14 | done 15 | -------------------------------------------------------------------------------- /tablestore/const.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | class Const(object): 4 | class ConstError(TypeError): 5 | pass 6 | class ConstCaseError(ConstError): 7 | pass 8 | 9 | def __setattr__(self, name, value): 10 | if name in self.__dict__.keys(): 11 | raise self.ConstError("can't change const.%s" % name) 12 | 13 | self.__dict__[name] = value 14 | 15 | import sys 16 | sys.modules[__name__] = Const() 17 | -------------------------------------------------------------------------------- /tablestore/const_module.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | import sys 3 | 4 | class ConstModule(object): 5 | class ConstError(TypeError): 6 | pass 7 | 8 | class ConstCaseError(ConstError): 9 | pass 10 | 11 | def __setattr__(self, name, value): 12 | if name in self.__dict__.keys(): 13 | raise self.ConstError("can't change const.%s" % name) 14 | self.__dict__[name] = value 15 | 16 | sys.modules[__name__] = ConstModule() 17 | -------------------------------------------------------------------------------- /tablestore/consts.py: -------------------------------------------------------------------------------- 1 | V4_SIGNATURE_SALT = 'ots' 2 | V4_SIGNATURE_PRODUCT = 'ots' 3 | V4_SIGNATURE_SEED = 'aliyun' 4 | V4_SIGNATURE_PREFIX = V4_SIGNATURE_SEED + '_v4' 5 | V4_SIGNATURE_CONSTANT = V4_SIGNATURE_SEED + '_v4_request' 6 | 7 | V4_SIGNATURE_SIGN_DATE_FORMAT = '%Y%m%d' 8 | 9 | OTS_HEADER_PREFIX = 'x-ots-' 10 | OTS_HEADER_SIGNATURE = 'x-ots-signature' 11 | OTS_HEADER_SIGNATURE_V4 = 'x-ots-signaturev4' 12 | OTS_HEADER_SIGN_DATE = 'x-ots-signdate' 13 | OTS_HEADER_SIGN_REGION = 'x-ots-signregion' 14 | -------------------------------------------------------------------------------- /flatc.sh: -------------------------------------------------------------------------------- 1 | ./bin/flatc-mac-22.9.24/flatc-mac --python -o "tablestore/flatbuffer/dataprotocol" tablestore/flatbuffer/sql.fbs 2 | mv tablestore/flatbuffer/dataprotocol/tablestore/flatbuffer/dataprotocol/* tablestore/flatbuffer/dataprotocol/ 3 | rm -rf tablestore/flatbuffer/dataprotocol/tablestore 4 | 5 | ./bin/flatc-mac-22.9.24/flatc-mac --python -o "tablestore/flatbuffer/timeseries" tablestore/flatbuffer/timeseries.fbs 6 | mv tablestore/flatbuffer/timeseries/tablestore/flatbuffer/timeseries/* tablestore/flatbuffer/timeseries/ 7 | rm -rf tablestore/flatbuffer/timeseries/tablestore -------------------------------------------------------------------------------- /examples/set_tls.py: -------------------------------------------------------------------------------- 1 | import os 2 | import ssl 3 | 4 | from tablestore import * 5 | from example_config import * 6 | 7 | # The following code shows how to set the TLS version used by OTSClient. 8 | 9 | access_key_id = OTS_ACCESS_KEY_ID 10 | access_key_secret = OTS_ACCESS_KEY_SECRET 11 | 12 | # Create a connection with TLS version 1.2 13 | ots_client = OTSClient(OTS_ENDPOINT, access_key_id, access_key_secret, OTS_INSTANCE, 14 | region=OTS_REGION, ssl_version=ssl.PROTOCOL_TLSv1_2) 15 | 16 | # do something 17 | resp = ots_client.list_table() 18 | print(resp) 19 | -------------------------------------------------------------------------------- /tests/enum_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from tests.lib.api_test_base import APITestBase 4 | from tablestore import metadata 5 | from tablestore.protobuf import search_pb2 6 | from enum import IntEnum 7 | import inspect 8 | 9 | class EnumTest(APITestBase): 10 | def setUp(self): 11 | pass # no need to set up client 12 | 13 | def tearDown(self): 14 | pass # no need to tearDown client 15 | 16 | def test_IntEnum_equal_int(self): 17 | self.assert_equal(metadata.HighlightEncoder.PLAIN_MODE, search_pb2.PLAIN_MODE) 18 | self.assert_equal(metadata.HighlightEncoder.HTML_MODE, search_pb2.HTML_MODE) 19 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "tablestore" 3 | version = "6.3.0" 4 | description = "Aliyun TableStore(OTS) SDK" 5 | readme = "README.md" 6 | license = "Apache License 2.0" 7 | homepage = "https://www.aliyun.com/product/ots" 8 | authors = ["xunjian.sl "] 9 | packages = [ 10 | { include = "tablestore"}, 11 | ] 12 | include = ["CHANGELOG.md"] 13 | 14 | [tool.poetry.dependencies] 15 | python = "^3.8" 16 | protobuf = ">=3.20.0, <=5.27.4" 17 | urllib3 = ">=1.14" 18 | certifi = ">=2016.2.28" 19 | future = ">=0.16.0" 20 | six = ">=1.11.0" 21 | flatbuffers = ">=22.9.24" 22 | numpy = ">=1.11.0" 23 | crc32c = ">=2.7.1" 24 | aiohttp = ">=3.8, <=3.10" 25 | 26 | [tool.poetry.group.dev.dependencies] 27 | pytest = "^8.3.5" 28 | 29 | [tool.pytest.ini_options] 30 | pythonpath = ["src"] 31 | 32 | [build-system] 33 | requires = ["poetry-core"] 34 | build-backend = "poetry.core.masonry.api" -------------------------------------------------------------------------------- /tablestore/flatbuffer/sql.fbs: -------------------------------------------------------------------------------- 1 | namespace tablestore.flatbuffer.dataprotocol; 2 | 3 | enum DataType:byte {NONE = 0, LONG = 1, BOOLEAN = 2, DOUBLE = 3, STRING = 4, BINARY = 5, STRING_RLE = 6} 4 | 5 | table BytesValue { 6 | value: [byte]; 7 | } 8 | 9 | // rle(run-length encoding) format, [a, a, a, b, c, d, a, a] would encode as 10 | // array: [a, b, c, d, a] 11 | // index_mapping: [0, 0, 0, 1, 2, 3, 4, 4] 12 | table RLEStringValues { 13 | array: [string]; 14 | index_mapping: [int32]; 15 | } 16 | 17 | table ColumnValues { 18 | is_nullvalues: [bool]; 19 | long_values: [long]; 20 | bool_values: [bool]; 21 | double_values: [double]; 22 | string_values: [string]; 23 | binary_values: [BytesValue]; 24 | rle_string_values: RLEStringValues; 25 | } 26 | 27 | table SQLResponseColumn { 28 | column_name: string; 29 | column_type: DataType; 30 | column_value: ColumnValues; 31 | } 32 | 33 | table SQLResponseColumns{ 34 | columns: [SQLResponseColumn]; 35 | row_count: int64; 36 | } 37 | 38 | root_type SQLResponseColumns; -------------------------------------------------------------------------------- /tablestore/flatbuffer/timeseries.fbs: -------------------------------------------------------------------------------- 1 | namespace tablestore.flatbuffer.timeseries; 2 | 3 | enum DataType:byte {NONE = 0, LONG = 1, BOOLEAN = 2, DOUBLE = 3, STRING = 4, BINARY = 5} 4 | 5 | table BytesValue { 6 | value: [byte]; 7 | } 8 | 9 | table FieldValues { 10 | long_values: [long]; 11 | bool_values: [bool]; 12 | double_values: [double]; 13 | string_values: [string]; 14 | binary_values: [BytesValue]; 15 | } 16 | 17 | table Tag { 18 | name: string; 19 | value: string; 20 | } 21 | 22 | table FlatBufferRowInGroup { 23 | data_source: string; 24 | tags: string; 25 | time: int64; 26 | field_values: FieldValues; 27 | meta_cache_update_time: uint32; 28 | tag_list: [Tag]; 29 | } 30 | 31 | table FlatBufferRowGroup { 32 | measurement_name: string; 33 | field_names: [string]; 34 | field_types: [DataType]; 35 | rows: [FlatBufferRowInGroup]; 36 | } 37 | 38 | table FlatBufferRows { 39 | row_groups: [FlatBufferRowGroup]; 40 | } 41 | 42 | root_type FlatBufferRows; 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017 aliyun.com 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | 8 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | Software. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /tablestore/credentials.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | 4 | class Credentials(object): 5 | def __init__(self, access_key_id, access_key_secret, security_token=None): 6 | self._access_key_id = access_key_id 7 | self._access_key_secret = access_key_secret 8 | self._security_token = security_token 9 | 10 | def get_access_key_id(self): 11 | return self._access_key_id 12 | 13 | def get_access_key_secret(self): 14 | return self._access_key_secret 15 | 16 | def get_security_token(self): 17 | return self._security_token 18 | 19 | 20 | class CredentialsProvider(ABC): 21 | @abstractmethod 22 | def get_credentials(self) -> Credentials: 23 | pass 24 | 25 | 26 | class StaticCredentialsProvider(CredentialsProvider): 27 | def __init__(self, access_key_id: str, access_key_secret: str = None, security_token: str = None): 28 | self._credentials = Credentials(access_key_id, access_key_secret, security_token) 29 | 30 | def get_credentials(self) -> Credentials: 31 | return self._credentials 32 | -------------------------------------------------------------------------------- /tablestore/types.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from tablestore.utils import DefaultJsonObject 4 | 5 | class PrimaryKey(DefaultJsonObject): 6 | def __init__(self): 7 | self.pks = [] 8 | 9 | def add_primary_key(self, pk): 10 | self.pks.append(pk) 11 | 12 | def get_primary_keys(self): 13 | return self.pks 14 | 15 | def get_primary_key_size(self): 16 | return len(self.pks) 17 | 18 | def get_primary_key(self, index): 19 | return self.pks[index] 20 | 21 | class PrimaryKeyColumn(DefaultJsonObject): 22 | def __init(self, name, value): 23 | self.name = name 24 | self.value = value 25 | 26 | def get_name(self, name): 27 | self.name = name 28 | 29 | def get_value(self, value): 30 | self.value = value 31 | 32 | class PrimaryKeyValue(DefaultJsonObject): 33 | def __init__(self, pk_type, value): 34 | self.type = pk_type 35 | self.value = value 36 | 37 | def get_type(self): 38 | return self.type 39 | 40 | def get_value(self): 41 | return self.value 42 | 43 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | build 3 | dist 4 | tablestore.egg-info 5 | tests/*.log 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | *.so 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | *.manifest 29 | *.spec 30 | pip-log.txt 31 | pip-delete-this-directory.txt 32 | htmlcov/ 33 | .tox/ 34 | .nox/ 35 | .coverage 36 | .coverage.* 37 | .cache 38 | nosetests.xml 39 | coverage.xml 40 | *.cover 41 | *.py,cover 42 | .hypothesis/ 43 | .pytest_cache/ 44 | cover/ 45 | *.mo 46 | *.pot 47 | *.log 48 | local_settings.py 49 | db.sqlite3 50 | db.sqlite3-journal 51 | instance/ 52 | .webassets-cache 53 | .scrapy 54 | docs/_build/ 55 | .pybuilder/ 56 | target/ 57 | .ipynb_checkpoints 58 | profile_default/ 59 | ipython_config.py 60 | __pypackages__/ 61 | celerybeat-schedule 62 | celerybeat.pid 63 | *.sage.py 64 | .env 65 | .venv 66 | env/ 67 | venv/ 68 | ENV/ 69 | env.bak/ 70 | venv.bak/ 71 | .spyderproject 72 | .spyproject 73 | .ropeproject 74 | /site 75 | .mypy_cache/ 76 | .dmypy.json 77 | dmypy.json 78 | .pyre/ 79 | .pytype/ 80 | cython_debug/ 81 | .idea 82 | .DS_Store 83 | .python-version 84 | -------------------------------------------------------------------------------- /tablestore/error.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | class OTSError(Exception): 4 | pass 5 | 6 | 7 | class OTSClientError(OTSError): 8 | 9 | def __init__(self, message, http_status = None): 10 | self.message = message 11 | self.http_status = http_status 12 | 13 | def get_http_status(self): 14 | return self.http_status 15 | 16 | def __str__(self): 17 | return self.message 18 | 19 | def get_error_message(self): 20 | return self.message 21 | 22 | 23 | class OTSServiceError(OTSError): 24 | 25 | def __init__(self, http_status, code, message, request_id = ''): 26 | self.http_status = http_status 27 | self.code = code 28 | self.message = message 29 | self.request_id = request_id 30 | 31 | def __str__(self): 32 | err_string = "ErrorCode: %s, ErrorMessage: %s, RequestID: %s" % ( 33 | self.code, self.message, self.request_id) 34 | return err_string 35 | 36 | def get_http_status(self): 37 | return self.http_status 38 | 39 | def get_error_code(self): 40 | return self.code 41 | 42 | def get_error_message(self): 43 | return self.message 44 | 45 | def get_request_id(self): 46 | return self.request_id 47 | -------------------------------------------------------------------------------- /tablestore/plainbuffer/plain_buffer_consts.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | import sys 4 | import platform 5 | import tablestore.const_module as const 6 | 7 | const.python_version = sys.version_info[0] 8 | 9 | const.HEADER = 0x75 10 | 11 | # tag type 12 | const.TAG_ROW_PK = chr(0x1) 13 | const.TAG_ROW_DATA = chr(0x2) 14 | const.TAG_CELL = chr(0x3) 15 | const.TAG_CELL_NAME = chr(0x4) 16 | const.TAG_CELL_VALUE = chr(0x5) 17 | const.TAG_CELL_TYPE = chr(0x6) 18 | const.TAG_CELL_TIMESTAMP = chr(0x7) 19 | const.TAG_DELETE_ROW_MARKER = chr(0x8) 20 | const.TAG_ROW_CHECKSUM = chr(0x9) 21 | const.TAG_CELL_CHECKSUM = chr(0x0A) 22 | 23 | # cell op type 24 | const.DELETE_ALL_VERSION = 0x1 25 | const.DELETE_ONE_VERSION = 0x3 26 | const.INCREMENT = 0x4 27 | 28 | # variant type 29 | const.VT_INTEGER = 0x0 30 | const.VT_DOUBLE = 0x1 31 | const.VT_BOOLEAN = 0x2 32 | const.VT_STRING = 0x3 33 | const.VT_NULL = 0x6 34 | const.VT_BLOB = 0x7 35 | const.VT_INF_MIN = 0x9 36 | const.VT_INF_MAX = 0xa 37 | const.VT_AUTO_INCREMENT = 0xb 38 | 39 | # othber 40 | const.LITTLE_ENDIAN_32_SIZE = 4 41 | const.LITTLE_ENDIAN_64_SIZE = 8 42 | const.MAX_BUFFER_SIZE = 64 * 1024 * 1024 43 | 44 | const.SYS_BITS = int(platform.architecture()[0][:2]) 45 | if const.SYS_BITS == 64: 46 | const.LITTLE_ENDIAN_SIZE = const.LITTLE_ENDIAN_64_SIZE 47 | elif const.SYS_BITS == 32: 48 | const.LITTLE_ENDIAN_SIZE = const.LITTLE_ENDIAN_32_SIZE 49 | else: 50 | const.LITTLE_ENDIAN_SIZE = 4 51 | 52 | 53 | -------------------------------------------------------------------------------- /tests_setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # The shell script works as expected when run on MacOS. 4 | 5 | # 第一部分: 设置目录 6 | DIR=$(cd $(dirname \$0); pwd) 7 | cd ${DIR} 8 | 9 | # 第二部分: 移动 tablestore/flatbuffer/dataprotocol 目录 10 | if [ -d "tablestore/flatbuffer/dataprotocol" ]; then 11 | cp -r tablestore/flatbuffer/dataprotocol ./ 12 | else 13 | echo "Directory tablestore/flatbuffer/dataprotocol does not exist." 14 | fi 15 | 16 | if [ -d "tablestore/flatbuffer/timeseries" ]; then 17 | cp -r tablestore/flatbuffer/timeseries ./ 18 | else 19 | echo "Directory tablestore/flatbuffer/timeseries does not exist." 20 | fi 21 | 22 | # 第三部分: 移动 protobuf/py3 目录下的文件 23 | if [ -d "tablestore/protobuf/py3" ]; then 24 | shopt -s nullglob 25 | for file in tablestore/protobuf/py3/*.py; do 26 | if [[ "$file" != *"__init__.py" ]]; then 27 | mv "$file" tablestore/protobuf/ 28 | fi 29 | done 30 | else 31 | echo "Directory tablestore/protobuf/py3 does not exist." 32 | fi 33 | 34 | # 第四部分: 替换 decoder.py 文件内的内容 35 | if [ -f "tablestore/decoder.py" ]; then 36 | sed -i '' 's/tablestore\.flatbuffer\.dataprotocol/dataprotocol/g' tablestore/decoder.py 37 | else 38 | echo "File tablestore/decoder.py does not exist." 39 | fi 40 | 41 | # 第五部分: 替换 flat_buffer_decoder.py 文件内的内容 42 | if [ -f "tablestore/flatbuffer/flat_buffer_decoder.py" ]; then 43 | sed -i '' 's/tablestore\.flatbuffer\.dataprotocol/dataprotocol/g' tablestore/flatbuffer/flat_buffer_decoder.py 44 | sed -i '' 's/\.dataprotocol/dataprotocol/g' tablestore/flatbuffer/flat_buffer_decoder.py 45 | else 46 | echo "File tablestore/flatbuffer/flat_buffer_decoder.py does not exist." 47 | fi 48 | 49 | echo "Script execution completed." 50 | -------------------------------------------------------------------------------- /tablestore/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | import json 4 | import datetime 5 | from enum import Enum 6 | import struct 7 | 8 | class MyEncoder(json.JSONEncoder): 9 | def default(self, obj): 10 | if isinstance(obj, Enum): 11 | return obj.name 12 | elif isinstance(obj, datetime.datetime): 13 | return obj.strftime("%Y-%m-%d %H:%M:%S") 14 | elif isinstance(obj, datetime.date): 15 | return obj.strftime("%Y-%m-%d") 16 | elif isinstance(obj, object): 17 | return obj.__dict__ 18 | return json.JSONEncoder.default(self, obj) 19 | 20 | class DefaultJsonObject(object): 21 | def __repr__(self): 22 | return json.dumps(self, cls=MyEncoder, indent=2) 23 | 24 | 25 | class VectorUtils: 26 | @staticmethod 27 | def floats_to_bytes(floats): 28 | if not isinstance(floats, (list, tuple)) or not all(isinstance(f, float) for f in floats): 29 | raise TypeError("Input must be a list/tuple of floats") 30 | if len(floats) == 0: 31 | raise ValueError("vector is empty") 32 | return bytearray(struct.pack('<' + 'f' * len(floats), *floats)) 33 | 34 | @staticmethod 35 | def bytes_to_floats(byte_data): 36 | if not isinstance(byte_data, bytearray): 37 | raise TypeError("Input must be a bytearray object") 38 | num_floats = len(byte_data) // 4 39 | if len(byte_data) % 4 != 0 or num_floats == 0: 40 | raise ValueError("bytes length is not multiple of 4(SIZE_OF_FLOAT32) or length is 0") 41 | floats = struct.unpack('<' + 'f' * num_floats, byte_data) 42 | return list(floats) 43 | 44 | def get_now_utc_datetime(): 45 | return datetime.datetime.now(datetime.timezone.utc) 46 | -------------------------------------------------------------------------------- /tests/decoder_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | import unittest 4 | 5 | 6 | from tests.lib.api_test_base import APITestBase 7 | from tablestore import decoder 8 | 9 | 10 | class DecoderTest(APITestBase): 11 | 12 | 13 | """DecoderTest""" 14 | 15 | def test_decode_timeseries_tag_or_attribute(self): 16 | d = decoder.OTSProtoBufferDecoder("utf-8") 17 | 18 | attri = d._parse_timeseries_tag_or_attribute("[]") 19 | self.assert_equal(len(attri), 0) 20 | 21 | attri = d._parse_timeseries_tag_or_attribute("[\"a=a1\",\"b=b2\",\"c=0.3\"]") 22 | self.assert_equal(len(attri), 3) 23 | try: 24 | d._parse_timeseries_tag_or_attribute("[a=a1\",\"b=b2\",\"c=0.3\"]") 25 | self.fail("should have failed") 26 | except Exception as e: 27 | self.assertTrue(e is not None) 28 | 29 | try: 30 | d._parse_timeseries_tag_or_attribute("[\"a==a1\",\"b=b2\",\"c=0.3\"]") 31 | self.fail("should have failed") 32 | except Exception as e: 33 | self.assertTrue(e is not None) 34 | 35 | try: 36 | d._parse_timeseries_tag_or_attribute("[\"a==a1\",\"b=b2=0.3\"]") 37 | self.fail("should have failed") 38 | except Exception as e: 39 | self.assertTrue(e is not None) 40 | 41 | try: 42 | d._parse_timeseries_tag_or_attribute("[\"a=a1\",]") 43 | self.fail("should have failed") 44 | except Exception as e: 45 | self.assertTrue(e is not None) 46 | 47 | try: 48 | d._parse_timeseries_tag_or_attribute("[\"a=a1\"") 49 | self.fail("should have failed") 50 | except Exception as e: 51 | self.assertTrue(e is not None) 52 | 53 | 54 | if __name__ == '__main__': 55 | unittest.main() -------------------------------------------------------------------------------- /tests/decoder_unittest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | import unittest 4 | 5 | 6 | from tests.lib.api_test_base import APITestBase 7 | from tablestore import decoder 8 | 9 | 10 | class DecoderTest(APITestBase): 11 | 12 | 13 | """DecoderTest""" 14 | 15 | def test_decode_timeseries_tag_or_attribute(self): 16 | d = decoder.OTSProtoBufferDecoder("utf-8") 17 | 18 | attri = d._parse_timeseries_tag_or_attribute("[]") 19 | self.assert_equal(len(attri), 0) 20 | 21 | attri = d._parse_timeseries_tag_or_attribute("[\"a=a1\",\"b=b2\",\"c=0.3\"]") 22 | self.assert_equal(len(attri), 3) 23 | try: 24 | d._parse_timeseries_tag_or_attribute("[a=a1\",\"b=b2\",\"c=0.3\"]") 25 | self.fail("should have failed") 26 | except Exception as e: 27 | self.assertTrue(e is not None) 28 | 29 | try: 30 | d._parse_timeseries_tag_or_attribute("[\"a==a1\",\"b=b2\",\"c=0.3\"]") 31 | self.fail("should have failed") 32 | except Exception as e: 33 | self.assertTrue(e is not None) 34 | 35 | try: 36 | d._parse_timeseries_tag_or_attribute("[\"a==a1\",\"b=b2=0.3\"]") 37 | self.fail("should have failed") 38 | except Exception as e: 39 | self.assertTrue(e is not None) 40 | 41 | try: 42 | d._parse_timeseries_tag_or_attribute("[\"a=a1\",]") 43 | self.fail("should have failed") 44 | except Exception as e: 45 | self.assertTrue(e is not None) 46 | 47 | try: 48 | d._parse_timeseries_tag_or_attribute("[\"a=a1\"") 49 | self.fail("should have failed") 50 | except Exception as e: 51 | self.assertTrue(e is not None) 52 | 53 | 54 | if __name__ == '__main__': 55 | unittest.main() -------------------------------------------------------------------------------- /examples/put_row.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from example_config import * 4 | 5 | from tablestore import * 6 | from tablestore.retry import WriteRetryPolicy 7 | 8 | import time 9 | 10 | table_name = 'OTSPutRowSimpleExample' 11 | 12 | 13 | def create_table(client): 14 | schema_of_primary_key = [('gid', 'INTEGER'), ('uid', 'INTEGER')] 15 | table_meta = TableMeta(table_name, schema_of_primary_key) 16 | table_options = TableOptions() 17 | reserved_throughput = ReservedThroughput(CapacityUnit(0, 0)) 18 | client.create_table(table_meta, table_options, reserved_throughput) 19 | print('Table has been created.') 20 | 21 | 22 | def delete_table(client): 23 | client.delete_table(table_name) 24 | print('Table \'%s\' has been deleted.' % table_name) 25 | 26 | 27 | def put_row(client): 28 | primary_key = [('gid', 1), ('uid', 101)] 29 | attribute_columns = [('name', '萧峰'), ('mobile', 15100000000), ('address', bytearray('China', 'utf-8')), 30 | ('female', False), ('age', 29.7)] 31 | row = Row(primary_key, attribute_columns) 32 | 33 | condition = Condition(RowExistenceExpectation.EXPECT_NOT_EXIST, 34 | SingleColumnCondition("age", 20, ComparatorType.EQUAL)) 35 | consumed, return_row = client.put_row(table_name, row, condition) 36 | print(u'Write succeed, consume %s write cu.' % consumed.write) 37 | 38 | 39 | if __name__ == '__main__': 40 | client = OTSClient(OTS_ENDPOINT, OTS_ACCESS_KEY_ID, OTS_ACCESS_KEY_SECRET, OTS_INSTANCE, sts_token=OTS_STS_TOKEN, 41 | region=OTS_REGION, retry_policy=WriteRetryPolicy()) 42 | try: 43 | delete_table(client) 44 | except: 45 | pass 46 | create_table(client) 47 | 48 | time.sleep(3) # wait for table ready 49 | 50 | put_row(client) 51 | delete_table(client) 52 | -------------------------------------------------------------------------------- /tablestore/protobuf/table_store_filter.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | package com.aliyun.tablestore.protocol; 3 | 4 | enum VariantType { 5 | VT_INTEGER = 0; 6 | VT_DOUBLE = 1; 7 | //VT_BOOLEAN = 2; 8 | VT_STRING = 3; 9 | VT_NULL = 6; 10 | VT_BLOB = 7; 11 | } 12 | 13 | message ValueTransferRule { 14 | required string regex = 1; 15 | optional VariantType cast_type = 2; 16 | } 17 | 18 | enum FilterType { 19 | FT_SINGLE_COLUMN_VALUE = 1; 20 | FT_COMPOSITE_COLUMN_VALUE = 2; 21 | FT_COLUMN_PAGINATION = 3; 22 | } 23 | 24 | enum ComparatorType { 25 | CT_EQUAL = 1; 26 | CT_NOT_EQUAL = 2; 27 | CT_GREATER_THAN = 3; 28 | CT_GREATER_EQUAL = 4; 29 | CT_LESS_THAN = 5; 30 | CT_LESS_EQUAL = 6; 31 | CT_EXIST = 7; 32 | CT_NOT_EXIST = 8; 33 | } 34 | 35 | message SingleColumnValueFilter { 36 | required ComparatorType comparator = 1; 37 | required string column_name = 2; 38 | required bytes column_value = 3; // Serialized SQLVariant 39 | required bool filter_if_missing = 4; 40 | required bool latest_version_only = 5; 41 | optional ValueTransferRule value_trans_rule = 6; 42 | } 43 | 44 | enum LogicalOperator { 45 | LO_NOT = 1; 46 | LO_AND = 2; 47 | LO_OR = 3; 48 | } 49 | 50 | message CompositeColumnValueFilter { 51 | required LogicalOperator combinator = 1; 52 | repeated Filter sub_filters = 2; 53 | } 54 | 55 | message ColumnPaginationFilter { 56 | required int32 offset = 1; 57 | required int32 limit = 2; 58 | } 59 | 60 | message Filter { 61 | required FilterType type = 1; 62 | required bytes filter = 2; // Serialized string of filter of the type 63 | } 64 | -------------------------------------------------------------------------------- /examples/put_get_timeseries_data.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from example_config import * 4 | from tablestore import * 5 | from tablestore.metadata import * 6 | import time 7 | 8 | 9 | def put_timeseries_data(client: OTSClient, table_name: str): 10 | tags = {"tag1": "t1", "tag2": "t2"} 11 | field1 = {"long_field": 1, "string_field": "string", "bool_field": True, "double_field": 0.3} 12 | field2 = {"binary_field2": bytearray(b'a')} 13 | key2 = TimeseriesKey("measure2", "datasource2", tags) 14 | key1 = TimeseriesKey("measure1", "datasource1", tags) 15 | time1 = time.time() 16 | row1 = TimeseriesRow(key1, field1, int(time1 * 1000000)) 17 | time2 = time.time() 18 | row2 = TimeseriesRow(key2, field2, int(time2 * 1000000)) 19 | rows = [row1, row2] 20 | 21 | client.put_timeseries_data(table_name, rows) 22 | 23 | 24 | def get_timeseries_data(client: OTSClient, table_name: str): 25 | request = GetTimeseriesDataRequest(table_name) 26 | tags = {"tag1": "t1", "tag2": "t2"} 27 | key1 = TimeseriesKey("measure1", "datasource1", tags) 28 | request.timeseriesKey = key1 29 | request.endTimeInUs = int(time.time() * 1000000) 30 | resp = client.get_timeseries_data(request) 31 | print(resp.rows) 32 | print(resp.nextToken) 33 | 34 | 35 | def create_timeseries_data(client: OTSClient, table_name: str): 36 | table_meta = TimeseriesTableMeta(table_name, TimeseriesTableOptions(172800), TimeseriesMetaOptions(None, False)) 37 | request = CreateTimeseriesTableRequest(table_meta) 38 | client.create_timeseries_table(request) 39 | 40 | 41 | if __name__ == '__main__': 42 | client = OTSClient(OTS_ENDPOINT, OTS_ACCESS_KEY_ID, OTS_ACCESS_KEY_SECRET, OTS_INSTANCE, region=OTS_REGION) 43 | table_name = "table_name" 44 | create_timeseries_data(client, table_name) 45 | put_timeseries_data(client, table_name) 46 | get_timeseries_data(client, table_name) -------------------------------------------------------------------------------- /tablestore/flatbuffer/timeseries/Tag.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: timeseries 4 | 5 | import flatbuffers 6 | from flatbuffers.compat import import_numpy 7 | np = import_numpy() 8 | 9 | class Tag(object): 10 | __slots__ = ['_tab'] 11 | 12 | @classmethod 13 | def GetRootAs(cls, buf, offset=0): 14 | n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) 15 | x = Tag() 16 | x.Init(buf, n + offset) 17 | return x 18 | 19 | @classmethod 20 | def GetRootAsTag(cls, buf, offset=0): 21 | """This method is deprecated. Please switch to GetRootAs.""" 22 | return cls.GetRootAs(buf, offset) 23 | # Tag 24 | def Init(self, buf, pos): 25 | self._tab = flatbuffers.table.Table(buf, pos) 26 | 27 | # Tag 28 | def Name(self): 29 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 30 | if o != 0: 31 | return self._tab.String(o + self._tab.Pos) 32 | return None 33 | 34 | # Tag 35 | def Value(self): 36 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 37 | if o != 0: 38 | return self._tab.String(o + self._tab.Pos) 39 | return None 40 | 41 | def TagStart(builder): builder.StartObject(2) 42 | def Start(builder): 43 | return TagStart(builder) 44 | def TagAddName(builder, name): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(name), 0) 45 | def AddName(builder, name): 46 | return TagAddName(builder, name) 47 | def TagAddValue(builder, value): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(value), 0) 48 | def AddValue(builder, value): 49 | return TagAddValue(builder, value) 50 | def TagEnd(builder): return builder.EndObject() 51 | def End(builder): 52 | return TagEnd(builder) -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import re 3 | import sys 4 | 5 | try: 6 | from setuptools import setup 7 | except ImportError: 8 | from distutils.core import setup 9 | 10 | version = '' 11 | with open('tablestore/__init__.py', 'r') as fd: 12 | version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', 13 | fd.read(), re.MULTILINE).group(1) 14 | 15 | if not version: 16 | raise RuntimeError('Cannot find version information') 17 | 18 | 19 | with open('README.rst', 'rb') as f: 20 | readme = f.read().decode('utf-8') 21 | 22 | setup( 23 | name='tablestore', 24 | version=version, 25 | description='Aliyun TableStore(OTS) SDK', 26 | long_description=readme, 27 | packages=['tablestore', 'tablestore.protobuf', 'tablestore.plainbuffer','tablestore.flatbuffer','tablestore.flatbuffer.timeseries','tablestore.flatbuffer.dataprotocol','dataprotocol'], 28 | package_dir={'tablestore.protobuf': 'tablestore/protobuf/','dataprotocol':'tablestore/flatbuffer/dataprotocol'}, 29 | install_requires=['enum34>=1.1.6', 'protobuf>=3.20.0,<=5.27.4', 'urllib3>=1.14', 'certifi>=2016.2.28', 'future>=0.16.0', 'six>=1.11.0', 'flatbuffers>=22.9.24', 'numpy>=1.11.0', 'crc32c>=2.7.1'], 30 | include_package_data=True, 31 | url='https://cn.aliyun.com/product/ots', 32 | classifiers=[ 33 | 'Development Status :: 5 - Production/Stable', 34 | 'Intended Audience :: Developers', 35 | 'License :: OSI Approved :: Apache Software License', 36 | 'Operating System :: OS Independent', 37 | 'Programming Language :: Python', 38 | 'Programming Language :: Python :: 3', 39 | 'Programming Language :: Python :: 3.8', 40 | 'Programming Language :: Python :: 3.9', 41 | 'Programming Language :: Python :: 3.10', 42 | 'Programming Language :: Python :: 3.11', 43 | 'Programming Language :: Python :: 3.12' 44 | ] 45 | ) 46 | 47 | -------------------------------------------------------------------------------- /tests/lib/restriction.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | MaxInstanceNameLength = 16 # The upper limit of Instance name length 4 | MaxTableNameLength = 255 # The upper limit of the Table name length 5 | MaxColumnNameLength = 255 # Maximum length of column names 6 | MaxInstanceCountForUser = 5 # The upper limit of Instances contained in an account 7 | MaxTableCountForInstance = 10 # The upper limit of the number of tables contained in an Instance 8 | MaxPKColumnNum = 4 # Upper limit of the number of columns included in the primary key 9 | MaxPKStringValueLength = 1024 # Upper limit of String type column size (primary key column) 10 | MaxNonPKStringValueLength = 64 * 1024# Upper limit of column size for String type (non-primary key column) 11 | MaxBinaryValueLength = 64 * 1024 # Upper limit for the size of Binary type column values 12 | MaxColumnCountForRow = 100 # Maximum number of columns in a row 13 | MaxColumnDataSizeForRow = 1024 * 1024 # The upper limit of the total size of columns in a row 14 | MaxReadWriteCapacityUnit = 5000 # Upper limit of Capacity Unit on the table 15 | MinReadWriteCapacityUnit = 0 # Lower limit of Capacity Unit on the table 16 | MaxRowCountForMultiGetRow = 100 # Upper limit on the number of rows for a single MultiGetRow operation 17 | MaxRowCountForMultiWriteRow = 200 # The upper limit of the number of rows for a single MultiWriteRow operation 18 | MaxRowCountForGetRange = 5000 # Maximum number of rows returned by a single Query 19 | MaxDataSizeForGetRange = 1024 * 1024 # Upper limit of the data size returned by a single Query 20 | MaxCUReduceTimeLimit = 4 # Maximum number of CU downscaling attempts 21 | 22 | CUUpdateTimeLongest = 60 # Longest response time for UpdateTableCU, unit: s 23 | 24 | CURestoreTimeInSec = 1 # This is not a restriction item, built-in variable, CU recovery time 25 | -------------------------------------------------------------------------------- /examples/create_timeseries_table.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from example_config import * 4 | from tablestore import * 5 | from tablestore.metadata import * 6 | import time 7 | 8 | 9 | def create_timeseries_table(client: OTSClient): 10 | tableOption = TimeseriesTableOptions(172800) 11 | metaOption = TimeseriesMetaOptions(None, False) 12 | tableMeta = TimeseriesTableMeta("table_name", tableOption, metaOption) 13 | analytical_store = TimeseriesAnalyticalStore("as", 2592000, SyncType.SYNC_TYPE_FULL) 14 | lastPointIndex = LastpointIndexMeta("last1") 15 | request = CreateTimeseriesTableRequest(tableMeta, [analytical_store], [lastPointIndex]) 16 | ret = client.create_timeseries_table(request) 17 | print(ret) 18 | 19 | 20 | def create_timeseries_table_with_user_define_key(client: OTSClient): 21 | tableOption = TimeseriesTableOptions(172800) 22 | metaOption = TimeseriesMetaOptions(None, False) 23 | timeseries_keys = ["a", "b"] 24 | field_primary_keys = [('gid', 'INTEGER'), ('uid', 'INTEGER')] 25 | tableMeta = TimeseriesTableMeta("table_name", tableOption, metaOption, timeseries_keys, field_primary_keys) 26 | analytical_store = TimeseriesAnalyticalStore("as", 2592000, SyncType.SYNC_TYPE_FULL) 27 | lastPointIndex = LastpointIndexMeta("last1") 28 | request = CreateTimeseriesTableRequest(tableMeta, [analytical_store], True, [lastPointIndex]) 29 | ret = client.create_timeseries_table(request) 30 | print(ret) 31 | 32 | 33 | def delete_timeseries_table(client: OTSClient): 34 | client.delete_timeseries_table("table_name") 35 | 36 | 37 | def describe_timeseries_table(client: OTSClient): 38 | client.describe_timeseries_table("table_name") 39 | 40 | 41 | if __name__ == '__main__': 42 | client = OTSClient(OTS_ENDPOINT, OTS_ACCESS_KEY_ID, OTS_ACCESS_KEY_SECRET, OTS_INSTANCE, region=OTS_REGION) 43 | create_timeseries_table(client) 44 | time.sleep(30) 45 | describe_timeseries_table(client) 46 | delete_timeseries_table(client) 47 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | Tablestore SDK for Python 版本记录 2 | =========================== 3 | 4 | Python SDK 的版本号遵循 `Semantic Versioning `_ 规则。 5 | 6 | Version 6.3.0 7 | ------------- 8 | - Support async client 9 | - Add async and sync mixed test by RandomOTSClient in tests/__init__.py 10 | 11 | Version 6.2.1 12 | ------------- 13 | - Support SingleColumnRegexCondition filter 14 | 15 | Version 6.2.0 16 | ------------- 17 | - 支持无 AK 方案，OTSClient 支持 credentials_provider 参数。 18 | - 使用 Poetry 管理项目，提高开发和发布效率。 19 | - Flatc 实时编译 fbs 文件。 20 | - 解决 dataprotocol 包不在 tablestore 项目下面的问题，避免包冲突。 21 | - 优化 ut 测试。 22 | 23 | Version 6.1.0 24 | ------------- 25 | - Support some timeseries api. 26 | - Update protobuf to ">=3.20.0,<=5.27.4" 27 | - Refine util shell 'protoc.sh' 28 | 29 | Version 6.0.1 30 | ------------- 31 | - Fix incompatible changes in delete_row 32 | 33 | Version 6.0.0 34 | ------------- 35 | 36 | - Update protobuf from 3.19.0 to 4.25.0 37 | - Support Python 3.8、Python 3.9、Python 3.10、Python 3.11、Python 3.12 38 | - Support Highlight 39 | 40 | Version 5.4.3 41 | ------------- 42 | 43 | - Support SearchIndex Knn Vector Query 44 | 45 | Version 5.2.1 46 | ------------- 47 | 48 | - Optimize SearchResponse 49 | 50 | Version 5.2.0 51 | ------------- 52 | 53 | - Support ParallelScan API 54 | - Support Max/Min/Avg/Sum/Count/DistinctCount 55 | - Support GroupBy API 56 | 57 | Version 4.3.5 58 | ------------- 59 | 60 | - Fix bytearray encode bug 61 | 62 | Version 4.3.4 63 | ------------- 64 | 65 | - replace protobuf-py3 by protobuf 66 | 67 | Version 4.3.2 68 | ------------- 69 | 70 | - remove crcmod 71 | 72 | Version 4.3.0 73 | ------------- 74 | 75 | - Support Python 3.3+ 76 | 77 | Version 4.2.0 78 | ------------- 79 | 80 | - Support STS 81 | 82 | Version 4.1.0 83 | ------------- 84 | 85 | - Support Python 2.6 86 | 87 | Version 4.0.0 88 | ------------- 89 | 90 | - 支持主键列自增功能 91 | - 支持多版本 92 | - 支持TTL 93 | - 增大重试时间 94 | 95 | Version 2.0.8 96 | ------------- 97 | 98 | - 支持https访问和证书验证 99 | 100 | Version 2.0.7 101 | ------------- 102 | 103 | - 根据按量计费方式，调整了示例代码中的预留CU设置 104 | 105 | Version 2.0.6 106 | ------------- 107 | 108 | - 调整了部分异常情况下的重试退避策略 109 | 110 | -------------------------------------------------------------------------------- /examples/timeseries_meta_operations.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from example_config import * 4 | from tablestore import * 5 | from tablestore.metadata import * 6 | import time 7 | 8 | tags = {"tag1": "t1", "tag2": "t2"} 9 | field1 = {"long_field": 1, "string_field": "string", "bool_field": True, "double_field": 0.3} 10 | field2 = {"binary_field2": b'a'} 11 | key2 = TimeseriesKey("measure2", "datasource2", tags) 12 | key1 = TimeseriesKey("measure1", "datasource1", tags) 13 | 14 | def create_timeseries_table(client: OTSClient, table_name: str): 15 | tableOption = TimeseriesTableOptions(172800) 16 | metaOption = TimeseriesMetaOptions(None, False) 17 | tableMeta = TimeseriesTableMeta(table_name, tableOption, metaOption) 18 | request = CreateTimeseriesTableRequest(tableMeta) 19 | ret = client.create_timeseries_table(request) 20 | print(ret) 21 | 22 | 23 | def get_timeseries_meta(client: OTSClient, table_name: str): 24 | request = QueryTimeseriesMetaRequest(table_name) 25 | resp = client.query_timeseries_meta(request) 26 | print(resp.nextToken) 27 | print(resp.timeseriesMetas) 28 | print(resp.totalHits) 29 | 30 | 31 | def update_timeseries_meta(client: OTSClient, table_name: str): 32 | attri = {"aaa": "bbb", "ccc": "dddd"} 33 | meta = TimeseriesMeta(key1, attri) 34 | req = UpdateTimeseriesMetaRequest(table_name, [meta]) 35 | client.update_timeseries_meta(req) 36 | 37 | def delete_timeseries_meta(client: OTSClient, table_name: str): 38 | req = DeleteTimeseriesMetaRequest(table_name, [key2]) 39 | client.delete_timeseries_meta(req) 40 | 41 | 42 | 43 | def put_timeseries_data(client: OTSClient, table_name: str): 44 | time1 = time.time() 45 | row1 = TimeseriesRow(key1, field1, int(time1 * 1000000)) 46 | time2 = time.time() 47 | row2 = TimeseriesRow(key2, field2, int(time2 * 1000000)) 48 | rows = [row1, row2] 49 | 50 | client.put_timeseries_data(table_name, rows) 51 | 52 | 53 | 54 | 55 | if __name__ == '__main__': 56 | client = OTSClient(OTS_ENDPOINT, OTS_ACCESS_KEY_ID, OTS_ACCESS_KEY_SECRET, OTS_INSTANCE, region=OTS_REGION) 57 | table_name = "table_name" 58 | create_timeseries_table(client, table_name) 59 | time.sleep(30) 60 | put_timeseries_data(client, table_name) 61 | 62 | get_timeseries_meta(client, table_name) 63 | update_timeseries_meta(client, table_name) 64 | delete_timeseries_meta(client, table_name) 65 | 66 | client.describe_timeseries_table(table_name) 67 | 68 | 69 | 70 | -------------------------------------------------------------------------------- /tablestore/flatbuffer/timeseries/BytesValue.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: timeseries 4 | 5 | import flatbuffers 6 | from flatbuffers.compat import import_numpy 7 | np = import_numpy() 8 | 9 | class BytesValue(object): 10 | __slots__ = ['_tab'] 11 | 12 | @classmethod 13 | def GetRootAs(cls, buf, offset=0): 14 | n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) 15 | x = BytesValue() 16 | x.Init(buf, n + offset) 17 | return x 18 | 19 | @classmethod 20 | def GetRootAsBytesValue(cls, buf, offset=0): 21 | """This method is deprecated. Please switch to GetRootAs.""" 22 | return cls.GetRootAs(buf, offset) 23 | # BytesValue 24 | def Init(self, buf, pos): 25 | self._tab = flatbuffers.table.Table(buf, pos) 26 | 27 | # BytesValue 28 | def Value(self, j): 29 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 30 | if o != 0: 31 | a = self._tab.Vector(o) 32 | return self._tab.Get(flatbuffers.number_types.Int8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1)) 33 | return 0 34 | 35 | # BytesValue 36 | def ValueAsNumpy(self): 37 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 38 | if o != 0: 39 | return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int8Flags, o) 40 | return 0 41 | 42 | # BytesValue 43 | def ValueLength(self): 44 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 45 | if o != 0: 46 | return self._tab.VectorLen(o) 47 | return 0 48 | 49 | # BytesValue 50 | def ValueIsNone(self): 51 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 52 | return o == 0 53 | 54 | def BytesValueStart(builder): builder.StartObject(1) 55 | def Start(builder): 56 | return BytesValueStart(builder) 57 | def BytesValueAddValue(builder, value): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(value), 0) 58 | def AddValue(builder, value): 59 | return BytesValueAddValue(builder, value) 60 | def BytesValueStartValueVector(builder, numElems): return builder.StartVector(1, numElems, 1) 61 | def StartValueVector(builder, numElems): 62 | return BytesValueStartValueVector(builder, numElems) 63 | def BytesValueEnd(builder): return builder.EndObject() 64 | def End(builder): 65 | return BytesValueEnd(builder) -------------------------------------------------------------------------------- /tablestore/flatbuffer/dataprotocol/BytesValue.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: dataprotocol 4 | 5 | import flatbuffers 6 | from flatbuffers.compat import import_numpy 7 | np = import_numpy() 8 | 9 | class BytesValue(object): 10 | __slots__ = ['_tab'] 11 | 12 | @classmethod 13 | def GetRootAs(cls, buf, offset=0): 14 | n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) 15 | x = BytesValue() 16 | x.Init(buf, n + offset) 17 | return x 18 | 19 | @classmethod 20 | def GetRootAsBytesValue(cls, buf, offset=0): 21 | """This method is deprecated. Please switch to GetRootAs.""" 22 | return cls.GetRootAs(buf, offset) 23 | # BytesValue 24 | def Init(self, buf, pos): 25 | self._tab = flatbuffers.table.Table(buf, pos) 26 | 27 | # BytesValue 28 | def Value(self, j): 29 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 30 | if o != 0: 31 | a = self._tab.Vector(o) 32 | return self._tab.Get(flatbuffers.number_types.Int8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1)) 33 | return 0 34 | 35 | # BytesValue 36 | def ValueAsNumpy(self): 37 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 38 | if o != 0: 39 | return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int8Flags, o) 40 | return 0 41 | 42 | # BytesValue 43 | def ValueLength(self): 44 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 45 | if o != 0: 46 | return self._tab.VectorLen(o) 47 | return 0 48 | 49 | # BytesValue 50 | def ValueIsNone(self): 51 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 52 | return o == 0 53 | 54 | def BytesValueStart(builder): builder.StartObject(1) 55 | def Start(builder): 56 | return BytesValueStart(builder) 57 | def BytesValueAddValue(builder, value): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(value), 0) 58 | def AddValue(builder, value): 59 | return BytesValueAddValue(builder, value) 60 | def BytesValueStartValueVector(builder, numElems): return builder.StartVector(1, numElems, 1) 61 | def StartValueVector(builder, numElems): 62 | return BytesValueStartValueVector(builder, numElems) 63 | def BytesValueEnd(builder): return builder.EndObject() 64 | def End(builder): 65 | return BytesValueEnd(builder) -------------------------------------------------------------------------------- /tablestore/flatbuffer/timeseries/FlatBufferRows.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: timeseries 4 | 5 | import flatbuffers 6 | from flatbuffers.compat import import_numpy 7 | np = import_numpy() 8 | 9 | class FlatBufferRows(object): 10 | __slots__ = ['_tab'] 11 | 12 | @classmethod 13 | def GetRootAs(cls, buf, offset=0): 14 | n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) 15 | x = FlatBufferRows() 16 | x.Init(buf, n + offset) 17 | return x 18 | 19 | @classmethod 20 | def GetRootAsFlatBufferRows(cls, buf, offset=0): 21 | """This method is deprecated. Please switch to GetRootAs.""" 22 | return cls.GetRootAs(buf, offset) 23 | # FlatBufferRows 24 | def Init(self, buf, pos): 25 | self._tab = flatbuffers.table.Table(buf, pos) 26 | 27 | # FlatBufferRows 28 | def RowGroups(self, j): 29 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 30 | if o != 0: 31 | x = self._tab.Vector(o) 32 | x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4 33 | x = self._tab.Indirect(x) 34 | from tablestore.flatbuffer.timeseries.FlatBufferRowGroup import FlatBufferRowGroup 35 | obj = FlatBufferRowGroup() 36 | obj.Init(self._tab.Bytes, x) 37 | return obj 38 | return None 39 | 40 | # FlatBufferRows 41 | def RowGroupsLength(self): 42 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 43 | if o != 0: 44 | return self._tab.VectorLen(o) 45 | return 0 46 | 47 | # FlatBufferRows 48 | def RowGroupsIsNone(self): 49 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 50 | return o == 0 51 | 52 | def FlatBufferRowsStart(builder): builder.StartObject(1) 53 | def Start(builder): 54 | return FlatBufferRowsStart(builder) 55 | def FlatBufferRowsAddRowGroups(builder, rowGroups): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(rowGroups), 0) 56 | def AddRowGroups(builder, rowGroups): 57 | return FlatBufferRowsAddRowGroups(builder, rowGroups) 58 | def FlatBufferRowsStartRowGroupsVector(builder, numElems): return builder.StartVector(4, numElems, 4) 59 | def StartRowGroupsVector(builder, numElems): 60 | return FlatBufferRowsStartRowGroupsVector(builder, numElems) 61 | def FlatBufferRowsEnd(builder): return builder.EndObject() 62 | def End(builder): 63 | return FlatBufferRowsEnd(builder) -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | import random 4 | from collections.abc import AsyncGenerator 5 | 6 | import tablestore 7 | from tablestore import OTSClient, AsyncOTSClient, CredentialsProvider 8 | 9 | 10 | class RandomOTSClient: 11 | def __init__(self, end_point, access_key_id=None, access_key_secret=None, instance_name=None, 12 | credentials_provider: CredentialsProvider = None, region: str = None, **kwargs): 13 | self.sync_client = OTSClient(end_point, access_key_id, access_key_secret, instance_name, credentials_provider, region, **kwargs) 14 | self.async_client = AsyncOTSClient(end_point, access_key_id, access_key_secret, instance_name, credentials_provider, region, **kwargs) 15 | 16 | self.logger = None 17 | self.set_logger() 18 | 19 | func_names = [ 20 | func_name for func_name in dir(OTSClient) 21 | if callable(getattr(OTSClient, func_name)) and not func_name.startswith('_') 22 | ] 23 | for func_name in func_names: 24 | setattr(self, func_name, self.generate_func_by_name(func_name)) 25 | 26 | def set_logger(self): 27 | self.logger = logging.getLogger('APITestBase') 28 | self.logger.setLevel(logging.INFO) 29 | 30 | fh = logging.FileHandler('tablestore_sdk_test.log') 31 | fh.setLevel(logging.INFO) 32 | 33 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') 34 | fh.setFormatter(formatter) 35 | 36 | self.logger.addHandler(fh) 37 | 38 | async def _run_async_client_func(self, func_name, *args, **kwargs): 39 | async with self.async_client: 40 | coroutine_object = getattr(self.async_client, func_name)(*args, **kwargs) 41 | if isinstance(coroutine_object, AsyncGenerator): 42 | result = [item async for item in coroutine_object] 43 | else: 44 | result = await coroutine_object 45 | 46 | return result 47 | 48 | def generate_func_by_name(self, func_name): 49 | def generated_func(*args, **kwargs): 50 | if random.choice([True, False]): 51 | self.logger.info(f'use sync_client for {func_name}') 52 | return getattr(self.sync_client, func_name)(*args, **kwargs) 53 | else: 54 | self.logger.info(f'use async_client for {func_name}') 55 | return asyncio.run(self._run_async_client_func(func_name, *args, **kwargs)) 56 | 57 | return generated_func 58 | 59 | tablestore.OTSClient = RandomOTSClient -------------------------------------------------------------------------------- /examples/delete_row.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from example_config import * 4 | from tablestore import * 5 | from tablestore.metadata import * 6 | import time 7 | 8 | table_name = 'python_sdk_5' 9 | 10 | 11 | def create_table(client): 12 | schema_of_primary_key = [('gid', 'INTEGER'), ('uid', 'STRING')] 13 | table_meta = TableMeta(table_name, schema_of_primary_key) 14 | table_option = TableOptions() 15 | reserved_throughput = ReservedThroughput(CapacityUnit(0, 0)) 16 | client.create_table(table_meta, table_option, reserved_throughput) 17 | print('Table has been created.') 18 | 19 | 20 | def delete_table(client): 21 | client.delete_table(table_name) 22 | print('Table \'%s\' has been deleted.' % table_name) 23 | 24 | 25 | def put_row(client): 26 | primary_key = [('gid', 1), ('uid', "101")] 27 | attribute_columns = [('name', 'John'), ('mobile', 15100000000), ('address', 'China'), ('age', 20)] 28 | row = Row(primary_key, attribute_columns) 29 | condition = Condition( 30 | RowExistenceExpectation.EXPECT_NOT_EXIST) # Expect not exist: put it into table only when this row is not exist. 31 | consumed, return_row = client.put_row(table_name, row) 32 | print('Write succeed, consume %s write cu.' % consumed.write) 33 | 34 | 35 | def delete_row(client): 36 | primary_key = [('gid', 1), ('uid', '101')] 37 | condition = Condition(RowExistenceExpectation.IGNORE, SingleColumnCondition("age", 25, ComparatorType.LESS_THAN)) 38 | consumed, return_row = client.delete_row(table_name, primary_key, condition) 39 | print('Delete succeed, consume %s write cu.' % consumed.write) 40 | 41 | 42 | def get_row(client): 43 | primary_key = [('gid', 1), ('uid', "101")] 44 | columns_to_get = ['name', 'address', 45 | 'age'] # given a list of columns to get, or empty list if you want to get entire row. 46 | consumed, return_row, next_token = client.get_row(table_name, primary_key, columns_to_get, None, 1) 47 | print('Read succeed, consume %s read cu.' % consumed.read) 48 | 49 | if return_row is not None: 50 | print('Value of attribute: %s' % return_row.attribute_columns) 51 | 52 | 53 | if __name__ == '__main__': 54 | client = OTSClient(OTS_ENDPOINT, OTS_ACCESS_KEY_ID, OTS_ACCESS_KEY_SECRET, OTS_INSTANCE, region=OTS_REGION) 55 | try: 56 | delete_table(client) 57 | except: 58 | pass 59 | create_table(client) 60 | 61 | time.sleep(3) # wait for table ready 62 | 63 | put_row(client) 64 | print('#### row before delete ####') 65 | get_row(client) 66 | delete_row(client) 67 | print('#### row after delete ####') 68 | get_row(client) 69 | 70 | delete_table(client) 71 | -------------------------------------------------------------------------------- /tablestore/flatbuffer/dataprotocol/SQLResponseColumn.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: dataprotocol 4 | 5 | import flatbuffers 6 | from flatbuffers.compat import import_numpy 7 | np = import_numpy() 8 | 9 | class SQLResponseColumn(object): 10 | __slots__ = ['_tab'] 11 | 12 | @classmethod 13 | def GetRootAs(cls, buf, offset=0): 14 | n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) 15 | x = SQLResponseColumn() 16 | x.Init(buf, n + offset) 17 | return x 18 | 19 | @classmethod 20 | def GetRootAsSQLResponseColumn(cls, buf, offset=0): 21 | """This method is deprecated. Please switch to GetRootAs.""" 22 | return cls.GetRootAs(buf, offset) 23 | # SQLResponseColumn 24 | def Init(self, buf, pos): 25 | self._tab = flatbuffers.table.Table(buf, pos) 26 | 27 | # SQLResponseColumn 28 | def ColumnName(self): 29 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 30 | if o != 0: 31 | return self._tab.String(o + self._tab.Pos) 32 | return None 33 | 34 | # SQLResponseColumn 35 | def ColumnType(self): 36 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 37 | if o != 0: 38 | return self._tab.Get(flatbuffers.number_types.Int8Flags, o + self._tab.Pos) 39 | return 0 40 | 41 | # SQLResponseColumn 42 | def ColumnValue(self): 43 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) 44 | if o != 0: 45 | x = self._tab.Indirect(o + self._tab.Pos) 46 | from tablestore.flatbuffer.dataprotocol.ColumnValues import ColumnValues 47 | obj = ColumnValues() 48 | obj.Init(self._tab.Bytes, x) 49 | return obj 50 | return None 51 | 52 | def SQLResponseColumnStart(builder): builder.StartObject(3) 53 | def Start(builder): 54 | return SQLResponseColumnStart(builder) 55 | def SQLResponseColumnAddColumnName(builder, columnName): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(columnName), 0) 56 | def AddColumnName(builder, columnName): 57 | return SQLResponseColumnAddColumnName(builder, columnName) 58 | def SQLResponseColumnAddColumnType(builder, columnType): builder.PrependInt8Slot(1, columnType, 0) 59 | def AddColumnType(builder, columnType): 60 | return SQLResponseColumnAddColumnType(builder, columnType) 61 | def SQLResponseColumnAddColumnValue(builder, columnValue): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(columnValue), 0) 62 | def AddColumnValue(builder, columnValue): 63 | return SQLResponseColumnAddColumnValue(builder, columnValue) 64 | def SQLResponseColumnEnd(builder): return builder.EndObject() 65 | def End(builder): 66 | return SQLResponseColumnEnd(builder) -------------------------------------------------------------------------------- /protoc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # Print color 5 | YELLOW='\033[33m' 6 | GREEN='\033[32m' 7 | RED='\033[31m' 8 | RESET='\033[0m' 9 | 10 | binary_linux="bin/protoc-25.0-linux-x86_64/protoc" 11 | binary_osx="bin/protoc-25.0-osx-universal_binary/protoc" 12 | 13 | # Check the current system 14 | system=$(uname -s | tr '[:upper:]' '[:lower:]') 15 | arch=$(uname -m | tr '[:upper:]' '[:lower:]') 16 | 17 | # Determine the appropriate binary file 18 | if [ "$system" == "linux" ] && [ "$arch" == "x86_64" ]; then 19 | binary=$binary_linux 20 | echo "use linux " 21 | elif [ "$system" == "darwin" ]; then 22 | binary=$binary_osx 23 | echo "use osx" 24 | else 25 | echo -e "${RED}当前系统既不是linux-x86_64，也不是osx。请到https://github.com/protocolbuffers/protobuf/releases/tag/ 下载可执行文件${RESET}" 26 | exit 1 27 | fi 28 | 29 | chmod +x "$binary" 30 | 31 | echo "protoc 版本是: $(./$binary --version)" 32 | 33 | 34 | check_and_replace_import_path_in_pb_py() { 35 | proto_file=$1 36 | py_file=$2 37 | # Check if the py file exists 38 | if [ -e "$py_file" ]; then 39 | # If the proto contains import, modify the import path in the generated py file 40 | if grep -q '^import ' "$proto_file"; then 41 | cur_sec=$(date '+%s') 42 | tmp_file=/tmp/temp${cur_sec}.py 43 | 44 | # Modify "import xxx_pb2" to "import tablestore.protobuf.xxx_pb2" 45 | sed 's/^import \([a-zA-Z0-9_]*\)_pb2/import tablestore.protobuf.\1_pb2/g' "$py_file" > "$tmp_file" 46 | # Modify "from xxx_pb2 import" to "from tablestore.protobuf.xxx_pb2 import" 47 | sed 's/^from \([a-zA-Z0-9_]*\)_pb2 import/from tablestore.protobuf.\1_pb2 import/g' "$tmp_file" > "$py_file" 48 | 49 | rm -f $tmp_file 50 | echo -e "${YELLOW}检测到${proto_file}中有import，已修改对应的py文件${py_file}中的import路径${RESET}" 51 | fi 52 | else 53 | echo -e "${RED}错误：${py_file}文件不存在${RESET}" 54 | fi 55 | } 56 | 57 | ./$binary --proto_path=tablestore/protobuf/ --python_out=tablestore/protobuf/ tablestore/protobuf/table_store.proto 58 | ./$binary --proto_path=tablestore/protobuf/ --python_out=tablestore/protobuf/ tablestore/protobuf/table_store_filter.proto 59 | ./$binary --proto_path=tablestore/protobuf/ --python_out=tablestore/protobuf/ tablestore/protobuf/search.proto 60 | ./$binary --proto_path=tablestore/protobuf/ --python_out=tablestore/protobuf/ tablestore/protobuf/timeseries.proto 61 | 62 | check_and_replace_import_path_in_pb_py tablestore/protobuf/table_store.proto tablestore/protobuf/table_store_pb2.py 63 | check_and_replace_import_path_in_pb_py tablestore/protobuf/table_store_filter.proto tablestore/protobuf/table_store_filter_pb2.py 64 | check_and_replace_import_path_in_pb_py tablestore/protobuf/search.proto tablestore/protobuf/search_pb2.py 65 | check_and_replace_import_path_in_pb_py tablestore/protobuf/timeseries.proto tablestore/protobuf/timeseries_pb2.py 66 | 67 | 68 | echo -e "${GREEN}所有命令执行完毕。${RESET}" -------------------------------------------------------------------------------- /examples/update_row.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from example_config import * 4 | from tablestore import * 5 | import time 6 | 7 | table_name = 'python_sdk_4' 8 | 9 | 10 | def create_table(client): 11 | schema_of_primary_key = [('gid', 'INTEGER'), ('uid', 'STRING')] 12 | table_meta = TableMeta(table_name, schema_of_primary_key) 13 | table_options = TableOptions() 14 | reserved_throughput = ReservedThroughput(CapacityUnit(0, 0)) 15 | client.create_table(table_meta, table_options, reserved_throughput) 16 | print('Table has been created.') 17 | 18 | 19 | def delete_table(client): 20 | client.delete_table(table_name) 21 | print('Table \'%s\' has been deleted.' % table_name) 22 | 23 | 24 | def put_row(client): 25 | primary_key = [('gid', 1), ('uid', "101")] 26 | attribute_columns = [('name', 'John'), ('mobile', 15100000000), ('address', 'China'), ('age', 20)] 27 | row = Row(primary_key, attribute_columns) 28 | condition = Condition( 29 | RowExistenceExpectation.EXPECT_NOT_EXIST) # Expect not exist: put it into table only when this row is not exist. 30 | consumed, return_row = client.put_row(table_name, row) 31 | print('Write succeed, consume %s write cu.' % consumed.write) 32 | 33 | 34 | def update_row(client): 35 | primary_key = [('gid', 1), ('uid', "101")] 36 | update_of_attribute_columns = { 37 | 'PUT': [('name', 'David'), ('address', 'Hongkong')], 38 | 'DELETE': [('address', None, 1488436949003)], 39 | 'DELETE_ALL': [('mobile'), ('age')], 40 | 'INCREMENT': [('counter', -1)] 41 | } 42 | row = Row(primary_key, update_of_attribute_columns) 43 | condition = Condition(RowExistenceExpectation.IGNORE, SingleColumnCondition("age", 20, 44 | ComparatorType.EQUAL)) # update row only when this row is exist 45 | consumed, return_row = client.update_row(table_name, row, condition) 46 | print('Update succeed, consume %s write cu.' % consumed.write) 47 | 48 | 49 | def get_row(client): 50 | primary_key = [('gid', 1), ('uid', '101')] 51 | columns_to_get = ['name', 'address', 'age', 52 | 'counter'] # given a list of columns to get, or empty list if you want to get entire row. 53 | consumed, return_row, next_token = client.get_row(table_name, primary_key, columns_to_get, None, 1) 54 | print('Read succeed, consume %s read cu.' % consumed.read) 55 | 56 | print('Value of attribute: %s' % return_row.attribute_columns) 57 | 58 | 59 | if __name__ == '__main__': 60 | client = OTSClient(OTS_ENDPOINT, OTS_ACCESS_KEY_ID, OTS_ACCESS_KEY_SECRET, OTS_INSTANCE, region=OTS_REGION) 61 | try: 62 | delete_table(client) 63 | except: 64 | pass 65 | create_table(client) 66 | 67 | # time.sleep(3) # wait for table ready 68 | put_row(client) 69 | print('#### row before update ####') 70 | get_row(client) 71 | update_row(client) 72 | print('#### row after update ####') 73 | get_row(client) 74 | delete_table(client) 75 | -------------------------------------------------------------------------------- /tablestore/flatbuffer/dataprotocol/SQLResponseColumns.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: dataprotocol 4 | 5 | import flatbuffers 6 | from flatbuffers.compat import import_numpy 7 | np = import_numpy() 8 | 9 | class SQLResponseColumns(object): 10 | __slots__ = ['_tab'] 11 | 12 | @classmethod 13 | def GetRootAs(cls, buf, offset=0): 14 | n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) 15 | x = SQLResponseColumns() 16 | x.Init(buf, n + offset) 17 | return x 18 | 19 | @classmethod 20 | def GetRootAsSQLResponseColumns(cls, buf, offset=0): 21 | """This method is deprecated. Please switch to GetRootAs.""" 22 | return cls.GetRootAs(buf, offset) 23 | # SQLResponseColumns 24 | def Init(self, buf, pos): 25 | self._tab = flatbuffers.table.Table(buf, pos) 26 | 27 | # SQLResponseColumns 28 | def Columns(self, j): 29 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 30 | if o != 0: 31 | x = self._tab.Vector(o) 32 | x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4 33 | x = self._tab.Indirect(x) 34 | from tablestore.flatbuffer.dataprotocol.SQLResponseColumn import SQLResponseColumn 35 | obj = SQLResponseColumn() 36 | obj.Init(self._tab.Bytes, x) 37 | return obj 38 | return None 39 | 40 | # SQLResponseColumns 41 | def ColumnsLength(self): 42 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 43 | if o != 0: 44 | return self._tab.VectorLen(o) 45 | return 0 46 | 47 | # SQLResponseColumns 48 | def ColumnsIsNone(self): 49 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 50 | return o == 0 51 | 52 | # SQLResponseColumns 53 | def RowCount(self): 54 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 55 | if o != 0: 56 | return self._tab.Get(flatbuffers.number_types.Int64Flags, o + self._tab.Pos) 57 | return 0 58 | 59 | def SQLResponseColumnsStart(builder): builder.StartObject(2) 60 | def Start(builder): 61 | return SQLResponseColumnsStart(builder) 62 | def SQLResponseColumnsAddColumns(builder, columns): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(columns), 0) 63 | def AddColumns(builder, columns): 64 | return SQLResponseColumnsAddColumns(builder, columns) 65 | def SQLResponseColumnsStartColumnsVector(builder, numElems): return builder.StartVector(4, numElems, 4) 66 | def StartColumnsVector(builder, numElems): 67 | return SQLResponseColumnsStartColumnsVector(builder, numElems) 68 | def SQLResponseColumnsAddRowCount(builder, rowCount): builder.PrependInt64Slot(1, rowCount, 0) 69 | def AddRowCount(builder, rowCount): 70 | return SQLResponseColumnsAddRowCount(builder, rowCount) 71 | def SQLResponseColumnsEnd(builder): return builder.EndObject() 72 | def End(builder): 73 | return SQLResponseColumnsEnd(builder) -------------------------------------------------------------------------------- /examples/sql_query.py: -------------------------------------------------------------------------------- 1 | from example_config import * 2 | from tablestore import * 3 | import time 4 | 5 | table_name = 'OTSSqlQuerySimpleExample' 6 | 7 | 8 | def exe_sql_query(client, query): 9 | row_list, table_comsume_list, search_comsume_list = client.exe_sql_query(query) 10 | if len(table_comsume_list) > 0: 11 | for table_comsume_unit in table_comsume_list: 12 | print('Read succeed, table %s consume %s read cu.' % (table_comsume_unit[0], table_comsume_unit[1].read)) 13 | if len(search_comsume_list) > 0: 14 | for search_comsume_unit in search_comsume_list: 15 | print('Read succeed, table %s consume %s read cu.' % (search_comsume_unit[0], search_comsume_unit[1].read)) 16 | for row in row_list: 17 | print(row.attribute_columns) 18 | 19 | 20 | create_queries = [ 21 | 'show tables', 22 | 'create table %s (uid VARCHAR(1024), pid BIGINT(20),name MEDIUMTEXT, age BIGINT(20), grade DOUBLE, PRIMARY KEY(uid,pid));' % table_name, 23 | 'desc %s' % table_name, 24 | ] 25 | 26 | select_and_delete_queries = [ 27 | 'select * from %s' % table_name, 28 | 'select * from %s where pid >3' % table_name, 29 | 'select grade from %s where uid = "7"' % table_name, 30 | 'drop mapping table %s' % table_name 31 | ] 32 | 33 | 34 | def create_table(client): 35 | table_meta = TableMeta(table_name, [('uid', 'STRING'), ('pid', 'INTEGER')]) 36 | table_options = TableOptions() 37 | reserved_throughput = ReservedThroughput(CapacityUnit(0, 0)) 38 | client.create_table(table_meta, table_options, reserved_throughput) 39 | time.sleep(5) 40 | 41 | 42 | def batch_write_row(client): 43 | # batch put 10 rows and update 10 rows on exist table, delete 10 rows on a not-exist table. 44 | put_row_items = [] 45 | for i in range(0, 10): 46 | primary_key = [('uid', str(i)), ('pid', i)] 47 | attribute_columns = [('name', 'somebody' + str(i)), ('age', i), ('grade', i + 0.2)] 48 | row = Row(primary_key, attribute_columns) 49 | condition = Condition(RowExistenceExpectation.IGNORE) 50 | item = PutRowItem(row, condition) 51 | put_row_items.append(item) 52 | 53 | request = BatchWriteRowRequest() 54 | request.add(TableInBatchWriteRowItem(table_name, put_row_items)) 55 | result = client.batch_write_row(request) 56 | 57 | print('Result status: %s' % (result.is_all_succeed())) 58 | print('check first table\'s put results:') 59 | succ, fail = result.get_put() 60 | for item in succ: 61 | print('Put succeed, consume %s write cu.' % item.consumed.write) 62 | for item in fail: 63 | print('Put failed, error code: %s, error message: %s' % (item.error_code, item.error_message)) 64 | 65 | 66 | def drop_table(client): 67 | client.delete_table(table_name) 68 | 69 | 70 | if __name__ == '__main__': 71 | client = OTSClient(OTS_ENDPOINT, OTS_ACCESS_KEY_ID, OTS_ACCESS_KEY_SECRET, OTS_INSTANCE, region=OTS_REGION) 72 | create_table(client) 73 | for query in create_queries: 74 | exe_sql_query(client, query) 75 | batch_write_row(client) 76 | for query in select_and_delete_queries: 77 | exe_sql_query(client, query) 78 | drop_table(client) 79 | -------------------------------------------------------------------------------- /tablestore/plainbuffer/plain_buffer_crc8.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | import sys 4 | import six 5 | 6 | CRC8_TABLE = [0x00, 0x07, 0x0e, 0x09, 0x1c, 0x1b, 0x12, 0x15, 7 | 0x38, 0x3f, 0x36, 0x31, 0x24, 0x23, 0x2a, 0x2d, 8 | 0x70, 0x77, 0x7e, 0x79, 0x6c, 0x6b, 0x62, 0x65, 9 | 0x48, 0x4f, 0x46, 0x41, 0x54, 0x53, 0x5a, 0x5d, 10 | 0xe0, 0xe7, 0xee, 0xe9, 0xfc, 0xfb, 0xf2, 0xf5, 11 | 0xd8, 0xdf, 0xd6, 0xd1, 0xc4, 0xc3, 0xca, 0xcd, 12 | 0x90, 0x97, 0x9e, 0x99, 0x8c, 0x8b, 0x82, 0x85, 13 | 0xa8, 0xaf, 0xa6, 0xa1, 0xb4, 0xb3, 0xba, 0xbd, 14 | 0xc7, 0xc0, 0xc9, 0xce, 0xdb, 0xdc, 0xd5, 0xd2, 15 | 0xff, 0xf8, 0xf1, 0xf6, 0xe3, 0xe4, 0xed, 0xea, 16 | 0xb7, 0xb0, 0xb9, 0xbe, 0xab, 0xac, 0xa5, 0xa2, 17 | 0x8f, 0x88, 0x81, 0x86, 0x93, 0x94, 0x9d, 0x9a, 18 | 0x27, 0x20, 0x29, 0x2e, 0x3b, 0x3c, 0x35, 0x32, 19 | 0x1f, 0x18, 0x11, 0x16, 0x03, 0x04, 0x0d, 0x0a, 20 | 0x57, 0x50, 0x59, 0x5e, 0x4b, 0x4c, 0x45, 0x42, 21 | 0x6f, 0x68, 0x61, 0x66, 0x73, 0x74, 0x7d, 0x7a, 22 | 0x89, 0x8e, 0x87, 0x80, 0x95, 0x92, 0x9b, 0x9c, 23 | 0xb1, 0xb6, 0xbf, 0xb8, 0xad, 0xaa, 0xa3, 0xa4, 24 | 0xf9, 0xfe, 0xf7, 0xf0, 0xe5, 0xe2, 0xeb, 0xec, 25 | 0xc1, 0xc6, 0xcf, 0xc8, 0xdd, 0xda, 0xd3, 0xd4, 26 | 0x69, 0x6e, 0x67, 0x60, 0x75, 0x72, 0x7b, 0x7c, 27 | 0x51, 0x56, 0x5f, 0x58, 0x4d, 0x4a, 0x43, 0x44, 28 | 0x19, 0x1e, 0x17, 0x10, 0x05, 0x02, 0x0b, 0x0c, 29 | 0x21, 0x26, 0x2f, 0x28, 0x3d, 0x3a, 0x33, 0x34, 30 | 0x4e, 0x49, 0x40, 0x47, 0x52, 0x55, 0x5c, 0x5b, 31 | 0x76, 0x71, 0x78, 0x7f, 0x6a, 0x6d, 0x64, 0x63, 32 | 0x3e, 0x39, 0x30, 0x37, 0x22, 0x25, 0x2c, 0x2b, 33 | 0x06, 0x01, 0x08, 0x0f, 0x1a, 0x1d, 0x14, 0x13, 34 | 0xae, 0xa9, 0xa0, 0xa7, 0xb2, 0xb5, 0xbc, 0xbb, 35 | 0x96, 0x91, 0x98, 0x9f, 0x8a, 0x8d, 0x84, 0x83, 36 | 0xde, 0xd9, 0xd0, 0xd7, 0xc2, 0xc5, 0xcc, 0xcb, 37 | 0xe6, 0xe1, 0xe8, 0xef, 0xfa, 0xfd, 0xf4, 0xf3] 38 | 39 | class PlainBufferCrc8(object): 40 | @staticmethod 41 | def crc_string(crc, bytes_): 42 | if isinstance(bytes_, six.text_type): 43 | bytes_ = bytes_.encode('utf-8') 44 | elif not isinstance(bytes_, six.binary_type): 45 | raise TypeError("must be string, actual:" + str(type(bytes_))) 46 | 47 | if sys.version_info[0] == 2: 48 | for byte in bytes_: 49 | crc = CRC8_TABLE[((crc & 0xff) ^ ord(byte))] 50 | else: 51 | for byte in bytes_: 52 | crc = CRC8_TABLE[((crc & 0xff) ^ byte)] 53 | return crc 54 | 55 | @staticmethod 56 | def crc_bytes(crc, bytes_): 57 | for byte in bytes_: 58 | crc = CRC8_TABLE[((crc & 0xff) ^ byte)] 59 | return crc 60 | 61 | @staticmethod 62 | def crc_int8(crc, byte): 63 | return CRC8_TABLE[((crc & 0xff)^byte)] 64 | 65 | @staticmethod 66 | def crc_int32(crc, byte): 67 | for i in range(0, 4): 68 | crc = PlainBufferCrc8.crc_int8(crc, (byte>>(i*8)) & 0xff) 69 | return crc 70 | 71 | @staticmethod 72 | def crc_int64(crc, byte): 73 | for i in range(0, 8): 74 | crc = PlainBufferCrc8.crc_int8(crc, (byte>>(i*8)) & 0xff) 75 | return crc 76 | 77 | __all__ = ['PlainBufferCrc8'] 78 | -------------------------------------------------------------------------------- /examples/secondary_index_operations.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from example_config import * 4 | from tablestore import * 5 | import time 6 | import json 7 | 8 | table_name = 'SecondaryIndexOperationExample' 9 | index_name_1 = 'index1' 10 | index_name_2 = 'index2' 11 | 12 | 13 | def create_table(client): 14 | print('Begin CreateTable') 15 | schema_of_primary_key = [('gid', 'INTEGER'), ('uid', 'STRING')] 16 | defined_columns = [('i', 'INTEGER'), ('bool', 'BOOLEAN'), ('d', 'DOUBLE'), ('s', 'STRING'), ('b', 'BINARY')] 17 | table_meta = TableMeta(table_name, schema_of_primary_key, defined_columns) 18 | table_option = TableOptions(-1, 1) 19 | reserved_throughput = ReservedThroughput(CapacityUnit(0, 0)) 20 | secondary_indexes = [ 21 | SecondaryIndexMeta(index_name_1, ['i', 's'], ['bool', 'b', 'd']), 22 | ] 23 | client.create_table(table_meta, table_option, reserved_throughput, secondary_indexes) 24 | print('Table has been created.') 25 | 26 | 27 | def create_index(client): 28 | print('Begin CreateIndex') 29 | index_meta = SecondaryIndexMeta(index_name_2, ['i', 's'], ['bool', 'b', 'd']) 30 | client.create_secondary_index(table_name, index_meta, False) 31 | print('Index has been created.') 32 | 33 | 34 | def describe_table(client): 35 | print('Begin DescribeTable') 36 | describe_response = client.describe_table(table_name) 37 | print('TableName: %s' % describe_response.table_meta.table_name) 38 | print('PrimaryKey: %s' % describe_response.table_meta.schema_of_primary_key) 39 | print('Reserved read throughput: %s' % describe_response.reserved_throughput_details.capacity_unit.read) 40 | print('Reserved write throughput: %s' % describe_response.reserved_throughput_details.capacity_unit.write) 41 | print('Last increase throughput time: %s' % describe_response.reserved_throughput_details.last_increase_time) 42 | print('Last decrease throughput time: %s' % describe_response.reserved_throughput_details.last_decrease_time) 43 | print('table options\'s time to live: %s' % describe_response.table_options.time_to_live) 44 | print('table options\'s max version: %s' % describe_response.table_options.max_version) 45 | print('table options\'s max_time_deviation: %s' % describe_response.table_options.max_time_deviation) 46 | print('Secondary indexes:') 47 | for secondary_index in describe_response.secondary_indexes: 48 | print('index name: %s' % secondary_index.index_name) 49 | print('primary key names: %s' % str(secondary_index.primary_key_names)) 50 | print('defined column names: %s' % str(secondary_index.defined_column_names)) 51 | print('End DescribeTable') 52 | 53 | 54 | def delete_index(client, index_name): 55 | print('Begin DeleteIndex') 56 | client.delete_secondary_index(table_name, index_name) 57 | print('End delete index.') 58 | 59 | 60 | def delete_table(client): 61 | print('Begin DeleteTable') 62 | client.delete_table(table_name) 63 | print('End DeleteTable') 64 | 65 | 66 | if __name__ == '__main__': 67 | client = OTSClient(OTS_ENDPOINT, OTS_ACCESS_KEY_ID, OTS_ACCESS_KEY_SECRET, OTS_INSTANCE, region=OTS_REGION) 68 | try: 69 | delete_table(client) 70 | except: 71 | pass 72 | 73 | create_table(client) 74 | 75 | # time.sleep(3) # wait for table ready 76 | create_index(client) 77 | describe_table(client) 78 | delete_index(client, index_name_1) 79 | describe_table(client) 80 | delete_index(client, index_name_2) 81 | delete_table(client) 82 | -------------------------------------------------------------------------------- /examples/table_operations.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from example_config import * 4 | from tablestore import * 5 | import time 6 | 7 | table_name = 'OTSTableOperationsSimpleExample' 8 | 9 | 10 | def create_table(client): 11 | print('Begin CreateTable') 12 | schema_of_primary_key = [('gid', 'INTEGER'), ('uid', 'STRING')] 13 | table_meta = TableMeta(table_name, schema_of_primary_key) 14 | table_option = TableOptions(-1, 2) 15 | reserved_throughput = ReservedThroughput(CapacityUnit(0, 0)) 16 | client.create_table(table_meta, table_option, reserved_throughput) 17 | print('Table has been created.') 18 | 19 | 20 | def list_table(client): 21 | print('Begin ListTable') 22 | tables = client.list_table() 23 | print('All the tables you have created:') 24 | for table in tables: 25 | print(table) 26 | print('End ListTable') 27 | 28 | 29 | def describe_table(client): 30 | print('Begin DescribeTable') 31 | describe_response = client.describe_table(table_name) 32 | print('TableName: %s' % describe_response.table_meta.table_name) 33 | print('PrimaryKey: %s' % describe_response.table_meta.schema_of_primary_key) 34 | print('Reserved read throughput: %s' % describe_response.reserved_throughput_details.capacity_unit.read) 35 | print('Reserved write throughput: %s' % describe_response.reserved_throughput_details.capacity_unit.write) 36 | print('Last increase throughput time: %s' % describe_response.reserved_throughput_details.last_increase_time) 37 | print('Last decrease throughput time: %s' % describe_response.reserved_throughput_details.last_decrease_time) 38 | print('table options\'s time to live: %s' % describe_response.table_options.time_to_live) 39 | print('table options\'s max version: %s' % describe_response.table_options.max_version) 40 | print('table options\'s max_time_deviation: %s' % describe_response.table_options.max_time_deviation) 41 | print('End DescribeTable') 42 | 43 | 44 | def update_table(client): 45 | print('Begin UpdateTable') 46 | table_option = TableOptions(100001, None, None) 47 | update_response = client.update_table(table_name, table_option, None) 48 | print('Reserved read throughput: %s' % update_response.reserved_throughput_details.capacity_unit.read) 49 | print('Reserved write throughput: %s' % update_response.reserved_throughput_details.capacity_unit.write) 50 | print('Last increase throughput time: %s' % update_response.reserved_throughput_details.last_increase_time) 51 | print('Last decrease throughput time: %s' % update_response.reserved_throughput_details.last_decrease_time) 52 | print('table options\'s time to live: %s' % update_response.table_options.time_to_live) 53 | print('table options\'s max version: %s' % update_response.table_options.max_version) 54 | print('table options\'s max_time_deviation: %s' % update_response.table_options.max_time_deviation) 55 | print('End UpdateTable') 56 | 57 | 58 | def delete_table(client): 59 | print('Begin DeleteTable') 60 | client.delete_table(table_name) 61 | print('Table \'%s\' has been deleted.' % table_name) 62 | 63 | 64 | if __name__ == '__main__': 65 | client = OTSClient(OTS_ENDPOINT, OTS_ACCESS_KEY_ID, OTS_ACCESS_KEY_SECRET, OTS_INSTANCE, region=OTS_REGION) 66 | try: 67 | delete_table(client) 68 | except: 69 | pass 70 | 71 | create_table(client) 72 | 73 | time.sleep(3) # wait for table ready 74 | 75 | list_table(client) 76 | describe_table(client) 77 | update_table(client) 78 | delete_table(client) 79 | -------------------------------------------------------------------------------- /tests/utils_test.py: -------------------------------------------------------------------------------- 1 | import random 2 | import unittest 3 | 4 | from tablestore.utils import VectorUtils 5 | 6 | 7 | def generate_random_floats(num_floats, lower_bound, upper_bound): 8 | """Generate a list of random floating-point numbers within a specified range and quantity.""" 9 | return [random.uniform(lower_bound, upper_bound) for _ in range(num_floats)] 10 | 11 | 12 | class VectorUtilsTest(unittest.TestCase): 13 | def test_floats_to_bytearray(self): 14 | # Test case 1: normal case 15 | floats = [1.0, 2.0, 3.0] 16 | bytes_list = VectorUtils.floats_to_bytes(floats) 17 | self.assertEqual(bytearray(b'\x00\x00\x80?\x00\x00\x00@\x00\x00@@'), bytes_list) 18 | # Test case 2: empty list 19 | floats = [] 20 | with self.assertRaisesRegex(ValueError, "vector is empty"): 21 | VectorUtils.floats_to_bytes(floats) 22 | # Test case 3: non-float list 23 | floats = [1, 2, 3] 24 | with self.assertRaisesRegex(TypeError, "Input must be a list/tuple of floats"): 25 | VectorUtils.floats_to_bytes(floats) 26 | # Test case 4: non-list or tuple 27 | floats = 1.0 28 | with self.assertRaisesRegex(TypeError, "Input must be a list/tuple of floats"): 29 | VectorUtils.floats_to_bytes(floats) 30 | # Test case 5: tuple 31 | floats = (1.0, 2.0, 3.0) 32 | bytes_list = VectorUtils.floats_to_bytes(floats) 33 | self.assertEqual(bytearray(b'\x00\x00\x80?\x00\x00\x00@\x00\x00@@'), bytes_list) 34 | # Test case 6: non-float tuple 35 | floats = (1, 2, 3) 36 | with self.assertRaisesRegex(TypeError, "Input must be a list/tuple of floats"): 37 | VectorUtils.floats_to_bytes(floats) 38 | # Test case 7: complex floats 39 | floats = [1.1, 2.22, 3.333, 4.4444] 40 | bytes_list = VectorUtils.floats_to_bytes(floats) 41 | self.assertEqual( 42 | bytearray(i % 256 for i in [-51, -52, -116, 63, 123, 20, 14, 64, -33, 79, 85, 64, -122, 56, -114, 64]), 43 | bytes_list) 44 | 45 | def test_bytes_to_floats(self): 46 | # Test case 1: normal case 47 | bytes_list = bytearray(b'\x00\x00\x80?\x00\x00\x00@\x00\x00@@') 48 | floats = VectorUtils.bytes_to_floats(bytes_list) 49 | self.assertEqual([1.0, 2.0, 3.0], floats) 50 | # Test case 2: not bytearray 51 | bytes_list = b'' 52 | with self.assertRaisesRegex(TypeError, "Input must be a bytearray object"): 53 | VectorUtils.bytes_to_floats(bytes_list) 54 | # Test case 3: invalid bytearray 55 | bytes_list = bytearray(b'\x00\x00\x80?\x00\x00\x00@\x00@@') 56 | with self.assertRaisesRegex(ValueError, 57 | "bytes length is not multiple of 4\\(SIZE_OF_FLOAT32\\) or length is 0"): 58 | VectorUtils.bytes_to_floats(bytes_list) 59 | # Test case 4: empty bytearray 60 | bytes_list = bytearray(b'') 61 | with self.assertRaisesRegex(ValueError, 62 | "bytes length is not multiple of 4\\(SIZE_OF_FLOAT32\\) or length is 0"): 63 | VectorUtils.bytes_to_floats(bytes_list) 64 | 65 | def test_bytes_and_floats_conversion(self): 66 | floats = generate_random_floats(random.randint(100, 1024), 0, 1) 67 | bytes_list = VectorUtils.floats_to_bytes(floats) 68 | floats_2 = VectorUtils.bytes_to_floats(bytes_list) 69 | self.assertEqual(len(floats), len(floats_2)) 70 | for i in range(len(floats)): 71 | # floats_2[i] may not equal to floats[i] due to precision loss 72 | self.assertAlmostEqual(floats[i], floats_2[i], places=7) 73 | 74 | 75 | if __name__ == '__main__': 76 | unittest.main() 77 | -------------------------------------------------------------------------------- /examples/batch_write_row.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from example_config import * 4 | from tablestore import * 5 | import time 6 | 7 | table_name = 'OTSBatchWriteRowSimpleExample' 8 | 9 | 10 | def create_table(client): 11 | schema_of_primary_key = [('gid', 'INTEGER'), ('uid', 'INTEGER')] 12 | table_meta = TableMeta(table_name, schema_of_primary_key) 13 | table_options = TableOptions() 14 | reserved_throughput = ReservedThroughput(CapacityUnit(0, 0)) 15 | client.create_table(table_meta, table_options, reserved_throughput) 16 | print('Table has been created.') 17 | 18 | 19 | def delete_table(client): 20 | client.delete_table(table_name) 21 | print('Table \'%s\' has been deleted.' % table_name) 22 | 23 | 24 | def batch_write_row(client): 25 | # batch put 10 rows and update 10 rows on exist table, delete 10 rows on a not-exist table. 26 | put_row_items = [] 27 | for i in range(0, 10): 28 | primary_key = [('gid', i), ('uid', i + 1)] 29 | attribute_columns = [('name', 'somebody' + str(i)), ('address', 'somewhere' + str(i)), ('age', i)] 30 | row = Row(primary_key, attribute_columns) 31 | condition = Condition(RowExistenceExpectation.IGNORE) 32 | item = PutRowItem(row, condition) 33 | put_row_items.append(item) 34 | 35 | for i in range(10, 20): 36 | primary_key = [('gid', i), ('uid', i + 1)] 37 | attribute_columns = {'put': [('name', 'somebody' + str(i)), ('address', 'somewhere' + str(i)), ('age', i)]} 38 | row = Row(primary_key, attribute_columns) 39 | condition = Condition(RowExistenceExpectation.IGNORE, SingleColumnCondition("age", i, ComparatorType.EQUAL)) 40 | item = UpdateRowItem(row, condition) 41 | put_row_items.append(item) 42 | 43 | delete_row_items = [] 44 | for i in range(10, 20): 45 | primary_key = [('gid', i), ('uid', i + 1)] 46 | row = Row(primary_key) 47 | condition = Condition(RowExistenceExpectation.IGNORE) 48 | item = DeleteRowItem(row, condition) 49 | delete_row_items.append(item) 50 | 51 | request = BatchWriteRowRequest() 52 | request.add(TableInBatchWriteRowItem(table_name, put_row_items)) 53 | request.add(TableInBatchWriteRowItem('notExistTable', delete_row_items)) 54 | result = client.batch_write_row(request) 55 | 56 | print('Result status: %s' % (result.is_all_succeed())) 57 | print('check first table\'s put results:') 58 | succ, fail = result.get_put() 59 | for item in succ: 60 | print('Put succeed, consume %s write cu.' % item.consumed.write) 61 | for item in fail: 62 | print('Put failed, error code: %s, error message: %s' % (item.error_code, item.error_message)) 63 | 64 | print('check first table\'s update results:') 65 | succ, fail = result.get_update() 66 | for item in succ: 67 | print('Update succeed, consume %s write cu.' % item.consumed.write) 68 | for item in fail: 69 | print('Update failed, error code: %s, error message: %s' % (item.error_code, item.error_message)) 70 | 71 | print('check second table\'s delete results:') 72 | succ, fail = result.get_delete() 73 | for item in succ: 74 | print('Delete succeed, consume %s write cu.' % item.consumed.write) 75 | for item in fail: 76 | print('Delete failed, error code: %s, error message: %s' % (item.error_code, item.error_message)) 77 | 78 | 79 | if __name__ == '__main__': 80 | client = OTSClient(OTS_ENDPOINT, OTS_ACCESS_KEY_ID, OTS_ACCESS_KEY_SECRET, OTS_INSTANCE, region=OTS_REGION) 81 | try: 82 | delete_table(client) 83 | except: 84 | pass 85 | create_table(client) 86 | 87 | time.sleep(3) # wait for table ready 88 | batch_write_row(client) 89 | delete_table(client) 90 | -------------------------------------------------------------------------------- /tablestore/protobuf/table_store_filter_pb2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by the protocol buffer compiler. DO NOT EDIT! 3 | # source: table_store_filter.proto 4 | # Protobuf Python Version: 4.25.0 5 | """Generated protocol buffer code.""" 6 | from google.protobuf import descriptor as _descriptor 7 | from google.protobuf import descriptor_pool as _descriptor_pool 8 | from google.protobuf import symbol_database as _symbol_database 9 | from google.protobuf.internal import builder as _builder 10 | # @@protoc_insertion_point(imports) 11 | 12 | _sym_db = _symbol_database.Default() 13 | 14 | 15 | 16 | 17 | DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18table_store_filter.proto\x12\x1e\x63om.aliyun.tablestore.protocol\"b\n\x11ValueTransferRule\x12\r\n\x05regex\x18\x01 \x02(\t\x12>\n\tcast_type\x18\x02 \x01(\x0e\x32+.com.aliyun.tablestore.protocol.VariantType\"\x8d\x02\n\x17SingleColumnValueFilter\x12\x42\n\ncomparator\x18\x01 \x02(\x0e\x32..com.aliyun.tablestore.protocol.ComparatorType\x12\x13\n\x0b\x63olumn_name\x18\x02 \x02(\t\x12\x14\n\x0c\x63olumn_value\x18\x03 \x02(\x0c\x12\x19\n\x11\x66ilter_if_missing\x18\x04 \x02(\x08\x12\x1b\n\x13latest_version_only\x18\x05 \x02(\x08\x12K\n\x10value_trans_rule\x18\x06 \x01(\x0b\x32\x31.com.aliyun.tablestore.protocol.ValueTransferRule\"\x9e\x01\n\x1a\x43ompositeColumnValueFilter\x12\x43\n\ncombinator\x18\x01 \x02(\x0e\x32/.com.aliyun.tablestore.protocol.LogicalOperator\x12;\n\x0bsub_filters\x18\x02 \x03(\x0b\x32&.com.aliyun.tablestore.protocol.Filter\"7\n\x16\x43olumnPaginationFilter\x12\x0e\n\x06offset\x18\x01 \x02(\x05\x12\r\n\x05limit\x18\x02 \x02(\x05\"R\n\x06\x46ilter\x12\x38\n\x04type\x18\x01 \x02(\x0e\x32*.com.aliyun.tablestore.protocol.FilterType\x12\x0e\n\x06\x66ilter\x18\x02 \x02(\x0c*U\n\x0bVariantType\x12\x0e\n\nVT_INTEGER\x10\x00\x12\r\n\tVT_DOUBLE\x10\x01\x12\r\n\tVT_STRING\x10\x03\x12\x0b\n\x07VT_NULL\x10\x06\x12\x0b\n\x07VT_BLOB\x10\x07*a\n\nFilterType\x12\x1a\n\x16\x46T_SINGLE_COLUMN_VALUE\x10\x01\x12\x1d\n\x19\x46T_COMPOSITE_COLUMN_VALUE\x10\x02\x12\x18\n\x14\x46T_COLUMN_PAGINATION\x10\x03*\xa0\x01\n\x0e\x43omparatorType\x12\x0c\n\x08\x43T_EQUAL\x10\x01\x12\x10\n\x0c\x43T_NOT_EQUAL\x10\x02\x12\x13\n\x0f\x43T_GREATER_THAN\x10\x03\x12\x14\n\x10\x43T_GREATER_EQUAL\x10\x04\x12\x10\n\x0c\x43T_LESS_THAN\x10\x05\x12\x11\n\rCT_LESS_EQUAL\x10\x06\x12\x0c\n\x08\x43T_EXIST\x10\x07\x12\x10\n\x0c\x43T_NOT_EXIST\x10\x08*4\n\x0fLogicalOperator\x12\n\n\x06LO_NOT\x10\x01\x12\n\n\x06LO_AND\x10\x02\x12\t\n\x05LO_OR\x10\x03') 18 | 19 | _globals = globals() 20 | _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) 21 | _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'table_store_filter_pb2', _globals) 22 | if _descriptor._USE_C_DESCRIPTORS == False: 23 | DESCRIPTOR._options = None 24 | _globals['_VARIANTTYPE']._serialized_start=734 25 | _globals['_VARIANTTYPE']._serialized_end=819 26 | _globals['_FILTERTYPE']._serialized_start=821 27 | _globals['_FILTERTYPE']._serialized_end=918 28 | _globals['_COMPARATORTYPE']._serialized_start=921 29 | _globals['_COMPARATORTYPE']._serialized_end=1081 30 | _globals['_LOGICALOPERATOR']._serialized_start=1083 31 | _globals['_LOGICALOPERATOR']._serialized_end=1135 32 | _globals['_VALUETRANSFERRULE']._serialized_start=60 33 | _globals['_VALUETRANSFERRULE']._serialized_end=158 34 | _globals['_SINGLECOLUMNVALUEFILTER']._serialized_start=161 35 | _globals['_SINGLECOLUMNVALUEFILTER']._serialized_end=430 36 | _globals['_COMPOSITECOLUMNVALUEFILTER']._serialized_start=433 37 | _globals['_COMPOSITECOLUMNVALUEFILTER']._serialized_end=591 38 | _globals['_COLUMNPAGINATIONFILTER']._serialized_start=593 39 | _globals['_COLUMNPAGINATIONFILTER']._serialized_end=648 40 | _globals['_FILTER']._serialized_start=650 41 | _globals['_FILTER']._serialized_end=732 42 | # @@protoc_insertion_point(module_scope) 43 | -------------------------------------------------------------------------------- /examples/batch_get_row.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from example_config import * 4 | from tablestore import * 5 | import time 6 | 7 | table_name_1 = 'OTSBatchGetRowSimpleExample_1' 8 | table_name_2 = 'OTSBatchGetRowSimpleExample_2' 9 | 10 | 11 | def create_table(client, table_name): 12 | schema_of_primary_key = [('gid', 'INTEGER'), ('uid', 'INTEGER')] 13 | table_meta = TableMeta(table_name, schema_of_primary_key) 14 | table_option = TableOptions() 15 | reserved_throughput = ReservedThroughput(CapacityUnit(0, 0)) 16 | client.create_table(table_meta, table_option, reserved_throughput) 17 | print('Table has been created.') 18 | 19 | 20 | def delete_table(client, table_name): 21 | try: 22 | client.delete_table(table_name) 23 | print('Table \'%s\' has been deleted.' % table_name) 24 | except: 25 | pass 26 | 27 | 28 | def put_row(client, table_name): 29 | for i in range(0, 10): 30 | primary_key = [('gid', i), ('uid', i + 1)] 31 | attribute_columns = [('name', 'John'), ('mobile', i), ('address', 'China'), ('age', i)] 32 | row = Row(primary_key, attribute_columns) 33 | condition = Condition( 34 | RowExistenceExpectation.EXPECT_NOT_EXIST) # Expect not exist: put it into table only when this row is not exist. 35 | consumed, return_row = client.put_row(table_name, row, condition) 36 | print(u'Write succeed, consume %s write cu.' % consumed.write) 37 | 38 | 39 | def batch_get_row(client): 40 | # try to get rows from two different tables 41 | columns_to_get = ['name', 'mobile', 'address', 'age'] 42 | rows_to_get = [] 43 | for i in range(0, 10): 44 | primary_key = [('gid', i), ('uid', i + 1)] 45 | rows_to_get.append(primary_key) 46 | 47 | cond = CompositeColumnCondition(LogicalOperator.AND) 48 | cond.add_sub_condition(SingleColumnCondition("name", "John", ComparatorType.EQUAL)) 49 | cond.add_sub_condition(SingleColumnCondition("address", 'China', ComparatorType.EQUAL)) 50 | 51 | request = BatchGetRowRequest() 52 | request.add(TableInBatchGetRowItem(table_name_1, rows_to_get, columns_to_get, cond, 1)) 53 | request.add(TableInBatchGetRowItem(table_name_2, rows_to_get, columns_to_get, cond, 1)) 54 | 55 | result = client.batch_get_row(request) 56 | 57 | print('Result status: %s' % (result.is_all_succeed())) 58 | 59 | table_result_0 = result.get_result_by_table(table_name_1) 60 | table_result_1 = result.get_result_by_table(table_name_2) 61 | 62 | print('Check first table\'s result:') 63 | for item in table_result_0: 64 | if item.is_ok: 65 | print('Read succeed, PrimaryKey: %s, Attributes: %s' % (item.row.primary_key, item.row.attribute_columns)) 66 | else: 67 | print('Read failed, error code: %s, error message: %s' % (item.error_code, item.error_message)) 68 | 69 | print('Check second table\'s result:') 70 | for item in table_result_1: 71 | if item.is_ok: 72 | print('Read succeed, PrimaryKey: %s, Attributes: %s' % (item.row.primary_key, item.row.attribute_columns)) 73 | else: 74 | print('Read failed, error code: %s, error message: %s' % (item.error_code, item.error_message)) 75 | 76 | 77 | if __name__ == '__main__': 78 | client = OTSClient(OTS_ENDPOINT, OTS_ACCESS_KEY_ID, OTS_ACCESS_KEY_SECRET, OTS_INSTANCE, region=OTS_REGION) 79 | delete_table(client, table_name_1) 80 | delete_table(client, table_name_2) 81 | 82 | create_table(client, table_name_1) 83 | create_table(client, table_name_2) 84 | 85 | time.sleep(3) # wait for table ready 86 | put_row(client, table_name_1) 87 | put_row(client, table_name_2) 88 | batch_get_row(client) 89 | delete_table(client, table_name_1) 90 | delete_table(client, table_name_2) 91 | -------------------------------------------------------------------------------- /tablestore/flatbuffer/dataprotocol/RLEStringValues.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: dataprotocol 4 | 5 | import flatbuffers 6 | from flatbuffers.compat import import_numpy 7 | np = import_numpy() 8 | 9 | class RLEStringValues(object): 10 | __slots__ = ['_tab'] 11 | 12 | @classmethod 13 | def GetRootAs(cls, buf, offset=0): 14 | n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) 15 | x = RLEStringValues() 16 | x.Init(buf, n + offset) 17 | return x 18 | 19 | @classmethod 20 | def GetRootAsRLEStringValues(cls, buf, offset=0): 21 | """This method is deprecated. Please switch to GetRootAs.""" 22 | return cls.GetRootAs(buf, offset) 23 | # RLEStringValues 24 | def Init(self, buf, pos): 25 | self._tab = flatbuffers.table.Table(buf, pos) 26 | 27 | # RLEStringValues 28 | def Array(self, j): 29 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 30 | if o != 0: 31 | a = self._tab.Vector(o) 32 | return self._tab.String(a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) 33 | return "" 34 | 35 | # RLEStringValues 36 | def ArrayLength(self): 37 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 38 | if o != 0: 39 | return self._tab.VectorLen(o) 40 | return 0 41 | 42 | # RLEStringValues 43 | def ArrayIsNone(self): 44 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 45 | return o == 0 46 | 47 | # RLEStringValues 48 | def IndexMapping(self, j): 49 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 50 | if o != 0: 51 | a = self._tab.Vector(o) 52 | return self._tab.Get(flatbuffers.number_types.Int32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) 53 | return 0 54 | 55 | # RLEStringValues 56 | def IndexMappingAsNumpy(self): 57 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 58 | if o != 0: 59 | return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o) 60 | return 0 61 | 62 | # RLEStringValues 63 | def IndexMappingLength(self): 64 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 65 | if o != 0: 66 | return self._tab.VectorLen(o) 67 | return 0 68 | 69 | # RLEStringValues 70 | def IndexMappingIsNone(self): 71 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 72 | return o == 0 73 | 74 | def RLEStringValuesStart(builder): builder.StartObject(2) 75 | def Start(builder): 76 | return RLEStringValuesStart(builder) 77 | def RLEStringValuesAddArray(builder, array): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(array), 0) 78 | def AddArray(builder, array): 79 | return RLEStringValuesAddArray(builder, array) 80 | def RLEStringValuesStartArrayVector(builder, numElems): return builder.StartVector(4, numElems, 4) 81 | def StartArrayVector(builder, numElems): 82 | return RLEStringValuesStartArrayVector(builder, numElems) 83 | def RLEStringValuesAddIndexMapping(builder, indexMapping): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(indexMapping), 0) 84 | def AddIndexMapping(builder, indexMapping): 85 | return RLEStringValuesAddIndexMapping(builder, indexMapping) 86 | def RLEStringValuesStartIndexMappingVector(builder, numElems): return builder.StartVector(4, numElems, 4) 87 | def StartIndexMappingVector(builder, numElems): 88 | return RLEStringValuesStartIndexMappingVector(builder, numElems) 89 | def RLEStringValuesEnd(builder): return builder.EndObject() 90 | def End(builder): 91 | return RLEStringValuesEnd(builder) -------------------------------------------------------------------------------- /tablestore/timeseries_condition.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | import six 4 | from enum import IntEnum 5 | from tablestore.protobuf import timeseries_pb2 6 | 7 | 8 | class MetaQueryCompositeOperator(IntEnum): 9 | OP_AND = 1 10 | OP_OR = 2 11 | OP_NOT = 3 12 | 13 | def to_pb(self): 14 | if self == MetaQueryCompositeOperator.OP_AND: 15 | return timeseries_pb2.MetaQueryCompositeOperator.OP_AND 16 | elif self == MetaQueryCompositeOperator.OP_OR: 17 | return timeseries_pb2.MetaQueryCompositeOperator.OP_OR 18 | elif self == MetaQueryCompositeOperator.OP_NOT: 19 | return timeseries_pb2.MetaQueryCompositeOperator.OP_NOT 20 | 21 | 22 | class MetaQuerySingleOperator(IntEnum): 23 | OP_EQUAL = 1 24 | OP_NOT_EQUAL = 2 25 | OP_GREATER_THAN = 3 26 | OP_GREATER_EQUAL = 4 27 | OP_LESS_THAN = 5 28 | OP_LESS_EQUAL = 6 29 | OP_PREFIX = 7 30 | 31 | def to_pb(self): 32 | if self == MetaQuerySingleOperator.OP_EQUAL: 33 | return timeseries_pb2.MetaQuerySingleOperator.OP_EQUAL 34 | elif self == MetaQuerySingleOperator.OP_GREATER_THAN: 35 | return timeseries_pb2.MetaQuerySingleOperator.OP_GREATER_THAN 36 | elif self == MetaQuerySingleOperator.OP_GREATER_EQUAL: 37 | return timeseries_pb2.MetaQuerySingleOperator.OP_GREATER_EQUAL 38 | elif self == MetaQuerySingleOperator.OP_LESS_THAN: 39 | return timeseries_pb2.MetaQuerySingleOperator.OP_LESS_THAN 40 | elif self == MetaQuerySingleOperator.OP_LESS_EQUAL: 41 | return timeseries_pb2.MetaQuerySingleOperator.OP_LESS_EQUAL 42 | elif self == MetaQuerySingleOperator.OP_PREFIX: 43 | return timeseries_pb2.MetaQuerySingleOperator.OP_PREFIX 44 | 45 | 46 | class MeasurementMetaQueryCondition(object): 47 | def __init__(self, operator: MetaQuerySingleOperator, value: str): 48 | self.operator = operator 49 | self.value = value 50 | 51 | def get_type(self): 52 | return timeseries_pb2.MetaQueryConditionType.MEASUREMENT_CONDITION 53 | 54 | 55 | class DataSourceMetaQueryCondition(object): 56 | def __init__(self, operator: MetaQuerySingleOperator, value: str): 57 | self.operator = operator 58 | self.value = value 59 | 60 | def get_type(self): 61 | return timeseries_pb2.MetaQueryConditionType.SOURCE_CONDITION 62 | 63 | 64 | class TagMetaQueryCondition(object): 65 | def __init__(self, operator: MetaQuerySingleOperator, tag_name: str, value: str): 66 | self.operator = operator 67 | self.tag_name = tag_name 68 | self.value = value 69 | 70 | def get_type(self): 71 | return timeseries_pb2.MetaQueryConditionType.TAG_CONDITION 72 | 73 | 74 | class UpdateTimeMetaQueryCondition(object): 75 | def __init__(self, operator: MetaQuerySingleOperator, time_in_us: int): 76 | self.operator = operator 77 | self.time_in_us = time_in_us 78 | 79 | def get_type(self): 80 | return timeseries_pb2.MetaQueryConditionType.UPDATE_TIME_CONDITION 81 | 82 | 83 | class AttributeMetaQueryCondition(object): 84 | def __init__(self, operator: MetaQuerySingleOperator, attribute_name: str, value: str): 85 | self.operator = operator 86 | self.attribute_name = attribute_name 87 | self.value = value 88 | 89 | def get_type(self): 90 | return timeseries_pb2.MetaQueryConditionType.ATTRIBUTE_CONDITION 91 | 92 | 93 | class CompositeMetaQueryCondition(object): 94 | def __init__(self, operator: MetaQueryCompositeOperator, sub_conditions: list): 95 | self.operator = operator 96 | self.subConditions = sub_conditions 97 | 98 | def get_type(self): 99 | return timeseries_pb2.MetaQueryConditionType.COMPOSITE_CONDITION 100 | 101 | 102 | 103 | -------------------------------------------------------------------------------- /tests/client_unittest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from tests.lib.api_test_base import APITestBase 4 | from tablestore import * 5 | from tablestore.protocol import OTSProtocol 6 | 7 | import logging 8 | 9 | 10 | class ClientTest(APITestBase): 11 | def setUp(self): 12 | pass # no need to set up client 13 | 14 | def tearDown(self): 15 | pass # no need to tearDown client 16 | 17 | def test_validate_parameter(self): 18 | test_endpoint = "https://test-inst.test-region.xxx" 19 | test_ak_id = "test_id" 20 | test_ak_secret = "test_key" 21 | test_instance = "test-inst" 22 | test_region = "test-region" 23 | 24 | # pass 25 | OTSClient(test_endpoint, test_ak_id, test_ak_secret, test_instance, region=test_region) 26 | OTSClient(test_endpoint, test_ak_id, test_ak_secret, test_instance, region=None) 27 | 28 | with self.assertRaisesRegex(OTSClientError, "end_point is not str or is empty."): 29 | OTSClient("", test_ak_id, test_ak_secret, test_instance, region=test_region) 30 | with self.assertRaisesRegex(OTSClientError, "end_point is not str or is empty."): 31 | OTSClient(1, test_ak_id, test_ak_secret, test_instance, region=test_region) 32 | 33 | with self.assertRaisesRegex(OTSClientError, "access_key_id is not str or is empty."): 34 | OTSClient(test_endpoint, "", test_ak_secret, test_instance, region=test_region) 35 | with self.assertRaisesRegex(OTSClientError, "access_key_id is not str or is empty."): 36 | OTSClient(test_endpoint, 1, test_ak_secret, test_instance, region=test_region) 37 | 38 | with self.assertRaisesRegex(OTSClientError, "access_key_secret is not str or is empty."): 39 | OTSClient(test_endpoint, test_ak_id, "", test_instance, region=test_region) 40 | with self.assertRaisesRegex(OTSClientError, "access_key_secret is not str or is empty."): 41 | OTSClient(test_endpoint, test_ak_id, 1, test_instance, region=test_region) 42 | 43 | with self.assertRaisesRegex(OTSClientError, "instance_name is not str or is empty."): 44 | OTSClient(test_endpoint, test_ak_id, test_ak_secret, "", region=test_region) 45 | with self.assertRaisesRegex(OTSClientError, "instance_name is not str or is empty."): 46 | OTSClient(test_endpoint, test_ak_id, test_ak_secret, 1, region=test_region) 47 | 48 | with self.assertRaisesRegex(OTSClientError, "region is not str or is empty."): 49 | OTSClient(test_endpoint, test_ak_id, test_ak_secret, test_instance, region="") 50 | with self.assertRaisesRegex(OTSClientError, "region is not str or is empty."): 51 | OTSClient(test_endpoint, test_ak_id, test_ak_secret, test_instance, region=1) 52 | 53 | with self.assertRaisesRegex(OTSClientError, "protocol of end_point must be 'http' or 'https', e.g. https://instance.cn-hangzhou.ots.aliyun.com."): 54 | test_client = OTSClient("tcp://instance.cn-hangzhou.ots.aliyun.com.", test_ak_id, test_ak_secret, test_instance, region=test_region) 55 | 56 | test_client = OTSClient(test_endpoint, test_ak_id, test_ak_secret, test_instance, region=test_region) 57 | self.assertEqual(test_ak_id, test_client.credentials_provider.get_credentials().get_access_key_id()) 58 | self.assertEqual(test_region, test_client._signer.region) 59 | self.assertEqual(True, test_client._signer.auto_update_v4_sign) 60 | 61 | test_date = "20250101" 62 | test_client = OTSClient(test_endpoint, test_ak_id, test_ak_secret, test_instance, region=test_region, sign_date=test_date, auto_update_v4_sign=False) 63 | self.assertEqual(test_ak_id, test_client.credentials_provider.get_credentials().get_access_key_id()) 64 | self.assertEqual(test_region, test_client._signer.region) 65 | self.assertEqual(test_date, test_client._signer.sign_date) 66 | self.assertEqual(False, test_client._signer.auto_update_v4_sign) 67 | -------------------------------------------------------------------------------- /tablestore/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __version__ = '6.3.0' 4 | __all__ = [ 5 | 'OTSClient', 6 | 'AsyncOTSClient', 7 | 8 | # Data Types 9 | 'INF_MIN', 10 | 'INF_MAX', 11 | 'PK_AUTO_INCR', 12 | 'TableMeta', 13 | 'TableOptions', 14 | 'CapacityUnit', 15 | 'ReservedThroughput', 16 | 'ReservedThroughputDetails', 17 | 'ColumnType', 18 | 'ReturnType', 19 | 'Column', 20 | 'Direction', 21 | 'UpdateTableResponse', 22 | 'DescribeTableResponse', 23 | 'RowDataItem', 24 | 'Condition', 25 | 'Row', 26 | 'RowItem', 27 | 'PutRowItem', 28 | 'UpdateRowItem', 29 | 'DeleteRowItem', 30 | 'BatchGetRowRequest', 31 | 'TableInBatchGetRowItem', 32 | 'BatchGetRowResponse', 33 | 'BatchWriteRowType', 34 | 'BatchWriteRowRequest', 35 | 'TableInBatchWriteRowItem', 36 | 'BatchWriteRowResponse', 37 | 'BatchWriteRowResponseItem', 38 | 'OTSClientError', 39 | 'OTSServiceError', 40 | 'DefaultRetryPolicy', 41 | 'LogicalOperator', 42 | 'ComparatorType', 43 | 'CastType', 44 | 'ColumnConditionType', 45 | 'ColumnCondition', 46 | 'CompositeColumnCondition', 47 | 'RegexRule', 48 | 'SingleColumnCondition', 49 | 'SingleColumnRegexCondition', 50 | 'RowExistenceExpectation', 51 | 'SearchIndexMeta', 52 | 'FieldSchema', 53 | 'VectorOptions', 54 | 'VectorDataType', 55 | 'VectorMetricType', 56 | 'FieldType', 57 | 'IndexSetting', 58 | 'Collapse', 59 | 'Sort', 60 | 'PrimaryKeySort', 61 | 'ScoreSort', 62 | 'GeoDistanceSort', 63 | 'FieldSort', 64 | 'DocSort', 65 | 'SortOrder', 66 | 'SortMode', 67 | 'ScoreMode', 68 | 'AnalyzerType', 69 | 'SingleWordAnalyzerParameter', 70 | 'SplitAnalyzerParameter', 71 | 'FuzzyAnalyzerParameter', 72 | 'Sorter', 73 | 'SyncStat', 74 | 'SyncPhase', 75 | 'QueryOperator', 76 | 'MatchQuery', 77 | 'MatchPhraseQuery', 78 | 'TermQuery', 79 | 'RangeQuery', 80 | 'PrefixQuery', 81 | 'BoolQuery', 82 | 'FunctionScoreQuery', 83 | 'NestedQuery', 84 | 'InnerHits', 85 | 'WildcardQuery', 86 | 'MatchAllQuery', 87 | 'GeoBoundingBoxQuery', 88 | 'GeoDistanceQuery', 89 | 'GeoPolygonQuery', 90 | 'TermsQuery', 91 | 'SearchQuery', 92 | 'ScanQuery', 93 | 'HighlightParameter', 94 | 'Highlight', 95 | 'HighlightEncoder', 96 | 'HighlightFragmentOrder', 97 | 'SearchHit', 98 | 'SearchInnerHit', 99 | 'HighlightResult', 100 | 'HighlightField', 101 | 'ColumnsToGet', 102 | 'ColumnReturnType', 103 | 'FieldValueFactor', 104 | 'GeoDistanceType', 105 | 'NestedFilter', 106 | 'DefinedColumnSchema', 107 | 'SecondaryIndexMeta', 108 | 'SecondaryIndexType', 109 | 'ExistsQuery', 110 | 'KnnVectorQuery', 111 | 'Agg', 112 | 'Max', 113 | 'Min', 114 | 'Avg', 115 | 'Sum', 116 | 'Count', 117 | 'DistinctCount', 118 | 'TopRows', 119 | 'Percentiles', 120 | 'AggResult', 121 | 'PercentilesResultItem', 122 | 'GroupKeySort', 123 | 'RowCountSort', 124 | 'SubAggSort', 125 | 'GeoPoint', 126 | 'FieldRange', 127 | 'BaseGroupBy', 128 | 'GroupByField', 129 | 'GroupByRange', 130 | 'GroupByFilter', 131 | 'GroupByGeoDistance', 132 | 'GroupByHistogram', 133 | 'GroupByResult', 134 | 'BaseGroupByResultItem', 135 | 'GroupByFieldResultItem', 136 | 'GroupByRangeResultItem', 137 | 'GroupByFilterResultItem', 138 | 'GroupByGeoDistanceResultItem', 139 | 'GroupByHistogramResultItem', 140 | ] 141 | 142 | 143 | from tablestore.client import OTSClient, AsyncOTSClient 144 | from tablestore.metadata import * 145 | from tablestore.aggregation import * 146 | from tablestore.group_by import * 147 | from tablestore.error import * 148 | from tablestore.retry import * 149 | from tablestore.credentials import * 150 | from tablestore.const_module import * 151 | -------------------------------------------------------------------------------- /tablestore/plainbuffer/plain_buffer_stream.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | import six 4 | import struct 5 | from builtins import int 6 | from tablestore.error import * 7 | from .plain_buffer_consts import * 8 | 9 | class PlainBufferInputStream(object): 10 | def __init__(self, data_buffer): 11 | self.buffer = data_buffer 12 | self.len = len(self.buffer) 13 | self.cur_pos = 0 14 | self.last_tag = chr(0) 15 | 16 | def read_tag(self): 17 | if len(self.buffer) == self.cur_pos: 18 | self.last_tag = 0 19 | else: 20 | self.last_tag = self.read_raw_byte() 21 | 22 | def check_last_tag_was(self, tag): 23 | return self.last_tag == tag 24 | 25 | def get_last_tag(self): 26 | return self.last_tag 27 | 28 | def read_raw_byte(self): 29 | if self.len != self.cur_pos: 30 | pos = self.cur_pos 31 | self.cur_pos += 1 32 | if const.python_version == 2: 33 | return self.buffer[pos] 34 | else: 35 | return chr(self.buffer[pos]) 36 | else: 37 | raise OTSClientError("Read raw byte encountered EOF.") 38 | 39 | def read_raw_little_endian64(self): 40 | return struct.unpack(' self.capacity: 114 | raise OTSClientError("The buffer is full.") 115 | if isinstance(value, six.text_type): 116 | value = value.encode('utf-8') 117 | self.buffer += bytearray(value) 118 | -------------------------------------------------------------------------------- /examples/pk_auto_incr.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from example_config import * 4 | 5 | from tablestore import * 6 | import tablestore.protobuf.table_store_pb2 as pb2 7 | import time 8 | 9 | table_name = 'OTSPkAutoIncrSimpleExample' 10 | 11 | 12 | def create_table(client): 13 | schema_of_primary_key = [('gid', 'INTEGER'), ('uid', 'INTEGER', PK_AUTO_INCR)] 14 | table_meta = TableMeta(table_name, schema_of_primary_key) 15 | table_options = TableOptions() 16 | reserved_throughput = ReservedThroughput(CapacityUnit(0, 0)) 17 | client.create_table(table_meta, table_options, reserved_throughput) 18 | print('Table has been created.') 19 | 20 | 21 | def describe_table(client): 22 | describe_response = client.describe_table(table_name) 23 | print('TableName: %s' % describe_response.table_meta.table_name) 24 | print('PrimaryKey: %s' % describe_response.table_meta.schema_of_primary_key) 25 | 26 | 27 | def delete_table(client): 28 | client.delete_table(table_name) 29 | print('Table \'%s\' has been deleted.' % table_name) 30 | 31 | 32 | def put_row(client): 33 | primary_key = [('gid', 1), ('uid', PK_AUTO_INCR)] 34 | attribute_columns = [('name', 'John'), ('mobile', 15100000000), ('address', 'China'), ('age', 20)] 35 | row = Row(primary_key, attribute_columns) 36 | 37 | # Expect not exist: put it into table only when this row is not exist. 38 | row.attribute_columns = [('name', 'John'), ('mobile', 15100000000), ('address', 'China'), ('age', 25)] 39 | consumed, return_row = client.put_row(table_name, row) 40 | print('Write succeed, consume %s write cu.' % consumed.write) 41 | 42 | consumed, return_row = client.put_row(table_name, row, return_type=ReturnType.RT_PK) 43 | print('Write succeed, consume %s write cu.' % consumed.write) 44 | print('Primary key:%s' % return_row.primary_key) 45 | 46 | 47 | def batch_write_row(client): 48 | put_row_items = [] 49 | for i in range(0, 10): 50 | primary_key = [('gid', i), ('uid', PK_AUTO_INCR)] 51 | attribute_columns = [('name', 'somebody' + str(i)), ('address', 'somewhere' + str(i)), ('age', i)] 52 | row = Row(primary_key, attribute_columns) 53 | condition = Condition(RowExistenceExpectation.IGNORE) 54 | item = PutRowItem(row, condition, return_type=ReturnType.RT_PK) 55 | put_row_items.append(item) 56 | 57 | request = BatchWriteRowRequest() 58 | request.add(TableInBatchWriteRowItem(table_name, put_row_items)) 59 | result = client.batch_write_row(request) 60 | 61 | print('Result status: %s' % (result.is_all_succeed())) 62 | print('check first table\'s put results:') 63 | succ, fail = result.get_put() 64 | for item in succ: 65 | print('Put succeed, primary key:%s.' % item.row.primary_key) 66 | for item in fail: 67 | print('Put failed, error code: %s, error message: %s' % (item.error_code, item.error_message)) 68 | 69 | 70 | def get_range(client): 71 | inclusive_start_primary_key = [('gid', INF_MIN), ('uid', INF_MIN)] 72 | exclusive_end_primary_key = [('gid', INF_MAX), ('uid', INF_MAX)] 73 | columns_to_get = [] 74 | limit = 90 75 | 76 | consumed, next_start_primary_key, row_list, next_token = client.get_range( 77 | table_name, Direction.FORWARD, 78 | inclusive_start_primary_key, exclusive_end_primary_key, 79 | columns_to_get, 80 | limit, 81 | column_filter=None, 82 | max_version=1 83 | ) 84 | for row in row_list: 85 | print(row.primary_key) 86 | 87 | 88 | if __name__ == '__main__': 89 | client = OTSClient(OTS_ENDPOINT, OTS_ACCESS_KEY_ID, OTS_ACCESS_KEY_SECRET, OTS_INSTANCE, region=OTS_REGION) 90 | try: 91 | delete_table(client) 92 | except: 93 | pass 94 | create_table(client) 95 | 96 | time.sleep(3) # wait for table ready 97 | 98 | describe_table(client) 99 | put_row(client) 100 | batch_write_row(client) 101 | get_range(client) 102 | delete_table(client) 103 | -------------------------------------------------------------------------------- /tablestore/connection.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | import ssl 3 | import time 4 | 5 | import aiohttp 6 | from aiohttp import ClientTimeout 7 | 8 | try: 9 | import httplib 10 | except ImportError: 11 | import http.client 12 | 13 | from urllib3.poolmanager import PoolManager 14 | from urllib3.connectionpool import HTTPConnectionPool 15 | import certifi 16 | 17 | from tablestore.error import * 18 | 19 | _NETWORK_IO_TIME_COUNT_FLAG = False 20 | _network_io_time = 0 21 | 22 | 23 | class ConnectionPool(object): 24 | 25 | NUM_POOLS = 5 # one pool per host, usually just 1 pool is needed 26 | # when redirect happens, one additional pool will be created 27 | 28 | def __init__(self, host, path, timeout=0, maxsize=50, client_ssl_version=None): 29 | self.host = host 30 | self.path = path 31 | 32 | self.pool = PoolManager( 33 | self.NUM_POOLS, 34 | headers=None, 35 | cert_reqs='CERT_REQUIRED', # Force certificate check 36 | ca_certs=certifi.where(), # Path to the Certifi bundle 37 | timeout=timeout, 38 | maxsize=maxsize, 39 | block=True, 40 | ssl_version=client_ssl_version 41 | ) 42 | 43 | def send_receive(self, url, request_headers, request_body): 44 | 45 | global _network_io_time 46 | 47 | if _NETWORK_IO_TIME_COUNT_FLAG: 48 | begin = time.time() 49 | 50 | response = self.pool.urlopen( 51 | 'POST', self.host + self.path + url, 52 | body=request_body, headers=request_headers, 53 | redirect=False, 54 | assert_same_host=False, 55 | ) 56 | 57 | if _NETWORK_IO_TIME_COUNT_FLAG: 58 | end = time.time() 59 | _network_io_time += end - begin 60 | 61 | # TODO error handling 62 | response_headers = dict(response.headers) 63 | response_body = response.data # TODO figure out why response.read() don't work 64 | 65 | return response.status, response.reason, response_headers, response_body 66 | 67 | 68 | class AsyncConnectionPool(object): 69 | 70 | def __init__(self, host, path, timeout=50, maxsize=50, keepalive_timeout=12, force_close=False, client_ssl_version=None): 71 | self.host = host 72 | self.path = path 73 | 74 | if isinstance(timeout, (list, tuple)): 75 | conn_timeout, read_timeout = timeout 76 | else: 77 | conn_timeout = read_timeout = timeout 78 | 79 | ssl_context = None 80 | if client_ssl_version is not None: 81 | ssl_context = ssl.create_default_context() 82 | ssl_context.minimum_version = client_ssl_version 83 | 84 | self.pool = aiohttp.ClientSession( 85 | timeout=ClientTimeout( 86 | sock_connect=conn_timeout, 87 | sock_read=read_timeout 88 | ), 89 | connector=aiohttp.TCPConnector( 90 | limit=maxsize, 91 | ssl_context=ssl_context, 92 | keepalive_timeout=keepalive_timeout, 93 | force_close=force_close, 94 | ) 95 | ) 96 | 97 | async def send_receive(self, url, request_headers, request_body): 98 | 99 | global _network_io_time 100 | 101 | if _NETWORK_IO_TIME_COUNT_FLAG: 102 | begin = time.time() 103 | 104 | async with self.pool.request( 105 | 'POST', self.host + self.path + url, 106 | data=request_body, 107 | headers=request_headers, 108 | allow_redirects=False, 109 | ) as response: 110 | response_body = await response.read() 111 | 112 | if _NETWORK_IO_TIME_COUNT_FLAG: 113 | end = time.time() 114 | _network_io_time += end - begin 115 | 116 | response_headers = dict(response.headers) 117 | 118 | return response.status, response.reason, response_headers, response_body 119 | 120 | async def close(self): 121 | await self.pool.close() -------------------------------------------------------------------------------- /examples/get_row.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from example_config import * 4 | from tablestore import * 5 | import time 6 | 7 | table_name = 'OTSGetRowSimpleExample' 8 | 9 | 10 | def create_table(client): 11 | schema_of_primary_key = [('uid', 'INTEGER'), ('gid', 'INTEGER')] 12 | table_meta = TableMeta(table_name, schema_of_primary_key) 13 | table_options = TableOptions() 14 | reserved_throughput = ReservedThroughput(CapacityUnit(0, 0)) 15 | client.create_table(table_meta, table_options, reserved_throughput) 16 | print('Table has been created.') 17 | 18 | 19 | def delete_table(client): 20 | client.delete_table(table_name) 21 | print('Table \'%s\' has been deleted.' % table_name) 22 | 23 | 24 | def put_row(client): 25 | primary_key = [('uid', 1), ('gid', 101)] 26 | attribute_columns = [('name', '杭州'), ('growth', 0.95), ('type', 'sub-provincial city'), ('postal code', 310000), 27 | ('Alibaba', True), ('chengdu', False)] 28 | row = Row(primary_key, attribute_columns) 29 | condition = Condition( 30 | RowExistenceExpectation.EXPECT_NOT_EXIST) # Expect not exist: put it into table only when this row is not exist. 31 | consumed, return_row = client.put_row(table_name, row, condition) 32 | print('Write succeed, consume %s write cu.' % consumed.write) 33 | 34 | 35 | def get_row(client): 36 | primary_key = [('uid', 1), ('gid', 101)] 37 | columns_to_get = [] # given a list of columns to get, or empty list if you want to get entire row. 38 | 39 | cond = CompositeColumnCondition(LogicalOperator.AND) 40 | cond.add_sub_condition(SingleColumnCondition("growth", 0.9, ComparatorType.NOT_EQUAL)) 41 | cond.add_sub_condition(SingleColumnCondition("name", '杭州', ComparatorType.EQUAL)) 42 | 43 | consumed, return_row, next_token = client.get_row(table_name, primary_key, columns_to_get, cond, 1) 44 | 45 | print('Read succeed, consume %s read cu.' % consumed.read) 46 | 47 | print('Value of primary key: %s' % return_row.primary_key) 48 | print('Value of attribute: %s' % return_row.attribute_columns) 49 | for att in return_row.attribute_columns: 50 | print('name:%s\tvalue:%s\ttimestamp:%d' % (att[0], att[1], att[2])) 51 | 52 | 53 | def get_row2(client): 54 | primary_key = [('uid', 1), ('gid', 101)] 55 | columns_to_get = [] 56 | 57 | cond = CompositeColumnCondition(LogicalOperator.AND) 58 | cond.add_sub_condition(SingleColumnCondition("growth", 0.9, ComparatorType.NOT_EQUAL)) 59 | cond.add_sub_condition(SingleColumnCondition("name", '杭州', ComparatorType.EQUAL)) 60 | 61 | consumed, return_row, next_token = client.get_row(table_name, primary_key, columns_to_get, cond, 1, 62 | start_column='Alibaba', end_column='name') 63 | 64 | print('Read succeed, consume %s read cu.' % consumed.read) 65 | 66 | print('Value of primary key: %s' % return_row.primary_key) 67 | print('Value of attribute: %s' % return_row.attribute_columns) 68 | 69 | def get_row3(client): 70 | primary_key = [('uid', 1), ('gid', 101)] 71 | columns_to_get = [] 72 | 73 | cond = CompositeColumnCondition(LogicalOperator.AND) 74 | cond.add_sub_condition(SingleColumnCondition("growth", 0.9, ComparatorType.NOT_EQUAL)) 75 | cond.add_sub_condition(SingleColumnRegexCondition("Alibaba", ComparatorType.EXIST, None)) 76 | 77 | consumed, return_row, next_token = client.get_row(table_name, primary_key, columns_to_get, cond, 1, 78 | start_column='Alibaba', end_column='name') 79 | 80 | print('Read succeed, consume %s read cu.' % consumed.read) 81 | 82 | print('Value of primary key: %s' % return_row.primary_key) 83 | print('Value of attribute: %s' % return_row.attribute_columns) 84 | 85 | 86 | if __name__ == '__main__': 87 | client = OTSClient(OTS_ENDPOINT, OTS_ACCESS_KEY_ID, OTS_ACCESS_KEY_SECRET, OTS_INSTANCE, region=OTS_REGION) 88 | try: 89 | delete_table(client) 90 | except: 91 | pass 92 | create_table(client) 93 | 94 | time.sleep(3) # wait for table ready 95 | put_row(client) 96 | get_row(client) 97 | get_row2(client) 98 | get_row3(client) 99 | delete_table(client) 100 | -------------------------------------------------------------------------------- /examples/transaction_and_abort.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from example_config import * 4 | from tablestore import * 5 | import time 6 | 7 | # Table support transaction must be set in advance, we can't create and use transaction immediately! 8 | table_name = 'TransactionTable' 9 | 10 | 11 | def start_transaction(): 12 | key = [('PK0', 1)] 13 | transaction_id = client.start_local_transaction(table_name, key) 14 | print ('Value of transaction id: %s' % transaction_id) 15 | 16 | return transaction_id 17 | 18 | 19 | def put_row(): 20 | primary_key = [('PK0', 1), ('PK1', 'transaction')] 21 | attribute_columns = [('value', 'origion value')] 22 | row = Row(primary_key, attribute_columns) 23 | condition = Condition(RowExistenceExpectation.IGNORE) 24 | consumed, return_row = client.put_row(table_name, row, condition) 25 | print ('Write succeed, consume %s write cu.' % consumed.write) 26 | 27 | 28 | def get_row(transaction_id): 29 | primary_key = [('PK0', 1), ('PK1', 'transaction')] 30 | columns_to_get = ['value'] 31 | 32 | consumed, return_row, next_token = client.get_row( 33 | table_name, primary_key, columns_to_get, None, 1, None, None, None, None, transaction_id 34 | ) 35 | 36 | for att in return_row.attribute_columns: 37 | print ('\tname:%s\tvalue:%s' % (att[0], att[1])) 38 | 39 | 40 | def update_row(transaction_id): 41 | primary_key = [('PK0', 1), ('PK1', 'transaction')] 42 | update_of_attribute_columns = { 43 | 'PUT': [('value', 'new value')] 44 | } 45 | row = Row(primary_key, update_of_attribute_columns) 46 | condition = Condition(RowExistenceExpectation.IGNORE) 47 | consumed, return_row = client.update_row(table_name, row, condition, None, transaction_id) 48 | print ('Update succeed, consume %s write cu.' % consumed.write) 49 | 50 | 51 | def get_range(transaction_id): 52 | inclusive_start_primary_key = [('PK0', 1), ('PK1', INF_MIN)] 53 | exclusive_end_primary_key = [('PK0', 1), ('PK1', INF_MAX)] 54 | columns_to_get = [] 55 | limit = 1 56 | 57 | consumed, next_start_primary_key, row_list, next_token = client.get_range( 58 | table_name, Direction.FORWARD, 59 | inclusive_start_primary_key, exclusive_end_primary_key, 60 | columns_to_get, 61 | limit, 62 | column_filter=None, 63 | max_version=1, 64 | transaction_id=transaction_id 65 | ) 66 | 67 | all_rows = [] 68 | all_rows.extend(row_list) 69 | 70 | for row in all_rows: 71 | print (row.primary_key, row.attribute_columns) 72 | print ('Total rows: ', len(all_rows)) 73 | 74 | 75 | def batch_write_row(transaction_id): 76 | put_row_items = [] 77 | 78 | primary_key = [('PK0', 1), ('PK1', 'transaction')] 79 | attribute_columns = {'put': [('batch', 'batch value')]} 80 | row = Row(primary_key, attribute_columns) 81 | condition = Condition(RowExistenceExpectation.IGNORE) 82 | item = UpdateRowItem(row, condition) 83 | put_row_items.append(item) 84 | 85 | request = BatchWriteRowRequest() 86 | request.add(TableInBatchWriteRowItem(table_name, put_row_items)) 87 | request.set_transaction_id(transaction_id) 88 | 89 | result = client.batch_write_row(request) 90 | 91 | print ('Result status: %s' % (result.is_all_succeed())) 92 | print ('check first table\'s put results:') 93 | 94 | succ, fail = result.get_update() 95 | for item in succ: 96 | print ('Update succeed, consume %s write cu.' % item.consumed.write) 97 | for item in fail: 98 | print ('Update failed, error code: %s, error message: %s' % (item.error_code, item.error_message)) 99 | 100 | 101 | def abort_transaction(transaction_id): 102 | client.abort_transaction(transaction_id) 103 | 104 | 105 | if __name__ == '__main__': 106 | client = OTSClient(OTS_ENDPOINT, OTS_ACCESS_KEY_ID, OTS_ACCESS_KEY_SECRET, OTS_INSTANCE, region=OTS_REGION) 107 | 108 | put_row() 109 | get_row(None) 110 | 111 | transaction_id = start_transaction() 112 | update_row(transaction_id) 113 | batch_write_row(transaction_id) 114 | 115 | print ('Get Origin Value') 116 | get_row(None) 117 | print ('Get Transaction Value') 118 | get_row(transaction_id) 119 | 120 | get_range(None) 121 | get_range(transaction_id) 122 | abort_transaction(transaction_id) 123 | 124 | print ('Get Final Value [Abort]') 125 | get_row(None) 126 | -------------------------------------------------------------------------------- /examples/transaction_and_commit.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from example_config import * 4 | from tablestore import * 5 | import time 6 | 7 | # Table support transaction must be set in advance, we can't create and use transaction immediately! 8 | table_name = 'TransactionTable' 9 | 10 | 11 | def start_transaction(): 12 | key = [('PK0', 1)] 13 | transaction_id = client.start_local_transaction(table_name, key) 14 | print ('Value of transaction id: %s' % transaction_id) 15 | 16 | return transaction_id 17 | 18 | 19 | def put_row(): 20 | primary_key = [('PK0', 1), ('PK1', 'transaction')] 21 | attribute_columns = [('value', 'origion value')] 22 | row = Row(primary_key, attribute_columns) 23 | condition = Condition(RowExistenceExpectation.IGNORE) 24 | consumed, return_row = client.put_row(table_name, row, condition) 25 | print ('Write succeed, consume %s write cu.' % consumed.write) 26 | 27 | 28 | def get_row(transaction_id): 29 | primary_key = [('PK0', 1), ('PK1', 'transaction')] 30 | columns_to_get = ['value', 'batch'] 31 | 32 | consumed, return_row, next_token = client.get_row( 33 | table_name, primary_key, columns_to_get, None, 1, transaction_id=transaction_id 34 | ) 35 | 36 | for att in return_row.attribute_columns: 37 | print ('\tname:%s\tvalue:%s' % (att[0], att[1])) 38 | 39 | 40 | def update_row(transaction_id): 41 | primary_key = [('PK0', 1), ('PK1', 'transaction')] 42 | update_of_attribute_columns = { 43 | 'PUT': [('value', 'new value')] 44 | } 45 | row = Row(primary_key, update_of_attribute_columns) 46 | condition = Condition(RowExistenceExpectation.IGNORE) 47 | consumed, return_row = client.update_row(table_name, row, condition, None, transaction_id) 48 | print ('Update succeed, consume %s write cu.' % consumed.write) 49 | 50 | 51 | def get_range(transaction_id): 52 | inclusive_start_primary_key = [('PK0', 1), ('PK1', INF_MIN)] 53 | exclusive_end_primary_key = [('PK0', 1), ('PK1', INF_MAX)] 54 | columns_to_get = [] 55 | limit = 1 56 | 57 | consumed, next_start_primary_key, row_list, next_token = client.get_range( 58 | table_name, Direction.FORWARD, 59 | inclusive_start_primary_key, exclusive_end_primary_key, 60 | columns_to_get, 61 | limit, 62 | column_filter=None, 63 | max_version=1, 64 | transaction_id=transaction_id 65 | ) 66 | 67 | all_rows = [] 68 | all_rows.extend(row_list) 69 | 70 | for row in all_rows: 71 | print (row.primary_key, row.attribute_columns) 72 | print ('Total rows: ', len(all_rows)) 73 | 74 | 75 | def batch_write_row(transaction_id): 76 | put_row_items = [] 77 | 78 | primary_key = [('PK0', 1), ('PK1', 'transaction')] 79 | attribute_columns = {'put': [('batch', 'batch value')]} 80 | row = Row(primary_key, attribute_columns) 81 | condition = Condition(RowExistenceExpectation.IGNORE) 82 | item = UpdateRowItem(row, condition) 83 | put_row_items.append(item) 84 | 85 | request = BatchWriteRowRequest() 86 | request.add(TableInBatchWriteRowItem(table_name, put_row_items)) 87 | request.set_transaction_id(transaction_id) 88 | 89 | result = client.batch_write_row(request) 90 | 91 | print ('Result status: %s'%(result.is_all_succeed())) 92 | print ('check first table\'s put results:') 93 | 94 | succ, fail = result.get_update() 95 | for item in succ: 96 | print ('Update succeed, consume %s write cu.' % item.consumed.write) 97 | for item in fail: 98 | print ('Update failed, error code: %s, error message: %s' % (item.error_code, item.error_message)) 99 | 100 | 101 | def commit_transaction(transaction_id): 102 | client.commit_transaction(transaction_id) 103 | 104 | 105 | if __name__ == '__main__': 106 | client = OTSClient(OTS_ENDPOINT, OTS_ACCESS_KEY_ID, OTS_ACCESS_KEY_SECRET, OTS_INSTANCE, region=OTS_REGION) 107 | 108 | put_row() 109 | get_row(None) 110 | 111 | transaction_id = start_transaction() 112 | update_row(transaction_id) 113 | batch_write_row(transaction_id) 114 | 115 | print ('Get Origin Value') 116 | get_row(None) 117 | print ('Get Transaction Value') 118 | get_row(transaction_id) 119 | 120 | get_range(None) 121 | get_range(transaction_id) 122 | commit_transaction(transaction_id) 123 | 124 | print ('Get Final Value [Commit]') 125 | get_row(None) 126 | -------------------------------------------------------------------------------- /tablestore/aggregation.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from tablestore.metadata import * 4 | from tablestore.plainbuffer.plain_buffer_builder import * 5 | import tablestore.protobuf.search_pb2 as search_pb2 6 | 7 | class Agg(object): 8 | 9 | def __init__(self, field, missing_value, name, agg_type): 10 | self.field = field 11 | self.missing = missing_value 12 | self.name = name 13 | self.type = agg_type 14 | 15 | def to_pb_str(self, proto): 16 | agg = proto 17 | agg.field_name = self.field 18 | 19 | if self.missing is not None: 20 | agg.missing = bytes(PlainBufferBuilder.serialize_column_value(self.missing)) 21 | 22 | return agg.SerializeToString() 23 | 24 | class Max(Agg): 25 | 26 | def __init__(self, field, missing_value = None, name = 'max'): 27 | Agg.__init__(self, field, missing_value, name, search_pb2.AGG_MAX) 28 | 29 | def to_pb_str(self): 30 | return Agg.to_pb_str(self, search_pb2.MaxAggregation()) 31 | 32 | 33 | class Min(Agg): 34 | 35 | def __init__(self, field, missing_value = None, name = 'min'): 36 | Agg.__init__(self, field, missing_value, name, search_pb2.AGG_MIN) 37 | 38 | def to_pb_str(self): 39 | return Agg.to_pb_str(self, search_pb2.MinAggregation()) 40 | 41 | 42 | class Avg(Agg): 43 | 44 | def __init__(self, field, missing_value = None, name = 'avg'): 45 | Agg.__init__(self, field, missing_value, name, search_pb2.AGG_AVG) 46 | 47 | def to_pb_str(self): 48 | return Agg.to_pb_str(self, search_pb2.AvgAggregation()) 49 | 50 | 51 | class Sum(Agg): 52 | 53 | def __init__(self, field, missing_value = None, name = 'sum'): 54 | Agg.__init__(self, field, missing_value, name, search_pb2.AGG_SUM) 55 | 56 | def to_pb_str(self): 57 | return Agg.to_pb_str(self, search_pb2.SumAggregation()) 58 | 59 | 60 | class Count(Agg): 61 | 62 | def __init__(self, field, name = 'count'): 63 | Agg.__init__(self, field, None, name, search_pb2.AGG_COUNT) 64 | 65 | def to_pb_str(self): 66 | return Agg.to_pb_str(self, search_pb2.SumAggregation()) 67 | 68 | 69 | class DistinctCount(Agg): 70 | 71 | def __init__(self, field, missing_value = None, name = 'distinct_count'): 72 | Agg.__init__(self, field, missing_value, name, search_pb2.AGG_DISTINCT_COUNT) 73 | 74 | def to_pb_str(self): 75 | return Agg.to_pb_str(self, search_pb2.DistinctCountAggregation()) 76 | 77 | class Percentiles(Agg): 78 | 79 | def __init__(self, field, percentiles_list, missing_value = None, name = 'percentiles'): 80 | Agg.__init__(self, field, missing_value, name, search_pb2.AGG_PERCENTILES) 81 | self.percentiles_list = percentiles_list 82 | 83 | def to_pb_str(self): 84 | agg = search_pb2.PercentilesAggregation() 85 | agg.field_name = self.field 86 | 87 | if self.missing is not None: 88 | agg.missing = bytes(PlainBufferBuilder.serialize_column_value(self.missing)) 89 | 90 | for percentile in self.percentiles_list: 91 | agg.percentiles.append(percentile) 92 | 93 | return agg.SerializeToString() 94 | 95 | 96 | """ 97 | TopRows: used in group_by 98 | """ 99 | class TopRows(object): 100 | 101 | def __init__(self, limit, sort, name = 'top_rows'): 102 | self.limit = limit 103 | self.sort = sort 104 | self.name = name 105 | self.type = search_pb2.AGG_TOP_ROWS 106 | 107 | def to_pb_str(self, encode_sort_func): 108 | agg = search_pb2.TopRowsAggregation() 109 | agg.limit = self.limit 110 | 111 | if self.sort is not None: 112 | for sorter in self.sort.sorters: 113 | encode_sort_func(agg.sort.sorter.add(), sorter) 114 | 115 | return agg.SerializeToString() 116 | 117 | """ 118 | AggreagtionType ValueType 119 | Max double 120 | Min double 121 | Sum double 122 | Count int64 123 | DistinctCount int64 124 | TopRows [Row] 125 | Percentiles [(key1, value1), (key2, value2)....] 126 | """ 127 | class AggResult(object): 128 | def __init__(self, name, value): 129 | self.name = name 130 | self.value = value 131 | 132 | class PercentilesResultItem(object): 133 | def __init__(self, key, value): 134 | self.key = key 135 | self.value = value 136 | 137 | -------------------------------------------------------------------------------- /examples/get_range.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from example_config import * 4 | from tablestore import * 5 | import time 6 | 7 | table_name = 'OTSGetRangeSimpleExample' 8 | 9 | 10 | def create_table(client): 11 | schema_of_primary_key = [('uid', 'INTEGER'), ('gid', 'BINARY')] 12 | table_meta = TableMeta(table_name, schema_of_primary_key) 13 | table_option = TableOptions() 14 | reserved_throughput = ReservedThroughput(CapacityUnit(0, 0)) 15 | client.create_table(table_meta, table_option, reserved_throughput) 16 | print('Table has been created.') 17 | 18 | 19 | def delete_table(client): 20 | client.delete_table(table_name) 21 | print('Table \'%s\' has been deleted.' % table_name) 22 | 23 | 24 | def put_row(client): 25 | for i in range(0, 100): 26 | primary_key = [('uid', i), ('gid', bytearray(str(i + 1), 'utf-8'))] 27 | attribute_columns = [('name', 'John'), ('mobile', i), ('address', 'China'), ('age', i)] 28 | row = Row(primary_key, attribute_columns) 29 | condition = Condition( 30 | RowExistenceExpectation.IGNORE) # Expect not exist: put it into table only when this row is not exist. 31 | consumed, return_row = client.put_row(table_name, row, condition) 32 | print('Write succeed, consume %s write cu.' % consumed.write) 33 | 34 | 35 | def get_range(client): 36 | ''' 37 | Scan table to get all the rows. 38 | It will not return you all once, you should continue read from next start primary key till next start primary key is None. 39 | ''' 40 | inclusive_start_primary_key = [('uid', INF_MIN), ('gid', INF_MIN)] 41 | exclusive_end_primary_key = [('uid', INF_MAX), ('gid', INF_MAX)] 42 | columns_to_get = [] 43 | limit = 90 44 | 45 | cond = CompositeColumnCondition(LogicalOperator.AND) 46 | cond.add_sub_condition(SingleColumnCondition("address", 'China', ComparatorType.EQUAL)) 47 | cond.add_sub_condition(SingleColumnCondition("age", 50, ComparatorType.LESS_THAN)) 48 | 49 | consumed, next_start_primary_key, row_list, next_token = client.get_range( 50 | table_name, Direction.FORWARD, 51 | inclusive_start_primary_key, exclusive_end_primary_key, 52 | columns_to_get, 53 | limit, 54 | column_filter=cond, 55 | max_version=1 56 | ) 57 | 58 | all_rows = [] 59 | all_rows.extend(row_list) 60 | while next_start_primary_key is not None: 61 | inclusive_start_primary_key = next_start_primary_key 62 | consumed, next_start_primary_key, row_list, next_token = client.get_range( 63 | table_name, Direction.FORWARD, 64 | inclusive_start_primary_key, exclusive_end_primary_key, 65 | columns_to_get, limit, 66 | column_filter=cond, 67 | max_version=1 68 | ) 69 | all_rows.extend(row_list) 70 | print('Read succeed, consume %s read cu.' % consumed.read) 71 | 72 | for row in all_rows: 73 | print(row.primary_key, row.attribute_columns) 74 | print('Total rows: ', len(all_rows)) 75 | 76 | 77 | def xget_range(client): 78 | ''' 79 | You can easily scan the range use xget_range, without handling next start primary key. 80 | ''' 81 | consumed_counter = CapacityUnit(0, 0) 82 | inclusive_start_primary_key = [('uid', INF_MIN), ('gid', INF_MIN)] 83 | exclusive_end_primary_key = [('uid', INF_MAX), ('gid', INF_MAX)] 84 | 85 | cond = CompositeColumnCondition(LogicalOperator.AND) 86 | cond.add_sub_condition(SingleColumnCondition("address", 'China', ComparatorType.EQUAL)) 87 | cond.add_sub_condition(SingleColumnCondition("age", 50, ComparatorType.GREATER_EQUAL)) 88 | 89 | columns_to_get = [] 90 | range_iter = client.xget_range( 91 | table_name, Direction.FORWARD, 92 | inclusive_start_primary_key, exclusive_end_primary_key, 93 | consumed_counter, columns_to_get, 100, 94 | column_filter=cond, max_version=1 95 | ) 96 | 97 | total_rows = 0 98 | for row in range_iter: 99 | print(row.primary_key, row.attribute_columns) 100 | total_rows += 1 101 | 102 | print('Total rows:', total_rows) 103 | 104 | 105 | if __name__ == '__main__': 106 | client = OTSClient(OTS_ENDPOINT, OTS_ACCESS_KEY_ID, OTS_ACCESS_KEY_SECRET, OTS_INSTANCE, region=OTS_REGION) 107 | try: 108 | delete_table(client) 109 | except: 110 | pass 111 | create_table(client) 112 | 113 | time.sleep(3) # wait for table ready 114 | put_row(client) 115 | get_range(client) 116 | xget_range(client) 117 | delete_table(client) 118 | -------------------------------------------------------------------------------- /tablestore/flatbuffer/flat_buffer_decoder.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from tablestore.metadata import * 4 | from tablestore.flatbuffer.dataprotocol.BytesValue import BytesValue 5 | from tablestore.flatbuffer.dataprotocol.DataType import * 6 | import sys 7 | import collections 8 | from tablestore.flatbuffer.dataprotocol.ColumnValues import * 9 | from tablestore.flatbuffer.dataprotocol.RLEStringValues import RLEStringValues 10 | 11 | 12 | class flat_buffer_decoder(object): 13 | @staticmethod 14 | def byte_to_str_decode(bt): 15 | if sys.version_info[0] == 2: 16 | return bt 17 | else: 18 | return bt.decode('UTF-8') 19 | 20 | @staticmethod 21 | def gen_meta_column(col_val:ColumnValues,col_tp:DataType): 22 | is_null_values = [] 23 | for i in range(col_val.IsNullvaluesLength()): 24 | is_null_values.append(col_val.IsNullvalues(i)) 25 | long_values = [] 26 | for i in range(col_val.LongValuesLength()): 27 | long_values.append(col_val.LongValues(i)) 28 | bool_values = [] 29 | for i in range(col_val.BoolValuesLength()): 30 | bool_values.append(col_val.BoolValues(i)) 31 | double_values = [] 32 | for i in range(col_val.DoubleValuesLength()): 33 | double_values.append(col_val.DoubleValues(i)) 34 | string_values = [] 35 | for i in range(col_val.StringValuesLength()): 36 | string_values.append(flat_buffer_decoder.byte_to_str_decode(col_val.StringValues(i))) 37 | binary_values = [] 38 | for i in range(col_val.BinaryValuesLength()): 39 | bytes_value = col_val.BinaryValues(i) 40 | binary_values.append(flat_buffer_decoder.gen_bytes_value(bytes_value)) 41 | rle_string_values = col_val.RleStringValues() 42 | 43 | values = get_column_val_by_tp(is_null_values, long_values, bool_values, double_values, string_values, binary_values, rle_string_values, col_tp) 44 | if col_tp == DataType.STRING_RLE: 45 | values = flat_buffer_decoder.gen_rle_string_values(values) 46 | if len(is_null_values) != len(values): 47 | raise ValueError("the length of unpacked values not equal to null map") 48 | 49 | return [None if is_null else val for is_null, val in zip(is_null_values, values)] 50 | 51 | @staticmethod 52 | def gen_rle_string_values(values:RLEStringValues): 53 | ret = [] 54 | for i in range(values.IndexMappingLength()): 55 | ret.append(values.Array(values.IndexMapping(i)).decode('UTF-8')) 56 | return ret 57 | 58 | @staticmethod 59 | def gen_bytes_value(bytes_value: Optional[BytesValue]): 60 | if bytes_value is None: 61 | return None 62 | value = [] 63 | for i in range(bytes_value.ValueLength()): 64 | value.append(bytes_value.Value(i)) 65 | return bytes(value) 66 | 67 | @staticmethod 68 | def format_flat_buffer_columns(columns): 69 | columns_meta = collections.defaultdict(list) 70 | for i in range(columns.ColumnsLength()): 71 | column = columns.Columns(i) 72 | col_name = flat_buffer_decoder.byte_to_str_decode(column.ColumnName()) 73 | col_tp = column.ColumnType() 74 | col_val = column.ColumnValue() 75 | columns_meta[col_name] = flat_buffer_decoder.gen_meta_column(col_val,col_tp) 76 | return columns_meta 77 | 78 | @staticmethod 79 | def columns_to_rows(columns_meta): 80 | res_list = [] 81 | column_len = sys.maxsize 82 | for key in columns_meta: 83 | column_len = min(len(columns_meta[key]),column_len) 84 | for i in range(column_len): 85 | tup = [] 86 | for key in columns_meta: 87 | tup.append((key,columns_meta[key][i])) 88 | row =Row(primary_key = [],attribute_columns=tup) 89 | res_list.append(row) 90 | return res_list 91 | 92 | 93 | def get_column_val_by_tp(is_null_values, long_values, bool_values, double_values, string_values, binary_values, rle_string_values, tp): 94 | if tp == DataType.NONE: 95 | return is_null_values 96 | if tp == DataType.LONG: 97 | return long_values 98 | if tp == DataType.BOOLEAN: 99 | return bool_values 100 | if tp == DataType.DOUBLE: 101 | return double_values 102 | if tp == DataType.STRING: 103 | return string_values 104 | if tp == DataType.BINARY: 105 | return binary_values 106 | if tp == DataType.STRING_RLE: 107 | return rle_string_values 108 | return None -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Aliyun Tablestore SDK for Python 2 | ================================== 3 | 4 | .. image:: https://img.shields.io/badge/license-apache2-brightgreen.svg 5 | :target: https://travis-ci.org/aliyun/aliyun-tablestore-python-sdk 6 | .. image:: https://badge.fury.io/gh/aliyun%2Faliyun-tablestore-python-sdk.svg 7 | :target: https://travis-ci.org/aliyun/aliyun-tablestore-python-sdk 8 | .. image:: https://travis-ci.org/aliyun/aliyun-tablestore-python-sdk.svg 9 | :target: https://travis-ci.org/aliyun/aliyun-tablestore-python-sdk 10 | 11 | 概述 12 | ---- 13 | 14 | - 此 Python SDK 基于 `阿里云表格存储服务 `_ API 构建。 15 | - 阿里云表格存储是构建在阿里云飞天分布式系统之上的 NoSQL 数据存储服务，提供海量结构化数据的存储和实时访问。 16 | 17 | 运行环境 18 | --------- 19 | 20 | - 安装 Python 即可运行，支持 python3.8、Python3.9、python3.10、python3.11、python3.12。 21 | 22 | 安装方法 23 | --------- 24 | 25 | PIP安装 26 | -------- 27 | 28 | .. code-block:: bash 29 | 30 | $ pip install tablestore 31 | 32 | Github安装 33 | ------------ 34 | 35 | 1. 下载源码 36 | 37 | .. code-block:: bash 38 | 39 | $ git clone https://github.com/aliyun/aliyun-tablestore-python-sdk.git 40 | 41 | 2. 构建 whl (构建好的whl文件在dist目录下) 42 | 43 | .. code-block:: bash 44 | 45 | $ poetry build 46 | 47 | 3. 安装 48 | 49 | .. code-block:: bash 50 | 51 | $ pip install dist/tablestore-{替换为实际版本}-py3-none-any.whl 52 | 53 | 54 | 源码安装 55 | -------- 56 | 57 | 1. 下载 SDK 发布包并解压 58 | 2. 构建 whl (构建好的whl文件在dist目录下) 59 | 60 | .. code-block:: bash 61 | 62 | $ poetry build 63 | 64 | 3. 安装 65 | 66 | .. code-block:: bash 67 | 68 | $ pip install dist/tablestore-{替换为实际版本}-py3-none-any.whl 69 | 70 | 示例代码 71 | --------- 72 | 73 | 表（Table）示例： 74 | 75 | - `表操作（表的创建、获取、更新和删除） `_ 76 | - `单行写（向表内写入一行数据） `_ 77 | - `单行读（从表内读出一样数据） `_ 78 | - `更新单行（更新某一行的部分字段） `_ 79 | - `删除某行（从表内删除某一行数据） `_ 80 | - `批量写（向多张表，一次性写入多行数据） `_ 81 | - `批量读（从多张表，一次性读出多行数据） `_ 82 | - `范围扫描（给定一个范围，扫描出该范围内的所有数据） `_ 83 | - `主键自增列（主键自动生成一个递增ID） `_ 84 | - `全局二级索引 `_ 85 | - `局部事务（提交事务） `_ 86 | - `局部事务（舍弃事务） `_ 87 | 88 | 多元索引（Search）示例： 89 | 90 | - `基础搜索 `_ 91 | - `并发圈选数据 `_ 92 | - `全文检索 `_ 93 | - `向量检索 `_ 94 | - `Max/Min/Sum/Avg/Count/DistinctCount 等 `_ 95 | - `GroupBy/Histogram 等 `_ 96 | 97 | 执行测试 98 | --------- 99 | 100 | **注意：测试 case 中会有清理某个实例下所有表的动作，所以请使用专门的测试实例来测试。** 101 | 102 | 1. 设置执行Case的配置 103 | 104 | .. code-block:: bash 105 | 106 | $ export OTS_TEST_ACCESS_KEY_ID= 107 | $ export OTS_TEST_ACCESS_KEY_SECRET= 108 | $ export OTS_TEST_ENDPOINT= 109 | $ export OTS_TEST_INSTANCE= 110 | 111 | 2. 运行case 112 | 113 | .. code-block:: bash 114 | 115 | $ poetry run pytest tests 116 | 117 | 贡献代码 118 | -------- 119 | - 我们非常欢迎大家为 Tablestore Python SDK 以及其他 Tablestore SDK 贡献代码。 120 | - 非常感谢 `@Wall-ee `_ 对 4.3.0 版本的贡献。 121 | 122 | 联系我们 123 | -------- 124 | - `阿里云 Tablestore 官方网站 `_ 125 | - `阿里云官网联系方式 `_ 126 | - `阿里云 Tablestore 官方文档 `_ 127 | 128 | 129 | -------------------------------------------------------------------------------- /tests/auth_unittest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from tests.lib.api_test_base import APITestBase 4 | from tablestore.credentials import StaticCredentialsProvider 5 | from tablestore.auth import * 6 | 7 | 8 | class AuthTest(APITestBase): 9 | def setUp(self): 10 | self.test_ak_id = "test_id" 11 | self.test_ak_secret = "test_key" 12 | self.test_sts_token = "test_token" 13 | self.test_encoding = "utf-8" 14 | self.test_region = "test-region" 15 | self.test_sign_date = "20250410" 16 | self.signature_string = "test_signature_string" 17 | self.test_query = "test_query" 18 | self.headers = {"x-ots-test": "test"} 19 | 20 | def tearDown(self): 21 | pass # no need to tearDown client 22 | 23 | def test_calculate_signature(self): 24 | actual_sha1_sign = b"C845e" + b"f7UjN" + b"GL0gE" + b"xNlQh" + b"p+3B" + b"/gY=" 25 | sha1_sign = call_signature_method_sha1(self.test_ak_secret, self.signature_string, self.test_encoding) 26 | self.assertEqual(28, len(sha1_sign)) 27 | self.assertEqual(actual_sha1_sign, sha1_sign) 28 | 29 | actual_sha256_sign = b"c+lCAaaQ" + b"VSCVlc0" + b"u0JBE" + b"PoIzy" + b"xplf4" + b"xEIBH" + b"8sdW" + b"UOjo=" 30 | sha256_sign = call_signature_method_sha256(self.test_ak_secret, self.signature_string, self.test_encoding) 31 | self.assertEqual(44, len(sha256_sign)) 32 | self.assertEqual(actual_sha256_sign, sha256_sign) 33 | 34 | def test_SignClass(self): 35 | cred = StaticCredentialsProvider(self.test_ak_id, self.test_ak_secret, self.test_sts_token) 36 | 37 | # test v2 sign 38 | v2_signer = SignV2(cred, self.test_encoding) 39 | self.assertEqual(cred, v2_signer.get_credentials_provider()) 40 | self.assertEqual(None, v2_signer.signing_key) 41 | self.assertEqual(self.test_encoding, v2_signer.encoding) 42 | v2_signer.gen_signing_key() 43 | self.assertEqual(self.test_ak_secret, v2_signer.signing_key) 44 | test_headers = self.headers.copy() 45 | v2_signer.make_request_signature_and_add_headers(self.test_query, test_headers) 46 | v2_request_signature = test_headers[consts.OTS_HEADER_SIGNATURE] 47 | actual_v2_request_signature = b"QDhzL" + b"v7VES" + b"BJtYQY4" + b"Li0Ih" + b"SUOdg=" 48 | self.assertEqual(28, len(v2_request_signature)) 49 | self.assertEqual(actual_v2_request_signature, v2_request_signature) 50 | v2_response_signature = v2_signer.make_response_signature(self.test_query, self.headers) 51 | actual_v2_response_signature = b"UjJK" + b"/SWed0" + b"n9o6J" + b"YxvAp" + b"HGaQ" + b"ABo=" 52 | self.assertEqual(actual_v2_response_signature, v2_response_signature) 53 | 54 | # test v4 sign 55 | with self.assertRaisesRegex(OTSClientError, "region is not str or is empty."): 56 | v4_signer = SignV4(cred, self.test_encoding) 57 | v4_signer = SignV4(cred, self.test_encoding, region=self.test_region, sign_date=self.test_sign_date) 58 | self.assertEqual(cred, v4_signer.get_credentials_provider()) 59 | self.assertEqual(None, v4_signer.signing_key) 60 | self.assertEqual(self.test_encoding, v4_signer.encoding) 61 | v4_signer.gen_signing_key() 62 | actual_v4_signing_key = b"nToxlXr" + b"xgCm0L" + b"5J0nr/q" + b"q/GmtgN9" + b"GVBhiR" + b"LzdL" + b"aVUP0=" 63 | self.assertEqual(actual_v4_signing_key, v4_signer.signing_key) 64 | test_headers = self.headers.copy() 65 | v4_signer.make_request_signature_and_add_headers(self.test_query, test_headers) 66 | self.assertEqual(self.test_region, test_headers[consts.OTS_HEADER_SIGN_REGION]) 67 | self.assertEqual(v4_signer.sign_date, test_headers[consts.OTS_HEADER_SIGN_DATE]) 68 | v4_request_signature = test_headers[consts.OTS_HEADER_SIGNATURE_V4] 69 | actual_v4_request_signature = b"yXnO" + b"pODWa" + b"U1EYAl" + b"LP3l25k" + b"sj010" + b"uGHS7" + b"uxIt5Q" + b"iwz4o=" 70 | self.assertEqual(44, len(v4_request_signature)) 71 | self.assertEqual(actual_v4_request_signature, v4_request_signature) 72 | v4_response_signature = v4_signer.make_response_signature(self.test_query, self.headers) 73 | actual_v4_response_signature = b"vIhaU" + b"Gwv/J" + b"Sg8ct" + b"LNyx" + b"bNeN" + b"v69A=" 74 | self.assertEqual(actual_v4_response_signature, v4_response_signature) 75 | 76 | # v2 sign and v4 sign use the same response sign method(sha1) 77 | origin_v2_signing_key = v2_signer.signing_key 78 | v2_signer.signing_key = v4_signer.signing_key # set signing key to same 79 | self.assertEqual( 80 | v2_signer.make_response_signature(self.test_query, self.headers), 81 | v4_signer.make_response_signature(self.test_query, self.headers) 82 | ) 83 | v2_signer.signing_key = origin_v2_signing_key 84 | -------------------------------------------------------------------------------- /examples/parallel_scan.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from example_config import * 4 | from tablestore import * 5 | import time 6 | import json 7 | import threadpool 8 | import threading 9 | 10 | table_name = 'ParallelScanExampleTable' 11 | index_name = 'search_index' 12 | client = None 13 | 14 | 15 | def fetch_rows_per_thread(query, session_id, current_thread_id, max_thread_num): 16 | token = None 17 | 18 | while True: 19 | try: 20 | scan_query = ScanQuery(query, limit=20, next_token=token, current_parallel_id=current_thread_id, 21 | max_parallel=max_thread_num, alive_time=30) 22 | 23 | response = client.parallel_scan( 24 | table_name, index_name, scan_query, session_id, 25 | columns_to_get=ColumnsToGet(return_type=ColumnReturnType.ALL_FROM_INDEX)) 26 | 27 | for row in response.rows: 28 | print("%s:%s" % (threading.currentThread().name, str(row))) 29 | 30 | if len(response.next_token) == 0: 31 | break 32 | else: 33 | token = response.next_token 34 | except OTSServiceError as e: 35 | print(e) 36 | except OTSClientError as e: 37 | print(e) 38 | 39 | 40 | def parallel_scan(table_name, index_name): 41 | response = client.compute_splits(table_name, index_name) 42 | 43 | query = TermQuery('d', 0.1) 44 | 45 | params = [] 46 | for i in range(response.splits_size): 47 | params.append((([query, response.session_id, i, response.splits_size], None))) 48 | 49 | pool = threadpool.ThreadPool(response.splits_size) 50 | requests = threadpool.makeRequests(fetch_rows_per_thread, params) 51 | [pool.putRequest(req) for req in requests] 52 | pool.wait() 53 | 54 | 55 | def prepare_data(rows_count): 56 | print('Begin prepare data: %d' % rows_count) 57 | for i in range(rows_count): 58 | pk = [('PK1', i), ('PK2', 'pk_' + str(i % 10))] 59 | lj = i / 100 60 | li = i % 100 61 | cols = [('k', 'key%03d' % i), ('t', 'this is ' + str(i)), 62 | ('g', '%f,%f' % (30.0 + 0.05 * lj, 114.0 + 0.05 * li)), ('ka', '["a", "b", "%d"]' % i), 63 | ('la', '[-1, %d]' % i), ('l', i), 64 | ('b', i % 2 == 0), ('d', 0.1), 65 | ('n', json.dumps([{'nk': 'key%03d' % i, 'nl': i, 'nt': 'this is in nested ' + str(i)}]))] 66 | 67 | client.put_row(table_name, Row(pk, cols)) 68 | 69 | print('End prepare data.') 70 | print('Wait for data sync to search index.') 71 | time.sleep(30) 72 | 73 | 74 | def prepare_table(): 75 | table_meta = TableMeta(table_name, [('PK1', 'INTEGER'), ('PK2', 'STRING')]) 76 | 77 | table_options = TableOptions() 78 | reserved_throughput = ReservedThroughput(CapacityUnit(0, 0)) 79 | client.create_table(table_meta, table_options, reserved_throughput) 80 | time.sleep(10) 81 | 82 | 83 | def prepare_index(index_name, with_nested=False): 84 | field_a = FieldSchema('k', FieldType.KEYWORD, index=True, enable_sort_and_agg=True, store=True) 85 | field_b = FieldSchema('t', FieldType.TEXT, index=True, store=True, analyzer=AnalyzerType.SINGLEWORD) 86 | field_c = FieldSchema('g', FieldType.GEOPOINT, index=True, store=True) 87 | field_d = FieldSchema('ka', FieldType.KEYWORD, index=True, is_array=True, store=True) 88 | field_e = FieldSchema('la', FieldType.LONG, index=True, is_array=True, store=True) 89 | field_f = FieldSchema('l', FieldType.LONG, index=True, store=True) 90 | field_g = FieldSchema('b', FieldType.BOOLEAN, index=True, store=True) 91 | field_h = FieldSchema('d', FieldType.DOUBLE, index=True, store=True) 92 | if with_nested: 93 | field_n = FieldSchema('n', FieldType.NESTED, sub_field_schemas=[ 94 | FieldSchema('nk', FieldType.KEYWORD, index=True, store=True), 95 | FieldSchema('nl', FieldType.LONG, index=True, store=True), 96 | FieldSchema('nt', FieldType.TEXT, index=True, store=True), 97 | ]) 98 | 99 | fields = [field_a, field_b, field_c, field_d, field_e, field_f, field_g, field_h] 100 | if with_nested: 101 | fields.append(field_n) 102 | index_setting = IndexSetting(routing_fields=['PK1']) 103 | index_sort = Sort(sorters=[PrimaryKeySort(SortOrder.ASC)]) if not with_nested else None 104 | index_meta = SearchIndexMeta(fields, index_setting=index_setting, index_sort=index_sort) # default with index sort 105 | client.create_search_index(table_name, index_name, index_meta) 106 | 107 | 108 | def delete_table(): 109 | try: 110 | client.delete_table(table_name) 111 | except: 112 | pass 113 | 114 | 115 | def delete_search_index(index_name): 116 | try: 117 | client.delete_search_index(table_name, index_name) 118 | except: 119 | pass 120 | 121 | 122 | if __name__ == '__main__': 123 | client = OTSClient(OTS_ENDPOINT, OTS_ACCESS_KEY_ID, OTS_ACCESS_KEY_SECRET, OTS_INSTANCE, region=OTS_REGION) 124 | delete_search_index(index_name) 125 | 126 | delete_table() 127 | 128 | prepare_table() 129 | prepare_index(index_name, with_nested=False) 130 | prepare_data(100) 131 | 132 | # perform parallel scan 133 | 134 | parallel_scan(table_name, index_name) 135 | 136 | delete_search_index(index_name) 137 | delete_table() 138 | -------------------------------------------------------------------------------- /tablestore/flatbuffer/timeseries/FlatBufferRowInGroup.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: timeseries 4 | 5 | import flatbuffers 6 | from flatbuffers.compat import import_numpy 7 | np = import_numpy() 8 | 9 | class FlatBufferRowInGroup(object): 10 | __slots__ = ['_tab'] 11 | 12 | @classmethod 13 | def GetRootAs(cls, buf, offset=0): 14 | n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) 15 | x = FlatBufferRowInGroup() 16 | x.Init(buf, n + offset) 17 | return x 18 | 19 | @classmethod 20 | def GetRootAsFlatBufferRowInGroup(cls, buf, offset=0): 21 | """This method is deprecated. Please switch to GetRootAs.""" 22 | return cls.GetRootAs(buf, offset) 23 | # FlatBufferRowInGroup 24 | def Init(self, buf, pos): 25 | self._tab = flatbuffers.table.Table(buf, pos) 26 | 27 | # FlatBufferRowInGroup 28 | def DataSource(self): 29 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 30 | if o != 0: 31 | return self._tab.String(o + self._tab.Pos) 32 | return None 33 | 34 | # FlatBufferRowInGroup 35 | def Tags(self): 36 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 37 | if o != 0: 38 | return self._tab.String(o + self._tab.Pos) 39 | return None 40 | 41 | # FlatBufferRowInGroup 42 | def Time(self): 43 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) 44 | if o != 0: 45 | return self._tab.Get(flatbuffers.number_types.Int64Flags, o + self._tab.Pos) 46 | return 0 47 | 48 | # FlatBufferRowInGroup 49 | def FieldValues(self): 50 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) 51 | if o != 0: 52 | x = self._tab.Indirect(o + self._tab.Pos) 53 | from tablestore.flatbuffer.timeseries.FieldValues import FieldValues 54 | obj = FieldValues() 55 | obj.Init(self._tab.Bytes, x) 56 | return obj 57 | return None 58 | 59 | # FlatBufferRowInGroup 60 | def MetaCacheUpdateTime(self): 61 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) 62 | if o != 0: 63 | return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos) 64 | return 0 65 | 66 | # FlatBufferRowInGroup 67 | def TagList(self, j): 68 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) 69 | if o != 0: 70 | x = self._tab.Vector(o) 71 | x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4 72 | x = self._tab.Indirect(x) 73 | from tablestore.flatbuffer.timeseries.Tag import Tag 74 | obj = Tag() 75 | obj.Init(self._tab.Bytes, x) 76 | return obj 77 | return None 78 | 79 | # FlatBufferRowInGroup 80 | def TagListLength(self): 81 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) 82 | if o != 0: 83 | return self._tab.VectorLen(o) 84 | return 0 85 | 86 | # FlatBufferRowInGroup 87 | def TagListIsNone(self): 88 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) 89 | return o == 0 90 | 91 | def FlatBufferRowInGroupStart(builder): builder.StartObject(6) 92 | def Start(builder): 93 | return FlatBufferRowInGroupStart(builder) 94 | def FlatBufferRowInGroupAddDataSource(builder, dataSource): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(dataSource), 0) 95 | def AddDataSource(builder, dataSource): 96 | return FlatBufferRowInGroupAddDataSource(builder, dataSource) 97 | def FlatBufferRowInGroupAddTags(builder, tags): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(tags), 0) 98 | def AddTags(builder, tags): 99 | return FlatBufferRowInGroupAddTags(builder, tags) 100 | def FlatBufferRowInGroupAddTime(builder, time): builder.PrependInt64Slot(2, time, 0) 101 | def AddTime(builder, time): 102 | return FlatBufferRowInGroupAddTime(builder, time) 103 | def FlatBufferRowInGroupAddFieldValues(builder, fieldValues): builder.PrependUOffsetTRelativeSlot(3, flatbuffers.number_types.UOffsetTFlags.py_type(fieldValues), 0) 104 | def AddFieldValues(builder, fieldValues): 105 | return FlatBufferRowInGroupAddFieldValues(builder, fieldValues) 106 | def FlatBufferRowInGroupAddMetaCacheUpdateTime(builder, metaCacheUpdateTime): builder.PrependUint32Slot(4, metaCacheUpdateTime, 0) 107 | def AddMetaCacheUpdateTime(builder, metaCacheUpdateTime): 108 | return FlatBufferRowInGroupAddMetaCacheUpdateTime(builder, metaCacheUpdateTime) 109 | def FlatBufferRowInGroupAddTagList(builder, tagList): builder.PrependUOffsetTRelativeSlot(5, flatbuffers.number_types.UOffsetTFlags.py_type(tagList), 0) 110 | def AddTagList(builder, tagList): 111 | return FlatBufferRowInGroupAddTagList(builder, tagList) 112 | def FlatBufferRowInGroupStartTagListVector(builder, numElems): return builder.StartVector(4, numElems, 4) 113 | def StartTagListVector(builder, numElems): 114 | return FlatBufferRowInGroupStartTagListVector(builder, numElems) 115 | def FlatBufferRowInGroupEnd(builder): return builder.EndObject() 116 | def End(builder): 117 | return FlatBufferRowInGroupEnd(builder) -------------------------------------------------------------------------------- /examples/vector_search.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from example_config import * 4 | from tablestore import * 5 | import time 6 | import json 7 | 8 | table_name = 'vector_search_example_table' 9 | index_name = 'search_index' 10 | client = None 11 | 12 | 13 | def _print_rows(request_id, rows, total_count): 14 | print('Request ID:%s' % request_id) 15 | 16 | for row in rows: 17 | print(row) 18 | 19 | print('Rows return: %d' % len(rows)) 20 | print('Total count: %d' % total_count) 21 | 22 | 23 | def knn_vector_query(table_name, index_name): 24 | print('********** Begin KNN Vector Query **********') 25 | 26 | query = KnnVectorQuery(field_name='vq', top_k=10, float32_query_vector=[5, -5, 10, -10], filter=MatchAllQuery()) 27 | sort = Sort(sorters=[ScoreSort(sort_order=SortOrder.DESC)]) 28 | search_response = client.search(table_name, index_name, 29 | SearchQuery(query, limit=10, get_total_count=False, sort=sort), 30 | ColumnsToGet(return_type=ColumnReturnType.ALL_FROM_INDEX)) 31 | _print_rows(search_response.request_id, search_response.rows, search_response.total_count) 32 | 33 | print('********** End KNN Vector Query **********') 34 | 35 | 36 | def prepare_data(rows_count): 37 | print('Begin prepare data: %d' % rows_count) 38 | for i in range(rows_count): 39 | pk = [('PK1', i)] 40 | 41 | cols = [('k', 'key%03d' % i), 42 | ('l', i), 43 | ('vq', '[%d, %d, %d, %d]' % (i + 5, i - 5, i + 10, i - 10))] 44 | 45 | client.put_row(table_name, Row(pk, cols)) 46 | 47 | print('End prepare data.') 48 | print('Wait for data sync to search index.') 49 | time.sleep(60) 50 | 51 | 52 | def prepare_table(): 53 | print('********** Begin CreateTable **********\n') 54 | 55 | table_meta = TableMeta(table_name, [('PK1', 'INTEGER')]) 56 | 57 | table_options = TableOptions(allow_update=False) 58 | reserved_throughput = ReservedThroughput(CapacityUnit(0, 0)) 59 | client.create_table(table_meta, table_options, reserved_throughput) 60 | 61 | print('********** End CreateTable **********\n') 62 | 63 | 64 | def prepare_index(index_name): 65 | print('********** Begin CreateSearchIndex **********\n') 66 | 67 | field_a = FieldSchema('k', FieldType.KEYWORD, index=True, enable_sort_and_agg=True, store=True) 68 | field_f = FieldSchema('l', FieldType.LONG, index=True, store=True) 69 | field_vq = FieldSchema("vq", FieldType.VECTOR, vector_options=VectorOptions( 70 | data_type=VectorDataType.VD_FLOAT_32, metric_type=VectorMetricType.VM_COSINE, dimension=4)) 71 | 72 | fields = [field_a, field_f, field_vq] 73 | 74 | index_setting = IndexSetting(routing_fields=['PK1']) 75 | index_sort = Sort(sorters=[PrimaryKeySort(SortOrder.ASC)]) 76 | index_meta = SearchIndexMeta(fields, index_setting=index_setting, index_sort=index_sort) # default with index sort 77 | client.create_search_index(table_name, index_name, index_meta) 78 | 79 | print('********** End CreateSearchIndex **********\n') 80 | 81 | 82 | def update_search_index(): 83 | print('********** Begin ListSearchIndex **********\n') 84 | 85 | index_meta = SearchIndexMeta(fields=None, time_to_live=24 * 3600 * 180) 86 | client.update_search_index(table_name, index_name, index_meta) 87 | print('End update search index') 88 | 89 | 90 | def list_search_index(): 91 | for table, index_name in client.list_search_index(table_name): 92 | print('%s, %s' % (table, index_name)) 93 | 94 | print('********** End ListSearchIndex **********\n') 95 | 96 | 97 | def describe_search_index(): 98 | print('********** Begin DescribeSearchIndex **********\n') 99 | 100 | index_meta, sync_stat = client.describe_search_index(table_name, index_name) 101 | print('sync stat: %s, %d' % (str(sync_stat.sync_phase), sync_stat.current_sync_timestamp)) 102 | print('index name: %s' % index_name) 103 | print('ttl: %ds' % (index_meta.time_to_live)) 104 | print('index fields:') 105 | print('name \t type \t\t indexed \t stored \t is_array \t allow_sort \t ') 106 | for field in index_meta.fields: 107 | print('%s\t%s\t%s\t\t%s\t\t%s\t%s' % (field.field_name, str(field.field_type), 108 | str(field.index), str(field.store), 109 | str(field.is_array), str(field.enable_sort_and_agg))) 110 | 111 | print('********** End DescribeSearchIndex **********\n') 112 | 113 | 114 | def delete_table(): 115 | print('********** Begin DeleteTable **********\n') 116 | 117 | try: 118 | client.delete_table(table_name) 119 | except: 120 | pass 121 | 122 | print('********** End DeleteTable **********\n') 123 | 124 | 125 | def delete_search_index(index_name): 126 | print('********** Begin DeleteSearchIndex **********\n') 127 | 128 | try: 129 | client.delete_search_index(table_name, index_name) 130 | except: 131 | pass 132 | print('********** End DeleteSearchIndex **********\n') 133 | 134 | 135 | if __name__ == '__main__': 136 | client = OTSClient(OTS_ENDPOINT, OTS_ACCESS_KEY_ID, OTS_ACCESS_KEY_SECRET, OTS_INSTANCE, region=OTS_REGION) 137 | delete_search_index(index_name) 138 | delete_table() 139 | 140 | prepare_table() 141 | prepare_index(index_name) 142 | prepare_data(100) 143 | list_search_index() 144 | describe_search_index() 145 | update_search_index() 146 | describe_search_index() 147 | 148 | # perform queries 149 | knn_vector_query(table_name, index_name) 150 | 151 | delete_search_index(index_name) 152 | delete_table() 153 | -------------------------------------------------------------------------------- /tests/timeseries_flat_buffer_unittest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | import unittest 4 | 5 | from timeseries import FlatBufferRows 6 | from tests.lib.api_test_base import APITestBase 7 | import time 8 | from tablestore.flatbuffer import timeseries_flat_buffer_encoder 9 | from tablestore import metadata 10 | from tablestore import * 11 | 12 | class TimeseriesFlatBufferTest(APITestBase): 13 | 14 | 15 | """TimeseriesFlatBufferTest""" 16 | 17 | def test_flat_buffer_encode(self): 18 | """测试flatbuffer编码正确性""" 19 | tags = {"tag1": "t1", "tag2": "t2"} 20 | field1 = {"long_field": 1, "string_field": "string", "bool_field": True, "doubel_field": 0.3, "doubel_field2":0.4} 21 | field2 = {"long_field2": 3, "string_field2": "string2", "doubel_field2": 0.4, "byte_field": bytearray(b'abc1')} 22 | key2 = metadata.TimeseriesKey("measure2", "datasource2", tags) 23 | key1 = metadata.TimeseriesKey("measure1", "datasource1", tags) 24 | time1 = time.time() 25 | row1 = metadata.TimeseriesRow(key1, field1, int(time1)) 26 | time2 = time.time() 27 | row2 = metadata.TimeseriesRow(key2, field2, int(time2)) 28 | rows = [row1, row2] 29 | 30 | resultbyte = timeseries_flat_buffer_encoder.get_column_val_by_tp("flatbuffertest", rows) 31 | 32 | flatBufferRows = FlatBufferRows.FlatBufferRows.GetRootAsFlatBufferRows(resultbyte) 33 | 34 | self.assert_equal(flatBufferRows.RowGroupsLength(), len(rows)) 35 | 36 | decode_row1 = flatBufferRows.RowGroups(0) 37 | self.assert_equal(decode_row1.FieldNamesLength(), len(field1)) 38 | self.assert_equal(decode_row1.FieldTypesLength(), len(field1)) 39 | self.assert_equal(decode_row1.MeasurementName(), "measure1") 40 | self.assert_equal(decode_row1.RowsLength(), 1) 41 | row_in_group1 = decode_row1.Rows(0) 42 | self.assert_equal(row_in_group1.DataSource(), "datasource1") 43 | self.assert_equal(row_in_group1.TagListLength(), len(tags)) 44 | self.assert_equal(row_in_group1.TagList(0).Name(), "tag1") 45 | self.assert_equal(row_in_group1.TagList(0).Value(), "t1") 46 | self.assert_equal(row_in_group1.Time(), int(time1)) 47 | self.assert_equal(row_in_group1.FieldValues().StringValues(0), "string") 48 | self.assert_equal(row_in_group1.FieldValues().LongValues(0), 1) 49 | self.assert_equal(row_in_group1.FieldValues().BoolValues(0), True) 50 | self.assert_equal(row_in_group1.FieldValues().DoubleValues(0), 0.3) 51 | self.assert_equal(row_in_group1.FieldValues().DoubleValues(1), 0.4) 52 | 53 | decode_row2 = flatBufferRows.RowGroups(1) 54 | self.assert_equal(decode_row2.FieldNamesLength(), len(field2)) 55 | self.assert_equal(decode_row2.FieldTypesLength(), len(field2)) 56 | self.assert_equal(decode_row2.MeasurementName(), "measure2") 57 | row_in_group2 = decode_row2.Rows(0) 58 | self.assert_equal(row_in_group2.DataSource(), "datasource2") 59 | self.assert_equal(row_in_group2.FieldValues().BinaryValues(0).ValueAsNumpy().tobytes(), b'abc1') 60 | 61 | 62 | tags = {"tag1": "t1", "tag2": "t2"} 63 | field1 = {"long_field": 1, "string_field": "string", "bool_field": True, "doubel_field": 0.3, 64 | "doubel_field2": 0.4, "string_field2": "string", "string_field3": "string", 65 | "byte_field1": bytearray(b'abc1'), "byte_field2": bytearray(b'abc2')} 66 | key1 = metadata.TimeseriesKey("measure1", "datasource1", tags) 67 | time1 = time.time() 68 | row1 = metadata.TimeseriesRow(key1, field1, int(time1)) 69 | rows = [row1] 70 | resultbyte = timeseries_flat_buffer_encoder.get_column_val_by_tp("flatbuffertest", rows) 71 | flatBufferRows = FlatBufferRows.FlatBufferRows.GetRootAsFlatBufferRows(resultbyte) 72 | self.assert_equal(flatBufferRows.RowGroupsLength(), len(rows)) 73 | 74 | decode_row1 = flatBufferRows.RowGroups(0) 75 | self.assert_equal(decode_row1.FieldNamesLength(), len(field1)) 76 | self.assert_equal(decode_row1.FieldTypesLength(), len(field1)) 77 | self.assert_equal(decode_row1.MeasurementName(), "measure1") 78 | self.assert_equal(decode_row1.RowsLength(), 1) 79 | row_in_group1 = decode_row1.Rows(0) 80 | self.assert_equal(row_in_group1.DataSource(), "datasource1") 81 | self.assert_equal(row_in_group1.TagListLength(), len(tags)) 82 | self.assert_equal(row_in_group1.TagList(0).Name(), "tag1") 83 | self.assert_equal(row_in_group1.TagList(0).Value(), "t1") 84 | self.assert_equal(row_in_group1.Time(), int(time1)) 85 | 86 | self.assert_equal(row_in_group1.FieldValues().DoubleValuesLength(), 2) 87 | self.assert_equal(row_in_group1.FieldValues().StringValuesLength(), 3) 88 | self.assert_equal(row_in_group1.FieldValues().BinaryValuesLength(), 2) 89 | self.assert_equal(row_in_group1.FieldValues().LongValuesLength(), 1) 90 | self.assert_equal(row_in_group1.FieldValues().BoolValuesLength(), 1) 91 | 92 | self.assert_equal(row_in_group1.FieldValues().StringValues(0), field1["string_field"]) 93 | self.assert_equal(row_in_group1.FieldValues().StringValues(1), field1["string_field2"]) 94 | self.assert_equal(row_in_group1.FieldValues().StringValues(2), field1["string_field3"]) 95 | self.assert_equal(row_in_group1.FieldValues().LongValues(0), 1) 96 | self.assert_equal(row_in_group1.FieldValues().BoolValues(0), True) 97 | self.assert_equal(row_in_group1.FieldValues().DoubleValues(0), 0.3) 98 | self.assert_equal(row_in_group1.FieldValues().BinaryValues(0).ValueAsNumpy().tobytes(), bytes(field1["byte_field1"])) 99 | 100 | if __name__ == '__main__': 101 | unittest.main() 102 | -------------------------------------------------------------------------------- /tests/timeseries_flat_buffer_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | import unittest 4 | 5 | from tablestore.flatbuffer.timeseries import FlatBufferRows 6 | from tests.lib.api_test_base import APITestBase 7 | import time 8 | from tablestore.flatbuffer import timeseries_flat_buffer_encoder 9 | from tablestore import metadata 10 | from tablestore import * 11 | 12 | class TimeseriesFlatBufferTest(APITestBase): 13 | 14 | 15 | """TimeseriesFlatBufferTest""" 16 | 17 | def test_flat_buffer_encode(self): 18 | """Test the correctness of flatbuffer encoding""" 19 | tags = {"tag1": "t1", "tag2": "t2"} 20 | field1 = {"long_field": 1, "string_field": "string", "bool_field": True, "doubel_field": 0.3, "doubel_field2":0.4} 21 | field2 = {"long_field2": 3, "string_field2": "string2", "doubel_field2": 0.4, "byte_field": bytearray(b'abc1')} 22 | key2 = metadata.TimeseriesKey("measure2", "datasource2", tags) 23 | key1 = metadata.TimeseriesKey("measure1", "datasource1", tags) 24 | time1 = time.time() 25 | row1 = metadata.TimeseriesRow(key1, field1, int(time1)) 26 | time2 = time.time() 27 | row2 = metadata.TimeseriesRow(key2, field2, int(time2)) 28 | rows = [row1, row2] 29 | 30 | resultbyte = timeseries_flat_buffer_encoder.get_column_val_by_tp("flatbuffertest", rows) 31 | 32 | flatBufferRows = FlatBufferRows.FlatBufferRows.GetRootAsFlatBufferRows(resultbyte) 33 | 34 | self.assert_equal(flatBufferRows.RowGroupsLength(), len(rows)) 35 | 36 | decode_row1 = flatBufferRows.RowGroups(0) 37 | self.assert_equal(decode_row1.FieldNamesLength(), len(field1)) 38 | self.assert_equal(decode_row1.FieldTypesLength(), len(field1)) 39 | self.assert_equal(decode_row1.MeasurementName(), "measure1") 40 | self.assert_equal(decode_row1.RowsLength(), 1) 41 | row_in_group1 = decode_row1.Rows(0) 42 | self.assert_equal(row_in_group1.DataSource(), "datasource1") 43 | self.assert_equal(row_in_group1.TagListLength(), len(tags)) 44 | self.assert_equal(row_in_group1.TagList(0).Name(), "tag1") 45 | self.assert_equal(row_in_group1.TagList(0).Value(), "t1") 46 | self.assert_equal(row_in_group1.Time(), int(time1)) 47 | self.assert_equal(row_in_group1.FieldValues().StringValues(0), "string") 48 | self.assert_equal(row_in_group1.FieldValues().LongValues(0), 1) 49 | self.assert_equal(row_in_group1.FieldValues().BoolValues(0), True) 50 | self.assert_equal(row_in_group1.FieldValues().DoubleValues(0), 0.3) 51 | self.assert_equal(row_in_group1.FieldValues().DoubleValues(1), 0.4) 52 | 53 | decode_row2 = flatBufferRows.RowGroups(1) 54 | self.assert_equal(decode_row2.FieldNamesLength(), len(field2)) 55 | self.assert_equal(decode_row2.FieldTypesLength(), len(field2)) 56 | self.assert_equal(decode_row2.MeasurementName(), "measure2") 57 | row_in_group2 = decode_row2.Rows(0) 58 | self.assert_equal(row_in_group2.DataSource(), "datasource2") 59 | self.assert_equal(row_in_group2.FieldValues().BinaryValues(0).ValueAsNumpy().tobytes(), b'abc1') 60 | 61 | 62 | tags = {"tag1": "t1", "tag2": "t2"} 63 | field1 = {"long_field": 1, "string_field": "string", "bool_field": True, "doubel_field": 0.3, 64 | "doubel_field2": 0.4, "string_field2": "string", "string_field3": "string", 65 | "byte_field1": bytearray(b'abc1'), "byte_field2": bytearray(b'abc2')} 66 | key1 = metadata.TimeseriesKey("measure1", "datasource1", tags) 67 | time1 = time.time() 68 | row1 = metadata.TimeseriesRow(key1, field1, int(time1)) 69 | rows = [row1] 70 | resultbyte = timeseries_flat_buffer_encoder.get_column_val_by_tp("flatbuffertest", rows) 71 | flatBufferRows = FlatBufferRows.FlatBufferRows.GetRootAsFlatBufferRows(resultbyte) 72 | self.assert_equal(flatBufferRows.RowGroupsLength(), len(rows)) 73 | 74 | decode_row1 = flatBufferRows.RowGroups(0) 75 | self.assert_equal(decode_row1.FieldNamesLength(), len(field1)) 76 | self.assert_equal(decode_row1.FieldTypesLength(), len(field1)) 77 | self.assert_equal(decode_row1.MeasurementName(), "measure1") 78 | self.assert_equal(decode_row1.RowsLength(), 1) 79 | row_in_group1 = decode_row1.Rows(0) 80 | self.assert_equal(row_in_group1.DataSource(), "datasource1") 81 | self.assert_equal(row_in_group1.TagListLength(), len(tags)) 82 | self.assert_equal(row_in_group1.TagList(0).Name(), "tag1") 83 | self.assert_equal(row_in_group1.TagList(0).Value(), "t1") 84 | self.assert_equal(row_in_group1.Time(), int(time1)) 85 | 86 | self.assert_equal(row_in_group1.FieldValues().DoubleValuesLength(), 2) 87 | self.assert_equal(row_in_group1.FieldValues().StringValuesLength(), 3) 88 | self.assert_equal(row_in_group1.FieldValues().BinaryValuesLength(), 2) 89 | self.assert_equal(row_in_group1.FieldValues().LongValuesLength(), 1) 90 | self.assert_equal(row_in_group1.FieldValues().BoolValuesLength(), 1) 91 | 92 | self.assert_equal(row_in_group1.FieldValues().StringValues(0), field1["string_field"]) 93 | self.assert_equal(row_in_group1.FieldValues().StringValues(1), field1["string_field2"]) 94 | self.assert_equal(row_in_group1.FieldValues().StringValues(2), field1["string_field3"]) 95 | self.assert_equal(row_in_group1.FieldValues().LongValues(0), 1) 96 | self.assert_equal(row_in_group1.FieldValues().BoolValues(0), True) 97 | self.assert_equal(row_in_group1.FieldValues().DoubleValues(0), 0.3) 98 | self.assert_equal(row_in_group1.FieldValues().BinaryValues(0).ValueAsNumpy().tobytes(), bytes(field1["byte_field1"])) 99 | 100 | if __name__ == '__main__': 101 | unittest.main() 102 | -------------------------------------------------------------------------------- /tablestore/flatbuffer/timeseries/FlatBufferRowGroup.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: timeseries 4 | 5 | import flatbuffers 6 | from flatbuffers.compat import import_numpy 7 | np = import_numpy() 8 | 9 | class FlatBufferRowGroup(object): 10 | __slots__ = ['_tab'] 11 | 12 | @classmethod 13 | def GetRootAs(cls, buf, offset=0): 14 | n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) 15 | x = FlatBufferRowGroup() 16 | x.Init(buf, n + offset) 17 | return x 18 | 19 | @classmethod 20 | def GetRootAsFlatBufferRowGroup(cls, buf, offset=0): 21 | """This method is deprecated. Please switch to GetRootAs.""" 22 | return cls.GetRootAs(buf, offset) 23 | # FlatBufferRowGroup 24 | def Init(self, buf, pos): 25 | self._tab = flatbuffers.table.Table(buf, pos) 26 | 27 | # FlatBufferRowGroup 28 | def MeasurementName(self): 29 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 30 | if o != 0: 31 | return self._tab.String(o + self._tab.Pos) 32 | return None 33 | 34 | # FlatBufferRowGroup 35 | def FieldNames(self, j): 36 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 37 | if o != 0: 38 | a = self._tab.Vector(o) 39 | return self._tab.String(a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) 40 | return "" 41 | 42 | # FlatBufferRowGroup 43 | def FieldNamesLength(self): 44 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 45 | if o != 0: 46 | return self._tab.VectorLen(o) 47 | return 0 48 | 49 | # FlatBufferRowGroup 50 | def FieldNamesIsNone(self): 51 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 52 | return o == 0 53 | 54 | # FlatBufferRowGroup 55 | def FieldTypes(self, j): 56 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) 57 | if o != 0: 58 | a = self._tab.Vector(o) 59 | return self._tab.Get(flatbuffers.number_types.Int8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1)) 60 | return 0 61 | 62 | # FlatBufferRowGroup 63 | def FieldTypesAsNumpy(self): 64 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) 65 | if o != 0: 66 | return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int8Flags, o) 67 | return 0 68 | 69 | # FlatBufferRowGroup 70 | def FieldTypesLength(self): 71 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) 72 | if o != 0: 73 | return self._tab.VectorLen(o) 74 | return 0 75 | 76 | # FlatBufferRowGroup 77 | def FieldTypesIsNone(self): 78 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) 79 | return o == 0 80 | 81 | # FlatBufferRowGroup 82 | def Rows(self, j): 83 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) 84 | if o != 0: 85 | x = self._tab.Vector(o) 86 | x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4 87 | x = self._tab.Indirect(x) 88 | from tablestore.flatbuffer.timeseries.FlatBufferRowInGroup import FlatBufferRowInGroup 89 | obj = FlatBufferRowInGroup() 90 | obj.Init(self._tab.Bytes, x) 91 | return obj 92 | return None 93 | 94 | # FlatBufferRowGroup 95 | def RowsLength(self): 96 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) 97 | if o != 0: 98 | return self._tab.VectorLen(o) 99 | return 0 100 | 101 | # FlatBufferRowGroup 102 | def RowsIsNone(self): 103 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) 104 | return o == 0 105 | 106 | def FlatBufferRowGroupStart(builder): builder.StartObject(4) 107 | def Start(builder): 108 | return FlatBufferRowGroupStart(builder) 109 | def FlatBufferRowGroupAddMeasurementName(builder, measurementName): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(measurementName), 0) 110 | def AddMeasurementName(builder, measurementName): 111 | return FlatBufferRowGroupAddMeasurementName(builder, measurementName) 112 | def FlatBufferRowGroupAddFieldNames(builder, fieldNames): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(fieldNames), 0) 113 | def AddFieldNames(builder, fieldNames): 114 | return FlatBufferRowGroupAddFieldNames(builder, fieldNames) 115 | def FlatBufferRowGroupStartFieldNamesVector(builder, numElems): return builder.StartVector(4, numElems, 4) 116 | def StartFieldNamesVector(builder, numElems): 117 | return FlatBufferRowGroupStartFieldNamesVector(builder, numElems) 118 | def FlatBufferRowGroupAddFieldTypes(builder, fieldTypes): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(fieldTypes), 0) 119 | def AddFieldTypes(builder, fieldTypes): 120 | return FlatBufferRowGroupAddFieldTypes(builder, fieldTypes) 121 | def FlatBufferRowGroupStartFieldTypesVector(builder, numElems): return builder.StartVector(1, numElems, 1) 122 | def StartFieldTypesVector(builder, numElems): 123 | return FlatBufferRowGroupStartFieldTypesVector(builder, numElems) 124 | def FlatBufferRowGroupAddRows(builder, rows): builder.PrependUOffsetTRelativeSlot(3, flatbuffers.number_types.UOffsetTFlags.py_type(rows), 0) 125 | def AddRows(builder, rows): 126 | return FlatBufferRowGroupAddRows(builder, rows) 127 | def FlatBufferRowGroupStartRowsVector(builder, numElems): return builder.StartVector(4, numElems, 4) 128 | def StartRowsVector(builder, numElems): 129 | return FlatBufferRowGroupStartRowsVector(builder, numElems) 130 | def FlatBufferRowGroupEnd(builder): return builder.EndObject() 131 | def End(builder): 132 | return FlatBufferRowGroupEnd(builder) -------------------------------------------------------------------------------- /tests/async_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | import asyncio 3 | 4 | from aiohttp import ClientTimeout 5 | 6 | from tablestore import * 7 | import unittest 8 | from unittest import IsolatedAsyncioTestCase 9 | from .lib import test_config 10 | 11 | 12 | class AsyncTest(IsolatedAsyncioTestCase): 13 | def setUp(self): 14 | self.table_name = 'AsyncTest' 15 | self.rows_count = 100 16 | 17 | self.async_client = AsyncOTSClient(test_config.OTS_ENDPOINT, test_config.OTS_ACCESS_KEY_ID, test_config.OTS_ACCESS_KEY_SECRET, test_config.OTS_INSTANCE, region=test_config.OTS_REGION) 18 | 19 | asyncio.run(self._prepare_empty_table()) 20 | 21 | def _get_batch_write_request(self, id_base = 0): 22 | put_row_items = [] 23 | 24 | for i in range(id_base, id_base + self.rows_count): 25 | primary_key = [('gid', i), ('uid', i + 1)] 26 | attribute_columns = [('name', 'somebody' + str(i)), ('age', i)] 27 | row = Row(primary_key, attribute_columns) 28 | condition = Condition(RowExistenceExpectation.IGNORE) 29 | item = PutRowItem(row, condition) 30 | put_row_items.append(item) 31 | 32 | batch_write_request = BatchWriteRowRequest() 33 | batch_write_request.add(TableInBatchWriteRowItem(self.table_name, put_row_items)) 34 | 35 | return batch_write_request 36 | 37 | def _get_batch_get_request(self, id_base = 0): 38 | columns_to_get = ['name', 'age'] 39 | rows_to_get = [] 40 | for i in range(id_base, id_base + self.rows_count): 41 | primary_key = [('gid', i), ('uid', i + 1)] 42 | rows_to_get.append(primary_key) 43 | 44 | batch_get_request = BatchGetRowRequest() 45 | batch_get_request.add(TableInBatchGetRowItem(self.table_name, rows_to_get, columns_to_get, max_version=1)) 46 | 47 | return batch_get_request 48 | 49 | async def _prepare_empty_table(self): 50 | if self.table_name in (await self.async_client.list_table()): 51 | await self.async_client.delete_table(self.table_name) 52 | 53 | schema_of_primary_key = [('gid', 'INTEGER'), ('uid', 'INTEGER')] 54 | table_meta = TableMeta(self.table_name, schema_of_primary_key) 55 | table_options = TableOptions() 56 | reserved_throughput = ReservedThroughput(CapacityUnit(0, 0)) 57 | await self.async_client.create_table(table_meta, table_options, reserved_throughput) 58 | 59 | await self.async_client.close() 60 | 61 | await asyncio.sleep(0.02) 62 | 63 | def _get_result_rows(self, result): 64 | return len([item for item in result.items[self.table_name] if item.row is not None]) 65 | 66 | async def test_batch_write(self): 67 | async with self.async_client: 68 | await self.async_client.batch_write_row(self._get_batch_write_request()) 69 | result = await self.async_client.batch_get_row(self._get_batch_get_request(id_base=0)) 70 | self.assertEqual(self._get_result_rows(result), self.rows_count) 71 | 72 | await self._prepare_empty_table() 73 | 74 | async def test_parallel(self): 75 | paragraph = 3 76 | 77 | async with self.async_client: 78 | tasks = [ 79 | asyncio.create_task(self.async_client.batch_write_row(self._get_batch_write_request(id_base=i * self.rows_count))) 80 | for i in range(paragraph) 81 | ] 82 | for task in tasks: 83 | await task 84 | 85 | tasks = [ 86 | self.async_client.batch_get_row(self._get_batch_get_request(id_base=i * self.rows_count)) 87 | for i in range(paragraph) 88 | ] 89 | results = await asyncio.gather(*tasks) 90 | 91 | for result in results: 92 | self.assertEqual(self._get_result_rows(result), self.rows_count) 93 | 94 | await self._prepare_empty_table() 95 | 96 | 97 | async def test_double_close(self): 98 | async with self.async_client: 99 | await self.async_client.batch_write_row(self._get_batch_write_request(id_base=0)) 100 | 101 | async with self.async_client: 102 | result = await self.async_client.batch_get_row(self._get_batch_get_request(id_base=0)) 103 | self.assertEqual(self._get_result_rows(result), self.rows_count) 104 | 105 | await self.async_client.batch_write_row(self._get_batch_write_request(id_base=0)) 106 | await self.async_client.close() 107 | 108 | result = await self.async_client.batch_get_row(self._get_batch_get_request(id_base=0)) 109 | self.assertEqual(self._get_result_rows(result), self.rows_count) 110 | await self.async_client.close() 111 | 112 | await self._prepare_empty_table() 113 | 114 | async def test_timeout_parameter(self): 115 | async with AsyncOTSClient( 116 | test_config.OTS_ENDPOINT, 117 | test_config.OTS_ACCESS_KEY_ID, 118 | test_config.OTS_ACCESS_KEY_SECRET, 119 | test_config.OTS_INSTANCE, 120 | region=test_config.OTS_REGION, 121 | socket_timeout=10 122 | ) as client: 123 | await client.batch_get_row(self._get_batch_get_request(id_base=0)) 124 | self.assertEqual(client._connection.pool._timeout, ClientTimeout(sock_connect=10, sock_read=10)) 125 | 126 | async with AsyncOTSClient( 127 | test_config.OTS_ENDPOINT, 128 | test_config.OTS_ACCESS_KEY_ID, 129 | test_config.OTS_ACCESS_KEY_SECRET, 130 | test_config.OTS_INSTANCE, 131 | region=test_config.OTS_REGION, 132 | socket_timeout=(10, 30) 133 | ) as client: 134 | await client.batch_get_row(self._get_batch_get_request(id_base=0)) 135 | self.assertEqual(client._connection.pool._timeout, ClientTimeout(sock_connect=10, sock_read=30)) 136 | 137 | if __name__ == '__main__': 138 | unittest.main() -------------------------------------------------------------------------------- /tablestore/retry.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | import random 3 | import math 4 | 5 | class RetryPolicy(object): 6 | """ 7 | The ```RetryPolicy``` is the interface for retry strategies, containing 2 unimplemented methods and their parameter lists. 8 | To implement a retry strategy, inherit from this class and implement its 2 methods. 9 | """ 10 | 11 | def should_retry(self, retry_times, exception, api_name): 12 | raise NotImplementedError() 13 | 14 | def get_retry_delay(self, retry_times, exception, api_name): 15 | raise NotImplementedError() 16 | 17 | 18 | class RetryUtil(object): 19 | 20 | @classmethod 21 | def should_retry_no_matter_which_api(cls, exception): 22 | error_code = exception.code 23 | error_message = exception.message 24 | 25 | if (error_code == "OTSRowOperationConflict" or 26 | error_code == "OTSNotEnoughCapacityUnit" or 27 | error_code == "OTSTableNotReady" or 28 | error_code == "OTSPartitionUnavailable" or 29 | error_code == "OTSServerBusy" or 30 | error_code == "OTSOperationThrottled"): 31 | return True 32 | 33 | if error_code == "OTSQuotaExhausted" and error_message == "Too frequent table operations.": 34 | return True 35 | 36 | return False 37 | 38 | @classmethod 39 | def is_repeatable_api(cls, api_name): 40 | return api_name in ['ListTable', 'DescribeTable', 'GetRow', 'BatchGetRow', 'GetRange', 'GetTimeseriesData', 'ListTimeseriesTable', 'DescribeTimeseriesTable', 'QueryTimeseriesMeta'] 41 | 42 | @classmethod 43 | def should_retry_when_api_repeatable(cls, retry_times, exception, api_name): 44 | error_code = exception.code 45 | error_message = exception.message 46 | http_status = exception.http_status 47 | 48 | if (error_code == "OTSTimeout" or 49 | error_code == "OTSInternalServerError" or 50 | error_code == "OTSServerUnavailable"): 51 | return True 52 | 53 | if (http_status == 500 or http_status == 502 or http_status == 503): 54 | return True 55 | 56 | # TODO handle network error & timeout 57 | return False 58 | 59 | @classmethod 60 | def is_server_throttling_exception(cls, exception): 61 | error_code = exception.code 62 | error_message = exception.message 63 | 64 | if (error_code == "OTSServerBusy" or 65 | error_code == "OTSNotEnoughCapacityUnit" or 66 | error_code == "OTSOperationThrottled"): 67 | return True 68 | 69 | if error_code == "OTSQuotaExhausted" and error_message == "Too frequent table operations.": 70 | return True 71 | 72 | return False 73 | 74 | 75 | class DefaultRetryPolicy(RetryPolicy): 76 | """ 77 | Default retry strategy 78 | The maximum number of retries is 20, and the maximum retry interval is 3 seconds. Retries are performed for throttling-related errors and internal server errors associated with read operations. 79 | """ 80 | 81 | # Maximum retry count 82 | max_retry_times = 20 83 | 84 | # Maximum retry interval, in seconds 85 | max_retry_delay = 3 86 | 87 | # Incremental multiplier for each retry interval 88 | scale_factor = 2 89 | 90 | # Two error initial retry intervals, in seconds 91 | server_throttling_exception_delay_factor = 0.5 92 | stability_exception_delay_factor = 0.2 93 | 94 | def _max_retry_time_reached(self, retry_times, exception, api_name): 95 | return retry_times >= self.max_retry_times 96 | 97 | def is_repeatable_api(self, api_name): 98 | return RetryUtil.is_repeatable_api(api_name) 99 | 100 | def _can_retry(self, retry_times, exception, api_name): 101 | 102 | if RetryUtil.should_retry_no_matter_which_api(exception): 103 | return True 104 | 105 | if self.is_repeatable_api(api_name) and RetryUtil.should_retry_when_api_repeatable(retry_times, exception, api_name): 106 | return True 107 | 108 | return False 109 | 110 | def get_retry_delay(self, retry_times, exception, api_name): 111 | 112 | if RetryUtil.is_server_throttling_exception(exception): 113 | delay_factor = self.server_throttling_exception_delay_factor 114 | else: 115 | delay_factor = self.stability_exception_delay_factor 116 | 117 | delay_limit = delay_factor * math.pow(self.scale_factor, retry_times) 118 | 119 | if delay_limit >= self.max_retry_delay: 120 | delay_limit = self.max_retry_delay 121 | 122 | real_delay = delay_limit * 0.5 + delay_limit * 0.5 * random.random() 123 | return real_delay 124 | 125 | def should_retry(self, retry_times, exception, api_name): 126 | 127 | if self._max_retry_time_reached(retry_times, exception, api_name): 128 | return False 129 | 130 | if self._can_retry(retry_times, exception, api_name): 131 | return True 132 | 133 | return False 134 | 135 | 136 | class NoRetryPolicy(RetryPolicy): 137 | """ 138 | A retry strategy that does not perform any retries. 139 | """ 140 | 141 | def get_retry_delay(self, retry_times, exception, api_name): 142 | return 0 143 | 144 | def should_retry(self, retry_times, exception, api_name): 145 | return False 146 | 147 | 148 | class NoDelayRetryPolicy(DefaultRetryPolicy): 149 | """ 150 | A retry strategy with no delay 151 | """ 152 | 153 | def get_retry_delay(self, retry_times, exception, api_name): 154 | return 0 155 | 156 | class WriteRetryPolicy(DefaultRetryPolicy): 157 | """ 158 | Compared to the default retry strategy, this strategy will also retry write operations. 159 | """ 160 | 161 | def is_repeatable_api(self, api_name): 162 | return api_name in ['ListTable', 'DescribeTable', 'GetRow', 'BatchGetRow', 'GetRange', 163 | 'PutRow', 'UpdatRow', 'DeleteRow', 'BatchWriteRow'] 164 | -------------------------------------------------------------------------------- /tablestore/auth.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | import hashlib 4 | import hmac 5 | import base64 6 | import six 7 | from abc import ABC, abstractmethod 8 | 9 | try: 10 | from urlparse import urlparse, parse_qsl 11 | from urllib import urlencode 12 | except ImportError: 13 | from urllib.parse import urlparse, parse_qsl, urlencode 14 | 15 | import tablestore.consts as consts 16 | import tablestore.utils as utils 17 | from tablestore.credentials import CredentialsProvider 18 | from tablestore.error import * 19 | 20 | 21 | def calculate_hmac(signing_key, signature_string, sign_method, encoding): 22 | if isinstance(signing_key, six.text_type): 23 | signing_key = signing_key.encode(encoding) 24 | if isinstance(signature_string, six.text_type): 25 | signature_string = signature_string.encode(encoding) 26 | return hmac.new(signing_key, signature_string, sign_method).digest() 27 | 28 | 29 | def call_signature_method_sha1(signing_key, signature_string, encoding): 30 | # The signature method is supposed to be HmacSHA1 31 | return base64.b64encode(calculate_hmac(signing_key, signature_string, hashlib.sha1, encoding)).decode(encoding) 32 | 33 | 34 | def call_signature_method_sha256(signing_key, signature_string, encoding): 35 | # The signature method is supposed to be HmacSHA256 36 | return base64.b64encode(calculate_hmac(signing_key, signature_string, hashlib.sha256, encoding)).decode(encoding) 37 | 38 | 39 | class SignBase(ABC): 40 | def __init__(self, credentials_provider: CredentialsProvider, encoding, **kwargs): 41 | self.credentials_provider = credentials_provider 42 | self.encoding = encoding 43 | self.signing_key = None 44 | 45 | def get_credentials_provider(self): 46 | return self.credentials_provider 47 | 48 | @staticmethod 49 | def _make_headers_string(headers): 50 | headers_item = ["%s:%s" % (k.lower(), v.strip()) for k, v in headers.items() 51 | if k.startswith(consts.OTS_HEADER_PREFIX)] 52 | return "\n".join(sorted(headers_item)) 53 | 54 | def _get_request_signature_string(self, query, headers): 55 | uri, param_string, query_string = urlparse(query)[2:5] 56 | 57 | # TODO a special query should be input to test query sorting, 58 | # because none of the current APIs uses query map, but the sorting 59 | # is required in the protocol document. 60 | query_pairs = parse_qsl(query_string) 61 | sorted_query = urlencode(sorted(query_pairs)) 62 | signature_string = uri + '\n' + 'POST' + '\n' + sorted_query + '\n' 63 | 64 | headers_string = self._make_headers_string(headers) 65 | signature_string += headers_string + '\n' 66 | return signature_string 67 | 68 | def make_response_signature(self, query, headers): 69 | uri = urlparse(query)[2] 70 | headers_string = self._make_headers_string(headers) 71 | signature_string = headers_string + '\n' + uri 72 | # Response signature use same signing key as request signature 73 | # But the signature method is supposed to be HmacSHA1 74 | signature = call_signature_method_sha1(self.signing_key, signature_string, self.encoding) 75 | return signature 76 | 77 | @abstractmethod 78 | def gen_signing_key(self): 79 | pass 80 | 81 | @abstractmethod 82 | def make_request_signature_and_add_headers(self, query, headers): 83 | pass 84 | 85 | 86 | class SignV2(SignBase): 87 | def __init__(self, credentials_provider: CredentialsProvider, encoding, **kwargs): 88 | SignBase.__init__(self, credentials_provider, encoding, **kwargs) 89 | 90 | def gen_signing_key(self): 91 | self.signing_key = self.credentials_provider.get_credentials().get_access_key_secret() 92 | 93 | def make_request_signature_and_add_headers(self, query, headers): 94 | signature_string = self._get_request_signature_string(query, headers) 95 | headers[consts.OTS_HEADER_SIGNATURE] = call_signature_method_sha1(self.signing_key, signature_string, 96 | self.encoding) 97 | 98 | 99 | class SignV4(SignBase): 100 | def __init__(self, credentials_provider: CredentialsProvider, encoding, **kwargs): 101 | SignBase.__init__(self, credentials_provider, encoding, **kwargs) 102 | self.user_key = None 103 | self.region = kwargs.get('region') 104 | if not isinstance(self.region, str) or self.region == '': 105 | raise OTSClientError('region is not str or is empty.') 106 | self.sign_date = kwargs.get('sign_date') 107 | self.auto_update_v4_sign = (kwargs.get('auto_update_v4_sign') is True) 108 | if self.sign_date is None: 109 | self.sign_date = utils.get_now_utc_datetime().strftime(consts.V4_SIGNATURE_SIGN_DATE_FORMAT) 110 | self.auto_update_v4_sign = True 111 | 112 | def gen_signing_key(self): 113 | # if the signing_key is None, we need to update signing_key. 114 | need_update = self.signing_key is None 115 | # if the user_key changes, we need to update signing_key. 116 | cur_user_key = self.credentials_provider.get_credentials().get_access_key_secret() 117 | if cur_user_key != self.user_key: 118 | self.user_key = cur_user_key 119 | need_update = True 120 | # for v4, only update the sign date and signing_key 121 | if self.auto_update_v4_sign: 122 | cur_date = utils.get_now_utc_datetime().strftime(consts.V4_SIGNATURE_SIGN_DATE_FORMAT) 123 | # if sign_date changes, we need to update signing_key. 124 | if cur_date != self.sign_date: 125 | self.sign_date = cur_date 126 | need_update = True 127 | if self.sign_date is None: 128 | raise OTSClientError('v4 sign_date is None.') 129 | if not need_update: 130 | return 131 | origin_signing_key = consts.V4_SIGNATURE_PREFIX + self.user_key 132 | first_signing_key = calculate_hmac(origin_signing_key, self.sign_date, hashlib.sha256, self.encoding) 133 | second_signing_key = calculate_hmac(first_signing_key, self.region, hashlib.sha256, self.encoding) 134 | third_signing_key = calculate_hmac(second_signing_key, consts.V4_SIGNATURE_PRODUCT, hashlib.sha256, 135 | self.encoding) 136 | fourth_signing_key = calculate_hmac(third_signing_key, consts.V4_SIGNATURE_CONSTANT, hashlib.sha256, 137 | self.encoding) 138 | self.signing_key = base64.b64encode(fourth_signing_key) 139 | 140 | def make_request_signature_and_add_headers(self, query, headers): 141 | headers[consts.OTS_HEADER_SIGN_DATE] = self.sign_date 142 | headers[consts.OTS_HEADER_SIGN_REGION] = self.region 143 | signature_string = self._get_request_signature_string(query, headers) 144 | signature_string += consts.V4_SIGNATURE_SALT 145 | headers[consts.OTS_HEADER_SIGNATURE_V4] = call_signature_method_sha256(self.signing_key, signature_string, 146 | self.encoding) 147 | -------------------------------------------------------------------------------- /tests/sql_query_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | import sys 4 | import unittest 5 | from tests.lib.api_test_base import APITestBase 6 | from tablestore import * 7 | import tests.lib.restriction as restriction 8 | import copy 9 | from tablestore.error import * 10 | import math 11 | import time 12 | 13 | if sys.getdefaultencoding() != 'utf-8': 14 | reload(sys) 15 | sys.setdefaultencoding('utf-8') 16 | 17 | def batch_write_row(client,table_name,tp): 18 | put_row_items = [] 19 | for i in range(0, 10): 20 | if tp == "sql_test": 21 | primary_key = [('uid',str(i)),('pid',i)] 22 | attribute_columns = [('name','somebody'+str(i)), ('age',i%3),('grade',i+0.2)] 23 | else: 24 | primary_key = [('uid',str(i)),('pid',bytearray(i))] 25 | attribute_columns = [('name','somebody'+str(i)), ('age',i%3),('grade',i+0.2),('isMale',i%2==0),('picture',bytearray(i))] 26 | row = Row(primary_key, attribute_columns) 27 | condition = Condition(RowExistenceExpectation.IGNORE) 28 | item = PutRowItem(row, condition) 29 | put_row_items.append(item) 30 | request = BatchWriteRowRequest() 31 | request.add(TableInBatchWriteRowItem(table_name, put_row_items)) 32 | client.batch_write_row(request) 33 | 34 | class SqlQueryTest(APITestBase): 35 | def test_sql_query(self): 36 | """Test the execution results of SQL""" 37 | table_name = 'SqlQuery' + self.get_python_version() 38 | self.delete_table_and_index() 39 | table_meta = TableMeta(table_name, [('uid', 'STRING'),('pid', 'INTEGER')]) 40 | reserved_throughput = ReservedThroughput(CapacityUnit( 41 | restriction.MinReadWriteCapacityUnit, 42 | restriction.MinReadWriteCapacityUnit 43 | )) 44 | table_options = TableOptions() 45 | self.client_test.create_table(table_meta, table_options, reserved_throughput) 46 | self.wait_for_partition_load(table_name) 47 | sql_queries={ 48 | "create_table" :'create table %s (uid VARCHAR(1024), pid BIGINT(20),name MEDIUMTEXT, age BIGINT(20), grade DOUBLE,PRIMARY KEY(uid,pid));' % table_name, 49 | "count":'select count(*) from %s' % table_name, 50 | "sum":'select sum(age) from %s' % table_name, 51 | "where":'select pid from %s where grade > 5.0' % table_name, 52 | "group_by":'select age,avg(grade) from %s group by age order by age' % table_name, 53 | "order_by":'select pid from %s order by grade limit 1' % table_name, 54 | "desc":'desc %s' % table_name, 55 | "drop_table" :'drop mapping table %s' % table_name 56 | } 57 | ground_truth={ 58 | "count":10, 59 | "sum":9.0, 60 | "where":[5,6,7,8,9], 61 | "group_by":[(0,4.7),(1,4.2),(2,5.2)], 62 | "order_by":0, 63 | "desc":[[('Default', None), ('Extra', ''), ('Field', 'age'), ('Key', ''), ('Null', 'YES'), ('Type', 'bigint(20)')], 64 | [('Default', None), ('Extra', ''), ('Field', 'grade'), ('Key', ''), ('Null', 'YES'), ('Type', 'double')], 65 | [('Default', None), ('Extra', ''), ('Field', 'name'), ('Key', ''), ('Null', 'YES'), ('Type', 'mediumtext')], 66 | [('Default', None), ('Extra', ''), ('Field', 'pid'), ('Key', 'PRI'), ('Null', 'NO'), ('Type', 'bigint(20)')], 67 | [('Default', None), ('Extra', ''), ('Field', 'uid'), ('Key', 'PRI'), ('Null', 'NO'), ('Type', 'varchar(1024)')]] 68 | } 69 | 70 | def sort_result(input): 71 | ret = [] 72 | for item in input: 73 | tmp = sorted(item,key=lambda x:x[0]) 74 | ret.append(tmp) 75 | ret.sort() 76 | return ret 77 | 78 | def get_result(row_list,tp): 79 | ret = [] 80 | for row in row_list: 81 | ret.append(row.attribute_columns) 82 | if tp == "count":return ret[0][0][1] 83 | elif tp == "sum":return ret[0][0][1] 84 | elif tp == "where": 85 | where_ret = [] 86 | for item in ret: 87 | where_ret.append(item[0][1]) 88 | return where_ret 89 | elif tp == "group_by": 90 | group_by_ret = [] 91 | for item in ret: 92 | group_by_ret.append((item[0][1],item[1][1])) 93 | return group_by_ret 94 | elif tp == "order_by":return ret[0][0][1] 95 | elif tp == "desc":return sort_result(ret) 96 | 97 | def exe_sql_query(query): 98 | return self.client_test.exe_sql_query(query) 99 | 100 | def CHECK(tp): 101 | row_list,_,_ = exe_sql_query(sql_queries[tp]) 102 | ret = get_result(row_list,tp) 103 | self.assert_equal(ret, ground_truth[tp]) 104 | 105 | batch_write_row(self.client_test,table_name,"sql_test") 106 | exe_sql_query(sql_queries["create_table"]) 107 | CHECK("count") 108 | CHECK("sum") 109 | CHECK("where") 110 | CHECK("group_by") 111 | CHECK("order_by") 112 | CHECK("desc") 113 | exe_sql_query(sql_queries["drop_table"]) 114 | self.client_test.delete_table(table_name) 115 | 116 | def test_fbs_decoder_types(self): 117 | """Test the fbs decoder result""" 118 | table_name = 'fbsDecoderTypes' + self.get_python_version() 119 | self.delete_table_and_index() 120 | table_meta = TableMeta(table_name, [('uid', 'STRING'),('pid', 'BINARY')]) 121 | reserved_throughput = ReservedThroughput(CapacityUnit( 122 | restriction.MinReadWriteCapacityUnit, 123 | restriction.MinReadWriteCapacityUnit 124 | )) 125 | table_options = TableOptions() 126 | self.client_test.create_table(table_meta, table_options, reserved_throughput) 127 | self.wait_for_partition_load(table_name) 128 | sql_queries={ 129 | "create_table" :'create table %s (uid VARCHAR(1024), pid VARBINARY(1024), name MEDIUMTEXT, age BIGINT(20), grade DOUBLE, isMale BOOLEAN, picture MEDIUMBLOB, PRIMARY KEY(uid,pid));' % table_name, 130 | "all":'select * from %s' % table_name, 131 | "drop_table" :'drop mapping table %s' % table_name 132 | } 133 | 134 | def get_result(row_list): 135 | ret = [] 136 | for row in row_list: 137 | ret.append(row.attribute_columns) 138 | return ret 139 | 140 | def exe_sql_query(query): 141 | return self.client_test.exe_sql_query(query) 142 | 143 | def CHECK(ret): 144 | for i in range(0, 10): 145 | gt = [('uid',str(i)),('pid',bytearray(i)),('name','somebody'+str(i)), ('age',i%3), 146 | ('grade',i+0.2),('isMale',i%2==0),('picture',bytearray(i))] 147 | self.assert_equal(ret[i].sort(), gt.sort()) 148 | 149 | batch_write_row(self.client_test,table_name,"fbs_test") 150 | exe_sql_query(sql_queries["create_table"]) 151 | row_list,_,_ = exe_sql_query(sql_queries["all"]) 152 | CHECK(get_result(row_list)) 153 | exe_sql_query(sql_queries["drop_table"]) 154 | self.client_test.delete_table(table_name) 155 | 156 | if __name__ == '__main__': 157 | unittest.main() --------------------------------------------------------------------------------