├── BEService ├── __init__.py ├── constants.py └── TalkService-remote ├── other version ├── GOLANG │ └── beservice │ │ ├── GoUnusedProtection__.go │ │ └── BEService-consts.go ├── RUBY │ └── b_e_service_constants.rb └── C++ │ ├── BEService_constants.cpp │ ├── BEService_constants.h │ └── TalkService_server.skeleton.cpp ├── README.md ├── thrift ├── __init__.py ├── transport │ ├── __init__.py │ ├── sslcompat.py │ ├── THttpClient.py │ ├── TSocket.py │ ├── TZlibTransport.py │ ├── TTwisted.py │ ├── THeaderTransport.py │ └── TTransport.py ├── protocol │ ├── __init__.py │ ├── TProtocolDecorator.py │ ├── TMultiplexedProtocol.py │ ├── TBase.py │ ├── THeaderProtocol.py │ ├── TBinaryProtocol.py │ ├── TProtocol.py │ └── TCompactProtocol.py ├── ext │ ├── binary.cpp │ ├── protocol.h │ ├── endian.h │ ├── compact.cpp │ ├── types.cpp │ ├── types.h │ ├── binary.h │ ├── module.cpp │ └── compact.h ├── TSCons.py ├── compat.py ├── TSerialization.py ├── TRecursive.py ├── TMultiplexedProcessor.py ├── Thrift.py └── TTornado.py ├── sample.py └── function.py /BEService/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ['ttypes', 'constants', 'TalkService'] 2 | -------------------------------------------------------------------------------- /other version/GOLANG/beservice/GoUnusedProtection__.go: -------------------------------------------------------------------------------- 1 | // Autogenerated by Thrift Compiler (0.13.0) 2 | // DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 3 | 4 | package beservice 5 | 6 | var GoUnusedProtection__ int; 7 | 8 | -------------------------------------------------------------------------------- /other version/RUBY/b_e_service_constants.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Autogenerated by Thrift Compiler (0.13.0) 3 | # 4 | # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | # 6 | 7 | require 'thrift' 8 | require 'b_e_service_types' 9 | 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SIMPLE-PROTECTV2 2 | - JUST SIMPLE LIB FOR SIMPLE PROTECT 3 | - UNOFFICIAL LIB BOT (USE WITH URE OWN RISK) 4 | - NEW FUNCTION, WITH FETCHOPS POLLING 5 | 6 | # APPRECIATE 7 | PLEASE GIVE START OR LITTLE DONATION (CONTACT ME) 8 | 9 | # CONTACT 10 | WA: +6289625658302 (JUST FOR ERROR REPORT, DONATION, AND CUSTOM REQUEST) 11 | -------------------------------------------------------------------------------- /other version/C++/BEService_constants.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.13.0) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | #include "BEService_constants.h" 8 | 9 | 10 | 11 | const BEServiceConstants g_BEService_constants; 12 | 13 | BEServiceConstants::BEServiceConstants() { 14 | } 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /BEService/constants.py: -------------------------------------------------------------------------------- 1 | # 2 | # Autogenerated by Thrift Compiler (0.13.0) 3 | # 4 | # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | # 6 | # options string: py 7 | # 8 | 9 | from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException 10 | from thrift.protocol.TProtocol import TProtocolException 11 | from thrift.TRecursive import fix_spec 12 | 13 | import sys 14 | from .ttypes import * 15 | -------------------------------------------------------------------------------- /other version/C++/BEService_constants.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.13.0) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | #ifndef BEService_CONSTANTS_H 8 | #define BEService_CONSTANTS_H 9 | 10 | #include "BEService_types.h" 11 | 12 | 13 | 14 | class BEServiceConstants { 15 | public: 16 | BEServiceConstants(); 17 | 18 | }; 19 | 20 | extern const BEServiceConstants g_BEService_constants; 21 | 22 | 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /other version/GOLANG/beservice/BEService-consts.go: -------------------------------------------------------------------------------- 1 | // Autogenerated by Thrift Compiler (0.13.0) 2 | // DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 3 | 4 | package beservice 5 | 6 | import( 7 | "bytes" 8 | "context" 9 | "reflect" 10 | "fmt" 11 | "github.com/apache/thrift/lib/go/thrift" 12 | ) 13 | 14 | // (needed to ensure safety because of naive import list construction.) 15 | var _ = thrift.ZERO 16 | var _ = fmt.Printf 17 | var _ = context.Background 18 | var _ = reflect.DeepEqual 19 | var _ = bytes.Equal 20 | 21 | 22 | func init() { 23 | } 24 | 25 | -------------------------------------------------------------------------------- /thrift/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | __all__ = ['Thrift', 'TSCons'] 21 | -------------------------------------------------------------------------------- /thrift/transport/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | __all__ = ['TTransport', 'TSocket', 'THttpClient', 'TZlibTransport'] 21 | -------------------------------------------------------------------------------- /thrift/protocol/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | __all__ = ['fastbinary', 'TBase', 'TBinaryProtocol', 'TCompactProtocol', 21 | 'TJSONProtocol', 'TProtocol', 'TProtocolDecorator'] 22 | -------------------------------------------------------------------------------- /thrift/protocol/TProtocolDecorator.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | 21 | class TProtocolDecorator(object): 22 | def __new__(cls, protocol, *args, **kwargs): 23 | decorated_cls = type(''.join(['Decorated', protocol.__class__.__name__]), 24 | (cls, protocol.__class__), 25 | protocol.__dict__) 26 | return object.__new__(decorated_cls) 27 | -------------------------------------------------------------------------------- /thrift/ext/binary.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | #include "ext/binary.h" 21 | namespace apache { 22 | namespace thrift { 23 | namespace py { 24 | 25 | bool BinaryProtocol::readFieldBegin(TType& type, int16_t& tag) { 26 | uint8_t b = 0; 27 | if (!readByte(b)) { 28 | return false; 29 | } 30 | type = static_cast(b); 31 | if (type == T_STOP) { 32 | return true; 33 | } 34 | return readI16(tag); 35 | } 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /thrift/TSCons.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | from os import path 21 | from SCons.Builder import Builder 22 | from six.moves import map 23 | 24 | 25 | def scons_env(env, add=''): 26 | opath = path.dirname(path.abspath('$TARGET')) 27 | lstr = 'thrift --gen cpp -o ' + opath + ' ' + add + ' $SOURCE' 28 | cppbuild = Builder(action=lstr) 29 | env.Append(BUILDERS={'ThriftCpp': cppbuild}) 30 | 31 | 32 | def gen_cpp(env, dir, file): 33 | scons_env(env) 34 | suffixes = ['_types.h', '_types.cpp'] 35 | targets = map(lambda s: 'gen-cpp/' + file + s, suffixes) 36 | return env.ThriftCpp(targets, dir + file + '.thrift') 37 | -------------------------------------------------------------------------------- /thrift/compat.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | import sys 21 | 22 | if sys.version_info[0] == 2: 23 | 24 | from cStringIO import StringIO as BufferIO 25 | 26 | def binary_to_str(bin_val): 27 | return bin_val 28 | 29 | def str_to_binary(str_val): 30 | return str_val 31 | 32 | def byte_index(bytes_val, i): 33 | return ord(bytes_val[i]) 34 | 35 | else: 36 | 37 | from io import BytesIO as BufferIO # noqa 38 | 39 | def binary_to_str(bin_val): 40 | return bin_val.decode('utf8') 41 | 42 | def str_to_binary(str_val): 43 | return bytes(str_val, 'utf8') 44 | 45 | def byte_index(bytes_val, i): 46 | return bytes_val[i] 47 | -------------------------------------------------------------------------------- /thrift/TSerialization.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | from .protocol import TBinaryProtocol 21 | from .transport import TTransport 22 | 23 | 24 | def serialize(thrift_object, 25 | protocol_factory=TBinaryProtocol.TBinaryProtocolFactory()): 26 | transport = TTransport.TMemoryBuffer() 27 | protocol = protocol_factory.getProtocol(transport) 28 | thrift_object.write(protocol) 29 | return transport.getvalue() 30 | 31 | 32 | def deserialize(base, 33 | buf, 34 | protocol_factory=TBinaryProtocol.TBinaryProtocolFactory()): 35 | transport = TTransport.TMemoryBuffer(buf) 36 | protocol = protocol_factory.getProtocol(transport) 37 | base.read(protocol) 38 | return base 39 | -------------------------------------------------------------------------------- /thrift/protocol/TMultiplexedProtocol.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | from thrift.Thrift import TMessageType 21 | from thrift.protocol import TProtocolDecorator 22 | 23 | SEPARATOR = ":" 24 | 25 | 26 | class TMultiplexedProtocol(TProtocolDecorator.TProtocolDecorator): 27 | def __init__(self, protocol, serviceName): 28 | self.serviceName = serviceName 29 | 30 | def writeMessageBegin(self, name, type, seqid): 31 | if (type == TMessageType.CALL or 32 | type == TMessageType.ONEWAY): 33 | super(TMultiplexedProtocol, self).writeMessageBegin( 34 | self.serviceName + SEPARATOR + name, 35 | type, 36 | seqid 37 | ) 38 | else: 39 | super(TMultiplexedProtocol, self).writeMessageBegin(name, type, seqid) 40 | -------------------------------------------------------------------------------- /sample.py: -------------------------------------------------------------------------------- 1 | from function import * 2 | 3 | 4 | cl = BE_Team(myToken="ucbfdc04ee82a01c534eeb5475ecaa984:aWF0OiAxNjA2ODQ2NTkwODM5Cg==..pYlS2EYYLeLJ4ZQlH5/nATzSGYs=", 5 | myApp="ANDROIDLITE\t2.14.0\tAndroid OS\t5.1.1") 6 | 7 | 8 | def worker(op): 9 | try: 10 | if op.type in [25, 26]: 11 | msg = op.message 12 | text = str(msg.text) 13 | msg_id = msg.id 14 | receiver = msg.to 15 | msg.from_ = msg._from 16 | sender = msg._from 17 | cmd = text.lower() 18 | if msg.toType == 0 and sender != cl.profile.mid: to = sender 19 | else: to = receiver 20 | 21 | if cmd == "ping": 22 | cl.sendMessage(to,'pong') 23 | 24 | if cmd == "speed": 25 | start = time.time() 26 | cl.sendMessage(to,'benchmark...') 27 | total = time.time()-start 28 | cl.sendMessage(to,str(total)) 29 | 30 | except Exception as catch: 31 | trace = catch.__traceback__ 32 | print("Error Name: "+str(trace.tb_frame.f_code.co_name)+"\nError Filename: "+str(trace.tb_frame.f_code.co_filename)+"\nError Line: "+str(trace.tb_lineno)+"\nError: "+str(catch)) 33 | 34 | with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: 35 | while True: 36 | try: 37 | ops = cl.fetchOps() 38 | for op in ops: 39 | if op.revision == -1 and op.param2 != None: 40 | cl.globalRev = int(op.param2.split("\x1e")[0]) 41 | if op.revision == -1 and op.param1 != None: 42 | cl.individualRev = int(op.param1.split("\x1e")[0]) 43 | cl.localRev = max(op.revision, cl.localRev) 44 | executor.submit(worker,op) 45 | except: 46 | pass 47 | 48 | -------------------------------------------------------------------------------- /thrift/ext/protocol.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | #ifndef THRIFT_PY_PROTOCOL_H 21 | #define THRIFT_PY_PROTOCOL_H 22 | 23 | #include "ext/types.h" 24 | #include 25 | #include 26 | 27 | namespace apache { 28 | namespace thrift { 29 | namespace py { 30 | 31 | template 32 | class ProtocolBase { 33 | 34 | public: 35 | ProtocolBase() 36 | : stringLimit_((std::numeric_limits::max)()), 37 | containerLimit_((std::numeric_limits::max)()), 38 | output_(NULL) {} 39 | inline virtual ~ProtocolBase(); 40 | 41 | bool prepareDecodeBufferFromTransport(PyObject* trans); 42 | 43 | PyObject* readStruct(PyObject* output, PyObject* klass, PyObject* spec_seq); 44 | 45 | bool prepareEncodeBuffer(); 46 | 47 | bool encodeValue(PyObject* value, TType type, PyObject* typeargs); 48 | 49 | PyObject* getEncodedValue(); 50 | 51 | long stringLimit() const { return stringLimit_; } 52 | void setStringLengthLimit(long limit) { stringLimit_ = limit; } 53 | 54 | long containerLimit() const { return containerLimit_; } 55 | void setContainerLengthLimit(long limit) { containerLimit_ = limit; } 56 | 57 | protected: 58 | bool readBytes(char** output, int len); 59 | 60 | bool readByte(uint8_t& val) { 61 | char* buf; 62 | if (!readBytes(&buf, 1)) { 63 | return false; 64 | } 65 | val = static_cast(buf[0]); 66 | return true; 67 | } 68 | 69 | bool writeBuffer(char* data, size_t len); 70 | 71 | void writeByte(uint8_t val) { writeBuffer(reinterpret_cast(&val), 1); } 72 | 73 | PyObject* decodeValue(TType type, PyObject* typeargs); 74 | 75 | bool skip(TType type); 76 | 77 | inline bool checkType(TType got, TType expected); 78 | inline bool checkLengthLimit(int32_t len, long limit); 79 | 80 | inline bool isUtf8(PyObject* typeargs); 81 | 82 | private: 83 | Impl* impl() { return static_cast(this); } 84 | 85 | long stringLimit_; 86 | long containerLimit_; 87 | EncodeBuffer* output_; 88 | DecodeBuffer input_; 89 | }; 90 | } 91 | } 92 | } 93 | 94 | #include "ext/protocol.tcc" 95 | 96 | #endif // THRIFT_PY_PROTOCOL_H 97 | -------------------------------------------------------------------------------- /thrift/protocol/TBase.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | from thrift.transport import TTransport 21 | 22 | 23 | class TBase(object): 24 | __slots__ = () 25 | 26 | def __repr__(self): 27 | L = ['%s=%r' % (key, getattr(self, key)) for key in self.__slots__] 28 | return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) 29 | 30 | def __eq__(self, other): 31 | if not isinstance(other, self.__class__): 32 | return False 33 | for attr in self.__slots__: 34 | my_val = getattr(self, attr) 35 | other_val = getattr(other, attr) 36 | if my_val != other_val: 37 | return False 38 | return True 39 | 40 | def __ne__(self, other): 41 | return not (self == other) 42 | 43 | def read(self, iprot): 44 | if (iprot._fast_decode is not None and 45 | isinstance(iprot.trans, TTransport.CReadableTransport) and 46 | self.thrift_spec is not None): 47 | iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) 48 | else: 49 | iprot.readStruct(self, self.thrift_spec) 50 | 51 | def write(self, oprot): 52 | if (oprot._fast_encode is not None and self.thrift_spec is not None): 53 | oprot.trans.write( 54 | oprot._fast_encode(self, [self.__class__, self.thrift_spec])) 55 | else: 56 | oprot.writeStruct(self, self.thrift_spec) 57 | 58 | 59 | class TExceptionBase(TBase, Exception): 60 | pass 61 | 62 | 63 | class TFrozenBase(TBase): 64 | def __setitem__(self, *args): 65 | raise TypeError("Can't modify frozen struct") 66 | 67 | def __delitem__(self, *args): 68 | raise TypeError("Can't modify frozen struct") 69 | 70 | def __hash__(self, *args): 71 | return hash(self.__class__) ^ hash(self.__slots__) 72 | 73 | @classmethod 74 | def read(cls, iprot): 75 | if (iprot._fast_decode is not None and 76 | isinstance(iprot.trans, TTransport.CReadableTransport) and 77 | cls.thrift_spec is not None): 78 | self = cls() 79 | return iprot._fast_decode(None, iprot, 80 | [self.__class__, self.thrift_spec]) 81 | else: 82 | return iprot.readStruct(cls, cls.thrift_spec, True) 83 | -------------------------------------------------------------------------------- /thrift/ext/endian.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | #ifndef THRIFT_PY_ENDIAN_H 21 | #define THRIFT_PY_ENDIAN_H 22 | 23 | #include 24 | 25 | #ifndef _WIN32 26 | #include 27 | #else 28 | #include 29 | #pragma comment(lib, "ws2_32.lib") 30 | #define BIG_ENDIAN (4321) 31 | #define LITTLE_ENDIAN (1234) 32 | #define BYTE_ORDER LITTLE_ENDIAN 33 | #define inline __inline 34 | #endif 35 | 36 | /* Fix endianness issues on Solaris */ 37 | #if defined(__SVR4) && defined(__sun) 38 | #if defined(__i386) && !defined(__i386__) 39 | #define __i386__ 40 | #endif 41 | 42 | #ifndef BIG_ENDIAN 43 | #define BIG_ENDIAN (4321) 44 | #endif 45 | #ifndef LITTLE_ENDIAN 46 | #define LITTLE_ENDIAN (1234) 47 | #endif 48 | 49 | /* I386 is LE, even on Solaris */ 50 | #if !defined(BYTE_ORDER) && defined(__i386__) 51 | #define BYTE_ORDER LITTLE_ENDIAN 52 | #endif 53 | #endif 54 | 55 | #ifndef __BYTE_ORDER 56 | #if defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && defined(BIG_ENDIAN) 57 | #define __BYTE_ORDER BYTE_ORDER 58 | #define __LITTLE_ENDIAN LITTLE_ENDIAN 59 | #define __BIG_ENDIAN BIG_ENDIAN 60 | #else 61 | #error "Cannot determine endianness" 62 | #endif 63 | #endif 64 | 65 | // Same comment as the enum. Sorry. 66 | #if __BYTE_ORDER == __BIG_ENDIAN 67 | #define ntohll(n) (n) 68 | #define htonll(n) (n) 69 | #if defined(__GNUC__) && defined(__GLIBC__) 70 | #include 71 | #define letohll(n) bswap_64(n) 72 | #define htolell(n) bswap_64(n) 73 | #else /* GNUC & GLIBC */ 74 | #define letohll(n) ((((unsigned long long)ntohl(n)) << 32) + ntohl(n >> 32)) 75 | #define htolell(n) ((((unsigned long long)htonl(n)) << 32) + htonl(n >> 32)) 76 | #endif 77 | #elif __BYTE_ORDER == __LITTLE_ENDIAN 78 | #if defined(__GNUC__) && defined(__GLIBC__) 79 | #include 80 | #define ntohll(n) bswap_64(n) 81 | #define htonll(n) bswap_64(n) 82 | #elif defined(_MSC_VER) 83 | #include 84 | #define ntohll(n) _byteswap_uint64(n) 85 | #define htonll(n) _byteswap_uint64(n) 86 | #else /* GNUC & GLIBC */ 87 | #define ntohll(n) ((((unsigned long long)ntohl(n)) << 32) + ntohl(n >> 32)) 88 | #define htonll(n) ((((unsigned long long)htonl(n)) << 32) + htonl(n >> 32)) 89 | #endif /* GNUC & GLIBC */ 90 | #define letohll(n) (n) 91 | #define htolell(n) (n) 92 | #else /* __BYTE_ORDER */ 93 | #error "Can't define htonll or ntohll!" 94 | #endif 95 | 96 | #endif // THRIFT_PY_ENDIAN_H 97 | -------------------------------------------------------------------------------- /thrift/ext/compact.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | #include "ext/compact.h" 21 | 22 | namespace apache { 23 | namespace thrift { 24 | namespace py { 25 | 26 | const uint8_t CompactProtocol::TTypeToCType[] = { 27 | CT_STOP, // T_STOP 28 | 0, // unused 29 | CT_BOOLEAN_TRUE, // T_BOOL 30 | CT_BYTE, // T_BYTE 31 | CT_DOUBLE, // T_DOUBLE 32 | 0, // unused 33 | CT_I16, // T_I16 34 | 0, // unused 35 | CT_I32, // T_I32 36 | 0, // unused 37 | CT_I64, // T_I64 38 | CT_BINARY, // T_STRING 39 | CT_STRUCT, // T_STRUCT 40 | CT_MAP, // T_MAP 41 | CT_SET, // T_SET 42 | CT_LIST, // T_LIST 43 | }; 44 | 45 | bool CompactProtocol::readFieldBegin(TType& type, int16_t& tag) { 46 | uint8_t b; 47 | if (!readByte(b)) { 48 | return false; 49 | } 50 | uint8_t ctype = b & 0xf; 51 | type = getTType(ctype); 52 | if (type == -1) { 53 | return false; 54 | } else if (type == T_STOP) { 55 | tag = 0; 56 | return true; 57 | } 58 | uint8_t diff = (b & 0xf0) >> 4; 59 | if (diff) { 60 | tag = readTags_.top() + diff; 61 | } else if (!readI16(tag)) { 62 | readTags_.top() = -1; 63 | return false; 64 | } 65 | if (ctype == CT_BOOLEAN_FALSE || ctype == CT_BOOLEAN_TRUE) { 66 | readBool_.exists = true; 67 | readBool_.value = ctype == CT_BOOLEAN_TRUE; 68 | } 69 | readTags_.top() = tag; 70 | return true; 71 | } 72 | 73 | TType CompactProtocol::getTType(uint8_t type) { 74 | switch (type) { 75 | case T_STOP: 76 | return T_STOP; 77 | case CT_BOOLEAN_FALSE: 78 | case CT_BOOLEAN_TRUE: 79 | return T_BOOL; 80 | case CT_BYTE: 81 | return T_BYTE; 82 | case CT_I16: 83 | return T_I16; 84 | case CT_I32: 85 | return T_I32; 86 | case CT_I64: 87 | return T_I64; 88 | case CT_DOUBLE: 89 | return T_DOUBLE; 90 | case CT_BINARY: 91 | return T_STRING; 92 | case CT_LIST: 93 | return T_LIST; 94 | case CT_SET: 95 | return T_SET; 96 | case CT_MAP: 97 | return T_MAP; 98 | case CT_STRUCT: 99 | return T_STRUCT; 100 | default: 101 | PyErr_Format(PyExc_TypeError, "don't know what type: %d", type); 102 | return static_cast(-1); 103 | } 104 | } 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /thrift/TRecursive.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | from __future__ import absolute_import 14 | from __future__ import division 15 | from __future__ import print_function 16 | from __future__ import unicode_literals 17 | 18 | from thrift.Thrift import TType 19 | 20 | TYPE_IDX = 1 21 | SPEC_ARGS_IDX = 3 22 | SPEC_ARGS_CLASS_REF_IDX = 0 23 | SPEC_ARGS_THRIFT_SPEC_IDX = 1 24 | 25 | 26 | def fix_spec(all_structs): 27 | """Wire up recursive references for all TStruct definitions inside of each thrift_spec.""" 28 | for struc in all_structs: 29 | spec = struc.thrift_spec 30 | for thrift_spec in spec: 31 | if thrift_spec is None: 32 | continue 33 | elif thrift_spec[TYPE_IDX] == TType.STRUCT: 34 | other = thrift_spec[SPEC_ARGS_IDX][SPEC_ARGS_CLASS_REF_IDX].thrift_spec 35 | thrift_spec[SPEC_ARGS_IDX][SPEC_ARGS_THRIFT_SPEC_IDX] = other 36 | elif thrift_spec[TYPE_IDX] in (TType.LIST, TType.SET): 37 | _fix_list_or_set(thrift_spec[SPEC_ARGS_IDX]) 38 | elif thrift_spec[TYPE_IDX] == TType.MAP: 39 | _fix_map(thrift_spec[SPEC_ARGS_IDX]) 40 | 41 | 42 | def _fix_list_or_set(element_type): 43 | # For a list or set, the thrift_spec entry looks like, 44 | # (1, TType.LIST, 'lister', (TType.STRUCT, [RecList, None], False), None, ), # 1 45 | # so ``element_type`` will be, 46 | # (TType.STRUCT, [RecList, None], False) 47 | if element_type[0] == TType.STRUCT: 48 | element_type[1][1] = element_type[1][0].thrift_spec 49 | elif element_type[0] in (TType.LIST, TType.SET): 50 | _fix_list_or_set(element_type[1]) 51 | elif element_type[0] == TType.MAP: 52 | _fix_map(element_type[1]) 53 | 54 | 55 | def _fix_map(element_type): 56 | # For a map of key -> value type, ``element_type`` will be, 57 | # (TType.I16, None, TType.STRUCT, [RecMapBasic, None], False), None, ) 58 | # which is just a normal struct definition. 59 | # 60 | # For a map of key -> list / set, ``element_type`` will be, 61 | # (TType.I16, None, TType.LIST, (TType.STRUCT, [RecMapList, None], False), False) 62 | # and we need to process the 3rd element as a list. 63 | # 64 | # For a map of key -> map, ``element_type`` will be, 65 | # (TType.I16, None, TType.MAP, (TType.I16, None, TType.STRUCT, 66 | # [RecMapMap, None], False), False) 67 | # and need to process 3rd element as a map. 68 | 69 | # Is the map key a struct? 70 | if element_type[0] == TType.STRUCT: 71 | element_type[1][1] = element_type[1][0].thrift_spec 72 | elif element_type[0] in (TType.LIST, TType.SET): 73 | _fix_list_or_set(element_type[1]) 74 | elif element_type[0] == TType.MAP: 75 | _fix_map(element_type[1]) 76 | 77 | # Is the map value a struct? 78 | if element_type[2] == TType.STRUCT: 79 | element_type[3][1] = element_type[3][0].thrift_spec 80 | elif element_type[2] in (TType.LIST, TType.SET): 81 | _fix_list_or_set(element_type[3]) 82 | elif element_type[2] == TType.MAP: 83 | _fix_map(element_type[3]) 84 | -------------------------------------------------------------------------------- /thrift/TMultiplexedProcessor.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | from thrift.Thrift import TProcessor, TMessageType 21 | from thrift.protocol import TProtocolDecorator, TMultiplexedProtocol 22 | from thrift.protocol.TProtocol import TProtocolException 23 | 24 | 25 | class TMultiplexedProcessor(TProcessor): 26 | def __init__(self): 27 | self.defaultProcessor = None 28 | self.services = {} 29 | 30 | def registerDefault(self, processor): 31 | """ 32 | If a non-multiplexed processor connects to the server and wants to 33 | communicate, use the given processor to handle it. This mechanism 34 | allows servers to upgrade from non-multiplexed to multiplexed in a 35 | backwards-compatible way and still handle old clients. 36 | """ 37 | self.defaultProcessor = processor 38 | 39 | def registerProcessor(self, serviceName, processor): 40 | self.services[serviceName] = processor 41 | 42 | def on_message_begin(self, func): 43 | for key in self.services.keys(): 44 | self.services[key].on_message_begin(func) 45 | 46 | def process(self, iprot, oprot): 47 | (name, type, seqid) = iprot.readMessageBegin() 48 | if type != TMessageType.CALL and type != TMessageType.ONEWAY: 49 | raise TProtocolException( 50 | TProtocolException.NOT_IMPLEMENTED, 51 | "TMultiplexedProtocol only supports CALL & ONEWAY") 52 | 53 | index = name.find(TMultiplexedProtocol.SEPARATOR) 54 | if index < 0: 55 | if self.defaultProcessor: 56 | return self.defaultProcessor.process( 57 | StoredMessageProtocol(iprot, (name, type, seqid)), oprot) 58 | else: 59 | raise TProtocolException( 60 | TProtocolException.NOT_IMPLEMENTED, 61 | "Service name not found in message name: " + name + ". " + 62 | "Did you forget to use TMultiplexedProtocol in your client?") 63 | 64 | serviceName = name[0:index] 65 | call = name[index + len(TMultiplexedProtocol.SEPARATOR):] 66 | if serviceName not in self.services: 67 | raise TProtocolException( 68 | TProtocolException.NOT_IMPLEMENTED, 69 | "Service name not found: " + serviceName + ". " + 70 | "Did you forget to call registerProcessor()?") 71 | 72 | standardMessage = (call, type, seqid) 73 | return self.services[serviceName].process( 74 | StoredMessageProtocol(iprot, standardMessage), oprot) 75 | 76 | 77 | class StoredMessageProtocol(TProtocolDecorator.TProtocolDecorator): 78 | def __init__(self, protocol, messageBegin): 79 | self.messageBegin = messageBegin 80 | 81 | def readMessageBegin(self): 82 | return self.messageBegin 83 | -------------------------------------------------------------------------------- /thrift/ext/types.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | #include "ext/types.h" 21 | #include "ext/protocol.h" 22 | 23 | namespace apache { 24 | namespace thrift { 25 | namespace py { 26 | 27 | PyObject* ThriftModule = NULL; 28 | 29 | #if PY_MAJOR_VERSION < 3 30 | char refill_signature[] = {'s', '#', 'i'}; 31 | #else 32 | const char* refill_signature = "y#i"; 33 | #endif 34 | 35 | bool parse_struct_item_spec(StructItemSpec* dest, PyObject* spec_tuple) { 36 | // i'd like to use ParseArgs here, but it seems to be a bottleneck. 37 | if (PyTuple_Size(spec_tuple) != 5) { 38 | PyErr_Format(PyExc_TypeError, "expecting 5 arguments for spec tuple but got %d", 39 | static_cast(PyTuple_Size(spec_tuple))); 40 | return false; 41 | } 42 | 43 | dest->tag = static_cast(PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 0))); 44 | if (INT_CONV_ERROR_OCCURRED(dest->tag)) { 45 | return false; 46 | } 47 | 48 | dest->type = static_cast(PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 1))); 49 | if (INT_CONV_ERROR_OCCURRED(dest->type)) { 50 | return false; 51 | } 52 | 53 | dest->attrname = PyTuple_GET_ITEM(spec_tuple, 2); 54 | dest->typeargs = PyTuple_GET_ITEM(spec_tuple, 3); 55 | dest->defval = PyTuple_GET_ITEM(spec_tuple, 4); 56 | return true; 57 | } 58 | 59 | bool parse_set_list_args(SetListTypeArgs* dest, PyObject* typeargs) { 60 | if (PyTuple_Size(typeargs) != 3) { 61 | PyErr_SetString(PyExc_TypeError, "expecting tuple of size 3 for list/set type args"); 62 | return false; 63 | } 64 | 65 | dest->element_type = static_cast(PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0))); 66 | if (INT_CONV_ERROR_OCCURRED(dest->element_type)) { 67 | return false; 68 | } 69 | 70 | dest->typeargs = PyTuple_GET_ITEM(typeargs, 1); 71 | 72 | dest->immutable = Py_True == PyTuple_GET_ITEM(typeargs, 2); 73 | 74 | return true; 75 | } 76 | 77 | bool parse_map_args(MapTypeArgs* dest, PyObject* typeargs) { 78 | if (PyTuple_Size(typeargs) != 5) { 79 | PyErr_SetString(PyExc_TypeError, "expecting 5 arguments for typeargs to map"); 80 | return false; 81 | } 82 | 83 | dest->ktag = static_cast(PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0))); 84 | if (INT_CONV_ERROR_OCCURRED(dest->ktag)) { 85 | return false; 86 | } 87 | 88 | dest->vtag = static_cast(PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 2))); 89 | if (INT_CONV_ERROR_OCCURRED(dest->vtag)) { 90 | return false; 91 | } 92 | 93 | dest->ktypeargs = PyTuple_GET_ITEM(typeargs, 1); 94 | dest->vtypeargs = PyTuple_GET_ITEM(typeargs, 3); 95 | dest->immutable = Py_True == PyTuple_GET_ITEM(typeargs, 4); 96 | 97 | return true; 98 | } 99 | 100 | bool parse_struct_args(StructTypeArgs* dest, PyObject* typeargs) { 101 | if (PyList_Size(typeargs) != 2) { 102 | PyErr_SetString(PyExc_TypeError, "expecting list of size 2 for struct args"); 103 | return false; 104 | } 105 | 106 | dest->klass = PyList_GET_ITEM(typeargs, 0); 107 | dest->spec = PyList_GET_ITEM(typeargs, 1); 108 | 109 | return true; 110 | } 111 | } 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /thrift/transport/sslcompat.py: -------------------------------------------------------------------------------- 1 | # 2 | # licensed to the apache software foundation (asf) under one 3 | # or more contributor license agreements. see the notice file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. the asf licenses this file 6 | # to you under the apache license, version 2.0 (the 7 | # "license"); you may not use this file except in compliance 8 | # with the license. you may obtain a copy of the license at 9 | # 10 | # http://www.apache.org/licenses/license-2.0 11 | # 12 | # unless required by applicable law or agreed to in writing, 13 | # software distributed under the license is distributed on an 14 | # "as is" basis, without warranties or conditions of any 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | import logging 21 | import sys 22 | 23 | from thrift.transport.TTransport import TTransportException 24 | 25 | logger = logging.getLogger(__name__) 26 | 27 | 28 | def legacy_validate_callback(cert, hostname): 29 | """legacy method to validate the peer's SSL certificate, and to check 30 | the commonName of the certificate to ensure it matches the hostname we 31 | used to make this connection. Does not support subjectAltName records 32 | in certificates. 33 | 34 | raises TTransportException if the certificate fails validation. 35 | """ 36 | if 'subject' not in cert: 37 | raise TTransportException( 38 | TTransportException.NOT_OPEN, 39 | 'No SSL certificate found from %s' % hostname) 40 | fields = cert['subject'] 41 | for field in fields: 42 | # ensure structure we get back is what we expect 43 | if not isinstance(field, tuple): 44 | continue 45 | cert_pair = field[0] 46 | if len(cert_pair) < 2: 47 | continue 48 | cert_key, cert_value = cert_pair[0:2] 49 | if cert_key != 'commonName': 50 | continue 51 | certhost = cert_value 52 | # this check should be performed by some sort of Access Manager 53 | if certhost == hostname: 54 | # success, cert commonName matches desired hostname 55 | return 56 | else: 57 | raise TTransportException( 58 | TTransportException.UNKNOWN, 59 | 'Hostname we connected to "%s" doesn\'t match certificate ' 60 | 'provided commonName "%s"' % (hostname, certhost)) 61 | raise TTransportException( 62 | TTransportException.UNKNOWN, 63 | 'Could not validate SSL certificate from host "%s". Cert=%s' 64 | % (hostname, cert)) 65 | 66 | 67 | def _optional_dependencies(): 68 | try: 69 | import ipaddress # noqa 70 | logger.debug('ipaddress module is available') 71 | ipaddr = True 72 | except ImportError: 73 | logger.warn('ipaddress module is unavailable') 74 | ipaddr = False 75 | 76 | if sys.hexversion < 0x030500F0: 77 | try: 78 | from backports.ssl_match_hostname import match_hostname, __version__ as ver 79 | ver = list(map(int, ver.split('.'))) 80 | logger.debug('backports.ssl_match_hostname module is available') 81 | match = match_hostname 82 | if ver[0] * 10 + ver[1] >= 35: 83 | return ipaddr, match 84 | else: 85 | logger.warn('backports.ssl_match_hostname module is too old') 86 | ipaddr = False 87 | except ImportError: 88 | logger.warn('backports.ssl_match_hostname is unavailable') 89 | ipaddr = False 90 | try: 91 | from ssl import match_hostname 92 | logger.debug('ssl.match_hostname is available') 93 | match = match_hostname 94 | except ImportError: 95 | logger.warn('using legacy validation callback') 96 | match = legacy_validate_callback 97 | return ipaddr, match 98 | 99 | 100 | _match_has_ipaddress, _match_hostname = _optional_dependencies() 101 | -------------------------------------------------------------------------------- /thrift/transport/THttpClient.py: -------------------------------------------------------------------------------- 1 | # Modif by BE-Team 2 | 3 | from io import BytesIO 4 | import os 5 | import ssl 6 | import sys 7 | import warnings 8 | import base64 9 | import time 10 | from six.moves import urllib 11 | import http.client as http_client 12 | from .TTransport import TTransportBase 13 | import six 14 | 15 | 16 | class THttpClient(TTransportBase): 17 | """Http implementation of TTransport base.""" 18 | 19 | def __init__(self, uri_or_host, port=None, path=None, cafile=None, cert_file=None, key_file=None, ssl_context=None): 20 | if port is not None: 21 | warnings.warn( 22 | "Please use the THttpClient('http{s}://host:port/path') constructor", 23 | DeprecationWarning, 24 | stacklevel=2) 25 | self.host = uri_or_host 26 | self.port = port 27 | assert path 28 | self.path = path 29 | self.scheme = 'http' 30 | else: 31 | parsed = urllib.parse.urlparse(uri_or_host) 32 | self.scheme = parsed.scheme 33 | assert self.scheme in ('http', 'https') 34 | if self.scheme == 'http': 35 | self.port = parsed.port or http_client.HTTP_PORT 36 | elif self.scheme == 'https': 37 | self.port = parsed.port or http_client.HTTPS_PORT 38 | self.certfile = cert_file 39 | self.keyfile = key_file 40 | self.context = ssl.create_default_context(cafile=cafile) if (cafile and not ssl_context) else ssl_context 41 | self.host = parsed.hostname 42 | self.path = parsed.path 43 | if parsed.query: 44 | self.path += '?%s' % parsed.query 45 | proxy = None 46 | self.realhost = self.realport = self.proxy_auth = None 47 | self.__wbuf = BytesIO() 48 | self.__rbuf = BytesIO() 49 | self.__http = None 50 | self.__http_response = None 51 | self.__timeout = None 52 | self.__custom_headers = None 53 | self.__time = time.time() 54 | 55 | 56 | def using_proxy(self): 57 | return self.realhost is not None 58 | 59 | def open(self): 60 | if self.scheme == 'http': 61 | self.__http = http_client.HTTPConnection(self.host, self.port, 62 | timeout=self.__timeout) 63 | if self.scheme == 'https': 64 | self.__http = http_client.HTTPSConnection(self.host, self.port, 65 | key_file=self.keyfile, 66 | cert_file=self.certfile, 67 | timeout=self.__timeout, 68 | context=self.context) 69 | 70 | def close(self): 71 | self.__http.close() 72 | self.__http = None 73 | self.__http_response = None 74 | 75 | def isOpen(self): 76 | return self.__http is not None 77 | 78 | def setTimeout(self, ms): 79 | if ms is None: 80 | self.__timeout = None 81 | else: 82 | self.__timeout = ms / 1000.0 83 | 84 | def setCustomHeaders(self, headers): 85 | self.__custom_headers = headers 86 | 87 | def read(self, sz): 88 | return self.__rbuf.read(sz) 89 | 90 | def write(self, buf): 91 | self.__wbuf.write(buf) 92 | 93 | def flush(self): 94 | if not self.isOpen(): 95 | self.open(); self.__time = time.time() 96 | elif time.time() - self.__time > 90: 97 | self.close(); self.open(); self.__time = time.time() 98 | data = self.__wbuf.getvalue() 99 | self.__wbuf.truncate(0) 100 | self.__wbuf.seek(0) 101 | self.__http.putrequest('POST', self.path) 102 | self.__http.putheader('Content-Type', 'application/x-thrift') 103 | self.__http.putheader('Content-Length', str(len(data))) 104 | if self.__custom_headers: 105 | for key, val in six.iteritems(self.__custom_headers): 106 | self.__http.putheader(key, val) 107 | self.__http.endheaders() 108 | self.__http.send(data) 109 | self.__rbuf = BytesIO(self.__http.getresponse().read()) 110 | -------------------------------------------------------------------------------- /thrift/ext/types.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | #ifndef THRIFT_PY_TYPES_H 21 | #define THRIFT_PY_TYPES_H 22 | 23 | #include 24 | 25 | #ifdef _MSC_VER 26 | #define __STDC_FORMAT_MACROS 27 | #define __STDC_LIMIT_MACROS 28 | #endif 29 | #include 30 | 31 | #if PY_MAJOR_VERSION >= 3 32 | 33 | #include 34 | 35 | // TODO: better macros 36 | #define PyInt_AsLong(v) PyLong_AsLong(v) 37 | #define PyInt_FromLong(v) PyLong_FromLong(v) 38 | 39 | #define PyString_InternFromString(v) PyUnicode_InternFromString(v) 40 | 41 | #endif 42 | 43 | #define INTERN_STRING(value) _intern_##value 44 | 45 | #define INT_CONV_ERROR_OCCURRED(v) (((v) == -1) && PyErr_Occurred()) 46 | 47 | extern "C" { 48 | extern PyObject* INTERN_STRING(TFrozenDict); 49 | extern PyObject* INTERN_STRING(cstringio_buf); 50 | extern PyObject* INTERN_STRING(cstringio_refill); 51 | } 52 | 53 | namespace apache { 54 | namespace thrift { 55 | namespace py { 56 | 57 | extern PyObject* ThriftModule; 58 | 59 | // Stolen out of TProtocol.h. 60 | // It would be a huge pain to have both get this from one place. 61 | enum TType { 62 | T_INVALID = -1, 63 | T_STOP = 0, 64 | T_VOID = 1, 65 | T_BOOL = 2, 66 | T_BYTE = 3, 67 | T_I08 = 3, 68 | T_I16 = 6, 69 | T_I32 = 8, 70 | T_U64 = 9, 71 | T_I64 = 10, 72 | T_DOUBLE = 4, 73 | T_STRING = 11, 74 | T_UTF7 = 11, 75 | T_STRUCT = 12, 76 | T_MAP = 13, 77 | T_SET = 14, 78 | T_LIST = 15, 79 | T_UTF8 = 16, 80 | T_UTF16 = 17 81 | }; 82 | 83 | // replace with unique_ptr when we're OK with C++11 84 | class ScopedPyObject { 85 | public: 86 | ScopedPyObject() : obj_(NULL) {} 87 | explicit ScopedPyObject(PyObject* py_object) : obj_(py_object) {} 88 | ~ScopedPyObject() { 89 | if (obj_) 90 | Py_DECREF(obj_); 91 | } 92 | PyObject* get() throw() { return obj_; } 93 | operator bool() { return obj_; } 94 | void reset(PyObject* py_object) throw() { 95 | if (obj_) 96 | Py_DECREF(obj_); 97 | obj_ = py_object; 98 | } 99 | PyObject* release() throw() { 100 | PyObject* tmp = obj_; 101 | obj_ = NULL; 102 | return tmp; 103 | } 104 | void swap(ScopedPyObject& other) throw() { 105 | ScopedPyObject tmp(other.release()); 106 | other.reset(release()); 107 | reset(tmp.release()); 108 | } 109 | 110 | private: 111 | ScopedPyObject(const ScopedPyObject&) {} 112 | ScopedPyObject& operator=(const ScopedPyObject&) { return *this; } 113 | 114 | PyObject* obj_; 115 | }; 116 | 117 | /** 118 | * A cache of the two key attributes of a CReadableTransport, 119 | * so we don't have to keep calling PyObject_GetAttr. 120 | */ 121 | struct DecodeBuffer { 122 | ScopedPyObject stringiobuf; 123 | ScopedPyObject refill_callable; 124 | }; 125 | 126 | #if PY_MAJOR_VERSION < 3 127 | extern char refill_signature[3]; 128 | typedef PyObject EncodeBuffer; 129 | #else 130 | extern const char* refill_signature; 131 | struct EncodeBuffer { 132 | std::vector buf; 133 | size_t pos; 134 | }; 135 | #endif 136 | 137 | /** 138 | * A cache of the spec_args for a set or list, 139 | * so we don't have to keep calling PyTuple_GET_ITEM. 140 | */ 141 | struct SetListTypeArgs { 142 | TType element_type; 143 | PyObject* typeargs; 144 | bool immutable; 145 | }; 146 | 147 | /** 148 | * A cache of the spec_args for a map, 149 | * so we don't have to keep calling PyTuple_GET_ITEM. 150 | */ 151 | struct MapTypeArgs { 152 | TType ktag; 153 | TType vtag; 154 | PyObject* ktypeargs; 155 | PyObject* vtypeargs; 156 | bool immutable; 157 | }; 158 | 159 | /** 160 | * A cache of the spec_args for a struct, 161 | * so we don't have to keep calling PyTuple_GET_ITEM. 162 | */ 163 | struct StructTypeArgs { 164 | PyObject* klass; 165 | PyObject* spec; 166 | bool immutable; 167 | }; 168 | 169 | /** 170 | * A cache of the item spec from a struct specification, 171 | * so we don't have to keep calling PyTuple_GET_ITEM. 172 | */ 173 | struct StructItemSpec { 174 | int tag; 175 | TType type; 176 | PyObject* attrname; 177 | PyObject* typeargs; 178 | PyObject* defval; 179 | }; 180 | 181 | bool parse_set_list_args(SetListTypeArgs* dest, PyObject* typeargs); 182 | 183 | bool parse_map_args(MapTypeArgs* dest, PyObject* typeargs); 184 | 185 | bool parse_struct_args(StructTypeArgs* dest, PyObject* typeargs); 186 | 187 | bool parse_struct_item_spec(StructItemSpec* dest, PyObject* spec_tuple); 188 | } 189 | } 190 | } 191 | 192 | #endif // THRIFT_PY_TYPES_H 193 | -------------------------------------------------------------------------------- /thrift/Thrift.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | import sys 21 | 22 | 23 | class TType(object): 24 | STOP = 0 25 | VOID = 1 26 | BOOL = 2 27 | BYTE = 3 28 | I08 = 3 29 | DOUBLE = 4 30 | I16 = 6 31 | I32 = 8 32 | I64 = 10 33 | STRING = 11 34 | UTF7 = 11 35 | STRUCT = 12 36 | MAP = 13 37 | SET = 14 38 | LIST = 15 39 | UTF8 = 16 40 | UTF16 = 17 41 | 42 | _VALUES_TO_NAMES = ( 43 | 'STOP', 44 | 'VOID', 45 | 'BOOL', 46 | 'BYTE', 47 | 'DOUBLE', 48 | None, 49 | 'I16', 50 | None, 51 | 'I32', 52 | None, 53 | 'I64', 54 | 'STRING', 55 | 'STRUCT', 56 | 'MAP', 57 | 'SET', 58 | 'LIST', 59 | 'UTF8', 60 | 'UTF16', 61 | ) 62 | 63 | 64 | class TMessageType(object): 65 | CALL = 1 66 | REPLY = 2 67 | EXCEPTION = 3 68 | ONEWAY = 4 69 | 70 | 71 | class TProcessor(object): 72 | """Base class for processor, which works on two streams.""" 73 | 74 | def process(self, iprot, oprot): 75 | """ 76 | Process a request. The normal behvaior is to have the 77 | processor invoke the correct handler and then it is the 78 | server's responsibility to write the response to oprot. 79 | """ 80 | pass 81 | 82 | def on_message_begin(self, func): 83 | """ 84 | Install a callback that receives (name, type, seqid) 85 | after the message header is read. 86 | """ 87 | pass 88 | 89 | 90 | class TException(Exception): 91 | """Base class for all thrift exceptions.""" 92 | 93 | # BaseException.message is deprecated in Python v[2.6,3.0) 94 | if (2, 6, 0) <= sys.version_info < (3, 0): 95 | def _get_message(self): 96 | return self._message 97 | 98 | def _set_message(self, message): 99 | self._message = message 100 | message = property(_get_message, _set_message) 101 | 102 | def __init__(self, message=None): 103 | Exception.__init__(self, message) 104 | self.message = message 105 | 106 | 107 | class TApplicationException(TException): 108 | """Application level thrift exceptions.""" 109 | 110 | UNKNOWN = 0 111 | UNKNOWN_METHOD = 1 112 | INVALID_MESSAGE_TYPE = 2 113 | WRONG_METHOD_NAME = 3 114 | BAD_SEQUENCE_ID = 4 115 | MISSING_RESULT = 5 116 | INTERNAL_ERROR = 6 117 | PROTOCOL_ERROR = 7 118 | INVALID_TRANSFORM = 8 119 | INVALID_PROTOCOL = 9 120 | UNSUPPORTED_CLIENT_TYPE = 10 121 | 122 | def __init__(self, type=UNKNOWN, message=None): 123 | TException.__init__(self, message) 124 | self.type = type 125 | 126 | def __str__(self): 127 | if self.message: 128 | return self.message 129 | elif self.type == self.UNKNOWN_METHOD: 130 | return 'Unknown method' 131 | elif self.type == self.INVALID_MESSAGE_TYPE: 132 | return 'Invalid message type' 133 | elif self.type == self.WRONG_METHOD_NAME: 134 | return 'Wrong method name' 135 | elif self.type == self.BAD_SEQUENCE_ID: 136 | return 'Bad sequence ID' 137 | elif self.type == self.MISSING_RESULT: 138 | return 'Missing result' 139 | elif self.type == self.INTERNAL_ERROR: 140 | return 'Internal error' 141 | elif self.type == self.PROTOCOL_ERROR: 142 | return 'Protocol error' 143 | elif self.type == self.INVALID_TRANSFORM: 144 | return 'Invalid transform' 145 | elif self.type == self.INVALID_PROTOCOL: 146 | return 'Invalid protocol' 147 | elif self.type == self.UNSUPPORTED_CLIENT_TYPE: 148 | return 'Unsupported client type' 149 | else: 150 | return 'Default (unknown) TApplicationException' 151 | 152 | def read(self, iprot): 153 | iprot.readStructBegin() 154 | while True: 155 | (fname, ftype, fid) = iprot.readFieldBegin() 156 | if ftype == TType.STOP: 157 | break 158 | if fid == 1: 159 | if ftype == TType.STRING: 160 | self.message = iprot.readString() 161 | else: 162 | iprot.skip(ftype) 163 | elif fid == 2: 164 | if ftype == TType.I32: 165 | self.type = iprot.readI32() 166 | else: 167 | iprot.skip(ftype) 168 | else: 169 | iprot.skip(ftype) 170 | iprot.readFieldEnd() 171 | iprot.readStructEnd() 172 | 173 | def write(self, oprot): 174 | oprot.writeStructBegin('TApplicationException') 175 | if self.message is not None: 176 | oprot.writeFieldBegin('message', TType.STRING, 1) 177 | oprot.writeString(self.message) 178 | oprot.writeFieldEnd() 179 | if self.type is not None: 180 | oprot.writeFieldBegin('type', TType.I32, 2) 181 | oprot.writeI32(self.type) 182 | oprot.writeFieldEnd() 183 | oprot.writeFieldStop() 184 | oprot.writeStructEnd() 185 | 186 | 187 | class TFrozenDict(dict): 188 | """A dictionary that is "frozen" like a frozenset""" 189 | 190 | def __init__(self, *args, **kwargs): 191 | super(TFrozenDict, self).__init__(*args, **kwargs) 192 | # Sort the items so they will be in a consistent order. 193 | # XOR in the hash of the class so we don't collide with 194 | # the hash of a list of tuples. 195 | self.__hashval = hash(TFrozenDict) ^ hash(tuple(sorted(self.items()))) 196 | 197 | def __setitem__(self, *args): 198 | raise TypeError("Can't modify frozen TFreezableDict") 199 | 200 | def __delitem__(self, *args): 201 | raise TypeError("Can't modify frozen TFreezableDict") 202 | 203 | def __hash__(self): 204 | return self.__hashval 205 | -------------------------------------------------------------------------------- /thrift/ext/binary.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | #ifndef THRIFT_PY_BINARY_H 21 | #define THRIFT_PY_BINARY_H 22 | 23 | #include 24 | #include "ext/protocol.h" 25 | #include "ext/endian.h" 26 | #include 27 | 28 | namespace apache { 29 | namespace thrift { 30 | namespace py { 31 | 32 | class BinaryProtocol : public ProtocolBase { 33 | public: 34 | virtual ~BinaryProtocol() {} 35 | 36 | void writeI8(int8_t val) { writeBuffer(reinterpret_cast(&val), sizeof(int8_t)); } 37 | 38 | void writeI16(int16_t val) { 39 | int16_t net = static_cast(htons(val)); 40 | writeBuffer(reinterpret_cast(&net), sizeof(int16_t)); 41 | } 42 | 43 | void writeI32(int32_t val) { 44 | int32_t net = static_cast(htonl(val)); 45 | writeBuffer(reinterpret_cast(&net), sizeof(int32_t)); 46 | } 47 | 48 | void writeI64(int64_t val) { 49 | int64_t net = static_cast(htonll(val)); 50 | writeBuffer(reinterpret_cast(&net), sizeof(int64_t)); 51 | } 52 | 53 | void writeDouble(double dub) { 54 | // Unfortunately, bitwise_cast doesn't work in C. Bad C! 55 | union { 56 | double f; 57 | int64_t t; 58 | } transfer; 59 | transfer.f = dub; 60 | writeI64(transfer.t); 61 | } 62 | 63 | void writeBool(int v) { writeByte(static_cast(v)); } 64 | 65 | void writeString(PyObject* value, int32_t len) { 66 | writeI32(len); 67 | writeBuffer(PyBytes_AS_STRING(value), len); 68 | } 69 | 70 | bool writeListBegin(PyObject* value, const SetListTypeArgs& parsedargs, int32_t len) { 71 | writeByte(parsedargs.element_type); 72 | writeI32(len); 73 | return true; 74 | } 75 | 76 | bool writeMapBegin(PyObject* value, const MapTypeArgs& parsedargs, int32_t len) { 77 | writeByte(parsedargs.ktag); 78 | writeByte(parsedargs.vtag); 79 | writeI32(len); 80 | return true; 81 | } 82 | 83 | bool writeStructBegin() { return true; } 84 | bool writeStructEnd() { return true; } 85 | bool writeField(PyObject* value, const StructItemSpec& parsedspec) { 86 | writeByte(static_cast(parsedspec.type)); 87 | writeI16(parsedspec.tag); 88 | return encodeValue(value, parsedspec.type, parsedspec.typeargs); 89 | } 90 | 91 | void writeFieldStop() { writeByte(static_cast(T_STOP)); } 92 | 93 | bool readBool(bool& val) { 94 | char* buf; 95 | if (!readBytes(&buf, 1)) { 96 | return false; 97 | } 98 | val = buf[0] == 1; 99 | return true; 100 | } 101 | 102 | bool readI8(int8_t& val) { 103 | char* buf; 104 | if (!readBytes(&buf, 1)) { 105 | return false; 106 | } 107 | val = buf[0]; 108 | return true; 109 | } 110 | 111 | bool readI16(int16_t& val) { 112 | char* buf; 113 | if (!readBytes(&buf, sizeof(int16_t))) { 114 | return false; 115 | } 116 | memcpy(&val, buf, sizeof(int16_t)); 117 | val = ntohs(val); 118 | return true; 119 | } 120 | 121 | bool readI32(int32_t& val) { 122 | char* buf; 123 | if (!readBytes(&buf, sizeof(int32_t))) { 124 | return false; 125 | } 126 | memcpy(&val, buf, sizeof(int32_t)); 127 | val = ntohl(val); 128 | return true; 129 | } 130 | 131 | bool readI64(int64_t& val) { 132 | char* buf; 133 | if (!readBytes(&buf, sizeof(int64_t))) { 134 | return false; 135 | } 136 | memcpy(&val, buf, sizeof(int64_t)); 137 | val = ntohll(val); 138 | return true; 139 | } 140 | 141 | bool readDouble(double& val) { 142 | union { 143 | int64_t f; 144 | double t; 145 | } transfer; 146 | 147 | if (!readI64(transfer.f)) { 148 | return false; 149 | } 150 | val = transfer.t; 151 | return true; 152 | } 153 | 154 | int32_t readString(char** buf) { 155 | int32_t len = 0; 156 | if (!readI32(len) || !checkLengthLimit(len, stringLimit()) || !readBytes(buf, len)) { 157 | return -1; 158 | } 159 | return len; 160 | } 161 | 162 | int32_t readListBegin(TType& etype) { 163 | int32_t len; 164 | uint8_t b = 0; 165 | if (!readByte(b) || !readI32(len) || !checkLengthLimit(len, containerLimit())) { 166 | return -1; 167 | } 168 | etype = static_cast(b); 169 | return len; 170 | } 171 | 172 | int32_t readMapBegin(TType& ktype, TType& vtype) { 173 | int32_t len; 174 | uint8_t k, v; 175 | if (!readByte(k) || !readByte(v) || !readI32(len) || !checkLengthLimit(len, containerLimit())) { 176 | return -1; 177 | } 178 | ktype = static_cast(k); 179 | vtype = static_cast(v); 180 | return len; 181 | } 182 | 183 | bool readStructBegin() { return true; } 184 | bool readStructEnd() { return true; } 185 | 186 | bool readFieldBegin(TType& type, int16_t& tag); 187 | 188 | #define SKIPBYTES(n) \ 189 | do { \ 190 | if (!readBytes(&dummy_buf_, (n))) { \ 191 | return false; \ 192 | } \ 193 | return true; \ 194 | } while (0) 195 | 196 | bool skipBool() { SKIPBYTES(1); } 197 | bool skipByte() { SKIPBYTES(1); } 198 | bool skipI16() { SKIPBYTES(2); } 199 | bool skipI32() { SKIPBYTES(4); } 200 | bool skipI64() { SKIPBYTES(8); } 201 | bool skipDouble() { SKIPBYTES(8); } 202 | bool skipString() { 203 | int32_t len; 204 | if (!readI32(len)) { 205 | return false; 206 | } 207 | SKIPBYTES(len); 208 | } 209 | #undef SKIPBYTES 210 | 211 | private: 212 | char* dummy_buf_; 213 | }; 214 | } 215 | } 216 | } 217 | #endif // THRIFT_PY_BINARY_H 218 | -------------------------------------------------------------------------------- /function.py: -------------------------------------------------------------------------------- 1 | from thrift.transport.THttpClient import THttpClient 2 | from thrift.protocol.TCompactProtocol import TCompactProtocol 3 | from BEService import TalkService 4 | from BEService.ttypes import * 5 | import time, json, requests, os, random, ast, datetime, sys, concurrent.futures 6 | 7 | class BE_Team: 8 | def __init__(self, myToken, myApp, pool=False): 9 | self.lineServer = "https://ga2.line.naver.jp" 10 | self.thisHeaders = {} 11 | splited = myApp.split("\t") 12 | self.thisHeaders["x-line-access"] = myToken 13 | self.thisHeaders["x-line-application"] = myApp 14 | self.thisHeaders["x-lal"] = "en_id" 15 | if splited[0] == "ANDROIDLITE": 16 | self.thisHeaders["user-agent"] = 'LLA/{} Mi5 {}'.format(splited[1], splited[3]) 17 | elif splited[0] == "CHROMEOS": 18 | self.thisHeaders["user-agent"] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36' 19 | elif splited[0] in ["IOS", "IOSIPAD"]: 20 | self.thisHeaders["user-agent"] = 'Line/{} Iphone8 {}'.format(splited[1], splited[3]) 21 | else : 22 | self.thisHeaders["user-agent"] = 'Line/{}'.format(splited[1]) 23 | self.talk = self.openTransport("/S4") 24 | self.polling = self.openTransport("/P4") 25 | self.profile = self.getProfile() 26 | self.serverTime = self.getServerTime() 27 | self.localRev = -1 28 | self.globalRev = 0 29 | self.individualRev = 0 30 | print("[ Login ] Display Name: " + self.profile.displayName) 31 | print("[ Login ] Auth Token: " + myToken) 32 | 33 | def openTransport(self, endPoint): 34 | transport = THttpClient(self.lineServer + endPoint) 35 | transport.setCustomHeaders(self.thisHeaders) 36 | protocol = TCompactProtocol(transport) 37 | return TalkService.Client(protocol) 38 | 39 | def acceptChatInvitation(self, chatMid): 40 | return self.talk.acceptChatInvitation(AcceptChatInvitationRequest(0,chatMid)) 41 | 42 | def acceptChatInvitationByTicket(self, chatMid, ticketId): 43 | return self.talk.acceptChatInvitationByTicket(AcceptChatInvitationByTicketRequest(0,chatMid,ticketId)) 44 | 45 | def blockContact(self, mid): 46 | return self.talk.blockContact(0,mid) 47 | 48 | def cancelChatInvitation(self,chatMid, targetUserMids): 49 | return self.talk.cancelChatInvitation(CancelChatInvitationRequest(0,chatMid,targetUserMids)) 50 | 51 | def createChat(self, name, targetUserMids, picturePath=""): 52 | return self.talk.createChat(CreateChatRequest(0,0,name,targetUserMids,picturePath)) 53 | 54 | def deleteSelfFromChat(self, chatMid): 55 | return self.talk.deleteSelfFromChat(DeleteSelfFromChatRequest(0,chatMid,"","","","")) 56 | 57 | def deleteOtherFromChat(self, chatMid, targetUserMids): 58 | return self.talk.deleteOtherFromChat(DeleteOtherFromChatRequest(0,chatMid,targetUserMids)) 59 | 60 | def fetchOperations(self, deviceId, offsetFrom): 61 | return self.polling.fetchOperations(FetchOperationsRequest(deviceId,offsetFrom)) 62 | 63 | def fetchOps(self): 64 | return self.polling.fetchOps(self.localRev,15,self.globalRev,self.individualRev) 65 | 66 | def findAndAddContactsByMid(self, mid, reference=""): 67 | return self.talk.findAndAddContactsByMid(0,mid,0,reference) 68 | 69 | def findAndAddContactsByUserid(self, searchId, reference=""): 70 | return self.talk.findAndAddContactsByUserid(0,searchId,reference) 71 | 72 | def findContactByUserid(self, userid): 73 | return self.talk.findContactByUserid(userid) 74 | 75 | def findChatByTicket(self, ticketId): 76 | return self.talk.findChatByTicket(FindChatByTicketRequest(ticketId)) 77 | 78 | def getAllChatMids(self, withMemberChats=True, withInvitedChats=True, syncReason=0): 79 | return self.talk.getAllChatMids(GetAllChatMidsRequest(withMemberChats,withInvitedChats),syncReason) 80 | 81 | def getProfile(self, syncReason=0): 82 | return self.talk.getProfile(syncReason) 83 | 84 | def getContact(self, mid): 85 | return self.talk.getContact(mid) 86 | 87 | def getCountryWithRequestIp(self): 88 | return self.talk.getCountryWithRequestIp() 89 | 90 | def getServerTime(self): 91 | return self.talk.getServerTime() 92 | 93 | def getContacts(self, mids): 94 | return self.talk.getContacts(mids) 95 | 96 | def getAllContactIds(self, syncReason=0): 97 | return self.talk.getAllContactIds(syncReason) 98 | 99 | def getChats(self, chatMids, withMembers=True, withInvitees=True): 100 | return self.talk.getChats(GetChatsRequest(chatMids,withMembers,withInvitees)) 101 | 102 | def inviteIntoChat(self, chatMid, targetUserMids=[]): 103 | return self.talk.inviteIntoChat(InviteIntoChatRequest(0,chatMid,targetUserMids)) 104 | 105 | def reissueChatTicket(self, chatMid): 106 | return self.talk.reissueChatTicket(ReissueChatTicketRequest(0,chatMid)) 107 | 108 | def rejectChatInvitation(self, chatMid): 109 | return self.talk.rejectChatInvitation(RejectChatInvitationRequest(0,chatMid)) 110 | 111 | def sendMessage(self, to, text, contentMetadata={}, contentType=0): 112 | msg = Message() 113 | msg.to, msg._from = to, self.profile.mid 114 | msg.text = text 115 | msg.contentType, msg.contentMetadata = contentType, contentMetadata 116 | return self.talk.sendMessage(0,msg) 117 | 118 | def sendMessageReply(self, to, text, msgId): 119 | msg = Message() 120 | msg.to, msg._from = to, self.profile.mid 121 | msg.text = text 122 | msg.contentType, msg.contentMetadata = 0, {} 123 | msg.relatedMessageId = msgId 124 | msg.messageRelationType = 3 125 | msg.relatedMessageServiceCode = 1 126 | return self.talk.sendMessage(0,msg) 127 | 128 | def sendMention(self, to, mid, text): 129 | mentiones = '{"S":"0","E":"3","M":'+json.dumps(mid)+'}' 130 | text_ = '@x {}'.format(text) 131 | return self.sendMessage(to, text_, contentMetadata={'MENTION':'{"MENTIONEES":['+mentiones+']}'}, contentType=0) 132 | 133 | def unsendMessage(self, messageId): 134 | return self.talk.unsendMessage(0,messageId) 135 | 136 | def updateChat(self, chat, updatedAttribute): 137 | return self.talk.updateChat(UpdateChatRequest(0,chat,updatedAttribute)) 138 | 139 | def updateProfileAttribute(self, attr, value): 140 | return self.talk.updateProfileAttribute(0,attr,value) 141 | 142 | -------------------------------------------------------------------------------- /other version/C++/TalkService_server.skeleton.cpp: -------------------------------------------------------------------------------- 1 | // This autogenerated skeleton file illustrates how to build a server. 2 | // You should copy it to another filename to avoid overwriting it. 3 | 4 | #include "TalkService.h" 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | using namespace ::apache::thrift; 11 | using namespace ::apache::thrift::protocol; 12 | using namespace ::apache::thrift::transport; 13 | using namespace ::apache::thrift::server; 14 | 15 | class TalkServiceHandler : virtual public TalkServiceIf { 16 | public: 17 | TalkServiceHandler() { 18 | // Your initialization goes here 19 | } 20 | 21 | void acceptChatInvitation(AcceptChatInvitationResponse& _return, const AcceptChatInvitationRequest& request) { 22 | // Your implementation goes here 23 | printf("acceptChatInvitation\n"); 24 | } 25 | 26 | void acceptChatInvitationByTicket(AcceptChatInvitationByTicketResponse& _return, const AcceptChatInvitationByTicketRequest& request) { 27 | // Your implementation goes here 28 | printf("acceptChatInvitationByTicket\n"); 29 | } 30 | 31 | void blockContact(const int32_t reqSeq, const std::string& id) { 32 | // Your implementation goes here 33 | printf("blockContact\n"); 34 | } 35 | 36 | void cancelChatInvitation(CancelChatInvitationResponse& _return, const CancelChatInvitationRequest& request) { 37 | // Your implementation goes here 38 | printf("cancelChatInvitation\n"); 39 | } 40 | 41 | void createChat(CreateChatResponse& _return, const CreateChatRequest& request) { 42 | // Your implementation goes here 43 | printf("createChat\n"); 44 | } 45 | 46 | void deleteSelfFromChat(DeleteSelfFromChatResponse& _return, const DeleteSelfFromChatRequest& request) { 47 | // Your implementation goes here 48 | printf("deleteSelfFromChat\n"); 49 | } 50 | 51 | void deleteOtherFromChat(DeleteOtherFromChatResponse& _return, const DeleteOtherFromChatRequest& request) { 52 | // Your implementation goes here 53 | printf("deleteOtherFromChat\n"); 54 | } 55 | 56 | void fetchOperations(FetchOperationsResponse& _return, const FetchOperationsRequest& request) { 57 | // Your implementation goes here 58 | printf("fetchOperations\n"); 59 | } 60 | 61 | void fetchOps(std::vector & _return, const int64_t localRev, const int32_t count, const int64_t globalRev, const int64_t individualRev) { 62 | // Your implementation goes here 63 | printf("fetchOps\n"); 64 | } 65 | 66 | void findAndAddContactsByMid(std::map & _return, const int32_t reqSeq, const std::string& mid, const ContactType::type type, const std::string& reference) { 67 | // Your implementation goes here 68 | printf("findAndAddContactsByMid\n"); 69 | } 70 | 71 | void findAndAddContactsByUserid(std::map & _return, const int32_t reqSeq, const std::string& searchId, const std::string& reference) { 72 | // Your implementation goes here 73 | printf("findAndAddContactsByUserid\n"); 74 | } 75 | 76 | void findContactByUserid(Contact& _return, const std::string& userid) { 77 | // Your implementation goes here 78 | printf("findContactByUserid\n"); 79 | } 80 | 81 | void findChatByTicket(FindChatByTicketResponse& _return, const FindChatByTicketRequest& request) { 82 | // Your implementation goes here 83 | printf("findChatByTicket\n"); 84 | } 85 | 86 | void getAllChatMids(GetAllChatMidsResponse& _return, const GetAllChatMidsRequest& request, const SyncReason::type syncReason) { 87 | // Your implementation goes here 88 | printf("getAllChatMids\n"); 89 | } 90 | 91 | void getProfile(Profile& _return, const SyncReason::type syncReason) { 92 | // Your implementation goes here 93 | printf("getProfile\n"); 94 | } 95 | 96 | void getContact(Contact& _return, const std::string& id) { 97 | // Your implementation goes here 98 | printf("getContact\n"); 99 | } 100 | 101 | void getCountryWithRequestIp(std::string& _return) { 102 | // Your implementation goes here 103 | printf("getCountryWithRequestIp\n"); 104 | } 105 | 106 | int64_t getServerTime() { 107 | // Your implementation goes here 108 | printf("getServerTime\n"); 109 | } 110 | 111 | void getContacts(std::vector & _return, const std::vector & ids) { 112 | // Your implementation goes here 113 | printf("getContacts\n"); 114 | } 115 | 116 | void getAllContactIds(std::vector & _return, const SyncReason::type syncReason) { 117 | // Your implementation goes here 118 | printf("getAllContactIds\n"); 119 | } 120 | 121 | void getChats(GetChatsResponse& _return, const GetChatsRequest& request) { 122 | // Your implementation goes here 123 | printf("getChats\n"); 124 | } 125 | 126 | void inviteIntoChat(InviteIntoChatResponse& _return, const InviteIntoChatRequest& request) { 127 | // Your implementation goes here 128 | printf("inviteIntoChat\n"); 129 | } 130 | 131 | void reissueChatTicket(ReissueChatTicketResponse& _return, const ReissueChatTicketRequest& request) { 132 | // Your implementation goes here 133 | printf("reissueChatTicket\n"); 134 | } 135 | 136 | void rejectChatInvitation(RejectChatInvitationResponse& _return, const RejectChatInvitationRequest& request) { 137 | // Your implementation goes here 138 | printf("rejectChatInvitation\n"); 139 | } 140 | 141 | void sendMessage(Message& _return, const int32_t seq, const Message& message) { 142 | // Your implementation goes here 143 | printf("sendMessage\n"); 144 | } 145 | 146 | void unsendMessage(const int32_t seq, const std::string& messageId) { 147 | // Your implementation goes here 148 | printf("unsendMessage\n"); 149 | } 150 | 151 | void updateChat(UpdateChatResponse& _return, const UpdateChatRequest& request) { 152 | // Your implementation goes here 153 | printf("updateChat\n"); 154 | } 155 | 156 | void updateProfileAttribute(const int32_t reqSeq, const ProfileAttribute::type attr, const std::string& value) { 157 | // Your implementation goes here 158 | printf("updateProfileAttribute\n"); 159 | } 160 | 161 | }; 162 | 163 | int main(int argc, char **argv) { 164 | int port = 9090; 165 | ::std::shared_ptr handler(new TalkServiceHandler()); 166 | ::std::shared_ptr processor(new TalkServiceProcessor(handler)); 167 | ::std::shared_ptr serverTransport(new TServerSocket(port)); 168 | ::std::shared_ptr transportFactory(new TBufferedTransportFactory()); 169 | ::std::shared_ptr protocolFactory(new TBinaryProtocolFactory()); 170 | 171 | TSimpleServer server(processor, serverTransport, transportFactory, protocolFactory); 172 | server.serve(); 173 | return 0; 174 | } 175 | 176 | -------------------------------------------------------------------------------- /thrift/ext/module.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | #include 21 | #include "types.h" 22 | #include "binary.h" 23 | #include "compact.h" 24 | #include 25 | #include 26 | 27 | // TODO(dreiss): defval appears to be unused. Look into removing it. 28 | // TODO(dreiss): Make parse_spec_args recursive, and cache the output 29 | // permanently in the object. (Malloc and orphan.) 30 | // TODO(dreiss): Why do we need cStringIO for reading, why not just char*? 31 | // Can cStringIO let us work with a BufferedTransport? 32 | // TODO(dreiss): Don't ignore the rv from cwrite (maybe). 33 | 34 | // Doing a benchmark shows that interning actually makes a difference, amazingly. 35 | 36 | /** Pointer to interned string to speed up attribute lookup. */ 37 | PyObject* INTERN_STRING(TFrozenDict); 38 | PyObject* INTERN_STRING(cstringio_buf); 39 | PyObject* INTERN_STRING(cstringio_refill); 40 | static PyObject* INTERN_STRING(string_length_limit); 41 | static PyObject* INTERN_STRING(container_length_limit); 42 | static PyObject* INTERN_STRING(trans); 43 | 44 | namespace apache { 45 | namespace thrift { 46 | namespace py { 47 | 48 | template 49 | static PyObject* encode_impl(PyObject* args) { 50 | if (!args) 51 | return NULL; 52 | 53 | PyObject* enc_obj = NULL; 54 | PyObject* type_args = NULL; 55 | if (!PyArg_ParseTuple(args, "OO", &enc_obj, &type_args)) { 56 | return NULL; 57 | } 58 | if (!enc_obj || !type_args) { 59 | return NULL; 60 | } 61 | 62 | T protocol; 63 | if (!protocol.prepareEncodeBuffer() || !protocol.encodeValue(enc_obj, T_STRUCT, type_args)) { 64 | return NULL; 65 | } 66 | 67 | return protocol.getEncodedValue(); 68 | } 69 | 70 | static inline long as_long_then_delete(PyObject* value, long default_value) { 71 | ScopedPyObject scope(value); 72 | long v = PyInt_AsLong(value); 73 | if (INT_CONV_ERROR_OCCURRED(v)) { 74 | PyErr_Clear(); 75 | return default_value; 76 | } 77 | return v; 78 | } 79 | 80 | template 81 | static PyObject* decode_impl(PyObject* args) { 82 | PyObject* output_obj = NULL; 83 | PyObject* oprot = NULL; 84 | PyObject* typeargs = NULL; 85 | if (!PyArg_ParseTuple(args, "OOO", &output_obj, &oprot, &typeargs)) { 86 | return NULL; 87 | } 88 | 89 | T protocol; 90 | int32_t default_limit = (std::numeric_limits::max)(); 91 | protocol.setStringLengthLimit( 92 | as_long_then_delete(PyObject_GetAttr(oprot, INTERN_STRING(string_length_limit)), 93 | default_limit)); 94 | protocol.setContainerLengthLimit( 95 | as_long_then_delete(PyObject_GetAttr(oprot, INTERN_STRING(container_length_limit)), 96 | default_limit)); 97 | ScopedPyObject transport(PyObject_GetAttr(oprot, INTERN_STRING(trans))); 98 | if (!transport) { 99 | return NULL; 100 | } 101 | 102 | StructTypeArgs parsedargs; 103 | if (!parse_struct_args(&parsedargs, typeargs)) { 104 | return NULL; 105 | } 106 | 107 | if (!protocol.prepareDecodeBufferFromTransport(transport.get())) { 108 | return NULL; 109 | } 110 | 111 | return protocol.readStruct(output_obj, parsedargs.klass, parsedargs.spec); 112 | } 113 | } 114 | } 115 | } 116 | 117 | using namespace apache::thrift::py; 118 | 119 | /* -- PYTHON MODULE SETUP STUFF --- */ 120 | 121 | extern "C" { 122 | 123 | static PyObject* encode_binary(PyObject*, PyObject* args) { 124 | return encode_impl(args); 125 | } 126 | 127 | static PyObject* decode_binary(PyObject*, PyObject* args) { 128 | return decode_impl(args); 129 | } 130 | 131 | static PyObject* encode_compact(PyObject*, PyObject* args) { 132 | return encode_impl(args); 133 | } 134 | 135 | static PyObject* decode_compact(PyObject*, PyObject* args) { 136 | return decode_impl(args); 137 | } 138 | 139 | static PyMethodDef ThriftFastBinaryMethods[] = { 140 | {"encode_binary", encode_binary, METH_VARARGS, ""}, 141 | {"decode_binary", decode_binary, METH_VARARGS, ""}, 142 | {"encode_compact", encode_compact, METH_VARARGS, ""}, 143 | {"decode_compact", decode_compact, METH_VARARGS, ""}, 144 | {NULL, NULL, 0, NULL} /* Sentinel */ 145 | }; 146 | 147 | #if PY_MAJOR_VERSION >= 3 148 | 149 | static struct PyModuleDef ThriftFastBinaryDef = {PyModuleDef_HEAD_INIT, 150 | "thrift.protocol.fastbinary", 151 | NULL, 152 | 0, 153 | ThriftFastBinaryMethods, 154 | NULL, 155 | NULL, 156 | NULL, 157 | NULL}; 158 | 159 | #define INITERROR return NULL; 160 | 161 | PyObject* PyInit_fastbinary() { 162 | 163 | #else 164 | 165 | #define INITERROR return; 166 | 167 | void initfastbinary() { 168 | 169 | PycString_IMPORT; 170 | if (PycStringIO == NULL) 171 | INITERROR 172 | 173 | #endif 174 | 175 | #define INIT_INTERN_STRING(value) \ 176 | do { \ 177 | INTERN_STRING(value) = PyString_InternFromString(#value); \ 178 | if (!INTERN_STRING(value)) \ 179 | INITERROR \ 180 | } while (0) 181 | 182 | INIT_INTERN_STRING(TFrozenDict); 183 | INIT_INTERN_STRING(cstringio_buf); 184 | INIT_INTERN_STRING(cstringio_refill); 185 | INIT_INTERN_STRING(string_length_limit); 186 | INIT_INTERN_STRING(container_length_limit); 187 | INIT_INTERN_STRING(trans); 188 | #undef INIT_INTERN_STRING 189 | 190 | PyObject* module = 191 | #if PY_MAJOR_VERSION >= 3 192 | PyModule_Create(&ThriftFastBinaryDef); 193 | #else 194 | Py_InitModule("thrift.protocol.fastbinary", ThriftFastBinaryMethods); 195 | #endif 196 | if (module == NULL) 197 | INITERROR; 198 | 199 | #if PY_MAJOR_VERSION >= 3 200 | return module; 201 | #endif 202 | } 203 | } 204 | -------------------------------------------------------------------------------- /thrift/TTornado.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | from __future__ import absolute_import 21 | import logging 22 | import socket 23 | import struct 24 | 25 | from .transport.TTransport import TTransportException, TTransportBase, TMemoryBuffer 26 | 27 | from io import BytesIO 28 | from collections import deque 29 | from contextlib import contextmanager 30 | from tornado import gen, iostream, ioloop, tcpserver, concurrent 31 | 32 | __all__ = ['TTornadoServer', 'TTornadoStreamTransport'] 33 | 34 | logger = logging.getLogger(__name__) 35 | 36 | 37 | class _Lock(object): 38 | def __init__(self): 39 | self._waiters = deque() 40 | 41 | def acquired(self): 42 | return len(self._waiters) > 0 43 | 44 | @gen.coroutine 45 | def acquire(self): 46 | blocker = self._waiters[-1] if self.acquired() else None 47 | future = concurrent.Future() 48 | self._waiters.append(future) 49 | if blocker: 50 | yield blocker 51 | 52 | raise gen.Return(self._lock_context()) 53 | 54 | def release(self): 55 | assert self.acquired(), 'Lock not aquired' 56 | future = self._waiters.popleft() 57 | future.set_result(None) 58 | 59 | @contextmanager 60 | def _lock_context(self): 61 | try: 62 | yield 63 | finally: 64 | self.release() 65 | 66 | 67 | class TTornadoStreamTransport(TTransportBase): 68 | """a framed, buffered transport over a Tornado stream""" 69 | def __init__(self, host, port, stream=None, io_loop=None): 70 | self.host = host 71 | self.port = port 72 | self.io_loop = io_loop or ioloop.IOLoop.current() 73 | self.__wbuf = BytesIO() 74 | self._read_lock = _Lock() 75 | 76 | # servers provide a ready-to-go stream 77 | self.stream = stream 78 | 79 | def with_timeout(self, timeout, future): 80 | return gen.with_timeout(timeout, future, self.io_loop) 81 | 82 | @gen.coroutine 83 | def open(self, timeout=None): 84 | logger.debug('socket connecting') 85 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) 86 | self.stream = iostream.IOStream(sock) 87 | 88 | try: 89 | connect = self.stream.connect((self.host, self.port)) 90 | if timeout is not None: 91 | yield self.with_timeout(timeout, connect) 92 | else: 93 | yield connect 94 | except (socket.error, IOError, ioloop.TimeoutError) as e: 95 | message = 'could not connect to {}:{} ({})'.format(self.host, self.port, e) 96 | raise TTransportException( 97 | type=TTransportException.NOT_OPEN, 98 | message=message) 99 | 100 | raise gen.Return(self) 101 | 102 | def set_close_callback(self, callback): 103 | """ 104 | Should be called only after open() returns 105 | """ 106 | self.stream.set_close_callback(callback) 107 | 108 | def close(self): 109 | # don't raise if we intend to close 110 | self.stream.set_close_callback(None) 111 | self.stream.close() 112 | 113 | def read(self, _): 114 | # The generated code for Tornado shouldn't do individual reads -- only 115 | # frames at a time 116 | assert False, "you're doing it wrong" 117 | 118 | @contextmanager 119 | def io_exception_context(self): 120 | try: 121 | yield 122 | except (socket.error, IOError) as e: 123 | raise TTransportException( 124 | type=TTransportException.END_OF_FILE, 125 | message=str(e)) 126 | except iostream.StreamBufferFullError as e: 127 | raise TTransportException( 128 | type=TTransportException.UNKNOWN, 129 | message=str(e)) 130 | 131 | @gen.coroutine 132 | def readFrame(self): 133 | # IOStream processes reads one at a time 134 | with (yield self._read_lock.acquire()): 135 | with self.io_exception_context(): 136 | frame_header = yield self.stream.read_bytes(4) 137 | if len(frame_header) == 0: 138 | raise iostream.StreamClosedError('Read zero bytes from stream') 139 | frame_length, = struct.unpack('!i', frame_header) 140 | frame = yield self.stream.read_bytes(frame_length) 141 | raise gen.Return(frame) 142 | 143 | def write(self, buf): 144 | self.__wbuf.write(buf) 145 | 146 | def flush(self): 147 | frame = self.__wbuf.getvalue() 148 | # reset wbuf before write/flush to preserve state on underlying failure 149 | frame_length = struct.pack('!i', len(frame)) 150 | self.__wbuf = BytesIO() 151 | with self.io_exception_context(): 152 | return self.stream.write(frame_length + frame) 153 | 154 | 155 | class TTornadoServer(tcpserver.TCPServer): 156 | def __init__(self, processor, iprot_factory, oprot_factory=None, 157 | *args, **kwargs): 158 | super(TTornadoServer, self).__init__(*args, **kwargs) 159 | 160 | self._processor = processor 161 | self._iprot_factory = iprot_factory 162 | self._oprot_factory = (oprot_factory if oprot_factory is not None 163 | else iprot_factory) 164 | 165 | @gen.coroutine 166 | def handle_stream(self, stream, address): 167 | host, port = address[:2] 168 | trans = TTornadoStreamTransport(host=host, port=port, stream=stream, 169 | io_loop=self.io_loop) 170 | oprot = self._oprot_factory.getProtocol(trans) 171 | 172 | try: 173 | while not trans.stream.closed(): 174 | try: 175 | frame = yield trans.readFrame() 176 | except TTransportException as e: 177 | if e.type == TTransportException.END_OF_FILE: 178 | break 179 | else: 180 | raise 181 | tr = TMemoryBuffer(frame) 182 | iprot = self._iprot_factory.getProtocol(tr) 183 | yield self._processor.process(iprot, oprot) 184 | except Exception: 185 | logger.exception('thrift exception in handle_stream') 186 | trans.close() 187 | 188 | logger.info('client disconnected %s:%d', host, port) 189 | -------------------------------------------------------------------------------- /thrift/protocol/THeaderProtocol.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | from thrift.protocol.TBinaryProtocol import TBinaryProtocolAccelerated 21 | from thrift.protocol.TCompactProtocol import TCompactProtocolAccelerated 22 | from thrift.protocol.TProtocol import TProtocolBase, TProtocolException, TProtocolFactory 23 | from thrift.Thrift import TApplicationException, TMessageType 24 | from thrift.transport.THeaderTransport import THeaderTransport, THeaderSubprotocolID, THeaderClientType 25 | 26 | 27 | PROTOCOLS_BY_ID = { 28 | THeaderSubprotocolID.BINARY: TBinaryProtocolAccelerated, 29 | THeaderSubprotocolID.COMPACT: TCompactProtocolAccelerated, 30 | } 31 | 32 | 33 | class THeaderProtocol(TProtocolBase): 34 | """A framed protocol with headers and payload transforms. 35 | 36 | THeaderProtocol frames other Thrift protocols and adds support for optional 37 | out-of-band headers. The currently supported subprotocols are 38 | TBinaryProtocol and TCompactProtocol. 39 | 40 | It's also possible to apply transforms to the encoded message payload. The 41 | only transform currently supported is to gzip. 42 | 43 | When used in a server, THeaderProtocol can accept messages from 44 | non-THeaderProtocol clients if allowed (see `allowed_client_types`). This 45 | includes framed and unframed transports and both TBinaryProtocol and 46 | TCompactProtocol. The server will respond in the appropriate dialect for 47 | the connected client. HTTP clients are not currently supported. 48 | 49 | THeaderProtocol does not currently support THTTPServer, TNonblockingServer, 50 | or TProcessPoolServer. 51 | 52 | See doc/specs/HeaderFormat.md for details of the wire format. 53 | 54 | """ 55 | 56 | def __init__(self, transport, allowed_client_types): 57 | # much of the actual work for THeaderProtocol happens down in 58 | # THeaderTransport since we need to do low-level shenanigans to detect 59 | # if the client is sending us headers or one of the headerless formats 60 | # we support. this wraps the real transport with the one that does all 61 | # the magic. 62 | if not isinstance(transport, THeaderTransport): 63 | transport = THeaderTransport(transport, allowed_client_types) 64 | super(THeaderProtocol, self).__init__(transport) 65 | self._set_protocol() 66 | 67 | def get_headers(self): 68 | return self.trans.get_headers() 69 | 70 | def set_header(self, key, value): 71 | self.trans.set_header(key, value) 72 | 73 | def clear_headers(self): 74 | self.trans.clear_headers() 75 | 76 | def add_transform(self, transform_id): 77 | self.trans.add_transform(transform_id) 78 | 79 | def writeMessageBegin(self, name, ttype, seqid): 80 | self.trans.sequence_id = seqid 81 | return self._protocol.writeMessageBegin(name, ttype, seqid) 82 | 83 | def writeMessageEnd(self): 84 | return self._protocol.writeMessageEnd() 85 | 86 | def writeStructBegin(self, name): 87 | return self._protocol.writeStructBegin(name) 88 | 89 | def writeStructEnd(self): 90 | return self._protocol.writeStructEnd() 91 | 92 | def writeFieldBegin(self, name, ttype, fid): 93 | return self._protocol.writeFieldBegin(name, ttype, fid) 94 | 95 | def writeFieldEnd(self): 96 | return self._protocol.writeFieldEnd() 97 | 98 | def writeFieldStop(self): 99 | return self._protocol.writeFieldStop() 100 | 101 | def writeMapBegin(self, ktype, vtype, size): 102 | return self._protocol.writeMapBegin(ktype, vtype, size) 103 | 104 | def writeMapEnd(self): 105 | return self._protocol.writeMapEnd() 106 | 107 | def writeListBegin(self, etype, size): 108 | return self._protocol.writeListBegin(etype, size) 109 | 110 | def writeListEnd(self): 111 | return self._protocol.writeListEnd() 112 | 113 | def writeSetBegin(self, etype, size): 114 | return self._protocol.writeSetBegin(etype, size) 115 | 116 | def writeSetEnd(self): 117 | return self._protocol.writeSetEnd() 118 | 119 | def writeBool(self, bool_val): 120 | return self._protocol.writeBool(bool_val) 121 | 122 | def writeByte(self, byte): 123 | return self._protocol.writeByte(byte) 124 | 125 | def writeI16(self, i16): 126 | return self._protocol.writeI16(i16) 127 | 128 | def writeI32(self, i32): 129 | return self._protocol.writeI32(i32) 130 | 131 | def writeI64(self, i64): 132 | return self._protocol.writeI64(i64) 133 | 134 | def writeDouble(self, dub): 135 | return self._protocol.writeDouble(dub) 136 | 137 | def writeBinary(self, str_val): 138 | return self._protocol.writeBinary(str_val) 139 | 140 | def _set_protocol(self): 141 | try: 142 | protocol_cls = PROTOCOLS_BY_ID[self.trans.protocol_id] 143 | except KeyError: 144 | raise TApplicationException( 145 | TProtocolException.INVALID_PROTOCOL, 146 | "Unknown protocol requested.", 147 | ) 148 | 149 | self._protocol = protocol_cls(self.trans) 150 | self._fast_encode = self._protocol._fast_encode 151 | self._fast_decode = self._protocol._fast_decode 152 | 153 | def readMessageBegin(self): 154 | try: 155 | self.trans.readFrame(0) 156 | self._set_protocol() 157 | except TApplicationException as exc: 158 | self._protocol.writeMessageBegin(b"", TMessageType.EXCEPTION, 0) 159 | exc.write(self._protocol) 160 | self._protocol.writeMessageEnd() 161 | self.trans.flush() 162 | 163 | return self._protocol.readMessageBegin() 164 | 165 | def readMessageEnd(self): 166 | return self._protocol.readMessageEnd() 167 | 168 | def readStructBegin(self): 169 | return self._protocol.readStructBegin() 170 | 171 | def readStructEnd(self): 172 | return self._protocol.readStructEnd() 173 | 174 | def readFieldBegin(self): 175 | return self._protocol.readFieldBegin() 176 | 177 | def readFieldEnd(self): 178 | return self._protocol.readFieldEnd() 179 | 180 | def readMapBegin(self): 181 | return self._protocol.readMapBegin() 182 | 183 | def readMapEnd(self): 184 | return self._protocol.readMapEnd() 185 | 186 | def readListBegin(self): 187 | return self._protocol.readListBegin() 188 | 189 | def readListEnd(self): 190 | return self._protocol.readListEnd() 191 | 192 | def readSetBegin(self): 193 | return self._protocol.readSetBegin() 194 | 195 | def readSetEnd(self): 196 | return self._protocol.readSetEnd() 197 | 198 | def readBool(self): 199 | return self._protocol.readBool() 200 | 201 | def readByte(self): 202 | return self._protocol.readByte() 203 | 204 | def readI16(self): 205 | return self._protocol.readI16() 206 | 207 | def readI32(self): 208 | return self._protocol.readI32() 209 | 210 | def readI64(self): 211 | return self._protocol.readI64() 212 | 213 | def readDouble(self): 214 | return self._protocol.readDouble() 215 | 216 | def readBinary(self): 217 | return self._protocol.readBinary() 218 | 219 | 220 | class THeaderProtocolFactory(TProtocolFactory): 221 | def __init__(self, allowed_client_types=(THeaderClientType.HEADERS,)): 222 | self.allowed_client_types = allowed_client_types 223 | 224 | def getProtocol(self, trans): 225 | return THeaderProtocol(trans, self.allowed_client_types) 226 | -------------------------------------------------------------------------------- /thrift/transport/TSocket.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | import errno 21 | import logging 22 | import os 23 | import socket 24 | import sys 25 | 26 | from .TTransport import TTransportBase, TTransportException, TServerTransportBase 27 | 28 | logger = logging.getLogger(__name__) 29 | 30 | 31 | class TSocketBase(TTransportBase): 32 | def _resolveAddr(self): 33 | if self._unix_socket is not None: 34 | return [(socket.AF_UNIX, socket.SOCK_STREAM, None, None, 35 | self._unix_socket)] 36 | else: 37 | return socket.getaddrinfo(self.host, 38 | self.port, 39 | self._socket_family, 40 | socket.SOCK_STREAM, 41 | 0, 42 | socket.AI_PASSIVE | socket.AI_ADDRCONFIG) 43 | 44 | def close(self): 45 | if self.handle: 46 | self.handle.close() 47 | self.handle = None 48 | 49 | 50 | class TSocket(TSocketBase): 51 | """Socket implementation of TTransport base.""" 52 | 53 | def __init__(self, host='localhost', port=9090, unix_socket=None, 54 | socket_family=socket.AF_UNSPEC, 55 | socket_keepalive=False): 56 | """Initialize a TSocket 57 | 58 | @param host(str) The host to connect to. 59 | @param port(int) The (TCP) port to connect to. 60 | @param unix_socket(str) The filename of a unix socket to connect to. 61 | (host and port will be ignored.) 62 | @param socket_family(int) The socket family to use with this socket. 63 | @param socket_keepalive(bool) enable TCP keepalive, default off. 64 | """ 65 | self.host = host 66 | self.port = port 67 | self.handle = None 68 | self._unix_socket = unix_socket 69 | self._timeout = None 70 | self._socket_family = socket_family 71 | self._socket_keepalive = socket_keepalive 72 | 73 | def setHandle(self, h): 74 | self.handle = h 75 | 76 | def isOpen(self): 77 | return self.handle is not None 78 | 79 | def setTimeout(self, ms): 80 | if ms is None: 81 | self._timeout = None 82 | else: 83 | self._timeout = ms / 1000.0 84 | 85 | if self.handle is not None: 86 | self.handle.settimeout(self._timeout) 87 | 88 | def _do_open(self, family, socktype): 89 | return socket.socket(family, socktype) 90 | 91 | @property 92 | def _address(self): 93 | return self._unix_socket if self._unix_socket else '%s:%d' % (self.host, self.port) 94 | 95 | def open(self): 96 | if self.handle: 97 | raise TTransportException(type=TTransportException.ALREADY_OPEN, message="already open") 98 | try: 99 | addrs = self._resolveAddr() 100 | except socket.gaierror as gai: 101 | msg = 'failed to resolve sockaddr for ' + str(self._address) 102 | logger.exception(msg) 103 | raise TTransportException(type=TTransportException.NOT_OPEN, message=msg, inner=gai) 104 | for family, socktype, _, _, sockaddr in addrs: 105 | handle = self._do_open(family, socktype) 106 | 107 | # TCP_KEEPALIVE 108 | if self._socket_keepalive: 109 | handle.setsockopt(socket.IPPROTO_TCP, socket.SO_KEEPALIVE, 1) 110 | 111 | handle.settimeout(self._timeout) 112 | try: 113 | handle.connect(sockaddr) 114 | self.handle = handle 115 | return 116 | except socket.error: 117 | handle.close() 118 | logger.info('Could not connect to %s', sockaddr, exc_info=True) 119 | msg = 'Could not connect to any of %s' % list(map(lambda a: a[4], 120 | addrs)) 121 | logger.error(msg) 122 | raise TTransportException(type=TTransportException.NOT_OPEN, message=msg) 123 | 124 | def read(self, sz): 125 | try: 126 | buff = self.handle.recv(sz) 127 | except socket.error as e: 128 | if (e.args[0] == errno.ECONNRESET and 129 | (sys.platform == 'darwin' or sys.platform.startswith('freebsd'))): 130 | # freebsd and Mach don't follow POSIX semantic of recv 131 | # and fail with ECONNRESET if peer performed shutdown. 132 | # See corresponding comment and code in TSocket::read() 133 | # in lib/cpp/src/transport/TSocket.cpp. 134 | self.close() 135 | # Trigger the check to raise the END_OF_FILE exception below. 136 | buff = '' 137 | elif e.args[0] == errno.ETIMEDOUT: 138 | raise TTransportException(type=TTransportException.TIMED_OUT, message="read timeout", inner=e) 139 | else: 140 | raise TTransportException(message="unexpected exception", inner=e) 141 | if len(buff) == 0: 142 | raise TTransportException(type=TTransportException.END_OF_FILE, 143 | message='TSocket read 0 bytes') 144 | return buff 145 | 146 | def write(self, buff): 147 | if not self.handle: 148 | raise TTransportException(type=TTransportException.NOT_OPEN, 149 | message='Transport not open') 150 | sent = 0 151 | have = len(buff) 152 | while sent < have: 153 | try: 154 | plus = self.handle.send(buff) 155 | if plus == 0: 156 | raise TTransportException(type=TTransportException.END_OF_FILE, 157 | message='TSocket sent 0 bytes') 158 | sent += plus 159 | buff = buff[plus:] 160 | except socket.error as e: 161 | raise TTransportException(message="unexpected exception", inner=e) 162 | 163 | def flush(self): 164 | pass 165 | 166 | 167 | class TServerSocket(TSocketBase, TServerTransportBase): 168 | """Socket implementation of TServerTransport base.""" 169 | 170 | def __init__(self, host=None, port=9090, unix_socket=None, socket_family=socket.AF_UNSPEC): 171 | self.host = host 172 | self.port = port 173 | self._unix_socket = unix_socket 174 | self._socket_family = socket_family 175 | self.handle = None 176 | self._backlog = 128 177 | 178 | def setBacklog(self, backlog=None): 179 | if not self.handle: 180 | self._backlog = backlog 181 | else: 182 | # We cann't update backlog when it is already listening, since the 183 | # handle has been created. 184 | logger.warn('You have to set backlog before listen.') 185 | 186 | def listen(self): 187 | res0 = self._resolveAddr() 188 | socket_family = self._socket_family == socket.AF_UNSPEC and socket.AF_INET6 or self._socket_family 189 | for res in res0: 190 | if res[0] is socket_family or res is res0[-1]: 191 | break 192 | 193 | # We need remove the old unix socket if the file exists and 194 | # nobody is listening on it. 195 | if self._unix_socket: 196 | tmp = socket.socket(res[0], res[1]) 197 | try: 198 | tmp.connect(res[4]) 199 | except socket.error as err: 200 | eno, message = err.args 201 | if eno == errno.ECONNREFUSED: 202 | os.unlink(res[4]) 203 | 204 | self.handle = socket.socket(res[0], res[1]) 205 | self.handle.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 206 | if hasattr(self.handle, 'settimeout'): 207 | self.handle.settimeout(None) 208 | self.handle.bind(res[4]) 209 | self.handle.listen(self._backlog) 210 | 211 | def accept(self): 212 | client, addr = self.handle.accept() 213 | result = TSocket() 214 | result.setHandle(client) 215 | return result 216 | -------------------------------------------------------------------------------- /thrift/transport/TZlibTransport.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | """TZlibTransport provides a compressed transport and transport factory 21 | class, using the python standard library zlib module to implement 22 | data compression. 23 | """ 24 | 25 | from __future__ import division 26 | import zlib 27 | from .TTransport import TTransportBase, CReadableTransport 28 | from ..compat import BufferIO 29 | 30 | 31 | class TZlibTransportFactory(object): 32 | """Factory transport that builds zlib compressed transports. 33 | 34 | This factory caches the last single client/transport that it was passed 35 | and returns the same TZlibTransport object that was created. 36 | 37 | This caching means the TServer class will get the _same_ transport 38 | object for both input and output transports from this factory. 39 | (For non-threaded scenarios only, since the cache only holds one object) 40 | 41 | The purpose of this caching is to allocate only one TZlibTransport where 42 | only one is really needed (since it must have separate read/write buffers), 43 | and makes the statistics from getCompSavings() and getCompRatio() 44 | easier to understand. 45 | """ 46 | # class scoped cache of last transport given and zlibtransport returned 47 | _last_trans = None 48 | _last_z = None 49 | 50 | def getTransport(self, trans, compresslevel=9): 51 | """Wrap a transport, trans, with the TZlibTransport 52 | compressed transport class, returning a new 53 | transport to the caller. 54 | 55 | @param compresslevel: The zlib compression level, ranging 56 | from 0 (no compression) to 9 (best compression). Defaults to 9. 57 | @type compresslevel: int 58 | 59 | This method returns a TZlibTransport which wraps the 60 | passed C{trans} TTransport derived instance. 61 | """ 62 | if trans == self._last_trans: 63 | return self._last_z 64 | ztrans = TZlibTransport(trans, compresslevel) 65 | self._last_trans = trans 66 | self._last_z = ztrans 67 | return ztrans 68 | 69 | 70 | class TZlibTransport(TTransportBase, CReadableTransport): 71 | """Class that wraps a transport with zlib, compressing writes 72 | and decompresses reads, using the python standard 73 | library zlib module. 74 | """ 75 | # Read buffer size for the python fastbinary C extension, 76 | # the TBinaryProtocolAccelerated class. 77 | DEFAULT_BUFFSIZE = 4096 78 | 79 | def __init__(self, trans, compresslevel=9): 80 | """Create a new TZlibTransport, wrapping C{trans}, another 81 | TTransport derived object. 82 | 83 | @param trans: A thrift transport object, i.e. a TSocket() object. 84 | @type trans: TTransport 85 | @param compresslevel: The zlib compression level, ranging 86 | from 0 (no compression) to 9 (best compression). Default is 9. 87 | @type compresslevel: int 88 | """ 89 | self.__trans = trans 90 | self.compresslevel = compresslevel 91 | self.__rbuf = BufferIO() 92 | self.__wbuf = BufferIO() 93 | self._init_zlib() 94 | self._init_stats() 95 | 96 | def _reinit_buffers(self): 97 | """Internal method to initialize/reset the internal StringIO objects 98 | for read and write buffers. 99 | """ 100 | self.__rbuf = BufferIO() 101 | self.__wbuf = BufferIO() 102 | 103 | def _init_stats(self): 104 | """Internal method to reset the internal statistics counters 105 | for compression ratios and bandwidth savings. 106 | """ 107 | self.bytes_in = 0 108 | self.bytes_out = 0 109 | self.bytes_in_comp = 0 110 | self.bytes_out_comp = 0 111 | 112 | def _init_zlib(self): 113 | """Internal method for setting up the zlib compression and 114 | decompression objects. 115 | """ 116 | self._zcomp_read = zlib.decompressobj() 117 | self._zcomp_write = zlib.compressobj(self.compresslevel) 118 | 119 | def getCompRatio(self): 120 | """Get the current measured compression ratios (in,out) from 121 | this transport. 122 | 123 | Returns a tuple of: 124 | (inbound_compression_ratio, outbound_compression_ratio) 125 | 126 | The compression ratios are computed as: 127 | compressed / uncompressed 128 | 129 | E.g., data that compresses by 10x will have a ratio of: 0.10 130 | and data that compresses to half of ts original size will 131 | have a ratio of 0.5 132 | 133 | None is returned if no bytes have yet been processed in 134 | a particular direction. 135 | """ 136 | r_percent, w_percent = (None, None) 137 | if self.bytes_in > 0: 138 | r_percent = self.bytes_in_comp / self.bytes_in 139 | if self.bytes_out > 0: 140 | w_percent = self.bytes_out_comp / self.bytes_out 141 | return (r_percent, w_percent) 142 | 143 | def getCompSavings(self): 144 | """Get the current count of saved bytes due to data 145 | compression. 146 | 147 | Returns a tuple of: 148 | (inbound_saved_bytes, outbound_saved_bytes) 149 | 150 | Note: if compression is actually expanding your 151 | data (only likely with very tiny thrift objects), then 152 | the values returned will be negative. 153 | """ 154 | r_saved = self.bytes_in - self.bytes_in_comp 155 | w_saved = self.bytes_out - self.bytes_out_comp 156 | return (r_saved, w_saved) 157 | 158 | def isOpen(self): 159 | """Return the underlying transport's open status""" 160 | return self.__trans.isOpen() 161 | 162 | def open(self): 163 | """Open the underlying transport""" 164 | self._init_stats() 165 | return self.__trans.open() 166 | 167 | def listen(self): 168 | """Invoke the underlying transport's listen() method""" 169 | self.__trans.listen() 170 | 171 | def accept(self): 172 | """Accept connections on the underlying transport""" 173 | return self.__trans.accept() 174 | 175 | def close(self): 176 | """Close the underlying transport,""" 177 | self._reinit_buffers() 178 | self._init_zlib() 179 | return self.__trans.close() 180 | 181 | def read(self, sz): 182 | """Read up to sz bytes from the decompressed bytes buffer, and 183 | read from the underlying transport if the decompression 184 | buffer is empty. 185 | """ 186 | ret = self.__rbuf.read(sz) 187 | if len(ret) > 0: 188 | return ret 189 | # keep reading from transport until something comes back 190 | while True: 191 | if self.readComp(sz): 192 | break 193 | ret = self.__rbuf.read(sz) 194 | return ret 195 | 196 | def readComp(self, sz): 197 | """Read compressed data from the underlying transport, then 198 | decompress it and append it to the internal StringIO read buffer 199 | """ 200 | zbuf = self.__trans.read(sz) 201 | zbuf = self._zcomp_read.unconsumed_tail + zbuf 202 | buf = self._zcomp_read.decompress(zbuf) 203 | self.bytes_in += len(zbuf) 204 | self.bytes_in_comp += len(buf) 205 | old = self.__rbuf.read() 206 | self.__rbuf = BufferIO(old + buf) 207 | if len(old) + len(buf) == 0: 208 | return False 209 | return True 210 | 211 | def write(self, buf): 212 | """Write some bytes, putting them into the internal write 213 | buffer for eventual compression. 214 | """ 215 | self.__wbuf.write(buf) 216 | 217 | def flush(self): 218 | """Flush any queued up data in the write buffer and ensure the 219 | compression buffer is flushed out to the underlying transport 220 | """ 221 | wout = self.__wbuf.getvalue() 222 | if len(wout) > 0: 223 | zbuf = self._zcomp_write.compress(wout) 224 | self.bytes_out += len(wout) 225 | self.bytes_out_comp += len(zbuf) 226 | else: 227 | zbuf = '' 228 | ztail = self._zcomp_write.flush(zlib.Z_SYNC_FLUSH) 229 | self.bytes_out_comp += len(ztail) 230 | if (len(zbuf) + len(ztail)) > 0: 231 | self.__wbuf = BufferIO() 232 | self.__trans.write(zbuf + ztail) 233 | self.__trans.flush() 234 | 235 | @property 236 | def cstringio_buf(self): 237 | """Implement the CReadableTransport interface""" 238 | return self.__rbuf 239 | 240 | def cstringio_refill(self, partialread, reqlen): 241 | """Implement the CReadableTransport interface for refill""" 242 | retstring = partialread 243 | if reqlen < self.DEFAULT_BUFFSIZE: 244 | retstring += self.read(self.DEFAULT_BUFFSIZE) 245 | while len(retstring) < reqlen: 246 | retstring += self.read(reqlen - len(retstring)) 247 | self.__rbuf = BufferIO(retstring) 248 | return self.__rbuf 249 | -------------------------------------------------------------------------------- /thrift/protocol/TBinaryProtocol.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | from .TProtocol import TType, TProtocolBase, TProtocolException, TProtocolFactory 21 | from struct import pack, unpack 22 | 23 | 24 | class TBinaryProtocol(TProtocolBase): 25 | """Binary implementation of the Thrift protocol driver.""" 26 | 27 | # NastyHaxx. Python 2.4+ on 32-bit machines forces hex constants to be 28 | # positive, converting this into a long. If we hardcode the int value 29 | # instead it'll stay in 32 bit-land. 30 | 31 | # VERSION_MASK = 0xffff0000 32 | VERSION_MASK = -65536 33 | 34 | # VERSION_1 = 0x80010000 35 | VERSION_1 = -2147418112 36 | 37 | TYPE_MASK = 0x000000ff 38 | 39 | def __init__(self, trans, strictRead=False, strictWrite=True, **kwargs): 40 | TProtocolBase.__init__(self, trans) 41 | self.strictRead = strictRead 42 | self.strictWrite = strictWrite 43 | self.string_length_limit = kwargs.get('string_length_limit', None) 44 | self.container_length_limit = kwargs.get('container_length_limit', None) 45 | 46 | def _check_string_length(self, length): 47 | self._check_length(self.string_length_limit, length) 48 | 49 | def _check_container_length(self, length): 50 | self._check_length(self.container_length_limit, length) 51 | 52 | def writeMessageBegin(self, name, type, seqid): 53 | if self.strictWrite: 54 | self.writeI32(TBinaryProtocol.VERSION_1 | type) 55 | self.writeString(name) 56 | self.writeI32(seqid) 57 | else: 58 | self.writeString(name) 59 | self.writeByte(type) 60 | self.writeI32(seqid) 61 | 62 | def writeMessageEnd(self): 63 | pass 64 | 65 | def writeStructBegin(self, name): 66 | pass 67 | 68 | def writeStructEnd(self): 69 | pass 70 | 71 | def writeFieldBegin(self, name, type, id): 72 | self.writeByte(type) 73 | self.writeI16(id) 74 | 75 | def writeFieldEnd(self): 76 | pass 77 | 78 | def writeFieldStop(self): 79 | self.writeByte(TType.STOP) 80 | 81 | def writeMapBegin(self, ktype, vtype, size): 82 | self.writeByte(ktype) 83 | self.writeByte(vtype) 84 | self.writeI32(size) 85 | 86 | def writeMapEnd(self): 87 | pass 88 | 89 | def writeListBegin(self, etype, size): 90 | self.writeByte(etype) 91 | self.writeI32(size) 92 | 93 | def writeListEnd(self): 94 | pass 95 | 96 | def writeSetBegin(self, etype, size): 97 | self.writeByte(etype) 98 | self.writeI32(size) 99 | 100 | def writeSetEnd(self): 101 | pass 102 | 103 | def writeBool(self, bool): 104 | if bool: 105 | self.writeByte(1) 106 | else: 107 | self.writeByte(0) 108 | 109 | def writeByte(self, byte): 110 | buff = pack("!b", byte) 111 | self.trans.write(buff) 112 | 113 | def writeI16(self, i16): 114 | buff = pack("!h", i16) 115 | self.trans.write(buff) 116 | 117 | def writeI32(self, i32): 118 | buff = pack("!i", i32) 119 | self.trans.write(buff) 120 | 121 | def writeI64(self, i64): 122 | buff = pack("!q", i64) 123 | self.trans.write(buff) 124 | 125 | def writeDouble(self, dub): 126 | buff = pack("!d", dub) 127 | self.trans.write(buff) 128 | 129 | def writeBinary(self, str): 130 | self.writeI32(len(str)) 131 | self.trans.write(str) 132 | 133 | def readMessageBegin(self): 134 | sz = self.readI32() 135 | if sz < 0: 136 | version = sz & TBinaryProtocol.VERSION_MASK 137 | if version != TBinaryProtocol.VERSION_1: 138 | raise TProtocolException( 139 | type=TProtocolException.BAD_VERSION, 140 | message='Bad version in readMessageBegin: %d' % (sz)) 141 | type = sz & TBinaryProtocol.TYPE_MASK 142 | name = self.readString() 143 | seqid = self.readI32() 144 | else: 145 | if self.strictRead: 146 | raise TProtocolException(type=TProtocolException.BAD_VERSION, 147 | message='No protocol version header') 148 | name = self.trans.readAll(sz) 149 | type = self.readByte() 150 | seqid = self.readI32() 151 | return (name, type, seqid) 152 | 153 | def readMessageEnd(self): 154 | pass 155 | 156 | def readStructBegin(self): 157 | pass 158 | 159 | def readStructEnd(self): 160 | pass 161 | 162 | def readFieldBegin(self): 163 | type = self.readByte() 164 | if type == TType.STOP: 165 | return (None, type, 0) 166 | id = self.readI16() 167 | return (None, type, id) 168 | 169 | def readFieldEnd(self): 170 | pass 171 | 172 | def readMapBegin(self): 173 | ktype = self.readByte() 174 | vtype = self.readByte() 175 | size = self.readI32() 176 | self._check_container_length(size) 177 | return (ktype, vtype, size) 178 | 179 | def readMapEnd(self): 180 | pass 181 | 182 | def readListBegin(self): 183 | etype = self.readByte() 184 | size = self.readI32() 185 | self._check_container_length(size) 186 | return (etype, size) 187 | 188 | def readListEnd(self): 189 | pass 190 | 191 | def readSetBegin(self): 192 | etype = self.readByte() 193 | size = self.readI32() 194 | self._check_container_length(size) 195 | return (etype, size) 196 | 197 | def readSetEnd(self): 198 | pass 199 | 200 | def readBool(self): 201 | byte = self.readByte() 202 | if byte == 0: 203 | return False 204 | return True 205 | 206 | def readByte(self): 207 | buff = self.trans.readAll(1) 208 | val, = unpack('!b', buff) 209 | return val 210 | 211 | def readI16(self): 212 | buff = self.trans.readAll(2) 213 | val, = unpack('!h', buff) 214 | return val 215 | 216 | def readI32(self): 217 | buff = self.trans.readAll(4) 218 | val, = unpack('!i', buff) 219 | return val 220 | 221 | def readI64(self): 222 | buff = self.trans.readAll(8) 223 | val, = unpack('!q', buff) 224 | return val 225 | 226 | def readDouble(self): 227 | buff = self.trans.readAll(8) 228 | val, = unpack('!d', buff) 229 | return val 230 | 231 | def readBinary(self): 232 | size = self.readI32() 233 | self._check_string_length(size) 234 | s = self.trans.readAll(size) 235 | return s 236 | 237 | 238 | class TBinaryProtocolFactory(TProtocolFactory): 239 | def __init__(self, strictRead=False, strictWrite=True, **kwargs): 240 | self.strictRead = strictRead 241 | self.strictWrite = strictWrite 242 | self.string_length_limit = kwargs.get('string_length_limit', None) 243 | self.container_length_limit = kwargs.get('container_length_limit', None) 244 | 245 | def getProtocol(self, trans): 246 | prot = TBinaryProtocol(trans, self.strictRead, self.strictWrite, 247 | string_length_limit=self.string_length_limit, 248 | container_length_limit=self.container_length_limit) 249 | return prot 250 | 251 | 252 | class TBinaryProtocolAccelerated(TBinaryProtocol): 253 | """C-Accelerated version of TBinaryProtocol. 254 | 255 | This class does not override any of TBinaryProtocol's methods, 256 | but the generated code recognizes it directly and will call into 257 | our C module to do the encoding, bypassing this object entirely. 258 | We inherit from TBinaryProtocol so that the normal TBinaryProtocol 259 | encoding can happen if the fastbinary module doesn't work for some 260 | reason. (TODO(dreiss): Make this happen sanely in more cases.) 261 | To disable this behavior, pass fallback=False constructor argument. 262 | 263 | In order to take advantage of the C module, just use 264 | TBinaryProtocolAccelerated instead of TBinaryProtocol. 265 | 266 | NOTE: This code was contributed by an external developer. 267 | The internal Thrift team has reviewed and tested it, 268 | but we cannot guarantee that it is production-ready. 269 | Please feel free to report bugs and/or success stories 270 | to the public mailing list. 271 | """ 272 | pass 273 | 274 | def __init__(self, *args, **kwargs): 275 | fallback = kwargs.pop('fallback', True) 276 | super(TBinaryProtocolAccelerated, self).__init__(*args, **kwargs) 277 | try: 278 | from thrift.protocol import fastbinary 279 | except ImportError: 280 | if not fallback: 281 | raise 282 | else: 283 | self._fast_decode = fastbinary.decode_binary 284 | self._fast_encode = fastbinary.encode_binary 285 | 286 | 287 | class TBinaryProtocolAcceleratedFactory(TProtocolFactory): 288 | def __init__(self, 289 | string_length_limit=None, 290 | container_length_limit=None, 291 | fallback=True): 292 | self.string_length_limit = string_length_limit 293 | self.container_length_limit = container_length_limit 294 | self._fallback = fallback 295 | 296 | def getProtocol(self, trans): 297 | return TBinaryProtocolAccelerated( 298 | trans, 299 | string_length_limit=self.string_length_limit, 300 | container_length_limit=self.container_length_limit, 301 | fallback=self._fallback) 302 | -------------------------------------------------------------------------------- /thrift/ext/compact.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | #ifndef THRIFT_PY_COMPACT_H 21 | #define THRIFT_PY_COMPACT_H 22 | 23 | #include 24 | #include "ext/protocol.h" 25 | #include "ext/endian.h" 26 | #include 27 | #include 28 | 29 | namespace apache { 30 | namespace thrift { 31 | namespace py { 32 | 33 | class CompactProtocol : public ProtocolBase { 34 | public: 35 | CompactProtocol() { readBool_.exists = false; } 36 | 37 | virtual ~CompactProtocol() {} 38 | 39 | void writeI8(int8_t val) { writeBuffer(reinterpret_cast(&val), 1); } 40 | 41 | void writeI16(int16_t val) { writeVarint(toZigZag(val)); } 42 | 43 | int writeI32(int32_t val) { return writeVarint(toZigZag(val)); } 44 | 45 | void writeI64(int64_t val) { writeVarint64(toZigZag64(val)); } 46 | 47 | void writeDouble(double dub) { 48 | union { 49 | double f; 50 | int64_t t; 51 | } transfer; 52 | transfer.f = htolell(dub); 53 | writeBuffer(reinterpret_cast(&transfer.t), sizeof(int64_t)); 54 | } 55 | 56 | void writeBool(int v) { writeByte(static_cast(v ? CT_BOOLEAN_TRUE : CT_BOOLEAN_FALSE)); } 57 | 58 | void writeString(PyObject* value, int32_t len) { 59 | writeVarint(len); 60 | writeBuffer(PyBytes_AS_STRING(value), len); 61 | } 62 | 63 | bool writeListBegin(PyObject* value, const SetListTypeArgs& args, int32_t len) { 64 | int ctype = toCompactType(args.element_type); 65 | if (len <= 14) { 66 | writeByte(static_cast(len << 4 | ctype)); 67 | } else { 68 | writeByte(0xf0 | ctype); 69 | writeVarint(len); 70 | } 71 | return true; 72 | } 73 | 74 | bool writeMapBegin(PyObject* value, const MapTypeArgs& args, int32_t len) { 75 | if (len == 0) { 76 | writeByte(0); 77 | return true; 78 | } 79 | int ctype = toCompactType(args.ktag) << 4 | toCompactType(args.vtag); 80 | writeVarint(len); 81 | writeByte(ctype); 82 | return true; 83 | } 84 | 85 | bool writeStructBegin() { 86 | writeTags_.push(0); 87 | return true; 88 | } 89 | bool writeStructEnd() { 90 | writeTags_.pop(); 91 | return true; 92 | } 93 | 94 | bool writeField(PyObject* value, const StructItemSpec& spec) { 95 | if (spec.type == T_BOOL) { 96 | doWriteFieldBegin(spec, PyObject_IsTrue(value) ? CT_BOOLEAN_TRUE : CT_BOOLEAN_FALSE); 97 | return true; 98 | } else { 99 | doWriteFieldBegin(spec, toCompactType(spec.type)); 100 | return encodeValue(value, spec.type, spec.typeargs); 101 | } 102 | } 103 | 104 | void writeFieldStop() { writeByte(0); } 105 | 106 | bool readBool(bool& val) { 107 | if (readBool_.exists) { 108 | readBool_.exists = false; 109 | val = readBool_.value; 110 | return true; 111 | } 112 | char* buf; 113 | if (!readBytes(&buf, 1)) { 114 | return false; 115 | } 116 | val = buf[0] == CT_BOOLEAN_TRUE; 117 | return true; 118 | } 119 | bool readI8(int8_t& val) { 120 | char* buf; 121 | if (!readBytes(&buf, 1)) { 122 | return false; 123 | } 124 | val = buf[0]; 125 | return true; 126 | } 127 | 128 | bool readI16(int16_t& val) { 129 | uint16_t uval; 130 | if (readVarint(uval)) { 131 | val = fromZigZag(uval); 132 | return true; 133 | } 134 | return false; 135 | } 136 | 137 | bool readI32(int32_t& val) { 138 | uint32_t uval; 139 | if (readVarint(uval)) { 140 | val = fromZigZag(uval); 141 | return true; 142 | } 143 | return false; 144 | } 145 | 146 | bool readI64(int64_t& val) { 147 | uint64_t uval; 148 | if (readVarint(uval)) { 149 | val = fromZigZag(uval); 150 | return true; 151 | } 152 | return false; 153 | } 154 | 155 | bool readDouble(double& val) { 156 | union { 157 | int64_t f; 158 | double t; 159 | } transfer; 160 | 161 | char* buf; 162 | if (!readBytes(&buf, 8)) { 163 | return false; 164 | } 165 | memcpy(&transfer.f, buf, sizeof(int64_t)); 166 | transfer.f = letohll(transfer.f); 167 | val = transfer.t; 168 | return true; 169 | } 170 | 171 | int32_t readString(char** buf) { 172 | uint32_t len; 173 | if (!readVarint(len) || !checkLengthLimit(len, stringLimit())) { 174 | return -1; 175 | } 176 | if (len == 0) { 177 | return 0; 178 | } 179 | if (!readBytes(buf, len)) { 180 | return -1; 181 | } 182 | return len; 183 | } 184 | 185 | int32_t readListBegin(TType& etype) { 186 | uint8_t b; 187 | if (!readByte(b)) { 188 | return -1; 189 | } 190 | etype = getTType(b & 0xf); 191 | if (etype == -1) { 192 | return -1; 193 | } 194 | uint32_t len = (b >> 4) & 0xf; 195 | if (len == 15 && !readVarint(len)) { 196 | return -1; 197 | } 198 | if (!checkLengthLimit(len, containerLimit())) { 199 | return -1; 200 | } 201 | return len; 202 | } 203 | 204 | int32_t readMapBegin(TType& ktype, TType& vtype) { 205 | uint32_t len; 206 | if (!readVarint(len) || !checkLengthLimit(len, containerLimit())) { 207 | return -1; 208 | } 209 | if (len != 0) { 210 | uint8_t kvType; 211 | if (!readByte(kvType)) { 212 | return -1; 213 | } 214 | ktype = getTType(kvType >> 4); 215 | vtype = getTType(kvType & 0xf); 216 | if (ktype == -1 || vtype == -1) { 217 | return -1; 218 | } 219 | } 220 | return len; 221 | } 222 | 223 | bool readStructBegin() { 224 | readTags_.push(0); 225 | return true; 226 | } 227 | bool readStructEnd() { 228 | readTags_.pop(); 229 | return true; 230 | } 231 | bool readFieldBegin(TType& type, int16_t& tag); 232 | 233 | bool skipBool() { 234 | bool val; 235 | return readBool(val); 236 | } 237 | #define SKIPBYTES(n) \ 238 | do { \ 239 | if (!readBytes(&dummy_buf_, (n))) { \ 240 | return false; \ 241 | } \ 242 | return true; \ 243 | } while (0) 244 | bool skipByte() { SKIPBYTES(1); } 245 | bool skipDouble() { SKIPBYTES(8); } 246 | bool skipI16() { 247 | int16_t val; 248 | return readI16(val); 249 | } 250 | bool skipI32() { 251 | int32_t val; 252 | return readI32(val); 253 | } 254 | bool skipI64() { 255 | int64_t val; 256 | return readI64(val); 257 | } 258 | bool skipString() { 259 | uint32_t len; 260 | if (!readVarint(len)) { 261 | return false; 262 | } 263 | SKIPBYTES(len); 264 | } 265 | #undef SKIPBYTES 266 | 267 | private: 268 | enum Types { 269 | CT_STOP = 0x00, 270 | CT_BOOLEAN_TRUE = 0x01, 271 | CT_BOOLEAN_FALSE = 0x02, 272 | CT_BYTE = 0x03, 273 | CT_I16 = 0x04, 274 | CT_I32 = 0x05, 275 | CT_I64 = 0x06, 276 | CT_DOUBLE = 0x07, 277 | CT_BINARY = 0x08, 278 | CT_LIST = 0x09, 279 | CT_SET = 0x0A, 280 | CT_MAP = 0x0B, 281 | CT_STRUCT = 0x0C 282 | }; 283 | 284 | static const uint8_t TTypeToCType[]; 285 | 286 | TType getTType(uint8_t type); 287 | 288 | int toCompactType(TType type) { 289 | int i = static_cast(type); 290 | return i < 16 ? TTypeToCType[i] : -1; 291 | } 292 | 293 | uint32_t toZigZag(int32_t val) { return (val >> 31) ^ (val << 1); } 294 | 295 | uint64_t toZigZag64(int64_t val) { return (val >> 63) ^ (val << 1); } 296 | 297 | int writeVarint(uint32_t val) { 298 | int cnt = 1; 299 | while (val & ~0x7fU) { 300 | writeByte(static_cast((val & 0x7fU) | 0x80U)); 301 | val >>= 7; 302 | ++cnt; 303 | } 304 | writeByte(static_cast(val)); 305 | return cnt; 306 | } 307 | 308 | int writeVarint64(uint64_t val) { 309 | int cnt = 1; 310 | while (val & ~0x7fULL) { 311 | writeByte(static_cast((val & 0x7fULL) | 0x80ULL)); 312 | val >>= 7; 313 | ++cnt; 314 | } 315 | writeByte(static_cast(val)); 316 | return cnt; 317 | } 318 | 319 | template 320 | bool readVarint(T& result) { 321 | uint8_t b; 322 | T val = 0; 323 | int shift = 0; 324 | for (int i = 0; i < Max; ++i) { 325 | if (!readByte(b)) { 326 | return false; 327 | } 328 | if (b & 0x80) { 329 | val |= static_cast(b & 0x7f) << shift; 330 | } else { 331 | val |= static_cast(b) << shift; 332 | result = val; 333 | return true; 334 | } 335 | shift += 7; 336 | } 337 | PyErr_Format(PyExc_OverflowError, "varint exceeded %d bytes", Max); 338 | return false; 339 | } 340 | 341 | template 342 | S fromZigZag(U val) { 343 | return (val >> 1) ^ static_cast(-static_cast(val & 1)); 344 | } 345 | 346 | void doWriteFieldBegin(const StructItemSpec& spec, int ctype) { 347 | int diff = spec.tag - writeTags_.top(); 348 | if (diff > 0 && diff <= 15) { 349 | writeByte(static_cast(diff << 4 | ctype)); 350 | } else { 351 | writeByte(static_cast(ctype)); 352 | writeI16(spec.tag); 353 | } 354 | writeTags_.top() = spec.tag; 355 | } 356 | 357 | std::stack writeTags_; 358 | std::stack readTags_; 359 | struct { 360 | bool exists; 361 | bool value; 362 | } readBool_; 363 | char* dummy_buf_; 364 | }; 365 | } 366 | } 367 | } 368 | #endif // THRIFT_PY_COMPACT_H 369 | -------------------------------------------------------------------------------- /BEService/TalkService-remote: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Autogenerated by Thrift Compiler (0.13.0) 4 | # 5 | # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 6 | # 7 | # options string: py 8 | # 9 | 10 | import sys 11 | import pprint 12 | if sys.version_info[0] > 2: 13 | from urllib.parse import urlparse 14 | else: 15 | from urlparse import urlparse 16 | from thrift.transport import TTransport, TSocket, TSSLSocket, THttpClient 17 | from thrift.protocol.TBinaryProtocol import TBinaryProtocol 18 | 19 | from BEService import TalkService 20 | from BEService.ttypes import * 21 | 22 | if len(sys.argv) <= 1 or sys.argv[1] == '--help': 23 | print('') 24 | print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] [-novalidate] [-ca_certs certs] [-keyfile keyfile] [-certfile certfile] function [arg1 [arg2...]]') 25 | print('') 26 | print('Functions:') 27 | print(' AcceptChatInvitationResponse acceptChatInvitation(AcceptChatInvitationRequest request)') 28 | print(' AcceptChatInvitationByTicketResponse acceptChatInvitationByTicket(AcceptChatInvitationByTicketRequest request)') 29 | print(' void blockContact(i32 reqSeq, string id)') 30 | print(' CancelChatInvitationResponse cancelChatInvitation(CancelChatInvitationRequest request)') 31 | print(' CreateChatResponse createChat(CreateChatRequest request)') 32 | print(' DeleteSelfFromChatResponse deleteSelfFromChat(DeleteSelfFromChatRequest request)') 33 | print(' DeleteOtherFromChatResponse deleteOtherFromChat(DeleteOtherFromChatRequest request)') 34 | print(' FetchOperationsResponse fetchOperations(FetchOperationsRequest request)') 35 | print(' fetchOps(i64 localRev, i32 count, i64 globalRev, i64 individualRev)') 36 | print(' findAndAddContactsByMid(i32 reqSeq, string mid, ContactType type, string reference)') 37 | print(' findAndAddContactsByUserid(i32 reqSeq, string searchId, string reference)') 38 | print(' Contact findContactByUserid(string userid)') 39 | print(' FindChatByTicketResponse findChatByTicket(FindChatByTicketRequest request)') 40 | print(' GetAllChatMidsResponse getAllChatMids(GetAllChatMidsRequest request, SyncReason syncReason)') 41 | print(' Profile getProfile(SyncReason syncReason)') 42 | print(' Contact getContact(string id)') 43 | print(' string getCountryWithRequestIp()') 44 | print(' i64 getServerTime()') 45 | print(' getContacts( ids)') 46 | print(' getAllContactIds(SyncReason syncReason)') 47 | print(' GetChatsResponse getChats(GetChatsRequest request)') 48 | print(' InviteIntoChatResponse inviteIntoChat(InviteIntoChatRequest request)') 49 | print(' ReissueChatTicketResponse reissueChatTicket(ReissueChatTicketRequest request)') 50 | print(' RejectChatInvitationResponse rejectChatInvitation(RejectChatInvitationRequest request)') 51 | print(' Message sendMessage(i32 seq, Message message)') 52 | print(' void unsendMessage(i32 seq, string messageId)') 53 | print(' UpdateChatResponse updateChat(UpdateChatRequest request)') 54 | print(' void updateProfileAttribute(i32 reqSeq, ProfileAttribute attr, string value)') 55 | print('') 56 | sys.exit(0) 57 | 58 | pp = pprint.PrettyPrinter(indent=2) 59 | host = 'localhost' 60 | port = 9090 61 | uri = '' 62 | framed = False 63 | ssl = False 64 | validate = True 65 | ca_certs = None 66 | keyfile = None 67 | certfile = None 68 | http = False 69 | argi = 1 70 | 71 | if sys.argv[argi] == '-h': 72 | parts = sys.argv[argi + 1].split(':') 73 | host = parts[0] 74 | if len(parts) > 1: 75 | port = int(parts[1]) 76 | argi += 2 77 | 78 | if sys.argv[argi] == '-u': 79 | url = urlparse(sys.argv[argi + 1]) 80 | parts = url[1].split(':') 81 | host = parts[0] 82 | if len(parts) > 1: 83 | port = int(parts[1]) 84 | else: 85 | port = 80 86 | uri = url[2] 87 | if url[4]: 88 | uri += '?%s' % url[4] 89 | http = True 90 | argi += 2 91 | 92 | if sys.argv[argi] == '-f' or sys.argv[argi] == '-framed': 93 | framed = True 94 | argi += 1 95 | 96 | if sys.argv[argi] == '-s' or sys.argv[argi] == '-ssl': 97 | ssl = True 98 | argi += 1 99 | 100 | if sys.argv[argi] == '-novalidate': 101 | validate = False 102 | argi += 1 103 | 104 | if sys.argv[argi] == '-ca_certs': 105 | ca_certs = sys.argv[argi+1] 106 | argi += 2 107 | 108 | if sys.argv[argi] == '-keyfile': 109 | keyfile = sys.argv[argi+1] 110 | argi += 2 111 | 112 | if sys.argv[argi] == '-certfile': 113 | certfile = sys.argv[argi+1] 114 | argi += 2 115 | 116 | cmd = sys.argv[argi] 117 | args = sys.argv[argi + 1:] 118 | 119 | if http: 120 | transport = THttpClient.THttpClient(host, port, uri) 121 | else: 122 | if ssl: 123 | socket = TSSLSocket.TSSLSocket(host, port, validate=validate, ca_certs=ca_certs, keyfile=keyfile, certfile=certfile) 124 | else: 125 | socket = TSocket.TSocket(host, port) 126 | if framed: 127 | transport = TTransport.TFramedTransport(socket) 128 | else: 129 | transport = TTransport.TBufferedTransport(socket) 130 | protocol = TBinaryProtocol(transport) 131 | client = TalkService.Client(protocol) 132 | transport.open() 133 | 134 | if cmd == 'acceptChatInvitation': 135 | if len(args) != 1: 136 | print('acceptChatInvitation requires 1 args') 137 | sys.exit(1) 138 | pp.pprint(client.acceptChatInvitation(eval(args[0]),)) 139 | 140 | elif cmd == 'acceptChatInvitationByTicket': 141 | if len(args) != 1: 142 | print('acceptChatInvitationByTicket requires 1 args') 143 | sys.exit(1) 144 | pp.pprint(client.acceptChatInvitationByTicket(eval(args[0]),)) 145 | 146 | elif cmd == 'blockContact': 147 | if len(args) != 2: 148 | print('blockContact requires 2 args') 149 | sys.exit(1) 150 | pp.pprint(client.blockContact(eval(args[0]), args[1],)) 151 | 152 | elif cmd == 'cancelChatInvitation': 153 | if len(args) != 1: 154 | print('cancelChatInvitation requires 1 args') 155 | sys.exit(1) 156 | pp.pprint(client.cancelChatInvitation(eval(args[0]),)) 157 | 158 | elif cmd == 'createChat': 159 | if len(args) != 1: 160 | print('createChat requires 1 args') 161 | sys.exit(1) 162 | pp.pprint(client.createChat(eval(args[0]),)) 163 | 164 | elif cmd == 'deleteSelfFromChat': 165 | if len(args) != 1: 166 | print('deleteSelfFromChat requires 1 args') 167 | sys.exit(1) 168 | pp.pprint(client.deleteSelfFromChat(eval(args[0]),)) 169 | 170 | elif cmd == 'deleteOtherFromChat': 171 | if len(args) != 1: 172 | print('deleteOtherFromChat requires 1 args') 173 | sys.exit(1) 174 | pp.pprint(client.deleteOtherFromChat(eval(args[0]),)) 175 | 176 | elif cmd == 'fetchOperations': 177 | if len(args) != 1: 178 | print('fetchOperations requires 1 args') 179 | sys.exit(1) 180 | pp.pprint(client.fetchOperations(eval(args[0]),)) 181 | 182 | elif cmd == 'fetchOps': 183 | if len(args) != 4: 184 | print('fetchOps requires 4 args') 185 | sys.exit(1) 186 | pp.pprint(client.fetchOps(eval(args[0]), eval(args[1]), eval(args[2]), eval(args[3]),)) 187 | 188 | elif cmd == 'findAndAddContactsByMid': 189 | if len(args) != 4: 190 | print('findAndAddContactsByMid requires 4 args') 191 | sys.exit(1) 192 | pp.pprint(client.findAndAddContactsByMid(eval(args[0]), args[1], eval(args[2]), args[3],)) 193 | 194 | elif cmd == 'findAndAddContactsByUserid': 195 | if len(args) != 3: 196 | print('findAndAddContactsByUserid requires 3 args') 197 | sys.exit(1) 198 | pp.pprint(client.findAndAddContactsByUserid(eval(args[0]), args[1], args[2],)) 199 | 200 | elif cmd == 'findContactByUserid': 201 | if len(args) != 1: 202 | print('findContactByUserid requires 1 args') 203 | sys.exit(1) 204 | pp.pprint(client.findContactByUserid(args[0],)) 205 | 206 | elif cmd == 'findChatByTicket': 207 | if len(args) != 1: 208 | print('findChatByTicket requires 1 args') 209 | sys.exit(1) 210 | pp.pprint(client.findChatByTicket(eval(args[0]),)) 211 | 212 | elif cmd == 'getAllChatMids': 213 | if len(args) != 2: 214 | print('getAllChatMids requires 2 args') 215 | sys.exit(1) 216 | pp.pprint(client.getAllChatMids(eval(args[0]), eval(args[1]),)) 217 | 218 | elif cmd == 'getProfile': 219 | if len(args) != 1: 220 | print('getProfile requires 1 args') 221 | sys.exit(1) 222 | pp.pprint(client.getProfile(eval(args[0]),)) 223 | 224 | elif cmd == 'getContact': 225 | if len(args) != 1: 226 | print('getContact requires 1 args') 227 | sys.exit(1) 228 | pp.pprint(client.getContact(args[0],)) 229 | 230 | elif cmd == 'getCountryWithRequestIp': 231 | if len(args) != 0: 232 | print('getCountryWithRequestIp requires 0 args') 233 | sys.exit(1) 234 | pp.pprint(client.getCountryWithRequestIp()) 235 | 236 | elif cmd == 'getServerTime': 237 | if len(args) != 0: 238 | print('getServerTime requires 0 args') 239 | sys.exit(1) 240 | pp.pprint(client.getServerTime()) 241 | 242 | elif cmd == 'getContacts': 243 | if len(args) != 1: 244 | print('getContacts requires 1 args') 245 | sys.exit(1) 246 | pp.pprint(client.getContacts(eval(args[0]),)) 247 | 248 | elif cmd == 'getAllContactIds': 249 | if len(args) != 1: 250 | print('getAllContactIds requires 1 args') 251 | sys.exit(1) 252 | pp.pprint(client.getAllContactIds(eval(args[0]),)) 253 | 254 | elif cmd == 'getChats': 255 | if len(args) != 1: 256 | print('getChats requires 1 args') 257 | sys.exit(1) 258 | pp.pprint(client.getChats(eval(args[0]),)) 259 | 260 | elif cmd == 'inviteIntoChat': 261 | if len(args) != 1: 262 | print('inviteIntoChat requires 1 args') 263 | sys.exit(1) 264 | pp.pprint(client.inviteIntoChat(eval(args[0]),)) 265 | 266 | elif cmd == 'reissueChatTicket': 267 | if len(args) != 1: 268 | print('reissueChatTicket requires 1 args') 269 | sys.exit(1) 270 | pp.pprint(client.reissueChatTicket(eval(args[0]),)) 271 | 272 | elif cmd == 'rejectChatInvitation': 273 | if len(args) != 1: 274 | print('rejectChatInvitation requires 1 args') 275 | sys.exit(1) 276 | pp.pprint(client.rejectChatInvitation(eval(args[0]),)) 277 | 278 | elif cmd == 'sendMessage': 279 | if len(args) != 2: 280 | print('sendMessage requires 2 args') 281 | sys.exit(1) 282 | pp.pprint(client.sendMessage(eval(args[0]), eval(args[1]),)) 283 | 284 | elif cmd == 'unsendMessage': 285 | if len(args) != 2: 286 | print('unsendMessage requires 2 args') 287 | sys.exit(1) 288 | pp.pprint(client.unsendMessage(eval(args[0]), args[1],)) 289 | 290 | elif cmd == 'updateChat': 291 | if len(args) != 1: 292 | print('updateChat requires 1 args') 293 | sys.exit(1) 294 | pp.pprint(client.updateChat(eval(args[0]),)) 295 | 296 | elif cmd == 'updateProfileAttribute': 297 | if len(args) != 3: 298 | print('updateProfileAttribute requires 3 args') 299 | sys.exit(1) 300 | pp.pprint(client.updateProfileAttribute(eval(args[0]), eval(args[1]), args[2],)) 301 | 302 | else: 303 | print('Unrecognized method %s' % cmd) 304 | sys.exit(1) 305 | 306 | transport.close() 307 | -------------------------------------------------------------------------------- /thrift/transport/TTwisted.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | from io import BytesIO 21 | import struct 22 | 23 | from zope.interface import implementer, Interface, Attribute 24 | from twisted.internet.protocol import ServerFactory, ClientFactory, \ 25 | connectionDone 26 | from twisted.internet import defer 27 | from twisted.internet.threads import deferToThread 28 | from twisted.protocols import basic 29 | from twisted.web import server, resource, http 30 | 31 | from thrift.transport import TTransport 32 | 33 | 34 | class TMessageSenderTransport(TTransport.TTransportBase): 35 | 36 | def __init__(self): 37 | self.__wbuf = BytesIO() 38 | 39 | def write(self, buf): 40 | self.__wbuf.write(buf) 41 | 42 | def flush(self): 43 | msg = self.__wbuf.getvalue() 44 | self.__wbuf = BytesIO() 45 | return self.sendMessage(msg) 46 | 47 | def sendMessage(self, message): 48 | raise NotImplementedError 49 | 50 | 51 | class TCallbackTransport(TMessageSenderTransport): 52 | 53 | def __init__(self, func): 54 | TMessageSenderTransport.__init__(self) 55 | self.func = func 56 | 57 | def sendMessage(self, message): 58 | return self.func(message) 59 | 60 | 61 | class ThriftClientProtocol(basic.Int32StringReceiver): 62 | 63 | MAX_LENGTH = 2 ** 31 - 1 64 | 65 | def __init__(self, client_class, iprot_factory, oprot_factory=None): 66 | self._client_class = client_class 67 | self._iprot_factory = iprot_factory 68 | if oprot_factory is None: 69 | self._oprot_factory = iprot_factory 70 | else: 71 | self._oprot_factory = oprot_factory 72 | 73 | self.recv_map = {} 74 | self.started = defer.Deferred() 75 | 76 | def dispatch(self, msg): 77 | self.sendString(msg) 78 | 79 | def connectionMade(self): 80 | tmo = TCallbackTransport(self.dispatch) 81 | self.client = self._client_class(tmo, self._oprot_factory) 82 | self.started.callback(self.client) 83 | 84 | def connectionLost(self, reason=connectionDone): 85 | # the called errbacks can add items to our client's _reqs, 86 | # so we need to use a tmp, and iterate until no more requests 87 | # are added during errbacks 88 | if self.client: 89 | tex = TTransport.TTransportException( 90 | type=TTransport.TTransportException.END_OF_FILE, 91 | message='Connection closed (%s)' % reason) 92 | while self.client._reqs: 93 | _, v = self.client._reqs.popitem() 94 | v.errback(tex) 95 | del self.client._reqs 96 | self.client = None 97 | 98 | def stringReceived(self, frame): 99 | tr = TTransport.TMemoryBuffer(frame) 100 | iprot = self._iprot_factory.getProtocol(tr) 101 | (fname, mtype, rseqid) = iprot.readMessageBegin() 102 | 103 | try: 104 | method = self.recv_map[fname] 105 | except KeyError: 106 | method = getattr(self.client, 'recv_' + fname) 107 | self.recv_map[fname] = method 108 | 109 | method(iprot, mtype, rseqid) 110 | 111 | 112 | class ThriftSASLClientProtocol(ThriftClientProtocol): 113 | 114 | START = 1 115 | OK = 2 116 | BAD = 3 117 | ERROR = 4 118 | COMPLETE = 5 119 | 120 | MAX_LENGTH = 2 ** 31 - 1 121 | 122 | def __init__(self, client_class, iprot_factory, oprot_factory=None, 123 | host=None, service=None, mechanism='GSSAPI', **sasl_kwargs): 124 | """ 125 | host: the name of the server, from a SASL perspective 126 | service: the name of the server's service, from a SASL perspective 127 | mechanism: the name of the preferred mechanism to use 128 | 129 | All other kwargs will be passed to the puresasl.client.SASLClient 130 | constructor. 131 | """ 132 | 133 | from puresasl.client import SASLClient 134 | self.SASLCLient = SASLClient 135 | 136 | ThriftClientProtocol.__init__(self, client_class, iprot_factory, oprot_factory) 137 | 138 | self._sasl_negotiation_deferred = None 139 | self._sasl_negotiation_status = None 140 | self.client = None 141 | 142 | if host is not None: 143 | self.createSASLClient(host, service, mechanism, **sasl_kwargs) 144 | 145 | def createSASLClient(self, host, service, mechanism, **kwargs): 146 | self.sasl = self.SASLClient(host, service, mechanism, **kwargs) 147 | 148 | def dispatch(self, msg): 149 | encoded = self.sasl.wrap(msg) 150 | len_and_encoded = ''.join((struct.pack('!i', len(encoded)), encoded)) 151 | ThriftClientProtocol.dispatch(self, len_and_encoded) 152 | 153 | @defer.inlineCallbacks 154 | def connectionMade(self): 155 | self._sendSASLMessage(self.START, self.sasl.mechanism) 156 | initial_message = yield deferToThread(self.sasl.process) 157 | self._sendSASLMessage(self.OK, initial_message) 158 | 159 | while True: 160 | status, challenge = yield self._receiveSASLMessage() 161 | if status == self.OK: 162 | response = yield deferToThread(self.sasl.process, challenge) 163 | self._sendSASLMessage(self.OK, response) 164 | elif status == self.COMPLETE: 165 | if not self.sasl.complete: 166 | msg = "The server erroneously indicated that SASL " \ 167 | "negotiation was complete" 168 | raise TTransport.TTransportException(msg, message=msg) 169 | else: 170 | break 171 | else: 172 | msg = "Bad SASL negotiation status: %d (%s)" % (status, challenge) 173 | raise TTransport.TTransportException(msg, message=msg) 174 | 175 | self._sasl_negotiation_deferred = None 176 | ThriftClientProtocol.connectionMade(self) 177 | 178 | def _sendSASLMessage(self, status, body): 179 | if body is None: 180 | body = "" 181 | header = struct.pack(">BI", status, len(body)) 182 | self.transport.write(header + body) 183 | 184 | def _receiveSASLMessage(self): 185 | self._sasl_negotiation_deferred = defer.Deferred() 186 | self._sasl_negotiation_status = None 187 | return self._sasl_negotiation_deferred 188 | 189 | def connectionLost(self, reason=connectionDone): 190 | if self.client: 191 | ThriftClientProtocol.connectionLost(self, reason) 192 | 193 | def dataReceived(self, data): 194 | if self._sasl_negotiation_deferred: 195 | # we got a sasl challenge in the format (status, length, challenge) 196 | # save the status, let IntNStringReceiver piece the challenge data together 197 | self._sasl_negotiation_status, = struct.unpack("B", data[0]) 198 | ThriftClientProtocol.dataReceived(self, data[1:]) 199 | else: 200 | # normal frame, let IntNStringReceiver piece it together 201 | ThriftClientProtocol.dataReceived(self, data) 202 | 203 | def stringReceived(self, frame): 204 | if self._sasl_negotiation_deferred: 205 | # the frame is just a SASL challenge 206 | response = (self._sasl_negotiation_status, frame) 207 | self._sasl_negotiation_deferred.callback(response) 208 | else: 209 | # there's a second 4 byte length prefix inside the frame 210 | decoded_frame = self.sasl.unwrap(frame[4:]) 211 | ThriftClientProtocol.stringReceived(self, decoded_frame) 212 | 213 | 214 | class ThriftServerProtocol(basic.Int32StringReceiver): 215 | 216 | MAX_LENGTH = 2 ** 31 - 1 217 | 218 | def dispatch(self, msg): 219 | self.sendString(msg) 220 | 221 | def processError(self, error): 222 | self.transport.loseConnection() 223 | 224 | def processOk(self, _, tmo): 225 | msg = tmo.getvalue() 226 | 227 | if len(msg) > 0: 228 | self.dispatch(msg) 229 | 230 | def stringReceived(self, frame): 231 | tmi = TTransport.TMemoryBuffer(frame) 232 | tmo = TTransport.TMemoryBuffer() 233 | 234 | iprot = self.factory.iprot_factory.getProtocol(tmi) 235 | oprot = self.factory.oprot_factory.getProtocol(tmo) 236 | 237 | d = self.factory.processor.process(iprot, oprot) 238 | d.addCallbacks(self.processOk, self.processError, 239 | callbackArgs=(tmo,)) 240 | 241 | 242 | class IThriftServerFactory(Interface): 243 | 244 | processor = Attribute("Thrift processor") 245 | 246 | iprot_factory = Attribute("Input protocol factory") 247 | 248 | oprot_factory = Attribute("Output protocol factory") 249 | 250 | 251 | class IThriftClientFactory(Interface): 252 | 253 | client_class = Attribute("Thrift client class") 254 | 255 | iprot_factory = Attribute("Input protocol factory") 256 | 257 | oprot_factory = Attribute("Output protocol factory") 258 | 259 | 260 | @implementer(IThriftServerFactory) 261 | class ThriftServerFactory(ServerFactory): 262 | 263 | protocol = ThriftServerProtocol 264 | 265 | def __init__(self, processor, iprot_factory, oprot_factory=None): 266 | self.processor = processor 267 | self.iprot_factory = iprot_factory 268 | if oprot_factory is None: 269 | self.oprot_factory = iprot_factory 270 | else: 271 | self.oprot_factory = oprot_factory 272 | 273 | 274 | @implementer(IThriftClientFactory) 275 | class ThriftClientFactory(ClientFactory): 276 | 277 | protocol = ThriftClientProtocol 278 | 279 | def __init__(self, client_class, iprot_factory, oprot_factory=None): 280 | self.client_class = client_class 281 | self.iprot_factory = iprot_factory 282 | if oprot_factory is None: 283 | self.oprot_factory = iprot_factory 284 | else: 285 | self.oprot_factory = oprot_factory 286 | 287 | def buildProtocol(self, addr): 288 | p = self.protocol(self.client_class, self.iprot_factory, 289 | self.oprot_factory) 290 | p.factory = self 291 | return p 292 | 293 | 294 | class ThriftResource(resource.Resource): 295 | 296 | allowedMethods = ('POST',) 297 | 298 | def __init__(self, processor, inputProtocolFactory, 299 | outputProtocolFactory=None): 300 | resource.Resource.__init__(self) 301 | self.inputProtocolFactory = inputProtocolFactory 302 | if outputProtocolFactory is None: 303 | self.outputProtocolFactory = inputProtocolFactory 304 | else: 305 | self.outputProtocolFactory = outputProtocolFactory 306 | self.processor = processor 307 | 308 | def getChild(self, path, request): 309 | return self 310 | 311 | def _cbProcess(self, _, request, tmo): 312 | msg = tmo.getvalue() 313 | request.setResponseCode(http.OK) 314 | request.setHeader("content-type", "application/x-thrift") 315 | request.write(msg) 316 | request.finish() 317 | 318 | def render_POST(self, request): 319 | request.content.seek(0, 0) 320 | data = request.content.read() 321 | tmi = TTransport.TMemoryBuffer(data) 322 | tmo = TTransport.TMemoryBuffer() 323 | 324 | iprot = self.inputProtocolFactory.getProtocol(tmi) 325 | oprot = self.outputProtocolFactory.getProtocol(tmo) 326 | 327 | d = self.processor.process(iprot, oprot) 328 | d.addCallback(self._cbProcess, request, tmo) 329 | return server.NOT_DONE_YET 330 | -------------------------------------------------------------------------------- /thrift/transport/THeaderTransport.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | import struct 21 | import zlib 22 | 23 | from thrift.compat import BufferIO, byte_index 24 | from thrift.protocol.TBinaryProtocol import TBinaryProtocol 25 | from thrift.protocol.TCompactProtocol import TCompactProtocol, readVarint, writeVarint 26 | from thrift.Thrift import TApplicationException 27 | from thrift.transport.TTransport import ( 28 | CReadableTransport, 29 | TMemoryBuffer, 30 | TTransportBase, 31 | TTransportException, 32 | ) 33 | 34 | 35 | U16 = struct.Struct("!H") 36 | I32 = struct.Struct("!i") 37 | HEADER_MAGIC = 0x0FFF 38 | HARD_MAX_FRAME_SIZE = 0x3FFFFFFF 39 | 40 | 41 | class THeaderClientType(object): 42 | HEADERS = 0x00 43 | 44 | FRAMED_BINARY = 0x01 45 | UNFRAMED_BINARY = 0x02 46 | 47 | FRAMED_COMPACT = 0x03 48 | UNFRAMED_COMPACT = 0x04 49 | 50 | 51 | class THeaderSubprotocolID(object): 52 | BINARY = 0x00 53 | COMPACT = 0x02 54 | 55 | 56 | class TInfoHeaderType(object): 57 | KEY_VALUE = 0x01 58 | 59 | 60 | class THeaderTransformID(object): 61 | ZLIB = 0x01 62 | 63 | 64 | READ_TRANSFORMS_BY_ID = { 65 | THeaderTransformID.ZLIB: zlib.decompress, 66 | } 67 | 68 | 69 | WRITE_TRANSFORMS_BY_ID = { 70 | THeaderTransformID.ZLIB: zlib.compress, 71 | } 72 | 73 | 74 | def _readString(trans): 75 | size = readVarint(trans) 76 | if size < 0: 77 | raise TTransportException( 78 | TTransportException.NEGATIVE_SIZE, 79 | "Negative length" 80 | ) 81 | return trans.read(size) 82 | 83 | 84 | def _writeString(trans, value): 85 | writeVarint(trans, len(value)) 86 | trans.write(value) 87 | 88 | 89 | class THeaderTransport(TTransportBase, CReadableTransport): 90 | def __init__(self, transport, allowed_client_types): 91 | self._transport = transport 92 | self._client_type = THeaderClientType.HEADERS 93 | self._allowed_client_types = allowed_client_types 94 | 95 | self._read_buffer = BufferIO(b"") 96 | self._read_headers = {} 97 | 98 | self._write_buffer = BufferIO() 99 | self._write_headers = {} 100 | self._write_transforms = [] 101 | 102 | self.flags = 0 103 | self.sequence_id = 0 104 | self._protocol_id = THeaderSubprotocolID.BINARY 105 | self._max_frame_size = HARD_MAX_FRAME_SIZE 106 | 107 | def isOpen(self): 108 | return self._transport.isOpen() 109 | 110 | def open(self): 111 | return self._transport.open() 112 | 113 | def close(self): 114 | return self._transport.close() 115 | 116 | def get_headers(self): 117 | return self._read_headers 118 | 119 | def set_header(self, key, value): 120 | if not isinstance(key, bytes): 121 | raise ValueError("header names must be bytes") 122 | if not isinstance(value, bytes): 123 | raise ValueError("header values must be bytes") 124 | self._write_headers[key] = value 125 | 126 | def clear_headers(self): 127 | self._write_headers.clear() 128 | 129 | def add_transform(self, transform_id): 130 | if transform_id not in WRITE_TRANSFORMS_BY_ID: 131 | raise ValueError("unknown transform") 132 | self._write_transforms.append(transform_id) 133 | 134 | def set_max_frame_size(self, size): 135 | if not 0 < size < HARD_MAX_FRAME_SIZE: 136 | raise ValueError("maximum frame size should be < %d and > 0" % HARD_MAX_FRAME_SIZE) 137 | self._max_frame_size = size 138 | 139 | @property 140 | def protocol_id(self): 141 | if self._client_type == THeaderClientType.HEADERS: 142 | return self._protocol_id 143 | elif self._client_type in (THeaderClientType.FRAMED_BINARY, THeaderClientType.UNFRAMED_BINARY): 144 | return THeaderSubprotocolID.BINARY 145 | elif self._client_type in (THeaderClientType.FRAMED_COMPACT, THeaderClientType.UNFRAMED_COMPACT): 146 | return THeaderSubprotocolID.COMPACT 147 | else: 148 | raise TTransportException( 149 | TTransportException.INVALID_CLIENT_TYPE, 150 | "Protocol ID not know for client type %d" % self._client_type, 151 | ) 152 | 153 | def read(self, sz): 154 | # if there are bytes left in the buffer, produce those first. 155 | bytes_read = self._read_buffer.read(sz) 156 | bytes_left_to_read = sz - len(bytes_read) 157 | if bytes_left_to_read == 0: 158 | return bytes_read 159 | 160 | # if we've determined this is an unframed client, just pass the read 161 | # through to the underlying transport until we're reset again at the 162 | # beginning of the next message. 163 | if self._client_type in (THeaderClientType.UNFRAMED_BINARY, THeaderClientType.UNFRAMED_COMPACT): 164 | return bytes_read + self._transport.read(bytes_left_to_read) 165 | 166 | # we're empty and (maybe) framed. fill the buffers with the next frame. 167 | self.readFrame(bytes_left_to_read) 168 | return bytes_read + self._read_buffer.read(bytes_left_to_read) 169 | 170 | def _set_client_type(self, client_type): 171 | if client_type not in self._allowed_client_types: 172 | raise TTransportException( 173 | TTransportException.INVALID_CLIENT_TYPE, 174 | "Client type %d not allowed by server." % client_type, 175 | ) 176 | self._client_type = client_type 177 | 178 | def readFrame(self, req_sz): 179 | # the first word could either be the length field of a framed message 180 | # or the first bytes of an unframed message. 181 | first_word = self._transport.readAll(I32.size) 182 | frame_size, = I32.unpack(first_word) 183 | is_unframed = False 184 | if frame_size & TBinaryProtocol.VERSION_MASK == TBinaryProtocol.VERSION_1: 185 | self._set_client_type(THeaderClientType.UNFRAMED_BINARY) 186 | is_unframed = True 187 | elif (byte_index(first_word, 0) == TCompactProtocol.PROTOCOL_ID and 188 | byte_index(first_word, 1) & TCompactProtocol.VERSION_MASK == TCompactProtocol.VERSION): 189 | self._set_client_type(THeaderClientType.UNFRAMED_COMPACT) 190 | is_unframed = True 191 | 192 | if is_unframed: 193 | bytes_left_to_read = req_sz - I32.size 194 | if bytes_left_to_read > 0: 195 | rest = self._transport.read(bytes_left_to_read) 196 | else: 197 | rest = b"" 198 | self._read_buffer = BufferIO(first_word + rest) 199 | return 200 | 201 | # ok, we're still here so we're framed. 202 | if frame_size > self._max_frame_size: 203 | raise TTransportException( 204 | TTransportException.SIZE_LIMIT, 205 | "Frame was too large.", 206 | ) 207 | read_buffer = BufferIO(self._transport.readAll(frame_size)) 208 | 209 | # the next word is either going to be the version field of a 210 | # binary/compact protocol message or the magic value + flags of a 211 | # header protocol message. 212 | second_word = read_buffer.read(I32.size) 213 | version, = I32.unpack(second_word) 214 | read_buffer.seek(0) 215 | if version >> 16 == HEADER_MAGIC: 216 | self._set_client_type(THeaderClientType.HEADERS) 217 | self._read_buffer = self._parse_header_format(read_buffer) 218 | elif version & TBinaryProtocol.VERSION_MASK == TBinaryProtocol.VERSION_1: 219 | self._set_client_type(THeaderClientType.FRAMED_BINARY) 220 | self._read_buffer = read_buffer 221 | elif (byte_index(second_word, 0) == TCompactProtocol.PROTOCOL_ID and 222 | byte_index(second_word, 1) & TCompactProtocol.VERSION_MASK == TCompactProtocol.VERSION): 223 | self._set_client_type(THeaderClientType.FRAMED_COMPACT) 224 | self._read_buffer = read_buffer 225 | else: 226 | raise TTransportException( 227 | TTransportException.INVALID_CLIENT_TYPE, 228 | "Could not detect client transport type.", 229 | ) 230 | 231 | def _parse_header_format(self, buffer): 232 | # make BufferIO look like TTransport for varint helpers 233 | buffer_transport = TMemoryBuffer() 234 | buffer_transport._buffer = buffer 235 | 236 | buffer.read(2) # discard the magic bytes 237 | self.flags, = U16.unpack(buffer.read(U16.size)) 238 | self.sequence_id, = I32.unpack(buffer.read(I32.size)) 239 | 240 | header_length = U16.unpack(buffer.read(U16.size))[0] * 4 241 | end_of_headers = buffer.tell() + header_length 242 | if end_of_headers > len(buffer.getvalue()): 243 | raise TTransportException( 244 | TTransportException.SIZE_LIMIT, 245 | "Header size is larger than whole frame.", 246 | ) 247 | 248 | self._protocol_id = readVarint(buffer_transport) 249 | 250 | transforms = [] 251 | transform_count = readVarint(buffer_transport) 252 | for _ in range(transform_count): 253 | transform_id = readVarint(buffer_transport) 254 | if transform_id not in READ_TRANSFORMS_BY_ID: 255 | raise TApplicationException( 256 | TApplicationException.INVALID_TRANSFORM, 257 | "Unknown transform: %d" % transform_id, 258 | ) 259 | transforms.append(transform_id) 260 | transforms.reverse() 261 | 262 | headers = {} 263 | while buffer.tell() < end_of_headers: 264 | header_type = readVarint(buffer_transport) 265 | if header_type == TInfoHeaderType.KEY_VALUE: 266 | count = readVarint(buffer_transport) 267 | for _ in range(count): 268 | key = _readString(buffer_transport) 269 | value = _readString(buffer_transport) 270 | headers[key] = value 271 | else: 272 | break # ignore unknown headers 273 | self._read_headers = headers 274 | 275 | # skip padding / anything we didn't understand 276 | buffer.seek(end_of_headers) 277 | 278 | payload = buffer.read() 279 | for transform_id in transforms: 280 | transform_fn = READ_TRANSFORMS_BY_ID[transform_id] 281 | payload = transform_fn(payload) 282 | return BufferIO(payload) 283 | 284 | def write(self, buf): 285 | self._write_buffer.write(buf) 286 | 287 | def flush(self): 288 | payload = self._write_buffer.getvalue() 289 | self._write_buffer = BufferIO() 290 | 291 | buffer = BufferIO() 292 | if self._client_type == THeaderClientType.HEADERS: 293 | for transform_id in self._write_transforms: 294 | transform_fn = WRITE_TRANSFORMS_BY_ID[transform_id] 295 | payload = transform_fn(payload) 296 | 297 | headers = BufferIO() 298 | writeVarint(headers, self._protocol_id) 299 | writeVarint(headers, len(self._write_transforms)) 300 | for transform_id in self._write_transforms: 301 | writeVarint(headers, transform_id) 302 | if self._write_headers: 303 | writeVarint(headers, TInfoHeaderType.KEY_VALUE) 304 | writeVarint(headers, len(self._write_headers)) 305 | for key, value in self._write_headers.items(): 306 | _writeString(headers, key) 307 | _writeString(headers, value) 308 | self._write_headers = {} 309 | padding_needed = (4 - (len(headers.getvalue()) % 4)) % 4 310 | headers.write(b"\x00" * padding_needed) 311 | header_bytes = headers.getvalue() 312 | 313 | buffer.write(I32.pack(10 + len(header_bytes) + len(payload))) 314 | buffer.write(U16.pack(HEADER_MAGIC)) 315 | buffer.write(U16.pack(self.flags)) 316 | buffer.write(I32.pack(self.sequence_id)) 317 | buffer.write(U16.pack(len(header_bytes) // 4)) 318 | buffer.write(header_bytes) 319 | buffer.write(payload) 320 | elif self._client_type in (THeaderClientType.FRAMED_BINARY, THeaderClientType.FRAMED_COMPACT): 321 | buffer.write(I32.pack(len(payload))) 322 | buffer.write(payload) 323 | elif self._client_type in (THeaderClientType.UNFRAMED_BINARY, THeaderClientType.UNFRAMED_COMPACT): 324 | buffer.write(payload) 325 | else: 326 | raise TTransportException( 327 | TTransportException.INVALID_CLIENT_TYPE, 328 | "Unknown client type.", 329 | ) 330 | 331 | # the frame length field doesn't count towards the frame payload size 332 | frame_bytes = buffer.getvalue() 333 | frame_payload_size = len(frame_bytes) - 4 334 | if frame_payload_size > self._max_frame_size: 335 | raise TTransportException( 336 | TTransportException.SIZE_LIMIT, 337 | "Attempting to send frame that is too large.", 338 | ) 339 | 340 | self._transport.write(frame_bytes) 341 | self._transport.flush() 342 | 343 | @property 344 | def cstringio_buf(self): 345 | return self._read_buffer 346 | 347 | def cstringio_refill(self, partialread, reqlen): 348 | result = bytearray(partialread) 349 | while len(result) < reqlen: 350 | result += self.read(reqlen - len(result)) 351 | self._read_buffer = BufferIO(result) 352 | return self._read_buffer 353 | -------------------------------------------------------------------------------- /thrift/transport/TTransport.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | from struct import pack, unpack 21 | from thrift.Thrift import TException 22 | from ..compat import BufferIO 23 | 24 | 25 | class TTransportException(TException): 26 | """Custom Transport Exception class""" 27 | 28 | UNKNOWN = 0 29 | NOT_OPEN = 1 30 | ALREADY_OPEN = 2 31 | TIMED_OUT = 3 32 | END_OF_FILE = 4 33 | NEGATIVE_SIZE = 5 34 | SIZE_LIMIT = 6 35 | INVALID_CLIENT_TYPE = 7 36 | 37 | def __init__(self, type=UNKNOWN, message=None, inner=None): 38 | TException.__init__(self, message) 39 | self.type = type 40 | self.inner = inner 41 | 42 | 43 | class TTransportBase(object): 44 | """Base class for Thrift transport layer.""" 45 | 46 | def isOpen(self): 47 | pass 48 | 49 | def open(self): 50 | pass 51 | 52 | def close(self): 53 | pass 54 | 55 | def read(self, sz): 56 | pass 57 | 58 | def readAll(self, sz): 59 | buff = b'' 60 | have = 0 61 | while (have < sz): 62 | chunk = self.read(sz - have) 63 | chunkLen = len(chunk) 64 | have += chunkLen 65 | buff += chunk 66 | 67 | if chunkLen == 0: 68 | raise EOFError() 69 | 70 | return buff 71 | 72 | def write(self, buf): 73 | pass 74 | 75 | def flush(self): 76 | pass 77 | 78 | 79 | # This class should be thought of as an interface. 80 | class CReadableTransport(object): 81 | """base class for transports that are readable from C""" 82 | 83 | # TODO(dreiss): Think about changing this interface to allow us to use 84 | # a (Python, not c) StringIO instead, because it allows 85 | # you to write after reading. 86 | 87 | # NOTE: This is a classic class, so properties will NOT work 88 | # correctly for setting. 89 | @property 90 | def cstringio_buf(self): 91 | """A cStringIO buffer that contains the current chunk we are reading.""" 92 | pass 93 | 94 | def cstringio_refill(self, partialread, reqlen): 95 | """Refills cstringio_buf. 96 | 97 | Returns the currently used buffer (which can but need not be the same as 98 | the old cstringio_buf). partialread is what the C code has read from the 99 | buffer, and should be inserted into the buffer before any more reads. The 100 | return value must be a new, not borrowed reference. Something along the 101 | lines of self._buf should be fine. 102 | 103 | If reqlen bytes can't be read, throw EOFError. 104 | """ 105 | pass 106 | 107 | 108 | class TServerTransportBase(object): 109 | """Base class for Thrift server transports.""" 110 | 111 | def listen(self): 112 | pass 113 | 114 | def accept(self): 115 | pass 116 | 117 | def close(self): 118 | pass 119 | 120 | 121 | class TTransportFactoryBase(object): 122 | """Base class for a Transport Factory""" 123 | 124 | def getTransport(self, trans): 125 | return trans 126 | 127 | 128 | class TBufferedTransportFactory(object): 129 | """Factory transport that builds buffered transports""" 130 | 131 | def getTransport(self, trans): 132 | buffered = TBufferedTransport(trans) 133 | return buffered 134 | 135 | 136 | class TBufferedTransport(TTransportBase, CReadableTransport): 137 | """Class that wraps another transport and buffers its I/O. 138 | 139 | The implementation uses a (configurable) fixed-size read buffer 140 | but buffers all writes until a flush is performed. 141 | """ 142 | DEFAULT_BUFFER = 4096 143 | 144 | def __init__(self, trans, rbuf_size=DEFAULT_BUFFER): 145 | self.__trans = trans 146 | self.__wbuf = BufferIO() 147 | # Pass string argument to initialize read buffer as cStringIO.InputType 148 | self.__rbuf = BufferIO(b'') 149 | self.__rbuf_size = rbuf_size 150 | 151 | def isOpen(self): 152 | return self.__trans.isOpen() 153 | 154 | def open(self): 155 | return self.__trans.open() 156 | 157 | def close(self): 158 | return self.__trans.close() 159 | 160 | def read(self, sz): 161 | ret = self.__rbuf.read(sz) 162 | if len(ret) != 0: 163 | return ret 164 | self.__rbuf = BufferIO(self.__trans.read(max(sz, self.__rbuf_size))) 165 | return self.__rbuf.read(sz) 166 | 167 | def write(self, buf): 168 | try: 169 | self.__wbuf.write(buf) 170 | except Exception as e: 171 | # on exception reset wbuf so it doesn't contain a partial function call 172 | self.__wbuf = BufferIO() 173 | raise e 174 | 175 | def flush(self): 176 | out = self.__wbuf.getvalue() 177 | # reset wbuf before write/flush to preserve state on underlying failure 178 | self.__wbuf = BufferIO() 179 | self.__trans.write(out) 180 | self.__trans.flush() 181 | 182 | # Implement the CReadableTransport interface. 183 | @property 184 | def cstringio_buf(self): 185 | return self.__rbuf 186 | 187 | def cstringio_refill(self, partialread, reqlen): 188 | retstring = partialread 189 | if reqlen < self.__rbuf_size: 190 | # try to make a read of as much as we can. 191 | retstring += self.__trans.read(self.__rbuf_size) 192 | 193 | # but make sure we do read reqlen bytes. 194 | if len(retstring) < reqlen: 195 | retstring += self.__trans.readAll(reqlen - len(retstring)) 196 | 197 | self.__rbuf = BufferIO(retstring) 198 | return self.__rbuf 199 | 200 | 201 | class TMemoryBuffer(TTransportBase, CReadableTransport): 202 | """Wraps a cBytesIO object as a TTransport. 203 | 204 | NOTE: Unlike the C++ version of this class, you cannot write to it 205 | then immediately read from it. If you want to read from a 206 | TMemoryBuffer, you must either pass a string to the constructor. 207 | TODO(dreiss): Make this work like the C++ version. 208 | """ 209 | 210 | def __init__(self, value=None, offset=0): 211 | """value -- a value to read from for stringio 212 | 213 | If value is set, this will be a transport for reading, 214 | otherwise, it is for writing""" 215 | if value is not None: 216 | self._buffer = BufferIO(value) 217 | else: 218 | self._buffer = BufferIO() 219 | if offset: 220 | self._buffer.seek(offset) 221 | 222 | def isOpen(self): 223 | return not self._buffer.closed 224 | 225 | def open(self): 226 | pass 227 | 228 | def close(self): 229 | self._buffer.close() 230 | 231 | def read(self, sz): 232 | return self._buffer.read(sz) 233 | 234 | def write(self, buf): 235 | self._buffer.write(buf) 236 | 237 | def flush(self): 238 | pass 239 | 240 | def getvalue(self): 241 | return self._buffer.getvalue() 242 | 243 | # Implement the CReadableTransport interface. 244 | @property 245 | def cstringio_buf(self): 246 | return self._buffer 247 | 248 | def cstringio_refill(self, partialread, reqlen): 249 | # only one shot at reading... 250 | raise EOFError() 251 | 252 | 253 | class TFramedTransportFactory(object): 254 | """Factory transport that builds framed transports""" 255 | 256 | def getTransport(self, trans): 257 | framed = TFramedTransport(trans) 258 | return framed 259 | 260 | 261 | class TFramedTransport(TTransportBase, CReadableTransport): 262 | """Class that wraps another transport and frames its I/O when writing.""" 263 | 264 | def __init__(self, trans,): 265 | self.__trans = trans 266 | self.__rbuf = BufferIO(b'') 267 | self.__wbuf = BufferIO() 268 | 269 | def isOpen(self): 270 | return self.__trans.isOpen() 271 | 272 | def open(self): 273 | return self.__trans.open() 274 | 275 | def close(self): 276 | return self.__trans.close() 277 | 278 | def read(self, sz): 279 | ret = self.__rbuf.read(sz) 280 | if len(ret) != 0: 281 | return ret 282 | 283 | self.readFrame() 284 | return self.__rbuf.read(sz) 285 | 286 | def readFrame(self): 287 | buff = self.__trans.readAll(4) 288 | sz, = unpack('!i', buff) 289 | self.__rbuf = BufferIO(self.__trans.readAll(sz)) 290 | 291 | def write(self, buf): 292 | self.__wbuf.write(buf) 293 | 294 | def flush(self): 295 | wout = self.__wbuf.getvalue() 296 | wsz = len(wout) 297 | # reset wbuf before write/flush to preserve state on underlying failure 298 | self.__wbuf = BufferIO() 299 | # N.B.: Doing this string concatenation is WAY cheaper than making 300 | # two separate calls to the underlying socket object. Socket writes in 301 | # Python turn out to be REALLY expensive, but it seems to do a pretty 302 | # good job of managing string buffer operations without excessive copies 303 | buf = pack("!i", wsz) + wout 304 | self.__trans.write(buf) 305 | self.__trans.flush() 306 | 307 | # Implement the CReadableTransport interface. 308 | @property 309 | def cstringio_buf(self): 310 | return self.__rbuf 311 | 312 | def cstringio_refill(self, prefix, reqlen): 313 | # self.__rbuf will already be empty here because fastbinary doesn't 314 | # ask for a refill until the previous buffer is empty. Therefore, 315 | # we can start reading new frames immediately. 316 | while len(prefix) < reqlen: 317 | self.readFrame() 318 | prefix += self.__rbuf.getvalue() 319 | self.__rbuf = BufferIO(prefix) 320 | return self.__rbuf 321 | 322 | 323 | class TFileObjectTransport(TTransportBase): 324 | """Wraps a file-like object to make it work as a Thrift transport.""" 325 | 326 | def __init__(self, fileobj): 327 | self.fileobj = fileobj 328 | 329 | def isOpen(self): 330 | return True 331 | 332 | def close(self): 333 | self.fileobj.close() 334 | 335 | def read(self, sz): 336 | return self.fileobj.read(sz) 337 | 338 | def write(self, buf): 339 | self.fileobj.write(buf) 340 | 341 | def flush(self): 342 | self.fileobj.flush() 343 | 344 | 345 | class TSaslClientTransport(TTransportBase, CReadableTransport): 346 | """ 347 | SASL transport 348 | """ 349 | 350 | START = 1 351 | OK = 2 352 | BAD = 3 353 | ERROR = 4 354 | COMPLETE = 5 355 | 356 | def __init__(self, transport, host, service, mechanism='GSSAPI', 357 | **sasl_kwargs): 358 | """ 359 | transport: an underlying transport to use, typically just a TSocket 360 | host: the name of the server, from a SASL perspective 361 | service: the name of the server's service, from a SASL perspective 362 | mechanism: the name of the preferred mechanism to use 363 | 364 | All other kwargs will be passed to the puresasl.client.SASLClient 365 | constructor. 366 | """ 367 | 368 | from puresasl.client import SASLClient 369 | 370 | self.transport = transport 371 | self.sasl = SASLClient(host, service, mechanism, **sasl_kwargs) 372 | 373 | self.__wbuf = BufferIO() 374 | self.__rbuf = BufferIO(b'') 375 | 376 | def open(self): 377 | if not self.transport.isOpen(): 378 | self.transport.open() 379 | 380 | self.send_sasl_msg(self.START, bytes(self.sasl.mechanism, 'ascii')) 381 | self.send_sasl_msg(self.OK, self.sasl.process()) 382 | 383 | while True: 384 | status, challenge = self.recv_sasl_msg() 385 | if status == self.OK: 386 | self.send_sasl_msg(self.OK, self.sasl.process(challenge)) 387 | elif status == self.COMPLETE: 388 | if not self.sasl.complete: 389 | raise TTransportException( 390 | TTransportException.NOT_OPEN, 391 | "The server erroneously indicated " 392 | "that SASL negotiation was complete") 393 | else: 394 | break 395 | else: 396 | raise TTransportException( 397 | TTransportException.NOT_OPEN, 398 | "Bad SASL negotiation status: %d (%s)" 399 | % (status, challenge)) 400 | 401 | def send_sasl_msg(self, status, body): 402 | header = pack(">BI", status, len(body)) 403 | self.transport.write(header + body) 404 | self.transport.flush() 405 | 406 | def recv_sasl_msg(self): 407 | header = self.transport.readAll(5) 408 | status, length = unpack(">BI", header) 409 | if length > 0: 410 | payload = self.transport.readAll(length) 411 | else: 412 | payload = "" 413 | return status, payload 414 | 415 | def write(self, data): 416 | self.__wbuf.write(data) 417 | 418 | def flush(self): 419 | data = self.__wbuf.getvalue() 420 | encoded = self.sasl.wrap(data) 421 | self.transport.write(pack("!i", len(encoded)) + encoded) 422 | self.transport.flush() 423 | self.__wbuf = BufferIO() 424 | 425 | def read(self, sz): 426 | ret = self.__rbuf.read(sz) 427 | if len(ret) != 0: 428 | return ret 429 | 430 | self._read_frame() 431 | return self.__rbuf.read(sz) 432 | 433 | def _read_frame(self): 434 | header = self.transport.readAll(4) 435 | length, = unpack('!i', header) 436 | encoded = self.transport.readAll(length) 437 | self.__rbuf = BufferIO(self.sasl.unwrap(encoded)) 438 | 439 | def close(self): 440 | self.sasl.dispose() 441 | self.transport.close() 442 | 443 | # based on TFramedTransport 444 | @property 445 | def cstringio_buf(self): 446 | return self.__rbuf 447 | 448 | def cstringio_refill(self, prefix, reqlen): 449 | # self.__rbuf will already be empty here because fastbinary doesn't 450 | # ask for a refill until the previous buffer is empty. Therefore, 451 | # we can start reading new frames immediately. 452 | while len(prefix) < reqlen: 453 | self._read_frame() 454 | prefix += self.__rbuf.getvalue() 455 | self.__rbuf = BufferIO(prefix) 456 | return self.__rbuf 457 | -------------------------------------------------------------------------------- /thrift/protocol/TProtocol.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | from thrift.Thrift import TException, TType, TFrozenDict 21 | from thrift.transport.TTransport import TTransportException 22 | from ..compat import binary_to_str, str_to_binary 23 | 24 | import six 25 | import sys 26 | from itertools import islice 27 | from six.moves import zip 28 | 29 | 30 | class TProtocolException(TException): 31 | """Custom Protocol Exception class""" 32 | 33 | UNKNOWN = 0 34 | INVALID_DATA = 1 35 | NEGATIVE_SIZE = 2 36 | SIZE_LIMIT = 3 37 | BAD_VERSION = 4 38 | NOT_IMPLEMENTED = 5 39 | DEPTH_LIMIT = 6 40 | INVALID_PROTOCOL = 7 41 | 42 | def __init__(self, type=UNKNOWN, message=None): 43 | TException.__init__(self, message) 44 | self.type = type 45 | 46 | 47 | class TProtocolBase(object): 48 | """Base class for Thrift protocol driver.""" 49 | 50 | def __init__(self, trans): 51 | self.trans = trans 52 | self._fast_decode = None 53 | self._fast_encode = None 54 | 55 | @staticmethod 56 | def _check_length(limit, length): 57 | if length < 0: 58 | raise TTransportException(TTransportException.NEGATIVE_SIZE, 59 | 'Negative length: %d' % length) 60 | if limit is not None and length > limit: 61 | raise TTransportException(TTransportException.SIZE_LIMIT, 62 | 'Length exceeded max allowed: %d' % limit) 63 | 64 | def writeMessageBegin(self, name, ttype, seqid): 65 | pass 66 | 67 | def writeMessageEnd(self): 68 | pass 69 | 70 | def writeStructBegin(self, name): 71 | pass 72 | 73 | def writeStructEnd(self): 74 | pass 75 | 76 | def writeFieldBegin(self, name, ttype, fid): 77 | pass 78 | 79 | def writeFieldEnd(self): 80 | pass 81 | 82 | def writeFieldStop(self): 83 | pass 84 | 85 | def writeMapBegin(self, ktype, vtype, size): 86 | pass 87 | 88 | def writeMapEnd(self): 89 | pass 90 | 91 | def writeListBegin(self, etype, size): 92 | pass 93 | 94 | def writeListEnd(self): 95 | pass 96 | 97 | def writeSetBegin(self, etype, size): 98 | pass 99 | 100 | def writeSetEnd(self): 101 | pass 102 | 103 | def writeBool(self, bool_val): 104 | pass 105 | 106 | def writeByte(self, byte): 107 | pass 108 | 109 | def writeI16(self, i16): 110 | pass 111 | 112 | def writeI32(self, i32): 113 | pass 114 | 115 | def writeI64(self, i64): 116 | pass 117 | 118 | def writeDouble(self, dub): 119 | pass 120 | 121 | def writeString(self, str_val): 122 | self.writeBinary(str_to_binary(str_val)) 123 | 124 | def writeBinary(self, str_val): 125 | pass 126 | 127 | def writeUtf8(self, str_val): 128 | self.writeString(str_val.encode('utf8')) 129 | 130 | def readMessageBegin(self): 131 | pass 132 | 133 | def readMessageEnd(self): 134 | pass 135 | 136 | def readStructBegin(self): 137 | pass 138 | 139 | def readStructEnd(self): 140 | pass 141 | 142 | def readFieldBegin(self): 143 | pass 144 | 145 | def readFieldEnd(self): 146 | pass 147 | 148 | def readMapBegin(self): 149 | pass 150 | 151 | def readMapEnd(self): 152 | pass 153 | 154 | def readListBegin(self): 155 | pass 156 | 157 | def readListEnd(self): 158 | pass 159 | 160 | def readSetBegin(self): 161 | pass 162 | 163 | def readSetEnd(self): 164 | pass 165 | 166 | def readBool(self): 167 | pass 168 | 169 | def readByte(self): 170 | pass 171 | 172 | def readI16(self): 173 | pass 174 | 175 | def readI32(self): 176 | pass 177 | 178 | def readI64(self): 179 | pass 180 | 181 | def readDouble(self): 182 | pass 183 | 184 | def readString(self): 185 | return binary_to_str(self.readBinary()) 186 | 187 | def readBinary(self): 188 | pass 189 | 190 | def readUtf8(self): 191 | return self.readString().decode('utf8') 192 | 193 | def skip(self, ttype): 194 | if ttype == TType.BOOL: 195 | self.readBool() 196 | elif ttype == TType.BYTE: 197 | self.readByte() 198 | elif ttype == TType.I16: 199 | self.readI16() 200 | elif ttype == TType.I32: 201 | self.readI32() 202 | elif ttype == TType.I64: 203 | self.readI64() 204 | elif ttype == TType.DOUBLE: 205 | self.readDouble() 206 | elif ttype == TType.STRING: 207 | self.readString() 208 | elif ttype == TType.STRUCT: 209 | name = self.readStructBegin() 210 | while True: 211 | (name, ttype, id) = self.readFieldBegin() 212 | if ttype == TType.STOP: 213 | break 214 | self.skip(ttype) 215 | self.readFieldEnd() 216 | self.readStructEnd() 217 | elif ttype == TType.MAP: 218 | (ktype, vtype, size) = self.readMapBegin() 219 | for i in range(size): 220 | self.skip(ktype) 221 | self.skip(vtype) 222 | self.readMapEnd() 223 | elif ttype == TType.SET: 224 | (etype, size) = self.readSetBegin() 225 | for i in range(size): 226 | self.skip(etype) 227 | self.readSetEnd() 228 | elif ttype == TType.LIST: 229 | (etype, size) = self.readListBegin() 230 | for i in range(size): 231 | self.skip(etype) 232 | self.readListEnd() 233 | else: 234 | raise TProtocolException( 235 | TProtocolException.INVALID_DATA, 236 | "invalid TType") 237 | 238 | # tuple of: ( 'reader method' name, is_container bool, 'writer_method' name ) 239 | _TTYPE_HANDLERS = ( 240 | (None, None, False), # 0 TType.STOP 241 | (None, None, False), # 1 TType.VOID # TODO: handle void? 242 | ('readBool', 'writeBool', False), # 2 TType.BOOL 243 | ('readByte', 'writeByte', False), # 3 TType.BYTE and I08 244 | ('readDouble', 'writeDouble', False), # 4 TType.DOUBLE 245 | (None, None, False), # 5 undefined 246 | ('readI16', 'writeI16', False), # 6 TType.I16 247 | (None, None, False), # 7 undefined 248 | ('readI32', 'writeI32', False), # 8 TType.I32 249 | (None, None, False), # 9 undefined 250 | ('readI64', 'writeI64', False), # 10 TType.I64 251 | ('readString', 'writeString', False), # 11 TType.STRING and UTF7 252 | ('readContainerStruct', 'writeContainerStruct', True), # 12 *.STRUCT 253 | ('readContainerMap', 'writeContainerMap', True), # 13 TType.MAP 254 | ('readContainerSet', 'writeContainerSet', True), # 14 TType.SET 255 | ('readContainerList', 'writeContainerList', True), # 15 TType.LIST 256 | (None, None, False), # 16 TType.UTF8 # TODO: handle utf8 types? 257 | (None, None, False) # 17 TType.UTF16 # TODO: handle utf16 types? 258 | ) 259 | 260 | def _ttype_handlers(self, ttype, spec): 261 | if spec == 'BINARY': 262 | if ttype != TType.STRING: 263 | raise TProtocolException(type=TProtocolException.INVALID_DATA, 264 | message='Invalid binary field type %d' % ttype) 265 | return ('readBinary', 'writeBinary', False) 266 | if sys.version_info[0] == 2 and spec == 'UTF8': 267 | if ttype != TType.STRING: 268 | raise TProtocolException(type=TProtocolException.INVALID_DATA, 269 | message='Invalid string field type %d' % ttype) 270 | return ('readUtf8', 'writeUtf8', False) 271 | return self._TTYPE_HANDLERS[ttype] if ttype < len(self._TTYPE_HANDLERS) else (None, None, False) 272 | 273 | def _read_by_ttype(self, ttype, spec, espec): 274 | reader_name, _, is_container = self._ttype_handlers(ttype, espec) 275 | if reader_name is None: 276 | raise TProtocolException(type=TProtocolException.INVALID_DATA, 277 | message='Invalid type %d' % (ttype)) 278 | reader_func = getattr(self, reader_name) 279 | read = (lambda: reader_func(espec)) if is_container else reader_func 280 | while True: 281 | yield read() 282 | 283 | def readFieldByTType(self, ttype, spec): 284 | return next(self._read_by_ttype(ttype, spec, spec)) 285 | 286 | def readContainerList(self, spec): 287 | ttype, tspec, is_immutable = spec 288 | (list_type, list_len) = self.readListBegin() 289 | # TODO: compare types we just decoded with thrift_spec 290 | elems = islice(self._read_by_ttype(ttype, spec, tspec), list_len) 291 | results = (tuple if is_immutable else list)(elems) 292 | self.readListEnd() 293 | return results 294 | 295 | def readContainerSet(self, spec): 296 | ttype, tspec, is_immutable = spec 297 | (set_type, set_len) = self.readSetBegin() 298 | # TODO: compare types we just decoded with thrift_spec 299 | elems = islice(self._read_by_ttype(ttype, spec, tspec), set_len) 300 | results = (frozenset if is_immutable else set)(elems) 301 | self.readSetEnd() 302 | return results 303 | 304 | def readContainerStruct(self, spec): 305 | (obj_class, obj_spec) = spec 306 | obj = obj_class() 307 | obj.read(self) 308 | return obj 309 | 310 | def readContainerMap(self, spec): 311 | ktype, kspec, vtype, vspec, is_immutable = spec 312 | (map_ktype, map_vtype, map_len) = self.readMapBegin() 313 | # TODO: compare types we just decoded with thrift_spec and 314 | # abort/skip if types disagree 315 | keys = self._read_by_ttype(ktype, spec, kspec) 316 | vals = self._read_by_ttype(vtype, spec, vspec) 317 | keyvals = islice(zip(keys, vals), map_len) 318 | results = (TFrozenDict if is_immutable else dict)(keyvals) 319 | self.readMapEnd() 320 | return results 321 | 322 | def readStruct(self, obj, thrift_spec, is_immutable=False): 323 | if is_immutable: 324 | fields = {} 325 | self.readStructBegin() 326 | while True: 327 | (fname, ftype, fid) = self.readFieldBegin() 328 | if ftype == TType.STOP: 329 | break 330 | try: 331 | field = thrift_spec[fid] 332 | except IndexError: 333 | self.skip(ftype) 334 | else: 335 | if field is not None and ftype == field[1]: 336 | fname = field[2] 337 | fspec = field[3] 338 | val = self.readFieldByTType(ftype, fspec) 339 | if is_immutable: 340 | fields[fname] = val 341 | else: 342 | setattr(obj, fname, val) 343 | else: 344 | self.skip(ftype) 345 | self.readFieldEnd() 346 | self.readStructEnd() 347 | if is_immutable: 348 | return obj(**fields) 349 | 350 | def writeContainerStruct(self, val, spec): 351 | val.write(self) 352 | 353 | def writeContainerList(self, val, spec): 354 | ttype, tspec, _ = spec 355 | self.writeListBegin(ttype, len(val)) 356 | for _ in self._write_by_ttype(ttype, val, spec, tspec): 357 | pass 358 | self.writeListEnd() 359 | 360 | def writeContainerSet(self, val, spec): 361 | ttype, tspec, _ = spec 362 | self.writeSetBegin(ttype, len(val)) 363 | for _ in self._write_by_ttype(ttype, val, spec, tspec): 364 | pass 365 | self.writeSetEnd() 366 | 367 | def writeContainerMap(self, val, spec): 368 | ktype, kspec, vtype, vspec, _ = spec 369 | self.writeMapBegin(ktype, vtype, len(val)) 370 | for _ in zip(self._write_by_ttype(ktype, six.iterkeys(val), spec, kspec), 371 | self._write_by_ttype(vtype, six.itervalues(val), spec, vspec)): 372 | pass 373 | self.writeMapEnd() 374 | 375 | def writeStruct(self, obj, thrift_spec): 376 | self.writeStructBegin(obj.__class__.__name__) 377 | for field in thrift_spec: 378 | if field is None: 379 | continue 380 | fname = field[2] 381 | val = getattr(obj, fname) 382 | if val is None: 383 | # skip writing out unset fields 384 | continue 385 | fid = field[0] 386 | ftype = field[1] 387 | fspec = field[3] 388 | self.writeFieldBegin(fname, ftype, fid) 389 | self.writeFieldByTType(ftype, val, fspec) 390 | self.writeFieldEnd() 391 | self.writeFieldStop() 392 | self.writeStructEnd() 393 | 394 | def _write_by_ttype(self, ttype, vals, spec, espec): 395 | _, writer_name, is_container = self._ttype_handlers(ttype, espec) 396 | writer_func = getattr(self, writer_name) 397 | write = (lambda v: writer_func(v, espec)) if is_container else writer_func 398 | for v in vals: 399 | yield write(v) 400 | 401 | def writeFieldByTType(self, ttype, val, spec): 402 | next(self._write_by_ttype(ttype, [val], spec, spec)) 403 | 404 | 405 | def checkIntegerLimits(i, bits): 406 | if bits == 8 and (i < -128 or i > 127): 407 | raise TProtocolException(TProtocolException.INVALID_DATA, 408 | "i8 requires -128 <= number <= 127") 409 | elif bits == 16 and (i < -32768 or i > 32767): 410 | raise TProtocolException(TProtocolException.INVALID_DATA, 411 | "i16 requires -32768 <= number <= 32767") 412 | elif bits == 32 and (i < -2147483648 or i > 2147483647): 413 | raise TProtocolException(TProtocolException.INVALID_DATA, 414 | "i32 requires -2147483648 <= number <= 2147483647") 415 | elif bits == 64 and (i < -9223372036854775808 or i > 9223372036854775807): 416 | raise TProtocolException(TProtocolException.INVALID_DATA, 417 | "i64 requires -9223372036854775808 <= number <= 9223372036854775807") 418 | 419 | 420 | class TProtocolFactory(object): 421 | def getProtocol(self, trans): 422 | pass 423 | -------------------------------------------------------------------------------- /thrift/protocol/TCompactProtocol.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | from .TProtocol import TType, TProtocolBase, TProtocolException, TProtocolFactory, checkIntegerLimits 21 | from struct import pack, unpack 22 | 23 | from ..compat import binary_to_str, str_to_binary 24 | 25 | __all__ = ['TCompactProtocol', 'TCompactProtocolFactory'] 26 | 27 | CLEAR = 0 28 | FIELD_WRITE = 1 29 | VALUE_WRITE = 2 30 | CONTAINER_WRITE = 3 31 | BOOL_WRITE = 4 32 | FIELD_READ = 5 33 | CONTAINER_READ = 6 34 | VALUE_READ = 7 35 | BOOL_READ = 8 36 | 37 | 38 | def make_helper(v_from, container): 39 | def helper(func): 40 | def nested(self, *args, **kwargs): 41 | assert self.state in (v_from, container), (self.state, v_from, container) 42 | return func(self, *args, **kwargs) 43 | return nested 44 | return helper 45 | 46 | 47 | writer = make_helper(VALUE_WRITE, CONTAINER_WRITE) 48 | reader = make_helper(VALUE_READ, CONTAINER_READ) 49 | 50 | 51 | def makeZigZag(n, bits): 52 | checkIntegerLimits(n, bits) 53 | return (n << 1) ^ (n >> (bits - 1)) 54 | 55 | 56 | def fromZigZag(n): 57 | return (n >> 1) ^ -(n & 1) 58 | 59 | 60 | def writeVarint(trans, n): 61 | assert n >= 0, "Input to TCompactProtocol writeVarint cannot be negative!" 62 | out = bytearray() 63 | while True: 64 | if n & ~0x7f == 0: 65 | out.append(n) 66 | break 67 | else: 68 | out.append((n & 0xff) | 0x80) 69 | n = n >> 7 70 | trans.write(bytes(out)) 71 | 72 | 73 | def readVarint(trans): 74 | result = 0 75 | shift = 0 76 | while True: 77 | x = trans.readAll(1) 78 | byte = ord(x) 79 | result |= (byte & 0x7f) << shift 80 | if byte >> 7 == 0: 81 | return result 82 | shift += 7 83 | 84 | 85 | class CompactType(object): 86 | STOP = 0x00 87 | TRUE = 0x01 88 | FALSE = 0x02 89 | BYTE = 0x03 90 | I16 = 0x04 91 | I32 = 0x05 92 | I64 = 0x06 93 | DOUBLE = 0x07 94 | BINARY = 0x08 95 | LIST = 0x09 96 | SET = 0x0A 97 | MAP = 0x0B 98 | STRUCT = 0x0C 99 | 100 | 101 | CTYPES = { 102 | TType.STOP: CompactType.STOP, 103 | TType.BOOL: CompactType.TRUE, # used for collection 104 | TType.BYTE: CompactType.BYTE, 105 | TType.I16: CompactType.I16, 106 | TType.I32: CompactType.I32, 107 | TType.I64: CompactType.I64, 108 | TType.DOUBLE: CompactType.DOUBLE, 109 | TType.STRING: CompactType.BINARY, 110 | TType.STRUCT: CompactType.STRUCT, 111 | TType.LIST: CompactType.LIST, 112 | TType.SET: CompactType.SET, 113 | TType.MAP: CompactType.MAP, 114 | } 115 | 116 | TTYPES = {} 117 | for k, v in CTYPES.items(): 118 | TTYPES[v] = k 119 | TTYPES[CompactType.FALSE] = TType.BOOL 120 | del k 121 | del v 122 | 123 | 124 | class TCompactProtocol(TProtocolBase): 125 | """Compact implementation of the Thrift protocol driver.""" 126 | 127 | PROTOCOL_ID = 0x82 128 | VERSION = 1 129 | VERSION_MASK = 0x1f 130 | TYPE_MASK = 0xe0 131 | TYPE_BITS = 0x07 132 | TYPE_SHIFT_AMOUNT = 5 133 | 134 | def __init__(self, trans, 135 | string_length_limit=None, 136 | container_length_limit=None): 137 | TProtocolBase.__init__(self, trans) 138 | self.state = CLEAR 139 | self.__last_fid = 0 140 | self.__bool_fid = None 141 | self.__bool_value = None 142 | self.__structs = [] 143 | self.__containers = [] 144 | self.string_length_limit = string_length_limit 145 | self.container_length_limit = container_length_limit 146 | 147 | def _check_string_length(self, length): 148 | self._check_length(self.string_length_limit, length) 149 | 150 | def _check_container_length(self, length): 151 | self._check_length(self.container_length_limit, length) 152 | 153 | def __writeVarint(self, n): 154 | writeVarint(self.trans, n) 155 | 156 | def writeMessageBegin(self, name, type, seqid): 157 | assert self.state == CLEAR 158 | self.__writeUByte(self.PROTOCOL_ID) 159 | self.__writeUByte(self.VERSION | (type << self.TYPE_SHIFT_AMOUNT)) 160 | # The sequence id is a signed 32-bit integer but the compact protocol 161 | # writes this out as a "var int" which is always positive, and attempting 162 | # to write a negative number results in an infinite loop, so we may 163 | # need to do some conversion here... 164 | tseqid = seqid 165 | if tseqid < 0: 166 | tseqid = 2147483648 + (2147483648 + tseqid) 167 | self.__writeVarint(tseqid) 168 | self.__writeBinary(str_to_binary(name)) 169 | self.state = VALUE_WRITE 170 | 171 | def writeMessageEnd(self): 172 | assert self.state == VALUE_WRITE 173 | self.state = CLEAR 174 | 175 | def writeStructBegin(self, name): 176 | assert self.state in (CLEAR, CONTAINER_WRITE, VALUE_WRITE), self.state 177 | self.__structs.append((self.state, self.__last_fid)) 178 | self.state = FIELD_WRITE 179 | self.__last_fid = 0 180 | 181 | def writeStructEnd(self): 182 | assert self.state == FIELD_WRITE 183 | self.state, self.__last_fid = self.__structs.pop() 184 | 185 | def writeFieldStop(self): 186 | self.__writeByte(0) 187 | 188 | def __writeFieldHeader(self, type, fid): 189 | delta = fid - self.__last_fid 190 | if 0 < delta <= 15: 191 | self.__writeUByte(delta << 4 | type) 192 | else: 193 | self.__writeByte(type) 194 | self.__writeI16(fid) 195 | self.__last_fid = fid 196 | 197 | def writeFieldBegin(self, name, type, fid): 198 | assert self.state == FIELD_WRITE, self.state 199 | if type == TType.BOOL: 200 | self.state = BOOL_WRITE 201 | self.__bool_fid = fid 202 | else: 203 | self.state = VALUE_WRITE 204 | self.__writeFieldHeader(CTYPES[type], fid) 205 | 206 | def writeFieldEnd(self): 207 | assert self.state in (VALUE_WRITE, BOOL_WRITE), self.state 208 | self.state = FIELD_WRITE 209 | 210 | def __writeUByte(self, byte): 211 | self.trans.write(pack('!B', byte)) 212 | 213 | def __writeByte(self, byte): 214 | self.trans.write(pack('!b', byte)) 215 | 216 | def __writeI16(self, i16): 217 | self.__writeVarint(makeZigZag(i16, 16)) 218 | 219 | def __writeSize(self, i32): 220 | self.__writeVarint(i32) 221 | 222 | def writeCollectionBegin(self, etype, size): 223 | assert self.state in (VALUE_WRITE, CONTAINER_WRITE), self.state 224 | if size <= 14: 225 | self.__writeUByte(size << 4 | CTYPES[etype]) 226 | else: 227 | self.__writeUByte(0xf0 | CTYPES[etype]) 228 | self.__writeSize(size) 229 | self.__containers.append(self.state) 230 | self.state = CONTAINER_WRITE 231 | writeSetBegin = writeCollectionBegin 232 | writeListBegin = writeCollectionBegin 233 | 234 | def writeMapBegin(self, ktype, vtype, size): 235 | assert self.state in (VALUE_WRITE, CONTAINER_WRITE), self.state 236 | if size == 0: 237 | self.__writeByte(0) 238 | else: 239 | self.__writeSize(size) 240 | self.__writeUByte(CTYPES[ktype] << 4 | CTYPES[vtype]) 241 | self.__containers.append(self.state) 242 | self.state = CONTAINER_WRITE 243 | 244 | def writeCollectionEnd(self): 245 | assert self.state == CONTAINER_WRITE, self.state 246 | self.state = self.__containers.pop() 247 | writeMapEnd = writeCollectionEnd 248 | writeSetEnd = writeCollectionEnd 249 | writeListEnd = writeCollectionEnd 250 | 251 | def writeBool(self, bool): 252 | if self.state == BOOL_WRITE: 253 | if bool: 254 | ctype = CompactType.TRUE 255 | else: 256 | ctype = CompactType.FALSE 257 | self.__writeFieldHeader(ctype, self.__bool_fid) 258 | elif self.state == CONTAINER_WRITE: 259 | if bool: 260 | self.__writeByte(CompactType.TRUE) 261 | else: 262 | self.__writeByte(CompactType.FALSE) 263 | else: 264 | raise AssertionError("Invalid state in compact protocol") 265 | 266 | writeByte = writer(__writeByte) 267 | writeI16 = writer(__writeI16) 268 | 269 | @writer 270 | def writeI32(self, i32): 271 | self.__writeVarint(makeZigZag(i32, 32)) 272 | 273 | @writer 274 | def writeI64(self, i64): 275 | self.__writeVarint(makeZigZag(i64, 64)) 276 | 277 | @writer 278 | def writeDouble(self, dub): 279 | self.trans.write(pack('> 4 292 | if delta == 0: 293 | fid = self.__readI16() 294 | else: 295 | fid = self.__last_fid + delta 296 | self.__last_fid = fid 297 | type = type & 0x0f 298 | if type == CompactType.TRUE: 299 | self.state = BOOL_READ 300 | self.__bool_value = True 301 | elif type == CompactType.FALSE: 302 | self.state = BOOL_READ 303 | self.__bool_value = False 304 | else: 305 | self.state = VALUE_READ 306 | return (None, self.__getTType(type), fid) 307 | 308 | def readFieldEnd(self): 309 | assert self.state in (VALUE_READ, BOOL_READ), self.state 310 | self.state = FIELD_READ 311 | 312 | def __readUByte(self): 313 | result, = unpack('!B', self.trans.readAll(1)) 314 | return result 315 | 316 | def __readByte(self): 317 | result, = unpack('!b', self.trans.readAll(1)) 318 | return result 319 | 320 | def __readVarint(self): 321 | return readVarint(self.trans) 322 | 323 | def __readZigZag(self): 324 | return fromZigZag(self.__readVarint()) 325 | 326 | def __readSize(self): 327 | result = self.__readVarint() 328 | if result < 0: 329 | raise TProtocolException("Length < 0") 330 | return result 331 | 332 | def readMessageBegin(self): 333 | assert self.state == CLEAR 334 | proto_id = self.__readUByte() 335 | if proto_id != self.PROTOCOL_ID: 336 | raise TProtocolException(TProtocolException.BAD_VERSION, 337 | 'Bad protocol id in the message: %d' % proto_id) 338 | ver_type = self.__readUByte() 339 | type = (ver_type >> self.TYPE_SHIFT_AMOUNT) & self.TYPE_BITS 340 | version = ver_type & self.VERSION_MASK 341 | if version != self.VERSION: 342 | raise TProtocolException(TProtocolException.BAD_VERSION, 343 | 'Bad version: %d (expect %d)' % (version, self.VERSION)) 344 | seqid = self.__readVarint() 345 | # the sequence is a compact "var int" which is treaded as unsigned, 346 | # however the sequence is actually signed... 347 | if seqid > 2147483647: 348 | seqid = -2147483648 - (2147483648 - seqid) 349 | name = binary_to_str(self.__readBinary()) 350 | return (name, type, seqid) 351 | 352 | def readMessageEnd(self): 353 | assert self.state == CLEAR 354 | assert len(self.__structs) == 0 355 | 356 | def readStructBegin(self): 357 | assert self.state in (CLEAR, CONTAINER_READ, VALUE_READ), self.state 358 | self.__structs.append((self.state, self.__last_fid)) 359 | self.state = FIELD_READ 360 | self.__last_fid = 0 361 | 362 | def readStructEnd(self): 363 | assert self.state == FIELD_READ 364 | self.state, self.__last_fid = self.__structs.pop() 365 | 366 | def readCollectionBegin(self): 367 | assert self.state in (VALUE_READ, CONTAINER_READ), self.state 368 | size_type = self.__readUByte() 369 | size = size_type >> 4 370 | type = self.__getTType(size_type) 371 | if size == 15: 372 | size = self.__readSize() 373 | self._check_container_length(size) 374 | self.__containers.append(self.state) 375 | self.state = CONTAINER_READ 376 | return type, size 377 | readSetBegin = readCollectionBegin 378 | readListBegin = readCollectionBegin 379 | 380 | def readMapBegin(self): 381 | assert self.state in (VALUE_READ, CONTAINER_READ), self.state 382 | size = self.__readSize() 383 | self._check_container_length(size) 384 | types = 0 385 | if size > 0: 386 | types = self.__readUByte() 387 | vtype = self.__getTType(types) 388 | ktype = self.__getTType(types >> 4) 389 | self.__containers.append(self.state) 390 | self.state = CONTAINER_READ 391 | return (ktype, vtype, size) 392 | 393 | def readCollectionEnd(self): 394 | assert self.state == CONTAINER_READ, self.state 395 | self.state = self.__containers.pop() 396 | readSetEnd = readCollectionEnd 397 | readListEnd = readCollectionEnd 398 | readMapEnd = readCollectionEnd 399 | 400 | def readBool(self): 401 | if self.state == BOOL_READ: 402 | return self.__bool_value == CompactType.TRUE 403 | elif self.state == CONTAINER_READ: 404 | return self.__readByte() == CompactType.TRUE 405 | else: 406 | raise AssertionError("Invalid state in compact protocol: %d" % 407 | self.state) 408 | 409 | readByte = reader(__readByte) 410 | __readI16 = __readZigZag 411 | readI16 = reader(__readZigZag) 412 | readI32 = reader(__readZigZag) 413 | readI64 = reader(__readZigZag) 414 | 415 | @reader 416 | def readDouble(self): 417 | buff = self.trans.readAll(8) 418 | val, = unpack('