├── ida_type_storage └── __init__.py ├── pymongo ├── auth.pyc ├── bulk.pyc ├── pool.pyc ├── common.pyc ├── cursor.pyc ├── errors.pyc ├── helpers.pyc ├── message.pyc ├── monitor.pyc ├── network.pyc ├── results.pyc ├── server.pyc ├── __init__.pyc ├── _cmessage.pyd ├── collection.pyc ├── database.pyc ├── ismaster.pyc ├── monitoring.pyc ├── monotonic.pyc ├── operations.pyc ├── response.pyc ├── settings.pyc ├── topology.pyc ├── uri_parser.pyc ├── mongo_client.pyc ├── server_type.pyc ├── ssl_context.pyc ├── ssl_support.pyc ├── thread_util.pyc ├── client_options.pyc ├── command_cursor.pyc ├── cursor_manager.pyc ├── son_manipulator.pyc ├── write_concern.pyc ├── periodic_executor.pyc ├── read_preferences.pyc ├── server_selectors.pyc ├── server_description.pyc ├── ssl_match_hostname.pyc ├── topology_description.pyc ├── mongo_replica_set_client.pyc ├── server_type.py ├── monotonic.py ├── cursor_manager.py ├── mongo_replica_set_client.py ├── response.py ├── __init__.py ├── settings.py ├── ssl_context.py ├── ssl_match_hostname.py ├── ismaster.py ├── thread_util.py ├── write_concern.py ├── server_description.py ├── server_selectors.py ├── periodic_executor.py ├── ssl_support.py ├── network.py ├── monitor.py ├── server.py ├── son_manipulator.py ├── client_options.py ├── errors.py ├── results.py ├── command_cursor.py └── operations.py ├── example ├── test_py ├── mfc_example_full.rar ├── test_wrapper.py ├── ex_func_chooser.py ├── test_decode.py ├── chooser.py ├── duplicated_view.py ├── duplicated_view.ui ├── ex_choose2.py ├── test.py ├── qt_test.py ├── create_struct.py ├── test_type_serial.py ├── log.txt ├── sqlite_test.py ├── test_create_struct.py └── test_db.py ├── README.md ├── .gitattributes └── .gitignore /ida_type_storage/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pymongo/auth.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/auth.pyc -------------------------------------------------------------------------------- /pymongo/bulk.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/bulk.pyc -------------------------------------------------------------------------------- /pymongo/pool.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/pool.pyc -------------------------------------------------------------------------------- /pymongo/common.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/common.pyc -------------------------------------------------------------------------------- /pymongo/cursor.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/cursor.pyc -------------------------------------------------------------------------------- /pymongo/errors.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/errors.pyc -------------------------------------------------------------------------------- /pymongo/helpers.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/helpers.pyc -------------------------------------------------------------------------------- /pymongo/message.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/message.pyc -------------------------------------------------------------------------------- /pymongo/monitor.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/monitor.pyc -------------------------------------------------------------------------------- /pymongo/network.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/network.pyc -------------------------------------------------------------------------------- /pymongo/results.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/results.pyc -------------------------------------------------------------------------------- /pymongo/server.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/server.pyc -------------------------------------------------------------------------------- /example/test_py: -------------------------------------------------------------------------------- 1 | f = open('C:\\Python27\\Lib\\site-packages\\PySide\\pyside-python2.7.dll', 'w') 2 | -------------------------------------------------------------------------------- /pymongo/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/__init__.pyc -------------------------------------------------------------------------------- /pymongo/_cmessage.pyd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/_cmessage.pyd -------------------------------------------------------------------------------- /pymongo/collection.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/collection.pyc -------------------------------------------------------------------------------- /pymongo/database.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/database.pyc -------------------------------------------------------------------------------- /pymongo/ismaster.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/ismaster.pyc -------------------------------------------------------------------------------- /pymongo/monitoring.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/monitoring.pyc -------------------------------------------------------------------------------- /pymongo/monotonic.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/monotonic.pyc -------------------------------------------------------------------------------- /pymongo/operations.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/operations.pyc -------------------------------------------------------------------------------- /pymongo/response.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/response.pyc -------------------------------------------------------------------------------- /pymongo/settings.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/settings.pyc -------------------------------------------------------------------------------- /pymongo/topology.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/topology.pyc -------------------------------------------------------------------------------- /pymongo/uri_parser.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/uri_parser.pyc -------------------------------------------------------------------------------- /pymongo/mongo_client.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/mongo_client.pyc -------------------------------------------------------------------------------- /pymongo/server_type.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/server_type.pyc -------------------------------------------------------------------------------- /pymongo/ssl_context.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/ssl_context.pyc -------------------------------------------------------------------------------- /pymongo/ssl_support.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/ssl_support.pyc -------------------------------------------------------------------------------- /pymongo/thread_util.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/thread_util.pyc -------------------------------------------------------------------------------- /pymongo/client_options.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/client_options.pyc -------------------------------------------------------------------------------- /pymongo/command_cursor.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/command_cursor.pyc -------------------------------------------------------------------------------- /pymongo/cursor_manager.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/cursor_manager.pyc -------------------------------------------------------------------------------- /pymongo/son_manipulator.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/son_manipulator.pyc -------------------------------------------------------------------------------- /pymongo/write_concern.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/write_concern.pyc -------------------------------------------------------------------------------- /example/mfc_example_full.rar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/example/mfc_example_full.rar -------------------------------------------------------------------------------- /pymongo/periodic_executor.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/periodic_executor.pyc -------------------------------------------------------------------------------- /pymongo/read_preferences.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/read_preferences.pyc -------------------------------------------------------------------------------- /pymongo/server_selectors.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/server_selectors.pyc -------------------------------------------------------------------------------- /pymongo/server_description.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/server_description.pyc -------------------------------------------------------------------------------- /pymongo/ssl_match_hostname.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/ssl_match_hostname.pyc -------------------------------------------------------------------------------- /pymongo/topology_description.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/topology_description.pyc -------------------------------------------------------------------------------- /example/test_wrapper.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | sys.path.append('F:\GitReps\ida_type_storage') 4 | from IdaTypeStringParser import * -------------------------------------------------------------------------------- /pymongo/mongo_replica_set_client.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigmar/ida_type_storage/HEAD/pymongo/mongo_replica_set_client.pyc -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Ida plugin for symbol sharing and storage. 2 | =========================================== 3 |
4 |
Requirements:
5 |
* PyMongo
6 |
* Mongodb
7 |
* Ida Pro (Tested on 6.8 version)
8 |
Install:
9 |
* Just move IdaTypeStringParser.py to your $(IDADIR)/plugins/
10 |
* Adds two items in file menu (Export types to storage/Import types from storage)
11 |
12 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Windows image file caches 2 | Thumbs.db 3 | ehthumbs.db 4 | 5 | # Folder config file 6 | Desktop.ini 7 | 8 | # Recycle Bin used on file shares 9 | $RECYCLE.BIN/ 10 | 11 | # Windows Installer files 12 | *.cab 13 | *.msi 14 | *.msm 15 | *.msp 16 | 17 | # Windows shortcuts 18 | *.lnk 19 | 20 | # ========================= 21 | # Operating System Files 22 | # ========================= 23 | 24 | # OSX 25 | # ========================= 26 | 27 | .DS_Store 28 | .AppleDouble 29 | .LSOverride 30 | 31 | # Thumbnails 32 | ._* 33 | 34 | # Files that might appear in the root of a volume 35 | .DocumentRevisions-V100 36 | .fseventsd 37 | .Spotlight-V100 38 | .TemporaryItems 39 | .Trashes 40 | .VolumeIcon.icns 41 | 42 | # Directories potentially created on remote AFP share 43 | .AppleDB 44 | .AppleDesktop 45 | Network Trash Folder 46 | Temporary Items 47 | .apdisk 48 | .idea 49 | *.pyc 50 | /pymongo/ 51 | -------------------------------------------------------------------------------- /pymongo/server_type.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Type codes for MongoDB servers.""" 16 | 17 | from collections import namedtuple 18 | 19 | 20 | SERVER_TYPE = namedtuple('ServerType', 21 | ['Unknown', 'Mongos', 'RSPrimary', 'RSSecondary', 22 | 'RSArbiter', 'RSOther', 'RSGhost', 23 | 'Standalone'])(*range(8)) 24 | -------------------------------------------------------------------------------- /pymongo/monotonic.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Time. Monotonic if possible. 16 | """ 17 | 18 | __all__ = ['time'] 19 | 20 | try: 21 | # Patches standard time module. 22 | # From https://pypi.python.org/pypi/Monotime. 23 | import monotime 24 | except ImportError: 25 | pass 26 | 27 | try: 28 | # Monotime or Python 3.3+. 29 | from time import monotonic as time 30 | except ImportError: 31 | # Not monotonic. 32 | from time import time 33 | -------------------------------------------------------------------------------- /example/ex_func_chooser.py: -------------------------------------------------------------------------------- 1 | import idaapi 2 | import idautils 3 | import idc 4 | 5 | class MyChoose2(Choose2): 6 | 7 | def __init__(self, title): 8 | Choose2.__init__(self, title, [ ["Address", 10 | Choose2.CHCOL_HEX], ["Name", 30 | Choose2.CHCOL_PLAIN] ], flags = 0,) 9 | self.modal = True 10 | self.n = 0 11 | self.icon = 41 12 | self.PopulateItems() 13 | 14 | def PopulateItems(self): 15 | self.items = [ [hex(x), GetFunctionName(x), x] for x in idautils.Functions() ] 16 | 17 | def OnClose(self): 18 | print "closed ", self.title 19 | 20 | def OnSelectLine(self, n): 21 | print n 22 | idc.Jump(self.items[n][2]) 23 | 24 | def OnGetLine(self, n): 25 | return self.items[n] 26 | 27 | def OnGetSize(self): 28 | return len(self.items) 29 | 30 | def OnDeleteLine(self, n): 31 | ea = self.items[n][2] 32 | idc.DelFunction(ea) 33 | return n 34 | 35 | def OnRefresh(self, n): 36 | self.PopulateItems() 37 | return n 38 | 39 | c = MyChoose2("My functions list") 40 | c.Show() -------------------------------------------------------------------------------- /example/test_decode.py: -------------------------------------------------------------------------------- 1 | import struct 2 | 3 | def encode_ordinal(ordinal): 4 | enc = [] 5 | enc.append(ordinal&0x7f|0x40) 6 | if ordinal > 0x3f: 7 | bt = ordinal 8 | bt = bt // 0x40 9 | enc.append(bt&0x7f|0x80) 10 | while bt > 0x7f: 11 | bt = bt // 0x80 12 | enc.append(bt&0x7f|0x80) 13 | stemp = "" 14 | for i in range(0,len(enc)): 15 | stemp = stemp + struct.pack("B",enc.pop(-1)) 16 | return stemp 17 | 18 | 19 | 20 | def decode_ordinal(enc): 21 | ord_num = 0 22 | i = 0 23 | fEnd = 0 24 | len = struct.unpack("B",enc[0]) 25 | for ch in enc: 26 | ch = ord(ch) 27 | if ch == 0: 28 | return 0 29 | ord_num = ord_num * 0x40 30 | if ch&0x80 != 0: 31 | ord_num = ord_num * 2 32 | ch = ch & 0x7f 33 | else: 34 | ch = ch & 0x3f 35 | fEnd = 1 36 | ord_num = ord_num | ch 37 | if fEnd > 0 or i >= len: 38 | break 39 | return ord_num 40 | 41 | 42 | print decode_ordinal('\x81\x80') 43 | temp = '' 44 | print encode_ordinal(13055).encode("hex") 45 | print encode_ordinal(0x200).encode("hex") 46 | -------------------------------------------------------------------------------- /example/chooser.py: -------------------------------------------------------------------------------- 1 | #--------------------------------------------------------------------- 2 | # Chooser test 3 | # 4 | # This script demonstrates the usage of the class-based chooser. 5 | # 6 | # Author: Gergely Erdelyi 7 | #--------------------------------------------------------------------- 8 | from idaapi import Choose 9 | 10 | # 11 | # Modal chooser 12 | # 13 | 14 | # Get a modal Choose instance 15 | chooser = Choose([], "MyChooser", 1) 16 | # List to choose from 17 | chooser.list = [ "First", "Second", "Third" ] 18 | # Set the width 19 | chooser.width = 50 20 | # Run the chooser 21 | ch = chooser.choose() 22 | # Print the results 23 | if ch > 0: 24 | print "You chose %d which is %s" % (ch, chooser.list[ch-1]) 25 | else: 26 | print "Escape from chooser" 27 | 28 | # 29 | # Normal chooser 30 | # 31 | class MyChoose(Choose): 32 | """ 33 | You have to subclass Chooser to override the enter() method 34 | """ 35 | def __init__(self, list=[], name="Choose"): 36 | Choose.__init__(self, list, name) 37 | # Set the width 38 | self.width = 50 39 | self.deflt = 1 40 | 41 | def enter(self, n): 42 | print "Enter called. Do some stuff here." 43 | print "The chosen item is %d = %s" % (n, self.list[n-1]) 44 | print "Now press ESC to leave." 45 | 46 | # Get a Choose instance 47 | chooser = MyChoose([ "First", "Second", "Third" ], "MyChoose") 48 | 49 | # Run the chooser 50 | ch = chooser.choose() 51 | -------------------------------------------------------------------------------- /pymongo/cursor_manager.py: -------------------------------------------------------------------------------- 1 | # Copyright 2009-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """A manager to handle when cursors are killed after they are closed. 16 | 17 | New cursor managers should be defined as subclasses of CursorManager and can be 18 | installed on a client by calling 19 | :meth:`~pymongo.mongo_client.MongoClient.set_cursor_manager`. 20 | 21 | .. versionchanged:: 3.0 22 | Undeprecated. :meth:`~pymongo.cursor_manager.CursorManager.close` now 23 | requires an `address` argument. The ``BatchCursorManager`` class is removed. 24 | """ 25 | 26 | import weakref 27 | from bson.py3compat import integer_types 28 | 29 | 30 | class CursorManager(object): 31 | """The cursor manager base class.""" 32 | 33 | def __init__(self, client): 34 | """Instantiate the manager. 35 | 36 | :Parameters: 37 | - `client`: a MongoClient 38 | """ 39 | self.__client = weakref.ref(client) 40 | 41 | def close(self, cursor_id, address): 42 | """Kill a cursor. 43 | 44 | Raises TypeError if cursor_id is not an instance of (int, long). 45 | 46 | :Parameters: 47 | - `cursor_id`: cursor id to close 48 | - `address`: the cursor's server's (host, port) pair 49 | 50 | .. versionchanged:: 3.0 51 | Now requires an `address` argument. 52 | """ 53 | if not isinstance(cursor_id, integer_types): 54 | raise TypeError("cursor_id must be an integer") 55 | 56 | self.__client().kill_cursors([cursor_id], address) 57 | -------------------------------------------------------------------------------- /pymongo/mongo_replica_set_client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you 4 | # may not use this file except in compliance with the License. You 5 | # may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. See the License for the specific language governing 13 | # permissions and limitations under the License. 14 | 15 | """Deprecated. See :doc:`/examples/high_availability`.""" 16 | 17 | import warnings 18 | 19 | from pymongo import mongo_client 20 | 21 | 22 | class MongoReplicaSetClient(mongo_client.MongoClient): 23 | """Deprecated alias for :class:`~pymongo.mongo_client.MongoClient`. 24 | 25 | :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient` 26 | will be removed in a future version of PyMongo. 27 | 28 | .. versionchanged:: 3.0 29 | :class:`~pymongo.mongo_client.MongoClient` is now the one and only 30 | client class for a standalone server, mongos, or replica set. 31 | It includes the functionality that had been split into 32 | :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`: it 33 | can connect to a replica set, discover all its members, and monitor 34 | the set for stepdowns, elections, and reconfigs. 35 | 36 | The ``refresh`` method is removed from 37 | :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`, 38 | as are the ``seeds`` and ``hosts`` properties. 39 | """ 40 | def __init__(self, *args, **kwargs): 41 | warnings.warn('MongoReplicaSetClient is deprecated, use MongoClient' 42 | ' to connect to a replica set', 43 | DeprecationWarning, stacklevel=2) 44 | 45 | super(MongoReplicaSetClient, self).__init__(*args, **kwargs) 46 | 47 | def __repr__(self): 48 | return "MongoReplicaSetClient(%s)" % (self._repr_helper(),) 49 | -------------------------------------------------------------------------------- /pymongo/response.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Represent a response from the server.""" 16 | 17 | 18 | class Response(object): 19 | __slots__ = ('_data', '_address', '_request_id', '_duration') 20 | 21 | def __init__(self, data, address, request_id, duration): 22 | """Represent a response from the server. 23 | 24 | :Parameters: 25 | - `data`: Raw BSON bytes. 26 | - `address`: (host, port) of the source server. 27 | - `request_id`: The request id of this operation. 28 | - `duration`: The duration of the operation. 29 | """ 30 | self._data = data 31 | self._address = address 32 | self._request_id = request_id 33 | self._duration = duration 34 | 35 | @property 36 | def data(self): 37 | """Server response's raw BSON bytes.""" 38 | return self._data 39 | 40 | @property 41 | def address(self): 42 | """(host, port) of the source server.""" 43 | return self._address 44 | 45 | @property 46 | def request_id(self): 47 | """The request id of this operation.""" 48 | return self._request_id 49 | 50 | @property 51 | def duration(self): 52 | """The duration of the operation.""" 53 | return self._duration 54 | 55 | 56 | class ExhaustResponse(Response): 57 | __slots__ = ('_socket_info', '_pool') 58 | 59 | def __init__( 60 | self, data, address, socket_info, pool, request_id, duration): 61 | """Represent a response to an exhaust cursor's initial query. 62 | 63 | :Parameters: 64 | - `data`: Raw BSON bytes. 65 | - `address`: (host, port) of the source server. 66 | - `socket_info`: The SocketInfo used for the initial query. 67 | - `pool`: The Pool from which the SocketInfo came. 68 | - `request_id`: The request id of this operation. 69 | - `duration`: The duration of the operation. 70 | """ 71 | super(ExhaustResponse, self).__init__(data, 72 | address, 73 | request_id, 74 | duration) 75 | self._socket_info = socket_info 76 | self._pool = pool 77 | 78 | @property 79 | def socket_info(self): 80 | """The SocketInfo used for the initial query. 81 | 82 | The server will send batches on this socket, without waiting for 83 | getMores from the client, until the result set is exhausted or there 84 | is an error. 85 | """ 86 | return self._socket_info 87 | 88 | @property 89 | def pool(self): 90 | """The Pool from which the SocketInfo came.""" 91 | return self._pool 92 | -------------------------------------------------------------------------------- /pymongo/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2009-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Python driver for MongoDB.""" 16 | 17 | ASCENDING = 1 18 | """Ascending sort order.""" 19 | DESCENDING = -1 20 | """Descending sort order.""" 21 | 22 | GEO2D = "2d" 23 | """Index specifier for a 2-dimensional `geospatial index`_. 24 | 25 | .. _geospatial index: http://docs.mongodb.org/manual/core/2d/ 26 | """ 27 | 28 | GEOHAYSTACK = "geoHaystack" 29 | """Index specifier for a 2-dimensional `haystack index`_. 30 | 31 | .. versionadded:: 2.1 32 | 33 | .. _haystack index: http://docs.mongodb.org/manual/core/geohaystack/ 34 | """ 35 | 36 | GEOSPHERE = "2dsphere" 37 | """Index specifier for a `spherical geospatial index`_. 38 | 39 | .. versionadded:: 2.5 40 | 41 | .. note:: 2dsphere indexing requires server version **>= 2.4.0**. 42 | 43 | .. _spherical geospatial index: http://docs.mongodb.org/manual/core/2dsphere/ 44 | """ 45 | 46 | HASHED = "hashed" 47 | """Index specifier for a `hashed index`_. 48 | 49 | .. versionadded:: 2.5 50 | 51 | .. note:: hashed indexing requires server version **>= 2.4.0**. 52 | 53 | .. _hashed index: http://docs.mongodb.org/manual/core/index-hashed/ 54 | """ 55 | 56 | TEXT = "text" 57 | """Index specifier for a `text index`_. 58 | 59 | .. versionadded:: 2.7.1 60 | 61 | .. note:: text search requires server version **>= 2.4.0**. 62 | 63 | .. _text index: http://docs.mongodb.org/manual/core/index-text/ 64 | """ 65 | 66 | OFF = 0 67 | """No database profiling.""" 68 | SLOW_ONLY = 1 69 | """Only profile slow operations.""" 70 | ALL = 2 71 | """Profile all operations.""" 72 | 73 | version_tuple = (3, 1, 1) 74 | 75 | def get_version_string(): 76 | if isinstance(version_tuple[-1], str): 77 | return '.'.join(map(str, version_tuple[:-1])) + version_tuple[-1] 78 | return '.'.join(map(str, version_tuple)) 79 | 80 | __version__ = version = get_version_string() 81 | """Current version of PyMongo.""" 82 | 83 | from pymongo.collection import ReturnDocument 84 | from pymongo.common import (MIN_SUPPORTED_WIRE_VERSION, 85 | MAX_SUPPORTED_WIRE_VERSION) 86 | from pymongo.cursor import CursorType 87 | from pymongo.mongo_client import MongoClient 88 | from pymongo.mongo_replica_set_client import MongoReplicaSetClient 89 | from pymongo.operations import (IndexModel, 90 | InsertOne, 91 | DeleteOne, 92 | DeleteMany, 93 | UpdateOne, 94 | UpdateMany, 95 | ReplaceOne) 96 | from pymongo.read_preferences import ReadPreference 97 | from pymongo.write_concern import WriteConcern 98 | 99 | def has_c(): 100 | """Is the C extension installed?""" 101 | try: 102 | from pymongo import _cmessage 103 | return True 104 | except ImportError: 105 | return False 106 | -------------------------------------------------------------------------------- /pymongo/settings.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you 4 | # may not use this file except in compliance with the License. You 5 | # may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. See the License for the specific language governing 13 | # permissions and limitations under the License. 14 | 15 | """Represent MongoClient's configuration.""" 16 | 17 | import threading 18 | 19 | from pymongo import monitor, pool 20 | from pymongo.common import LOCAL_THRESHOLD_MS, SERVER_SELECTION_TIMEOUT 21 | from pymongo.topology_description import TOPOLOGY_TYPE 22 | from pymongo.pool import PoolOptions 23 | from pymongo.server_description import ServerDescription 24 | 25 | 26 | class TopologySettings(object): 27 | def __init__(self, 28 | seeds=None, 29 | replica_set_name=None, 30 | pool_class=None, 31 | pool_options=None, 32 | monitor_class=None, 33 | condition_class=None, 34 | local_threshold_ms=LOCAL_THRESHOLD_MS, 35 | server_selection_timeout=SERVER_SELECTION_TIMEOUT): 36 | """Represent MongoClient's configuration. 37 | 38 | Take a list of (host, port) pairs and optional replica set name. 39 | """ 40 | self._seeds = seeds or [('localhost', 27017)] 41 | self._replica_set_name = replica_set_name 42 | self._pool_class = pool_class or pool.Pool 43 | self._pool_options = pool_options or PoolOptions() 44 | self._monitor_class = monitor_class or monitor.Monitor 45 | self._condition_class = condition_class or threading.Condition 46 | self._local_threshold_ms = local_threshold_ms 47 | self._server_selection_timeout = server_selection_timeout 48 | self._direct = (len(self._seeds) == 1 and not replica_set_name) 49 | 50 | @property 51 | def seeds(self): 52 | """List of server addresses.""" 53 | return self._seeds 54 | 55 | @property 56 | def replica_set_name(self): 57 | return self._replica_set_name 58 | 59 | @property 60 | def pool_class(self): 61 | return self._pool_class 62 | 63 | @property 64 | def pool_options(self): 65 | return self._pool_options 66 | 67 | @property 68 | def monitor_class(self): 69 | return self._monitor_class 70 | 71 | @property 72 | def condition_class(self): 73 | return self._condition_class 74 | 75 | @property 76 | def local_threshold_ms(self): 77 | return self._local_threshold_ms 78 | 79 | @property 80 | def server_selection_timeout(self): 81 | return self._server_selection_timeout 82 | 83 | @property 84 | def direct(self): 85 | """Connect directly to a single server, or use a set of servers? 86 | 87 | True if there is one seed and no replica_set_name. 88 | """ 89 | return self._direct 90 | 91 | def get_topology_type(self): 92 | if self.direct: 93 | return TOPOLOGY_TYPE.Single 94 | elif self.replica_set_name is not None: 95 | return TOPOLOGY_TYPE.ReplicaSetNoPrimary 96 | else: 97 | return TOPOLOGY_TYPE.Unknown 98 | 99 | def get_server_descriptions(self): 100 | """Initial dict of (address, ServerDescription) for all seeds.""" 101 | return dict([ 102 | (address, ServerDescription(address)) 103 | for address in self.seeds]) 104 | -------------------------------------------------------------------------------- /example/duplicated_view.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Form implementation generated from reading ui file 'duplicated_view.ui' 4 | # 5 | # Created: Mon Oct 26 08:03:15 2015 6 | # by: pyside-uic 0.2.15 running on PySide 1.2.4 7 | # 8 | # WARNING! All changes made in this file will be lost! 9 | 10 | from PySide import QtCore, QtGui 11 | 12 | class Ui_Dialog(object): 13 | def setupUi(self, Dialog): 14 | Dialog.setObjectName("Dialog") 15 | Dialog.resize(1088, 844) 16 | self.buttonBox = QtGui.QDialogButtonBox(Dialog) 17 | self.buttonBox.setGeometry(QtCore.QRect(910, 810, 171, 31)) 18 | self.buttonBox.setOrientation(QtCore.Qt.Horizontal) 19 | self.buttonBox.setStandardButtons(QtGui.QDialogButtonBox.Cancel|QtGui.QDialogButtonBox.Ok) 20 | self.buttonBox.setObjectName("buttonBox") 21 | self.textEdit_struc1 = QtGui.QTextEdit(Dialog) 22 | self.textEdit_struc1.setGeometry(QtCore.QRect(10, 60, 501, 361)) 23 | self.textEdit_struc1.setObjectName("textEdit_struc1") 24 | self.textEdit_struc2 = QtGui.QTextEdit(Dialog) 25 | self.textEdit_struc2.setGeometry(QtCore.QRect(580, 60, 501, 361)) 26 | self.textEdit_struc2.setObjectName("textEdit_struc2") 27 | self.textEdit_struc3 = QtGui.QTextEdit(Dialog) 28 | self.textEdit_struc3.setGeometry(QtCore.QRect(270, 470, 531, 361)) 29 | self.textEdit_struc3.setObjectName("textEdit_struc3") 30 | self.label_struc1 = QtGui.QLabel(Dialog) 31 | self.label_struc1.setGeometry(QtCore.QRect(200, 20, 61, 20)) 32 | self.label_struc1.setObjectName("label_struc1") 33 | self.label_struc2 = QtGui.QLabel(Dialog) 34 | self.label_struc2.setGeometry(QtCore.QRect(780, 20, 61, 20)) 35 | self.label_struc2.setObjectName("label_struc2") 36 | self.label_strucMerged = QtGui.QLabel(Dialog) 37 | self.label_strucMerged.setGeometry(QtCore.QRect(510, 440, 91, 20)) 38 | self.label_strucMerged.setObjectName("label_strucMerged") 39 | self.pushButton_importStruc1 = QtGui.QPushButton(Dialog) 40 | self.pushButton_importStruc1.setGeometry(QtCore.QRect(310, 430, 101, 23)) 41 | self.pushButton_importStruc1.setObjectName("pushButton_importStruc1") 42 | self.pushButton_importStruc2 = QtGui.QPushButton(Dialog) 43 | self.pushButton_importStruc2.setGeometry(QtCore.QRect(660, 430, 101, 23)) 44 | self.pushButton_importStruc2.setObjectName("pushButton_importStruc2") 45 | self.pushButton_useMerged = QtGui.QPushButton(Dialog) 46 | self.pushButton_useMerged.setGeometry(QtCore.QRect(830, 640, 121, 23)) 47 | self.pushButton_useMerged.setObjectName("pushButton_useMerged") 48 | 49 | self.retranslateUi(Dialog) 50 | QtCore.QObject.connect(self.buttonBox, QtCore.SIGNAL("accepted()"), Dialog.accept) 51 | QtCore.QObject.connect(self.buttonBox, QtCore.SIGNAL("rejected()"), Dialog.reject) 52 | QtCore.QMetaObject.connectSlotsByName(Dialog) 53 | 54 | def retranslateUi(self, Dialog): 55 | Dialog.setWindowTitle(QtGui.QApplication.translate("Dialog", "Dialog", None, QtGui.QApplication.UnicodeUTF8)) 56 | self.label_struc1.setText(QtGui.QApplication.translate("Dialog", "Structure 1", None, QtGui.QApplication.UnicodeUTF8)) 57 | self.label_struc2.setText(QtGui.QApplication.translate("Dialog", "Structure 2", None, QtGui.QApplication.UnicodeUTF8)) 58 | self.label_strucMerged.setText(QtGui.QApplication.translate("Dialog", "Merged structure", None, QtGui.QApplication.UnicodeUTF8)) 59 | self.pushButton_importStruc1.setText(QtGui.QApplication.translate("Dialog", "Import struture 1", None, QtGui.QApplication.UnicodeUTF8)) 60 | self.pushButton_importStruc2.setText(QtGui.QApplication.translate("Dialog", "Import struture 2", None, QtGui.QApplication.UnicodeUTF8)) 61 | self.pushButton_useMerged.setText(QtGui.QApplication.translate("Dialog", "Use merged structure", None, QtGui.QApplication.UnicodeUTF8)) 62 | 63 | -------------------------------------------------------------------------------- /pymongo/ssl_context.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you 4 | # may not use this file except in compliance with the License. You 5 | # may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. See the License for the specific language governing 13 | # permissions and limitations under the License. 14 | 15 | """A fake SSLContext implementation.""" 16 | 17 | try: 18 | import ssl 19 | except ImportError: 20 | pass 21 | 22 | 23 | class SSLContext(object): 24 | """A fake SSLContext. 25 | 26 | This implements an API similar to ssl.SSLContext from python 3.2 27 | but does not implement methods or properties that would be 28 | incompatible with ssl.wrap_socket from python 2.6. 29 | 30 | You must pass protocol which must be one of the PROTOCOL_* constants 31 | defined in the ssl module. ssl.PROTOCOL_SSLv23 is recommended for maximum 32 | interoperability. 33 | """ 34 | 35 | __slots__ = ('_cafile', '_certfile', 36 | '_keyfile', '_protocol', '_verify_mode') 37 | 38 | def __init__(self, protocol): 39 | self._cafile = None 40 | self._certfile = None 41 | self._keyfile = None 42 | self._protocol = protocol 43 | self._verify_mode = ssl.CERT_NONE 44 | 45 | @property 46 | def protocol(self): 47 | """The protocol version chosen when constructing the context. 48 | This attribute is read-only. 49 | """ 50 | return self._protocol 51 | 52 | def __get_verify_mode(self): 53 | """Whether to try to verify other peers' certificates and how to 54 | behave if verification fails. This attribute must be one of 55 | ssl.CERT_NONE, ssl.CERT_OPTIONAL or ssl.CERT_REQUIRED. 56 | """ 57 | return self._verify_mode 58 | 59 | def __set_verify_mode(self, value): 60 | """Setter for verify_mode.""" 61 | self._verify_mode = value 62 | 63 | verify_mode = property(__get_verify_mode, __set_verify_mode) 64 | 65 | def load_cert_chain(self, certfile, keyfile=None): 66 | """Load a private key and the corresponding certificate. The certfile 67 | string must be the path to a single file in PEM format containing the 68 | certificate as well as any number of CA certificates needed to 69 | establish the certificate's authenticity. The keyfile string, if 70 | present, must point to a file containing the private key. Otherwise 71 | the private key will be taken from certfile as well. 72 | """ 73 | self._certfile = certfile 74 | self._keyfile = keyfile 75 | 76 | def load_verify_locations(self, cafile=None, dummy=None): 77 | """Load a set of "certification authority"(CA) certificates used to 78 | validate other peers' certificates when `~verify_mode` is other than 79 | ssl.CERT_NONE. 80 | """ 81 | self._cafile = cafile 82 | 83 | def wrap_socket(self, sock, server_side=False, 84 | do_handshake_on_connect=True, 85 | suppress_ragged_eofs=True, dummy=None): 86 | """Wrap an existing Python socket sock and return an ssl.SSLSocket 87 | object. 88 | """ 89 | return ssl.wrap_socket(sock, keyfile=self._keyfile, 90 | certfile=self._certfile, 91 | server_side=server_side, 92 | cert_reqs=self._verify_mode, 93 | ssl_version=self._protocol, 94 | ca_certs=self._cafile, 95 | do_handshake_on_connect=do_handshake_on_connect, 96 | suppress_ragged_eofs=suppress_ragged_eofs) 97 | 98 | -------------------------------------------------------------------------------- /pymongo/ssl_match_hostname.py: -------------------------------------------------------------------------------- 1 | # Backport of the match_hostname logic introduced in python 3.2 2 | # http://hg.python.org/releasing/3.3.5/file/993955b807b3/Lib/ssl.py 3 | 4 | import re 5 | 6 | 7 | class CertificateError(ValueError): 8 | pass 9 | 10 | 11 | def _dnsname_match(dn, hostname, max_wildcards=1): 12 | """Matching according to RFC 6125, section 6.4.3 13 | 14 | http://tools.ietf.org/html/rfc6125#section-6.4.3 15 | """ 16 | pats = [] 17 | if not dn: 18 | return False 19 | 20 | parts = dn.split(r'.') 21 | leftmost = parts[0] 22 | remainder = parts[1:] 23 | 24 | wildcards = leftmost.count('*') 25 | if wildcards > max_wildcards: 26 | # Issue #17980: avoid denials of service by refusing more 27 | # than one wildcard per fragment. A survey of established 28 | # policy among SSL implementations showed it to be a 29 | # reasonable choice. 30 | raise CertificateError( 31 | "too many wildcards in certificate DNS name: " + repr(dn)) 32 | 33 | # speed up common case w/o wildcards 34 | if not wildcards: 35 | return dn.lower() == hostname.lower() 36 | 37 | # RFC 6125, section 6.4.3, subitem 1. 38 | # The client SHOULD NOT attempt to match a presented identifier in which 39 | # the wildcard character comprises a label other than the left-most label. 40 | if leftmost == '*': 41 | # When '*' is a fragment by itself, it matches a non-empty dotless 42 | # fragment. 43 | pats.append('[^.]+') 44 | elif leftmost.startswith('xn--') or hostname.startswith('xn--'): 45 | # RFC 6125, section 6.4.3, subitem 3. 46 | # The client SHOULD NOT attempt to match a presented identifier 47 | # where the wildcard character is embedded within an A-label or 48 | # U-label of an internationalized domain name. 49 | pats.append(re.escape(leftmost)) 50 | else: 51 | # Otherwise, '*' matches any dotless string, e.g. www* 52 | pats.append(re.escape(leftmost).replace(r'\*', '[^.]*')) 53 | 54 | # add the remaining fragments, ignore any wildcards 55 | for frag in remainder: 56 | pats.append(re.escape(frag)) 57 | 58 | pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) 59 | return pat.match(hostname) 60 | 61 | 62 | def match_hostname(cert, hostname): 63 | """Verify that *cert* (in decoded format as returned by 64 | SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125 65 | rules are followed, but IP addresses are not accepted for *hostname*. 66 | 67 | CertificateError is raised on failure. On success, the function 68 | returns nothing. 69 | """ 70 | if not cert: 71 | raise ValueError("empty or no certificate") 72 | dnsnames = [] 73 | san = cert.get('subjectAltName', ()) 74 | for key, value in san: 75 | if key == 'DNS': 76 | if _dnsname_match(value, hostname): 77 | return 78 | dnsnames.append(value) 79 | if not dnsnames: 80 | # The subject is only checked when there is no dNSName entry 81 | # in subjectAltName 82 | for sub in cert.get('subject', ()): 83 | for key, value in sub: 84 | # XXX according to RFC 2818, the most specific Common Name 85 | # must be used. 86 | if key == 'commonName': 87 | if _dnsname_match(value, hostname): 88 | return 89 | dnsnames.append(value) 90 | if len(dnsnames) > 1: 91 | raise CertificateError("hostname %r " 92 | "doesn't match either of %s" 93 | % (hostname, ', '.join(map(repr, dnsnames)))) 94 | elif len(dnsnames) == 1: 95 | raise CertificateError("hostname %r " 96 | "doesn't match %r" 97 | % (hostname, dnsnames[0])) 98 | else: 99 | raise CertificateError("no appropriate commonName or " 100 | "subjectAltName fields were found") 101 | -------------------------------------------------------------------------------- /pymongo/ismaster.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Parse a response to the 'ismaster' command.""" 16 | 17 | import itertools 18 | 19 | from bson.py3compat import imap 20 | from pymongo import common 21 | from pymongo.server_type import SERVER_TYPE 22 | 23 | 24 | def _get_server_type(doc): 25 | """Determine the server type from an ismaster response.""" 26 | if not doc.get('ok'): 27 | return SERVER_TYPE.Unknown 28 | 29 | if doc.get('isreplicaset'): 30 | return SERVER_TYPE.RSGhost 31 | elif doc.get('setName'): 32 | if doc.get('hidden'): 33 | return SERVER_TYPE.RSOther 34 | elif doc.get('ismaster'): 35 | return SERVER_TYPE.RSPrimary 36 | elif doc.get('secondary'): 37 | return SERVER_TYPE.RSSecondary 38 | elif doc.get('arbiterOnly'): 39 | return SERVER_TYPE.RSArbiter 40 | else: 41 | return SERVER_TYPE.RSOther 42 | elif doc.get('msg') == 'isdbgrid': 43 | return SERVER_TYPE.Mongos 44 | else: 45 | return SERVER_TYPE.Standalone 46 | 47 | 48 | class IsMaster(object): 49 | __slots__ = ('_doc', '_server_type', '_is_writable', '_is_readable') 50 | 51 | def __init__(self, doc): 52 | """Parse an ismaster response from the server.""" 53 | self._server_type = _get_server_type(doc) 54 | self._doc = doc 55 | self._is_writable = self._server_type in ( 56 | SERVER_TYPE.RSPrimary, 57 | SERVER_TYPE.Standalone, 58 | SERVER_TYPE.Mongos) 59 | 60 | self._is_readable = ( 61 | self.server_type == SERVER_TYPE.RSSecondary 62 | or self._is_writable) 63 | 64 | @property 65 | def server_type(self): 66 | return self._server_type 67 | 68 | @property 69 | def all_hosts(self): 70 | """List of hosts, passives, and arbiters known to this server.""" 71 | return set(imap(common.clean_node, itertools.chain( 72 | self._doc.get('hosts', []), 73 | self._doc.get('passives', []), 74 | self._doc.get('arbiters', [])))) 75 | 76 | @property 77 | def tags(self): 78 | """Replica set member tags or empty dict.""" 79 | return self._doc.get('tags', {}) 80 | 81 | @property 82 | def primary(self): 83 | """This server's opinion about who the primary is, or None.""" 84 | if self._doc.get('primary'): 85 | return common.partition_node(self._doc['primary']) 86 | else: 87 | return None 88 | 89 | @property 90 | def replica_set_name(self): 91 | """Replica set name or None.""" 92 | return self._doc.get('setName') 93 | 94 | @property 95 | def max_bson_size(self): 96 | return self._doc.get('maxBsonObjectSize', common.MAX_BSON_SIZE) 97 | 98 | @property 99 | def max_message_size(self): 100 | return self._doc.get('maxMessageSizeBytes', 2 * self.max_bson_size) 101 | 102 | @property 103 | def max_write_batch_size(self): 104 | return self._doc.get('maxWriteBatchSize', common.MAX_WRITE_BATCH_SIZE) 105 | 106 | @property 107 | def min_wire_version(self): 108 | return self._doc.get('minWireVersion', common.MIN_WIRE_VERSION) 109 | 110 | @property 111 | def max_wire_version(self): 112 | return self._doc.get('maxWireVersion', common.MAX_WIRE_VERSION) 113 | 114 | @property 115 | def election_id(self): 116 | return self._doc.get('electionId') 117 | 118 | @property 119 | def is_writable(self): 120 | return self._is_writable 121 | 122 | @property 123 | def is_readable(self): 124 | return self._is_readable 125 | 126 | @property 127 | def me(self): 128 | me = self._doc.get('me') 129 | if me: 130 | return common.clean_node(me) 131 | -------------------------------------------------------------------------------- /pymongo/thread_util.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Utilities for multi-threading support.""" 16 | 17 | import threading 18 | try: 19 | from time import monotonic as _time 20 | except ImportError: 21 | from time import time as _time 22 | 23 | from pymongo.monotonic import time as _time 24 | from pymongo.errors import ExceededMaxWaiters 25 | 26 | 27 | ### Begin backport from CPython 3.2 for timeout support for Semaphore.acquire 28 | class Semaphore: 29 | 30 | # After Tim Peters' semaphore class, but not quite the same (no maximum) 31 | 32 | def __init__(self, value=1): 33 | if value < 0: 34 | raise ValueError("semaphore initial value must be >= 0") 35 | self._cond = threading.Condition(threading.Lock()) 36 | self._value = value 37 | 38 | def acquire(self, blocking=True, timeout=None): 39 | if not blocking and timeout is not None: 40 | raise ValueError("can't specify timeout for non-blocking acquire") 41 | rc = False 42 | endtime = None 43 | self._cond.acquire() 44 | while self._value == 0: 45 | if not blocking: 46 | break 47 | if timeout is not None: 48 | if endtime is None: 49 | endtime = _time() + timeout 50 | else: 51 | timeout = endtime - _time() 52 | if timeout <= 0: 53 | break 54 | self._cond.wait(timeout) 55 | else: 56 | self._value = self._value - 1 57 | rc = True 58 | self._cond.release() 59 | return rc 60 | 61 | __enter__ = acquire 62 | 63 | def release(self): 64 | self._cond.acquire() 65 | self._value = self._value + 1 66 | self._cond.notify() 67 | self._cond.release() 68 | 69 | def __exit__(self, t, v, tb): 70 | self.release() 71 | 72 | @property 73 | def counter(self): 74 | return self._value 75 | 76 | 77 | class BoundedSemaphore(Semaphore): 78 | """Semaphore that checks that # releases is <= # acquires""" 79 | def __init__(self, value=1): 80 | Semaphore.__init__(self, value) 81 | self._initial_value = value 82 | 83 | def release(self): 84 | if self._value >= self._initial_value: 85 | raise ValueError("Semaphore released too many times") 86 | return Semaphore.release(self) 87 | ### End backport from CPython 3.2 88 | 89 | 90 | class DummySemaphore(object): 91 | def __init__(self, value=None): 92 | pass 93 | 94 | def acquire(self, blocking=True, timeout=None): 95 | return True 96 | 97 | def release(self): 98 | pass 99 | 100 | 101 | class MaxWaitersBoundedSemaphore(object): 102 | def __init__(self, semaphore_class, value=1, max_waiters=1): 103 | self.waiter_semaphore = semaphore_class(max_waiters) 104 | self.semaphore = semaphore_class(value) 105 | 106 | def acquire(self, blocking=True, timeout=None): 107 | if not self.waiter_semaphore.acquire(False): 108 | raise ExceededMaxWaiters() 109 | try: 110 | return self.semaphore.acquire(blocking, timeout) 111 | finally: 112 | self.waiter_semaphore.release() 113 | 114 | def __getattr__(self, name): 115 | return getattr(self.semaphore, name) 116 | 117 | 118 | class MaxWaitersBoundedSemaphoreThread(MaxWaitersBoundedSemaphore): 119 | def __init__(self, value=1, max_waiters=1): 120 | MaxWaitersBoundedSemaphore.__init__( 121 | self, BoundedSemaphore, value, max_waiters) 122 | 123 | 124 | def create_semaphore(max_size, max_waiters): 125 | if max_size is None: 126 | return DummySemaphore() 127 | else: 128 | if max_waiters is None: 129 | return BoundedSemaphore(max_size) 130 | else: 131 | return MaxWaitersBoundedSemaphoreThread(max_size, max_waiters) 132 | -------------------------------------------------------------------------------- /example/duplicated_view.ui: -------------------------------------------------------------------------------- 1 | 2 | 3 | Dialog 4 | 5 | 6 | 7 | 0 8 | 0 9 | 1088 10 | 844 11 | 12 | 13 | 14 | Dialog 15 | 16 | 17 | 18 | 19 | 910 20 | 810 21 | 171 22 | 31 23 | 24 | 25 | 26 | Qt::Horizontal 27 | 28 | 29 | QDialogButtonBox::Cancel|QDialogButtonBox::Ok 30 | 31 | 32 | 33 | 34 | 35 | 10 36 | 60 37 | 501 38 | 361 39 | 40 | 41 | 42 | 43 | 44 | 45 | 580 46 | 60 47 | 501 48 | 361 49 | 50 | 51 | 52 | 53 | 54 | 55 | 270 56 | 470 57 | 531 58 | 361 59 | 60 | 61 | 62 | 63 | 64 | 65 | 200 66 | 20 67 | 61 68 | 20 69 | 70 | 71 | 72 | Structure 1 73 | 74 | 75 | 76 | 77 | 78 | 780 79 | 20 80 | 61 81 | 20 82 | 83 | 84 | 85 | Structure 2 86 | 87 | 88 | 89 | 90 | 91 | 510 92 | 440 93 | 91 94 | 20 95 | 96 | 97 | 98 | Merged structure 99 | 100 | 101 | 102 | 103 | 104 | 310 105 | 430 106 | 101 107 | 23 108 | 109 | 110 | 111 | Import struture 1 112 | 113 | 114 | 115 | 116 | 117 | 660 118 | 430 119 | 101 120 | 23 121 | 122 | 123 | 124 | Import struture 2 125 | 126 | 127 | 128 | 129 | 130 | 830 131 | 640 132 | 121 133 | 23 134 | 135 | 136 | 137 | Use merged structure 138 | 139 | 140 | 141 | 142 | 143 | 144 | buttonBox 145 | accepted() 146 | Dialog 147 | accept() 148 | 149 | 150 | 248 151 | 254 152 | 153 | 154 | 157 155 | 274 156 | 157 | 158 | 159 | 160 | buttonBox 161 | rejected() 162 | Dialog 163 | reject() 164 | 165 | 166 | 316 167 | 260 168 | 169 | 170 | 286 171 | 274 172 | 173 | 174 | 175 | 176 | 177 | -------------------------------------------------------------------------------- /example/ex_choose2.py: -------------------------------------------------------------------------------- 1 | import idaapi 2 | from idaapi import Choose2 3 | 4 | # 5 | 6 | 7 | class chooser_handler_t(idaapi.action_handler_t): 8 | def __init__(self, thing): 9 | idaapi.action_handler_t.__init__(self) 10 | self.thing = thing 11 | 12 | def activate(self, ctx): 13 | sel = [] 14 | for i in xrange(len(ctx.chooser_selection)): 15 | sel.append(str(ctx.chooser_selection.at(i))) 16 | print "command %s selected @ %s" % (self.thing, ", ".join(sel)) 17 | 18 | def update(self, ctx): 19 | return idaapi.AST_ENABLE_FOR_FORM if idaapi.is_chooser_tform(ctx.form_type) else idaapi.AST_DISABLE_FOR_FORM 20 | 21 | 22 | class MyChoose2(Choose2): 23 | 24 | def __init__(self, title, nb = 5, flags=0x2, width=None, height=None, embedded=False, modal=False): 25 | Choose2.__init__( 26 | self, 27 | title, 28 | [ ["Address", 10], ["Name", 30] ], 29 | flags = flags, 30 | width = width, 31 | height = height, 32 | embedded = embedded) 33 | self.n = 0 34 | self.items = [ self.make_item() for x in xrange(0, nb+1) ] 35 | self.icon = 5 36 | self.selcount = 0 37 | self.modal = modal 38 | self.popup_names = ["Inzert", "Del leet", "Ehdeet", "Ree frech"] 39 | 40 | print("created %s" % str(self)) 41 | 42 | def OnClose(self): 43 | print "closed", str(self) 44 | 45 | def OnEditLine(self, n): 46 | self.items[n][1] = self.items[n][1] + "*" 47 | print("editing %d" % n) 48 | 49 | def OnInsertLine(self): 50 | self.items.append(self.make_item()) 51 | print("insert line") 52 | 53 | def OnSelectLine(self, n): 54 | self.selcount += 1 55 | Warning("[%02d] selectline '%s'" % (self.selcount, n)) 56 | 57 | def OnGetLine(self, n): 58 | # print("getline %d" % n) 59 | return self.items[n] 60 | 61 | def OnGetSize(self): 62 | n = len(self.items) 63 | # print("getsize -> %d" % n) 64 | return n 65 | 66 | def OnDeleteLine(self, n): 67 | 68 | if n > 0: 69 | print("del %d " % n) 70 | print self.items[n] 71 | del self.items[n] 72 | return n 73 | 74 | def OnCommand(self, n, cmd_id): 75 | # """Return int ; check add_chooser_command()""" 76 | print n 77 | print "cmd_id = %d"%cmd_id 78 | return 0 79 | 80 | def OnRefresh(self, n): 81 | # print("refresh %d" % n) 82 | return n 83 | 84 | def OnGetIcon(self, n): 85 | r = self.items[n] 86 | t = self.icon + r[1].count("*") 87 | # print "geticon", n, t 88 | return t 89 | 90 | def show(self): 91 | return self.Show(self.modal) >= 0 92 | 93 | def make_item(self): 94 | r = [str(self.n), "func_%04d" % self.n] 95 | self.n += 1 96 | return r 97 | 98 | def OnGetLineAttr(self, n): 99 | # print("getlineattr %d" % n) 100 | if n == 1: 101 | return [0xFF0000, 0] 102 | 103 | def OnSelectionChange(self, sel_list): 104 | print sel_list 105 | 106 | 107 | 108 | # ----------------------------------------------------------------------- 109 | def test_choose2(modal=False): 110 | global c 111 | c = MyChoose2("Choose2 - sample 1", nb=10, modal=modal) 112 | r = c.show() 113 | print r 114 | form = idaapi.get_current_tform() 115 | for thing in ["A", "B"]: 116 | idaapi.attach_action_to_popup(form, None, "choose2:act%s" % thing) 117 | 118 | # ----------------------------------------------------------------------- 119 | def test_choose2_embedded(): 120 | global c 121 | c = MyChoose2("Choose2 - embedded", nb=12, embedded = True, width=123, height=222) 122 | r = c.Embedded() 123 | if r == 1: 124 | try: 125 | if test_embedded: 126 | o, sel = _idaapi.choose2_get_embedded(c) 127 | print("o=%s, type(o)=%s" % (str(o), type(o))) 128 | test_embedded(o) 129 | finally: 130 | c.Close() 131 | 132 | # ----------------------------------------------------------------------- 133 | if __name__ == '__main__': 134 | 135 | # Register actions 136 | for thing in ["A", "B"]: 137 | actname = "choose2:act%s" % thing 138 | idaapi.register_action( 139 | idaapi.action_desc_t( 140 | actname, 141 | "command %s" % thing, 142 | chooser_handler_t(thing))) 143 | 144 | #test_choose2_embedded() 145 | test_choose2(True) 146 | 147 | # 148 | -------------------------------------------------------------------------------- /pymongo/write_concern.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Tools for working with write concerns.""" 16 | 17 | from bson.py3compat import integer_types, string_type 18 | from pymongo.errors import ConfigurationError 19 | 20 | class WriteConcern(object): 21 | """WriteConcern 22 | 23 | :Parameters: 24 | - `w`: (integer or string) Used with replication, write operations 25 | will block until they have been replicated to the specified number 26 | or tagged set of servers. `w=` always includes the replica 27 | set primary (e.g. w=3 means write to the primary and wait until 28 | replicated to **two** secondaries). **w=0 disables acknowledgement 29 | of write operations and can not be used with other write concern 30 | options.** 31 | - `wtimeout`: (integer) Used in conjunction with `w`. Specify a value 32 | in milliseconds to control how long to wait for write propagation 33 | to complete. If replication does not complete in the given 34 | timeframe, a timeout exception is raised. 35 | - `j`: If ``True`` block until write operations have been committed 36 | to the journal. Cannot be used in combination with `fsync`. Prior 37 | to MongoDB 2.6 this option was ignored if the server was running 38 | without journaling. Starting with MongoDB 2.6 write operations will 39 | fail with an exception if this option is used when the server is 40 | running without journaling. 41 | - `fsync`: If ``True`` and the server is running without journaling, 42 | blocks until the server has synced all data files to disk. If the 43 | server is running with journaling, this acts the same as the `j` 44 | option, blocking until write operations have been committed to the 45 | journal. Cannot be used in combination with `j`. 46 | """ 47 | 48 | __slots__ = ("__document", "__acknowledged") 49 | 50 | def __init__(self, w=None, wtimeout=None, j=None, fsync=None): 51 | self.__document = {} 52 | self.__acknowledged = True 53 | 54 | if wtimeout is not None: 55 | if not isinstance(wtimeout, integer_types): 56 | raise TypeError("wtimeout must be an integer") 57 | self.__document["wtimeout"] = wtimeout 58 | 59 | if j is not None: 60 | if not isinstance(j, bool): 61 | raise TypeError("j must be True or False") 62 | self.__document["j"] = j 63 | 64 | if fsync is not None: 65 | if not isinstance(fsync, bool): 66 | raise TypeError("fsync must be True or False") 67 | if j and fsync: 68 | raise ConfigurationError("Can't set both j " 69 | "and fsync at the same time") 70 | self.__document["fsync"] = fsync 71 | 72 | if self.__document and w == 0: 73 | raise ConfigurationError("Can not use w value " 74 | "of 0 with other options") 75 | if w is not None: 76 | if isinstance(w, integer_types): 77 | self.__acknowledged = w > 0 78 | elif not isinstance(w, string_type): 79 | raise TypeError("w must be an integer or string") 80 | self.__document["w"] = w 81 | 82 | @property 83 | def document(self): 84 | """The document representation of this write concern. 85 | 86 | .. note:: 87 | :class:`WriteConcern` is immutable. Mutating the value of 88 | :attr:`document` does not mutate this :class:`WriteConcern`. 89 | """ 90 | return self.__document.copy() 91 | 92 | @property 93 | def acknowledged(self): 94 | """If ``True`` write operations will wait for acknowledgement before 95 | returning. 96 | """ 97 | return self.__acknowledged 98 | 99 | def __repr__(self): 100 | return ("WriteConcern(%s)" % ( 101 | ", ".join("%s=%s" % kvt for kvt in self.document.items()),)) 102 | 103 | def __eq__(self, other): 104 | return self.document == other.document 105 | 106 | def __ne__(self, other): 107 | return self.document != other.document 108 | 109 | -------------------------------------------------------------------------------- /pymongo/server_description.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Represent one server in the topology.""" 16 | 17 | from pymongo.server_type import SERVER_TYPE 18 | from pymongo.ismaster import IsMaster 19 | 20 | 21 | class ServerDescription(object): 22 | """Immutable representation of one server. 23 | 24 | :Parameters: 25 | - `address`: A (host, port) pair 26 | - `ismaster`: Optional IsMaster instance 27 | - `round_trip_time`: Optional float 28 | - `error`: Optional, the last error attempting to connect to the server 29 | """ 30 | 31 | __slots__ = ( 32 | '_address', '_server_type', '_all_hosts', '_tags', '_replica_set_name', 33 | '_primary', '_max_bson_size', '_max_message_size', 34 | '_max_write_batch_size', '_min_wire_version', '_max_wire_version', 35 | '_round_trip_time', '_me', '_is_writable', '_is_readable', '_error', 36 | '_election_id') 37 | 38 | def __init__( 39 | self, 40 | address, 41 | ismaster=None, 42 | round_trip_time=None, 43 | error=None): 44 | self._address = address 45 | if not ismaster: 46 | ismaster = IsMaster({}) 47 | 48 | self._server_type = ismaster.server_type 49 | self._all_hosts = ismaster.all_hosts 50 | self._tags = ismaster.tags 51 | self._replica_set_name = ismaster.replica_set_name 52 | self._primary = ismaster.primary 53 | self._max_bson_size = ismaster.max_bson_size 54 | self._max_message_size = ismaster.max_message_size 55 | self._max_write_batch_size = ismaster.max_write_batch_size 56 | self._min_wire_version = ismaster.min_wire_version 57 | self._max_wire_version = ismaster.max_wire_version 58 | self._election_id = ismaster.election_id 59 | self._is_writable = ismaster.is_writable 60 | self._is_readable = ismaster.is_readable 61 | self._round_trip_time = round_trip_time 62 | self._me = ismaster.me 63 | self._error = error 64 | 65 | @property 66 | def address(self): 67 | return self._address 68 | 69 | @property 70 | def server_type(self): 71 | return self._server_type 72 | 73 | @property 74 | def all_hosts(self): 75 | """List of hosts, passives, and arbiters known to this server.""" 76 | return self._all_hosts 77 | 78 | @property 79 | def tags(self): 80 | return self._tags 81 | 82 | @property 83 | def replica_set_name(self): 84 | """Replica set name or None.""" 85 | return self._replica_set_name 86 | 87 | @property 88 | def primary(self): 89 | """This server's opinion about who the primary is, or None.""" 90 | return self._primary 91 | 92 | @property 93 | def max_bson_size(self): 94 | return self._max_bson_size 95 | 96 | @property 97 | def max_message_size(self): 98 | return self._max_message_size 99 | 100 | @property 101 | def max_write_batch_size(self): 102 | return self._max_write_batch_size 103 | 104 | @property 105 | def min_wire_version(self): 106 | return self._min_wire_version 107 | 108 | @property 109 | def max_wire_version(self): 110 | return self._max_wire_version 111 | 112 | @property 113 | def election_id(self): 114 | return self._election_id 115 | 116 | @property 117 | def me(self): 118 | return self._me 119 | 120 | @property 121 | def round_trip_time(self): 122 | """The current average latency or None.""" 123 | # This override is for unittesting only! 124 | if self._address in self._host_to_round_trip_time: 125 | return self._host_to_round_trip_time[self._address] 126 | 127 | return self._round_trip_time 128 | 129 | @property 130 | def error(self): 131 | """The last error attempting to connect to the server, or None.""" 132 | return self._error 133 | 134 | @property 135 | def is_writable(self): 136 | return self._is_writable 137 | 138 | @property 139 | def is_readable(self): 140 | return self._is_readable 141 | 142 | @property 143 | def is_server_type_known(self): 144 | return self.server_type != SERVER_TYPE.Unknown 145 | 146 | # For unittesting only. Use under no circumstances! 147 | _host_to_round_trip_time = {} 148 | -------------------------------------------------------------------------------- /pymongo/server_selectors.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you 4 | # may not use this file except in compliance with the License. You 5 | # may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. See the License for the specific language governing 13 | # permissions and limitations under the License. 14 | 15 | """Criteria to select some ServerDescriptions out of a list.""" 16 | 17 | from pymongo.server_type import SERVER_TYPE 18 | 19 | 20 | def any_server_selector(server_descriptions): 21 | return server_descriptions 22 | 23 | 24 | def readable_server_selector(server_descriptions): 25 | return [s for s in server_descriptions if s.is_readable] 26 | 27 | 28 | def writable_server_selector(server_descriptions): 29 | return [s for s in server_descriptions if s.is_writable] 30 | 31 | 32 | def secondary_server_selector(server_descriptions): 33 | return [s for s in server_descriptions 34 | if s.server_type == SERVER_TYPE.RSSecondary] 35 | 36 | 37 | def arbiter_server_selector(server_descriptions): 38 | return [s for s in server_descriptions 39 | if s.server_type == SERVER_TYPE.RSArbiter] 40 | 41 | 42 | def writable_preferred_server_selector(server_descriptions): 43 | """Like PrimaryPreferred but doesn't use tags or latency.""" 44 | return ( 45 | writable_server_selector(server_descriptions) or 46 | secondary_server_selector(server_descriptions)) 47 | 48 | 49 | def single_tag_set_server_selector(tag_set, server_descriptions): 50 | """All servers matching one tag set. 51 | 52 | A tag set is a dict. A server matches if its tags are a superset: 53 | A server tagged {'a': '1', 'b': '2'} matches the tag set {'a': '1'}. 54 | 55 | The empty tag set {} matches any server. 56 | 57 | The `server_descriptions` passed to this function should have 58 | non-readable servers (e.g. RSGhost, RSArbiter, Unknown) filtered 59 | out (e.g. by readable_server_selector or secondary_server_selector) 60 | first. 61 | """ 62 | def tags_match(server_tags): 63 | for key, value in tag_set.items(): 64 | if key not in server_tags or server_tags[key] != value: 65 | return False 66 | 67 | return True 68 | 69 | return [s for s in server_descriptions if tags_match(s.tags)] 70 | 71 | 72 | def tag_sets_server_selector(tag_sets, server_descriptions): 73 | """All servers match a list of tag sets. 74 | 75 | tag_sets is a list of dicts. The empty tag set {} matches any server, 76 | and may be provided at the end of the list as a fallback. So 77 | [{'a': 'value'}, {}] expresses a preference for servers tagged 78 | {'a': 'value'}, but accepts any server if none matches the first 79 | preference. 80 | 81 | The `server_descriptions` passed to this function should have 82 | non-readable servers (e.g. RSGhost, RSArbiter, Unknown) filtered 83 | out (e.g. by readable_server_selector or secondary_server_selector) 84 | first. 85 | """ 86 | for tag_set in tag_sets: 87 | selected = single_tag_set_server_selector(tag_set, server_descriptions) 88 | if selected: 89 | return selected 90 | 91 | return [] 92 | 93 | 94 | def apply_local_threshold(latency_ms, server_descriptions): 95 | """All servers with round trip times within latency_ms of the fastest one. 96 | 97 | No ServerDescription's round_trip_time can be None. 98 | 99 | The `server_descriptions` passed to this function should have 100 | non-readable servers (e.g. RSGhost, RSArbiter, Unknown) filtered 101 | out (e.g. by readable_server_selector or secondary_server_selector) 102 | first. 103 | """ 104 | if not server_descriptions: 105 | # Avoid ValueError from min() with empty sequence. 106 | return [] 107 | 108 | # round_trip_time is in seconds. 109 | if any(s for s in server_descriptions if s.round_trip_time is None): 110 | raise ValueError("Not all servers' round trip times are known") 111 | 112 | fastest = min(s.round_trip_time for s in server_descriptions) 113 | return [ 114 | s for s in server_descriptions 115 | if (s.round_trip_time - fastest) <= latency_ms / 1000.] 116 | 117 | 118 | def secondary_with_tags_server_selector(tag_sets, server_descriptions): 119 | """All near-enough secondaries matching the tag sets.""" 120 | return tag_sets_server_selector( 121 | tag_sets, secondary_server_selector(server_descriptions)) 122 | 123 | 124 | def member_with_tags_server_selector(tag_sets, server_descriptions): 125 | """All near-enough members matching the tag sets.""" 126 | return tag_sets_server_selector( 127 | tag_sets, readable_server_selector(server_descriptions)) 128 | -------------------------------------------------------------------------------- /pymongo/periodic_executor.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you 4 | # may not use this file except in compliance with the License. You 5 | # may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. See the License for the specific language governing 13 | # permissions and limitations under the License. 14 | 15 | """Run a target function on a background thread.""" 16 | 17 | import atexit 18 | import threading 19 | import time 20 | import weakref 21 | 22 | from pymongo.monotonic import time as _time 23 | 24 | 25 | class PeriodicExecutor(object): 26 | def __init__(self, interval, min_interval, target, name=None): 27 | """"Run a target function periodically on a background thread. 28 | 29 | If the target's return value is false, the executor stops. 30 | 31 | :Parameters: 32 | - `interval`: Seconds between calls to `target`. 33 | - `min_interval`: Minimum seconds between calls if `wake` is 34 | called very often. 35 | - `target`: A function. 36 | - `name`: A name to give the underlying thread. 37 | """ 38 | # threading.Event and its internal condition variable are expensive 39 | # in Python 2, see PYTHON-983. Use a boolean to know when to wake. 40 | # The executor's design is constrained by several Python issues, see 41 | # "periodic_executor.rst" in this repository. 42 | self._event = False 43 | self._interval = interval 44 | self._min_interval = min_interval 45 | self._target = target 46 | self._stopped = False 47 | self._thread = None 48 | self._name = name 49 | 50 | def open(self): 51 | """Start. Multiple calls have no effect. 52 | 53 | Not safe to call from multiple threads at once. 54 | """ 55 | self._stopped = False 56 | started = False 57 | try: 58 | started = self._thread and self._thread.is_alive() 59 | except ReferenceError: 60 | # Thread terminated. 61 | pass 62 | 63 | if not started: 64 | thread = threading.Thread(target=self._run, name=self._name) 65 | thread.daemon = True 66 | self._thread = weakref.proxy(thread) 67 | _register_executor(self) 68 | thread.start() 69 | 70 | def close(self, dummy=None): 71 | """Stop. To restart, call open(). 72 | 73 | The dummy parameter allows an executor's close method to be a weakref 74 | callback; see monitor.py. 75 | """ 76 | self._stopped = True 77 | 78 | def join(self, timeout=None): 79 | if self._thread is not None: 80 | try: 81 | self._thread.join(timeout) 82 | except ReferenceError: 83 | # Thread already terminated. 84 | pass 85 | 86 | def wake(self): 87 | """Execute the target function soon.""" 88 | self._event = True 89 | 90 | def _run(self): 91 | while not self._stopped: 92 | try: 93 | if not self._target(): 94 | self._stopped = True 95 | break 96 | except: 97 | self._stopped = True 98 | raise 99 | 100 | deadline = _time() + self._interval 101 | 102 | while not self._stopped and _time() < deadline: 103 | time.sleep(self._min_interval) 104 | if self._event: 105 | break # Early wake. 106 | 107 | self._event = False 108 | 109 | 110 | # _EXECUTORS has a weakref to each running PeriodicExecutor. Once started, 111 | # an executor is kept alive by a strong reference from its thread and perhaps 112 | # from other objects. When the thread dies and all other referrers are freed, 113 | # the executor is freed and removed from _EXECUTORS. If any threads are 114 | # running when the interpreter begins to shut down, we try to halt and join 115 | # them to avoid spurious errors. 116 | _EXECUTORS = set() 117 | 118 | 119 | def _register_executor(executor): 120 | ref = weakref.ref(executor, _on_executor_deleted) 121 | _EXECUTORS.add(ref) 122 | 123 | 124 | def _on_executor_deleted(ref): 125 | _EXECUTORS.remove(ref) 126 | 127 | 128 | def _shutdown_executors(): 129 | # Copy the set. Stopping threads has the side effect of removing executors. 130 | executors = list(_EXECUTORS) 131 | 132 | # First signal all executors to close... 133 | for ref in executors: 134 | executor = ref() 135 | if executor: 136 | executor.close() 137 | 138 | # ...then try to join them. 139 | for ref in executors: 140 | executor = ref() 141 | if executor: 142 | executor.join(1) 143 | 144 | executor = None 145 | 146 | atexit.register(_shutdown_executors) 147 | -------------------------------------------------------------------------------- /example/test.py: -------------------------------------------------------------------------------- 1 | str = '''struct __cppobj CList : CObject 2 | { 3 | CList::CNode *m_pNodeHead; 4 | CList::CNode *m_pNodeTail; 5 | int m_nCount; 6 | CList::CNode *m_pNodeFree; 7 | CPlex *m_pBlocks; 8 | int m_nBlockSize; 9 | };''' 10 | specials = ["__cppobj", "__unaligned", "__declspec"] 11 | def CheckSpecial(data): 12 | for special in specials: 13 | if data.startswith(special): 14 | return True 15 | return False 16 | 17 | def ExtractSpecs(data): 18 | specs = () 19 | while CheckSpecial(data): 20 | temp = data.strip(" \n;").split(" ",1) 21 | specs = specs + (temp[0].strip(" \n;"),) 22 | data = temp[1].strip(" \n;") 23 | return specs, data 24 | 25 | 26 | 27 | 28 | class AA(object): 29 | def __init__(self): 30 | self.l = [1,2,3,4,5,6,7,8] 31 | self.g = [] 32 | 33 | def GetL(self): 34 | return self.l 35 | 36 | def PutToG(self,l,f): 37 | self.g.append(l*f) 38 | 39 | def Do(self): 40 | print map(lambda x:self.PutToG(x,10),self.GetL()) 41 | print self.g 42 | 43 | 44 | def test(func): 45 | print func 46 | func() 47 | 48 | test(AA().Do) 49 | 50 | exit(0) 51 | 52 | import re 53 | str2 = '''ATL::CStringT > >''' 54 | #str2 = '''CMap > >,wchar_t const *,ATL::CStringT > >,wchar_t const *>::CPair''' 55 | str2 = '''CMap > >,wchar_t const *,ATL::CStringT > >,wchar_t const *>::CAssoc : CMap > >,wchar_t const *,ATL::CStringT > >,wchar_t const *>::CPair''' 56 | #str2 = '''__unaligned __declspec(align(1)) CMapPtrToPtr *m_siteMap;''' 57 | 58 | def getline(): 59 | global str 60 | if str == "": 61 | return "" 62 | ret = str.split('\n',1) 63 | if len(ret) == 1: 64 | str = "" 65 | return ret[0] 66 | str = ret[1] 67 | return ret[0] 68 | 69 | #print str.lower().startswith("struct") 70 | #ret = str.split('\n',1) 71 | #str = ret[1] 72 | #print str.split('{\n',1)[1].split('\n}',1)[0].split('\n') 73 | # while getline() != "": 74 | # print "ok" 75 | 76 | p = re.compile(r'\w*(<.*>)\w*') 77 | tok = p.findall(str2) 78 | print tok 79 | print tok[0].strip("<>").rstrip("<>") 80 | 81 | def SplitSTL(Stl_str): 82 | ch_op = 0 83 | ch_cl = 0 84 | start_pos = 0 85 | end_pos = 0 86 | fin = [] 87 | i = 0 88 | for ch in Stl_str: 89 | if ch == "<": 90 | if ch_op == 0: 91 | start_pos = i 92 | ch_op += 1 93 | elif ch == ">": 94 | ch_cl += 1 95 | if ch_op == ch_cl and ch_op != 0: 96 | end_pos = i 97 | print Stl_str[start_pos:end_pos] 98 | i += 1 99 | 100 | #for name in type_names: 101 | 102 | # SplitSTL(tok[0].strip("<>").rstrip("<>")) 103 | 104 | # while len(tok): 105 | # print tok 106 | # tok = p.findall(tok[0].strip("<>").rstrip("<>")) 107 | # print tok 108 | # print p.findall(tok[0].strip("<>")) 109 | # print tok[0][1:tok[0].find(",")] 110 | # str2 = str2.replace(tok[0],"") 111 | # print str2 112 | # tok = p.findall(str2) 113 | # print tok 114 | f = '''struct 115 | { 116 | unsigned __int16 e_magic; 117 | unsigned __int16 e_cblp; 118 | unsigned __int16 e_cp; 119 | unsigned __int16 e_crlc; 120 | unsigned __int16 e_cparhdr; 121 | unsigned __int16 e_minalloc; 122 | unsigned __int16 e_maxalloc; 123 | unsigned __int16 e_ss; 124 | unsigned __int16 e_sp; 125 | unsigned __int16 e_csum; 126 | unsigned __int16 e_ip; 127 | unsigned __int16 e_cs; 128 | unsigned __int16 e_lfarlc; 129 | unsigned __int16 e_ovno; 130 | unsigned __int16 e_res[4]; 131 | unsigned __int16 e_oemid; 132 | unsigned __int16 e_oeminfo; 133 | unsigned __int16 e_res2[10]; 134 | int e_lfanew; 135 | }''' 136 | # p = re.compile(r'\w*(<.*>)\w*') 137 | # tok = p.findall(str2) 138 | # print tok 139 | # while len(tok) > 0: 140 | 141 | 142 | 143 | # r = f.split("\n",1) 144 | # f = r[0] + " " + name + "\n" + r[1] 145 | # print f 146 | #print ExtractSpecs(str2) 147 | 148 | # class TypeStructureMember(object): 149 | # def __init__(self,name,type_name,funcFlag = False,specs = ()): 150 | # self.name = name 151 | # self.type_name = type_name 152 | # self.funcFlag = funcFlag 153 | # self.specs = specs 154 | # self.depended = -1 155 | # 156 | # 157 | # a = [TypeStructureMember("1","11"),TypeStructureMember("2","22")] 158 | # b = {} 159 | # for elem in a: 160 | # b[elem.name] = elem 161 | # 162 | # for elem in b: 163 | # b[elem].type_name = b[elem].type_name +"AAA" 164 | # 165 | # for elem in a: 166 | # print elem.type_name 167 | 168 | # str2 = '''CMap > >,wchar_t const *,CDocument *,CDocument *>::CPair, IClassFactory, ATL::CComObjectRootEx''' 169 | # def ParseSuffix(data): 170 | # i = 0 171 | # idx = 0 172 | # while True: 173 | # if data.find("<") != -1: 174 | # i = i + 1 175 | # idx = idx + data.find("<")+1 176 | # data = data[data.find("<")+1:] 177 | # continue 178 | # if data.find(">") != -1: 179 | # i = i - 1 180 | # idx = idx + data.find(">")+1 181 | # data = data[data.find(">")+1:] 182 | # if i == 0: break 183 | # 184 | # print str2.split(", ") 185 | # ParseSuffix(str2) -------------------------------------------------------------------------------- /pymongo/ssl_support.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you 4 | # may not use this file except in compliance with the License. You 5 | # may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. See the License for the specific language governing 13 | # permissions and limitations under the License. 14 | 15 | """Support for SSL in PyMongo.""" 16 | 17 | import atexit 18 | import sys 19 | import threading 20 | 21 | HAVE_SSL = True 22 | try: 23 | import ssl 24 | except ImportError: 25 | HAVE_SSL = False 26 | 27 | HAVE_CERTIFI = False 28 | try: 29 | import certifi 30 | HAVE_CERTIFI = True 31 | except ImportError: 32 | pass 33 | 34 | HAVE_WINCERTSTORE = False 35 | try: 36 | from wincertstore import CertFile 37 | HAVE_WINCERTSTORE = True 38 | except ImportError: 39 | pass 40 | 41 | from bson.py3compat import string_type 42 | from pymongo.errors import ConfigurationError 43 | 44 | _WINCERTSLOCK = threading.Lock() 45 | _WINCERTS = None 46 | 47 | if HAVE_SSL: 48 | try: 49 | # Python 3.2 and above. 50 | from ssl import SSLContext 51 | except ImportError: 52 | from pymongo.ssl_context import SSLContext 53 | 54 | def validate_cert_reqs(option, value): 55 | """Validate the cert reqs are valid. It must be None or one of the 56 | three values ``ssl.CERT_NONE``, ``ssl.CERT_OPTIONAL`` or 57 | ``ssl.CERT_REQUIRED``. 58 | """ 59 | if value is None: 60 | return value 61 | elif isinstance(value, string_type) and hasattr(ssl, value): 62 | value = getattr(ssl, value) 63 | 64 | if value in (ssl.CERT_NONE, ssl.CERT_OPTIONAL, ssl.CERT_REQUIRED): 65 | return value 66 | raise ValueError("The value of %s must be one of: " 67 | "`ssl.CERT_NONE`, `ssl.CERT_OPTIONAL` or " 68 | "`ssl.CERT_REQUIRED" % (option,)) 69 | 70 | def _load_wincerts(): 71 | """Set _WINCERTS to an instance of wincertstore.Certfile.""" 72 | global _WINCERTS 73 | 74 | certfile = CertFile() 75 | certfile.addstore("CA") 76 | certfile.addstore("ROOT") 77 | atexit.register(certfile.close) 78 | 79 | _WINCERTS = certfile 80 | 81 | # XXX: Possible future work. 82 | # - Support CRL files? Only supported by CPython >= 2.7.9 and >= 3.4 83 | # http://bugs.python.org/issue8813 84 | # - OCSP? Not supported by python at all. 85 | # http://bugs.python.org/issue17123 86 | # - Setting OP_NO_COMPRESSION? The server doesn't yet. 87 | # - Adding an ssl_context keyword argument to MongoClient? This might 88 | # be useful for sites that have unusual requirements rather than 89 | # trying to expose every SSLContext option through a keyword/uri 90 | # parameter. 91 | def get_ssl_context(*args): 92 | """Create and return an SSLContext object.""" 93 | certfile, keyfile, ca_certs, cert_reqs = args 94 | # Note PROTOCOL_SSLv23 is about the most misleading name imaginable. 95 | # This configures the server and client to negotiate the 96 | # highest protocol version they both support. A very good thing. 97 | ctx = SSLContext(ssl.PROTOCOL_SSLv23) 98 | if hasattr(ctx, "options"): 99 | # Explicitly disable SSLv2 and SSLv3. Note that up to 100 | # date versions of MongoDB 2.4 and above already do this, 101 | # python disables SSLv2 by default in >= 2.7.7 and >= 3.3.4 102 | # and SSLv3 in >= 3.4.3. There is no way for us to do this 103 | # explicitly for python 2.6 or 2.7 before 2.7.9. 104 | ctx.options |= getattr(ssl, "OP_NO_SSLv2", 0) 105 | ctx.options |= getattr(ssl, "OP_NO_SSLv3", 0) 106 | if certfile is not None: 107 | ctx.load_cert_chain(certfile, keyfile) 108 | if ca_certs is not None: 109 | ctx.load_verify_locations(ca_certs) 110 | elif cert_reqs != ssl.CERT_NONE: 111 | # CPython >= 2.7.9 or >= 3.4.0, pypy >= 2.5.1 112 | if hasattr(ctx, "load_default_certs"): 113 | ctx.load_default_certs() 114 | # Python >= 3.2.0, useless on Windows. 115 | elif (sys.platform != "win32" and 116 | hasattr(ctx, "set_default_verify_paths")): 117 | ctx.set_default_verify_paths() 118 | elif sys.platform == "win32" and HAVE_WINCERTSTORE: 119 | with _WINCERTSLOCK: 120 | if _WINCERTS is None: 121 | _load_wincerts() 122 | ctx.load_verify_locations(_WINCERTS.name) 123 | elif HAVE_CERTIFI: 124 | ctx.load_verify_locations(certifi.where()) 125 | else: 126 | raise ConfigurationError( 127 | "`ssl_cert_reqs` is not ssl.CERT_NONE and no system " 128 | "CA certificates could be loaded. `ssl_ca_certs` is " 129 | "required.") 130 | ctx.verify_mode = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs 131 | return ctx 132 | else: 133 | def validate_cert_reqs(option, dummy): 134 | """No ssl module, raise ConfigurationError.""" 135 | raise ConfigurationError("The value of %s is set but can't be " 136 | "validated. The ssl module is not available" 137 | % (option,)) 138 | 139 | def get_ssl_context(*dummy): 140 | """No ssl module, raise ConfigurationError.""" 141 | raise ConfigurationError("The ssl module is not available.") 142 | -------------------------------------------------------------------------------- /pymongo/network.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Internal network layer helper methods.""" 16 | 17 | import datetime 18 | import select 19 | import struct 20 | 21 | _HAS_POLL = True 22 | _poller = None 23 | _EVENT_MASK = 0 24 | try: 25 | from select import poll 26 | _poller = poll() 27 | _EVENT_MASK = (select.POLLIN | select.POLLPRI | select.POLLERR | 28 | select.POLLHUP | select.POLLNVAL) 29 | except ImportError: 30 | _HAS_POLL = False 31 | 32 | from pymongo import helpers, message 33 | from pymongo.errors import AutoReconnect, NotMasterError, OperationFailure 34 | 35 | _UNPACK_INT = struct.Struct(" max_bson_size + message._COMMAND_OVERHEAD): 76 | message._raise_document_too_large( 77 | name, size, max_bson_size + message._COMMAND_OVERHEAD) 78 | 79 | if publish: 80 | encoding_duration = datetime.datetime.now() - start 81 | listeners.publish_command_start(orig, dbname, request_id, address) 82 | start = datetime.datetime.now() 83 | 84 | try: 85 | sock.sendall(msg) 86 | response = receive_message(sock, 1, request_id) 87 | unpacked = helpers._unpack_response( 88 | response, codec_options=codec_options) 89 | 90 | response_doc = unpacked['data'][0] 91 | if check: 92 | msg = "command %s on namespace %s failed: %%s" % ( 93 | repr(spec).replace("%", "%%"), ns) 94 | helpers._check_command_response(response_doc, msg, allowable_errors) 95 | except Exception as exc: 96 | if publish: 97 | duration = (datetime.datetime.now() - start) + encoding_duration 98 | if isinstance(exc, (NotMasterError, OperationFailure)): 99 | failure = exc.details 100 | else: 101 | failure = message._convert_exception(exc) 102 | listeners.publish_command_failure( 103 | duration, failure, name, request_id, address) 104 | raise 105 | if publish: 106 | duration = (datetime.datetime.now() - start) + encoding_duration 107 | listeners.publish_command_success( 108 | duration, response_doc, name, request_id, address) 109 | return response_doc 110 | 111 | 112 | def receive_message(sock, operation, request_id): 113 | """Receive a raw BSON message or raise socket.error.""" 114 | header = _receive_data_on_socket(sock, 16) 115 | length = _UNPACK_INT(header[:4])[0] 116 | 117 | actual_op = _UNPACK_INT(header[12:])[0] 118 | assert operation == actual_op, ("wire protocol error: " 119 | "unknown opcode %r" % (actual_op,)) 120 | # No request_id for exhaust cursor "getMore". 121 | if request_id is not None: 122 | response_id = _UNPACK_INT(header[8:12])[0] 123 | assert request_id == response_id, ( 124 | "wire protocol error: got response id %r but expected %r" 125 | % (response_id, request_id)) 126 | 127 | assert length > 16, ("wire protocol error: message length is shorter" 128 | " than standard message header: %r" % (length,)) 129 | 130 | return _receive_data_on_socket(sock, length - 16) 131 | 132 | 133 | def _receive_data_on_socket(sock, length): 134 | msg = b"" 135 | while length: 136 | chunk = sock.recv(length) 137 | if chunk == b"": 138 | raise AutoReconnect("connection closed") 139 | 140 | length -= len(chunk) 141 | msg += chunk 142 | 143 | return msg 144 | 145 | 146 | def socket_closed(sock): 147 | """Return True if we know socket has been closed, False otherwise. 148 | """ 149 | try: 150 | if _HAS_POLL: 151 | _poller.register(sock, _EVENT_MASK) 152 | rd = _poller.poll(0) 153 | _poller.unregister(sock) 154 | else: 155 | rd, _, _ = select.select([sock], [], [], 0) 156 | # Any exception here is equally bad (select.error, ValueError, etc.). 157 | except: 158 | return True 159 | return len(rd) > 0 160 | -------------------------------------------------------------------------------- /pymongo/monitor.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you 4 | # may not use this file except in compliance with the License. You 5 | # may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. See the License for the specific language governing 13 | # permissions and limitations under the License. 14 | 15 | """Class to monitor a MongoDB server on a background thread.""" 16 | 17 | import weakref 18 | 19 | from bson.codec_options import DEFAULT_CODEC_OPTIONS 20 | from pymongo import common, helpers, message, periodic_executor 21 | from pymongo.server_type import SERVER_TYPE 22 | from pymongo.ismaster import IsMaster 23 | from pymongo.monotonic import time as _time 24 | from pymongo.read_preferences import MovingAverage 25 | from pymongo.server_description import ServerDescription 26 | 27 | 28 | class Monitor(object): 29 | def __init__( 30 | self, 31 | server_description, 32 | topology, 33 | pool, 34 | topology_settings): 35 | """Class to monitor a MongoDB server on a background thread. 36 | 37 | Pass an initial ServerDescription, a Topology, a Pool, and 38 | TopologySettings. 39 | 40 | The Topology is weakly referenced. The Pool must be exclusive to this 41 | Monitor. 42 | """ 43 | self._server_description = server_description 44 | self._pool = pool 45 | self._settings = topology_settings 46 | self._avg_round_trip_time = MovingAverage() 47 | 48 | # We strongly reference the executor and it weakly references us via 49 | # this closure. When the monitor is freed, stop the executor soon. 50 | def target(): 51 | monitor = self_ref() 52 | if monitor is None: 53 | return False # Stop the executor. 54 | Monitor._run(monitor) 55 | return True 56 | 57 | executor = periodic_executor.PeriodicExecutor( 58 | interval=common.HEARTBEAT_FREQUENCY, 59 | min_interval=common.MIN_HEARTBEAT_INTERVAL, 60 | target=target, 61 | name="pymongo_server_monitor_thread") 62 | 63 | self._executor = executor 64 | 65 | # Avoid cycles. When self or topology is freed, stop executor soon. 66 | self_ref = weakref.ref(self, executor.close) 67 | self._topology = weakref.proxy(topology, executor.close) 68 | 69 | def open(self): 70 | """Start monitoring, or restart after a fork. 71 | 72 | Multiple calls have no effect. 73 | """ 74 | self._executor.open() 75 | 76 | def close(self): 77 | """Close and stop monitoring. 78 | 79 | open() restarts the monitor after closing. 80 | """ 81 | self._executor.close() 82 | 83 | # Increment the pool_id and maybe close the socket. If the executor 84 | # thread has the socket checked out, it will be closed when checked in. 85 | self._pool.reset() 86 | 87 | def join(self, timeout=None): 88 | self._executor.join(timeout) 89 | 90 | def request_check(self): 91 | """If the monitor is sleeping, wake and check the server soon.""" 92 | self._executor.wake() 93 | 94 | def _run(self): 95 | try: 96 | self._server_description = self._check_with_retry() 97 | self._topology.on_change(self._server_description) 98 | except ReferenceError: 99 | # Topology was garbage-collected. 100 | self.close() 101 | 102 | def _check_with_retry(self): 103 | """Call ismaster once or twice. Reset server's pool on error. 104 | 105 | Returns a ServerDescription. 106 | """ 107 | # According to the spec, if an ismaster call fails we reset the 108 | # server's pool. If a server was once connected, change its type 109 | # to Unknown only after retrying once. 110 | address = self._server_description.address 111 | retry = self._server_description.server_type != SERVER_TYPE.Unknown 112 | 113 | try: 114 | return self._check_once() 115 | except ReferenceError: 116 | raise 117 | except Exception as error: 118 | self._topology.reset_pool(address) 119 | default = ServerDescription(address, error=error) 120 | if not retry: 121 | self._avg_round_trip_time.reset() 122 | # Server type defaults to Unknown. 123 | return default 124 | 125 | # Try a second and final time. If it fails return original error. 126 | try: 127 | return self._check_once() 128 | except ReferenceError: 129 | raise 130 | except Exception: 131 | self._avg_round_trip_time.reset() 132 | return default 133 | 134 | def _check_once(self): 135 | """A single attempt to call ismaster. 136 | 137 | Returns a ServerDescription, or raises an exception. 138 | """ 139 | with self._pool.get_socket({}) as sock_info: 140 | response, round_trip_time = self._check_with_socket(sock_info) 141 | self._avg_round_trip_time.add_sample(round_trip_time) 142 | sd = ServerDescription( 143 | address=self._server_description.address, 144 | ismaster=response, 145 | round_trip_time=self._avg_round_trip_time.get()) 146 | 147 | return sd 148 | 149 | def _check_with_socket(self, sock_info): 150 | """Return (IsMaster, round_trip_time). 151 | 152 | Can raise ConnectionFailure or OperationFailure. 153 | """ 154 | start = _time() 155 | request_id, msg, max_doc_size = message.query( 156 | 0, 'admin.$cmd', 0, -1, {'ismaster': 1}, 157 | None, DEFAULT_CODEC_OPTIONS) 158 | 159 | # TODO: use sock_info.command() 160 | sock_info.send_message(msg, max_doc_size) 161 | raw_response = sock_info.receive_message(1, request_id) 162 | result = helpers._unpack_response(raw_response) 163 | return IsMaster(result['data'][0]), _time() - start 164 | -------------------------------------------------------------------------------- /pymongo/server.py: -------------------------------------------------------------------------------- 1 | # Copyright 2009-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you 4 | # may not use this file except in compliance with the License. You 5 | # may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. See the License for the specific language governing 13 | # permissions and limitations under the License. 14 | 15 | """Communicate with one MongoDB server in a topology.""" 16 | 17 | import contextlib 18 | 19 | from datetime import datetime 20 | 21 | from pymongo.message import _convert_exception 22 | from pymongo.response import Response, ExhaustResponse 23 | from pymongo.server_type import SERVER_TYPE 24 | 25 | 26 | class Server(object): 27 | def __init__(self, server_description, pool, monitor): 28 | """Represent one MongoDB server.""" 29 | self._description = server_description 30 | self._pool = pool 31 | self._monitor = monitor 32 | 33 | def open(self): 34 | """Start monitoring, or restart after a fork. 35 | 36 | Multiple calls have no effect. 37 | """ 38 | self._monitor.open() 39 | 40 | def reset(self): 41 | """Clear the connection pool.""" 42 | self.pool.reset() 43 | 44 | def close(self): 45 | """Clear the connection pool and stop the monitor. 46 | 47 | Reconnect with open(). 48 | """ 49 | self._monitor.close() 50 | self._pool.reset() 51 | 52 | def request_check(self): 53 | """Check the server's state soon.""" 54 | self._monitor.request_check() 55 | 56 | def send_message(self, message, all_credentials): 57 | """Send an unacknowledged message to MongoDB. 58 | 59 | Can raise ConnectionFailure. 60 | 61 | :Parameters: 62 | - `message`: (request_id, data). 63 | - `all_credentials`: dict, maps auth source to MongoCredential. 64 | """ 65 | _, data, max_doc_size = self._split_message(message) 66 | with self.get_socket(all_credentials) as sock_info: 67 | sock_info.send_message(data, max_doc_size) 68 | 69 | def send_message_with_response( 70 | self, 71 | operation, 72 | set_slave_okay, 73 | all_credentials, 74 | listeners, 75 | exhaust=False): 76 | """Send a message to MongoDB and return a Response object. 77 | 78 | Can raise ConnectionFailure. 79 | 80 | :Parameters: 81 | - `operation`: A _Query or _GetMore object. 82 | - `set_slave_okay`: Pass to operation.get_message. 83 | - `all_credentials`: dict, maps auth source to MongoCredential. 84 | - `exhaust` (optional): If True, the socket used stays checked out. 85 | It is returned along with its Pool in the Response. 86 | """ 87 | with self.get_socket(all_credentials, exhaust) as sock_info: 88 | 89 | duration = None 90 | publish = listeners.enabled_for_commands 91 | if publish: 92 | start = datetime.now() 93 | 94 | message = operation.get_message( 95 | set_slave_okay, sock_info.is_mongos) 96 | request_id, data, max_doc_size = self._split_message(message) 97 | 98 | if publish: 99 | encoding_duration = datetime.now() - start 100 | cmd, dbn = operation.as_command() 101 | listeners.publish_command_start( 102 | cmd, dbn, request_id, sock_info.address) 103 | start = datetime.now() 104 | 105 | try: 106 | sock_info.send_message(data, max_doc_size) 107 | response_data = sock_info.receive_message(1, request_id) 108 | except Exception as exc: 109 | if publish: 110 | duration = (datetime.now() - start) + encoding_duration 111 | failure = _convert_exception(exc) 112 | listeners.publish_command_failure( 113 | duration, failure, next(iter(cmd)), request_id, 114 | sock_info.address) 115 | raise 116 | 117 | if publish: 118 | duration = (datetime.now() - start) + encoding_duration 119 | 120 | if exhaust: 121 | return ExhaustResponse( 122 | data=response_data, 123 | address=self._description.address, 124 | socket_info=sock_info, 125 | pool=self._pool, 126 | duration=duration, 127 | request_id=request_id) 128 | else: 129 | return Response( 130 | data=response_data, 131 | address=self._description.address, 132 | duration=duration, 133 | request_id=request_id) 134 | 135 | @contextlib.contextmanager 136 | def get_socket(self, all_credentials, checkout=False): 137 | with self.pool.get_socket(all_credentials, checkout) as sock_info: 138 | yield sock_info 139 | 140 | @property 141 | def description(self): 142 | return self._description 143 | 144 | @description.setter 145 | def description(self, server_description): 146 | assert server_description.address == self._description.address 147 | self._description = server_description 148 | 149 | @property 150 | def pool(self): 151 | return self._pool 152 | 153 | def _split_message(self, message): 154 | """Return request_id, data, max_doc_size. 155 | 156 | :Parameters: 157 | - `message`: (request_id, data, max_doc_size) or (request_id, data) 158 | """ 159 | if len(message) == 3: 160 | return message 161 | else: 162 | # get_more and kill_cursors messages don't include BSON documents. 163 | request_id, data = message 164 | return request_id, data, 0 165 | 166 | def __str__(self): 167 | d = self._description 168 | return '' % ( 169 | d.address[0], d.address[1], 170 | SERVER_TYPE._fields[d.server_type]) 171 | -------------------------------------------------------------------------------- /pymongo/son_manipulator.py: -------------------------------------------------------------------------------- 1 | # Copyright 2009-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Manipulators that can edit SON objects as they enter and exit a database. 16 | 17 | New manipulators should be defined as subclasses of SONManipulator and can be 18 | installed on a database by calling 19 | `pymongo.database.Database.add_son_manipulator`.""" 20 | 21 | import collections 22 | 23 | from bson.dbref import DBRef 24 | from bson.objectid import ObjectId 25 | from bson.son import SON 26 | 27 | 28 | class SONManipulator(object): 29 | """A base son manipulator. 30 | 31 | This manipulator just saves and restores objects without changing them. 32 | """ 33 | 34 | def will_copy(self): 35 | """Will this SON manipulator make a copy of the incoming document? 36 | 37 | Derived classes that do need to make a copy should override this 38 | method, returning True instead of False. All non-copying manipulators 39 | will be applied first (so that the user's document will be updated 40 | appropriately), followed by copying manipulators. 41 | """ 42 | return False 43 | 44 | def transform_incoming(self, son, collection): 45 | """Manipulate an incoming SON object. 46 | 47 | :Parameters: 48 | - `son`: the SON object to be inserted into the database 49 | - `collection`: the collection the object is being inserted into 50 | """ 51 | if self.will_copy(): 52 | return SON(son) 53 | return son 54 | 55 | def transform_outgoing(self, son, collection): 56 | """Manipulate an outgoing SON object. 57 | 58 | :Parameters: 59 | - `son`: the SON object being retrieved from the database 60 | - `collection`: the collection this object was stored in 61 | """ 62 | if self.will_copy(): 63 | return SON(son) 64 | return son 65 | 66 | 67 | class ObjectIdInjector(SONManipulator): 68 | """A son manipulator that adds the _id field if it is missing. 69 | 70 | .. versionchanged:: 2.7 71 | ObjectIdInjector is no longer used by PyMongo, but remains in this 72 | module for backwards compatibility. 73 | """ 74 | 75 | def transform_incoming(self, son, collection): 76 | """Add an _id field if it is missing. 77 | """ 78 | if not "_id" in son: 79 | son["_id"] = ObjectId() 80 | return son 81 | 82 | 83 | # This is now handled during BSON encoding (for performance reasons), 84 | # but I'm keeping this here as a reference for those implementing new 85 | # SONManipulators. 86 | class ObjectIdShuffler(SONManipulator): 87 | """A son manipulator that moves _id to the first position. 88 | """ 89 | 90 | def will_copy(self): 91 | """We need to copy to be sure that we are dealing with SON, not a dict. 92 | """ 93 | return True 94 | 95 | def transform_incoming(self, son, collection): 96 | """Move _id to the front if it's there. 97 | """ 98 | if not "_id" in son: 99 | return son 100 | transformed = SON({"_id": son["_id"]}) 101 | transformed.update(son) 102 | return transformed 103 | 104 | 105 | class NamespaceInjector(SONManipulator): 106 | """A son manipulator that adds the _ns field. 107 | """ 108 | 109 | def transform_incoming(self, son, collection): 110 | """Add the _ns field to the incoming object 111 | """ 112 | son["_ns"] = collection.name 113 | return son 114 | 115 | 116 | class AutoReference(SONManipulator): 117 | """Transparently reference and de-reference already saved embedded objects. 118 | 119 | This manipulator should probably only be used when the NamespaceInjector is 120 | also being used, otherwise it doesn't make too much sense - documents can 121 | only be auto-referenced if they have an *_ns* field. 122 | 123 | NOTE: this will behave poorly if you have a circular reference. 124 | 125 | TODO: this only works for documents that are in the same database. To fix 126 | this we'll need to add a DatabaseInjector that adds *_db* and then make 127 | use of the optional *database* support for DBRefs. 128 | """ 129 | 130 | def __init__(self, db): 131 | self.database = db 132 | 133 | def will_copy(self): 134 | """We need to copy so the user's document doesn't get transformed refs. 135 | """ 136 | return True 137 | 138 | def transform_incoming(self, son, collection): 139 | """Replace embedded documents with DBRefs. 140 | """ 141 | 142 | def transform_value(value): 143 | if isinstance(value, collections.MutableMapping): 144 | if "_id" in value and "_ns" in value: 145 | return DBRef(value["_ns"], transform_value(value["_id"])) 146 | else: 147 | return transform_dict(SON(value)) 148 | elif isinstance(value, list): 149 | return [transform_value(v) for v in value] 150 | return value 151 | 152 | def transform_dict(object): 153 | for (key, value) in object.items(): 154 | object[key] = transform_value(value) 155 | return object 156 | 157 | return transform_dict(SON(son)) 158 | 159 | def transform_outgoing(self, son, collection): 160 | """Replace DBRefs with embedded documents. 161 | """ 162 | 163 | def transform_value(value): 164 | if isinstance(value, DBRef): 165 | return self.database.dereference(value) 166 | elif isinstance(value, list): 167 | return [transform_value(v) for v in value] 168 | elif isinstance(value, collections.MutableMapping): 169 | return transform_dict(SON(value)) 170 | return value 171 | 172 | def transform_dict(object): 173 | for (key, value) in object.items(): 174 | object[key] = transform_value(value) 175 | return object 176 | 177 | return transform_dict(SON(son)) 178 | 179 | # TODO make a generic translator for custom types. Take encode, decode, 180 | # should_encode and should_decode functions and just encode and decode where 181 | # necessary. See examples/custom_type.py for where this would be useful. 182 | # Alternatively it could take a should_encode, to_binary, from_binary and 183 | # binary subtype. 184 | -------------------------------------------------------------------------------- /pymongo/client_options.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you 4 | # may not use this file except in compliance with the License. You 5 | # may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. See the License for the specific language governing 13 | # permissions and limitations under the License. 14 | 15 | """Tools to parse mongo client options.""" 16 | 17 | from bson.codec_options import _parse_codec_options 18 | from pymongo.auth import _build_credentials_tuple 19 | from pymongo.common import validate_boolean 20 | from pymongo import common 21 | from pymongo.errors import ConfigurationError 22 | from pymongo.monitoring import _EventListeners 23 | from pymongo.pool import PoolOptions 24 | from pymongo.read_preferences import make_read_preference 25 | from pymongo.ssl_support import get_ssl_context 26 | from pymongo.write_concern import WriteConcern 27 | 28 | 29 | def _parse_credentials(username, password, database, options): 30 | """Parse authentication credentials.""" 31 | if username is None: 32 | return None 33 | mechanism = options.get('authmechanism', 'DEFAULT') 34 | source = options.get('authsource', database or 'admin') 35 | return _build_credentials_tuple( 36 | mechanism, source, username, password, options) 37 | 38 | 39 | def _parse_read_preference(options): 40 | """Parse read preference options.""" 41 | if 'read_preference' in options: 42 | return options['read_preference'] 43 | 44 | mode = options.get('readpreference', 0) 45 | tags = options.get('readpreferencetags') 46 | return make_read_preference(mode, tags) 47 | 48 | 49 | def _parse_write_concern(options): 50 | """Parse write concern options.""" 51 | concern = options.get('w') 52 | wtimeout = options.get('wtimeout') 53 | j = options.get('j', options.get('journal')) 54 | fsync = options.get('fsync') 55 | return WriteConcern(concern, wtimeout, j, fsync) 56 | 57 | 58 | def _parse_ssl_options(options): 59 | """Parse ssl options.""" 60 | use_ssl = options.get('ssl') 61 | if use_ssl is not None: 62 | validate_boolean('ssl', use_ssl) 63 | 64 | certfile = options.get('ssl_certfile') 65 | keyfile = options.get('ssl_keyfile') 66 | ca_certs = options.get('ssl_ca_certs') 67 | cert_reqs = options.get('ssl_cert_reqs') 68 | match_hostname = options.get('ssl_match_hostname', True) 69 | 70 | ssl_kwarg_keys = [k for k in options 71 | if k.startswith('ssl_') and options[k]] 72 | if use_ssl == False and ssl_kwarg_keys: 73 | raise ConfigurationError("ssl has not been enabled but the " 74 | "following ssl parameters have been set: " 75 | "%s. Please set `ssl=True` or remove." 76 | % ', '.join(ssl_kwarg_keys)) 77 | 78 | if ssl_kwarg_keys and use_ssl is None: 79 | # ssl options imply ssl = True 80 | use_ssl = True 81 | 82 | if use_ssl is True: 83 | ctx = get_ssl_context(certfile, keyfile, ca_certs, cert_reqs) 84 | return ctx, match_hostname 85 | return None, match_hostname 86 | 87 | 88 | def _parse_pool_options(options): 89 | """Parse connection pool options.""" 90 | max_pool_size = options.get('maxpoolsize', common.MAX_POOL_SIZE) 91 | connect_timeout = options.get('connecttimeoutms', common.CONNECT_TIMEOUT) 92 | socket_keepalive = options.get('socketkeepalive', False) 93 | socket_timeout = options.get('sockettimeoutms') 94 | wait_queue_timeout = options.get('waitqueuetimeoutms') 95 | wait_queue_multiple = options.get('waitqueuemultiple') 96 | event_listeners = options.get('event_listeners') 97 | ssl_context, ssl_match_hostname = _parse_ssl_options(options) 98 | return PoolOptions(max_pool_size, 99 | connect_timeout, socket_timeout, 100 | wait_queue_timeout, wait_queue_multiple, 101 | ssl_context, ssl_match_hostname, socket_keepalive, 102 | _EventListeners(event_listeners)) 103 | 104 | 105 | class ClientOptions(object): 106 | 107 | """ClientOptions""" 108 | 109 | def __init__(self, username, password, database, options): 110 | self.__options = options 111 | 112 | self.__codec_options = _parse_codec_options(options) 113 | self.__credentials = _parse_credentials( 114 | username, password, database, options) 115 | self.__local_threshold_ms = options.get( 116 | 'localthresholdms', common.LOCAL_THRESHOLD_MS) 117 | # self.__server_selection_timeout is in seconds. Must use full name for 118 | # common.SERVER_SELECTION_TIMEOUT because it is set directly by tests. 119 | self.__server_selection_timeout = options.get( 120 | 'serverselectiontimeoutms', common.SERVER_SELECTION_TIMEOUT) 121 | self.__pool_options = _parse_pool_options(options) 122 | self.__read_preference = _parse_read_preference(options) 123 | self.__replica_set_name = options.get('replicaset') 124 | self.__write_concern = _parse_write_concern(options) 125 | self.__connect = options.get('connect') 126 | 127 | @property 128 | def _options(self): 129 | """The original options used to create this ClientOptions.""" 130 | return self.__options 131 | 132 | @property 133 | def connect(self): 134 | """Whether to begin discovering a MongoDB topology automatically.""" 135 | return self.__connect 136 | 137 | @property 138 | def codec_options(self): 139 | """A :class:`~bson.codec_options.CodecOptions` instance.""" 140 | return self.__codec_options 141 | 142 | @property 143 | def credentials(self): 144 | """A :class:`~pymongo.auth.MongoCredentials` instance or None.""" 145 | return self.__credentials 146 | 147 | @property 148 | def local_threshold_ms(self): 149 | """The local threshold for this instance.""" 150 | return self.__local_threshold_ms 151 | 152 | @property 153 | def server_selection_timeout(self): 154 | """The server selection timeout for this instance in seconds.""" 155 | return self.__server_selection_timeout 156 | 157 | @property 158 | def pool_options(self): 159 | """A :class:`~pymongo.pool.PoolOptions` instance.""" 160 | return self.__pool_options 161 | 162 | @property 163 | def read_preference(self): 164 | """A read preference instance.""" 165 | return self.__read_preference 166 | 167 | @property 168 | def replica_set_name(self): 169 | """Replica set name or None.""" 170 | return self.__replica_set_name 171 | 172 | @property 173 | def write_concern(self): 174 | """A :class:`~pymongo.write_concern.WriteConcern` instance.""" 175 | return self.__write_concern 176 | -------------------------------------------------------------------------------- /pymongo/errors.py: -------------------------------------------------------------------------------- 1 | # Copyright 2009-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Exceptions raised by PyMongo.""" 16 | 17 | from bson.errors import * 18 | 19 | try: 20 | from ssl import CertificateError 21 | except ImportError: 22 | from pymongo.ssl_match_hostname import CertificateError 23 | 24 | 25 | class PyMongoError(Exception): 26 | """Base class for all PyMongo exceptions.""" 27 | 28 | 29 | class ConnectionFailure(PyMongoError): 30 | """Raised when a connection to the database cannot be made or is lost.""" 31 | 32 | 33 | class AutoReconnect(ConnectionFailure): 34 | """Raised when a connection to the database is lost and an attempt to 35 | auto-reconnect will be made. 36 | 37 | In order to auto-reconnect you must handle this exception, recognizing that 38 | the operation which caused it has not necessarily succeeded. Future 39 | operations will attempt to open a new connection to the database (and 40 | will continue to raise this exception until the first successful 41 | connection is made). 42 | 43 | Subclass of :exc:`~pymongo.errors.ConnectionFailure`. 44 | """ 45 | def __init__(self, message='', errors=None): 46 | self.errors = self.details = errors or [] 47 | ConnectionFailure.__init__(self, message) 48 | 49 | 50 | class NetworkTimeout(AutoReconnect): 51 | """An operation on an open connection exceeded socketTimeoutMS. 52 | 53 | The remaining connections in the pool stay open. In the case of a write 54 | operation, you cannot know whether it succeeded or failed. 55 | 56 | Subclass of :exc:`~pymongo.errors.AutoReconnect`. 57 | """ 58 | 59 | 60 | class NotMasterError(AutoReconnect): 61 | """The server responded "not master" or "node is recovering". 62 | 63 | These errors result from a query, write, or command. The operation failed 64 | because the client thought it was using the primary but the primary has 65 | stepped down, or the client thought it was using a healthy secondary but 66 | the secondary is stale and trying to recover. 67 | 68 | The client launches a refresh operation on a background thread, to update 69 | its view of the server as soon as possible after throwing this exception. 70 | 71 | Subclass of :exc:`~pymongo.errors.AutoReconnect`. 72 | """ 73 | 74 | 75 | class ServerSelectionTimeoutError(AutoReconnect): 76 | """Thrown when no MongoDB server is available for an operation 77 | 78 | If there is no suitable server for an operation PyMongo tries for 79 | ``serverSelectionTimeoutMS`` (default 30 seconds) to find one, then 80 | throws this exception. For example, it is thrown after attempting an 81 | operation when PyMongo cannot connect to any server, or if you attempt 82 | an insert into a replica set that has no primary and does not elect one 83 | within the timeout window, or if you attempt to query with a Read 84 | Preference that the replica set cannot satisfy. 85 | """ 86 | 87 | 88 | class ConfigurationError(PyMongoError): 89 | """Raised when something is incorrectly configured. 90 | """ 91 | 92 | 93 | class OperationFailure(PyMongoError): 94 | """Raised when a database operation fails. 95 | 96 | .. versionadded:: 2.7 97 | The :attr:`details` attribute. 98 | """ 99 | 100 | def __init__(self, error, code=None, details=None): 101 | self.__code = code 102 | self.__details = details 103 | PyMongoError.__init__(self, error) 104 | 105 | @property 106 | def code(self): 107 | """The error code returned by the server, if any. 108 | """ 109 | return self.__code 110 | 111 | @property 112 | def details(self): 113 | """The complete error document returned by the server. 114 | 115 | Depending on the error that occurred, the error document 116 | may include useful information beyond just the error 117 | message. When connected to a mongos the error document 118 | may contain one or more subdocuments if errors occurred 119 | on multiple shards. 120 | """ 121 | return self.__details 122 | 123 | 124 | class CursorNotFound(OperationFailure): 125 | """Raised while iterating query results if the cursor is 126 | invalidated on the server. 127 | 128 | .. versionadded:: 2.7 129 | """ 130 | 131 | 132 | class ExecutionTimeout(OperationFailure): 133 | """Raised when a database operation times out, exceeding the $maxTimeMS 134 | set in the query or command option. 135 | 136 | .. note:: Requires server version **>= 2.6.0** 137 | 138 | .. versionadded:: 2.7 139 | """ 140 | 141 | 142 | class WriteConcernError(OperationFailure): 143 | """Base exception type for errors raised due to write concern. 144 | 145 | .. versionadded:: 3.0 146 | """ 147 | 148 | 149 | class WriteError(OperationFailure): 150 | """Base exception type for errors raised during write operations. 151 | 152 | .. versionadded:: 3.0 153 | """ 154 | 155 | 156 | class WTimeoutError(WriteConcernError): 157 | """Raised when a database operation times out (i.e. wtimeout expires) 158 | before replication completes. 159 | 160 | With newer versions of MongoDB the `details` attribute may include 161 | write concern fields like 'n', 'updatedExisting', or 'writtenTo'. 162 | 163 | .. versionadded:: 2.7 164 | """ 165 | 166 | 167 | class DuplicateKeyError(WriteError): 168 | """Raised when an insert or update fails due to a duplicate key error.""" 169 | 170 | 171 | class BulkWriteError(OperationFailure): 172 | """Exception class for bulk write errors. 173 | 174 | .. versionadded:: 2.7 175 | """ 176 | def __init__(self, results): 177 | OperationFailure.__init__( 178 | self, "batch op errors occurred", 65, results) 179 | 180 | 181 | class InvalidOperation(PyMongoError): 182 | """Raised when a client attempts to perform an invalid operation.""" 183 | 184 | 185 | class InvalidName(PyMongoError): 186 | """Raised when an invalid name is used.""" 187 | 188 | 189 | class CollectionInvalid(PyMongoError): 190 | """Raised when collection validation fails.""" 191 | 192 | 193 | class InvalidURI(ConfigurationError): 194 | """Raised when trying to parse an invalid mongodb URI.""" 195 | 196 | 197 | class ExceededMaxWaiters(Exception): 198 | """Raised when a thread tries to get a connection from a pool and 199 | ``maxPoolSize * waitQueueMultiple`` threads are already waiting. 200 | 201 | .. versionadded:: 2.6 202 | """ 203 | pass 204 | 205 | 206 | class DocumentTooLarge(InvalidDocument): 207 | """Raised when an encoded document is too large for the connected server. 208 | """ 209 | pass 210 | -------------------------------------------------------------------------------- /example/qt_test.py: -------------------------------------------------------------------------------- 1 | from PySide.QtCore import * 2 | from PySide.QtGui import * 3 | import sys 4 | from PySide import QtCore, QtGui 5 | 6 | 7 | class Ui_Dialog(object): 8 | def setupUi(self, Dialog): 9 | Dialog.setObjectName("Dialog") 10 | Dialog.resize(1088, 844) 11 | self.buttonBox = QtGui.QDialogButtonBox(Dialog) 12 | self.buttonBox.setGeometry(QtCore.QRect(910, 810, 171, 31)) 13 | self.buttonBox.setOrientation(QtCore.Qt.Horizontal) 14 | self.buttonBox.setStandardButtons(QtGui.QDialogButtonBox.Cancel|QtGui.QDialogButtonBox.Ok) 15 | self.buttonBox.setObjectName("buttonBox") 16 | self.textEdit_struc1 = QtGui.QTextEdit(Dialog) 17 | self.textEdit_struc1.setGeometry(QtCore.QRect(10, 60, 501, 361)) 18 | self.textEdit_struc1.setObjectName("textEdit_struc1") 19 | self.textEdit_struc2 = QtGui.QTextEdit(Dialog) 20 | self.textEdit_struc2.setGeometry(QtCore.QRect(580, 60, 501, 361)) 21 | self.textEdit_struc2.setObjectName("textEdit_struc2") 22 | self.textEdit_struc3 = QtGui.QTextEdit(Dialog) 23 | self.textEdit_struc3.setGeometry(QtCore.QRect(270, 470, 531, 361)) 24 | self.textEdit_struc3.setObjectName("textEdit_struc3") 25 | self.label_struc1 = QtGui.QLabel(Dialog) 26 | self.label_struc1.setGeometry(QtCore.QRect(200, 20, 61, 20)) 27 | self.label_struc1.setObjectName("label_struc1") 28 | self.label_struc2 = QtGui.QLabel(Dialog) 29 | self.label_struc2.setGeometry(QtCore.QRect(780, 20, 61, 20)) 30 | self.label_struc2.setObjectName("label_struc2") 31 | self.label_strucMerged = QtGui.QLabel(Dialog) 32 | self.label_strucMerged.setGeometry(QtCore.QRect(510, 440, 91, 20)) 33 | self.label_strucMerged.setObjectName("label_strucMerged") 34 | self.pushButton_importStruc1 = QtGui.QPushButton(Dialog) 35 | self.pushButton_importStruc1.setGeometry(QtCore.QRect(310, 430, 101, 23)) 36 | self.pushButton_importStruc1.setObjectName("pushButton_importStruc1") 37 | self.pushButton_importStruc2 = QtGui.QPushButton(Dialog) 38 | self.pushButton_importStruc2.setGeometry(QtCore.QRect(660, 430, 101, 23)) 39 | self.pushButton_importStruc2.setObjectName("pushButton_importStruc2") 40 | self.pushButton_useMerged = QtGui.QPushButton(Dialog) 41 | self.pushButton_useMerged.setGeometry(QtCore.QRect(830, 640, 121, 23)) 42 | self.pushButton_useMerged.setObjectName("pushButton_useMerged") 43 | 44 | self.retranslateUi(Dialog) 45 | #QtCore.QObject.connect(self.buttonBox, QtCore.SIGNAL("accepted()"), Dialog.accept) 46 | #QtCore.QObject.connect(self.buttonBox, QtCore.SIGNAL("rejected()"), Dialog.reject) 47 | QtCore.QMetaObject.connectSlotsByName(Dialog) 48 | 49 | def retranslateUi(self, Dialog): 50 | Dialog.setWindowTitle(QtGui.QApplication.translate("Dialog", "Dialog", None, QtGui.QApplication.UnicodeUTF8)) 51 | self.label_struc1.setText(QtGui.QApplication.translate("Dialog", "Structure 1", None, QtGui.QApplication.UnicodeUTF8)) 52 | self.label_struc2.setText(QtGui.QApplication.translate("Dialog", "Structure 2", None, QtGui.QApplication.UnicodeUTF8)) 53 | self.label_strucMerged.setText(QtGui.QApplication.translate("Dialog", "Merged structure", None, QtGui.QApplication.UnicodeUTF8)) 54 | self.pushButton_importStruc1.setText(QtGui.QApplication.translate("Dialog", "Import struture 1", None, QtGui.QApplication.UnicodeUTF8)) 55 | self.pushButton_importStruc2.setText(QtGui.QApplication.translate("Dialog", "Import struture 2", None, QtGui.QApplication.UnicodeUTF8)) 56 | self.pushButton_useMerged.setText(QtGui.QApplication.translate("Dialog", "Use merged structure", None, QtGui.QApplication.UnicodeUTF8)) 57 | 58 | class TestDialog(QDialog): 59 | def __init__(self, parent=None): 60 | super(TestDialog, self).__init__(parent) 61 | self.ui = Ui_Dialog() 62 | self.ui.setupUi(self) 63 | self.connect(self.ui.pushButton_importStruc1, SIGNAL("clicked()"), self.testOK) 64 | self.connect(self.ui.pushButton_importStruc2, SIGNAL("clicked()"), self.testOK) 65 | self.connect(self.ui.pushButton_useMerged, SIGNAL("clicked()"), self.testOK) 66 | self.ui.textEdit_struc1.setText("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") 67 | self.connect(self.ui.buttonBox,SIGNAL("accepted()"),self.testOK) 68 | self.connect(self.ui.buttonBox,SIGNAL("rejected()"),self.testOK) 69 | self.connect(self.ui.buttonBox,SIGNAL("accepted()"),self.accept) 70 | self.connect(self.ui.buttonBox,SIGNAL("rejected()"),self.reject) 71 | 72 | 73 | def testOK(self): 74 | print ("Ok") 75 | print (self.ui.textEdit_struc1.toPlainText()) 76 | 77 | 78 | 79 | class MyWindow(QDialog): 80 | def __init__(self, parent=None): 81 | super(MyWindow, self).__init__(parent) 82 | 83 | self.setWindowTitle("Send to CMD") 84 | 85 | self.check1 = QCheckBox("Activate Variable") 86 | self.variable = QLineEdit() 87 | self.finalcommand = QLineEdit() 88 | self.clearCommand = QPushButton("Clear") 89 | self.sendCommand = QPushButton("Send") 90 | self.clearOnSend = QCheckBox("Clear on Send") 91 | 92 | self.process = QProcess() 93 | self.console = QTextEdit(self) 94 | 95 | layout = QVBoxLayout() 96 | layout.addWidget(self.check1) 97 | layout.addWidget(self.variable) 98 | layout.addWidget(self.finalcommand) 99 | layout.addWidget(self.clearOnSend) 100 | layout.addWidget(self.clearCommand) 101 | layout.addWidget(self.sendCommand) 102 | layout.addWidget(self.console) 103 | self.setLayout(layout) 104 | 105 | self.connect(self.check1, SIGNAL("clicked()"), self.appendText) 106 | self.variable.textChanged.connect(self.appendText) 107 | 108 | self.clearCommand.clicked.connect(self.Clear) 109 | self.sendCommand.clicked.connect(self.Send) 110 | 111 | def appendText(self): 112 | if self.check1.isChecked(): 113 | TEXT1 = "Dir" + ' ' + str(self.variable.text()) 114 | else: 115 | TEXT1 = "" 116 | self.finalcommand.setText(str(TEXT1)) 117 | 118 | def Clear(self): 119 | if self.clearCommand.isEnabled(): 120 | self.console.clear() 121 | 122 | def Send(self): 123 | if self.clearOnSend.isChecked(): 124 | self.console.clear() 125 | FCTS = "cmd.exe /c" + " " + str(self.finalcommand.text()) 126 | self.process.readyReadStandardOutput.connect(self.readConsole) 127 | self.process.start(FCTS) 128 | if not self.process.waitForStarted(0): 129 | return False 130 | if not self.process.waitForFinished(0): 131 | return False 132 | 133 | def readConsole(self): 134 | #self.console.setText(str(self.process.readAllStandardOutput())) 135 | self.console.append(str(self.process.readAllStandardOutput())) 136 | 137 | def test_show(): 138 | global app 139 | form.show() 140 | app.exec_() 141 | 142 | form.show() 143 | app.exec_() 144 | 145 | app = QApplication(sys.argv) 146 | # form = MyWindow() 147 | form = TestDialog(None) 148 | # form.show() 149 | 150 | form.show() 151 | app.exec_() 152 | 153 | form.show() 154 | app.exec_() -------------------------------------------------------------------------------- /example/create_struct.py: -------------------------------------------------------------------------------- 1 | import struct 2 | import ctypes 3 | import idaapi 4 | import idc 5 | import sys 6 | import time 7 | from idaapi import * 8 | from idc import * 9 | fDebug = False 10 | 11 | if fDebug: 12 | import pydevd 13 | 14 | 15 | def make_field_str(field_num,field_size,pad = 0): 16 | ret = "" 17 | for i in range(0,field_num): 18 | ret += struct.pack(">B",len("field_%X"%(i*field_size))+1) + "field_%X"%(i*field_size) 19 | k = 1 20 | while pad > 0: 21 | ret += struct.pack(">B",len("field_%X"%(i*field_size+k))+1) + "field_%X"%(i*field_size+k) 22 | pad -=1 23 | k +=1 24 | return ret 25 | 26 | def encode_size(num): 27 | enc = 0 28 | if num > 0xF: 29 | t, pad = divmod(num, 0x10) 30 | if t < 0x100: 31 | enc = 0x8100|(pad<<11)|t 32 | return struct.pack(">BB",enc>>8,enc&0xFF) 33 | else: 34 | t1, t2, t3 = (0,0,0) 35 | t1, pad = divmod(num,0x400) 36 | t3 = pad 37 | if pad > 7: 38 | t2, t3 = divmod(pad,8) 39 | return "\xFF\xFF" + struct.pack(">BBB",t1|0x80,t2|0x80,t3<<3|0x40) 40 | else: 41 | return struct.pack(">B",num<<3|1) 42 | 43 | def decode_size(size_str): 44 | l = 0 45 | if size_str[:2] == "\xFF\xFF": 46 | l += 2 47 | size_str = size_str[2:] 48 | b1 = ord(size_str[0]) 49 | l +=1 50 | if b1&0x80: 51 | b2 = ord(size_str[1]) 52 | l += 1 53 | if b2&0x80: 54 | b3 = ord(size_str[2]) 55 | l += 1 56 | if b3&0x40: 57 | t1 = (b1&0x7f)*0x400 58 | t2 = (b2&0x7f)*8 59 | t3 = (b3&0x3f)>>3 60 | return (l,t1+t2+t3) 61 | else: 62 | return None 63 | t1 = b2*0x10 64 | t2 = (b1&0x7f)>>3 65 | return (l,t1+t2) 66 | return (l,b1>>3) 67 | 68 | def make_type_string(field_num,field_size,pad = 0): 69 | ret = "\x0d" + encode_size(field_num) 70 | if field_size == 1: 71 | t = "\x32" 72 | elif field_size == 2: 73 | t = "\x03" 74 | elif field_size == 8: 75 | t = "\x05" 76 | else: 77 | t = "\x07" 78 | ret += t*field_num 79 | if pad > 0: 80 | ret += "\x32"*pad 81 | return ret 82 | 83 | ############################################################ 84 | # Several type-related functions aren't accessibly via IDAPython 85 | # so have to do things with ctypes 86 | idaname = "ida64" if idc.__EA64__ else "ida" 87 | if sys.platform == "win32": 88 | g_dll = ctypes.windll[idaname + ".wll"] 89 | elif sys.platform == "linux2": 90 | g_dll = ctypes.cdll["lib" + idaname + ".so"] 91 | elif sys.platform == "darwin": 92 | g_dll = ctypes.cdll["lib" + idaname + ".dylib"] 93 | 94 | wrapperTypeString = '\x0d\x01\x01' 95 | 96 | class qtype(Structure): 97 | _fields_ = [("ptr",ctypes.c_char_p),("cur_size",ctypes.c_int),("max_size",ctypes.c_int)] 98 | 99 | 100 | ############################################################ 101 | # Specifying function types for a few IDA SDK functions to keep the 102 | # pointer-to-pointer args clear. 103 | 104 | 105 | d_get_named_type = g_dll.get_named_type 106 | d_get_named_type.argtypes = [ 107 | ctypes.c_void_p, #const til_t *ti, 108 | ctypes.c_char_p, #const char *name, 109 | ctypes.c_int, #int ntf_flags, 110 | ctypes.POINTER(ctypes.POINTER(ctypes.c_ubyte)), #const type_t **type=NULL, 111 | ctypes.POINTER(ctypes.POINTER(ctypes.c_ubyte)), #const p_list **fields=NULL, 112 | ctypes.POINTER(ctypes.POINTER(ctypes.c_ubyte)), #const char **cmt=NULL, 113 | ctypes.POINTER(ctypes.POINTER(ctypes.c_ubyte)), #const p_list **fieldcmts=NULL, 114 | ctypes.POINTER(ctypes.c_ulong), #sclass_t *sclass=NULL, 115 | ctypes.POINTER(ctypes.c_ulong), #uint32 *value=NULL); 116 | ] 117 | 118 | ############################################################ 119 | d_get_numbered_type = g_dll.get_numbered_type 120 | d_get_numbered_type.argtypes = [ 121 | ctypes.c_void_p, #const til_t *ti, 122 | ctypes.c_int, #uint32 ordinal, 123 | ctypes.POINTER(ctypes.c_char_p), #const type_t **type=NULL, 124 | ctypes.POINTER(ctypes.c_char_p), #const p_list **fields=NULL, 125 | ctypes.POINTER(ctypes.c_char_p), #const char **cmt=NULL, 126 | ctypes.POINTER(ctypes.c_char_p), #const p_list **fieldcmts=NULL, 127 | ctypes.POINTER(ctypes.c_ulong), #sclass_t *sclass=NULL 128 | ] 129 | 130 | d_set_numbered_type = g_dll.set_numbered_type 131 | d_set_numbered_type.argtypes = [ 132 | ctypes.c_void_p, #til_t *ti, 133 | ctypes.c_int, #uint32 ordinal, 134 | ctypes.c_int, #int ntf_flags, 135 | ctypes.c_char_p, #const char *name, 136 | ctypes.c_char_p, #const type_t *type, 137 | ctypes.c_char_p, #const p_list *fields=NULL, 138 | ctypes.c_char_p, #const char *cmt=NULL, 139 | ctypes.c_char_p, #const p_list *fldcmts=NULL, 140 | ctypes.POINTER(ctypes.c_ulong), #const sclass_t *sclass=NULL 141 | ] 142 | 143 | my_til = ctypes.c_void_p.in_dll(g_dll, 'idati') 144 | my_ti = idaapi.cvar.idati 145 | 146 | 147 | 148 | 149 | 150 | 151 | def create_struct_type(struc_size,name,field_size = 4,fAllign = True): 152 | if fDebug == True: 153 | pydevd.settrace('127.0.0.1', port=31337, stdoutToServer=True, stderrToServer=True, suspend=False) 154 | idx = get_type_ordinal(my_ti,name) 155 | flags = 1 156 | if idx != 0: 157 | answer = AskYN(0,"A structure for %s already exists. Are you sure you want to remake it?"%name) 158 | if answer == 1: 159 | flags = 4 160 | else: 161 | return 162 | fields_num, pad = divmod(struc_size, field_size) 163 | if fAllign and pad: 164 | fields_num += 1 165 | pad = 0 166 | typ_type = ctypes.c_char_p(make_type_string(fields_num, field_size,pad)) 167 | typ_fields = ctypes.c_char_p(make_field_str(fields_num, field_size,pad)) 168 | typ_cmt = ctypes.c_char_p("") 169 | typ_fieldcmts = ctypes.c_char_p("") 170 | sclass = ctypes.c_ulong(0) 171 | compact_til(my_ti) 172 | if idx == 0: 173 | idx = alloc_type_ordinal(my_ti) 174 | ret = d_set_numbered_type( 175 | my_til, 176 | idx, 177 | flags, 178 | ctypes.c_char_p(name), 179 | typ_type, 180 | typ_fields, 181 | typ_cmt, 182 | typ_fieldcmts, 183 | ctypes.byref(sclass) 184 | ) 185 | if ret != 0: 186 | start = time.time() 187 | import_type(my_ti,-1,name,IMPTYPE_OVERRIDE) 188 | print "Time for import = %f"%(time.time() - start) 189 | else: 190 | Warning("set_numbered_type error") 191 | 192 | 193 | class CreateStructForm(Form): 194 | """Simple Form to test multilinetext and combo box controls""" 195 | def __init__(self): 196 | Form.__init__(self, r"""STARTITEM 0 197 | Create struct 198 | 199 | 200 | {gAlign}> 201 | 202 | """, { 203 | 'cStrArg':Form.StringInput(), 204 | 'numSize':Form.NumericInput(tp=Form.FT_HEX), 205 | 'numFieldSize':Form.DropdownListControl( 206 | items=["1", "2", "4", "8"], 207 | readonly=False, 208 | selval="4"), 209 | 'gAlign': Form.ChkGroupControl(("ckAlign",)), 210 | }) 211 | 212 | def Go(self,size = 0): 213 | self.Compile() 214 | self.ckAlign.checked = True 215 | #f.numFieldSize.value = 4 216 | self.numSize.value = size 217 | ok = self.Execute() 218 | #print "Ok = %d"%ok 219 | if ok == 1: 220 | #print sel 221 | #print len(sel) 222 | print "Name = %s, size = %d, field size = %d, isAligh = %s"%(self.cStrArg.value,self.numSize.value,int(self.numFieldSize.value),"True" if self.ckAlign.checked else "False") 223 | create_struct_type(self.numSize.value,self.cStrArg.value,int(self.numFieldSize.value),self.ckAlign.checked) 224 | return 225 | return 226 | 227 | def OnFormChange(self, fid): 228 | 229 | return 1 230 | 231 | f = CreateStructForm() 232 | # print f.Go("aaaaaaaaa",'bbbbbbbbbb') 233 | f.Go() 234 | -------------------------------------------------------------------------------- /pymongo/results.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Result class definitions.""" 16 | 17 | from pymongo.errors import InvalidOperation 18 | 19 | 20 | class _WriteResult(object): 21 | """Base class for write result classes.""" 22 | 23 | def __init__(self, acknowledged): 24 | self.__acknowledged = acknowledged 25 | 26 | def _raise_if_unacknowledged(self, property_name): 27 | """Raise an exception on property access if unacknowledged.""" 28 | if not self.__acknowledged: 29 | raise InvalidOperation("A value for %s is not available when " 30 | "the write is unacknowledged. Check the " 31 | "acknowledged attribute to avoid this " 32 | "error." % (property_name,)) 33 | 34 | @property 35 | def acknowledged(self): 36 | """Is this the result of an acknowledged write operation? 37 | 38 | The :attr:`acknowledged` attribute will be ``False`` when using 39 | ``WriteConcern(w=0)``, otherwise ``True``. 40 | 41 | .. note:: 42 | If the :attr:`acknowledged` attribute is ``False`` all other 43 | attibutes of this class will raise 44 | :class:`~pymongo.errors.InvalidOperation` when accessed. Values for 45 | other attributes cannot be determined if the write operation was 46 | unacknowledged. 47 | 48 | .. seealso:: 49 | :class:`~pymongo.write_concern.WriteConcern` 50 | """ 51 | return self.__acknowledged 52 | 53 | 54 | class InsertOneResult(_WriteResult): 55 | """The return type for :meth:`~pymongo.collection.Collection.insert_one`. 56 | """ 57 | 58 | __slots__ = ("__inserted_id", "__acknowledged") 59 | 60 | def __init__(self, inserted_id, acknowledged): 61 | self.__inserted_id = inserted_id 62 | super(InsertOneResult, self).__init__(acknowledged) 63 | 64 | @property 65 | def inserted_id(self): 66 | """The inserted document's _id.""" 67 | return self.__inserted_id 68 | 69 | 70 | class InsertManyResult(_WriteResult): 71 | """The return type for :meth:`~pymongo.collection.Collection.insert_many`. 72 | """ 73 | 74 | __slots__ = ("__inserted_ids", "__acknowledged") 75 | 76 | def __init__(self, inserted_ids, acknowledged): 77 | self.__inserted_ids = inserted_ids 78 | super(InsertManyResult, self).__init__(acknowledged) 79 | 80 | @property 81 | def inserted_ids(self): 82 | """A list of _ids of the inserted documents, in the order provided. 83 | 84 | .. note:: If ``False`` is passed for the `ordered` parameter to 85 | :meth:`~pymongo.collection.Collection.insert_many` the server 86 | may have inserted the documents in a different order than what 87 | is presented here. 88 | """ 89 | return self.__inserted_ids 90 | 91 | 92 | class UpdateResult(_WriteResult): 93 | """The return type for :meth:`~pymongo.collection.Collection.update_one`, 94 | :meth:`~pymongo.collection.Collection.update_many`, and 95 | :meth:`~pymongo.collection.Collection.replace_one`. 96 | """ 97 | 98 | __slots__ = ("__raw_result", "__acknowledged") 99 | 100 | def __init__(self, raw_result, acknowledged): 101 | self.__raw_result = raw_result 102 | super(UpdateResult, self).__init__(acknowledged) 103 | 104 | @property 105 | def raw_result(self): 106 | """The raw result document returned by the server.""" 107 | return self.__raw_result 108 | 109 | @property 110 | def matched_count(self): 111 | """The number of documents matched for this update.""" 112 | self._raise_if_unacknowledged("matched_count") 113 | if self.upserted_id is not None: 114 | return 0 115 | return self.__raw_result.get("n", 0) 116 | 117 | @property 118 | def modified_count(self): 119 | """The number of documents modified. 120 | 121 | .. note:: modified_count is only reported by MongoDB 2.6 and later. 122 | When connected to an earlier server version, or in certain mixed 123 | version sharding configurations, this attribute will be set to 124 | ``None``. 125 | """ 126 | self._raise_if_unacknowledged("modified_count") 127 | return self.__raw_result.get("nModified") 128 | 129 | @property 130 | def upserted_id(self): 131 | """The _id of the inserted document if an upsert took place. Otherwise 132 | ``None``. 133 | """ 134 | self._raise_if_unacknowledged("upserted_id") 135 | return self.__raw_result.get("upserted") 136 | 137 | 138 | class DeleteResult(_WriteResult): 139 | """The return type for :meth:`~pymongo.collection.Collection.delete_one` 140 | and :meth:`~pymongo.collection.Collection.delete_many`""" 141 | 142 | __slots__ = ("__raw_result", "__acknowledged") 143 | 144 | def __init__(self, raw_result, acknowledged): 145 | self.__raw_result = raw_result 146 | super(DeleteResult, self).__init__(acknowledged) 147 | 148 | @property 149 | def raw_result(self): 150 | """The raw result document returned by the server.""" 151 | return self.__raw_result 152 | 153 | @property 154 | def deleted_count(self): 155 | """The number of documents deleted.""" 156 | self._raise_if_unacknowledged("deleted_count") 157 | return self.__raw_result.get("n", 0) 158 | 159 | 160 | class BulkWriteResult(_WriteResult): 161 | """An object wrapper for bulk API write results.""" 162 | 163 | __slots__ = ("__bulk_api_result", "__acknowledged") 164 | 165 | def __init__(self, bulk_api_result, acknowledged): 166 | """Create a BulkWriteResult instance. 167 | 168 | :Parameters: 169 | - `bulk_api_result`: A result dict from the bulk API 170 | - `acknowledged`: Was this write result acknowledged? If ``False`` 171 | then all properties of this object will raise 172 | :exc:`~pymongo.errors.InvalidOperation`. 173 | """ 174 | self.__bulk_api_result = bulk_api_result 175 | super(BulkWriteResult, self).__init__(acknowledged) 176 | 177 | @property 178 | def bulk_api_result(self): 179 | """The raw bulk API result.""" 180 | return self.__bulk_api_result 181 | 182 | @property 183 | def inserted_count(self): 184 | """The number of documents inserted.""" 185 | self._raise_if_unacknowledged("inserted_count") 186 | return self.__bulk_api_result.get("nInserted") 187 | 188 | @property 189 | def matched_count(self): 190 | """The number of documents matched for an update.""" 191 | self._raise_if_unacknowledged("matched_count") 192 | return self.__bulk_api_result.get("nMatched") 193 | 194 | @property 195 | def modified_count(self): 196 | """The number of documents modified. 197 | 198 | .. note:: modified_count is only reported by MongoDB 2.6 and later. 199 | When connected to an earlier server version, or in certain mixed 200 | version sharding configurations, this attribute will be set to 201 | ``None``. 202 | """ 203 | self._raise_if_unacknowledged("modified_count") 204 | return self.__bulk_api_result.get("nModified") 205 | 206 | @property 207 | def deleted_count(self): 208 | """The number of documents deleted.""" 209 | self._raise_if_unacknowledged("deleted_count") 210 | return self.__bulk_api_result.get("nRemoved") 211 | 212 | @property 213 | def upserted_count(self): 214 | """The number of documents upserted.""" 215 | self._raise_if_unacknowledged("upserted_count") 216 | return self.__bulk_api_result.get("nUpserted") 217 | 218 | @property 219 | def upserted_ids(self): 220 | """A map of operation index to the _id of the upserted document.""" 221 | self._raise_if_unacknowledged("upserted_ids") 222 | if self.__bulk_api_result: 223 | return dict((upsert["index"], upsert["_id"]) 224 | for upsert in self.bulk_api_result["upserted"]) 225 | -------------------------------------------------------------------------------- /example/test_type_serial.py: -------------------------------------------------------------------------------- 1 | from idaapi import * 2 | from idc import * 3 | import idc 4 | import ctypes 5 | import struct 6 | import pydevd 7 | import pickle 8 | from pymongo import * 9 | from bson import * 10 | from IdaTypeStringParser import * 11 | 12 | ############################################################ 13 | # Several type-related functions aren't accessibly via IDAPython 14 | # so have to do things with ctypes 15 | idaname = "ida64" if idc.__EA64__ else "ida" 16 | if sys.platform == "win32": 17 | g_dll = ctypes.windll[idaname + ".wll"] 18 | elif sys.platform == "linux2": 19 | g_dll = ctypes.cdll["lib" + idaname + ".so"] 20 | elif sys.platform == "darwin": 21 | g_dll = ctypes.cdll["lib" + idaname + ".dylib"] 22 | 23 | wrapperTypeString = '\x0d\x01\x01' 24 | 25 | class qtype(Structure): 26 | _fields_ = [("ptr",ctypes.c_char_p),("cur_size",ctypes.c_int),("max_size",ctypes.c_int)] 27 | 28 | str1 = '''struct _IMAGE_DOS_HEADER 29 | { 30 | unsigned __int16 e_magic; 31 | unsigned __int16 e_cblp; 32 | unsigned __int16 e_cp; 33 | unsigned __int16 e_crlc; 34 | unsigned __int16 e_cparhdr; 35 | unsigned __int16 e_minalloc; 36 | aaa* e_maxalloc; 37 | unsigned __int16 e_ss; 38 | unsigned __int16 e_sp; 39 | unsigned __int16 e_csum; 40 | unsigned __int16 e_ip; 41 | unsigned __int16 e_cs; 42 | unsigned __int16 e_lfarlc; 43 | unsigned __int16 e_ovno; 44 | unsigned __int16 e_res[4]; 45 | unsigned __int16 e_oemid; 46 | unsigned __int16 e_oeminfo; 47 | unsigned __int16 e_res2[10]; 48 | int e_lfanew; 49 | }''' 50 | ############################################################ 51 | # Specifying function types for a few IDA SDK functions to keep the 52 | # pointer-to-pointer args clear. 53 | 54 | c_serialize_tinfo = g_dll.serialize_tinfo 55 | c_serialize_tinfo.argtypes = [ 56 | ctypes.POINTER(qtype), #qtype *type 57 | ctypes.POINTER(qtype), #qtype *fields 58 | ctypes.POINTER(qtype), #qtype *fldcmts 59 | ctypes.POINTER(ctypes.c_ulong), #const tinfo_t *tif 60 | ctypes.c_int #int sudt_flags 61 | ] 62 | 63 | c_new_til = g_dll.new_til 64 | c_new_til.argtyped = [ 65 | c_char_p, #const char *name 66 | c_char_p #const char *desc 67 | ] 68 | c_new_til.restype = c_void_p 69 | 70 | c_parse_decl2 = g_dll.parse_decl2 71 | parse_decl2.argtypes = [ 72 | c_void_p, #param til type library to use 73 | c_char_p, #param decl C declaration to parse 74 | ctypes.POINTER(qtype), #param[out] name declared name 75 | ctypes.POINTER(ctypes.c_ulong), #param[out] tif type info 76 | ctypes.c_int #param flags combination of \ref PT_ 77 | ] 78 | 79 | c_deserialize_tinfo = g_dll.deserialize_tinfo 80 | c_deserialize_tinfo.argtypes = [ 81 | ctypes.POINTER(ctypes.c_ulong), #tinfo_t *tif 82 | ctypes.c_void_p, #const til_t *til 83 | ctypes.POINTER(ctypes.c_char_p), #const type_t **ptype 84 | ctypes.POINTER(ctypes.c_char_p), #const p_list **pfields 85 | ctypes.POINTER(ctypes.c_char_p) #const p_list **pfldcmts 86 | ] 87 | 88 | c_print_tinfo = g_dll.print_tinfo 89 | c_print_tinfo.argtypes = [ 90 | ctypes.POINTER(qtype), #qstring *result 91 | ctypes.c_char_p, #const char *prefix 92 | ctypes.c_int, #int indent 93 | ctypes.c_int, #int cmtindent 94 | ctypes.c_int, #int flags 95 | ctypes.POINTER(ctypes.c_ulong), #const tinfo_t *tif 96 | ctypes.c_char_p, #const char *name 97 | ctypes.c_char_p #const char *cmt 98 | ] 99 | 100 | get_named_type = g_dll.get_named_type 101 | get_named_type.argtypes = [ 102 | ctypes.c_void_p, #const til_t *ti, 103 | ctypes.c_char_p, #const char *name, 104 | ctypes.c_int, #int ntf_flags, 105 | ctypes.POINTER(ctypes.POINTER(ctypes.c_ubyte)), #const type_t **type=NULL, 106 | ctypes.POINTER(ctypes.POINTER(ctypes.c_ubyte)), #const p_list **fields=NULL, 107 | ctypes.POINTER(ctypes.POINTER(ctypes.c_ubyte)), #const char **cmt=NULL, 108 | ctypes.POINTER(ctypes.POINTER(ctypes.c_ubyte)), #const p_list **fieldcmts=NULL, 109 | ctypes.POINTER(ctypes.c_ulong), #sclass_t *sclass=NULL, 110 | ctypes.POINTER(ctypes.c_ulong), #uint32 *value=NULL); 111 | ] 112 | 113 | print_type_to_one_line = g_dll.print_type_to_one_line 114 | print_type_to_one_line.argtypes = [ 115 | ctypes.c_char_p, #char *buf, 116 | ctypes.c_ulong, #size_t bufsize, 117 | ctypes.c_void_p, #const til_t *ti, 118 | ctypes.POINTER(ctypes.c_ubyte), #const type_t *pt, 119 | ctypes.c_char_p, #const char *name = NULL, 120 | ctypes.POINTER(ctypes.c_ubyte), #const char *cmt = NULL, 121 | ctypes.POINTER(ctypes.c_ubyte), #const p_list *field_names = NULL, 122 | ctypes.POINTER(ctypes.c_ubyte), #const p_list *field_cmts = NULL); 123 | ] 124 | 125 | ############################################################ 126 | get_numbered_type = g_dll.get_numbered_type 127 | get_numbered_type.argtypes = [ 128 | ctypes.c_void_p, #const til_t *ti, 129 | ctypes.c_int, #uint32 ordinal, 130 | ctypes.POINTER(ctypes.c_char_p), #const type_t **type=NULL, 131 | ctypes.POINTER(ctypes.c_char_p), #const p_list **fields=NULL, 132 | ctypes.POINTER(ctypes.c_char_p), #const char **cmt=NULL, 133 | ctypes.POINTER(ctypes.c_char_p), #const p_list **fieldcmts=NULL, 134 | ctypes.POINTER(ctypes.c_ulong), #sclass_t *sclass=NULL 135 | ] 136 | 137 | set_numbered_type = g_dll.set_numbered_type 138 | set_numbered_type.argtypes = [ 139 | ctypes.c_void_p, #til_t *ti, 140 | ctypes.c_int, #uint32 ordinal, 141 | ctypes.c_int, #int ntf_flags, 142 | ctypes.c_char_p, #const char *name, 143 | ctypes.c_char_p, #const type_t *type, 144 | ctypes.c_char_p, #const p_list *fields=NULL, 145 | ctypes.c_char_p, #const char *cmt=NULL, 146 | ctypes.c_char_p, #const p_list *fldcmts=NULL, 147 | ctypes.POINTER(ctypes.c_ulong), #const sclass_t *sclass=NULL 148 | ] 149 | 150 | my_til = ctypes.c_void_p.in_dll(g_dll, 'idati') 151 | my_ti = idaapi.cvar.idati 152 | 153 | 154 | 155 | 156 | ITSP = IdaTypeStringParser() 157 | t = ITSP.ImportLocalType(51) 158 | tif = tinfo_t() 159 | #print t.fieldcmts 160 | #print t.cmt 161 | print tif.deserialize(idaapi.cvar.idati,t.TypeString,t.TypeFields,t.cmt) 162 | #print tif 163 | name = "" 164 | #print tif.get_type_name() 165 | #print name 166 | print tif._print(None,idaapi.PRTYPE_MULTI) 167 | exit() 168 | tif2 = ctypes.c_ulong() 169 | tif3 = ctypes.c_ulong() 170 | print "-------------------------------------------------------------------------------------------------------------------------------------------------" 171 | print c_deserialize_tinfo(byref(tif2),my_til,ctypes.c_char_p(t.TypeString),ctypes.c_char_p(t.TypeFields),ctypes.c_char_p(t.fieldcmts)) 172 | ret = qtype() 173 | ret.cur_size = 0 174 | ret.max_size = 0 175 | # print c_print_tinfo(byref(ret),ctypes.c_char_p(),0,0,idaapi.PRTYPE_MULTI,byref(tif2),ctypes.c_char_p(),ctypes.c_char_p()) 176 | # print ret.ptr 177 | r = idc_print_type(t.TypeString,t.TypeFields,t.name,PRTYPE_MULTI|PRTYPE_TYPE).strip() 178 | # r = ret.ptr.split("\n",1) 179 | # ret.ptr = r[0] + " " + t.name + "\n" + r[1] 180 | til = new_til("temp_til","temp") 181 | print "New till success" 182 | if len(r) != 0 and r[-1] != ';': 183 | r = r + ';' 184 | print r 185 | r = idc_parse_decl(til,r,0) 186 | if r is not None: 187 | name, type_str, fields_str = r 188 | print name 189 | print type_str.encode("hex") 190 | print t.TypeString.encode("hex") 191 | print fields_str.encode("hex") 192 | print t.TypeFields.encode("hex") 193 | # type_string = qtype() 194 | # type_fields = qtype() 195 | # type_fieldcmts = qtype() 196 | # print c_serialize_tinfo(byref(type_string),byref(type_fields),byref(type_fieldcmts),byref(tif3),SUDT_FAST|SUDT_TRUNC) 197 | # print type_string.ptr.encode("hex") 198 | # print t.TypeString.encode("hex") 199 | # print type_fields.ptr.encode("hex") 200 | # print t.TypeFields.encode("hex") 201 | 202 | r = idc_parse_decl(til,str1,0) 203 | if r is not None: 204 | name, type_str, fields_str = r 205 | print name 206 | print type_str.encode("hex") 207 | print t.TypeString.encode("hex") 208 | print fields_str.encode("hex") 209 | print t.TypeFields.encode("hex") 210 | 211 | free_til(til) 212 | -------------------------------------------------------------------------------- /pymongo/command_cursor.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """CommandCursor class to iterate over command results.""" 16 | 17 | import datetime 18 | 19 | from collections import deque 20 | 21 | from bson.py3compat import integer_types 22 | from pymongo import helpers 23 | from pymongo.errors import AutoReconnect, NotMasterError, OperationFailure 24 | from pymongo.message import _CursorAddress, _GetMore, _convert_exception 25 | 26 | 27 | class CommandCursor(object): 28 | """A cursor / iterator over command cursors. 29 | """ 30 | 31 | def __init__(self, collection, cursor_info, address, retrieved=0): 32 | """Create a new command cursor. 33 | """ 34 | self.__collection = collection 35 | self.__id = cursor_info['id'] 36 | self.__address = address 37 | self.__data = deque(cursor_info['firstBatch']) 38 | self.__retrieved = retrieved 39 | self.__batch_size = 0 40 | self.__killed = (self.__id == 0) 41 | 42 | if "ns" in cursor_info: 43 | self.__ns = cursor_info["ns"] 44 | else: 45 | self.__ns = collection.full_name 46 | 47 | def __del__(self): 48 | if self.__id and not self.__killed: 49 | self.__die() 50 | 51 | def __die(self): 52 | """Closes this cursor. 53 | """ 54 | if self.__id and not self.__killed: 55 | self.__collection.database.client.close_cursor( 56 | self.__id, _CursorAddress(self.__address, self.__ns)) 57 | self.__killed = True 58 | 59 | def close(self): 60 | """Explicitly close / kill this cursor. Required for PyPy, Jython and 61 | other Python implementations that don't use reference counting 62 | garbage collection. 63 | """ 64 | self.__die() 65 | 66 | def batch_size(self, batch_size): 67 | """Limits the number of documents returned in one batch. Each batch 68 | requires a round trip to the server. It can be adjusted to optimize 69 | performance and limit data transfer. 70 | 71 | .. note:: batch_size can not override MongoDB's internal limits on the 72 | amount of data it will return to the client in a single batch (i.e 73 | if you set batch size to 1,000,000,000, MongoDB will currently only 74 | return 4-16MB of results per batch). 75 | 76 | Raises :exc:`TypeError` if `batch_size` is not an integer. 77 | Raises :exc:`ValueError` if `batch_size` is less than ``0``. 78 | 79 | :Parameters: 80 | - `batch_size`: The size of each batch of results requested. 81 | """ 82 | if not isinstance(batch_size, integer_types): 83 | raise TypeError("batch_size must be an integer") 84 | if batch_size < 0: 85 | raise ValueError("batch_size must be >= 0") 86 | 87 | self.__batch_size = batch_size == 1 and 2 or batch_size 88 | return self 89 | 90 | def __send_message(self, operation): 91 | """Send a getmore message and handle the response. 92 | """ 93 | client = self.__collection.database.client 94 | listeners = client._event_listeners 95 | publish = listeners.enabled_for_commands 96 | try: 97 | response = client._send_message_with_response( 98 | operation, address=self.__address) 99 | except AutoReconnect: 100 | # Don't try to send kill cursors on another socket 101 | # or to another server. It can cause a _pinValue 102 | # assertion on some server releases if we get here 103 | # due to a socket timeout. 104 | self.__killed = True 105 | raise 106 | 107 | cmd_duration = response.duration 108 | rqst_id = response.request_id 109 | if publish: 110 | start = datetime.datetime.now() 111 | try: 112 | doc = helpers._unpack_response(response.data, 113 | self.__id, 114 | self.__collection.codec_options) 115 | except OperationFailure as exc: 116 | self.__killed = True 117 | 118 | if publish: 119 | duration = (datetime.datetime.now() - start) + cmd_duration 120 | listeners.publish_command_failure( 121 | duration, exc.details, "getMore", rqst_id, self.__address) 122 | 123 | raise 124 | except NotMasterError as exc: 125 | # Don't send kill cursors to another server after a "not master" 126 | # error. It's completely pointless. 127 | self.__killed = True 128 | 129 | if publish: 130 | duration = (datetime.datetime.now() - start) + cmd_duration 131 | listeners.publish_command_failure( 132 | duration, exc.details, "getMore", rqst_id, self.__address) 133 | 134 | client._reset_server_and_request_check(self.address) 135 | raise 136 | except Exception as exc: 137 | if publish: 138 | duration = (datetime.datetime.now() - start) + cmd_duration 139 | listeners.publish_command_failure( 140 | duration, _convert_exception(exc), "getMore", rqst_id, 141 | self.__address) 142 | raise 143 | 144 | if publish: 145 | duration = (datetime.datetime.now() - start) + cmd_duration 146 | # Must publish in getMore command response format. 147 | res = {"cursor": {"id": doc["cursor_id"], 148 | "ns": self.__collection.full_name, 149 | "nextBatch": doc["data"]}, 150 | "ok": 1} 151 | listeners.publish_command_success( 152 | duration, res, "getMore", rqst_id, self.__address) 153 | 154 | self.__id = doc["cursor_id"] 155 | if self.__id == 0: 156 | self.__killed = True 157 | 158 | self.__retrieved += doc["number_returned"] 159 | self.__data = deque(doc["data"]) 160 | 161 | def _refresh(self): 162 | """Refreshes the cursor with more data from the server. 163 | 164 | Returns the length of self.__data after refresh. Will exit early if 165 | self.__data is already non-empty. Raises OperationFailure when the 166 | cursor cannot be refreshed due to an error on the query. 167 | """ 168 | if len(self.__data) or self.__killed: 169 | return len(self.__data) 170 | 171 | if self.__id: # Get More 172 | self.__send_message( 173 | _GetMore(self.__ns, self.__batch_size, self.__id)) 174 | 175 | else: # Cursor id is zero nothing else to return 176 | self.__killed = True 177 | 178 | return len(self.__data) 179 | 180 | @property 181 | def alive(self): 182 | """Does this cursor have the potential to return more data? 183 | 184 | Even if :attr:`alive` is ``True``, :meth:`next` can raise 185 | :exc:`StopIteration`. Best to use a for loop:: 186 | 187 | for doc in collection.aggregate(pipeline): 188 | print(doc) 189 | 190 | .. note:: :attr:`alive` can be True while iterating a cursor from 191 | a failed server. In this case :attr:`alive` will return False after 192 | :meth:`next` fails to retrieve the next batch of results from the 193 | server. 194 | """ 195 | return bool(len(self.__data) or (not self.__killed)) 196 | 197 | @property 198 | def cursor_id(self): 199 | """Returns the id of the cursor.""" 200 | return self.__id 201 | 202 | @property 203 | def address(self): 204 | """The (host, port) of the server used, or None. 205 | 206 | .. versionadded:: 3.0 207 | """ 208 | return self.__address 209 | 210 | def __iter__(self): 211 | return self 212 | 213 | def next(self): 214 | """Advance the cursor.""" 215 | if len(self.__data) or self._refresh(): 216 | coll = self.__collection 217 | return coll.database._fix_outgoing(self.__data.popleft(), coll) 218 | else: 219 | raise StopIteration 220 | 221 | __next__ = next 222 | 223 | def __enter__(self): 224 | return self 225 | 226 | def __exit__(self, exc_type, exc_val, exc_tb): 227 | self.__die() 228 | -------------------------------------------------------------------------------- /example/log.txt: -------------------------------------------------------------------------------- 1 | 1 2 | AfxSig 3 | type: 2d 9a 01 80 40 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 9f ff ff fe 6d 41 43 43 41 41 41 41 41 41 41 44 41 41 41 41 41 41 45 41 41 41 41 41 42 41 41 9f ff ff ff 73 4f 41 41 41 42 41 41 41 42 9f ff ff ff 7f 4b 41 41 41 9f ff ff ff 47 6f 4b 41 41 41 9f ff ff ff 70 41 9f ff ff ff 4e 41 57 41 41 4b 9f ff ff ff 6c 9f ff ff ff 7c 45 9f ff ff ff 7d 9f ff ff ff 7f 4d 9f ff ff ff 70 68 9f ff ff ff 6e 9f ff ff ff 62 71 9f ff ff ff 76 4 | '-\x9a\x01\x80@AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\x9f\xff\xff\xfemACCAAAAAAADAAAAAAEAAAAABAA\x9f\xff\xff\xffsOAAABAAAB\x9f\xff\xff\xff\x7fKAAA\x9f\xff\xff\xffGoKAAA\x9f\xff\xff\xffpA\x9f\xff\xff\xffNAWAAK\x9f\xff\xff\xffl\x9f\xff\xff\xff|E\x9f\xff\xff\xff}\x9f\xff\xff\xff\x7fM\x9f\xff\xff\xffph\x9f\xff\xff\xffn\x9f\xff\xff\xffbq\x9f\xff\xff\xffv' 5 | fields: 0b 41 66 78 53 69 67 5f 65 6e 64 0d 41 66 78 53 69 67 5f 62 5f 44 5f 76 0d 41 66 78 53 69 67 5f 62 5f 62 5f 76 0d 41 66 78 53 69 67 5f 62 5f 75 5f 76 0d 41 66 78 53 69 67 5f 62 5f 68 5f 76 0e 41 66 78 53 69 67 5f 62 5f 57 5f 75 75 1a 41 66 78 53 69 67 5f 62 5f 57 5f 43 4f 50 59 44 41 54 41 53 54 52 55 43 54 14 41 66 78 53 69 67 5f 62 5f 76 5f 48 45 4c 50 49 4e 46 4f 10 41 66 78 53 69 67 5f 43 54 4c 43 4f 4c 4f 52 18 41 66 78 53 69 67 5f 43 54 4c 43 4f 4c 4f 52 5f 52 45 46 4c 45 43 54 0f 41 66 78 53 69 67 5f 69 5f 75 5f 57 5f 75 0e 41 66 78 53 69 67 5f 69 5f 75 75 5f 76 0e 41 66 78 53 69 67 5f 69 5f 57 5f 75 75 0d 41 66 78 53 69 67 5f 69 5f 76 5f 73 0d 41 66 78 53 69 67 5f 6c 5f 77 5f 6c 0e 41 66 78 53 69 67 5f 6c 5f 75 75 5f 4d 0d 41 66 78 53 69 67 5f 76 5f 62 5f 68 0d 41 66 78 53 69 67 5f 76 5f 68 5f 76 0d 41 66 78 53 69 67 5f 76 5f 68 5f 68 0d 41 66 78 53 69 67 5f 76 5f 76 5f 76 0d 41 66 78 53 69 67 5f 76 5f 75 5f 76 0d 41 66 78 53 69 67 5f 76 5f 75 5f 75 0e 41 66 78 53 69 67 5f 76 5f 75 75 5f 76 0e 41 66 78 53 69 67 5f 76 5f 76 5f 69 69 0e 41 66 78 53 69 67 5f 76 5f 75 5f 75 75 0e 41 66 78 53 69 67 5f 76 5f 75 5f 69 69 0d 41 66 78 53 69 67 5f 76 5f 75 5f 57 0d 41 66 78 53 69 67 5f 69 5f 75 5f 76 0d 41 66 78 53 69 67 5f 75 5f 75 5f 76 0d 41 66 78 53 69 67 5f 62 5f 76 5f 76 0d 41 66 78 53 69 67 5f 76 5f 77 5f 6c 13 41 66 78 53 69 67 5f 4d 44 49 41 43 54 49 56 41 54 45 0d 41 66 78 53 69 67 5f 76 5f 44 5f 76 0d 41 66 78 53 69 67 5f 76 5f 4d 5f 76 0e 41 66 78 53 69 67 5f 76 5f 4d 5f 75 62 0d 41 66 78 53 69 67 5f 76 5f 57 5f 76 0d 41 66 78 53 69 67 5f 76 5f 76 5f 57 0e 41 66 78 53 69 67 5f 76 5f 57 5f 75 75 0d 41 66 78 53 69 67 5f 76 5f 57 5f 70 0d 41 66 78 53 69 67 5f 76 5f 57 5f 68 0d 41 66 78 53 69 67 5f 43 5f 76 5f 76 10 41 66 78 53 69 67 5f 41 43 54 49 56 41 54 45 0e 41 66 78 53 69 67 5f 53 43 52 4f 4c 4c 16 41 66 78 53 69 67 5f 53 43 52 4f 4c 4c 5f 52 45 46 4c 45 43 54 0d 41 66 78 53 69 67 5f 76 5f 76 5f 73 0e 41 66 78 53 69 67 5f 76 5f 75 5f 63 73 11 41 66 78 53 69 67 5f 4f 57 4e 45 52 44 52 41 57 0d 41 66 78 53 69 67 5f 69 5f 69 5f 73 0d 41 66 78 53 69 67 5f 75 5f 76 5f 70 0d 41 66 78 53 69 67 5f 75 5f 76 5f 76 1c 41 66 78 53 69 67 5f 76 5f 62 5f 4e 43 43 41 4c 43 53 49 5a 45 50 41 52 41 4d 53 15 41 66 78 53 69 67 5f 76 5f 76 5f 57 49 4e 44 4f 57 50 4f 53 0e 41 66 78 53 69 67 5f 76 5f 75 75 5f 4d 0d 41 66 78 53 69 67 5f 76 5f 75 5f 70 0e 41 66 78 53 69 67 5f 53 49 5a 49 4e 47 12 41 66 78 53 69 67 5f 4d 4f 55 53 45 57 48 45 45 4c 13 41 66 78 53 69 67 5f 4d 4f 55 53 45 48 57 48 45 45 4c 0c 41 66 78 53 69 67 43 6d 64 5f 76 0c 41 66 78 53 69 67 43 6d 64 5f 62 10 41 66 78 53 69 67 43 6d 64 5f 52 41 4e 47 45 0d 41 66 78 53 69 67 43 6d 64 5f 45 58 0f 41 66 78 53 69 67 4e 6f 74 69 66 79 5f 76 0f 41 66 78 53 69 67 4e 6f 74 69 66 79 5f 62 13 41 66 78 53 69 67 4e 6f 74 69 66 79 5f 52 41 4e 47 45 10 41 66 78 53 69 67 4e 6f 74 69 66 79 5f 45 58 0c 41 66 78 53 69 67 43 6d 64 55 49 12 41 66 78 53 69 67 43 6d 64 55 49 5f 52 41 4e 47 45 0f 41 66 78 53 69 67 43 6d 64 5f 76 5f 70 76 0f 41 66 78 53 69 67 43 6d 64 5f 62 5f 70 76 09 41 66 78 53 69 67 5f 6c 0b 41 66 78 53 69 67 5f 6c 5f 70 0d 41 66 78 53 69 67 5f 75 5f 57 5f 75 0d 41 66 78 53 69 67 5f 76 5f 75 5f 4d 0d 41 66 78 53 69 67 5f 75 5f 75 5f 4d 1d 41 66 78 53 69 67 5f 75 5f 76 5f 4d 45 4e 55 47 45 54 4f 42 4a 45 43 54 49 4e 46 4f 0d 41 66 78 53 69 67 5f 76 5f 4d 5f 75 19 41 66 78 53 69 67 5f 76 5f 75 5f 4c 50 4d 44 49 4e 45 58 54 4d 45 4e 55 12 41 66 78 53 69 67 5f 41 50 50 43 4f 4d 4d 41 4e 44 10 41 66 78 53 69 67 5f 52 41 57 49 4e 50 55 54 0d 41 66 78 53 69 67 5f 75 5f 75 5f 75 15 41 66 78 53 69 67 5f 4d 4f 55 53 45 5f 58 42 55 54 54 4f 4e 17 41 66 78 53 69 67 5f 4d 4f 55 53 45 5f 4e 43 58 42 55 54 54 4f 4e 17 41 66 78 53 69 67 5f 49 4e 50 55 54 4c 41 4e 47 43 48 41 4e 47 45 0f 41 66 78 53 69 67 5f 76 5f 75 5f 68 6b 6c 19 41 66 78 53 69 67 5f 49 4e 50 55 54 44 45 56 49 43 45 43 48 41 4e 47 45 0a 41 66 78 53 69 67 5f 62 44 0a 41 66 78 53 69 67 5f 62 62 0c 41 66 78 53 69 67 5f 62 57 77 77 0c 41 66 78 53 69 67 5f 68 44 57 77 0b 41 66 78 53 69 67 5f 68 44 77 0c 41 66 78 53 69 67 5f 69 77 57 77 0b 41 66 78 53 69 67 5f 69 77 77 0c 41 66 78 53 69 67 5f 69 57 77 77 0a 41 66 78 53 69 67 5f 69 73 0b 41 66 78 53 69 67 5f 6c 77 6c 0c 41 66 78 53 69 67 5f 6c 77 77 4d 0a 41 66 78 53 69 67 5f 76 76 0a 41 66 78 53 69 67 5f 76 77 0b 41 66 78 53 69 67 5f 76 77 77 0c 41 66 78 53 69 67 5f 76 77 77 32 0c 41 66 78 53 69 67 5f 76 76 69 69 0c 41 66 78 53 69 67 5f 76 77 77 77 0c 41 66 78 53 69 67 5f 76 77 69 69 0b 41 66 78 53 69 67 5f 76 77 6c 0c 41 66 78 53 69 67 5f 76 62 57 57 0a 41 66 78 53 69 67 5f 76 44 0a 41 66 78 53 69 67 5f 76 4d 0c 41 66 78 53 69 67 5f 76 4d 77 62 0a 41 66 78 53 69 67 5f 76 57 0c 41 66 78 53 69 67 5f 76 57 77 77 0b 41 66 78 53 69 67 5f 76 57 70 0b 41 66 78 53 69 67 5f 76 57 68 0b 41 66 78 53 69 67 5f 76 77 57 0c 41 66 78 53 69 67 5f 76 77 57 62 0c 41 66 78 53 69 67 5f 76 77 77 57 0c 41 66 78 53 69 67 5f 76 77 77 78 0a 41 66 78 53 69 67 5f 76 73 0e 41 66 78 53 69 67 5f 76 4f 57 4e 45 52 0b 41 66 78 53 69 67 5f 69 69 73 0a 41 66 78 53 69 67 5f 77 70 0a 41 66 78 53 69 67 5f 77 76 0c 41 66 78 53 69 67 5f 76 50 4f 53 0d 41 66 78 53 69 67 5f 76 43 41 4c 43 10 41 66 78 53 69 67 5f 76 4e 4d 48 44 52 70 6c 10 41 66 78 53 69 67 5f 62 4e 4d 48 44 52 70 6c 11 41 66 78 53 69 67 5f 76 77 4e 4d 48 44 52 70 6c 11 41 66 78 53 69 67 5f 62 77 4e 4d 48 44 52 70 6c 11 41 66 78 53 69 67 5f 62 48 45 4c 50 49 4e 46 4f 10 41 66 78 53 69 67 5f 76 77 53 49 5a 49 4e 47 0d 41 66 78 53 69 67 5f 63 6d 64 75 69 0e 41 66 78 53 69 67 5f 63 6d 64 75 69 77 0b 41 66 78 53 69 67 5f 76 70 76 0b 41 66 78 53 69 67 5f 62 70 76 0c 41 66 78 53 69 67 5f 76 77 77 68 0b 41 66 78 53 69 67 5f 76 77 70 0a 41 66 78 53 69 67 5f 62 77 0a 41 66 78 53 69 67 5f 62 68 0a 41 66 78 53 69 67 5f 69 77 0a 41 66 78 53 69 67 5f 77 77 0a 41 66 78 53 69 67 5f 62 76 0a 41 66 78 53 69 67 5f 68 76 0a 41 66 78 53 69 67 5f 76 62 0b 41 66 78 53 69 67 5f 76 62 68 0b 41 66 78 53 69 67 5f 76 62 77 0b 41 66 78 53 69 67 5f 76 68 68 0a 41 66 78 53 69 67 5f 76 68 0c 41 66 78 53 69 67 5f 76 69 53 53 0b 41 66 78 53 69 67 5f 62 77 6c 10 41 66 78 53 69 67 5f 76 77 4d 4f 56 49 4e 47 0b 41 66 78 53 69 67 5f 76 57 32 0d 41 66 78 53 69 67 5f 62 57 43 44 53 0c 41 66 78 53 69 67 5f 62 77 73 70 0b 41 66 78 53 69 67 5f 76 77 73 6 | '\x0bAfxSig_end\rAfxSig_b_D_v\rAfxSig_b_b_v\rAfxSig_b_u_v\rAfxSig_b_h_v\x0eAfxSig_b_W_uu\x1aAfxSig_b_W_COPYDATASTRUCT\x14AfxSig_b_v_HELPINFO\x10AfxSig_CTLCOLOR\x18AfxSig_CTLCOLOR_REFLECT\x0fAfxSig_i_u_W_u\x0eAfxSig_i_uu_v\x0eAfxSig_i_W_uu\rAfxSig_i_v_s\rAfxSig_l_w_l\x0eAfxSig_l_uu_M\rAfxSig_v_b_h\rAfxSig_v_h_v\rAfxSig_v_h_h\rAfxSig_v_v_v\rAfxSig_v_u_v\rAfxSig_v_u_u\x0eAfxSig_v_uu_v\x0eAfxSig_v_v_ii\x0eAfxSig_v_u_uu\x0eAfxSig_v_u_ii\rAfxSig_v_u_W\rAfxSig_i_u_v\rAfxSig_u_u_v\rAfxSig_b_v_v\rAfxSig_v_w_l\x13AfxSig_MDIACTIVATE\rAfxSig_v_D_v\rAfxSig_v_M_v\x0eAfxSig_v_M_ub\rAfxSig_v_W_v\rAfxSig_v_v_W\x0eAfxSig_v_W_uu\rAfxSig_v_W_p\rAfxSig_v_W_h\rAfxSig_C_v_v\x10AfxSig_ACTIVATE\x0eAfxSig_SCROLL\x16AfxSig_SCROLL_REFLECT\rAfxSig_v_v_s\x0eAfxSig_v_u_cs\x11AfxSig_OWNERDRAW\rAfxSig_i_i_s\rAfxSig_u_v_p\rAfxSig_u_v_v\x1cAfxSig_v_b_NCCALCSIZEPARAMS\x15AfxSig_v_v_WINDOWPOS\x0eAfxSig_v_uu_M\rAfxSig_v_u_p\x0eAfxSig_SIZING\x12AfxSig_MOUSEWHEEL\x13AfxSig_MOUSEHWHEEL\x0cAfxSigCmd_v\x0cAfxSigCmd_b\x10AfxSigCmd_RANGE\rAfxSigCmd_EX\x0fAfxSigNotify_v\x0fAfxSigNotify_b\x13AfxSigNotify_RANGE\x10AfxSigNotify_EX\x0cAfxSigCmdUI\x12AfxSigCmdUI_RANGE\x0fAfxSigCmd_v_pv\x0fAfxSigCmd_b_pv\tAfxSig_l\x0bAfxSig_l_p\rAfxSig_u_W_u\rAfxSig_v_u_M\rAfxSig_u_u_M\x1dAfxSig_u_v_MENUGETOBJECTINFO\rAfxSig_v_M_u\x19AfxSig_v_u_LPMDINEXTMENU\x12AfxSig_APPCOMMAND\x10AfxSig_RAWINPUT\rAfxSig_u_u_u\x15AfxSig_MOUSE_XBUTTON\x17AfxSig_MOUSE_NCXBUTTON\x17AfxSig_INPUTLANGCHANGE\x0fAfxSig_v_u_hkl\x19AfxSig_INPUTDEVICECHANGE\nAfxSig_bD\nAfxSig_bb\x0cAfxSig_bWww\x0cAfxSig_hDWw\x0bAfxSig_hDw\x0cAfxSig_iwWw\x0bAfxSig_iww\x0cAfxSig_iWww\nAfxSig_is\x0bAfxSig_lwl\x0cAfxSig_lwwM\nAfxSig_vv\nAfxSig_vw\x0bAfxSig_vww\x0cAfxSig_vww2\x0cAfxSig_vvii\x0cAfxSig_vwww\x0cAfxSig_vwii\x0bAfxSig_vwl\x0cAfxSig_vbWW\nAfxSig_vD\nAfxSig_vM\x0cAfxSig_vMwb\nAfxSig_vW\x0cAfxSig_vWww\x0bAfxSig_vWp\x0bAfxSig_vWh\x0bAfxSig_vwW\x0cAfxSig_vwWb\x0cAfxSig_vwwW\x0cAfxSig_vwwx\nAfxSig_vs\x0eAfxSig_vOWNER\x0bAfxSig_iis\nAfxSig_wp\nAfxSig_wv\x0cAfxSig_vPOS\rAfxSig_vCALC\x10AfxSig_vNMHDRpl\x10AfxSig_bNMHDRpl\x11AfxSig_vwNMHDRpl\x11AfxSig_bwNMHDRpl\x11AfxSig_bHELPINFO\x10AfxSig_vwSIZING\rAfxSig_cmdui\x0eAfxSig_cmduiw\x0bAfxSig_vpv\x0bAfxSig_bpv\x0cAfxSig_vwwh\x0bAfxSig_vwp\nAfxSig_bw\nAfxSig_bh\nAfxSig_iw\nAfxSig_ww\nAfxSig_bv\nAfxSig_hv\nAfxSig_vb\x0bAfxSig_vbh\x0bAfxSig_vbw\x0bAfxSig_vhh\nAfxSig_vh\x0cAfxSig_viSS\x0bAfxSig_bwl\x10AfxSig_vwMOVING\x0bAfxSig_vW2\rAfxSig_bWCDS\x0cAfxSig_bwsp\x0bAfxSig_vws' 7 | 8 | 9 | -------------------------------------------------------------------------------- /example/sqlite_test.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | #from IdaTypeStringParser import LocalType 3 | import pickle 4 | 5 | import sys, collections 6 | 7 | # sys.path.append("C:\work\GitReps\ida_type_storage\\") 8 | # from IdaTypeStorage import Storage_sqlite 9 | 10 | 11 | 12 | class Storage_sqlite(object): 13 | 14 | actual_cols = ['name', 'TypeString', 'TypeFields', 'cmt', 'fieldcmts', 'sclass', 'parsedList', 'depends', 15 | 'depends_ordinals', "flags"] 16 | 17 | def __init__(self,db_name,project_name = ""): 18 | self.db_name = db_name 19 | self.project_name = project_name 20 | if self.project_name != "" and not self.isTableExist(self.project_name): 21 | self.request(r"CREATE TABLE '%s' (name text, TypeString text, TypeFields text, cmt text, fieldcmts text, sclass text, parsedList text, depends text, depends_ordinals text)"%(self.project_name)) 22 | 23 | def isTableExist(self,name): 24 | return True if len(self.request(r"SELECT name FROM sqlite_master WHERE type='table' AND name=?;",(name,))) == 1 else False 25 | 26 | def connect(self,project_name): 27 | self.project_name = project_name 28 | if self.project_name != "" and not self.isTableExist(self.project_name): 29 | self.request(r"CREATE TABLE '%s' (name text, TypeString text, TypeFields text, cmt text, fieldcmts text, sclass text, parsedList text, depends text, depends_ordinals text)"%(self.project_name)) 30 | 31 | def request(self,req_str,vals = ()): 32 | if type(vals) != tuple: 33 | vals = (vals,) 34 | conn = sqlite3.connect(self.db_name) 35 | c = conn.cursor() 36 | if len(vals) == 0: 37 | res = c.execute(req_str) 38 | else: 39 | res = c.execute(req_str,vals) 40 | res = res.fetchall() 41 | conn.commit() 42 | conn.close() 43 | return res 44 | 45 | def modify_ret(self,res): 46 | if len(res) > 0 and len(res[0]) == 1: 47 | ret = [] 48 | for el in res: 49 | ret.append(el[0].encode("ascii")) 50 | return ret 51 | elif len(res) == 1 and len(res[0]) > 1: 52 | ret = [] 53 | for el in res[0]: 54 | ret.append(el.encode("ascii")) 55 | return ret 56 | return res 57 | 58 | def GetAllProjects(self): 59 | return self.modify_ret(self.request(r"SELECT name FROM sqlite_master WHERE type='table'")) 60 | 61 | def GetAllNames(self): 62 | return self.modify_ret(self.request(r"SELECT name FROM %s"%self.project_name)) 63 | 64 | 65 | def deleteProject(self,name = ""): 66 | if name == "": 67 | name = self.project_name 68 | self.request(r"drop table '%s'"%(name)) 69 | self.project_name = "" 70 | 71 | def close_storage(self): 72 | pass 73 | 74 | def to_dict(self,res): 75 | ser_dic = collections.OrderedDict() 76 | ser_dic['name'] = res[0] 77 | ser_dic['TypeString'] = res[1] 78 | ser_dic['TypeFields'] = res[2] 79 | ser_dic['cmt'] = res[3] 80 | ser_dic['fieldcmts'] = res[4] 81 | ser_dic['sclass'] = pickle.loads(res[5].encode("ascii").decode("base64")) 82 | ser_dic['parsedList'] = pickle.loads(res[6].encode("ascii").decode("base64")) 83 | ser_dic['depends'] = pickle.loads(res[7].encode("ascii").decode("base64")) 84 | ser_dic['depends_ordinals'] = pickle.loads(res[8].encode("ascii").decode("base64")) 85 | return ser_dic 86 | 87 | def putToStorage(self,t): 88 | ser_dic = t.to_dict() 89 | try: 90 | self.request(r"INSERT INTO '%s' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"%(self.project_name),(ser_dic['name'],ser_dic['TypeString'],ser_dic['TypeFields'],ser_dic['cmt'],ser_dic['fieldcmts'],pickle.dumps(ser_dic["sclass"]).encode("base64"),pickle.dumps(ser_dic["parsedList"]).encode("base64"),pickle.dumps(ser_dic["depends"]).encode("base64"),pickle.dumps(ser_dic["depends_ordinals"]).encode("base64"))) 91 | except: 92 | Warning("Exception on sqlite putToStorage") 93 | 94 | def getFromStorage(self,name): 95 | res = [] 96 | try: 97 | res = self.request(r"SELECT * FROM '%s' WHERE name=?"%(self.project_name),(name,)) 98 | if len(res) == 0: 99 | return None 100 | elif len(res) > 1: 101 | raise NameError("getFromStorage: Type duplication or error. Count = %d" % len(res)) 102 | else: 103 | return LocalType().from_dict(self.to_dict(res[0])) 104 | except: 105 | Warning("Exception on sqlite getFromStorage") 106 | return None 107 | 108 | def isExist(self,name): 109 | res = self.request(r"SELECT * FROM '%s' WHERE name=?"%(self.project_name), (name,)) 110 | if len(res) == 0: 111 | return False 112 | elif len(res) == 1: 113 | return True 114 | else: 115 | raise NameError("isExist: Type duplication or error. Count = %d" % (len(res))) 116 | 117 | 118 | def updateType(self,name,t): 119 | ser_dic = t.to_dict() 120 | try: 121 | self.request(r"UPDATE '%s' SET name = ?, TypeString = ?, TypeFields = ?, cmt = ?, fieldcmts = ?, sclass = ?, parsedList = ?, depends = ?, depends_ordinals = ? WHERE name = ?"%(self.project_name), (ser_dic['name'], ser_dic['TypeString'], ser_dic['TypeFields'], ser_dic['cmt'], 122 | ser_dic['fieldcmts'], pickle.dumps(ser_dic["sclass"]).encode("base64"), 123 | pickle.dumps(ser_dic["parsedList"]).encode("base64"), pickle.dumps(ser_dic["depends"]).encode("base64"), 124 | pickle.dumps(ser_dic["depends_ordinals"]).encode("base64"),name)) 125 | return True 126 | except: 127 | Warning("Exception on sqlite updateType") 128 | return False 129 | 130 | 131 | def isActual(self): 132 | if self.project_name != "": 133 | curr_cols = [] 134 | for inf in self.modify_ret(self.request(r"PRAGMA table_info(%s)" % self.project_name)): 135 | curr_cols.append(inf[1].encode("ascii")) 136 | print curr_cols 137 | return curr_cols == self.actual_cols 138 | return True 139 | 140 | def update_table(self): 141 | self.request(r"ALTER TABLE %s ADD COLUMN flags INTEGER DEFAULT 0;"%self.project_name) 142 | ret = self.request(r"SELECT name,TypeString FROM %s"%self.project_name) 143 | for name, ts in ret: 144 | name = name.encode("ascii") 145 | print name, ts.decode("base64").encode("HEX") 146 | 147 | 148 | 149 | 150 | # def modify_ret(res): 151 | # if len(res) > 0 and len(res[0]) == 1: 152 | # ret = [] 153 | # for el in res: 154 | # ret.append(el[0].encode("ascii")) 155 | # return ret 156 | # return res 157 | # 158 | # conn = sqlite3.connect('example.db') 159 | # 160 | # c = conn.cursor() 161 | # #c.execute(r"CREATE TABLE 'stocks2' (date text, trans text, symbol text, qty real, price real)") 162 | # res = conn.execute("SELECT name FROM sqlite_master WHERE type='table';") 163 | # print modify_ret(res.fetchall()) 164 | # c.execute(r"INSERT INTO stocks2 VALUES ('2006-01-06','BUY',?,100,35.14)",('$NORMAL_STATE$29162',)) 165 | # conn.commit() 166 | # print c.execute(r"select date from %s WHERE symbol = ?"%('stocks2'),("$NORMAL_STATE$29162",)).fetchall() 167 | # print c.execute(r"select * from %s "%('stocks2')).fetchall() 168 | # 169 | # 170 | # exit(0) 171 | # res = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name=?;",('stocks',)) 172 | # print modify_ret(res.fetchall()) 173 | # 174 | # # Insert a row of data 175 | # #c.execute("INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)") 176 | # c.execute("INSERT INTO stocks VALUES ('2006-01-06','BUY','aaa3',100,35.14)") 177 | # 178 | # # Save (commit) the changes 179 | # conn.commit() 180 | # print c.execute("select date from %s WHERE symbol = ?"%('stocks'),("aaa",)).fetchall() 181 | # print modify_ret(c.execute("select date from %s WHERE symbol = ?"%('stocks'),("aaa",)).fetchall()) 182 | # print c.execute("select * from %s "%('stocks')).fetchall() 183 | # print c.execute("delete from %s WHERE symbol ='aaa34'"%('stocks')).fetchall() 184 | # conn.commit() 185 | # print c.execute("select date from %s "%('stocks')).fetchall() 186 | # #t = raw_input() 187 | # 188 | # conn.close() 189 | # 190 | # actual_cols = ['name', 'TypeString', 'TypeFields', 'cmt', 'fieldcmts', 'sclass', 'parsedList', 'depends', 'depends_ordinals', "flags"] 191 | # cols = [] 192 | # db = Storage_sqlite("F:\IDA 7.2\TypeStorage.db","test") 193 | # print db.GetAllNames() 194 | # print db.GetAllProjects() 195 | # 196 | # print db.isActual() 197 | # #db.update_table() 198 | # #print db.isActual() 199 | # print db.request(r"SELECT flags FROM test2") 200 | # 201 | # print db.request(r"UPDATE test2 SET flags = ? WHERE name == ?",(11,"TPropInfo")) 202 | # 203 | # print db.request(r"SELECT flags FROM test2 WHERE (flags & 3) == 3") 204 | 205 | names = ["ATL::CDataSource","ATL::CDataSource","BBBBB","CMFCToolBarNameDialog"] 206 | 207 | conn = sqlite3.connect("F:\IDA 7.2\TypeStorage.db") 208 | c = conn.cursor() 209 | res = c.execute(r"SELECT name FROM test WHERE name=?",names) 210 | res = res.fetchall() 211 | conn.commit() 212 | conn.close() 213 | 214 | print res -------------------------------------------------------------------------------- /pymongo/operations.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Operation class definitions.""" 16 | 17 | from pymongo.common import validate_boolean, validate_is_mapping 18 | from pymongo.helpers import _gen_index_name, _index_document, _index_list 19 | 20 | 21 | class _WriteOp(object): 22 | """Private base class for all write operations.""" 23 | 24 | __slots__ = ("_filter", "_doc", "_upsert") 25 | 26 | def __init__(self, filter=None, doc=None, upsert=None): 27 | if filter is not None: 28 | validate_is_mapping("filter", filter) 29 | if upsert is not None: 30 | validate_boolean("upsert", upsert) 31 | self._filter = filter 32 | self._doc = doc 33 | self._upsert = upsert 34 | 35 | def __eq__(self, other): 36 | if type(other) == type(self): 37 | return (other._filter, other._doc, other._upsert) == \ 38 | (self._filter, self._doc, self._upsert) 39 | return NotImplemented 40 | 41 | def __ne__(self, other): 42 | return not self == other 43 | 44 | 45 | class InsertOne(_WriteOp): 46 | """Represents an insert_one operation.""" 47 | 48 | def __init__(self, document): 49 | """Create an InsertOne instance. 50 | 51 | For use with :meth:`~pymongo.collection.Collection.bulk_write`. 52 | 53 | :Parameters: 54 | - `document`: The document to insert. If the document is missing an 55 | _id field one will be added. 56 | """ 57 | super(InsertOne, self).__init__(doc=document) 58 | 59 | def _add_to_bulk(self, bulkobj): 60 | """Add this operation to the _Bulk instance `bulkobj`.""" 61 | bulkobj.add_insert(self._doc) 62 | 63 | def __repr__(self): 64 | return "InsertOne(%r)" % (self._doc,) 65 | 66 | 67 | class DeleteOne(_WriteOp): 68 | """Represents a delete_one operation.""" 69 | 70 | def __init__(self, filter): 71 | """Create a DeleteOne instance. 72 | 73 | For use with :meth:`~pymongo.collection.Collection.bulk_write`. 74 | 75 | :Parameters: 76 | - `filter`: A query that matches the document to delete. 77 | """ 78 | super(DeleteOne, self).__init__(filter) 79 | 80 | def _add_to_bulk(self, bulkobj): 81 | """Add this operation to the _Bulk instance `bulkobj`.""" 82 | bulkobj.add_delete(self._filter, 1) 83 | 84 | def __repr__(self): 85 | return "DeleteOne(%r)" % (self._filter,) 86 | 87 | 88 | class DeleteMany(_WriteOp): 89 | """Represents a delete_many operation.""" 90 | 91 | def __init__(self, filter): 92 | """Create a DeleteMany instance. 93 | 94 | For use with :meth:`~pymongo.collection.Collection.bulk_write`. 95 | 96 | :Parameters: 97 | - `filter`: A query that matches the documents to delete. 98 | """ 99 | super(DeleteMany, self).__init__(filter) 100 | 101 | def _add_to_bulk(self, bulkobj): 102 | """Add this operation to the _Bulk instance `bulkobj`.""" 103 | bulkobj.add_delete(self._filter, 0) 104 | 105 | def __repr__(self): 106 | return "DeleteMany(%r)" % (self._filter,) 107 | 108 | 109 | class ReplaceOne(_WriteOp): 110 | """Represents a replace_one operation.""" 111 | 112 | def __init__(self, filter, replacement, upsert=False): 113 | """Create a ReplaceOne instance. 114 | 115 | For use with :meth:`~pymongo.collection.Collection.bulk_write`. 116 | 117 | :Parameters: 118 | - `filter`: A query that matches the document to replace. 119 | - `replacement`: The new document. 120 | - `upsert` (optional): If ``True``, perform an insert if no documents 121 | match the filter. 122 | """ 123 | super(ReplaceOne, self).__init__(filter, replacement, upsert) 124 | 125 | def _add_to_bulk(self, bulkobj): 126 | """Add this operation to the _Bulk instance `bulkobj`.""" 127 | bulkobj.add_replace(self._filter, self._doc, self._upsert) 128 | 129 | def __repr__(self): 130 | return "ReplaceOne(%r, %r, %r)" % (self._filter, 131 | self._doc, 132 | self._upsert) 133 | 134 | 135 | class UpdateOne(_WriteOp): 136 | """Represents an update_one operation.""" 137 | 138 | def __init__(self, filter, update, upsert=False): 139 | """Represents an update_one operation. 140 | 141 | For use with :meth:`~pymongo.collection.Collection.bulk_write`. 142 | 143 | :Parameters: 144 | - `filter`: A query that matches the document to update. 145 | - `update`: The modifications to apply. 146 | - `upsert` (optional): If ``True``, perform an insert if no documents 147 | match the filter. 148 | """ 149 | super(UpdateOne, self).__init__(filter, update, upsert) 150 | 151 | def _add_to_bulk(self, bulkobj): 152 | """Add this operation to the _Bulk instance `bulkobj`.""" 153 | bulkobj.add_update(self._filter, self._doc, False, self._upsert) 154 | 155 | def __repr__(self): 156 | return "UpdateOne(%r, %r, %r)" % (self._filter, 157 | self._doc, 158 | self._upsert) 159 | 160 | 161 | class UpdateMany(_WriteOp): 162 | """Represents an update_many operation.""" 163 | 164 | def __init__(self, filter, update, upsert=False): 165 | """Create an UpdateMany instance. 166 | 167 | For use with :meth:`~pymongo.collection.Collection.bulk_write`. 168 | 169 | :Parameters: 170 | - `filter`: A query that matches the documents to update. 171 | - `update`: The modifications to apply. 172 | - `upsert` (optional): If ``True``, perform an insert if no documents 173 | match the filter. 174 | """ 175 | super(UpdateMany, self).__init__(filter, update, upsert) 176 | 177 | def _add_to_bulk(self, bulkobj): 178 | """Add this operation to the _Bulk instance `bulkobj`.""" 179 | bulkobj.add_update(self._filter, self._doc, True, self._upsert) 180 | 181 | def __repr__(self): 182 | return "UpdateMany(%r, %r, %r)" % (self._filter, 183 | self._doc, 184 | self._upsert) 185 | 186 | 187 | class IndexModel(object): 188 | """Represents an index to create.""" 189 | 190 | __slots__ = ("__document",) 191 | 192 | def __init__(self, keys, **kwargs): 193 | """Create an Index instance. 194 | 195 | For use with :meth:`~pymongo.collection.Collection.create_indexes`. 196 | 197 | Takes either a single key or a list of (key, direction) pairs. 198 | The key(s) must be an instance of :class:`basestring` 199 | (:class:`str` in python 3), and the direction(s) must be one of 200 | (:data:`~pymongo.ASCENDING`, :data:`~pymongo.DESCENDING`, 201 | :data:`~pymongo.GEO2D`, :data:`~pymongo.GEOHAYSTACK`, 202 | :data:`~pymongo.GEOSPHERE`, :data:`~pymongo.HASHED`, 203 | :data:`~pymongo.TEXT`). 204 | 205 | Valid options include, but are not limited to: 206 | 207 | - `name`: custom name to use for this index - if none is 208 | given, a name will be generated. 209 | - `unique`: if ``True`` creates a uniqueness constraint on the index. 210 | - `background`: if ``True`` this index should be created in the 211 | background. 212 | - `sparse`: if ``True``, omit from the index any documents that lack 213 | the indexed field. 214 | - `bucketSize`: for use with geoHaystack indexes. 215 | Number of documents to group together within a certain proximity 216 | to a given longitude and latitude. 217 | - `min`: minimum value for keys in a :data:`~pymongo.GEO2D` 218 | index. 219 | - `max`: maximum value for keys in a :data:`~pymongo.GEO2D` 220 | index. 221 | - `expireAfterSeconds`: Used to create an expiring (TTL) 222 | collection. MongoDB will automatically delete documents from 223 | this collection after seconds. The indexed field must 224 | be a UTC datetime or the data will not expire. 225 | 226 | See the MongoDB documentation for a full list of supported options by 227 | server version. 228 | 229 | :Parameters: 230 | - `keys`: a single key or a list of (key, direction) 231 | pairs specifying the index to create 232 | - `**kwargs` (optional): any additional index creation 233 | options (see the above list) should be passed as keyword 234 | arguments 235 | """ 236 | keys = _index_list(keys) 237 | if "name" not in kwargs: 238 | kwargs["name"] = _gen_index_name(keys) 239 | kwargs["key"] = _index_document(keys) 240 | self.__document = kwargs 241 | 242 | @property 243 | def document(self): 244 | """An index document suitable for passing to the createIndexes 245 | command. 246 | """ 247 | return self.__document 248 | -------------------------------------------------------------------------------- /example/test_create_struct.py: -------------------------------------------------------------------------------- 1 | import struct 2 | import ctypes 3 | import idaapi 4 | import idc 5 | import sys 6 | 7 | def make_field_str(field_num,field_size,pad = 0): 8 | ret = "" 9 | for i in range(0,field_num): 10 | ret += struct.pack(">B",len("field_%X"%(i*field_size))+1) + "field_%X"%(i*field_size) 11 | k = 1 12 | while pad > 0: 13 | ret += struct.pack(">B",len("field_%X"%(i*field_size+k))+1) + "field_%X"%(i*field_size+k) 14 | pad -=1 15 | k +=1 16 | return ret 17 | 18 | def encode_size(num): 19 | enc = 0 20 | if num > 0xF: 21 | t, pad = divmod(num, 0x10) 22 | if t < 0x100: 23 | enc = 0x8100|(pad<<11)|t 24 | return struct.pack(">BB",enc>>8,enc&0xFF) 25 | else: 26 | t1, t2, t3 = (0,0,0) 27 | t1, pad = divmod(num,0x400) 28 | t3 = pad 29 | if pad > 7: 30 | t2, t3 = divmod(pad,8) 31 | return "\xFF\xFF" + struct.pack(">BBB",t1|0x80,t2|0x80,t3<<3|0x40) 32 | else: 33 | return struct.pack(">B",num<<3|1) 34 | 35 | def decode_size(size_str): 36 | l = 0 37 | if size_str[:2] == "\xFF\xFF": 38 | l += 2 39 | size_str = size_str[2:] 40 | b1 = ord(size_str[0]) 41 | l +=1 42 | if b1&0x80: 43 | b2 = ord(size_str[1]) 44 | l += 1 45 | if b2&0x80: 46 | b3 = ord(size_str[2]) 47 | l += 1 48 | if b3&0x40: 49 | t1 = (b1&0x7f)*0x400 50 | t2 = (b2&0x7f)*8 51 | t3 = (b3&0x3f)>>3 52 | return (l,t1+t2+t3) 53 | else: 54 | return None 55 | t1 = b2*0x10 56 | t2 = (b1&0x7f)>>3 57 | return (l,t1+t2) 58 | return (l,b1>>3) 59 | 60 | def make_type_string(field_num,field_size,pad = 0): 61 | ret = "\x0d" + encode_size(field_num + pad) 62 | if field_size == 1: 63 | t = "\x32" 64 | elif field_size == 2: 65 | t = "\x03" 66 | elif field_size == 8: 67 | t = "\x05" 68 | else: 69 | t = "\x07" 70 | ret += t*field_num 71 | if pad > 0: 72 | ret += "\x32"*pad 73 | return ret 74 | 75 | print encode_size(9).encode("HEX") 76 | print decode_size("ffff9ee248".decode("HEX")) 77 | print decode_size("8101".decode("HEX")) 78 | print decode_size("69".decode("HEX")) 79 | # print make_field_str(0x10,4).__repr__() 80 | print decode_size("ffffFFFF78".decode("HEX")) 81 | print encode_size(131071).encode("HEX") 82 | 83 | 84 | 85 | exit(1) 86 | 87 | ############################################################ 88 | # Several type-related functions aren't accessibly via IDAPython 89 | # so have to do things with ctypes 90 | idaname = "ida64" if idc.__EA64__ else "ida" 91 | if sys.platform == "win32": 92 | g_dll = ctypes.windll[idaname + ".wll"] 93 | elif sys.platform == "linux2": 94 | g_dll = ctypes.cdll["lib" + idaname + ".so"] 95 | elif sys.platform == "darwin": 96 | g_dll = ctypes.cdll["lib" + idaname + ".dylib"] 97 | 98 | wrapperTypeString = '\x0d\x01\x01' 99 | 100 | class qtype(Structure): 101 | _fields_ = [("ptr",ctypes.c_char_p),("cur_size",ctypes.c_int),("max_size",ctypes.c_int)] 102 | 103 | 104 | ############################################################ 105 | # Specifying function types for a few IDA SDK functions to keep the 106 | # pointer-to-pointer args clear. 107 | 108 | 109 | d_get_named_type = g_dll.get_named_type 110 | d_get_named_type.argtypes = [ 111 | ctypes.c_void_p, #const til_t *ti, 112 | ctypes.c_char_p, #const char *name, 113 | ctypes.c_int, #int ntf_flags, 114 | ctypes.POINTER(ctypes.POINTER(ctypes.c_ubyte)), #const type_t **type=NULL, 115 | ctypes.POINTER(ctypes.POINTER(ctypes.c_ubyte)), #const p_list **fields=NULL, 116 | ctypes.POINTER(ctypes.POINTER(ctypes.c_ubyte)), #const char **cmt=NULL, 117 | ctypes.POINTER(ctypes.POINTER(ctypes.c_ubyte)), #const p_list **fieldcmts=NULL, 118 | ctypes.POINTER(ctypes.c_ulong), #sclass_t *sclass=NULL, 119 | ctypes.POINTER(ctypes.c_ulong), #uint32 *value=NULL); 120 | ] 121 | 122 | ############################################################ 123 | d_get_numbered_type = g_dll.get_numbered_type 124 | d_get_numbered_type.argtypes = [ 125 | ctypes.c_void_p, #const til_t *ti, 126 | ctypes.c_int, #uint32 ordinal, 127 | ctypes.POINTER(ctypes.c_char_p), #const type_t **type=NULL, 128 | ctypes.POINTER(ctypes.c_char_p), #const p_list **fields=NULL, 129 | ctypes.POINTER(ctypes.c_char_p), #const char **cmt=NULL, 130 | ctypes.POINTER(ctypes.c_char_p), #const p_list **fieldcmts=NULL, 131 | ctypes.POINTER(ctypes.c_ulong), #sclass_t *sclass=NULL 132 | ] 133 | 134 | d_set_numbered_type = g_dll.set_numbered_type 135 | d_set_numbered_type.argtypes = [ 136 | ctypes.c_void_p, #til_t *ti, 137 | ctypes.c_int, #uint32 ordinal, 138 | ctypes.c_int, #int ntf_flags, 139 | ctypes.c_char_p, #const char *name, 140 | ctypes.c_char_p, #const type_t *type, 141 | ctypes.c_char_p, #const p_list *fields=NULL, 142 | ctypes.c_char_p, #const char *cmt=NULL, 143 | ctypes.c_char_p, #const p_list *fldcmts=NULL, 144 | ctypes.POINTER(ctypes.c_ulong), #const sclass_t *sclass=NULL 145 | ] 146 | 147 | my_til = ctypes.c_void_p.in_dll(g_dll, 'idati') 148 | my_ti = idaapi.cvar.idati 149 | 150 | 151 | 152 | 153 | from idaapi import * 154 | from idc import * 155 | import time 156 | 157 | def create_struct_type(struc_size,name,field_size = 4,fAllign = True): 158 | struct_id = GetStrucIdByName(name) 159 | if struct_id != BADADDR: 160 | answer = AskYN(0,"A structure for %s already exists. Are you sure you want to remake it?"%name) 161 | if answer == 1: 162 | DelStruc(struct_id) 163 | else: 164 | return 165 | fields_num, pad = divmod(struc_size, field_size) 166 | if fAllign and pad: 167 | fields_num += 1 168 | pad = 0 169 | typ_type = ctypes.c_char_p(make_type_string(struc_size, field_size,pad)) 170 | typ_fields = ctypes.c_char_p(make_field_str(struc_size, field_size,pad)) 171 | typ_cmt = ctypes.c_char_p("") 172 | typ_fieldcmts = ctypes.c_char_p("") 173 | sclass = ctypes.c_ulong(0) 174 | compact_til(my_ti) 175 | idx = alloc_type_ordinal(my_ti) 176 | ret = d_set_numbered_type( 177 | my_til, 178 | idx, 179 | 0x1, 180 | ctypes.c_char_p(name), 181 | typ_type, 182 | typ_fields, 183 | typ_cmt, 184 | typ_fieldcmts, 185 | ctypes.byref(sclass) 186 | ) 187 | if ret != 0: 188 | Til2Idb(BADADDR,name) 189 | else: 190 | Warning("set_numbered_type error") 191 | 192 | 193 | 194 | def create_struct1(struc_size=0,name = "test1",field_size = 4,fAllign = True): 195 | FieldsNum = struc_size 196 | if FieldsNum == 0 or FieldsNum is None: 197 | return 198 | #elif FieldsNum%4 != 0: 199 | # FieldsNum = FieldsNum + (4 - FieldsNum%4) 200 | #FieldsNum /= 4 201 | FieldsNum, pad = divmod(FieldsNum,field_size) 202 | print "FieldsNum = %d"%FieldsNum 203 | struct_id = AddStrucEx(-1,name,0) 204 | struct_id = GetStrucIdByName(name) 205 | if struct_id == BADADDR: 206 | Warning("Could not create the structure!.\nPlease check the entered name.") 207 | return 208 | i = 0 209 | for i in range(0,FieldsNum): 210 | if AddStrucMember(struct_id,"field_%X"%(i*4),-1,FF_DWRD|FF_DATA,-1,4) != 0: 211 | Warning("Error adding member") 212 | return 213 | if i%500 == 0: 214 | print "Current field %d"%i 215 | if pad == 2: 216 | if AddStrucMember(struct_id,"field_%X"%(i*4+4),-1,FF_WORD|FF_DATA,-1,2) != 0: 217 | Warning("Error adding member") 218 | return 219 | elif pad == 1: 220 | if AddStrucMember(struct_id,"field_%X"%(i*4+4),-1,FF_BYTE|FF_DATA,-1,1) != 0: 221 | Warning("Error adding member") 222 | return 223 | elif pad == 3: 224 | if AddStrucMember(struct_id,"field_%X"%(i*4+4),-1,FF_WORD|FF_DATA,-1,2) != 0: 225 | Warning("Error adding member") 226 | return 227 | if AddStrucMember(struct_id,"field_%X"%(i*4+6),-1,FF_BYTE|FF_DATA,-1,1) != 0: 228 | Warning("Error adding member") 229 | return 230 | 231 | return 232 | 233 | def create_struct2(struc_size=0,name = "test2",field_size = 4,fAllign = True): 234 | fields_num, pad = divmod(struc_size,field_size) 235 | fields_num += 1 236 | if fAllign and pad > 0: 237 | fields_num += 1 238 | pad = 0 239 | tid = add_struc(BADADDR,name,False) 240 | sptr = get_struc(get_struc_id(name)) 241 | i = 0 242 | add_struc_member(sptr, "field_%X" % (fields_num - 1), (fields_num - 1), FF_DWRD | FF_DATA, None, field_size) 243 | fields_num -= 1 244 | for i in range(0, fields_num): 245 | add_struc_member(sptr,"field_%X"%(((fields_num - 1)- i)*field_size),((fields_num - 1)- i)*field_size,FF_DWRD|FF_DATA,None,field_size) 246 | return 247 | 248 | 249 | 250 | 251 | 252 | 253 | struct_id = GetStrucIdByName("test1") 254 | if struct_id != BADADDR: 255 | DelStruc(struct_id) 256 | struct_id = GetStrucIdByName("test2") 257 | if struct_id != BADADDR: 258 | DelStruc(struct_id) 259 | print "Start create_struct1" 260 | start = time.time() 261 | create_struct1(0x1000) 262 | work_time = time.time() - start 263 | print "create_struct1 worked %f"%work_time 264 | print "Start create_struct2" 265 | start = time.time() 266 | create_struct2(0x1000) 267 | work_time = time.time() - start 268 | print "create_struct2 worked %f"%work_time -------------------------------------------------------------------------------- /example/test_db.py: -------------------------------------------------------------------------------- 1 | from pymongo import * 2 | import struct, pickle 3 | import ctypes 4 | from bson import * 5 | import json, ast 6 | 7 | #ti = idaapi.cvar.idati 8 | 9 | 10 | 11 | class TinfoReader: 12 | def __init__(self, tp): 13 | self.pos = 0 14 | self.tp = tp 15 | 16 | def read_byte(self): 17 | (result,) = struct.unpack("B", self.tp[self.pos]) 18 | self.pos += 1 19 | return result 20 | 21 | def read_string(self,cb): 22 | ret = self.tp[self.pos:self.pos+cb] 23 | self.pos += cb 24 | return ret 25 | 26 | def keep_going(self): 27 | return self.pos < len(self.tp) 28 | 29 | def encode_ordinal_to_string(ordinal): 30 | enc = [] 31 | enc.append(ordinal&0x7f|0x40) 32 | if ordinal > 0x3f: 33 | bt = ordinal 34 | bt = bt // 0x40 35 | enc.append(bt&0x7f|0x80) 36 | while bt > 0x7f: 37 | bt = bt // 0x80 38 | enc.append(bt&0x7f|0x80) 39 | # stemp = struct.pack("B",len(enc)+2) + "#" 40 | stemp = [] 41 | stemp.append(len(enc)+2) 42 | stemp.append("#") 43 | # for i in range(0,len(enc)): 44 | # stemp = stemp + struct.pack("B",enc.pop(-1)) 45 | stemp = stemp + enc.reverse() 46 | return stemp 47 | 48 | def decode_ordinal_string(enc): 49 | if enc[1] == "#": 50 | ord_num = 0 51 | i = 0 52 | fEnd = 0 53 | str_len = struct.unpack("B",enc[0])[0] - 2 54 | #print len 55 | for ch in enc[2:]: 56 | ch = ord(ch) 57 | if ch == 0: 58 | return 0 59 | ord_num = ord_num * 0x40 60 | if ch&0x80 != 0: 61 | ord_num = ord_num * 2 62 | ch = ch & 0x7f 63 | else: 64 | ch = ch & 0x3f 65 | fEnd = 1 66 | ord_num = ord_num | ch 67 | i = i + 1 68 | if fEnd > 0 or i >= str_len: 69 | break 70 | return ord_num 71 | return 0 72 | 73 | def decode_ordinal(enc): 74 | ord_num = 0 75 | i = 0 76 | fEnd = 0 77 | len = struct.unpack("B",enc[0]) 78 | for ch in enc: 79 | ch = ord(ch) 80 | if ch == 0: 81 | return 0 82 | ord_num = ord_num * 0x40 83 | if ch&0x80 != 0: 84 | ord_num = ord_num * 2 85 | ch = ch & 0x7f 86 | else: 87 | ch = ch & 0x3f 88 | fEnd = 1 89 | ord_num = ord_num | ch 90 | if fEnd > 0 or i >= len: 91 | break 92 | return ord_num 93 | 94 | def encode_ordinal(ordinal): 95 | enc = [] 96 | enc.append(ordinal&0x7f|0x40) 97 | if ordinal > 0x3f: 98 | bt = ordinal 99 | bt = bt // 0x40 100 | enc.append(bt&0x7f|0x80) 101 | while bt > 0x7f: 102 | bt = bt // 0x80 103 | enc.append(bt&0x7f|0x80) 104 | stemp = [] 105 | for i in range(0,len(enc)): 106 | stemp = stemp + struct.pack("B",enc.pop(-1)) 107 | return stemp 108 | 109 | class LocalType(object): 110 | def __init__(self, name = "", TypeString = "", TypeFields = "",cmt = "", fieldcmts = "", sclass = "", parsedList = [], depends = []): 111 | self.TypeString = TypeString 112 | self.TypeFields = TypeFields 113 | self.cmt = cmt 114 | self.fieldcmts = fieldcmts 115 | self.sclass = sclass 116 | self.name = name 117 | self.parsedList = [] 118 | self.depends = [] 119 | 120 | self.parsedList = self.ParseTypeString(TypeString) 121 | 122 | # def __init__(self,ser_dict): 123 | # self.from_dict(ser_dict) 124 | 125 | 126 | 127 | # def __init__(self, idx): 128 | # self.name = None 129 | # self.parsedList = [] 130 | # self.TypeString = None 131 | # self.TypeFields = None 132 | # self.cmt = None 133 | # self.fieldcmts = None 134 | # self.sclass = None 135 | # self.depends = [] 136 | 137 | 138 | # def find_type_by_name(self, name): 139 | # ordinal = get_type_ordinal(ti,name) 140 | 141 | # def GetTypeString(self): 142 | # 143 | # the_bytes = [] 144 | # for thing in self.parsedList: 145 | # if type(thing) == int: # if it's a byte, just put it back in 146 | # the_bytes.append(thing) 147 | # else: 148 | # the_bytes.append(ord("=")) # a type starts with = 149 | # ordinal = get_type_ordinal(ti,thing["local_type"]) # get the ordinal of the Local Type based on its name 150 | # if ordinal > 0: 151 | # the_bytes = the_bytes + encode_ordinal(ordinal) 152 | # else: 153 | # raise "Depends local type not in IDB" 154 | # packed = struct.pack("%dB" % len(the_bytes), *the_bytes) 155 | # return packed 156 | 157 | # def ParseTypeString(self,type_string): 158 | # tp = TinfoReader(type_string) 159 | # # print idc_print_type(type_, fields, "fun_name", 0) 160 | # # print type_.encode("string_escape") 161 | # output = [] 162 | # """ 163 | # Attempt to copy the tinfo from a location, replacing any Local Types with our own representation of them. 164 | # Pass all other bytes through as-is. 165 | # """ 166 | # while tp.keep_going(): 167 | # a_byte = tp.read_byte() 168 | # unwritten_bytes = [a_byte] 169 | # if a_byte == ord("="): # a type begins 170 | # ordinal_length = tp.read_byte() 171 | # number_marker = tp.read_byte() 172 | # #unwritten_bytes.append(number_marker) 173 | # if number_marker == ord("#"): # this is a Local Type referred to by its ordinal 174 | # ordinal = decode_ordinal(struct.pack("B",ordinal_length) + "#" + tp.read_string(ordinal_length-2)) 175 | # t = GetLocalTypeName(ordinal) 176 | # output.append({"local_type": t}) 177 | # if t not in self.depends: 178 | # self.depends.append(t) 179 | # continue 180 | # unwritten_bytes.append(ordinal_length) 181 | # unwritten_bytes.append(number_marker) 182 | # 183 | # output += unwritten_bytes # put all the bytes we didn't consume into the output as-is 184 | # 185 | # return output 186 | 187 | def to_dict(self): 188 | ser_dic = {} 189 | ser_dic['name'] = self.name 190 | ser_dic['TypeString'] = self.TypeString.encode("base64") 191 | ser_dic['TypeFields'] = self.TypeFields.encode("base64") 192 | ser_dic['cmt'] = self.cmt.encode("base64") 193 | ser_dic['fieldcmts'] = self.fieldcmts.encode("base64") 194 | ser_dic['sclass'] = self.sclass 195 | ser_dic['parsedList'] = self.parsedList 196 | ser_dic['depends'] = self.depends 197 | return ser_dic 198 | 199 | def from_dict(self,ser_dic): 200 | self.name = ser_dic['name'].encode("ascii") 201 | self.TypeString = ser_dic['TypeString'].encode("ascii").decode("base64") 202 | self.TypeFields = ser_dic['TypeFields'].encode("ascii").decode("base64") 203 | self.cmt = ser_dic['cmt'].encode("ascii").decode("base64") 204 | self.fieldcmts = ser_dic['fieldcmts'].encode("ascii").decode("base64") 205 | self.sclass = ser_dic['sclass'] 206 | self.parsedList = ser_dic['parsedList'] 207 | self.depends = ser_dic['depends'] 208 | return self 209 | 210 | class Storage(object): 211 | def __init__(self,ip = 'localhost',port = 27017 ,target_collection = "main_storage"): 212 | self.client = MongoClient(ip, port) 213 | self.db = self.client["LocalTypesStorage"] 214 | self.collection = self.db[target_collection] 215 | 216 | def putToStorage(self,t): 217 | self.collection.insert_one(t.to_dict()) 218 | 219 | def getFromStorage(self,name): 220 | res = collection.find({"name":name}) 221 | if res.count() == 1: 222 | return LocalType().from_dict(res[0]) 223 | elif res.count() == 0: 224 | return None 225 | else: 226 | raise "getFromStorage: Type duplication or error. Count = %d"%(res.count()) 227 | 228 | def isExist(self,name): 229 | res = collection.find({"name":name}) 230 | if res.count() == 1: 231 | return True 232 | elif res.count() == 0: 233 | return False 234 | else: 235 | raise "isExist: Type duplication or error. Count = %d"%(res.count()) 236 | 237 | def updateType(self,name,t): 238 | ret = collection.replace_one({'name':name},t) 239 | if ret.matched_count == 1: 240 | return True 241 | elif ret.matched_count == 0: 242 | return False 243 | else: 244 | raise "updateType: Type duplication or error. Count = %d"%(ret.count()) 245 | 246 | 247 | 248 | 249 | client = MongoClient('localhost', 27017) 250 | db = client['test_db'] 251 | print db.collection_names(include_system_collections=False) 252 | db['test-coll'].drop() 253 | print db.collection_names(include_system_collections=False) 254 | collection = db['test-coll2'] 255 | f = open("F:\IdaTextTypesParser\cache.dat","rb") 256 | LocalTypeMap = pickle.load(f) 257 | f.close() 258 | collection.drop() 259 | print len(LocalTypeMap) 260 | for name in LocalTypeMap: 261 | collection.insert_one(LocalTypeMap[name].to_dict()) 262 | 263 | print collection.find({"name":"_CERT_AUTHORITY_KEY_ID2_INFO"}).count() 264 | print collection.find({"name":"_CERT_AUTHORITY_KEY_ID2_INFO"})[0] 265 | for t in collection.find({"name":"_CERT_AUTHORITY_KEY_ID2_INFO"}): 266 | print t 267 | print t["name"] 268 | l = t["name"] 269 | print type(l) 270 | l = t["depends"][0] 271 | print l 272 | print type(l) 273 | 274 | t['name'] = t['name'] + "_modifed" 275 | id = t['_id'] 276 | collection.replace_one({'name':"_CERT_AUTHORITY_KEY_ID2_INFO"},t) 277 | for t in collection.find({"_id":id}): 278 | print t 279 | 280 | client.close() 281 | 282 | 283 | --------------------------------------------------------------------------------