├── gp ├── tests │ ├── __init__.py │ ├── .svn │ │ ├── text-base │ │ │ ├── __init__.py.svn-base │ │ │ ├── gp.test.data.svn-base │ │ │ ├── test_config.py.svn-base │ │ │ ├── client_test.py.svn-base │ │ │ ├── crud_fuzzer.py.svn-base │ │ │ ├── arc_fuzzer.py.svn-base │ │ │ ├── tcp_fuzzer.py.svn-base │ │ │ ├── fuzzer_base.py.svn-base │ │ │ ├── core_test.py.svn-base │ │ │ ├── test_base.py.svn-base │ │ │ ├── slave_test.py.svn-base │ │ │ ├── mysql_test.py.svn-base │ │ │ └── server_test.py.svn-base │ │ ├── all-wcprops │ │ └── entries │ ├── gp.test.data │ ├── __init__.pyc │ ├── test_config.py │ ├── client_test.py │ ├── crud_fuzzer.py │ ├── arc_fuzzer.py │ ├── tcp_fuzzer.py │ ├── fuzzer_base.py │ ├── core_test.py │ ├── test_base.py │ ├── slave_test.py │ ├── mysql_test.py │ ├── server_test.py │ └── mediawiki_test.py ├── __init__.py ├── mysql.pyc ├── __init__.pyc ├── .svn │ ├── text-base │ │ ├── __init__.py.svn-base │ │ └── mediawiki.py.svn-base │ ├── all-wcprops │ └── entries ├── mediawiki.pyc └── mediawiki.py ├── .gitignore └── README.md /gp/tests/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /gp/tests/.svn/text-base/__init__.py.svn-base: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | SciTE.properties 2 | *.pyc 3 | scite-session 4 | -------------------------------------------------------------------------------- /gp/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = [ 'client', 'mysql', 'mediawiki' ] 2 | -------------------------------------------------------------------------------- /gp/tests/gp.test.data: -------------------------------------------------------------------------------- 1 | 1,11 2 | 1, 12 3 | 11, 111 4 | 11 112 5 | -------------------------------------------------------------------------------- /gp/mysql.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wmde/graphcare/master/gp/mysql.pyc -------------------------------------------------------------------------------- /gp/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wmde/graphcare/master/gp/__init__.pyc -------------------------------------------------------------------------------- /gp/.svn/text-base/__init__.py.svn-base: -------------------------------------------------------------------------------- 1 | __all__ = [ 'client', 'mysql', 'mediawiki' ] 2 | -------------------------------------------------------------------------------- /gp/mediawiki.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wmde/graphcare/master/gp/mediawiki.pyc -------------------------------------------------------------------------------- /gp/tests/.svn/text-base/gp.test.data.svn-base: -------------------------------------------------------------------------------- 1 | 1,11 2 | 1, 12 3 | 11, 111 4 | 11 112 5 | -------------------------------------------------------------------------------- /gp/tests/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wmde/graphcare/master/gp/tests/__init__.pyc -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GraphCare 2 | 3 | This script ensures that a Graphserv instance is running and updates its graphcore processes with category/page data from Wikipedia. It is started periodically from a cron job on catgraph hosts. See --help for more info. 4 | 5 | ## Issue tracker 6 | Please file bugs and feature requests on [Phabricator] (https://phabricator.wikimedia.org/maniphest/task/create/?projects=tcb-team,catgraph&title=%5BGraphCare%5D). 7 | -------------------------------------------------------------------------------- /gp/tests/test_config.py: -------------------------------------------------------------------------------- 1 | from gp.client import * 2 | 3 | #graphcore binary for slave tests 4 | test_graphcore_path = '/home/daniel/src/graphserv/graphcore/graphcore'; 5 | 6 | #graphserv coordinates for client tests 7 | test_graphserv_host = 'localhost'; 8 | test_graphserv_port = PORT; 9 | 10 | #logins for test accounts 11 | test_admin = 'fred'; 12 | test_admin_password = 'test'; 13 | 14 | test_master = 'jules'; 15 | test_master_password = 'test'; 16 | 17 | #mysql login for testing MySQLGlue 18 | test_mysql_host = 'localhost'; 19 | test_mysql_user = 'gptest'; 20 | test_mysql_password = 'gptest'; 21 | test_mysql_database = 'gptest'; 22 | 23 | #mediawiki database info 24 | test_mediawiki_table_prefix = 'mw_'; 25 | -------------------------------------------------------------------------------- /gp/tests/.svn/text-base/test_config.py.svn-base: -------------------------------------------------------------------------------- 1 | from gp.client import * 2 | 3 | #graphcore binary for slave tests 4 | test_graphcore_path = '/home/daniel/src/graphserv/graphcore/graphcore'; 5 | 6 | #graphserv coordinates for client tests 7 | test_graphserv_host = 'localhost'; 8 | test_graphserv_port = PORT; 9 | 10 | #logins for test accounts 11 | test_admin = 'fred'; 12 | test_admin_password = 'test'; 13 | 14 | test_master = 'jules'; 15 | test_master_password = 'test'; 16 | 17 | #mysql login for testing MySQLGlue 18 | test_mysql_host = 'localhost'; 19 | test_mysql_user = 'gptest'; 20 | test_mysql_password = 'gptest'; 21 | test_mysql_database = 'gptest'; 22 | 23 | #mediawiki database info 24 | test_mediawiki_table_prefix = 'mw_'; 25 | -------------------------------------------------------------------------------- /gp/.svn/all-wcprops: -------------------------------------------------------------------------------- 1 | K 25 2 | svn:wc:ra_dav:version-url 3 | V 65 4 | /svnroot/daniel/!svn/ver/625/duesenstuff/trunk/gpClient/python/gp 5 | END 6 | client.py 7 | K 25 8 | svn:wc:ra_dav:version-url 9 | V 75 10 | /svnroot/daniel/!svn/ver/616/duesenstuff/trunk/gpClient/python/gp/client.py 11 | END 12 | __init__.py 13 | K 25 14 | svn:wc:ra_dav:version-url 15 | V 77 16 | /svnroot/daniel/!svn/ver/488/duesenstuff/trunk/gpClient/python/gp/__init__.py 17 | END 18 | mediawiki.py 19 | K 25 20 | svn:wc:ra_dav:version-url 21 | V 78 22 | /svnroot/daniel/!svn/ver/625/duesenstuff/trunk/gpClient/python/gp/mediawiki.py 23 | END 24 | mysql.py 25 | K 25 26 | svn:wc:ra_dav:version-url 27 | V 74 28 | /svnroot/daniel/!svn/ver/625/duesenstuff/trunk/gpClient/python/gp/mysql.py 29 | END 30 | -------------------------------------------------------------------------------- /gp/tests/client_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 3 | 4 | import unittest 5 | import os 6 | import tempfile 7 | from test_base import * 8 | from gp.client import * 9 | 10 | test_graph_name = 'test' + str(os.getpid()) 11 | TestFilePrefix = '/tmp/gptest-' + str(os.getpid()) 12 | 13 | class ClientTest (ClientTestBase, unittest.TestCase): 14 | """Test the TCP client connection. 15 | 16 | Client Connection Tests 17 | currently none. Could test handling of TCP issues, etc 18 | 19 | @TODO: (optionally) start server instance here! 20 | let it die when the test script dies. 21 | 22 | @TODO: CLI interface behaviour of server (port config, etc) 23 | 24 | """ 25 | 26 | 27 | if __name__ == '__main__': 28 | unittest.main() 29 | -------------------------------------------------------------------------------- /gp/tests/.svn/text-base/client_test.py.svn-base: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 3 | 4 | import unittest 5 | import os 6 | import tempfile 7 | from test_base import * 8 | from gp.client import * 9 | 10 | test_graph_name = 'test' + str(os.getpid()) 11 | TestFilePrefix = '/tmp/gptest-' + str(os.getpid()) 12 | 13 | class ClientTest (ClientTestBase, unittest.TestCase): 14 | """Test the TCP client connection. 15 | 16 | Client Connection Tests 17 | currently none. Could test handling of TCP issues, etc 18 | 19 | @TODO: (optionally) start server instance here! 20 | let it die when the test script dies. 21 | 22 | @TODO: CLI interface behaviour of server (port config, etc) 23 | 24 | """ 25 | 26 | 27 | if __name__ == '__main__': 28 | unittest.main() 29 | -------------------------------------------------------------------------------- /gp/tests/crud_fuzzer.py: -------------------------------------------------------------------------------- 1 | from fuzzer_base import * 2 | from gp.client import * 3 | from test_config import * 4 | import sys 5 | 6 | fuzz_commands = ( "list-roots", "list-successors", "add-arcs", "stats", 7 | "22", "-11111", "xyz", "0", "!", "&", "%", "#", "",) 8 | 9 | fuzz_args = ( "1", "2", "11", "22", "-11111", "xyz", "0", "!", 10 | "&", "%", "#", "",) 11 | 12 | 13 | class CrudFuzzer (FuzzerBase): 14 | """Test the TCP client connection""" 15 | 16 | def prepare(self): 17 | #self.gp.add_arcs(((1, 2), (1, 11), (2, 22),)) 18 | self.gp.add_arcs(((1, 2), (1, 11), (2, 22),)) 19 | 20 | 21 | 22 | def doFuzz(self): 23 | global fuzz_commands 24 | global fuzz_args 25 | 26 | cmd = "" 27 | 28 | cmd = cmd + fuzz_pick(fuzz_commands) 29 | cmd = cmd + " " 30 | cmd = cmd + fuzz_pick(fuzz_args) 31 | cmd = cmd + " " 32 | cmd = cmd + fuzz_pick(fuzz_args) 33 | 34 | try: 35 | self.gp.execute(cmd) 36 | return True 37 | except gpProcessorException, ex: 38 | pass # noop 39 | except gpUsageException, ex: 40 | pass # noop 41 | 42 | return False 43 | 44 | if __name__ == '__main__': 45 | 46 | fuzzer = CrudFuzzer() 47 | 48 | fuzzer.run(sys.argv) 49 | -------------------------------------------------------------------------------- /gp/tests/.svn/text-base/crud_fuzzer.py.svn-base: -------------------------------------------------------------------------------- 1 | from fuzzer_base import * 2 | from gp.client import * 3 | from test_config import * 4 | import sys 5 | 6 | fuzz_commands = ( "list-roots", "list-successors", "add-arcs", "stats", 7 | "22", "-11111", "xyz", "0", "!", "&", "%", "#", "",) 8 | 9 | fuzz_args = ( "1", "2", "11", "22", "-11111", "xyz", "0", "!", 10 | "&", "%", "#", "",) 11 | 12 | 13 | class CrudFuzzer (FuzzerBase): 14 | """Test the TCP client connection""" 15 | 16 | def prepare(self): 17 | #self.gp.add_arcs(((1, 2), (1, 11), (2, 22),)) 18 | self.gp.add_arcs(((1, 2), (1, 11), (2, 22),)) 19 | 20 | 21 | 22 | def doFuzz(self): 23 | global fuzz_commands 24 | global fuzz_args 25 | 26 | cmd = "" 27 | 28 | cmd = cmd + fuzz_pick(fuzz_commands) 29 | cmd = cmd + " " 30 | cmd = cmd + fuzz_pick(fuzz_args) 31 | cmd = cmd + " " 32 | cmd = cmd + fuzz_pick(fuzz_args) 33 | 34 | try: 35 | self.gp.execute(cmd) 36 | return True 37 | except gpProcessorException, ex: 38 | pass # noop 39 | except gpUsageException, ex: 40 | pass # noop 41 | 42 | return False 43 | 44 | if __name__ == '__main__': 45 | 46 | fuzzer = CrudFuzzer() 47 | 48 | fuzzer.run(sys.argv) 49 | -------------------------------------------------------------------------------- /gp/.svn/entries: -------------------------------------------------------------------------------- 1 | 10 2 | 3 | dir 4 | 630 5 | https://svn.toolserver.org/svnroot/daniel/duesenstuff/trunk/gpClient/python/gp 6 | https://svn.toolserver.org/svnroot/daniel 7 | 8 | 9 | 10 | 2012-02-14T15:08:17.070981Z 11 | 625 12 | daniel 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 9f2c43bc-b3c0-43f4-b155-41619b16f219 28 | 29 | tests 30 | dir 31 | 32 | client.py 33 | file 34 | 35 | 36 | 37 | 38 | 2012-07-27T13:23:46.676858Z 39 | 309da5b2baa57173583d063f9a01e46c 40 | 2012-02-10T16:03:58.030164Z 41 | 616 42 | daniel 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 76801 65 | 66 | __init__.py 67 | file 68 | 69 | 70 | 71 | 72 | 2012-07-27T13:23:46.676858Z 73 | ab2978e80e312edaaa9c344d02508e0a 74 | 2011-09-16T08:56:04.680087Z 75 | 488 76 | daniel 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 45 99 | 100 | mediawiki.py 101 | file 102 | 103 | 104 | 105 | 106 | 2012-07-27T13:23:46.676858Z 107 | 018a84ed8841639a42ec220ee099ad09 108 | 2012-02-14T15:08:17.070981Z 109 | 625 110 | daniel 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 17020 133 | 134 | mysql.py 135 | file 136 | 137 | 138 | 139 | 140 | 2012-07-27T13:23:46.676858Z 141 | 79338e28867b6f2e78f3957e500e0957 142 | 2012-02-14T15:08:17.070981Z 143 | 625 144 | daniel 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 23271 167 | 168 | -------------------------------------------------------------------------------- /gp/tests/arc_fuzzer.py: -------------------------------------------------------------------------------- 1 | from fuzzer_base import FuzzerBase, test_graph_name 2 | from gp.client import Connection 3 | from test_config import * 4 | import random 5 | import sys 6 | 7 | class ArcFuzzer(FuzzerBase): 8 | """Tests the TCP client connection""" 9 | 10 | def __init__(self): 11 | self.offset = 1 12 | FuzzerBase.__init__(self) 13 | 14 | def prepare(self): 15 | # in case we use a persistent graph, fund an unused offset 16 | Range = range(self.offset,10) 17 | for i in Range: 18 | if not self.gp.capture_list_successors(i): 19 | self.gp.add_arcs(((i, i+1),)) 20 | print "fuzz offset: %d (%s)" % (i, test_graph_name) 21 | return 22 | 23 | self.offset = i + 1 24 | #? self.offset verstehen! 25 | 26 | exit("no free offset left (or " 27 | + test_graph_name + "needs purging)") 28 | 29 | 30 | def random_node(self): 31 | return random.randint(10, 1000) * 10 + self.offset 32 | 33 | 34 | def random_arcs(self, n=0): 35 | if not n: 36 | n = random.randint(2, 80) 37 | arcs = [] 38 | for i in range(0, n): 39 | a = self.random_node() 40 | b = self.random_node() 41 | arcs.append((a, b)) 42 | return arcs 43 | 44 | def random_set(self, n=0): 45 | if not n: 46 | n = random.randint(2, 80) 47 | arcs = [] 48 | for i in range(0, n): 49 | x = self.random_node() 50 | arcs.append(x) 51 | return arcs 52 | 53 | def doFuzz(self): 54 | self.gp.add_arcs(self.random_arcs()) 55 | self.gp.remove_arcs(self.random_arcs()) 56 | 57 | self.gp.replace_successors(self.random_node(), self.random_set()) 58 | self.gp.replace_predecessors(self.random_node(), self.random_set()) 59 | 60 | return True 61 | 62 | 63 | if __name__ == '__main__': 64 | 65 | fuzzer = ArcFuzzer() 66 | 67 | fuzzer.run(sys.argv) 68 | -------------------------------------------------------------------------------- /gp/tests/.svn/text-base/arc_fuzzer.py.svn-base: -------------------------------------------------------------------------------- 1 | from fuzzer_base import FuzzerBase, test_graph_name 2 | from gp.client import Connection 3 | from test_config import * 4 | import random 5 | import sys 6 | 7 | class ArcFuzzer(FuzzerBase): 8 | """Tests the TCP client connection""" 9 | 10 | def __init__(self): 11 | self.offset = 1 12 | FuzzerBase.__init__(self) 13 | 14 | def prepare(self): 15 | # in case we use a persistent graph, fund an unused offset 16 | Range = range(self.offset,10) 17 | for i in Range: 18 | if not self.gp.capture_list_successors(i): 19 | self.gp.add_arcs(((i, i+1),)) 20 | print "fuzz offset: %d (%s)" % (i, test_graph_name) 21 | return 22 | 23 | self.offset = i + 1 24 | #? self.offset verstehen! 25 | 26 | exit("no free offset left (or " 27 | + test_graph_name + "needs purging)") 28 | 29 | 30 | def random_node(self): 31 | return random.randint(10, 1000) * 10 + self.offset 32 | 33 | 34 | def random_arcs(self, n=0): 35 | if not n: 36 | n = random.randint(2, 80) 37 | arcs = [] 38 | for i in range(0, n): 39 | a = self.random_node() 40 | b = self.random_node() 41 | arcs.append((a, b)) 42 | return arcs 43 | 44 | def random_set(self, n=0): 45 | if not n: 46 | n = random.randint(2, 80) 47 | arcs = [] 48 | for i in range(0, n): 49 | x = self.random_node() 50 | arcs.append(x) 51 | return arcs 52 | 53 | def doFuzz(self): 54 | self.gp.add_arcs(self.random_arcs()) 55 | self.gp.remove_arcs(self.random_arcs()) 56 | 57 | self.gp.replace_successors(self.random_node(), self.random_set()) 58 | self.gp.replace_predecessors(self.random_node(), self.random_set()) 59 | 60 | return True 61 | 62 | 63 | if __name__ == '__main__': 64 | 65 | fuzzer = ArcFuzzer() 66 | 67 | fuzzer.run(sys.argv) 68 | -------------------------------------------------------------------------------- /gp/tests/.svn/all-wcprops: -------------------------------------------------------------------------------- 1 | K 25 2 | svn:wc:ra_dav:version-url 3 | V 71 4 | /svnroot/daniel/!svn/ver/584/duesenstuff/trunk/gpClient/python/gp/tests 5 | END 6 | arc_fuzzer.py 7 | K 25 8 | svn:wc:ra_dav:version-url 9 | V 85 10 | /svnroot/daniel/!svn/ver/503/duesenstuff/trunk/gpClient/python/gp/tests/arc_fuzzer.py 11 | END 12 | tcp_fuzzer.py 13 | K 25 14 | svn:wc:ra_dav:version-url 15 | V 85 16 | /svnroot/daniel/!svn/ver/503/duesenstuff/trunk/gpClient/python/gp/tests/tcp_fuzzer.py 17 | END 18 | client_test.py 19 | K 25 20 | svn:wc:ra_dav:version-url 21 | V 86 22 | /svnroot/daniel/!svn/ver/479/duesenstuff/trunk/gpClient/python/gp/tests/client_test.py 23 | END 24 | __init__.py 25 | K 25 26 | svn:wc:ra_dav:version-url 27 | V 83 28 | /svnroot/daniel/!svn/ver/488/duesenstuff/trunk/gpClient/python/gp/tests/__init__.py 29 | END 30 | fuzzer_base.py 31 | K 25 32 | svn:wc:ra_dav:version-url 33 | V 86 34 | /svnroot/daniel/!svn/ver/503/duesenstuff/trunk/gpClient/python/gp/tests/fuzzer_base.py 35 | END 36 | crud_fuzzer.py 37 | K 25 38 | svn:wc:ra_dav:version-url 39 | V 86 40 | /svnroot/daniel/!svn/ver/503/duesenstuff/trunk/gpClient/python/gp/tests/crud_fuzzer.py 41 | END 42 | mediawiki_test.py 43 | K 25 44 | svn:wc:ra_dav:version-url 45 | V 89 46 | /svnroot/daniel/!svn/ver/488/duesenstuff/trunk/gpClient/python/gp/tests/mediawiki_test.py 47 | END 48 | mysql_test.py 49 | K 25 50 | svn:wc:ra_dav:version-url 51 | V 85 52 | /svnroot/daniel/!svn/ver/584/duesenstuff/trunk/gpClient/python/gp/tests/mysql_test.py 53 | END 54 | test_config.py 55 | K 25 56 | svn:wc:ra_dav:version-url 57 | V 86 58 | /svnroot/daniel/!svn/ver/482/duesenstuff/trunk/gpClient/python/gp/tests/test_config.py 59 | END 60 | gp.test.data 61 | K 25 62 | svn:wc:ra_dav:version-url 63 | V 84 64 | /svnroot/daniel/!svn/ver/474/duesenstuff/trunk/gpClient/python/gp/tests/gp.test.data 65 | END 66 | server_test.py 67 | K 25 68 | svn:wc:ra_dav:version-url 69 | V 86 70 | /svnroot/daniel/!svn/ver/529/duesenstuff/trunk/gpClient/python/gp/tests/server_test.py 71 | END 72 | core_test.py 73 | K 25 74 | svn:wc:ra_dav:version-url 75 | V 84 76 | /svnroot/daniel/!svn/ver/571/duesenstuff/trunk/gpClient/python/gp/tests/core_test.py 77 | END 78 | slave_test.py 79 | K 25 80 | svn:wc:ra_dav:version-url 81 | V 85 82 | /svnroot/daniel/!svn/ver/482/duesenstuff/trunk/gpClient/python/gp/tests/slave_test.py 83 | END 84 | test_base.py 85 | K 25 86 | svn:wc:ra_dav:version-url 87 | V 84 88 | /svnroot/daniel/!svn/ver/486/duesenstuff/trunk/gpClient/python/gp/tests/test_base.py 89 | END 90 | -------------------------------------------------------------------------------- /gp/tests/tcp_fuzzer.py: -------------------------------------------------------------------------------- 1 | from crud_fuzzer import * 2 | from gp.client import * 3 | from test_config import * 4 | import sys 5 | import threading 6 | import random 7 | import time 8 | 9 | class KillerThread ( threading.Thread ): 10 | """randomly kills client socket""" 11 | 12 | def __init__(self, fuzzer): 13 | super(KillerThread, self).__init__() 14 | 15 | self.stopped = False 16 | self.fuzzer = fuzzer 17 | self.delay = (0.01, 0.2) 18 | 19 | def kill_connection(self): 20 | if ( self.fuzzer.gp 21 | and self.fuzzer.gp.transport.socket ): 22 | 23 | try: 24 | self.fuzzer.kill_lock.acquire() 25 | 26 | self.fuzzer.gp.transport.hin.close() 27 | self.fuzzer.gp.transport.hout.close() 28 | self.fuzzer.gp.transport.socket.close() 29 | 30 | self.fuzzer.killed = True 31 | self.fuzzer.blip("!") 32 | finally: 33 | self.fuzzer.kill_lock.release() 34 | 35 | def run ( self ): 36 | while not self.stopped: 37 | d = random.random() * ( self.delay[1] - self.delay[0] ) + self.delay[0] 38 | time.sleep( d ) 39 | 40 | if not self.stopped: 41 | self.kill_connection() 42 | 43 | def stop(self): 44 | self.stopped = True 45 | 46 | 47 | class TcpFuzzer (CrudFuzzer): 48 | """Test server stability with unstable TCP client connection""" 49 | 50 | def __init__(self): 51 | super(TcpFuzzer, self).__init__() 52 | 53 | self.kill_lock = threading.Lock() 54 | 55 | self.killer = KillerThread( self ) 56 | self.killer.daemon = True 57 | 58 | self.killed = False 59 | 60 | def run(self, argv): 61 | try: 62 | super(TcpFuzzer, self).run(argv) 63 | finally: 64 | self.killer.stop() 65 | 66 | def connect(self): 67 | super(TcpFuzzer, self).connect() 68 | 69 | # force 1 byte chunks, to increase the probability of 70 | # incomplete writes. 71 | self.gp.transport.chunk_size = 1 72 | 73 | def doFuzz(self): 74 | try: 75 | self.kill_lock.acquire() 76 | 77 | if not self.killer.is_alive(): 78 | self.killer.start() 79 | 80 | if self.killed: 81 | self.connect() 82 | self.prepare() 83 | self.killed = False 84 | 85 | self.blip("*") 86 | finally: 87 | self.kill_lock.release() 88 | 89 | try: 90 | return super(TcpFuzzer, self).doFuzz() 91 | except: 92 | pass #whatever 93 | 94 | return False 95 | 96 | if __name__ == '__main__': 97 | fuzzer = TcpFuzzer() 98 | 99 | fuzzer.run(sys.argv) 100 | -------------------------------------------------------------------------------- /gp/tests/.svn/text-base/tcp_fuzzer.py.svn-base: -------------------------------------------------------------------------------- 1 | from crud_fuzzer import * 2 | from gp.client import * 3 | from test_config import * 4 | import sys 5 | import threading 6 | import random 7 | import time 8 | 9 | class KillerThread ( threading.Thread ): 10 | """randomly kills client socket""" 11 | 12 | def __init__(self, fuzzer): 13 | super(KillerThread, self).__init__() 14 | 15 | self.stopped = False 16 | self.fuzzer = fuzzer 17 | self.delay = (0.01, 0.2) 18 | 19 | def kill_connection(self): 20 | if ( self.fuzzer.gp 21 | and self.fuzzer.gp.transport.socket ): 22 | 23 | try: 24 | self.fuzzer.kill_lock.acquire() 25 | 26 | self.fuzzer.gp.transport.hin.close() 27 | self.fuzzer.gp.transport.hout.close() 28 | self.fuzzer.gp.transport.socket.close() 29 | 30 | self.fuzzer.killed = True 31 | self.fuzzer.blip("!") 32 | finally: 33 | self.fuzzer.kill_lock.release() 34 | 35 | def run ( self ): 36 | while not self.stopped: 37 | d = random.random() * ( self.delay[1] - self.delay[0] ) + self.delay[0] 38 | time.sleep( d ) 39 | 40 | if not self.stopped: 41 | self.kill_connection() 42 | 43 | def stop(self): 44 | self.stopped = True 45 | 46 | 47 | class TcpFuzzer (CrudFuzzer): 48 | """Test server stability with unstable TCP client connection""" 49 | 50 | def __init__(self): 51 | super(TcpFuzzer, self).__init__() 52 | 53 | self.kill_lock = threading.Lock() 54 | 55 | self.killer = KillerThread( self ) 56 | self.killer.daemon = True 57 | 58 | self.killed = False 59 | 60 | def run(self, argv): 61 | try: 62 | super(TcpFuzzer, self).run(argv) 63 | finally: 64 | self.killer.stop() 65 | 66 | def connect(self): 67 | super(TcpFuzzer, self).connect() 68 | 69 | # force 1 byte chunks, to increase the probability of 70 | # incomplete writes. 71 | self.gp.transport.chunk_size = 1 72 | 73 | def doFuzz(self): 74 | try: 75 | self.kill_lock.acquire() 76 | 77 | if not self.killer.is_alive(): 78 | self.killer.start() 79 | 80 | if self.killed: 81 | self.connect() 82 | self.prepare() 83 | self.killed = False 84 | 85 | self.blip("*") 86 | finally: 87 | self.kill_lock.release() 88 | 89 | try: 90 | return super(TcpFuzzer, self).doFuzz() 91 | except: 92 | pass #whatever 93 | 94 | return False 95 | 96 | if __name__ == '__main__': 97 | fuzzer = TcpFuzzer() 98 | 99 | fuzzer.run(sys.argv) 100 | -------------------------------------------------------------------------------- /gp/tests/fuzzer_base.py: -------------------------------------------------------------------------------- 1 | from gp.client import Connection 2 | from gp.client import gpException 3 | from test_config import * 4 | import test_config 5 | import os, sys 6 | import random 7 | import time 8 | 9 | test_graph_name = 'test' + str(os.getpid()) 10 | 11 | def fuzz_pick( a ): 12 | i = random.randint(0, len(a)-1) 13 | return a[i] 14 | 15 | 16 | class FuzzerBase (object): # abstract 17 | """Test the TCP client connection""" 18 | 19 | def __init__(self): 20 | self.graph = None 21 | self.useTempGraph = True 22 | 23 | def blip( self, s ): 24 | sys.stdout.write( s ) 25 | sys.stdout.flush() 26 | 27 | def newConnection(self): 28 | gp = Connection.new_client_connection(None, 29 | test_graphserv_host, test_graphserv_port ) 30 | gp.connect() 31 | return gp 32 | 33 | def connect(self): 34 | if not self.graph: 35 | self.graph = test_graph_name 36 | 37 | try: 38 | self.gp = self.newConnection() 39 | except gpException as ex: 40 | print("Unable to connect to " 41 | + test_graphserv_host + ":" + str(test_graphserv_port) 42 | + ", please make sure the graphserv process is running " 43 | + "and check the test_graphserv_host and " 44 | + "test_graphserv_port configuration options in " 45 | + "test_config.py.") 46 | print("Original error: " + str(ex)) 47 | quit(11) 48 | 49 | try: 50 | self.gp.authorize( 'password', 51 | test_admin + ":" + test_admin_password) 52 | except gpException, ex: 53 | print("Unable to connect to authorize as " 54 | + test_admin + ", please check the test_admin and " 55 | + "test_admin_password configuration options in " 56 | + "test_config.py.") 57 | print("Original error: " + str(ex)) 58 | quit(12) 59 | 60 | if self.useTempGraph: 61 | self.gp.try_create_graph( self.graph ) 62 | 63 | try: 64 | self.gp.use_graph( self.graph ) 65 | except gpException, ex: 66 | print("Unable to use graph self.graph, please check the " 67 | + "test_graph_name configuration option in test_config.py " 68 | + "as well as the privileges of user " + test_admin + ".") 69 | print("Original error: " + ex.getMessage()) 70 | quit(13) 71 | 72 | def disconnect(self): 73 | global test_admin, test_admin_password 74 | 75 | if self.useTempGraph and self.graph: 76 | self.gp.try_drop_graph(self.graph) #? gp OK? 77 | 78 | def prepare(self): 79 | pass #noop 80 | 81 | def doFuzz(self): #abstract 82 | raise NotImplementedError( 83 | "FuzzerBase.doFuzz() not implemented.") 84 | 85 | def run(self, argv): 86 | self.connect() 87 | self.prepare() 88 | 89 | n = None 90 | if len(argv) > 1: 91 | n = int(argv[1]) 92 | if not n: 93 | n = 100 94 | 95 | for k in range(n): 96 | for i in range(100): 97 | ok = self.doFuzz() 98 | if ok: 99 | self.blip("+") 100 | else: 101 | self.blip("-") 102 | 103 | self.blip("\n"); 104 | # time.sleep(1) #? Muss wieder rein! 105 | 106 | self.disconnect() 107 | -------------------------------------------------------------------------------- /gp/tests/.svn/text-base/fuzzer_base.py.svn-base: -------------------------------------------------------------------------------- 1 | from gp.client import Connection 2 | from gp.client import gpException 3 | from test_config import * 4 | import test_config 5 | import os, sys 6 | import random 7 | import time 8 | 9 | test_graph_name = 'test' + str(os.getpid()) 10 | 11 | def fuzz_pick( a ): 12 | i = random.randint(0, len(a)-1) 13 | return a[i] 14 | 15 | 16 | class FuzzerBase (object): # abstract 17 | """Test the TCP client connection""" 18 | 19 | def __init__(self): 20 | self.graph = None 21 | self.useTempGraph = True 22 | 23 | def blip( self, s ): 24 | sys.stdout.write( s ) 25 | sys.stdout.flush() 26 | 27 | def newConnection(self): 28 | gp = Connection.new_client_connection(None, 29 | test_graphserv_host, test_graphserv_port ) 30 | gp.connect() 31 | return gp 32 | 33 | def connect(self): 34 | if not self.graph: 35 | self.graph = test_graph_name 36 | 37 | try: 38 | self.gp = self.newConnection() 39 | except gpException as ex: 40 | print("Unable to connect to " 41 | + test_graphserv_host + ":" + str(test_graphserv_port) 42 | + ", please make sure the graphserv process is running " 43 | + "and check the test_graphserv_host and " 44 | + "test_graphserv_port configuration options in " 45 | + "test_config.py.") 46 | print("Original error: " + str(ex)) 47 | quit(11) 48 | 49 | try: 50 | self.gp.authorize( 'password', 51 | test_admin + ":" + test_admin_password) 52 | except gpException, ex: 53 | print("Unable to connect to authorize as " 54 | + test_admin + ", please check the test_admin and " 55 | + "test_admin_password configuration options in " 56 | + "test_config.py.") 57 | print("Original error: " + str(ex)) 58 | quit(12) 59 | 60 | if self.useTempGraph: 61 | self.gp.try_create_graph( self.graph ) 62 | 63 | try: 64 | self.gp.use_graph( self.graph ) 65 | except gpException, ex: 66 | print("Unable to use graph self.graph, please check the " 67 | + "test_graph_name configuration option in test_config.py " 68 | + "as well as the privileges of user " + test_admin + ".") 69 | print("Original error: " + ex.getMessage()) 70 | quit(13) 71 | 72 | def disconnect(self): 73 | global test_admin, test_admin_password 74 | 75 | if self.useTempGraph and self.graph: 76 | self.gp.try_drop_graph(self.graph) #? gp OK? 77 | 78 | def prepare(self): 79 | pass #noop 80 | 81 | def doFuzz(self): #abstract 82 | raise NotImplementedError( 83 | "FuzzerBase.doFuzz() not implemented.") 84 | 85 | def run(self, argv): 86 | self.connect() 87 | self.prepare() 88 | 89 | n = None 90 | if len(argv) > 1: 91 | n = int(argv[1]) 92 | if not n: 93 | n = 100 94 | 95 | for k in range(n): 96 | for i in range(100): 97 | ok = self.doFuzz() 98 | if ok: 99 | self.blip("+") 100 | else: 101 | self.blip("-") 102 | 103 | self.blip("\n"); 104 | # time.sleep(1) #? Muss wieder rein! 105 | 106 | self.disconnect() 107 | -------------------------------------------------------------------------------- /gp/tests/.svn/entries: -------------------------------------------------------------------------------- 1 | 10 2 | 3 | dir 4 | 630 5 | https://svn.toolserver.org/svnroot/daniel/duesenstuff/trunk/gpClient/python/gp/tests 6 | https://svn.toolserver.org/svnroot/daniel 7 | 8 | 9 | 10 | 2012-01-04T20:22:52.801376Z 11 | 584 12 | daniel 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 9f2c43bc-b3c0-43f4-b155-41619b16f219 28 | 29 | __init__.py 30 | file 31 | 32 | 33 | 34 | 35 | 2012-07-27T13:23:46.492858Z 36 | 68b329da9893e34099c7d8ad5cb9c940 37 | 2011-09-16T08:56:04.680087Z 38 | 488 39 | daniel 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 1 62 | 63 | fuzzer_base.py 64 | file 65 | 66 | 67 | 68 | 69 | 2012-07-27T13:23:46.492858Z 70 | fc335301d3e503338c0fc1e19a6bb2df 71 | 2011-12-08T12:26:53.600009Z 72 | 503 73 | daniel 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 3108 96 | 97 | crud_fuzzer.py 98 | file 99 | 100 | 101 | 102 | 103 | 2012-07-27T13:23:46.492858Z 104 | eaedd5d4fb4f6a292faeafed9bb40072 105 | 2011-12-08T12:26:53.600009Z 106 | 503 107 | daniel 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 1111 130 | 131 | mediawiki_test.py 132 | file 133 | 134 | 135 | 136 | 137 | 2012-07-27T13:23:46.492858Z 138 | f4fc3f91e21bc2c435572bf938fa15f5 139 | 2011-09-16T08:56:04.680087Z 140 | 488 141 | daniel 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 18781 164 | 165 | mysql_test.py 166 | file 167 | 168 | 169 | 170 | 171 | 2012-07-27T13:23:46.492858Z 172 | 3a8e8ca82bd2fbf070c10d0cb8164bef 173 | 2012-01-04T20:22:52.801376Z 174 | 584 175 | daniel 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 12534 198 | 199 | test_config.py 200 | file 201 | 202 | 203 | 204 | 205 | 2012-07-27T13:23:46.492858Z 206 | 35588ba14ff5e13128ab307d6bdeebab 207 | 2011-09-14T16:06:08.555694Z 208 | 482 209 | daniel 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 591 232 | 233 | gp.test.data 234 | file 235 | 236 | 237 | 238 | 239 | 2012-07-27T13:23:46.492858Z 240 | eecff6058c548a54df4099a59fa8a963 241 | 2011-09-13T15:27:07.312368Z 242 | 474 243 | daniel 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 28 266 | 267 | server_test.py 268 | file 269 | 270 | 271 | 272 | 273 | 2012-07-27T13:23:46.492858Z 274 | cf91fa51fed2af5a3e50609e46049f3a 275 | 2011-12-16T16:36:00.392792Z 276 | 529 277 | daniel 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | 16228 300 | 301 | core_test.py 302 | file 303 | 304 | 305 | 306 | 307 | 2012-07-27T13:23:46.492858Z 308 | fdb6646a4bf0bc1e856e8e4c60bf8b1c 309 | 2012-01-03T20:10:34.209454Z 310 | 571 311 | daniel 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 5613 334 | 335 | slave_test.py 336 | file 337 | 338 | 339 | 340 | 341 | 2012-07-27T13:23:46.488858Z 342 | 2918ab8f0c7f90b122b5303f3991b3bc 343 | 2011-09-14T16:06:08.555694Z 344 | 482 345 | daniel 346 | 347 | 348 | 349 | 350 | 351 | 352 | 353 | 354 | 355 | 356 | 357 | 358 | 359 | 360 | 361 | 362 | 363 | 364 | 365 | 366 | 367 | 11673 368 | 369 | test_base.py 370 | file 371 | 372 | 373 | 374 | 375 | 2012-07-27T13:23:46.492858Z 376 | 8d2ca90259ba9e77f1524644bdf33365 377 | 2011-09-15T12:38:27.528042Z 378 | 486 379 | daniel 380 | 381 | 382 | 383 | 384 | 385 | 386 | 387 | 388 | 389 | 390 | 391 | 392 | 393 | 394 | 395 | 396 | 397 | 398 | 399 | 400 | 401 | 9694 402 | 403 | arc_fuzzer.py 404 | file 405 | 406 | 407 | 408 | 409 | 2012-07-27T13:23:46.492858Z 410 | 6433b892a7a9dacfe634dd62a0784ef6 411 | 2011-12-08T12:26:53.600009Z 412 | 503 413 | daniel 414 | 415 | 416 | 417 | 418 | 419 | 420 | 421 | 422 | 423 | 424 | 425 | 426 | 427 | 428 | 429 | 430 | 431 | 432 | 433 | 434 | 435 | 1819 436 | 437 | tcp_fuzzer.py 438 | file 439 | 440 | 441 | 442 | 443 | 2012-07-27T13:23:46.492858Z 444 | e2c5e8d0281893c4a7b319a9a9d6790f 445 | 2011-12-08T12:26:53.600009Z 446 | 503 447 | daniel 448 | 449 | 450 | 451 | 452 | 453 | 454 | 455 | 456 | 457 | 458 | 459 | 460 | 461 | 462 | 463 | 464 | 465 | 466 | 467 | 468 | 469 | 2608 470 | 471 | client_test.py 472 | file 473 | 474 | 475 | 476 | 477 | 2012-07-27T13:23:46.492858Z 478 | 2753f6d9fad6ceeb9f027b0c4305c688 479 | 2011-09-13T15:48:54.869004Z 480 | 479 481 | daniel 482 | 483 | 484 | 485 | 486 | 487 | 488 | 489 | 490 | 491 | 492 | 493 | 494 | 495 | 496 | 497 | 498 | 499 | 500 | 501 | 502 | 503 | 638 504 | 505 | -------------------------------------------------------------------------------- /gp/tests/core_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 3 | 4 | import unittest 5 | import os 6 | import tempfile 7 | from test_base import * 8 | from test_config import * 9 | from gp.client import * 10 | 11 | class CoreTest (SlaveTestBase, unittest.TestCase): 12 | """ 13 | Tests core functions via client lib 14 | """ 15 | 16 | #### Core Functions ###############################/ 17 | 18 | def test_addArcs(self): 19 | self.gp.add_arcs( ( 20 | ( 1, 11 ), 21 | ( 1, 12 ), 22 | ( 11, 111 ), 23 | ( 11, 112 ), 24 | ) ) 25 | 26 | self.assertStatsValue( 'ArcCount', 4 ) 27 | 28 | arcs = self.gp.capture_list_successors( 1 ) 29 | 30 | self.assertTrue( TestBase.setEquals( arcs, ( 31 | ( 11, ), 32 | ( 12, ), 33 | ) ), "sucessors of (1)" ) 34 | 35 | arcs = self.gp.capture_list_successors( 11 ) 36 | self.assertTrue( TestBase.setEquals( arcs, ( 37 | ( 111, ), 38 | ( 112, ), 39 | ) ), "sucessors of (2)" ) 40 | 41 | # ------------------------------------------------------ 42 | 43 | self.gp.add_arcs( ( 44 | ( 1, 11 ), 45 | ( 11, 112 ), 46 | ( 2, 21 ), 47 | ) ) 48 | 49 | self.assertStatsValue( 'ArcCount', 5 ) 50 | 51 | arcs = self.gp.capture_list_successors( 2 ) 52 | self.assertTrue( TestBase.setEquals( arcs, ( 53 | ( 21, ), 54 | ) ), "sucessors of (2)" ) 55 | 56 | 57 | 58 | def test_clear(self): 59 | self.gp.add_arcs( ( 60 | ( 1, 11 ), 61 | ( 1, 12 ), 62 | ( 11, 111 ), 63 | ( 11, 112 ), 64 | ) ) 65 | 66 | self.assertStatsValue( 'ArcCount', 4 ) 67 | 68 | self.gp.clear() 69 | 70 | arcs = self.gp.capture_list_successors( 1 ) 71 | 72 | self.assertEmpty( arcs ) 73 | self.assertStatsValue( 'ArcCount', 0 ) 74 | 75 | #-------------------------------------------- 76 | self.gp.add_arcs( ( 77 | ( 1, 11 ), 78 | ( 1, 12 ), 79 | ( 11, 111 ), 80 | ( 11, 112 ), 81 | ) ) 82 | 83 | self.assertStatsValue( 'ArcCount', 4 ) 84 | 85 | 86 | def test_traverseSuccessors(self): 87 | self.gp.add_arcs( ( 88 | ( 1, 11 ), 89 | ( 1, 12 ), 90 | ( 11, 111 ), 91 | ( 11, 112 ), 92 | ( 111, 1111 ), 93 | ( 111, 1112 ), 94 | ( 112, 1121 ), 95 | ) ) 96 | 97 | self.assertStatsValue( 'ArcCount', 7 ) 98 | 99 | #-------------------------------------------- 100 | succ = self.gp.capture_traverse_successors( 11, 5 ) 101 | 102 | self.assertEquals( [ (11,), (111,), (112,), (1111,), (1112,), (1121,), ], succ ) 103 | 104 | 105 | def test_traverseSuccessorsWithout(self): 106 | self.gp.add_arcs( [ 107 | ( 1, 11 ), 108 | ( 1, 12 ), 109 | ( 11, 111 ), 110 | ( 11, 112 ), 111 | ( 111, 1111 ), 112 | ( 111, 1112 ), 113 | ( 112, 1121 ), 114 | ] ) 115 | 116 | self.assertStatsValue( 'ArcCount', 7 ) 117 | 118 | #-------------------------------------------- 119 | succ = self.gp.capture_traverse_successors_without( 11, 5, 111, 5 ) 120 | 121 | self.assertEquals( [ (11,), (112,), (1121,), ], succ ) 122 | 123 | def test_setMeta(self): 124 | #define var 125 | self.gp.set_meta("foo", 1234) 126 | val = self.gp.get_meta_value("foo") 127 | self.assertEquals( "1234", val ) 128 | 129 | #redefine var 130 | self.gp.set_meta("foo", "bla/bla") 131 | val = self.gp.get_meta_value("foo") 132 | self.assertEquals( "bla/bla", val ) 133 | 134 | # test bad ----------------------------------------- 135 | try: 136 | self.gp.set_meta("...", 1234) 137 | self.fail( "exception expected" ) 138 | except gpException as ex: 139 | pass 140 | 141 | try: 142 | self.gp.set_meta("x y", 1234) 143 | self.fail( "exception expected" ) 144 | except gpException as ex: 145 | pass 146 | 147 | try: 148 | self.gp._set_meta(" ", 1234) 149 | self.fail( "exception expected" ) 150 | except gpException as ex: 151 | pass 152 | 153 | try: 154 | self.gp.set_meta("foo", "bla bla") 155 | self.fail( "exception expected" ) 156 | except gpException as ex: 157 | pass 158 | 159 | try: 160 | self.gp.set_meta("foo", "2<3") 161 | self.fail( "exception expected" ) 162 | except gpException as ex: 163 | pass 164 | 165 | def test_getMeta(self): 166 | #get undefined 167 | val = self.gp.try_get_meta_value("foo") 168 | self.assertEquals( False, val ) 169 | 170 | #set var, and get value 171 | self.gp.set_meta("foo", "xxx") 172 | val = self.gp.get_meta_value("foo") 173 | self.assertEquals( "xxx", val ) 174 | 175 | #remove var, then get value 176 | self.gp.remove_meta("foo") 177 | val = self.gp.try_get_meta_value("foo") 178 | self.assertEquals( False, val ) 179 | 180 | def test_removeMeta(self): 181 | #remove undefined 182 | ok = self.gp.try_remove_meta("foo") 183 | self.assertEquals( False, ok ) 184 | 185 | #set var, then remove it 186 | self.gp.set_meta("foo", "xxx") 187 | ok = self.gp.try_remove_meta("foo") 188 | self.assertEquals( "OK", ok ) 189 | 190 | def test_listMeta(self): 191 | # assert empty 192 | meta = self.gp.capture_list_meta() 193 | self.assertEmpty( meta ) 194 | 195 | # add one, assert list 196 | self.gp.set_meta("foo", 1234) 197 | meta = self.gp.capture_list_meta_map() 198 | self.assertEquals( { "foo": 1234}, meta ) 199 | 200 | # remove one, assert empty 201 | self.gp.remove_meta("foo") 202 | meta = self.gp.capture_list_meta() 203 | self.assertEmpty( meta ) 204 | 205 | 206 | #TODO: add all the tests we have in the talkback test suit 207 | 208 | 209 | if __name__ == '__main__': 210 | unittest.main() 211 | 212 | 213 | -------------------------------------------------------------------------------- /gp/tests/.svn/text-base/core_test.py.svn-base: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 3 | 4 | import unittest 5 | import os 6 | import tempfile 7 | from test_base import * 8 | from test_config import * 9 | from gp.client import * 10 | 11 | class CoreTest (SlaveTestBase, unittest.TestCase): 12 | """ 13 | Tests core functions via client lib 14 | """ 15 | 16 | #### Core Functions ###############################/ 17 | 18 | def test_addArcs(self): 19 | self.gp.add_arcs( ( 20 | ( 1, 11 ), 21 | ( 1, 12 ), 22 | ( 11, 111 ), 23 | ( 11, 112 ), 24 | ) ) 25 | 26 | self.assertStatsValue( 'ArcCount', 4 ) 27 | 28 | arcs = self.gp.capture_list_successors( 1 ) 29 | 30 | self.assertTrue( TestBase.setEquals( arcs, ( 31 | ( 11, ), 32 | ( 12, ), 33 | ) ), "sucessors of (1)" ) 34 | 35 | arcs = self.gp.capture_list_successors( 11 ) 36 | self.assertTrue( TestBase.setEquals( arcs, ( 37 | ( 111, ), 38 | ( 112, ), 39 | ) ), "sucessors of (2)" ) 40 | 41 | # ------------------------------------------------------ 42 | 43 | self.gp.add_arcs( ( 44 | ( 1, 11 ), 45 | ( 11, 112 ), 46 | ( 2, 21 ), 47 | ) ) 48 | 49 | self.assertStatsValue( 'ArcCount', 5 ) 50 | 51 | arcs = self.gp.capture_list_successors( 2 ) 52 | self.assertTrue( TestBase.setEquals( arcs, ( 53 | ( 21, ), 54 | ) ), "sucessors of (2)" ) 55 | 56 | 57 | 58 | def test_clear(self): 59 | self.gp.add_arcs( ( 60 | ( 1, 11 ), 61 | ( 1, 12 ), 62 | ( 11, 111 ), 63 | ( 11, 112 ), 64 | ) ) 65 | 66 | self.assertStatsValue( 'ArcCount', 4 ) 67 | 68 | self.gp.clear() 69 | 70 | arcs = self.gp.capture_list_successors( 1 ) 71 | 72 | self.assertEmpty( arcs ) 73 | self.assertStatsValue( 'ArcCount', 0 ) 74 | 75 | #-------------------------------------------- 76 | self.gp.add_arcs( ( 77 | ( 1, 11 ), 78 | ( 1, 12 ), 79 | ( 11, 111 ), 80 | ( 11, 112 ), 81 | ) ) 82 | 83 | self.assertStatsValue( 'ArcCount', 4 ) 84 | 85 | 86 | def test_traverseSuccessors(self): 87 | self.gp.add_arcs( ( 88 | ( 1, 11 ), 89 | ( 1, 12 ), 90 | ( 11, 111 ), 91 | ( 11, 112 ), 92 | ( 111, 1111 ), 93 | ( 111, 1112 ), 94 | ( 112, 1121 ), 95 | ) ) 96 | 97 | self.assertStatsValue( 'ArcCount', 7 ) 98 | 99 | #-------------------------------------------- 100 | succ = self.gp.capture_traverse_successors( 11, 5 ) 101 | 102 | self.assertEquals( [ (11,), (111,), (112,), (1111,), (1112,), (1121,), ], succ ) 103 | 104 | 105 | def test_traverseSuccessorsWithout(self): 106 | self.gp.add_arcs( [ 107 | ( 1, 11 ), 108 | ( 1, 12 ), 109 | ( 11, 111 ), 110 | ( 11, 112 ), 111 | ( 111, 1111 ), 112 | ( 111, 1112 ), 113 | ( 112, 1121 ), 114 | ] ) 115 | 116 | self.assertStatsValue( 'ArcCount', 7 ) 117 | 118 | #-------------------------------------------- 119 | succ = self.gp.capture_traverse_successors_without( 11, 5, 111, 5 ) 120 | 121 | self.assertEquals( [ (11,), (112,), (1121,), ], succ ) 122 | 123 | def test_setMeta(self): 124 | #define var 125 | self.gp.set_meta("foo", 1234) 126 | val = self.gp.get_meta_value("foo") 127 | self.assertEquals( "1234", val ) 128 | 129 | #redefine var 130 | self.gp.set_meta("foo", "bla/bla") 131 | val = self.gp.get_meta_value("foo") 132 | self.assertEquals( "bla/bla", val ) 133 | 134 | # test bad ----------------------------------------- 135 | try: 136 | self.gp.set_meta("...", 1234) 137 | self.fail( "exception expected" ) 138 | except gpException as ex: 139 | pass 140 | 141 | try: 142 | self.gp.set_meta("x y", 1234) 143 | self.fail( "exception expected" ) 144 | except gpException as ex: 145 | pass 146 | 147 | try: 148 | self.gp._set_meta(" ", 1234) 149 | self.fail( "exception expected" ) 150 | except gpException as ex: 151 | pass 152 | 153 | try: 154 | self.gp.set_meta("foo", "bla bla") 155 | self.fail( "exception expected" ) 156 | except gpException as ex: 157 | pass 158 | 159 | try: 160 | self.gp.set_meta("foo", "2<3") 161 | self.fail( "exception expected" ) 162 | except gpException as ex: 163 | pass 164 | 165 | def test_getMeta(self): 166 | #get undefined 167 | val = self.gp.try_get_meta_value("foo") 168 | self.assertEquals( False, val ) 169 | 170 | #set var, and get value 171 | self.gp.set_meta("foo", "xxx") 172 | val = self.gp.get_meta_value("foo") 173 | self.assertEquals( "xxx", val ) 174 | 175 | #remove var, then get value 176 | self.gp.remove_meta("foo") 177 | val = self.gp.try_get_meta_value("foo") 178 | self.assertEquals( False, val ) 179 | 180 | def test_removeMeta(self): 181 | #remove undefined 182 | ok = self.gp.try_remove_meta("foo") 183 | self.assertEquals( False, ok ) 184 | 185 | #set var, then remove it 186 | self.gp.set_meta("foo", "xxx") 187 | ok = self.gp.try_remove_meta("foo") 188 | self.assertEquals( "OK", ok ) 189 | 190 | def test_listMeta(self): 191 | # assert empty 192 | meta = self.gp.capture_list_meta() 193 | self.assertEmpty( meta ) 194 | 195 | # add one, assert list 196 | self.gp.set_meta("foo", 1234) 197 | meta = self.gp.capture_list_meta_map() 198 | self.assertEquals( { "foo": 1234}, meta ) 199 | 200 | # remove one, assert empty 201 | self.gp.remove_meta("foo") 202 | meta = self.gp.capture_list_meta() 203 | self.assertEmpty( meta ) 204 | 205 | 206 | #TODO: add all the tests we have in the talkback test suit 207 | 208 | 209 | if __name__ == '__main__': 210 | unittest.main() 211 | 212 | 213 | -------------------------------------------------------------------------------- /gp/tests/test_base.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os, sys, traceback 3 | from gp.client import * 4 | from test_config import * 5 | 6 | test_graph_name = 'test' + str(os.getpid()) 7 | 8 | def suicide( code = 1 ): 9 | os._exit(code) 10 | 11 | class TestBase: 12 | """A few static methods to compare lists and sets recursively, 13 | so the equality of entire data structures can be asserted. 14 | """ 15 | 16 | @staticmethod 17 | def setContains( a, w ): 18 | """ Checks if a is an element of w. If a is a list, tuple, set or dict, 19 | a recursive comparison is performed. 20 | """ 21 | 22 | found = False 23 | for v in a: 24 | if type(v) in (tuple, list): 25 | try: 26 | if TestBase.arrayEquals( v, w ): 27 | found = True 28 | break 29 | except (TypeError, AttributeError), e: 30 | #perhaps w wasn't iterable 31 | pass 32 | elif type(v) == set: 33 | try: 34 | if TestBase.setEquals( v, w ): 35 | found = True 36 | break 37 | except (TypeError, AttributeError), e: 38 | #perhaps w wasn't iterable 39 | pass 40 | elif type(v) == dict: 41 | raise Exception("deep dictionary comparison not yet implemented") 42 | else: 43 | if v == w: 44 | found = True 45 | 46 | return found 47 | 48 | 49 | @staticmethod 50 | def setEquals( a, b ): 51 | """ determins if a and b contain the same elements. a and b my be sets, 52 | lists or tuples, but do not need to have the same type. 53 | """ 54 | 55 | if len(a) != len(b): 56 | return False 57 | 58 | for v in a: 59 | if not TestBase.setContains(b, v): 60 | return False 61 | 62 | return True 63 | 64 | @staticmethod 65 | def arrayEquals( a, b ): 66 | if len(a) != len(b): 67 | return False 68 | 69 | for k in range(len(a)): 70 | v = a[k] 71 | w = b[k] 72 | 73 | if type(v) in (tuple, list, set): 74 | #WARNING: no protection against circular array references 75 | 76 | try: 77 | if not TestBase.arrayEquals( w, v ): 78 | return False 79 | except TypeError, AttributeError: 80 | #perhaps w wasn't iterable 81 | return False 82 | elif type(v) == dict: 83 | try: 84 | if not TestBase.dictEquals( w, v ): 85 | return False 86 | except TypeError, AttributeError: 87 | #perhaps w wasn't a dict 88 | return False 89 | elif w != v: 90 | return False 91 | 92 | return True 93 | 94 | @staticmethod 95 | def dictEquals( a, b ): 96 | if len(a) != len(b): 97 | return False 98 | 99 | for k, v in a.items(): 100 | if not k in b: 101 | return False 102 | 103 | w = b[k] 104 | 105 | if type(v) in (tuple, list, set): 106 | #WARNING: no protection against circular array references 107 | 108 | try: 109 | if not TestBase.arrayEquals( w, v ): 110 | return False 111 | except TypeError, AttributeError: 112 | #perhaps w wasn't iterable 113 | return False 114 | elif type(v) == dict: 115 | try: 116 | if not TestBase.dictEquals( w, v ): 117 | return False 118 | except TypeError, AttributeError: 119 | #perhaps w wasn't a dict 120 | return False 121 | elif w != v: 122 | return False 123 | 124 | return True 125 | 126 | 127 | class ConnectionTestBase(TestBase): 128 | """Abstract base class with basic Connection tests. 129 | 130 | These need to pass for all types of connections. 131 | @Note: lib functionality like try and capture is tested in 132 | SlaveTest, because it doesn't need to be tested separately 133 | for all types of connections 134 | 135 | """ 136 | 137 | def setUp(self): #abstract 138 | raise NotImplementedError('subclasses must override setUp() to' 139 | + ' store a Connection in self.gp') 140 | 141 | def tearDown(self): 142 | if self.gp: 143 | self.gp.close() 144 | 145 | def test_ping(self): 146 | pong = self.gp.ping() 147 | #? nix assert? 148 | 149 | def test_stats(self): 150 | stats = self.gp.capture_stats() 151 | stats = pairs2map(stats) 152 | self.assertEqual(stats['ArcCount'], 0, "arc count should be zero") 153 | 154 | def test_dataSetHandling(self): 155 | self.gp.add_arcs((( 1, 11 ),( 1, 12 ),( 11, 111 ),( 11, 112 ),)) 156 | self.assertStatus('OK') 157 | self.assertStatsValue('ArcCount', 4) 158 | arcs = self.gp.capture_list_successors(1) 159 | self.assertTrue(ConnectionTestBase.setEquals( 160 | arcs, [(11,), (12,),]), "sucessors of (1)" ) 161 | arcs = self.gp.capture_list_successors(11) 162 | self.assertTrue(ConnectionTestBase.setEquals( 163 | arcs, [(111,), (112,),]), "sucessors of (2)" ) 164 | 165 | 166 | #### utility ###################################################### 167 | def assertNone(self, value, msg = None): 168 | if value is not None: 169 | if msg is None: 170 | msg = "expected None, found %s" % value 171 | 172 | self.fail(msg) 173 | 174 | def assertEmpty(self, value, msg = None): 175 | if value: 176 | if msg is None: 177 | msg = "expected value to be empty, found %s" % value 178 | 179 | self.fail(msg) 180 | 181 | def assertNotNone(self, value, msg = None): 182 | if value is None: 183 | if msg is None: 184 | msg = "found None where not expected" 185 | 186 | self.fail(msg) 187 | 188 | def assertContains(self, k, array, msg = None): 189 | if not k in array: 190 | if msg is None: 191 | msg = "Key %s not found in %s" % (k, array) 192 | 193 | self.fail(msg) 194 | 195 | def assertStatsValue(self, field, value): 196 | stats = self.gp.capture_stats() 197 | stats = pairs2map(stats) 198 | self.assertEquals(value, stats[field], "status[" + field + "]") 199 | 200 | def assertSessionValue(self, field, value): 201 | stats = self.gp.capture_session_info() 202 | stats = pairs2map(stats) 203 | self.assertEquals(value, stats[field], "session_info[" + field + "]") 204 | 205 | def assertStatus(self, value, message=None): 206 | status = self.gp.getStatus() 207 | self.assertEquals(value, status, message) 208 | 209 | 210 | class SlaveTestBase(ConnectionTestBase): #abstract 211 | 212 | def setUp(self): 213 | self.dump = PipeSink(sys.stdout) 214 | 215 | try: 216 | self.gp = Connection.new_slave_connection(test_graphcore_path) 217 | self.gp.connect() 218 | except gpException, ex: 219 | print("Unable to launch graphcore instance from " 220 | + "%s, please make sure graphcore is " % test_graphcore_path 221 | + "installed and check the test_graphcore_path " 222 | + "configuration options in test_config.py.") 223 | print("Original error: " + str(ex)) 224 | traceback.print_exc(); 225 | suicide(10) 226 | 227 | 228 | class ClientTestBase(ConnectionTestBase): #abstract 229 | 230 | def setUp(self): 231 | try: 232 | self.gp = self.newConnection() 233 | except gpException, ex: 234 | print("Unable to connect to " 235 | + "%s:%s, please make sure " % (test_graphserv_host, test_graphserv_port) 236 | + "the graphserv process is running and check the " 237 | + "test_graphserv_host and test_graphserv_port configuration " 238 | + "options in test_config.py.") 239 | print("Original error: " + str(ex)) 240 | traceback.print_exc(); 241 | suicide(11) 242 | try: 243 | self.gp.authorize( 244 | 'password', test_admin + ":" + test_admin_password) 245 | except gpException, ex: 246 | print("Unable to connect to authorize as %s " % test_admin 247 | + ", please check the test_admin and test_admin_password " 248 | + "configuration options in test_config.py.") 249 | print("Original error: " + str(ex)) 250 | traceback.print_exc(); 251 | suicide(12) 252 | try: 253 | self.gp.create_graph(test_graph_name) 254 | except gpException, ex: 255 | print("Unable to create graph %s" % test_graph_name 256 | + ", please check the test_graph_name configuration option " 257 | + "in test_config.py as well as the privileges of user " 258 | + test_admin + ".") 259 | print("Original error: " + str(ex)) 260 | traceback.print_exc(); 261 | suicide(13) 262 | 263 | self.gp.use_graph(test_graph_name) 264 | # if use_graph throws an error, let it rip. it really shouldn't 265 | # happen and it's not a confiugration problem 266 | 267 | def newConnection(self): 268 | gp = Connection.new_client_connection( 269 | None, test_graphserv_host, test_graphserv_port) 270 | gp.connect() 271 | return gp 272 | 273 | def tearDown(self): 274 | try: 275 | self.gp.drop_graph(test_graph_name) 276 | except gpProtocolException, ex: 277 | #failed to remove graph, maybe the connection is gone? try again. 278 | try: 279 | gp = self.newConnection() 280 | gp.authorize('password', 281 | test_admin + ":" + test_admin_password) 282 | gp.drop_graph(test_graph_name) 283 | except gpException, ex: 284 | # just give up 285 | # pass 286 | raise ex 287 | 288 | ConnectionTestBase.tearDown(self) 289 | 290 | -------------------------------------------------------------------------------- /gp/tests/.svn/text-base/test_base.py.svn-base: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os, sys, traceback 3 | from gp.client import * 4 | from test_config import * 5 | 6 | test_graph_name = 'test' + str(os.getpid()) 7 | 8 | def suicide( code = 1 ): 9 | os._exit(code) 10 | 11 | class TestBase: 12 | """A few static methods to compare lists and sets recursively, 13 | so the equality of entire data structures can be asserted. 14 | """ 15 | 16 | @staticmethod 17 | def setContains( a, w ): 18 | """ Checks if a is an element of w. If a is a list, tuple, set or dict, 19 | a recursive comparison is performed. 20 | """ 21 | 22 | found = False 23 | for v in a: 24 | if type(v) in (tuple, list): 25 | try: 26 | if TestBase.arrayEquals( v, w ): 27 | found = True 28 | break 29 | except (TypeError, AttributeError), e: 30 | #perhaps w wasn't iterable 31 | pass 32 | elif type(v) == set: 33 | try: 34 | if TestBase.setEquals( v, w ): 35 | found = True 36 | break 37 | except (TypeError, AttributeError), e: 38 | #perhaps w wasn't iterable 39 | pass 40 | elif type(v) == dict: 41 | raise Exception("deep dictionary comparison not yet implemented") 42 | else: 43 | if v == w: 44 | found = True 45 | 46 | return found 47 | 48 | 49 | @staticmethod 50 | def setEquals( a, b ): 51 | """ determins if a and b contain the same elements. a and b my be sets, 52 | lists or tuples, but do not need to have the same type. 53 | """ 54 | 55 | if len(a) != len(b): 56 | return False 57 | 58 | for v in a: 59 | if not TestBase.setContains(b, v): 60 | return False 61 | 62 | return True 63 | 64 | @staticmethod 65 | def arrayEquals( a, b ): 66 | if len(a) != len(b): 67 | return False 68 | 69 | for k in range(len(a)): 70 | v = a[k] 71 | w = b[k] 72 | 73 | if type(v) in (tuple, list, set): 74 | #WARNING: no protection against circular array references 75 | 76 | try: 77 | if not TestBase.arrayEquals( w, v ): 78 | return False 79 | except TypeError, AttributeError: 80 | #perhaps w wasn't iterable 81 | return False 82 | elif type(v) == dict: 83 | try: 84 | if not TestBase.dictEquals( w, v ): 85 | return False 86 | except TypeError, AttributeError: 87 | #perhaps w wasn't a dict 88 | return False 89 | elif w != v: 90 | return False 91 | 92 | return True 93 | 94 | @staticmethod 95 | def dictEquals( a, b ): 96 | if len(a) != len(b): 97 | return False 98 | 99 | for k, v in a.items(): 100 | if not k in b: 101 | return False 102 | 103 | w = b[k] 104 | 105 | if type(v) in (tuple, list, set): 106 | #WARNING: no protection against circular array references 107 | 108 | try: 109 | if not TestBase.arrayEquals( w, v ): 110 | return False 111 | except TypeError, AttributeError: 112 | #perhaps w wasn't iterable 113 | return False 114 | elif type(v) == dict: 115 | try: 116 | if not TestBase.dictEquals( w, v ): 117 | return False 118 | except TypeError, AttributeError: 119 | #perhaps w wasn't a dict 120 | return False 121 | elif w != v: 122 | return False 123 | 124 | return True 125 | 126 | 127 | class ConnectionTestBase(TestBase): 128 | """Abstract base class with basic Connection tests. 129 | 130 | These need to pass for all types of connections. 131 | @Note: lib functionality like try and capture is tested in 132 | SlaveTest, because it doesn't need to be tested separately 133 | for all types of connections 134 | 135 | """ 136 | 137 | def setUp(self): #abstract 138 | raise NotImplementedError('subclasses must override setUp() to' 139 | + ' store a Connection in self.gp') 140 | 141 | def tearDown(self): 142 | if self.gp: 143 | self.gp.close() 144 | 145 | def test_ping(self): 146 | pong = self.gp.ping() 147 | #? nix assert? 148 | 149 | def test_stats(self): 150 | stats = self.gp.capture_stats() 151 | stats = pairs2map(stats) 152 | self.assertEqual(stats['ArcCount'], 0, "arc count should be zero") 153 | 154 | def test_dataSetHandling(self): 155 | self.gp.add_arcs((( 1, 11 ),( 1, 12 ),( 11, 111 ),( 11, 112 ),)) 156 | self.assertStatus('OK') 157 | self.assertStatsValue('ArcCount', 4) 158 | arcs = self.gp.capture_list_successors(1) 159 | self.assertTrue(ConnectionTestBase.setEquals( 160 | arcs, [(11,), (12,),]), "sucessors of (1)" ) 161 | arcs = self.gp.capture_list_successors(11) 162 | self.assertTrue(ConnectionTestBase.setEquals( 163 | arcs, [(111,), (112,),]), "sucessors of (2)" ) 164 | 165 | 166 | #### utility ###################################################### 167 | def assertNone(self, value, msg = None): 168 | if value is not None: 169 | if msg is None: 170 | msg = "expected None, found %s" % value 171 | 172 | self.fail(msg) 173 | 174 | def assertEmpty(self, value, msg = None): 175 | if value: 176 | if msg is None: 177 | msg = "expected value to be empty, found %s" % value 178 | 179 | self.fail(msg) 180 | 181 | def assertNotNone(self, value, msg = None): 182 | if value is None: 183 | if msg is None: 184 | msg = "found None where not expected" 185 | 186 | self.fail(msg) 187 | 188 | def assertContains(self, k, array, msg = None): 189 | if not k in array: 190 | if msg is None: 191 | msg = "Key %s not found in %s" % (k, array) 192 | 193 | self.fail(msg) 194 | 195 | def assertStatsValue(self, field, value): 196 | stats = self.gp.capture_stats() 197 | stats = pairs2map(stats) 198 | self.assertEquals(value, stats[field], "status[" + field + "]") 199 | 200 | def assertSessionValue(self, field, value): 201 | stats = self.gp.capture_session_info() 202 | stats = pairs2map(stats) 203 | self.assertEquals(value, stats[field], "session_info[" + field + "]") 204 | 205 | def assertStatus(self, value, message=None): 206 | status = self.gp.getStatus() 207 | self.assertEquals(value, status, message) 208 | 209 | 210 | class SlaveTestBase(ConnectionTestBase): #abstract 211 | 212 | def setUp(self): 213 | self.dump = PipeSink(sys.stdout) 214 | 215 | try: 216 | self.gp = Connection.new_slave_connection(test_graphcore_path) 217 | self.gp.connect() 218 | except gpException, ex: 219 | print("Unable to launch graphcore instance from " 220 | + "%s, please make sure graphcore is " % test_graphcore_path 221 | + "installed and check the test_graphcore_path " 222 | + "configuration options in test_config.py.") 223 | print("Original error: " + str(ex)) 224 | traceback.print_exc(); 225 | suicide(10) 226 | 227 | 228 | class ClientTestBase(ConnectionTestBase): #abstract 229 | 230 | def setUp(self): 231 | try: 232 | self.gp = self.newConnection() 233 | except gpException, ex: 234 | print("Unable to connect to " 235 | + "%s:%s, please make sure " % (test_graphserv_host, test_graphserv_port) 236 | + "the graphserv process is running and check the " 237 | + "test_graphserv_host and test_graphserv_port configuration " 238 | + "options in test_config.py.") 239 | print("Original error: " + str(ex)) 240 | traceback.print_exc(); 241 | suicide(11) 242 | try: 243 | self.gp.authorize( 244 | 'password', test_admin + ":" + test_admin_password) 245 | except gpException, ex: 246 | print("Unable to connect to authorize as %s " % test_admin 247 | + ", please check the test_admin and test_admin_password " 248 | + "configuration options in test_config.py.") 249 | print("Original error: " + str(ex)) 250 | traceback.print_exc(); 251 | suicide(12) 252 | try: 253 | self.gp.create_graph(test_graph_name) 254 | except gpException, ex: 255 | print("Unable to create graph %s" % test_graph_name 256 | + ", please check the test_graph_name configuration option " 257 | + "in test_config.py as well as the privileges of user " 258 | + test_admin + ".") 259 | print("Original error: " + str(ex)) 260 | traceback.print_exc(); 261 | suicide(13) 262 | 263 | self.gp.use_graph(test_graph_name) 264 | # if use_graph throws an error, let it rip. it really shouldn't 265 | # happen and it's not a confiugration problem 266 | 267 | def newConnection(self): 268 | gp = Connection.new_client_connection( 269 | None, test_graphserv_host, test_graphserv_port) 270 | gp.connect() 271 | return gp 272 | 273 | def tearDown(self): 274 | try: 275 | self.gp.drop_graph(test_graph_name) 276 | except gpProtocolException, ex: 277 | #failed to remove graph, maybe the connection is gone? try again. 278 | try: 279 | gp = self.newConnection() 280 | gp.authorize('password', 281 | test_admin + ":" + test_admin_password) 282 | gp.drop_graph(test_graph_name) 283 | except gpException, ex: 284 | # just give up 285 | # pass 286 | raise ex 287 | 288 | ConnectionTestBase.tearDown(self) 289 | 290 | -------------------------------------------------------------------------------- /gp/tests/slave_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 3 | 4 | import unittest 5 | import tempfile 6 | from test_base import * 7 | from gp.client import * 8 | 9 | 10 | def read_lines( filepath ): 11 | f = open(filepath, 'ra') 12 | lines = [ s for s in f ] 13 | 14 | f.close() 15 | return lines 16 | 17 | 18 | # Tests a connection to a slave process, as well as general client 19 | # lib functionality. 20 | 21 | class SlaveTest (SlaveTestBase, unittest.TestCase): 22 | """Client Lib Functions 23 | 24 | Tested here, not in ConnectionTestBase, because we only need to test 25 | is once, not for every type of connection 26 | 27 | @TODO: test getStatusMessage, isClosed, etc 28 | 29 | """ 30 | 31 | def test_try(self): 32 | x = self.gp.try_foo() 33 | self.assertFalse(x) 34 | self.assertEquals('FAILED', self.gp.getStatus()) 35 | 36 | def test_capture(self): 37 | """empty data""" 38 | a = self.gp.capture_list_roots() 39 | self.assertStatus( 'OK' ) 40 | self.assertNotNone( a ) 41 | self.assertIsInstance(a, (list,tuple)) 42 | 43 | self.assertEquals( 0, len(a), 44 | "number of items in the result should be 0!" ) 45 | 46 | # single column data 47 | self.gp.add_arcs(((1, 11 ),(1, 12 ), (11, 111 ), (11, 112 ),)) 48 | a = self.gp.capture_list_successors( 1 ) 49 | self.assertStatus( 'OK' ) 50 | self.assertNotNone( a ) 51 | self.assertIsInstance(a, (list,tuple)) 52 | 53 | a = array_column( a, 0 ) 54 | self.assertEquals( [11, 12], a, 55 | "unexpected response for list_successors(1): %s" % a ) 56 | 57 | # two column data 58 | a = self.gp.capture_stats() 59 | self.assertNotNone( a ) 60 | 61 | a = pairs2map( a ) 62 | self.assertContains( 'ArcCount', a, 63 | "contents: %s" % a ) 64 | self.assertEquals( a['ArcCount'], 4 ) 65 | 66 | # two column data as map 67 | a = self.gp.capture_stats_map() 68 | self.assertContains( 'ArcCount', a, 69 | "contents: %s" % a ) 70 | self.assertEquals( a['ArcCount'], 4 ) 71 | 72 | # capture none 73 | a = self.gp.capture_traverse_successors( 77, 5 ) 74 | self.assertStatus( 'NONE' ) 75 | self.assertNone( a ) 76 | 77 | # capture on command with no output 78 | a = self.gp.capture_clear() 79 | self.assertStatus( 'OK' ) 80 | self.assertTrue( a ) 81 | 82 | # capture throwing error 83 | try: 84 | x = self.gp.capture_foo() 85 | self.fail("capturing output of an unknown command should " 86 | + "trigger an error") 87 | except gpProcessorException, e: 88 | # this is the expected outcome: the connection is closed. 89 | pass 90 | 91 | # capture with try 92 | x = self.gp.try_capture_foo() 93 | # should not trigger an exception... 94 | 95 | def dummyCallHandler(self, gp, params): 96 | if ( params['command'] == 'dummy' ): 97 | params['result'] = "test" 98 | return False 99 | return True 100 | 101 | 102 | def test_callHandler(self): 103 | h = self.dummyCallHandler 104 | self.gp.addCallHandler(h) 105 | 106 | st = self.gp.capture_stats() 107 | 108 | if st == False: 109 | self.fail("capture_stats failed and returned False") 110 | elif st == True: 111 | self.fail("capture_stats failed to capture, returned True") 112 | 113 | self.assertTrue(type(st) in (list, tuple), 114 | 'capture_stats is expected to return a list, got %s!' % type(st)) 115 | 116 | x = self.gp.dummy() 117 | self.assertEquals('test', x) 118 | 119 | 120 | def assertCommandAccepted(self, cmd, src=None, sink=None): 121 | s = re.sub('/\s+/s', ' ', str( cmd )) 122 | 123 | try: 124 | x = self.gp.execute(cmd, src, sink) 125 | raise Exception( 126 | "dummy command should have failed in core: %s" % s) 127 | except gpUsageException, e: 128 | self.fail("command syntax should be accepted by client: %s" % s) 129 | except gpProcessorException, e: 130 | # ok, should fail in core, but be accepted by client side validator 131 | pass 132 | 133 | def assertCommandRejected(self, cmd, src=None, sink=None): 134 | s = re.sub('/\s+/s', ' ', str(cmd)) 135 | 136 | try: 137 | x = self.gp.execute(cmd, src, sink) 138 | self.fail("bad command should be detected: %s" % s) 139 | except gpUsageException, e: 140 | pass # ok 141 | except gpProcessorException, e: 142 | self.fail( 143 | ( "bad command should have been detected by the client: %s" 144 | + "; core message: %s" ) % (s, e.getMessage()) ) 145 | 146 | def test_commandValidation(self): 147 | self.assertCommandRejected( '' ) 148 | self.assertCommandRejected(('', 23) ) 149 | 150 | self.assertCommandRejected( None ) 151 | self.assertCommandRejected((None, 23) ) 152 | 153 | self.assertCommandRejected( False ) 154 | self.assertCommandRejected((False, 23) ) 155 | 156 | self.assertCommandRejected(() ) 157 | self.assertCommandRejected((('foo',), ) ) 158 | 159 | self.assertCommandRejected( '123' ) 160 | self.assertCommandRejected(('123',) ) 161 | 162 | self.assertCommandAccepted( ' x ' ) 163 | self.assertCommandRejected((' x ',) ) 164 | 165 | self.assertCommandRejected( 'y' ) 166 | self.assertCommandRejected((' y ',) ) 167 | 168 | self.assertCommandRejected(('a:b') ) 169 | # 'a:b' is legal as an argument, but nut as a command name! 170 | 171 | self.assertCommandAccepted( 'x' ) 172 | self.assertCommandAccepted(('x',) ) 173 | 174 | self.assertCommandAccepted( 'xyz' ) 175 | self.assertCommandAccepted(('xyz',) ) 176 | 177 | self.assertCommandAccepted( 'x7y' ) 178 | self.assertCommandAccepted(('x7y',) ) 179 | 180 | chars = "\r\n\t\0\x09^\"§\$%/()[]\{\}=?'`\\*+~.,;@\xDD" 181 | for ch in chars: 182 | s = "a " + ch + " b" 183 | 184 | self.assertCommandRejected( s ) 185 | self.assertCommandRejected((s,) ) 186 | 187 | 188 | chars = " !&<>|#:" 189 | for ch in chars: 190 | s = "a " + ch + " b" 191 | 192 | self.assertCommandRejected((s,) ) 193 | 194 | 195 | # operators ----------------------------------------- 196 | self.assertCommandAccepted( 'clear && clear' ) 197 | self.assertCommandAccepted( 'clear !&& clear' ) 198 | 199 | 200 | # pipes disallowed ----------------------------------------- 201 | self.gp.allowPipes = False 202 | 203 | self.assertCommandRejected( 'clear > /tmp/test' ) 204 | self.assertCommandRejected( 'clear < /tmp/test' ) 205 | 206 | # pipes allowed ----------------------------------------- 207 | self.gp.allowPipes = True 208 | 209 | self.assertCommandAccepted( 'clear > /tmp/test' ) 210 | self.assertCommandAccepted( 'clear < /tmp/test' ) 211 | 212 | # pipes conflict ----------------------------------------- 213 | self.assertCommandRejected( 'clear > /tmp/test', None, 214 | NullSink.instance ) 215 | self.assertCommandRejected( 'clear < /tmp/test', 216 | NullSource.instance, None ) 217 | 218 | 219 | def assertArgumentAccepted(self, arg ): 220 | s = re.sub('/\s+/s', ' ', str(arg)) 221 | 222 | try: 223 | x = self.gp.execute(('foo', arg) ) 224 | raise Exception("dummy command should have failed in core: " 225 | + "foo %s" % s) 226 | except gpUsageException, e: 227 | self.fail("argument should be accepted by client: %s" % s) 228 | except gpProcessorException, e: 229 | pass 230 | # ok, should fail in core, but be accepted by client side 231 | # validator 232 | 233 | def assertArgumentRejected( self, arg ): 234 | s = re.sub('/\s+/s', ' ', str(arg)) 235 | try: 236 | x = self.gp.execute(('foo', arg) ) 237 | self.fail("malformed argument should be detected: %s" % s) 238 | except gpUsageException, e: 239 | pass 240 | # ok 241 | except gpProcessorException, e: 242 | self.fail("malformed argument should have been detected " 243 | + "by the client: %s; core message: %s" 244 | % (s, e.getMessage()) ) 245 | 246 | 247 | def test_argumentValidation(self): 248 | self.assertArgumentRejected( '' ) 249 | self.assertArgumentRejected( None ) 250 | self.assertArgumentRejected( False ) 251 | self.assertArgumentRejected( ' x ' ) 252 | 253 | # self.gp.setTimeout(2); # has no effect for pipes 254 | self.assertArgumentAccepted( 'x:y' ) 255 | # needed for password auth! 256 | # NOTE: This is broken in graphcore (but works via graphserv)! 257 | 258 | self.assertArgumentAccepted( '123' ) 259 | self.assertArgumentAccepted( 'x' ) 260 | self.assertArgumentAccepted( 'xyz' ) 261 | self.assertArgumentAccepted( 'x7y' ) 262 | self.assertArgumentAccepted( '7x7' ) 263 | 264 | chars = " \r\n\t\0\x09^!\"§\$%&/()[]\ \ =?'#`\\*+~., ;<>|@\xDD" 265 | for ch in chars: 266 | s = "a " + ch + " b" 267 | 268 | self.assertArgumentRejected(s) 269 | 270 | 271 | 272 | # // Client Lib I/O /////////////////////////////////////////////////////////////// 273 | # Tested here, not in ConnectionTestBase, because we only need to test is once, not for every type of connection 274 | # Note: ArraySource and ArraySink are used implicitly all the time in the tests, no need to test them separately. 275 | 276 | def test_fileSource(self): 277 | f = os.path.dirname(os.path.abspath(__file__)) + '/gp.test.data' 278 | src = FileSource(f) 279 | 280 | self.gp.add_arcs( src ) 281 | 282 | self.assertStatus( 'OK' ) 283 | self.assertStatsValue( 'ArcCount', 4 ) 284 | 285 | arcs = self.gp.capture_list_successors( 1 ) 286 | 287 | self.assertTrue( ConnectionTestBase.setEquals( 288 | arcs, [(11, ), (12, ),]), "sucessors of (1): expected [(11,), (12,)], got %s" % arcs ) 289 | 290 | arcs = self.gp.capture_list_successors( 11 ) 291 | self.assertTrue( ConnectionTestBase.setEquals( 292 | arcs, [(111, ), (112, ),]), "sucessors of (2): expected [(111,), (112,)], got %s" % arcs ) 293 | 294 | 295 | def test_fileSink(self): 296 | 297 | # set up the sink 298 | f = tempfile.mktemp(suffix='gpt') 299 | sink = FileSink(f, False, "\n") 300 | 301 | # generate output 302 | self.gp.add_arcs(((1, 11 ), (1, 12 ), (11, 111 ), (11, 112 ),)) 303 | 304 | ok = self.gp.traverse_successors(1, 2, sink) 305 | sink.close() 306 | 307 | # make sure we can read the file 308 | self.assertStatus('OK') 309 | self.assertEquals('OK', ok) 310 | 311 | # compare actual file contents 312 | rows = read_lines(f) 313 | 314 | self.assertNotEquals(False, rows, 315 | "could not get file contents of %s" % f) 316 | self.assertNotNone(rows, "could not get file contents of %s " % f) 317 | 318 | expected =("1\n", "11\n", "12\n", "111\n", "112\n",) 319 | self.assertTrue( ConnectionTestBase.setEquals( 320 | expected, rows), 'bad content in outfile: %s, expected %s' 321 | % ( rows, expected ) ) 322 | 323 | #cleanup 324 | try: 325 | unlink(f) 326 | except: 327 | pass 328 | 329 | def test_nullSource(self): 330 | self.gp.add_arcs( NullSource.instance ) 331 | self.assertStatus( 'OK' ) 332 | 333 | 334 | def test_nullSink(self): 335 | # generate output 336 | self.gp.add_arcs(((1, 11 ), (1, 12 ), (11, 111 ), (11, 112 ),)) 337 | 338 | ok = self.gp.traverse_successors(1, 2, NullSink.instance) 339 | self.assertStatus('OK') 340 | 341 | 342 | # //// Slave Connection Tests /////////////////////////////////////////////////// 343 | # currently none. could check if the process really dies after quit, etc 344 | # TODO: test checkPeer, etc 345 | 346 | if __name__ == '__main__': 347 | unittest.main() 348 | -------------------------------------------------------------------------------- /gp/tests/.svn/text-base/slave_test.py.svn-base: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 3 | 4 | import unittest 5 | import tempfile 6 | from test_base import * 7 | from gp.client import * 8 | 9 | 10 | def read_lines( filepath ): 11 | f = open(filepath, 'ra') 12 | lines = [ s for s in f ] 13 | 14 | f.close() 15 | return lines 16 | 17 | 18 | # Tests a connection to a slave process, as well as general client 19 | # lib functionality. 20 | 21 | class SlaveTest (SlaveTestBase, unittest.TestCase): 22 | """Client Lib Functions 23 | 24 | Tested here, not in ConnectionTestBase, because we only need to test 25 | is once, not for every type of connection 26 | 27 | @TODO: test getStatusMessage, isClosed, etc 28 | 29 | """ 30 | 31 | def test_try(self): 32 | x = self.gp.try_foo() 33 | self.assertFalse(x) 34 | self.assertEquals('FAILED', self.gp.getStatus()) 35 | 36 | def test_capture(self): 37 | """empty data""" 38 | a = self.gp.capture_list_roots() 39 | self.assertStatus( 'OK' ) 40 | self.assertNotNone( a ) 41 | self.assertIsInstance(a, (list,tuple)) 42 | 43 | self.assertEquals( 0, len(a), 44 | "number of items in the result should be 0!" ) 45 | 46 | # single column data 47 | self.gp.add_arcs(((1, 11 ),(1, 12 ), (11, 111 ), (11, 112 ),)) 48 | a = self.gp.capture_list_successors( 1 ) 49 | self.assertStatus( 'OK' ) 50 | self.assertNotNone( a ) 51 | self.assertIsInstance(a, (list,tuple)) 52 | 53 | a = array_column( a, 0 ) 54 | self.assertEquals( [11, 12], a, 55 | "unexpected response for list_successors(1): %s" % a ) 56 | 57 | # two column data 58 | a = self.gp.capture_stats() 59 | self.assertNotNone( a ) 60 | 61 | a = pairs2map( a ) 62 | self.assertContains( 'ArcCount', a, 63 | "contents: %s" % a ) 64 | self.assertEquals( a['ArcCount'], 4 ) 65 | 66 | # two column data as map 67 | a = self.gp.capture_stats_map() 68 | self.assertContains( 'ArcCount', a, 69 | "contents: %s" % a ) 70 | self.assertEquals( a['ArcCount'], 4 ) 71 | 72 | # capture none 73 | a = self.gp.capture_traverse_successors( 77, 5 ) 74 | self.assertStatus( 'NONE' ) 75 | self.assertNone( a ) 76 | 77 | # capture on command with no output 78 | a = self.gp.capture_clear() 79 | self.assertStatus( 'OK' ) 80 | self.assertTrue( a ) 81 | 82 | # capture throwing error 83 | try: 84 | x = self.gp.capture_foo() 85 | self.fail("capturing output of an unknown command should " 86 | + "trigger an error") 87 | except gpProcessorException, e: 88 | # this is the expected outcome: the connection is closed. 89 | pass 90 | 91 | # capture with try 92 | x = self.gp.try_capture_foo() 93 | # should not trigger an exception... 94 | 95 | def dummyCallHandler(self, gp, params): 96 | if ( params['command'] == 'dummy' ): 97 | params['result'] = "test" 98 | return False 99 | return True 100 | 101 | 102 | def test_callHandler(self): 103 | h = self.dummyCallHandler 104 | self.gp.addCallHandler(h) 105 | 106 | st = self.gp.capture_stats() 107 | 108 | if st == False: 109 | self.fail("capture_stats failed and returned False") 110 | elif st == True: 111 | self.fail("capture_stats failed to capture, returned True") 112 | 113 | self.assertTrue(type(st) in (list, tuple), 114 | 'capture_stats is expected to return a list, got %s!' % type(st)) 115 | 116 | x = self.gp.dummy() 117 | self.assertEquals('test', x) 118 | 119 | 120 | def assertCommandAccepted(self, cmd, src=None, sink=None): 121 | s = re.sub('/\s+/s', ' ', str( cmd )) 122 | 123 | try: 124 | x = self.gp.execute(cmd, src, sink) 125 | raise Exception( 126 | "dummy command should have failed in core: %s" % s) 127 | except gpUsageException, e: 128 | self.fail("command syntax should be accepted by client: %s" % s) 129 | except gpProcessorException, e: 130 | # ok, should fail in core, but be accepted by client side validator 131 | pass 132 | 133 | def assertCommandRejected(self, cmd, src=None, sink=None): 134 | s = re.sub('/\s+/s', ' ', str(cmd)) 135 | 136 | try: 137 | x = self.gp.execute(cmd, src, sink) 138 | self.fail("bad command should be detected: %s" % s) 139 | except gpUsageException, e: 140 | pass # ok 141 | except gpProcessorException, e: 142 | self.fail( 143 | ( "bad command should have been detected by the client: %s" 144 | + "; core message: %s" ) % (s, e.getMessage()) ) 145 | 146 | def test_commandValidation(self): 147 | self.assertCommandRejected( '' ) 148 | self.assertCommandRejected(('', 23) ) 149 | 150 | self.assertCommandRejected( None ) 151 | self.assertCommandRejected((None, 23) ) 152 | 153 | self.assertCommandRejected( False ) 154 | self.assertCommandRejected((False, 23) ) 155 | 156 | self.assertCommandRejected(() ) 157 | self.assertCommandRejected((('foo',), ) ) 158 | 159 | self.assertCommandRejected( '123' ) 160 | self.assertCommandRejected(('123',) ) 161 | 162 | self.assertCommandAccepted( ' x ' ) 163 | self.assertCommandRejected((' x ',) ) 164 | 165 | self.assertCommandRejected( 'y' ) 166 | self.assertCommandRejected((' y ',) ) 167 | 168 | self.assertCommandRejected(('a:b') ) 169 | # 'a:b' is legal as an argument, but nut as a command name! 170 | 171 | self.assertCommandAccepted( 'x' ) 172 | self.assertCommandAccepted(('x',) ) 173 | 174 | self.assertCommandAccepted( 'xyz' ) 175 | self.assertCommandAccepted(('xyz',) ) 176 | 177 | self.assertCommandAccepted( 'x7y' ) 178 | self.assertCommandAccepted(('x7y',) ) 179 | 180 | chars = "\r\n\t\0\x09^\"§\$%/()[]\{\}=?'`\\*+~.,;@\xDD" 181 | for ch in chars: 182 | s = "a " + ch + " b" 183 | 184 | self.assertCommandRejected( s ) 185 | self.assertCommandRejected((s,) ) 186 | 187 | 188 | chars = " !&<>|#:" 189 | for ch in chars: 190 | s = "a " + ch + " b" 191 | 192 | self.assertCommandRejected((s,) ) 193 | 194 | 195 | # operators ----------------------------------------- 196 | self.assertCommandAccepted( 'clear && clear' ) 197 | self.assertCommandAccepted( 'clear !&& clear' ) 198 | 199 | 200 | # pipes disallowed ----------------------------------------- 201 | self.gp.allowPipes = False 202 | 203 | self.assertCommandRejected( 'clear > /tmp/test' ) 204 | self.assertCommandRejected( 'clear < /tmp/test' ) 205 | 206 | # pipes allowed ----------------------------------------- 207 | self.gp.allowPipes = True 208 | 209 | self.assertCommandAccepted( 'clear > /tmp/test' ) 210 | self.assertCommandAccepted( 'clear < /tmp/test' ) 211 | 212 | # pipes conflict ----------------------------------------- 213 | self.assertCommandRejected( 'clear > /tmp/test', None, 214 | NullSink.instance ) 215 | self.assertCommandRejected( 'clear < /tmp/test', 216 | NullSource.instance, None ) 217 | 218 | 219 | def assertArgumentAccepted(self, arg ): 220 | s = re.sub('/\s+/s', ' ', str(arg)) 221 | 222 | try: 223 | x = self.gp.execute(('foo', arg) ) 224 | raise Exception("dummy command should have failed in core: " 225 | + "foo %s" % s) 226 | except gpUsageException, e: 227 | self.fail("argument should be accepted by client: %s" % s) 228 | except gpProcessorException, e: 229 | pass 230 | # ok, should fail in core, but be accepted by client side 231 | # validator 232 | 233 | def assertArgumentRejected( self, arg ): 234 | s = re.sub('/\s+/s', ' ', str(arg)) 235 | try: 236 | x = self.gp.execute(('foo', arg) ) 237 | self.fail("malformed argument should be detected: %s" % s) 238 | except gpUsageException, e: 239 | pass 240 | # ok 241 | except gpProcessorException, e: 242 | self.fail("malformed argument should have been detected " 243 | + "by the client: %s; core message: %s" 244 | % (s, e.getMessage()) ) 245 | 246 | 247 | def test_argumentValidation(self): 248 | self.assertArgumentRejected( '' ) 249 | self.assertArgumentRejected( None ) 250 | self.assertArgumentRejected( False ) 251 | self.assertArgumentRejected( ' x ' ) 252 | 253 | # self.gp.setTimeout(2); # has no effect for pipes 254 | self.assertArgumentAccepted( 'x:y' ) 255 | # needed for password auth! 256 | # NOTE: This is broken in graphcore (but works via graphserv)! 257 | 258 | self.assertArgumentAccepted( '123' ) 259 | self.assertArgumentAccepted( 'x' ) 260 | self.assertArgumentAccepted( 'xyz' ) 261 | self.assertArgumentAccepted( 'x7y' ) 262 | self.assertArgumentAccepted( '7x7' ) 263 | 264 | chars = " \r\n\t\0\x09^!\"§\$%&/()[]\ \ =?'#`\\*+~., ;<>|@\xDD" 265 | for ch in chars: 266 | s = "a " + ch + " b" 267 | 268 | self.assertArgumentRejected(s) 269 | 270 | 271 | 272 | # // Client Lib I/O /////////////////////////////////////////////////////////////// 273 | # Tested here, not in ConnectionTestBase, because we only need to test is once, not for every type of connection 274 | # Note: ArraySource and ArraySink are used implicitly all the time in the tests, no need to test them separately. 275 | 276 | def test_fileSource(self): 277 | f = os.path.dirname(os.path.abspath(__file__)) + '/gp.test.data' 278 | src = FileSource(f) 279 | 280 | self.gp.add_arcs( src ) 281 | 282 | self.assertStatus( 'OK' ) 283 | self.assertStatsValue( 'ArcCount', 4 ) 284 | 285 | arcs = self.gp.capture_list_successors( 1 ) 286 | 287 | self.assertTrue( ConnectionTestBase.setEquals( 288 | arcs, [(11, ), (12, ),]), "sucessors of (1): expected [(11,), (12,)], got %s" % arcs ) 289 | 290 | arcs = self.gp.capture_list_successors( 11 ) 291 | self.assertTrue( ConnectionTestBase.setEquals( 292 | arcs, [(111, ), (112, ),]), "sucessors of (2): expected [(111,), (112,)], got %s" % arcs ) 293 | 294 | 295 | def test_fileSink(self): 296 | 297 | # set up the sink 298 | f = tempfile.mktemp(suffix='gpt') 299 | sink = FileSink(f, False, "\n") 300 | 301 | # generate output 302 | self.gp.add_arcs(((1, 11 ), (1, 12 ), (11, 111 ), (11, 112 ),)) 303 | 304 | ok = self.gp.traverse_successors(1, 2, sink) 305 | sink.close() 306 | 307 | # make sure we can read the file 308 | self.assertStatus('OK') 309 | self.assertEquals('OK', ok) 310 | 311 | # compare actual file contents 312 | rows = read_lines(f) 313 | 314 | self.assertNotEquals(False, rows, 315 | "could not get file contents of %s" % f) 316 | self.assertNotNone(rows, "could not get file contents of %s " % f) 317 | 318 | expected =("1\n", "11\n", "12\n", "111\n", "112\n",) 319 | self.assertTrue( ConnectionTestBase.setEquals( 320 | expected, rows), 'bad content in outfile: %s, expected %s' 321 | % ( rows, expected ) ) 322 | 323 | #cleanup 324 | try: 325 | unlink(f) 326 | except: 327 | pass 328 | 329 | def test_nullSource(self): 330 | self.gp.add_arcs( NullSource.instance ) 331 | self.assertStatus( 'OK' ) 332 | 333 | 334 | def test_nullSink(self): 335 | # generate output 336 | self.gp.add_arcs(((1, 11 ), (1, 12 ), (11, 111 ), (11, 112 ),)) 337 | 338 | ok = self.gp.traverse_successors(1, 2, NullSink.instance) 339 | self.assertStatus('OK') 340 | 341 | 342 | # //// Slave Connection Tests /////////////////////////////////////////////////// 343 | # currently none. could check if the process really dies after quit, etc 344 | # TODO: test checkPeer, etc 345 | 346 | if __name__ == '__main__': 347 | unittest.main() 348 | -------------------------------------------------------------------------------- /gp/tests/mysql_test.py: -------------------------------------------------------------------------------- 1 | from test_base import * 2 | from gp.client import * 3 | from gp.mysql import * 4 | from gp.mysql import _fetch_dict 5 | 6 | import unittest 7 | import sys 8 | 9 | class MySQLTest (SlaveTestBase, unittest.TestCase): 10 | mysql = None 11 | 12 | def setUp(self): 13 | self.dump = PipeSink( sys.stdout ) 14 | 15 | try: 16 | self.gp = MySQLGlue.new_slave_connection( test_graphcore_path ) 17 | self.gp.connect() 18 | except gpException as ex: 19 | print "Unable to launch graphcore instance from %s, please make sure graphcore is installed and check the test_graphcore_path configuration options in test_config.py.\nOriginal error: %s " % (test_graphcore_path, ex.getMessage() ) 20 | suicide(10) 21 | 22 | try: 23 | self.gp.mysql_connect(test_mysql_host, test_mysql_user, test_mysql_password, test_mysql_database) 24 | except gpException as ex: 25 | print "Unable to connect to database %s on MySQL host %s as %s, please make sure MySQL is running and check the test_mysql_host and related configuration options in test_cofig.py.\nOriginal error: %s " % (test_mysql_database, test_mysql_host, test_mysql_user, ex.getMessage() ) 26 | suicide(10) 27 | 28 | def _make_table( self, table, fieldSpec ): 29 | sql = "CREATE TEMPORARY TABLE IF NOT EXISTS " + table 30 | sql += "(" 31 | sql += fieldSpec 32 | sql += ")" 33 | 34 | self.gp.mysql_query(sql) 35 | 36 | sql = "TRUNCATE TABLE " + table 37 | self.gp.mysql_query(sql) 38 | 39 | 40 | def test_Source(self): 41 | self._make_table( "test", "a INT NOT NULL, b INT NOT NULL" ) 42 | self.gp.mysql_query( "INSERT INTO test VALUES (3, 8)" ) 43 | self.gp.mysql_query( "INSERT INTO test VALUES (7, 9)" ) 44 | self.gp.mysql_query( "INSERT INTO test VALUES (11, 11)" ) 45 | 46 | #----------------------------------------------------------- 47 | src = self.gp.make_source( MySQLTable("test", "a", "b") ) 48 | 49 | self.assertEquals(( 3, 8 ), src.next() , "expected row to be 3,8 " ) 50 | self.assertEquals(( 7, 9 ), src.next() , "expected row to be 7,9" ) 51 | self.assertEquals(( 11, 11 ), src.next() , "expected row to be 11,11" ) 52 | 53 | try: 54 | r = src.next() 55 | self.fail( "expected no more rows, got %s " % (r, ) ) 56 | except StopIteration: 57 | pass 58 | 59 | src.close() 60 | 61 | #----------------------------------------------------------- 62 | src = self.gp.make_source( MySQLSelect("select a from test where a > 7") ) 63 | 64 | self.assertEquals((11,), src.next() , "expected row to be 11" ) 65 | 66 | try: 67 | r = src.next() 68 | self.fail( "expected no more rows, got %s " % (r, ) ) 69 | except StopIteration: 70 | pass 71 | 72 | src.close() 73 | 74 | 75 | def test_SelectInto(self): 76 | self._make_table( "test", "a INT NOT NULL, b INT NOT NULL" ) 77 | self.gp.mysql_query( "INSERT INTO test VALUES (3, 8)" ) 78 | self.gp.mysql_query( "INSERT INTO test VALUES (7, 9)" ) 79 | self.gp.mysql_query( "INSERT INTO test VALUES (11, 11)" ) 80 | 81 | #----------------------------------------------------------- 82 | sink = ArraySink() 83 | self.gp.select_into( "select a, b from test order by a, b", sink ) 84 | 85 | data = sink.getData() 86 | 87 | self.assertEquals([( 3, 8 ), ( 7, 9 ), ( 11, 11 )], data ) 88 | 89 | 90 | def test_UnbufferedSelectInto(self): 91 | self._make_table( "test", "a INT NOT NULL, b INT NOT NULL" ) 92 | self.gp.set_unbuffered(True) 93 | self.gp.mysql_query( "INSERT INTO test VALUES (3, 8)" ) 94 | self.gp.mysql_query( "INSERT INTO test VALUES (7, 9)" ) 95 | self.gp.mysql_query( "INSERT INTO test VALUES (11, 11)" ) 96 | 97 | #----------------------------------------------------------- 98 | sink = ArraySink() 99 | self.gp.select_into( "select a, b from test order by a, b", sink ) 100 | 101 | data = sink.getData() 102 | 103 | self.assertEquals( [ ( 3, 8 ), ( 7, 9 ), ( 11, 11 ) ], data ) 104 | 105 | def assertNextRowEquals(self, expected, res): 106 | row = _fetch_dict(res) 107 | self.assertTrue( TestBase.dictEquals(expected, row), "expected row to be %s, got %s" % (expected, row) ) 108 | 109 | def test_TempSink(self): 110 | snk = self.gp.make_temp_sink( MySQLTable("?", "a", "b") ) 111 | table = snk.getTable() 112 | 113 | snk.putRow( (4,5) ) 114 | snk.putRow( (6,7) ) 115 | snk.close() 116 | 117 | res = self.gp.mysql_query( "SELECT a, b FROM " + table.get_name() + " ORDER BY a, b") 118 | 119 | self.assertNextRowEquals({ 'a': 4, 'b': 5 }, res ) 120 | self.assertNextRowEquals({ 'a': 6, 'b': 7 }, res ) 121 | self.assertFalse( res.fetchone(), "expected next row to be false" ) 122 | 123 | res.close() 124 | 125 | snk.drop() 126 | 127 | 128 | def test_AddArcsFromSourceObject(self): 129 | self._make_table( "test", "a INT NOT NULL, b INT NOT NULL" ) 130 | self.gp.mysql_query( "INSERT INTO test VALUES (1, 11)" ) 131 | self.gp.mysql_query( "INSERT INTO test VALUES (1, 12)" ) 132 | self.gp.mysql_query( "INSERT INTO test VALUES (11, 111)" ) 133 | self.gp.mysql_query( "INSERT INTO test VALUES (11, 112)" ) 134 | 135 | #----------------------------------------------------------- 136 | src = self.gp.make_source( MySQLTable("test", "a", "b") ) 137 | self.gp.add_arcs( src ) 138 | src.close() 139 | 140 | self.assertStatus( 'OK' ) 141 | self.assertStatsValue( 'ArcCount', 4 ) 142 | 143 | arcs = self.gp.capture_list_successors( 1 ); 144 | self.assertTrue( ConnectionTestBase.setEquals( arcs, [ 145 | ( 11, ), 146 | ( 12, ), 147 | ] ), "sucessors of (1)" ) 148 | 149 | arcs = self.gp.capture_list_successors( 11 ) 150 | self.assertTrue( ConnectionTestBase.setEquals( arcs, [ 151 | ( 111, ), 152 | ( 112, ), 153 | ] ), "sucessors of (2)" ) 154 | 155 | 156 | def test_AddArcsFromSourceShorthand(self): 157 | self._make_table( "test", "a INT NOT NULL, b INT NOT NULL" ) 158 | self.gp.mysql_query( "INSERT INTO test VALUES (1, 11)" ) 159 | self.gp.mysql_query( "INSERT INTO test VALUES (1, 12)" ) 160 | self.gp.mysql_query( "INSERT INTO test VALUES (11, 111)" ) 161 | self.gp.mysql_query( "INSERT INTO test VALUES (11, 112)" ) 162 | 163 | #----------------------------------------------------------- 164 | src = self.gp.add_arcs_from( "test a b" ) 165 | src.close() 166 | 167 | self.assertStatus( 'OK' ) 168 | self.assertStatsValue( 'ArcCount', 4 ) 169 | 170 | arcs = self.gp.capture_list_successors( 1 ); 171 | self.assertTrue( ConnectionTestBase.setEquals( arcs, [ 172 | ( 11, ), 173 | ( 12, ), 174 | ] ), "sucessors of (1)" ) 175 | 176 | arcs = self.gp.capture_list_successors( 11 ) 177 | self.assertTrue( ConnectionTestBase.setEquals( arcs, [ 178 | ( 111, ), 179 | ( 112, ), 180 | ] ), "sucessors of (2)" ) 181 | 182 | #----------------------------------------------------------- 183 | self.gp.clear() 184 | stats = self.gp.capture_stats_map() 185 | self.assertEquals( 0, stats['ArcCount'], "ArcCount" ) 186 | 187 | #self.gp.setDebug(True) 188 | src = self.gp.add_arcs_from( "select a, b from test" ) 189 | src.close() 190 | 191 | stats = self.gp.capture_stats_map() 192 | self.assertEquals( 4, stats['ArcCount'], "ArcCount" ) 193 | 194 | #----------------------------------------------------------- 195 | self.gp.clear() 196 | 197 | src = self.gp.add_arcs_from( ("test", "a", "b") ) 198 | src.close() 199 | 200 | self.assertStatsValue( 'ArcCount', 4 ) 201 | 202 | #----------------------------------------------------------- 203 | self.gp.clear() 204 | 205 | src = self.gp.add_arcs_from( MySQLTable("test", "a", "b") ) 206 | src.close() 207 | 208 | self.assertStatsValue( 'ArcCount', 4 ) 209 | 210 | 211 | def testSuccessorsToSinkObject(self): 212 | self.gp.add_arcs( [ 213 | ( 1, 11 ), 214 | ( 1, 12 ), 215 | ( 11, 111 ), 216 | ( 11, 112 ), 217 | ] ) 218 | 219 | #----------------------------------------------------------- 220 | snk = self.gp.make_temp_sink( MySQLTable("?", "n") ) 221 | src = self.gp.traverse_successors( 1, 8, snk ) 222 | snk.close() 223 | table = snk.getTable() 224 | 225 | res = self.gp.mysql_query( "SELECT n FROM "+table.get_name()+" ORDER BY n") 226 | 227 | self.assertNextRowEquals({ 'n': 1L }, res ) 228 | self.assertNextRowEquals({ 'n': 11L }, res ) 229 | self.assertNextRowEquals({ 'n': 12L }, res ) 230 | self.assertNextRowEquals({ 'n': 111L }, res ) 231 | self.assertNextRowEquals({ 'n': 112L }, res ) 232 | self.assertFalse( res.fetchone(), "expected next row to be False" ) 233 | 234 | res.close() 235 | 236 | #----------------------------------------------------------- 237 | self.gp.set_max_allowed_packet(6); #force inserter to flush intermittedly 238 | 239 | snk = self.gp.make_temp_sink( MySQLTable("?", "n") ) 240 | src = self.gp.traverse_successors( 1, 8, snk ) 241 | snk.close() 242 | table = snk.getTable() 243 | 244 | res = self.gp.mysql_query( "SELECT n FROM "+table.get_name()+" ORDER BY n") 245 | 246 | self.assertNextRowEquals({ 'n': 1 }, res ) 247 | self.assertNextRowEquals({ 'n': 11 }, res ) 248 | self.assertNextRowEquals({ 'n': 12 }, res ) 249 | self.assertNextRowEquals({ 'n': 111 }, res ) 250 | self.assertNextRowEquals({ 'n': 112 }, res ) 251 | self.assertFalse( res.fetchone(), "expected next row to be False" ) 252 | 253 | res.close() 254 | 255 | 256 | def test_SuccessorsToSinkShorthand(self): 257 | self.gp.add_arcs( [ 258 | ( 1, 11 ), 259 | ( 1, 12 ), 260 | ( 11, 111 ), 261 | ( 11, 112 ), 262 | ] ) 263 | 264 | #----------------------------------------------------------- 265 | snk = self.gp.traverse_successors_into( 1, 8, "? n" ) 266 | snk.close() 267 | table = snk.getTable() 268 | 269 | res = self.gp.mysql_query( "SELECT n FROM "+table.get_name()+" ORDER BY n") 270 | 271 | self.assertNextRowEquals({ 'n': 1 }, res ) 272 | self.assertNextRowEquals({ 'n': 11 }, res ) 273 | self.assertNextRowEquals({ 'n': 12 }, res ) 274 | self.assertNextRowEquals({ 'n': 111 }, res ) 275 | self.assertNextRowEquals({ 'n': 112 }, res ) 276 | self.assertFalse( res.fetchone(), "expected next row to be False" ) 277 | 278 | res.close() 279 | snk.drop() 280 | 281 | #--------------------------------------------------------- 282 | snk = self.gp.traverse_successors_into( 1, 8, ( "?", "n" ) ) 283 | snk.close() 284 | table = snk.getTable() 285 | 286 | res = self.gp.mysql_query( "SELECT n FROM "+table.get_name()+" ORDER BY n") 287 | 288 | self.assertNextRowEquals({ 'n': 1 }, res ) 289 | self.assertNextRowEquals({ 'n': 11 }, res ) 290 | self.assertNextRowEquals({ 'n': 12 }, res ) 291 | self.assertNextRowEquals({ 'n': 111 }, res ) 292 | self.assertNextRowEquals({ 'n': 112 }, res ) 293 | self.assertFalse( res.fetchone(), "expected next row to be False" ) 294 | 295 | res.close() 296 | snk.drop() 297 | 298 | #--------------------------------------------------------- 299 | snk = self.gp.traverse_successors_into( 1, 8, MySQLTable("?", "n") ) 300 | snk.close() 301 | table = snk.getTable() 302 | 303 | res = self.gp.mysql_query( "SELECT n FROM "+table.get_name()+" ORDER BY n") 304 | 305 | self.assertNextRowEquals({ 'n': 1 }, res ) 306 | self.assertNextRowEquals({ 'n': 11 }, res ) 307 | self.assertNextRowEquals({ 'n': 12 }, res ) 308 | self.assertNextRowEquals({ 'n': 111 }, res ) 309 | self.assertNextRowEquals({ 'n': 112 }, res ) 310 | self.assertFalse( res.fetchone(), "expected next row to be False" ) 311 | 312 | res.close() 313 | snk.drop() 314 | 315 | #--------------------------------------------------------- 316 | self._make_table( "test_n", "n INT NOT NULL" ) 317 | 318 | table = MySQLTable("test_n", "n") 319 | snk = self.gp.traverse_successors_into( 1, 8, table ) 320 | snk.close() 321 | 322 | res = self.gp.mysql_query( "SELECT n FROM "+table.get_name()+" ORDER BY n") 323 | 324 | self.assertNextRowEquals({ 'n': 1 }, res ) 325 | self.assertNextRowEquals({ 'n': 11 }, res ) 326 | self.assertNextRowEquals({ 'n': 12 }, res ) 327 | self.assertNextRowEquals({ 'n': 111 }, res ) 328 | self.assertNextRowEquals({ 'n': 112 }, res ) 329 | self.assertFalse( res.fetchone(), "expected next row to be False" ) 330 | 331 | res.close() 332 | 333 | if __name__ == '__main__': 334 | unittest.main() 335 | -------------------------------------------------------------------------------- /gp/tests/.svn/text-base/mysql_test.py.svn-base: -------------------------------------------------------------------------------- 1 | from test_base import * 2 | from gp.client import * 3 | from gp.mysql import * 4 | from gp.mysql import _fetch_dict 5 | 6 | import unittest 7 | import sys 8 | 9 | class MySQLTest (SlaveTestBase, unittest.TestCase): 10 | mysql = None 11 | 12 | def setUp(self): 13 | self.dump = PipeSink( sys.stdout ) 14 | 15 | try: 16 | self.gp = MySQLGlue.new_slave_connection( test_graphcore_path ) 17 | self.gp.connect() 18 | except gpException as ex: 19 | print "Unable to launch graphcore instance from %s, please make sure graphcore is installed and check the test_graphcore_path configuration options in test_config.py.\nOriginal error: %s " % (test_graphcore_path, ex.getMessage() ) 20 | suicide(10) 21 | 22 | try: 23 | self.gp.mysql_connect(test_mysql_host, test_mysql_user, test_mysql_password, test_mysql_database) 24 | except gpException as ex: 25 | print "Unable to connect to database %s on MySQL host %s as %s, please make sure MySQL is running and check the test_mysql_host and related configuration options in test_cofig.py.\nOriginal error: %s " % (test_mysql_database, test_mysql_host, test_mysql_user, ex.getMessage() ) 26 | suicide(10) 27 | 28 | def _make_table( self, table, fieldSpec ): 29 | sql = "CREATE TEMPORARY TABLE IF NOT EXISTS " + table 30 | sql += "(" 31 | sql += fieldSpec 32 | sql += ")" 33 | 34 | self.gp.mysql_query(sql) 35 | 36 | sql = "TRUNCATE TABLE " + table 37 | self.gp.mysql_query(sql) 38 | 39 | 40 | def test_Source(self): 41 | self._make_table( "test", "a INT NOT NULL, b INT NOT NULL" ) 42 | self.gp.mysql_query( "INSERT INTO test VALUES (3, 8)" ) 43 | self.gp.mysql_query( "INSERT INTO test VALUES (7, 9)" ) 44 | self.gp.mysql_query( "INSERT INTO test VALUES (11, 11)" ) 45 | 46 | #----------------------------------------------------------- 47 | src = self.gp.make_source( MySQLTable("test", "a", "b") ) 48 | 49 | self.assertEquals(( 3, 8 ), src.next() , "expected row to be 3,8 " ) 50 | self.assertEquals(( 7, 9 ), src.next() , "expected row to be 7,9" ) 51 | self.assertEquals(( 11, 11 ), src.next() , "expected row to be 11,11" ) 52 | 53 | try: 54 | r = src.next() 55 | self.fail( "expected no more rows, got %s " % (r, ) ) 56 | except StopIteration: 57 | pass 58 | 59 | src.close() 60 | 61 | #----------------------------------------------------------- 62 | src = self.gp.make_source( MySQLSelect("select a from test where a > 7") ) 63 | 64 | self.assertEquals((11,), src.next() , "expected row to be 11" ) 65 | 66 | try: 67 | r = src.next() 68 | self.fail( "expected no more rows, got %s " % (r, ) ) 69 | except StopIteration: 70 | pass 71 | 72 | src.close() 73 | 74 | 75 | def test_SelectInto(self): 76 | self._make_table( "test", "a INT NOT NULL, b INT NOT NULL" ) 77 | self.gp.mysql_query( "INSERT INTO test VALUES (3, 8)" ) 78 | self.gp.mysql_query( "INSERT INTO test VALUES (7, 9)" ) 79 | self.gp.mysql_query( "INSERT INTO test VALUES (11, 11)" ) 80 | 81 | #----------------------------------------------------------- 82 | sink = ArraySink() 83 | self.gp.select_into( "select a, b from test order by a, b", sink ) 84 | 85 | data = sink.getData() 86 | 87 | self.assertEquals([( 3, 8 ), ( 7, 9 ), ( 11, 11 )], data ) 88 | 89 | 90 | def test_UnbufferedSelectInto(self): 91 | self._make_table( "test", "a INT NOT NULL, b INT NOT NULL" ) 92 | self.gp.set_unbuffered(True) 93 | self.gp.mysql_query( "INSERT INTO test VALUES (3, 8)" ) 94 | self.gp.mysql_query( "INSERT INTO test VALUES (7, 9)" ) 95 | self.gp.mysql_query( "INSERT INTO test VALUES (11, 11)" ) 96 | 97 | #----------------------------------------------------------- 98 | sink = ArraySink() 99 | self.gp.select_into( "select a, b from test order by a, b", sink ) 100 | 101 | data = sink.getData() 102 | 103 | self.assertEquals( [ ( 3, 8 ), ( 7, 9 ), ( 11, 11 ) ], data ) 104 | 105 | def assertNextRowEquals(self, expected, res): 106 | row = _fetch_dict(res) 107 | self.assertTrue( TestBase.dictEquals(expected, row), "expected row to be %s, got %s" % (expected, row) ) 108 | 109 | def test_TempSink(self): 110 | snk = self.gp.make_temp_sink( MySQLTable("?", "a", "b") ) 111 | table = snk.getTable() 112 | 113 | snk.putRow( (4,5) ) 114 | snk.putRow( (6,7) ) 115 | snk.close() 116 | 117 | res = self.gp.mysql_query( "SELECT a, b FROM " + table.get_name() + " ORDER BY a, b") 118 | 119 | self.assertNextRowEquals({ 'a': 4, 'b': 5 }, res ) 120 | self.assertNextRowEquals({ 'a': 6, 'b': 7 }, res ) 121 | self.assertFalse( res.fetchone(), "expected next row to be false" ) 122 | 123 | res.close() 124 | 125 | snk.drop() 126 | 127 | 128 | def test_AddArcsFromSourceObject(self): 129 | self._make_table( "test", "a INT NOT NULL, b INT NOT NULL" ) 130 | self.gp.mysql_query( "INSERT INTO test VALUES (1, 11)" ) 131 | self.gp.mysql_query( "INSERT INTO test VALUES (1, 12)" ) 132 | self.gp.mysql_query( "INSERT INTO test VALUES (11, 111)" ) 133 | self.gp.mysql_query( "INSERT INTO test VALUES (11, 112)" ) 134 | 135 | #----------------------------------------------------------- 136 | src = self.gp.make_source( MySQLTable("test", "a", "b") ) 137 | self.gp.add_arcs( src ) 138 | src.close() 139 | 140 | self.assertStatus( 'OK' ) 141 | self.assertStatsValue( 'ArcCount', 4 ) 142 | 143 | arcs = self.gp.capture_list_successors( 1 ); 144 | self.assertTrue( ConnectionTestBase.setEquals( arcs, [ 145 | ( 11, ), 146 | ( 12, ), 147 | ] ), "sucessors of (1)" ) 148 | 149 | arcs = self.gp.capture_list_successors( 11 ) 150 | self.assertTrue( ConnectionTestBase.setEquals( arcs, [ 151 | ( 111, ), 152 | ( 112, ), 153 | ] ), "sucessors of (2)" ) 154 | 155 | 156 | def test_AddArcsFromSourceShorthand(self): 157 | self._make_table( "test", "a INT NOT NULL, b INT NOT NULL" ) 158 | self.gp.mysql_query( "INSERT INTO test VALUES (1, 11)" ) 159 | self.gp.mysql_query( "INSERT INTO test VALUES (1, 12)" ) 160 | self.gp.mysql_query( "INSERT INTO test VALUES (11, 111)" ) 161 | self.gp.mysql_query( "INSERT INTO test VALUES (11, 112)" ) 162 | 163 | #----------------------------------------------------------- 164 | src = self.gp.add_arcs_from( "test a b" ) 165 | src.close() 166 | 167 | self.assertStatus( 'OK' ) 168 | self.assertStatsValue( 'ArcCount', 4 ) 169 | 170 | arcs = self.gp.capture_list_successors( 1 ); 171 | self.assertTrue( ConnectionTestBase.setEquals( arcs, [ 172 | ( 11, ), 173 | ( 12, ), 174 | ] ), "sucessors of (1)" ) 175 | 176 | arcs = self.gp.capture_list_successors( 11 ) 177 | self.assertTrue( ConnectionTestBase.setEquals( arcs, [ 178 | ( 111, ), 179 | ( 112, ), 180 | ] ), "sucessors of (2)" ) 181 | 182 | #----------------------------------------------------------- 183 | self.gp.clear() 184 | stats = self.gp.capture_stats_map() 185 | self.assertEquals( 0, stats['ArcCount'], "ArcCount" ) 186 | 187 | #self.gp.setDebug(True) 188 | src = self.gp.add_arcs_from( "select a, b from test" ) 189 | src.close() 190 | 191 | stats = self.gp.capture_stats_map() 192 | self.assertEquals( 4, stats['ArcCount'], "ArcCount" ) 193 | 194 | #----------------------------------------------------------- 195 | self.gp.clear() 196 | 197 | src = self.gp.add_arcs_from( ("test", "a", "b") ) 198 | src.close() 199 | 200 | self.assertStatsValue( 'ArcCount', 4 ) 201 | 202 | #----------------------------------------------------------- 203 | self.gp.clear() 204 | 205 | src = self.gp.add_arcs_from( MySQLTable("test", "a", "b") ) 206 | src.close() 207 | 208 | self.assertStatsValue( 'ArcCount', 4 ) 209 | 210 | 211 | def testSuccessorsToSinkObject(self): 212 | self.gp.add_arcs( [ 213 | ( 1, 11 ), 214 | ( 1, 12 ), 215 | ( 11, 111 ), 216 | ( 11, 112 ), 217 | ] ) 218 | 219 | #----------------------------------------------------------- 220 | snk = self.gp.make_temp_sink( MySQLTable("?", "n") ) 221 | src = self.gp.traverse_successors( 1, 8, snk ) 222 | snk.close() 223 | table = snk.getTable() 224 | 225 | res = self.gp.mysql_query( "SELECT n FROM "+table.get_name()+" ORDER BY n") 226 | 227 | self.assertNextRowEquals({ 'n': 1L }, res ) 228 | self.assertNextRowEquals({ 'n': 11L }, res ) 229 | self.assertNextRowEquals({ 'n': 12L }, res ) 230 | self.assertNextRowEquals({ 'n': 111L }, res ) 231 | self.assertNextRowEquals({ 'n': 112L }, res ) 232 | self.assertFalse( res.fetchone(), "expected next row to be False" ) 233 | 234 | res.close() 235 | 236 | #----------------------------------------------------------- 237 | self.gp.set_max_allowed_packet(6); #force inserter to flush intermittedly 238 | 239 | snk = self.gp.make_temp_sink( MySQLTable("?", "n") ) 240 | src = self.gp.traverse_successors( 1, 8, snk ) 241 | snk.close() 242 | table = snk.getTable() 243 | 244 | res = self.gp.mysql_query( "SELECT n FROM "+table.get_name()+" ORDER BY n") 245 | 246 | self.assertNextRowEquals({ 'n': 1 }, res ) 247 | self.assertNextRowEquals({ 'n': 11 }, res ) 248 | self.assertNextRowEquals({ 'n': 12 }, res ) 249 | self.assertNextRowEquals({ 'n': 111 }, res ) 250 | self.assertNextRowEquals({ 'n': 112 }, res ) 251 | self.assertFalse( res.fetchone(), "expected next row to be False" ) 252 | 253 | res.close() 254 | 255 | 256 | def test_SuccessorsToSinkShorthand(self): 257 | self.gp.add_arcs( [ 258 | ( 1, 11 ), 259 | ( 1, 12 ), 260 | ( 11, 111 ), 261 | ( 11, 112 ), 262 | ] ) 263 | 264 | #----------------------------------------------------------- 265 | snk = self.gp.traverse_successors_into( 1, 8, "? n" ) 266 | snk.close() 267 | table = snk.getTable() 268 | 269 | res = self.gp.mysql_query( "SELECT n FROM "+table.get_name()+" ORDER BY n") 270 | 271 | self.assertNextRowEquals({ 'n': 1 }, res ) 272 | self.assertNextRowEquals({ 'n': 11 }, res ) 273 | self.assertNextRowEquals({ 'n': 12 }, res ) 274 | self.assertNextRowEquals({ 'n': 111 }, res ) 275 | self.assertNextRowEquals({ 'n': 112 }, res ) 276 | self.assertFalse( res.fetchone(), "expected next row to be False" ) 277 | 278 | res.close() 279 | snk.drop() 280 | 281 | #--------------------------------------------------------- 282 | snk = self.gp.traverse_successors_into( 1, 8, ( "?", "n" ) ) 283 | snk.close() 284 | table = snk.getTable() 285 | 286 | res = self.gp.mysql_query( "SELECT n FROM "+table.get_name()+" ORDER BY n") 287 | 288 | self.assertNextRowEquals({ 'n': 1 }, res ) 289 | self.assertNextRowEquals({ 'n': 11 }, res ) 290 | self.assertNextRowEquals({ 'n': 12 }, res ) 291 | self.assertNextRowEquals({ 'n': 111 }, res ) 292 | self.assertNextRowEquals({ 'n': 112 }, res ) 293 | self.assertFalse( res.fetchone(), "expected next row to be False" ) 294 | 295 | res.close() 296 | snk.drop() 297 | 298 | #--------------------------------------------------------- 299 | snk = self.gp.traverse_successors_into( 1, 8, MySQLTable("?", "n") ) 300 | snk.close() 301 | table = snk.getTable() 302 | 303 | res = self.gp.mysql_query( "SELECT n FROM "+table.get_name()+" ORDER BY n") 304 | 305 | self.assertNextRowEquals({ 'n': 1 }, res ) 306 | self.assertNextRowEquals({ 'n': 11 }, res ) 307 | self.assertNextRowEquals({ 'n': 12 }, res ) 308 | self.assertNextRowEquals({ 'n': 111 }, res ) 309 | self.assertNextRowEquals({ 'n': 112 }, res ) 310 | self.assertFalse( res.fetchone(), "expected next row to be False" ) 311 | 312 | res.close() 313 | snk.drop() 314 | 315 | #--------------------------------------------------------- 316 | self._make_table( "test_n", "n INT NOT NULL" ) 317 | 318 | table = MySQLTable("test_n", "n") 319 | snk = self.gp.traverse_successors_into( 1, 8, table ) 320 | snk.close() 321 | 322 | res = self.gp.mysql_query( "SELECT n FROM "+table.get_name()+" ORDER BY n") 323 | 324 | self.assertNextRowEquals({ 'n': 1 }, res ) 325 | self.assertNextRowEquals({ 'n': 11 }, res ) 326 | self.assertNextRowEquals({ 'n': 12 }, res ) 327 | self.assertNextRowEquals({ 'n': 111 }, res ) 328 | self.assertNextRowEquals({ 'n': 112 }, res ) 329 | self.assertFalse( res.fetchone(), "expected next row to be False" ) 330 | 331 | res.close() 332 | 333 | if __name__ == '__main__': 334 | unittest.main() 335 | -------------------------------------------------------------------------------- /gp/tests/server_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 3 | 4 | import unittest 5 | import os 6 | import tempfile 7 | from test_base import * 8 | from gp.client import * 9 | 10 | test_graph_name = 'test' + str(os.getpid()) 11 | TestFilePrefix = '/tmp/gptest-' + str(os.getpid()) 12 | 13 | 14 | class ServerTest (ClientTestBase, unittest.TestCase): 15 | """Test server functions via client lib.""" 16 | 17 | def test_createGraph(self): 18 | """Graph management functions""" 19 | 20 | global test_graph_name 21 | 22 | name = test_graph_name + "_2" 23 | 24 | # create the graph 25 | self.gp.create_graph(name) 26 | 27 | #make sure we can't create it twice 28 | ok = self.gp.try_create_graph(name) 29 | self.assertFalse( ok, "should not be able to create graph again when it already exists" ) 30 | 31 | # see if we can use the graph from another connection 32 | gp2 = self.newConnection() 33 | 34 | gp2.use_graph(name) 35 | 36 | # see if we can drop the graph while it's used 37 | self.gp.drop_graph(name) 38 | 39 | #TODO: gp2 should now report errors, because the grpah is gone. test that. 40 | 41 | # see if we can re-create the graph after it was dropped 42 | self.gp.create_graph(name) 43 | self.gp.drop_graph(name) 44 | 45 | #TODO: test name restrictions 46 | 47 | 48 | def test_createNameRestrictions(self): 49 | global test_graph_name 50 | 51 | self.gp.strictArguments = False 52 | # disable strict client-side validation 53 | 54 | try: 55 | n = '' 56 | ok = self.gp.create_graph(n) 57 | self.fail("empty graph names should be forbidden!" ) 58 | except gpException, ex: 59 | pass 60 | # ok 61 | 62 | 63 | n = '1337' 64 | ok = self.gp.try_create_graph(n) 65 | self.assertFalse(ok, "numeric graph names should be forbidden! (name: `" + n + "`)" ) 66 | 67 | n = '1337' + test_graph_name 68 | ok = self.gp.try_create_graph(n) 69 | self.assertFalse( ok, 70 | "graph names starting with a number should be forbidden! (name: `" 71 | + n + "`)" ) 72 | 73 | chars = " \r\n\t\0\x09^!\"§\$%&/()[]\ \ =?'#`\\*+~.:, ;<>|@" 74 | for ch in chars: 75 | try: 76 | n = test_graph_name + ch + "test" 77 | ok = self.gp.create_graph(n) 78 | self.fail("graph names containing `" 79 | + ch + "` should be forbidden! (name: `" 80 | + n + "`)" ) 81 | except gpException, ex: 82 | pass 83 | # ok 84 | try: 85 | n = ch + test_graph_name 86 | ok = self.gp.create_graph(n) 87 | self.fail("graph names starting with `" 88 | + ch + "` should be forbidden! (name: `" + n + "`)") 89 | except gpException, ex: 90 | pass 91 | # ok 92 | 93 | n = 'test1337' + test_graph_name 94 | ok = self.gp.try_create_graph(n) 95 | self.assertEquals( 'OK', ok, 96 | "graph names containing numbers should be allowd! (name: `" 97 | + n+ "`)") 98 | self.gp.try_drop_graph(n) 99 | 100 | chars = '-_8' 101 | for ch in chars: 102 | n = 'test' + ch + test_graph_name 103 | ok = self.gp.try_create_graph(n) 104 | self.assertEquals( 'OK', ok, "graph names containing `" 105 | + ch + "` should be allowd! (name: `" + n + "`)") 106 | self.gp.try_drop_graph(n) 107 | 108 | def test_dropGraph(self): 109 | global test_graph_name 110 | 111 | name = test_graph_name + "_2" 112 | 113 | self.gp.create_graph(name) 114 | self.gp.drop_graph(name) 115 | 116 | ok = self.gp.try_use_graph(name) 117 | self.assertFalse( ok, 118 | "should not be able to use graph after dropping it" ) 119 | ok = self.gp.try_drop_graph(name) 120 | self.assertEquals( 'NONE', ok, "should not be able to drop " 121 | + "graph again after it was already dropped." ) 122 | 123 | def test_listGraphs(self): 124 | global test_graph_name 125 | 126 | gp2 = self.newConnection() 127 | graphs = gp2.capture_list_graphs() 128 | graphs = array_column(graphs, 0) 129 | self.assertTrue( test_graph_name in graphs, 130 | "test table test_graph_name should be in the list" ) 131 | 132 | self.gp.drop_graph(test_graph_name) 133 | graphs = gp2.capture_list_graphs() 134 | #print "graphs: " . var_export($graphs, true) . "\n" 135 | 136 | graphs = array_column( graphs, 0 ) 137 | 138 | #print "graphs: " . var_export($graphs, true) . "\n" 139 | 140 | #print "containes: " . var_export(ConnectionTestBase::setContains( $graphs, test_graph_name ), true) . "\n" 141 | 142 | self.assertFalse( 143 | ConnectionTestBase.setContains(graphs, test_graph_name), 144 | "test table test_graph_name should no longer be in the list" ) 145 | 146 | def test_shutdown(self): 147 | global test_graph_name 148 | 149 | gp2 = self.newConnection() 150 | gp2.use_graph(test_graph_name) 151 | gp2.stats() 152 | 153 | self.assertSessionValue('ConnectedGraph', test_graph_name) 154 | 155 | self.gp.shutdown() # <------------------ 156 | # self.assertSessionValue('ConnectedGraph', 'None'); 157 | # nice, but not reliable. race condition. 158 | 159 | self.gp.try_stats() 160 | self.assertEquals( 'FAILED', self.gp.getStatus(), 161 | 'fetching stats should fail after shutdown' ) 162 | 163 | gp2.try_stats() 164 | self.assertEquals( 'FAILED', gp2.getStatus(), 165 | 'fetching stats should fail after shutdown' ) 166 | gp2.close() 167 | 168 | gp3 = self.newConnection() 169 | gp3.try_use_graph(test_graph_name) 170 | self.assertEquals( 'FAILED', gp3.getStatus(), 171 | 'graph should be unavailable after shutdown' ) 172 | gp3.close() 173 | 174 | 175 | def test_quit(self): 176 | global test_graph_name 177 | 178 | gp2 = self.newConnection() 179 | gp2.use_graph(test_graph_name) 180 | gp2.stats() 181 | 182 | self.assertSessionValue('ConnectedGraph', test_graph_name) 183 | 184 | self.gp.quit() # <------------------ 185 | self.assertStatus('OK') 186 | 187 | try: 188 | self.gp.try_stats() 189 | self.fail( 'connection should be unusable after quit' ) 190 | except gpProtocolException, e: 191 | pass 192 | # ok 193 | 194 | 195 | gp2.stats() 196 | self.assertEquals( 'OK', gp2.getStatus(), 197 | 'connection should still be usable by others after quit; response: %s' % gp2.getResponse() ) 198 | gp2.close() 199 | 200 | gp3 = self.newConnection() 201 | gp3.use_graph(test_graph_name) 202 | self.assertEquals( 'OK', gp3.getStatus(), 203 | 'graph should still be available to others after quit; response: %s' % gp2.getResponse() ) 204 | gp3.close() 205 | 206 | 207 | # privileges 208 | def test_createGraphPrivilege(self): 209 | global test_graph_name 210 | global test_admin, test_admin_password 211 | global test_master, test_master_password 212 | 213 | name = test_graph_name + "_2" 214 | 215 | gp = self.newConnection() 216 | 217 | ok = gp.try_create_graph(name) 218 | self.assertFalse( ok, 219 | "should not be able to create a graph without authorizing" ) 220 | 221 | gp.authorize('password', 222 | test_master + ":" + test_master_password) 223 | ok = gp.try_create_graph(name) 224 | self.assertFalse( ok, 225 | "should not be able to create a graph without admin privileges" ) 226 | 227 | gp.authorize('password', 228 | test_admin + ":" + test_admin_password) 229 | # re-authenticate 230 | ok = gp.create_graph(name) 231 | self.assertEquals( ok, 'OK', 232 | "should be able to create graph with admin privileges; response: %s" % gp.getResponse() ) 233 | 234 | gp.try_drop_graph(name) 235 | # cleanup 236 | 237 | 238 | def test_dropGraphPrivilege(self): 239 | global test_graph_name 240 | global test_admin, test_admin_password 241 | global test_master, test_master_password 242 | 243 | name = test_graph_name 244 | 245 | gp = self.newConnection() 246 | 247 | ok = gp.try_drop_graph(name) 248 | self.assertFalse( ok, "should not be able to drop a graph without authorizing" ) 249 | 250 | gp.authorize('password', 251 | test_master + ":" + test_master_password) 252 | ok = gp.try_drop_graph(name) 253 | self.assertFalse( ok, 254 | "should not be able to drop a graph without admin privileges" ) 255 | 256 | gp.authorize('password', 257 | test_admin + ":" + test_admin_password) 258 | # re-authenticate 259 | ok = gp.drop_graph(name) 260 | self.assertEquals( ok, 'OK', 261 | "should be able to drop graph with admin privileges; response: %s" % gp.getResponse() ) 262 | 263 | def test_inputPipingPrivilege(self): 264 | global test_graph_name, test_graphserv_host 265 | global test_admin, test_admin_password 266 | global test_master, test_master_password 267 | 268 | #XXX: this uses local files, so it will always fail 269 | # if the server isn't on localhost! 270 | if test_graphserv_host != 'localhost': 271 | return None 272 | 273 | f = os.path.dirname(os.path.abspath(__file__)) + '/gp.test.data' 274 | 275 | gp = self.newConnection() 276 | gp.use_graph(test_graph_name) 277 | gp.allowPipes = True 278 | 279 | gp.authorize('password', 280 | test_master + ":" + test_master_password) 281 | 282 | try: 283 | ok = gp.execute("add-arcs < " + f) 284 | self.fail( 285 | "should not be able to pipe without admin privileges!" ) 286 | except gpProcessorException, ex: 287 | self.assertEquals( 'DENIED', gp.getStatus(), 288 | "piping should be denied, not fail. Message: " 289 | + str(ex)) 290 | 291 | 292 | gp.authorize('password', test_admin + ":" + test_admin_password) 293 | # re-authenticate 294 | ok = gp.execute("add-arcs < " + f) 295 | self.assertEquals( ok, 'OK', 296 | "should be able to pipe with admin privileges; response: %s" % gp.getResponse() ) 297 | 298 | 299 | def test_outputPipingPrivilege(self): 300 | global test_graph_name, test_graphserv_host 301 | global test_admin, test_admin_password 302 | global test_master, test_master_password 303 | 304 | #XXX: this uses local files, so it will always fail 305 | # if the server isn't on localhost! 306 | if test_graphserv_host != 'localhost': 307 | return None 308 | 309 | f = tempfile.mktemp(suffix='gpt') 310 | 311 | gp = self.newConnection() 312 | gp.use_graph(test_graph_name) 313 | gp.allowPipes = True 314 | 315 | try: 316 | ok = gp.execute("list-roots > " + f) 317 | self.fail( 318 | "should not be able to pipe without admin privileges!" ) 319 | except gpProcessorException, ex: 320 | self.assertEquals( 'DENIED', gp.getStatus(), 321 | "piping should be denied, not fail. Message: " 322 | + str(ex)) 323 | 324 | gp.authorize( 325 | 'password', test_admin + ":" + test_admin_password) 326 | # re-authenticate 327 | ok = gp.execute("list-roots > " + f) 328 | self.assertEquals( 329 | ok, 'OK', "should be able to pipe with admin privileges; response: %s" % gp.getResponse() ) 330 | 331 | try: 332 | unlink(f) 333 | # cleanup 334 | except: 335 | pass 336 | 337 | def test_addArcsPrivilege(self): 338 | global test_graph_name 339 | global test_master, test_master_password 340 | 341 | gp = self.newConnection() 342 | gp.use_graph(test_graph_name) 343 | 344 | ok = gp.try_add_arcs(((1, 11 ), (1, 12 ) ) ) 345 | self.assertFalse( 346 | ok, "should not be able to add arcs without authorizing" ) 347 | self.assertEquals('DENIED', gp.getStatus(), 348 | "command should be denied, not fail" ) 349 | 350 | gp.authorize('password', 351 | test_master + ":" + test_master_password) 352 | ok = gp.try_add_arcs(((1, 11 ), (1, 12 ) ) ) 353 | self.assertEquals( 'OK', ok, 354 | "should be able to add arcs with updater privileges; response: %s" % gp.getResponse() ) 355 | 356 | def test_removeArcsPrivilege(self): 357 | global test_graph_name 358 | global test_master, test_master_password 359 | 360 | self.gp.add_arcs(((1, 11 ), (1, 12 ) ) ) 361 | # add some arcs as admin 362 | 363 | gp = self.newConnection() 364 | gp.use_graph(test_graph_name) 365 | 366 | ok = gp.try_remove_arcs(((1, 11 ), ) ) 367 | self.assertFalse( ok, 368 | "should not be able to delete arcs without authorizing" ) 369 | self.assertEquals( 'DENIED', gp.getStatus(), 370 | "command should be denied, not fail" ) 371 | 372 | gp.authorize('password', 373 | test_master + ":" + test_master_password) 374 | 375 | ok = gp.try_remove_arcs(((1, 11 ), ) ) 376 | self.assertEquals( 'OK', ok, 377 | "should be able to delete arcs with updater privileges; response: %s" % gp.getResponse() ) 378 | 379 | def test_replaceSuccessorsPrivilege(self): 380 | global test_graph_name 381 | global test_master, test_master_password 382 | 383 | self.gp.add_arcs(((1, 11 ), (1, 12 ) ) ) 384 | # add some arcs as admin 385 | 386 | gp = self.newConnection() 387 | gp.use_graph(test_graph_name) 388 | 389 | ok = gp.try_replace_successors( 1, (17, ) ) 390 | self.assertFalse( ok, 391 | "should not be able to replace arcs without authorizing" ) 392 | self.assertEquals( 'DENIED', gp.getStatus(), 393 | "command should be denied, not fail" ) 394 | 395 | gp.authorize('password', 396 | test_master + ":" + test_master_password) 397 | ok = gp.try_replace_successors( 1, (17, ) ) 398 | self.assertEquals( 'OK', ok, 399 | "should be able to replace arcs with updater privileges; response: %s" % gp.getResponse() ) 400 | 401 | def test_replacePredecessorsPrivilege(self): 402 | global test_graph_name 403 | global test_master, test_master_password 404 | 405 | self.gp.add_arcs(((1, 11 ), (1, 12 ) ) ) 406 | # add some arcs as admin 407 | 408 | gp = self.newConnection() 409 | gp.use_graph(test_graph_name) 410 | 411 | ok = gp.try_replace_predecessors( 1, (17, ) ) 412 | self.assertFalse( ok, 413 | "should not be able to replace arcs without authorizing" ) 414 | self.assertEquals( 'DENIED', gp.getStatus(), 415 | "command should be denied, not fail" ) 416 | 417 | gp.authorize('password', 418 | test_master + ":" + test_master_password) 419 | ok = gp.try_replace_predecessors( 1, (17, ) ) 420 | self.assertEquals( 'OK', ok, 421 | "should be able to replace arcs with updater privileges; response: %s" % gp.getResponse() ) 422 | 423 | def testClearPrivilege(self): 424 | global test_graph_name 425 | global test_admin, test_admin_password 426 | global test_master, test_master_password 427 | 428 | gp = self.newConnection() 429 | gp.use_graph(test_graph_name) 430 | 431 | ok = gp.try_clear() 432 | self.assertFalse( ok, 433 | "should not be able to clear a graph without authorizing" ) 434 | 435 | gp.authorize('password', 436 | test_master + ":" + test_master_password) 437 | ok = gp.try_clear() 438 | self.assertEquals( ok, 'OK', 439 | "should be able to clear graph with updater privileges" ) 440 | 441 | gp.authorize('password', 442 | test_admin + ":" + test_admin_password) 443 | # re-authenticate 444 | ok = gp.try_clear() 445 | self.assertEquals( ok, 'OK', 446 | "should be able to clear graph with admin privileges" ) 447 | 448 | def test_shutdownPrivilege(self): 449 | global test_graph_name 450 | global test_admin, test_admin_password 451 | global test_master, test_master_password 452 | 453 | gp = self.newConnection() 454 | gp.use_graph(test_graph_name) 455 | 456 | ok = gp.try_shutdown() 457 | self.assertFalse( ok, 458 | "should not be able to shut down a graph without authorizing" ) 459 | 460 | gp.authorize('password', 461 | test_master + ":" + test_master_password) 462 | ok = gp.try_shutdown() 463 | self.assertFalse( ok, "should not be able to shut down a graph " 464 | + "without admin privileges" ) 465 | 466 | gp.authorize('password', 467 | test_admin + ":" + test_admin_password) 468 | # re-authenticate 469 | ok = gp.try_shutdown() 470 | self.assertEquals( ok, 'OK', 471 | "should be able to shut down graph with admin privileges" ) 472 | 473 | 474 | def test_traverseSuccessorsWithout(self): 475 | self.gp.add_arcs( [ 476 | ( 1, 11 ), 477 | ( 1, 12 ), 478 | ( 11, 111 ), 479 | ( 11, 112 ), 480 | ( 111, 1111 ), 481 | ( 111, 1112 ), 482 | ( 112, 1121 ), 483 | ] ) 484 | 485 | self.assertStatsValue( 'ArcCount', 7 ) 486 | 487 | #-------------------------------------------- 488 | succ = self.gp.capture_traverse_successors_without( 11, 5, 111, 5 ) 489 | 490 | self.assertEquals( [ (11,), (112,), (1121,), ], succ ) 491 | 492 | 493 | 494 | 495 | #TODO: (optionally) start server instance here! let it die when the test script dies. 496 | 497 | #TODO: CLI interface behaviour of server (port config, etc) 498 | 499 | if __name__ == '__main__': 500 | unittest.main() 501 | -------------------------------------------------------------------------------- /gp/tests/.svn/text-base/server_test.py.svn-base: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 3 | 4 | import unittest 5 | import os 6 | import tempfile 7 | from test_base import * 8 | from gp.client import * 9 | 10 | test_graph_name = 'test' + str(os.getpid()) 11 | TestFilePrefix = '/tmp/gptest-' + str(os.getpid()) 12 | 13 | 14 | class ServerTest (ClientTestBase, unittest.TestCase): 15 | """Test server functions via client lib.""" 16 | 17 | def test_createGraph(self): 18 | """Graph management functions""" 19 | 20 | global test_graph_name 21 | 22 | name = test_graph_name + "_2" 23 | 24 | # create the graph 25 | self.gp.create_graph(name) 26 | 27 | #make sure we can't create it twice 28 | ok = self.gp.try_create_graph(name) 29 | self.assertFalse( ok, "should not be able to create graph again when it already exists" ) 30 | 31 | # see if we can use the graph from another connection 32 | gp2 = self.newConnection() 33 | 34 | gp2.use_graph(name) 35 | 36 | # see if we can drop the graph while it's used 37 | self.gp.drop_graph(name) 38 | 39 | #TODO: gp2 should now report errors, because the grpah is gone. test that. 40 | 41 | # see if we can re-create the graph after it was dropped 42 | self.gp.create_graph(name) 43 | self.gp.drop_graph(name) 44 | 45 | #TODO: test name restrictions 46 | 47 | 48 | def test_createNameRestrictions(self): 49 | global test_graph_name 50 | 51 | self.gp.strictArguments = False 52 | # disable strict client-side validation 53 | 54 | try: 55 | n = '' 56 | ok = self.gp.create_graph(n) 57 | self.fail("empty graph names should be forbidden!" ) 58 | except gpException, ex: 59 | pass 60 | # ok 61 | 62 | 63 | n = '1337' 64 | ok = self.gp.try_create_graph(n) 65 | self.assertFalse(ok, "numeric graph names should be forbidden! (name: `" + n + "`)" ) 66 | 67 | n = '1337' + test_graph_name 68 | ok = self.gp.try_create_graph(n) 69 | self.assertFalse( ok, 70 | "graph names starting with a number should be forbidden! (name: `" 71 | + n + "`)" ) 72 | 73 | chars = " \r\n\t\0\x09^!\"§\$%&/()[]\ \ =?'#`\\*+~.:, ;<>|@" 74 | for ch in chars: 75 | try: 76 | n = test_graph_name + ch + "test" 77 | ok = self.gp.create_graph(n) 78 | self.fail("graph names containing `" 79 | + ch + "` should be forbidden! (name: `" 80 | + n + "`)" ) 81 | except gpException, ex: 82 | pass 83 | # ok 84 | try: 85 | n = ch + test_graph_name 86 | ok = self.gp.create_graph(n) 87 | self.fail("graph names starting with `" 88 | + ch + "` should be forbidden! (name: `" + n + "`)") 89 | except gpException, ex: 90 | pass 91 | # ok 92 | 93 | n = 'test1337' + test_graph_name 94 | ok = self.gp.try_create_graph(n) 95 | self.assertEquals( 'OK', ok, 96 | "graph names containing numbers should be allowd! (name: `" 97 | + n+ "`)") 98 | self.gp.try_drop_graph(n) 99 | 100 | chars = '-_8' 101 | for ch in chars: 102 | n = 'test' + ch + test_graph_name 103 | ok = self.gp.try_create_graph(n) 104 | self.assertEquals( 'OK', ok, "graph names containing `" 105 | + ch + "` should be allowd! (name: `" + n + "`)") 106 | self.gp.try_drop_graph(n) 107 | 108 | def test_dropGraph(self): 109 | global test_graph_name 110 | 111 | name = test_graph_name + "_2" 112 | 113 | self.gp.create_graph(name) 114 | self.gp.drop_graph(name) 115 | 116 | ok = self.gp.try_use_graph(name) 117 | self.assertFalse( ok, 118 | "should not be able to use graph after dropping it" ) 119 | ok = self.gp.try_drop_graph(name) 120 | self.assertEquals( 'NONE', ok, "should not be able to drop " 121 | + "graph again after it was already dropped." ) 122 | 123 | def test_listGraphs(self): 124 | global test_graph_name 125 | 126 | gp2 = self.newConnection() 127 | graphs = gp2.capture_list_graphs() 128 | graphs = array_column(graphs, 0) 129 | self.assertTrue( test_graph_name in graphs, 130 | "test table test_graph_name should be in the list" ) 131 | 132 | self.gp.drop_graph(test_graph_name) 133 | graphs = gp2.capture_list_graphs() 134 | #print "graphs: " . var_export($graphs, true) . "\n" 135 | 136 | graphs = array_column( graphs, 0 ) 137 | 138 | #print "graphs: " . var_export($graphs, true) . "\n" 139 | 140 | #print "containes: " . var_export(ConnectionTestBase::setContains( $graphs, test_graph_name ), true) . "\n" 141 | 142 | self.assertFalse( 143 | ConnectionTestBase.setContains(graphs, test_graph_name), 144 | "test table test_graph_name should no longer be in the list" ) 145 | 146 | def test_shutdown(self): 147 | global test_graph_name 148 | 149 | gp2 = self.newConnection() 150 | gp2.use_graph(test_graph_name) 151 | gp2.stats() 152 | 153 | self.assertSessionValue('ConnectedGraph', test_graph_name) 154 | 155 | self.gp.shutdown() # <------------------ 156 | # self.assertSessionValue('ConnectedGraph', 'None'); 157 | # nice, but not reliable. race condition. 158 | 159 | self.gp.try_stats() 160 | self.assertEquals( 'FAILED', self.gp.getStatus(), 161 | 'fetching stats should fail after shutdown' ) 162 | 163 | gp2.try_stats() 164 | self.assertEquals( 'FAILED', gp2.getStatus(), 165 | 'fetching stats should fail after shutdown' ) 166 | gp2.close() 167 | 168 | gp3 = self.newConnection() 169 | gp3.try_use_graph(test_graph_name) 170 | self.assertEquals( 'FAILED', gp3.getStatus(), 171 | 'graph should be unavailable after shutdown' ) 172 | gp3.close() 173 | 174 | 175 | def test_quit(self): 176 | global test_graph_name 177 | 178 | gp2 = self.newConnection() 179 | gp2.use_graph(test_graph_name) 180 | gp2.stats() 181 | 182 | self.assertSessionValue('ConnectedGraph', test_graph_name) 183 | 184 | self.gp.quit() # <------------------ 185 | self.assertStatus('OK') 186 | 187 | try: 188 | self.gp.try_stats() 189 | self.fail( 'connection should be unusable after quit' ) 190 | except gpProtocolException, e: 191 | pass 192 | # ok 193 | 194 | 195 | gp2.stats() 196 | self.assertEquals( 'OK', gp2.getStatus(), 197 | 'connection should still be usable by others after quit; response: %s' % gp2.getResponse() ) 198 | gp2.close() 199 | 200 | gp3 = self.newConnection() 201 | gp3.use_graph(test_graph_name) 202 | self.assertEquals( 'OK', gp3.getStatus(), 203 | 'graph should still be available to others after quit; response: %s' % gp2.getResponse() ) 204 | gp3.close() 205 | 206 | 207 | # privileges 208 | def test_createGraphPrivilege(self): 209 | global test_graph_name 210 | global test_admin, test_admin_password 211 | global test_master, test_master_password 212 | 213 | name = test_graph_name + "_2" 214 | 215 | gp = self.newConnection() 216 | 217 | ok = gp.try_create_graph(name) 218 | self.assertFalse( ok, 219 | "should not be able to create a graph without authorizing" ) 220 | 221 | gp.authorize('password', 222 | test_master + ":" + test_master_password) 223 | ok = gp.try_create_graph(name) 224 | self.assertFalse( ok, 225 | "should not be able to create a graph without admin privileges" ) 226 | 227 | gp.authorize('password', 228 | test_admin + ":" + test_admin_password) 229 | # re-authenticate 230 | ok = gp.create_graph(name) 231 | self.assertEquals( ok, 'OK', 232 | "should be able to create graph with admin privileges; response: %s" % gp.getResponse() ) 233 | 234 | gp.try_drop_graph(name) 235 | # cleanup 236 | 237 | 238 | def test_dropGraphPrivilege(self): 239 | global test_graph_name 240 | global test_admin, test_admin_password 241 | global test_master, test_master_password 242 | 243 | name = test_graph_name 244 | 245 | gp = self.newConnection() 246 | 247 | ok = gp.try_drop_graph(name) 248 | self.assertFalse( ok, "should not be able to drop a graph without authorizing" ) 249 | 250 | gp.authorize('password', 251 | test_master + ":" + test_master_password) 252 | ok = gp.try_drop_graph(name) 253 | self.assertFalse( ok, 254 | "should not be able to drop a graph without admin privileges" ) 255 | 256 | gp.authorize('password', 257 | test_admin + ":" + test_admin_password) 258 | # re-authenticate 259 | ok = gp.drop_graph(name) 260 | self.assertEquals( ok, 'OK', 261 | "should be able to drop graph with admin privileges; response: %s" % gp.getResponse() ) 262 | 263 | def test_inputPipingPrivilege(self): 264 | global test_graph_name, test_graphserv_host 265 | global test_admin, test_admin_password 266 | global test_master, test_master_password 267 | 268 | #XXX: this uses local files, so it will always fail 269 | # if the server isn't on localhost! 270 | if test_graphserv_host != 'localhost': 271 | return None 272 | 273 | f = os.path.dirname(os.path.abspath(__file__)) + '/gp.test.data' 274 | 275 | gp = self.newConnection() 276 | gp.use_graph(test_graph_name) 277 | gp.allowPipes = True 278 | 279 | gp.authorize('password', 280 | test_master + ":" + test_master_password) 281 | 282 | try: 283 | ok = gp.execute("add-arcs < " + f) 284 | self.fail( 285 | "should not be able to pipe without admin privileges!" ) 286 | except gpProcessorException, ex: 287 | self.assertEquals( 'DENIED', gp.getStatus(), 288 | "piping should be denied, not fail. Message: " 289 | + str(ex)) 290 | 291 | 292 | gp.authorize('password', test_admin + ":" + test_admin_password) 293 | # re-authenticate 294 | ok = gp.execute("add-arcs < " + f) 295 | self.assertEquals( ok, 'OK', 296 | "should be able to pipe with admin privileges; response: %s" % gp.getResponse() ) 297 | 298 | 299 | def test_outputPipingPrivilege(self): 300 | global test_graph_name, test_graphserv_host 301 | global test_admin, test_admin_password 302 | global test_master, test_master_password 303 | 304 | #XXX: this uses local files, so it will always fail 305 | # if the server isn't on localhost! 306 | if test_graphserv_host != 'localhost': 307 | return None 308 | 309 | f = tempfile.mktemp(suffix='gpt') 310 | 311 | gp = self.newConnection() 312 | gp.use_graph(test_graph_name) 313 | gp.allowPipes = True 314 | 315 | try: 316 | ok = gp.execute("list-roots > " + f) 317 | self.fail( 318 | "should not be able to pipe without admin privileges!" ) 319 | except gpProcessorException, ex: 320 | self.assertEquals( 'DENIED', gp.getStatus(), 321 | "piping should be denied, not fail. Message: " 322 | + str(ex)) 323 | 324 | gp.authorize( 325 | 'password', test_admin + ":" + test_admin_password) 326 | # re-authenticate 327 | ok = gp.execute("list-roots > " + f) 328 | self.assertEquals( 329 | ok, 'OK', "should be able to pipe with admin privileges; response: %s" % gp.getResponse() ) 330 | 331 | try: 332 | unlink(f) 333 | # cleanup 334 | except: 335 | pass 336 | 337 | def test_addArcsPrivilege(self): 338 | global test_graph_name 339 | global test_master, test_master_password 340 | 341 | gp = self.newConnection() 342 | gp.use_graph(test_graph_name) 343 | 344 | ok = gp.try_add_arcs(((1, 11 ), (1, 12 ) ) ) 345 | self.assertFalse( 346 | ok, "should not be able to add arcs without authorizing" ) 347 | self.assertEquals('DENIED', gp.getStatus(), 348 | "command should be denied, not fail" ) 349 | 350 | gp.authorize('password', 351 | test_master + ":" + test_master_password) 352 | ok = gp.try_add_arcs(((1, 11 ), (1, 12 ) ) ) 353 | self.assertEquals( 'OK', ok, 354 | "should be able to add arcs with updater privileges; response: %s" % gp.getResponse() ) 355 | 356 | def test_removeArcsPrivilege(self): 357 | global test_graph_name 358 | global test_master, test_master_password 359 | 360 | self.gp.add_arcs(((1, 11 ), (1, 12 ) ) ) 361 | # add some arcs as admin 362 | 363 | gp = self.newConnection() 364 | gp.use_graph(test_graph_name) 365 | 366 | ok = gp.try_remove_arcs(((1, 11 ), ) ) 367 | self.assertFalse( ok, 368 | "should not be able to delete arcs without authorizing" ) 369 | self.assertEquals( 'DENIED', gp.getStatus(), 370 | "command should be denied, not fail" ) 371 | 372 | gp.authorize('password', 373 | test_master + ":" + test_master_password) 374 | 375 | ok = gp.try_remove_arcs(((1, 11 ), ) ) 376 | self.assertEquals( 'OK', ok, 377 | "should be able to delete arcs with updater privileges; response: %s" % gp.getResponse() ) 378 | 379 | def test_replaceSuccessorsPrivilege(self): 380 | global test_graph_name 381 | global test_master, test_master_password 382 | 383 | self.gp.add_arcs(((1, 11 ), (1, 12 ) ) ) 384 | # add some arcs as admin 385 | 386 | gp = self.newConnection() 387 | gp.use_graph(test_graph_name) 388 | 389 | ok = gp.try_replace_successors( 1, (17, ) ) 390 | self.assertFalse( ok, 391 | "should not be able to replace arcs without authorizing" ) 392 | self.assertEquals( 'DENIED', gp.getStatus(), 393 | "command should be denied, not fail" ) 394 | 395 | gp.authorize('password', 396 | test_master + ":" + test_master_password) 397 | ok = gp.try_replace_successors( 1, (17, ) ) 398 | self.assertEquals( 'OK', ok, 399 | "should be able to replace arcs with updater privileges; response: %s" % gp.getResponse() ) 400 | 401 | def test_replacePredecessorsPrivilege(self): 402 | global test_graph_name 403 | global test_master, test_master_password 404 | 405 | self.gp.add_arcs(((1, 11 ), (1, 12 ) ) ) 406 | # add some arcs as admin 407 | 408 | gp = self.newConnection() 409 | gp.use_graph(test_graph_name) 410 | 411 | ok = gp.try_replace_predecessors( 1, (17, ) ) 412 | self.assertFalse( ok, 413 | "should not be able to replace arcs without authorizing" ) 414 | self.assertEquals( 'DENIED', gp.getStatus(), 415 | "command should be denied, not fail" ) 416 | 417 | gp.authorize('password', 418 | test_master + ":" + test_master_password) 419 | ok = gp.try_replace_predecessors( 1, (17, ) ) 420 | self.assertEquals( 'OK', ok, 421 | "should be able to replace arcs with updater privileges; response: %s" % gp.getResponse() ) 422 | 423 | def testClearPrivilege(self): 424 | global test_graph_name 425 | global test_admin, test_admin_password 426 | global test_master, test_master_password 427 | 428 | gp = self.newConnection() 429 | gp.use_graph(test_graph_name) 430 | 431 | ok = gp.try_clear() 432 | self.assertFalse( ok, 433 | "should not be able to clear a graph without authorizing" ) 434 | 435 | gp.authorize('password', 436 | test_master + ":" + test_master_password) 437 | ok = gp.try_clear() 438 | self.assertEquals( ok, 'OK', 439 | "should be able to clear graph with updater privileges" ) 440 | 441 | gp.authorize('password', 442 | test_admin + ":" + test_admin_password) 443 | # re-authenticate 444 | ok = gp.try_clear() 445 | self.assertEquals( ok, 'OK', 446 | "should be able to clear graph with admin privileges" ) 447 | 448 | def test_shutdownPrivilege(self): 449 | global test_graph_name 450 | global test_admin, test_admin_password 451 | global test_master, test_master_password 452 | 453 | gp = self.newConnection() 454 | gp.use_graph(test_graph_name) 455 | 456 | ok = gp.try_shutdown() 457 | self.assertFalse( ok, 458 | "should not be able to shut down a graph without authorizing" ) 459 | 460 | gp.authorize('password', 461 | test_master + ":" + test_master_password) 462 | ok = gp.try_shutdown() 463 | self.assertFalse( ok, "should not be able to shut down a graph " 464 | + "without admin privileges" ) 465 | 466 | gp.authorize('password', 467 | test_admin + ":" + test_admin_password) 468 | # re-authenticate 469 | ok = gp.try_shutdown() 470 | self.assertEquals( ok, 'OK', 471 | "should be able to shut down graph with admin privileges" ) 472 | 473 | 474 | def test_traverseSuccessorsWithout(self): 475 | self.gp.add_arcs( [ 476 | ( 1, 11 ), 477 | ( 1, 12 ), 478 | ( 11, 111 ), 479 | ( 11, 112 ), 480 | ( 111, 1111 ), 481 | ( 111, 1112 ), 482 | ( 112, 1121 ), 483 | ] ) 484 | 485 | self.assertStatsValue( 'ArcCount', 7 ) 486 | 487 | #-------------------------------------------- 488 | succ = self.gp.capture_traverse_successors_without( 11, 5, 111, 5 ) 489 | 490 | self.assertEquals( [ (11,), (112,), (1121,), ], succ ) 491 | 492 | 493 | 494 | 495 | #TODO: (optionally) start server instance here! let it die when the test script dies. 496 | 497 | #TODO: CLI interface behaviour of server (port config, etc) 498 | 499 | if __name__ == '__main__': 500 | unittest.main() 501 | -------------------------------------------------------------------------------- /gp/mediawiki.py: -------------------------------------------------------------------------------- 1 | from client import * 2 | from mysql import * 3 | 4 | import re 5 | 6 | NS_MAIN = 0 7 | NS_TALK = 1 8 | NS_USER = 2 9 | NS_USER_TALK = 3 10 | NS_PROJECT = 4 11 | NS_PROJECT_TALK = 5 12 | NS_FILE = 6 13 | NS_FILE_TALK = 7 14 | NS_MEDIAWIKI = 8 15 | NS_MEDIAWIKI_TALK = 9 16 | NS_TEMPLATE = 10 17 | NS_TEMPLATE_TALK = 11 18 | NS_HELP = 12 19 | NS_HELP_TALK = 13 20 | NS_CATEGORY = 14 21 | NS_CATEGORY_TALK = 15 22 | 23 | 24 | class MediaWikiGlue (MySQLGlue) : 25 | 26 | def __init__( self, transport, graphname = None ) : 27 | super(MediaWikiGlue, self).__init__(transport, graphname) 28 | 29 | self.table_prefix = "" 30 | 31 | #h = array( self, 'gp_mediawiki_exec_handler' ) 32 | #self.addExecHandler( h ) 33 | 34 | 35 | def set_table_prefix ( self, prefix ) : 36 | self.table_prefix = prefix 37 | 38 | 39 | def get_db_key ( self, name ) : 40 | if name is None or name == False: 41 | raise gpUsageException("name must not be empty!") 42 | 43 | #TODO: use native MediaWiki method if available 44 | name = name.strip() 45 | 46 | if name == "": 47 | raise gpUsageException("name must not be empty!") 48 | 49 | name = re.sub(' ', '_', name) 50 | 51 | result = name[0].upper() + name[1:] #FIXME: unreliable, handle unicode! 52 | 53 | return name 54 | 55 | 56 | def wiki_table ( self, name ) : 57 | return self.table_prefix + name 58 | 59 | 60 | def get_page_id ( self, ns, title ) : 61 | sql = "select page_id from " + self.wiki_table( "page" ) 62 | sql += " where page_namespace = %i" % int(ns) 63 | sql += " and page_title = " + self.quote_string( self.get_db_key(title) ) 64 | 65 | id = self.mysql_query_value( sql ) 66 | return id 67 | 68 | 69 | def add_arcs_from_category_structure ( self, ) : 70 | sql = "select C.page_id as parent, P.page_id as child" 71 | sql += " from " + self.wiki_table( "page" ) + " as P " 72 | sql += " join " + self.wiki_table( "categorylinks" ) + " as X " 73 | sql += " on X.cl_from = P.page_id " 74 | sql += " join " + self.wiki_table( "page" ) + " as C " 75 | sql += " on C.page_namespace = %i" % NS_CATEGORY 76 | sql += " and C.page_title = X.cl_to " 77 | sql += " where P.page_namespace = %i" % NS_CATEGORY 78 | 79 | src = self.make_source( MySQLSelect( sql ) ) 80 | 81 | self.add_arcs( src ) 82 | src.close() 83 | 84 | 85 | def get_subcategories ( self, cat, depth, without = None, without_depth = None ) : 86 | sink = ArraySink() 87 | 88 | id = self.get_page_id( NS_CATEGORY, cat ) 89 | if ( not id ): return 'NONE' 90 | 91 | if ( without ): without_id = self.get_page_id( NS_CATEGORY, without ) 92 | else: without_id = False 93 | 94 | temp = self.make_temp_sink( MySQLTable('?', 'id') ) 95 | 96 | if ( without_id ) : 97 | if ( not without_depth ): without_depth = depth 98 | status = self.traverse_successors_without( id, depth, without_id, without_depth, temp ) 99 | else : 100 | status = self.traverse_successors( id, depth, temp ) 101 | 102 | 103 | temp.close() 104 | 105 | if ( status == 'OK' ) : 106 | sql = "select page_title " 107 | sql += " from " + self.wiki_table( "page" ) 108 | sql += " join " + temp.getTable().get_name() 109 | sql += " on id = page_id " 110 | sql += " where page_namespace = %i" % NS_CATEGORY # should be redundant 111 | sql += " order by page_id " 112 | 113 | self.select_into( sql , sink) 114 | 115 | 116 | temp.drop() 117 | 118 | return sink.getData() 119 | 120 | @staticmethod 121 | def new_client_connection( graphname, host = False, port = False ) : 122 | return MediaWikiGlue( ClientTransport(host, port), graphname ) #FIXME: PORT graphname stuff to PHP! 123 | 124 | @staticmethod 125 | def new_slave_connection( command, cwd = None, env = None ) : 126 | return MediaWikiGlue( SlaveTransport(command, cwd, env), None ) 127 | 128 | 129 | 130 | 131 | class PageSet : 132 | 133 | def __init__ ( self, glue, table = "?", id_field = "page_id", namespace_field = "page_namespace", title_field = "page_title", big = True ) : 134 | self.big = big 135 | 136 | self.glue = glue 137 | self.table = table 138 | 139 | self.id_field = id_field 140 | self.namespace_field = namespace_field 141 | self.title_field = title_field 142 | 143 | self.table_obj = MySQLTable( self.table, self.id_field, self.namespace_field, self.title_field ) 144 | self.table_obj.set_field_definition( self.id_field, "INT NOT NULL") 145 | self.table_obj.set_field_definition( self.namespace_field, "INT DEFAULT NULL") 146 | self.table_obj.set_field_definition( self.title_field, "VARCHAR(255) BINARY DEFAULT NULL") 147 | self.table_obj.add_key_definition( "PRIMARY KEY (" + self.id_field + ")" ) 148 | self.table_obj.add_key_definition( "UNIQUE KEY (" + self.namespace_field + ", " + self.title_field + ")" ) 149 | 150 | self.table_id_obj = MySQLTable( self.table, self.id_field ) 151 | self.table_id_obj.add_key_definition( "PRIMARY KEY (" + self.id_field + ")" ) 152 | 153 | 154 | def set_expect_big ( self, big ) : 155 | self.big = big 156 | 157 | 158 | def get_table ( self, ) : 159 | return self.table_obj 160 | 161 | 162 | def create_table ( self, ) : 163 | table = self.table 164 | t = "" 165 | 166 | if ( not table or table == '?' ) : 167 | table = "gp_temp_%s" % self.glue.next_id() 168 | t = " TEMPORARY " 169 | 170 | 171 | sql = "CREATE " + t + " TABLE " + table 172 | sql += "(" 173 | sql += self.table_obj.get_field_definitions() 174 | sql += ")" 175 | 176 | self._update(sql) 177 | 178 | self.table = table 179 | self.table_obj.set_name( self.table ) 180 | self.table_id_obj.set_name( self.table ) 181 | 182 | return table 183 | 184 | 185 | 186 | def _query( self, sql, **kwargs ) : 187 | if not 'unbuffered' in kwargs: 188 | kwargs['unbuffered'] = self.big 189 | 190 | return self.glue.mysql_query(sql, **kwargs) #TODO: port kwargs to PHP 191 | 192 | def _update( self, sql, **kwargs ) : #TODO: port to PHP; use in PHP! 193 | return self.glue.mysql_update(sql, **kwargs) 194 | 195 | def add_from_select ( self, select, comment = None ) : 196 | sql= "REPLACE INTO " + self.table + " " 197 | sql += "( " 198 | sql += self.id_field + ", " 199 | sql += self.namespace_field + ", " 200 | sql += self.title_field + " ) " 201 | sql += select 202 | 203 | return self._update( sql, comment = comment ) 204 | 205 | 206 | def delete_where ( self, where, comment = None ) : 207 | sql= "DELETE FROM " + self.table + " " 208 | sql += where 209 | 210 | return self._update( sql, comment = comment ) 211 | 212 | 213 | def delete_using ( self, using, tableAlias = "T", comment = None ) : 214 | sql= "DELETE FROM " + tableAlias + " " 215 | sql += "USING " + self.table + " AS " + tableAlias + " " 216 | sql += using 217 | 218 | return self._update( sql, comment = comment ) 219 | 220 | 221 | def resolve_ids ( self, comment = None ) : 222 | #NOTE: MySQL can't perform self-joins on temp tables. so we need to copy the ids to another temp table first. 223 | t = MySQLTable("?", "page_id") 224 | t.add_key_definition("PRIMARY KEY (page_id)") 225 | 226 | tmp = self.glue.make_temp_table( t ) 227 | 228 | sql = tmp.get_insert(True) 229 | sql += "SELECT " + self.id_field 230 | sql += " FROM " + self.table 231 | sql += " WHERE page_title IS NULL" 232 | 233 | self._update( sql ); #copy page ids with no page title into temp table 234 | 235 | sql = "SELECT P.page_id, P.page_namespace, P.page_title " 236 | sql += " FROM " + self.glue.wiki_table("page") + " AS P " 237 | sql += " JOIN " + tmp.get_name() + " AS T ON T.page_id = P.page_id" 238 | 239 | self.add_from_select( sql, comment = comment ) #TODO: port comment to PHP 240 | 241 | self.glue.drop_temp_table( tmp ) 242 | return True 243 | 244 | 245 | def make_sink ( self, ) : 246 | sink = self.glue.make_sink( self.table_obj ) 247 | return sink 248 | 249 | 250 | def make_id_sink ( self, ) : 251 | sink = self.glue.make_sink( self.table_id_obj ) 252 | return sink 253 | 254 | 255 | def make_id_source ( self, ns = None ) : 256 | return self.make_source( ns, True ) 257 | 258 | 259 | def make_source ( self, ns = None, ids_only = False, auto_order = False ) : #TODO: PORT auto_order to PHP 260 | t = self.table_id_obj if ids_only else self.table_obj 261 | 262 | if ( ns is not None ) : 263 | select = t._get_select() 264 | 265 | if ( isinstance(ns, (tuple, list, set)) ): select += " where page_namespace in " + self.glue.as_list( ns ) 266 | else: select += " where page_namespace = %i" % int(ns) 267 | 268 | t = MySQLSelect(select) 269 | 270 | 271 | src = self.glue.make_source( t, big = self.big, auto_order = auto_order ) 272 | return src 273 | 274 | 275 | def capture ( self, ns = None, data = None ) : 276 | sink = ArraySink( data ) 277 | self.copy_to_sink( ns, sink ) 278 | return sink.getData() 279 | 280 | 281 | def capture_ids ( self, ns = None, data = None ) : 282 | sink = ArraySink( data ) 283 | self.copy_ids_to_sink( ns, sink ) 284 | return sink.getData() 285 | 286 | 287 | def copy_to_sink ( self, ns, sink ) : 288 | src = self.make_source(ns) 289 | c = self.glue.copy(src, sink, "~") 290 | src.close() 291 | return c 292 | 293 | 294 | def copy_ids_to_sink ( self, ns, sink ) : 295 | src = self.make_id_source(ns) 296 | c = self.glue.copy(src, sink, "~") 297 | src.close() 298 | return c 299 | 300 | 301 | def add_source ( self, src ) : 302 | sink = self.make_sink() 303 | c = self.glue.copy( src, sink, "+" ) 304 | sink.close() 305 | return c 306 | 307 | 308 | def add_page_set ( self, set ) : 309 | select = set.get_table()._get_select() 310 | return self.add_from_select( select ) 311 | 312 | 313 | def subtract_page_set ( self, set ) : 314 | t = set.get_table() 315 | return self.subtract_table( t ) 316 | 317 | 318 | def subtract_source ( self, src ): #XXX: must be a 1 column id source... 319 | t = MySQLTable("?", "page_id") 320 | sink = self.glue.make_temp_sink( t ) 321 | t = sink.getTable() 322 | 323 | self.glue.copy( src, sink, "+" ) 324 | 325 | ok = self.subtract_table(t, "page_id") 326 | 327 | self.glue.drop_temp_table(t) 328 | sink.close() 329 | 330 | return ok 331 | 332 | 333 | def retain_page_set ( self, set ) : 334 | t = set.get_table() 335 | return self.retain_table( t ) 336 | 337 | 338 | def retain_source ( self, src ) : #XXX: must be a 1 column id source... 339 | t = MySQLTable("?", "page_id") 340 | sink = self.glue.make_temp_sink( t ) 341 | t = sink.getTable() 342 | 343 | self.glue.copy( src, sink, "+" ) 344 | 345 | ok = self.retain_table(t, "page_id") 346 | 347 | self.glue.drop_temp_table(t) 348 | sink.close() 349 | 350 | return ok 351 | 352 | 353 | def subtract_table ( self, table, id_field = None ) : 354 | if ( not id_field ): id_field = table.get_field1() 355 | 356 | sql = "DELETE FROM T " 357 | sql += " USING " + self.table + " AS T " 358 | sql += " JOIN " + table.get_name() + " AS R " 359 | sql += " ON T." + self.id_field + " = R." + id_field 360 | 361 | self._update(sql) 362 | return True 363 | 364 | 365 | def retain_table ( self, table, id_field = None ) : 366 | if ( not id_field ): id_field = table.get_field1() 367 | 368 | sql = "DELETE FROM T " 369 | sql += " USING " + self.table + " AS T " 370 | sql += " LEFT JOIN " + table.get_name() + " AS R " 371 | sql += " ON T." + self.id_field + " = R." + id_field 372 | sql += " WHERE R." + id_field + " IS NULL" 373 | 374 | self._update(sql) 375 | return True 376 | 377 | 378 | def remove_page ( self, ns, title ) : 379 | sql = "DELETE FROM " + self.table 380 | sql += " WHERE " + self.namespace_field + " = %i" % int(ns) 381 | sql += " AND " + self.title_field + " = " + self.glue.quote_string(title) 382 | 383 | self._update(sql) 384 | return True 385 | 386 | 387 | def remove_page_id ( self, id ) : 388 | sql = "DELETE FROM " + self.table 389 | sql += " WHERE " + self.id_field + " = %i" % int(id) 390 | 391 | self._update(sql) 392 | return True 393 | 394 | 395 | def strip_namespace ( self, ns, inverse = False ) : 396 | sql = "DELETE FROM " + self.table 397 | sql += " WHERE " + self.namespace_field 398 | 399 | if ( isinstance(ns, (tuple, list, set)) ): sql += ( " not in " if inverse else " in " ) + self.glue.as_list( ns ) 400 | else: sql += ( " != " if inverse else " = " ) + str(int(ns)) 401 | 402 | self._update(sql) 403 | return True 404 | 405 | 406 | def retain_namespace ( self, ns ) : 407 | return self.strip_namespace( ns, True ) 408 | 409 | 410 | def add_page ( self, id, ns, title ) : 411 | if ( not id ): id = self.glue.get_page_id( NS_CATEGORY, cat ) 412 | 413 | values = array(id, ns, title) 414 | 415 | sql = self.table_obj.insert_command() 416 | sql += " VALUES " 417 | sql += self.glue.as_list(values) 418 | 419 | self._update( sql ) 420 | return True 421 | 422 | 423 | def add_page_id ( self, id ) : 424 | values = array(id) 425 | 426 | sql = "INSERT IGNORE INTO " + self.table 427 | sql += " ( " + self.id_field + " ) " 428 | sql += " VALUES " 429 | sql += self.glue.as_list(values) 430 | 431 | self._update( sql ) 432 | return True 433 | 434 | 435 | def expand_categories ( self, ns = None, comment = None ) : 436 | #NOTE: MySQL can't perform self-joins on temp tables. so we need to copy the category names to another temp table first. 437 | t = MySQLTable("?", "cat_title") 438 | t.set_field_definition("cat_title", "VARCHAR(255) BINARY NOT NULL") 439 | t.add_key_definition("PRIMARY KEY (cat_title)") 440 | 441 | tmp = self.glue.make_temp_table( t ) 442 | 443 | sql = tmp.get_insert(True) 444 | sql += " select page_title " 445 | sql += " from " + self.table + " as T " 446 | sql += " where page_namespace = %i " % NS_CATEGORY 447 | 448 | self._update( sql ) 449 | #self.glue.dump_query("select * from " +tmp.get_name()) 450 | 451 | # ---------------------------------------------------------- 452 | sql = "select P.page_id, P.page_namespace, P.page_title " 453 | sql += " from " + self.glue.wiki_table( "page" ) + " as P " 454 | sql += " join " + self.glue.wiki_table( "categorylinks" ) + " as X " 455 | sql += " on X.cl_from = P.page_id " 456 | sql += " join " + tmp.get_name() + " as T " 457 | sql += " on T.cat_title = X.cl_to " 458 | 459 | if (ns is not None) : 460 | if ( isinstance(ns, (tuple, list, set)) ): sql += " where P.page_namespace in " + self.glue.as_list( ns ) 461 | else: sql += " where P.page_namespace = %i" % int(ns) 462 | 463 | 464 | #self.glue.dump_query(sql) 465 | self.add_from_select( sql, comment = comment ) #TODO: port comment to PHP 466 | 467 | #self.glue.dump_query("select * from " +self.table) 468 | self.glue.drop_temp_table( tmp ) 469 | return True 470 | 471 | 472 | def add_subcategories ( self, cat, depth, without = None, without_depth = None ) : 473 | self._add_subcategory_ids(cat, depth, without, without_depth) 474 | self.resolve_ids() 475 | return True 476 | 477 | 478 | def _add_subcategory_ids( self, cat, depth, without = None, without_depth = None ) : 479 | id = self.glue.get_page_id( NS_CATEGORY, cat ) 480 | if ( not id ): return False 481 | 482 | if ( without ): without_id = self.glue.get_page_id( NS_CATEGORY, without ) 483 | else: without_id = False 484 | 485 | sink = self.make_id_sink() 486 | 487 | if ( without_id ) : 488 | if ( not without_depth ): without_depth = depth 489 | status = self.glue.traverse_successors_without( id, depth, without_id, without_depth, sink ) 490 | else : 491 | status = self.glue.traverse_successors( id, depth, sink ) 492 | 493 | 494 | sink.close() 495 | return True 496 | 497 | def get_size(self): 498 | res = self._query("SELECT COUNT(*) FROM " + self.table) 499 | try: 500 | row = res.fetchone() 501 | finally: 502 | res.close() 503 | 504 | return row[0] 505 | 506 | def add_pages_in ( self, cat, ns, depth, comment = None ) : 507 | self.get_size() 508 | 509 | if ( not self.add_subcategories(cat, depth) ): 510 | return False 511 | 512 | self.get_size() # ?! 513 | 514 | self.expand_categories(ns, comment = comment) 515 | return True 516 | 517 | 518 | def add_pages_transclusing ( self, tag, ns = None, comment = None ) : 519 | if ( ns is None ): ns = NS_TEMPLATE 520 | tag = self.glue.get_db_key( tag ) 521 | 522 | sql = " SELECT page_id, page_namespace, page_title " 523 | sql += " FROM " + self.glue.wiki_table( "page" ) 524 | sql += " JOIN " + self.glue.wiki_table( "templatelinks" ) 525 | sql += " ON tl_from = page_id " 526 | sql += " WHERE tl_namespace = %i" % int(ns) 527 | sql += " AND tl_title = " + self.glue.quote_string(tag) 528 | 529 | return self.add_from_select(sql, comment = comment) 530 | 531 | 532 | def clear ( self, ) : 533 | sql = "TRUNCATE " + self.table 534 | self._update(sql) 535 | return True 536 | 537 | 538 | def dispose ( self, ) : 539 | sql = "DROP TEMPORARY TABLE " + self.table 540 | self._update(sql) 541 | return True 542 | 543 | 544 | 545 | -------------------------------------------------------------------------------- /gp/.svn/text-base/mediawiki.py.svn-base: -------------------------------------------------------------------------------- 1 | from client import * 2 | from mysql import * 3 | 4 | import re 5 | 6 | NS_MAIN = 0 7 | NS_TALK = 1 8 | NS_USER = 2 9 | NS_USER_TALK = 3 10 | NS_PROJECT = 4 11 | NS_PROJECT_TALK = 5 12 | NS_FILE = 6 13 | NS_FILE_TALK = 7 14 | NS_MEDIAWIKI = 8 15 | NS_MEDIAWIKI_TALK = 9 16 | NS_TEMPLATE = 10 17 | NS_TEMPLATE_TALK = 11 18 | NS_HELP = 12 19 | NS_HELP_TALK = 13 20 | NS_CATEGORY = 14 21 | NS_CATEGORY_TALK = 15 22 | 23 | 24 | class MediaWikiGlue (MySQLGlue) : 25 | 26 | def __init__( self, transport, graphname = None ) : 27 | super(MediaWikiGlue, self).__init__(transport, graphname) 28 | 29 | self.table_prefix = "" 30 | 31 | #h = array( self, 'gp_mediawiki_exec_handler' ) 32 | #self.addExecHandler( h ) 33 | 34 | 35 | def set_table_prefix ( self, prefix ) : 36 | self.table_prefix = prefix 37 | 38 | 39 | def get_db_key ( self, name ) : 40 | if name is None or name == False: 41 | raise gpUsageException("name must not be empty!") 42 | 43 | #TODO: use native MediaWiki method if available 44 | name = name.strip() 45 | 46 | if name == "": 47 | raise gpUsageException("name must not be empty!") 48 | 49 | name = re.sub(' ', '_', name) 50 | 51 | result = name[0].upper() + name[1:] #FIXME: unreliable, handle unicode! 52 | 53 | return name 54 | 55 | 56 | def wiki_table ( self, name ) : 57 | return self.table_prefix + name 58 | 59 | 60 | def get_page_id ( self, ns, title ) : 61 | sql = "select page_id from " + self.wiki_table( "page" ) 62 | sql += " where page_namespace = %i" % int(ns) 63 | sql += " and page_title = " + self.quote_string( self.get_db_key(title) ) 64 | 65 | id = self.mysql_query_value( sql ) 66 | return id 67 | 68 | 69 | def add_arcs_from_category_structure ( self, ) : 70 | sql = "select C.page_id as parent, P.page_id as child" 71 | sql += " from " + self.wiki_table( "page" ) + " as P " 72 | sql += " join " + self.wiki_table( "categorylinks" ) + " as X " 73 | sql += " on X.cl_from = P.page_id " 74 | sql += " join " + self.wiki_table( "page" ) + " as C " 75 | sql += " on C.page_namespace = %i" % NS_CATEGORY 76 | sql += " and C.page_title = X.cl_to " 77 | sql += " where P.page_namespace = %i" % NS_CATEGORY 78 | 79 | src = self.make_source( MySQLSelect( sql ) ) 80 | 81 | self.add_arcs( src ) 82 | src.close() 83 | 84 | 85 | def get_subcategories ( self, cat, depth, without = None, without_depth = None ) : 86 | sink = ArraySink() 87 | 88 | id = self.get_page_id( NS_CATEGORY, cat ) 89 | if ( not id ): return 'NONE' 90 | 91 | if ( without ): without_id = self.get_page_id( NS_CATEGORY, without ) 92 | else: without_id = False 93 | 94 | temp = self.make_temp_sink( MySQLTable('?', 'id') ) 95 | 96 | if ( without_id ) : 97 | if ( not without_depth ): without_depth = depth 98 | status = self.traverse_successors_without( id, depth, without_id, without_depth, temp ) 99 | else : 100 | status = self.traverse_successors( id, depth, temp ) 101 | 102 | 103 | temp.close() 104 | 105 | if ( status == 'OK' ) : 106 | sql = "select page_title " 107 | sql += " from " + self.wiki_table( "page" ) 108 | sql += " join " + temp.getTable().get_name() 109 | sql += " on id = page_id " 110 | sql += " where page_namespace = %i" % NS_CATEGORY # should be redundant 111 | sql += " order by page_id " 112 | 113 | self.select_into( sql , sink) 114 | 115 | 116 | temp.drop() 117 | 118 | return sink.getData() 119 | 120 | @staticmethod 121 | def new_client_connection( graphname, host = False, port = False ) : 122 | return MediaWikiGlue( ClientTransport(host, port), graphname ) #FIXME: PORT graphname stuff to PHP! 123 | 124 | @staticmethod 125 | def new_slave_connection( command, cwd = None, env = None ) : 126 | return MediaWikiGlue( SlaveTransport(command, cwd, env), None ) 127 | 128 | 129 | 130 | 131 | class PageSet : 132 | 133 | def __init__ ( self, glue, table = "?", id_field = "page_id", namespace_field = "page_namespace", title_field = "page_title", big = True ) : 134 | self.big = big 135 | 136 | self.glue = glue 137 | self.table = table 138 | 139 | self.id_field = id_field 140 | self.namespace_field = namespace_field 141 | self.title_field = title_field 142 | 143 | self.table_obj = MySQLTable( self.table, self.id_field, self.namespace_field, self.title_field ) 144 | self.table_obj.set_field_definition( self.id_field, "INT NOT NULL") 145 | self.table_obj.set_field_definition( self.namespace_field, "INT DEFAULT NULL") 146 | self.table_obj.set_field_definition( self.title_field, "VARCHAR(255) BINARY DEFAULT NULL") 147 | self.table_obj.add_key_definition( "PRIMARY KEY (" + self.id_field + ")" ) 148 | self.table_obj.add_key_definition( "UNIQUE KEY (" + self.namespace_field + ", " + self.title_field + ")" ) 149 | 150 | self.table_id_obj = MySQLTable( self.table, self.id_field ) 151 | self.table_id_obj.add_key_definition( "PRIMARY KEY (" + self.id_field + ")" ) 152 | 153 | 154 | def set_expect_big ( self, big ) : 155 | self.big = big 156 | 157 | 158 | def get_table ( self, ) : 159 | return self.table_obj 160 | 161 | 162 | def create_table ( self, ) : 163 | table = self.table 164 | t = "" 165 | 166 | if ( not table or table == '?' ) : 167 | table = "gp_temp_%s" % self.glue.next_id() 168 | t = " TEMPORARY " 169 | 170 | 171 | sql = "CREATE " + t + " TABLE " + table 172 | sql += "(" 173 | sql += self.table_obj.get_field_definitions() 174 | sql += ")" 175 | 176 | self._update(sql) 177 | 178 | self.table = table 179 | self.table_obj.set_name( self.table ) 180 | self.table_id_obj.set_name( self.table ) 181 | 182 | return table 183 | 184 | 185 | 186 | def _query( self, sql, **kwargs ) : 187 | if not 'unbuffered' in kwargs: 188 | kwargs['unbuffered'] = self.big 189 | 190 | return self.glue.mysql_query(sql, **kwargs) #TODO: port kwargs to PHP 191 | 192 | def _update( self, sql, **kwargs ) : #TODO: port to PHP; use in PHP! 193 | return self.glue.mysql_update(sql, **kwargs) 194 | 195 | def add_from_select ( self, select, comment = None ) : 196 | sql= "REPLACE INTO " + self.table + " " 197 | sql += "( " 198 | sql += self.id_field + ", " 199 | sql += self.namespace_field + ", " 200 | sql += self.title_field + " ) " 201 | sql += select 202 | 203 | return self._update( sql, comment = comment ) 204 | 205 | 206 | def delete_where ( self, where, comment = None ) : 207 | sql= "DELETE FROM " + self.table + " " 208 | sql += where 209 | 210 | return self._update( sql, comment = comment ) 211 | 212 | 213 | def delete_using ( self, using, tableAlias = "T", comment = None ) : 214 | sql= "DELETE FROM " + tableAlias + " " 215 | sql += "USING " + self.table + " AS " + tableAlias + " " 216 | sql += using 217 | 218 | return self._update( sql, comment = comment ) 219 | 220 | 221 | def resolve_ids ( self, comment = None ) : 222 | #NOTE: MySQL can't perform self-joins on temp tables. so we need to copy the ids to another temp table first. 223 | t = MySQLTable("?", "page_id") 224 | t.add_key_definition("PRIMARY KEY (page_id)") 225 | 226 | tmp = self.glue.make_temp_table( t ) 227 | 228 | sql = tmp.get_insert(True) 229 | sql += "SELECT " + self.id_field 230 | sql += " FROM " + self.table 231 | sql += " WHERE page_title IS NULL" 232 | 233 | self._update( sql ); #copy page ids with no page title into temp table 234 | 235 | sql = "SELECT P.page_id, P.page_namespace, P.page_title " 236 | sql += " FROM " + self.glue.wiki_table("page") + " AS P " 237 | sql += " JOIN " + tmp.get_name() + " AS T ON T.page_id = P.page_id" 238 | 239 | self.add_from_select( sql, comment = comment ) #TODO: port comment to PHP 240 | 241 | self.glue.drop_temp_table( tmp ) 242 | return True 243 | 244 | 245 | def make_sink ( self, ) : 246 | sink = self.glue.make_sink( self.table_obj ) 247 | return sink 248 | 249 | 250 | def make_id_sink ( self, ) : 251 | sink = self.glue.make_sink( self.table_id_obj ) 252 | return sink 253 | 254 | 255 | def make_id_source ( self, ns = None ) : 256 | return self.make_source( ns, True ) 257 | 258 | 259 | def make_source ( self, ns = None, ids_only = False, auto_order = False ) : #TODO: PORT auto_order to PHP 260 | t = self.table_id_obj if ids_only else self.table_obj 261 | 262 | if ( ns is not None ) : 263 | select = t._get_select() 264 | 265 | if ( isinstance(ns, (tuple, list, set)) ): select += " where page_namespace in " + self.glue.as_list( ns ) 266 | else: select += " where page_namespace = %i" % int(ns) 267 | 268 | t = MySQLSelect(select) 269 | 270 | 271 | src = self.glue.make_source( t, big = self.big, auto_order = auto_order ) 272 | return src 273 | 274 | 275 | def capture ( self, ns = None, data = None ) : 276 | sink = ArraySink( data ) 277 | self.copy_to_sink( ns, sink ) 278 | return sink.getData() 279 | 280 | 281 | def capture_ids ( self, ns = None, data = None ) : 282 | sink = ArraySink( data ) 283 | self.copy_ids_to_sink( ns, sink ) 284 | return sink.getData() 285 | 286 | 287 | def copy_to_sink ( self, ns, sink ) : 288 | src = self.make_source(ns) 289 | c = self.glue.copy(src, sink, "~") 290 | src.close() 291 | return c 292 | 293 | 294 | def copy_ids_to_sink ( self, ns, sink ) : 295 | src = self.make_id_source(ns) 296 | c = self.glue.copy(src, sink, "~") 297 | src.close() 298 | return c 299 | 300 | 301 | def add_source ( self, src ) : 302 | sink = self.make_sink() 303 | c = self.glue.copy( src, sink, "+" ) 304 | sink.close() 305 | return c 306 | 307 | 308 | def add_page_set ( self, set ) : 309 | select = set.get_table()._get_select() 310 | return self.add_from_select( select ) 311 | 312 | 313 | def subtract_page_set ( self, set ) : 314 | t = set.get_table() 315 | return self.subtract_table( t ) 316 | 317 | 318 | def subtract_source ( self, src ): #XXX: must be a 1 column id source... 319 | t = MySQLTable("?", "page_id") 320 | sink = self.glue.make_temp_sink( t ) 321 | t = sink.getTable() 322 | 323 | self.glue.copy( src, sink, "+" ) 324 | 325 | ok = self.subtract_table(t, "page_id") 326 | 327 | self.glue.drop_temp_table(t) 328 | sink.close() 329 | 330 | return ok 331 | 332 | 333 | def retain_page_set ( self, set ) : 334 | t = set.get_table() 335 | return self.retain_table( t ) 336 | 337 | 338 | def retain_source ( self, src ) : #XXX: must be a 1 column id source... 339 | t = MySQLTable("?", "page_id") 340 | sink = self.glue.make_temp_sink( t ) 341 | t = sink.getTable() 342 | 343 | self.glue.copy( src, sink, "+" ) 344 | 345 | ok = self.retain_table(t, "page_id") 346 | 347 | self.glue.drop_temp_table(t) 348 | sink.close() 349 | 350 | return ok 351 | 352 | 353 | def subtract_table ( self, table, id_field = None ) : 354 | if ( not id_field ): id_field = table.get_field1() 355 | 356 | sql = "DELETE FROM T " 357 | sql += " USING " + self.table + " AS T " 358 | sql += " JOIN " + table.get_name() + " AS R " 359 | sql += " ON T." + self.id_field + " = R." + id_field 360 | 361 | self._update(sql) 362 | return True 363 | 364 | 365 | def retain_table ( self, table, id_field = None ) : 366 | if ( not id_field ): id_field = table.get_field1() 367 | 368 | sql = "DELETE FROM T " 369 | sql += " USING " + self.table + " AS T " 370 | sql += " LEFT JOIN " + table.get_name() + " AS R " 371 | sql += " ON T." + self.id_field + " = R." + id_field 372 | sql += " WHERE R." + id_field + " IS NULL" 373 | 374 | self._update(sql) 375 | return True 376 | 377 | 378 | def remove_page ( self, ns, title ) : 379 | sql = "DELETE FROM " + self.table 380 | sql += " WHERE " + self.namespace_field + " = %i" % int(ns) 381 | sql += " AND " + self.title_field + " = " + self.glue.quote_string(title) 382 | 383 | self._update(sql) 384 | return True 385 | 386 | 387 | def remove_page_id ( self, id ) : 388 | sql = "DELETE FROM " + self.table 389 | sql += " WHERE " + self.id_field + " = %i" % int(id) 390 | 391 | self._update(sql) 392 | return True 393 | 394 | 395 | def strip_namespace ( self, ns, inverse = False ) : 396 | sql = "DELETE FROM " + self.table 397 | sql += " WHERE " + self.namespace_field 398 | 399 | if ( isinstance(ns, (tuple, list, set)) ): sql += ( " not in " if inverse else " in " ) + self.glue.as_list( ns ) 400 | else: sql += ( " != " if inverse else " = " ) + str(int(ns)) 401 | 402 | self._update(sql) 403 | return True 404 | 405 | 406 | def retain_namespace ( self, ns ) : 407 | return self.strip_namespace( ns, True ) 408 | 409 | 410 | def add_page ( self, id, ns, title ) : 411 | if ( not id ): id = self.glue.get_page_id( NS_CATEGORY, cat ) 412 | 413 | values = array(id, ns, title) 414 | 415 | sql = self.table_obj.insert_command() 416 | sql += " VALUES " 417 | sql += self.glue.as_list(values) 418 | 419 | self._update( sql ) 420 | return True 421 | 422 | 423 | def add_page_id ( self, id ) : 424 | values = array(id) 425 | 426 | sql = "INSERT IGNORE INTO " + self.table 427 | sql += " ( " + self.id_field + " ) " 428 | sql += " VALUES " 429 | sql += self.glue.as_list(values) 430 | 431 | self._update( sql ) 432 | return True 433 | 434 | 435 | def expand_categories ( self, ns = None, comment = None ) : 436 | #NOTE: MySQL can't perform self-joins on temp tables. so we need to copy the category names to another temp table first. 437 | t = MySQLTable("?", "cat_title") 438 | t.set_field_definition("cat_title", "VARCHAR(255) BINARY NOT NULL") 439 | t.add_key_definition("PRIMARY KEY (cat_title)") 440 | 441 | tmp = self.glue.make_temp_table( t ) 442 | 443 | sql = tmp.get_insert(True) 444 | sql += " select page_title " 445 | sql += " from " + self.table + " as T " 446 | sql += " where page_namespace = %i " % NS_CATEGORY 447 | 448 | self._update( sql ) 449 | #self.glue.dump_query("select * from " +tmp.get_name()) 450 | 451 | # ---------------------------------------------------------- 452 | sql = "select P.page_id, P.page_namespace, P.page_title " 453 | sql += " from " + self.glue.wiki_table( "page" ) + " as P " 454 | sql += " join " + self.glue.wiki_table( "categorylinks" ) + " as X " 455 | sql += " on X.cl_from = P.page_id " 456 | sql += " join " + tmp.get_name() + " as T " 457 | sql += " on T.cat_title = X.cl_to " 458 | 459 | if (ns is not None) : 460 | if ( isinstance(ns, (tuple, list, set)) ): sql += " where P.page_namespace in " + self.glue.as_list( ns ) 461 | else: sql += " where P.page_namespace = %i" % int(ns) 462 | 463 | 464 | #self.glue.dump_query(sql) 465 | self.add_from_select( sql, comment = comment ) #TODO: port comment to PHP 466 | 467 | #self.glue.dump_query("select * from " +self.table) 468 | self.glue.drop_temp_table( tmp ) 469 | return True 470 | 471 | 472 | def add_subcategories ( self, cat, depth, without = None, without_depth = None ) : 473 | self._add_subcategory_ids(cat, depth, without, without_depth) 474 | self.resolve_ids() 475 | return True 476 | 477 | 478 | def _add_subcategory_ids( self, cat, depth, without = None, without_depth = None ) : 479 | id = self.glue.get_page_id( NS_CATEGORY, cat ) 480 | if ( not id ): return False 481 | 482 | if ( without ): without_id = self.glue.get_page_id( NS_CATEGORY, without ) 483 | else: without_id = False 484 | 485 | sink = self.make_id_sink() 486 | 487 | if ( without_id ) : 488 | if ( not without_depth ): without_depth = depth 489 | status = self.glue.traverse_successors_without( id, depth, without_id, without_depth, sink ) 490 | else : 491 | status = self.glue.traverse_successors( id, depth, sink ) 492 | 493 | 494 | sink.close() 495 | return True 496 | 497 | def get_size(self): 498 | res = self._query("SELECT COUNT(*) FROM " + self.table) 499 | try: 500 | row = res.fetchone() 501 | finally: 502 | res.close() 503 | 504 | return row[0] 505 | 506 | def add_pages_in ( self, cat, ns, depth, comment = None ) : 507 | self.get_size() 508 | 509 | if ( not self.add_subcategories(cat, depth) ): 510 | return False 511 | 512 | self.get_size() # ?! 513 | 514 | self.expand_categories(ns, comment = comment) 515 | return True 516 | 517 | 518 | def add_pages_transclusing ( self, tag, ns = None, comment = None ) : 519 | if ( ns is None ): ns = NS_TEMPLATE 520 | tag = self.glue.get_db_key( tag ) 521 | 522 | sql = " SELECT page_id, page_namespace, page_title " 523 | sql += " FROM " + self.glue.wiki_table( "page" ) 524 | sql += " JOIN " + self.glue.wiki_table( "templatelinks" ) 525 | sql += " ON tl_from = page_id " 526 | sql += " WHERE tl_namespace = %i" % int(ns) 527 | sql += " AND tl_title = " + self.glue.quote_string(tag) 528 | 529 | return self.add_from_select(sql, comment = comment) 530 | 531 | 532 | def clear ( self, ) : 533 | sql = "TRUNCATE " + self.table 534 | self._update(sql) 535 | return True 536 | 537 | 538 | def dispose ( self, ) : 539 | sql = "DROP TEMPORARY TABLE " + self.table 540 | self._update(sql) 541 | return True 542 | 543 | 544 | 545 | -------------------------------------------------------------------------------- /gp/tests/mediawiki_test.py: -------------------------------------------------------------------------------- 1 | from gp.mediawiki import * 2 | from gp.client import * 3 | from gp.mysql import * 4 | from test_base import * 5 | 6 | import unittest 7 | import sys 8 | 9 | class MediaWikiTest (SlaveTestBase, unittest.TestCase): 10 | 11 | def setUp(self) : 12 | self.dump = PipeSink( sys.stdout ) 13 | 14 | try : 15 | self.gp = MediaWikiGlue.new_slave_connection( test_graphcore_path ) 16 | self.gp.connect() 17 | except gpException as ex: 18 | print "Unable to launch graphcore instance from %s, please make sure graphcore is installed and check the test_graphcore_path configuration options in test_config.py.\nOriginal error: %s " % (test_graphcore_path, ex.getMessage() ) 19 | suicide(10) 20 | 21 | 22 | try : 23 | self.gp.mysql_connect( test_mysql_host, test_mysql_user, test_mysql_password, test_mysql_database ) 24 | self.gp.set_table_prefix( test_mediawiki_table_prefix ) 25 | except gpException as ex: 26 | print "Unable to connect to database %s on MySQL host %s as %s, please make sure MySQL is running and check the test_mysql_host and related configuration options in test_cofig.py.\nOriginal error: %s " % (test_mysql_database, test_mysql_host, test_mysql_user, ex.getMessage() ) 27 | suicide(10) 28 | 29 | 30 | 31 | def _makeTable( self, table, fieldSpec, temp = False ) : 32 | t = " TEMPORARY " if temp else "" 33 | sql = "CREATE " + t + " TABLE IF NOT EXISTS " + table 34 | sql += "(" 35 | sql += fieldSpec 36 | sql += ")" 37 | 38 | self.gp.mysql_query(sql) 39 | 40 | sql = "TRUNCATE TABLE " + table 41 | self.gp.mysql_query(sql) 42 | 43 | 44 | def _makeWikiTable( self, name, spec ) : 45 | name = test_mediawiki_table_prefix + name 46 | 47 | self._makeTable( name, spec ) 48 | return name 49 | 50 | 51 | def _makeWikiStructure( self ) : 52 | p = self._makeWikiTable( "page", "page_id INT NOT NULL, page_namespace INT NOT NULL, page_title VARCHAR(255) NOT NULL, PRIMARY KEY (page_id), UNIQUE KEY (page_namespace, page_title)" ) 53 | self.gp.mysql_query( "TRUNCATE " + p ) 54 | 55 | self.gp.mysql_query( "INSERT INTO " + p + " VALUES (1, " + str(NS_MAIN) + ", 'Main_Page')" ) 56 | self.gp.mysql_query( "INSERT INTO " + p + " VALUES (2, " + str(NS_PROJECT) + ", 'Help_Out')" ) 57 | 58 | self.gp.mysql_query( "INSERT INTO " + p + " VALUES (10, " + str(NS_CATEGORY) + ", 'ROOT')" ) 59 | self.gp.mysql_query( "INSERT INTO " + p + " VALUES (20, " + str(NS_CATEGORY) + ", 'Portals')" ) 60 | self.gp.mysql_query( "INSERT INTO " + p + " VALUES (110, " + str(NS_CATEGORY) + ", 'Topics')" ) 61 | self.gp.mysql_query( "INSERT INTO " + p + " VALUES (1110, " + str(NS_CATEGORY) + ", 'Beer')" ) 62 | self.gp.mysql_query( "INSERT INTO " + p + " VALUES (1111, " + str(NS_MAIN) + ", 'Lager')" ) 63 | self.gp.mysql_query( "INSERT INTO " + p + " VALUES (1112, " + str(NS_MAIN) + ", 'Pils')" ) 64 | self.gp.mysql_query( "INSERT INTO " + p + " VALUES (2110, " + str(NS_CATEGORY) + ", 'Cheese')" ) 65 | self.gp.mysql_query( "INSERT INTO " + p + " VALUES (120, " + str(NS_CATEGORY) + ", 'Maintenance')" ) 66 | self.gp.mysql_query( "INSERT INTO " + p + " VALUES (1120, " + str(NS_CATEGORY) + ", 'Bad_Cheese')" ) 67 | self.gp.mysql_query( "INSERT INTO " + p + " VALUES (1122, " + str(NS_MAIN) + ", 'Toe_Cheese')" ) 68 | self.gp.mysql_query( "INSERT INTO " + p + " VALUES (333, " + str(NS_TEMPLATE) + ", 'Yuck')" ) 69 | 70 | cl = self._makeWikiTable( "categorylinks", "cl_from INT NOT NULL, cl_to VARCHAR(255) NOT NULL, PRIMARY KEY (cl_from, cl_to), INDEX cl_to (cl_to)" ) 71 | self.gp.mysql_query( "TRUNCATE " + cl ) 72 | 73 | self.gp.mysql_query( "INSERT INTO " + cl + " VALUES (1, 'Portals')" ) 74 | self.gp.mysql_query( "INSERT INTO " + cl + " VALUES (2, 'Portals')" ) 75 | self.gp.mysql_query( "INSERT INTO " + cl + " VALUES (20, 'ROOT')" ) 76 | self.gp.mysql_query( "INSERT INTO " + cl + " VALUES (120, 'ROOT')" ) 77 | self.gp.mysql_query( "INSERT INTO " + cl + " VALUES (110, 'ROOT')" ) 78 | self.gp.mysql_query( "INSERT INTO " + cl + " VALUES (1110, 'Topics')" ) 79 | self.gp.mysql_query( "INSERT INTO " + cl + " VALUES (2110, 'Topics')" ) 80 | self.gp.mysql_query( "INSERT INTO " + cl + " VALUES (1111, 'Beer')" ) 81 | self.gp.mysql_query( "INSERT INTO " + cl + " VALUES (1112, 'Beer')" ) 82 | self.gp.mysql_query( "INSERT INTO " + cl + " VALUES (1120, 'Maintenance')" ) 83 | self.gp.mysql_query( "INSERT INTO " + cl + " VALUES (1120, 'Cheese')" ) 84 | self.gp.mysql_query( "INSERT INTO " + cl + " VALUES (1120, 'Cruft')" ) 85 | self.gp.mysql_query( "INSERT INTO " + cl + " VALUES (1122, 'Bad_Cheese')" ) 86 | 87 | tl = self._makeWikiTable( "templatelinks", "tl_from INT NOT NULL, tl_namespace INT NOT NULL, tl_title VARCHAR(255) NOT NULL, PRIMARY KEY (tl_from, tl_namespace, tl_title), INDEX tl_to (tl_namespace, tl_title)" ) 88 | self.gp.mysql_query( "TRUNCATE " + tl ) 89 | 90 | self.gp.mysql_query( "INSERT INTO " + tl + " VALUES (1122, " + str(NS_TEMPLATE) + ", 'Yuck')" ) 91 | self.gp.mysql_query( "INSERT INTO " + tl + " VALUES (1111, " + str(NS_TEMPLATE) + ", 'Yuck')" ) 92 | 93 | 94 | 95 | ########################################### 96 | 97 | def test_TraverseSuccessors( self ) : 98 | self.gp.add_arcs( [ 99 | ( 1, 11 ), 100 | ( 1, 12 ), 101 | ( 11, 111 ), 102 | ( 11, 112 ), 103 | ( 111, 1111 ), 104 | ( 111, 1112 ), 105 | ( 112, 1121 ), 106 | ] ) 107 | 108 | self.assertStatsValue( 'ArcCount', 7 ) 109 | 110 | #-------------------------------------------- 111 | succ = self.gp.capture_traverse_successors( 11, 5 ) 112 | 113 | self.assertEquals( [ ( 11, ), ( 111, ), ( 112, ), ( 1111, ), ( 1112, ), ( 1121, ), ], succ ) 114 | 115 | 116 | ########################################### 117 | 118 | def test_AddArcsFromCategoryStructure( self ) : 119 | self._makeWikiStructure() 120 | 121 | #----------------------------------------------------------- 122 | self.gp.add_arcs_from_category_structure() 123 | 124 | #----------------------------------------------------------- 125 | a = self.gp.capture_list_successors( 10 ) 126 | self.assertEquals([( 20, ), ( 110, ), ( 120, )], a ) 127 | 128 | a = self.gp.capture_list_predecessors( 1120 ) 129 | self.assertEquals([( 120, ), ( 2110, )], a ) 130 | 131 | a = self.gp.capture_traverse_successors( 110, 5 ) 132 | self.assertEquals([( 110, ), ( 1110, ), ( 2110, ), ( 1120, )], a ) 133 | 134 | 135 | def test_GetSubcategories( self ) : 136 | self._makeWikiStructure() 137 | self.gp.add_arcs_from_category_structure() 138 | 139 | #----------------------------------------------------------- 140 | a = self.gp.get_subcategories("topics", 5) 141 | self.assertEquals([( "Topics", ), 142 | ( "Beer", ), 143 | ( "Bad_Cheese", ), 144 | ( "Cheese", )], a ) 145 | 146 | #----------------------------------------------------------- 147 | a = self.gp.get_subcategories("topics", 5, "maintenance") 148 | self.assertEquals([( "Topics", ), 149 | ( "Beer", ), 150 | ( "Cheese", )], a ) 151 | 152 | 153 | ########################################### 154 | def test_AddSubcategories( self ) : 155 | self._makeWikiStructure() 156 | self.gp.add_arcs_from_category_structure() 157 | 158 | pages = PageSet(self.gp) 159 | pages.create_table() 160 | 161 | #----------------------------------------------------------- 162 | pages.clear() 163 | ok = pages.add_subcategories("topics", 5) 164 | self.assertTrue( ok ) 165 | 166 | a = pages.capture() 167 | self.assertEquals([(110, NS_CATEGORY, "Topics"), 168 | (1110, NS_CATEGORY, "Beer"), 169 | (1120, NS_CATEGORY, "Bad_Cheese"), 170 | (2110, NS_CATEGORY, "Cheese")], a ) 171 | 172 | #----------------------------------------------------------- 173 | pages.clear() 174 | ok = pages.add_subcategories("Portals", 5) 175 | self.assertTrue( ok ) 176 | 177 | a = pages.capture() 178 | self.assertEquals([(20, NS_CATEGORY, "Portals")], a ) 179 | 180 | #----------------------------------------------------------- 181 | pages.dispose() 182 | 183 | 184 | def test_AddPagesTranscluding( self ) : 185 | self._makeWikiStructure() 186 | self.gp.add_arcs_from_category_structure() 187 | 188 | pages = PageSet(self.gp) 189 | pages.create_table() 190 | 191 | #----------------------------------------------------------- 192 | pages.clear() 193 | ok = pages.add_pages_transclusing("yuck") 194 | self.assertTrue( ok ) 195 | 196 | a = pages.capture() 197 | self.assertEquals([(1111, NS_MAIN, "Lager"), 198 | (1122, NS_MAIN, "Toe_Cheese")], a ) 199 | 200 | #----------------------------------------------------------- 201 | pages.dispose() 202 | 203 | 204 | def test_AddPagesIn( self ) : 205 | self._makeWikiStructure() 206 | self.gp.add_arcs_from_category_structure() 207 | 208 | pages = PageSet(self.gp) 209 | pages.create_table() 210 | 211 | #----------------------------------------------------------- 212 | pages.clear() 213 | ok = pages.add_pages_in("topics", None, 5) 214 | self.assertTrue( ok ) 215 | 216 | a = pages.capture() 217 | expected = [ (110, NS_CATEGORY, "Topics"), 218 | (1110, NS_CATEGORY, "Beer"), 219 | (1111, NS_MAIN, "Lager"), 220 | (1112, NS_MAIN, "Pils"), 221 | (1120, NS_CATEGORY, "Bad_Cheese"), 222 | (1122, NS_MAIN, "Toe_Cheese"), 223 | (2110, NS_CATEGORY, "Cheese") ] 224 | 225 | self.assertEquals(expected, a ) 226 | 227 | #----------------------------------------------------------- 228 | pages.clear() 229 | ok = pages.add_pages_in("topics", None, 5) 230 | self.assertTrue( ok ) 231 | 232 | a = pages.capture( NS_MAIN ) 233 | self.assertEquals([(1111, NS_MAIN, "Lager"), 234 | (1112, NS_MAIN, "Pils"), 235 | (1122, NS_MAIN, "Toe_Cheese")], a ) 236 | 237 | #----------------------------------------------------------- 238 | pages.clear() 239 | ok = pages.add_pages_in("Portals", NS_MAIN, 5) 240 | self.assertTrue( ok ) 241 | 242 | a = pages.capture() 243 | self.assertEquals([(1, NS_MAIN, "Main_Page"), 244 | (20, NS_CATEGORY, "Portals")], a ) 245 | 246 | #----------------------------------------------------------- 247 | pages.clear() 248 | ok = pages.add_pages_in("portals", (NS_MAIN, NS_PROJECT), 5) 249 | self.assertTrue( ok ) 250 | 251 | a = pages.capture( (NS_MAIN, NS_PROJECT) ) 252 | self.assertEquals([(1, NS_MAIN, "Main_Page"), 253 | (2, NS_PROJECT, "Help_Out")], a ) 254 | 255 | #----------------------------------------------------------- 256 | pages.dispose() 257 | 258 | 259 | def test_BufferedAddPagesIn( self ) : 260 | self._makeWikiStructure() 261 | self.gp.add_arcs_from_category_structure() 262 | 263 | pages = PageSet(self.gp) 264 | pages.set_expect_big(False) 265 | pages.create_table() 266 | 267 | #----------------------------------------------------------- 268 | pages.clear() 269 | ok = pages.add_pages_in("topics", None, 5) 270 | self.assertTrue( ok ) 271 | 272 | a = pages.capture() 273 | expected = [(110, NS_CATEGORY, "Topics"), 274 | (1110, NS_CATEGORY, "Beer"), 275 | (1111, NS_MAIN, "Lager"), 276 | (1112, NS_MAIN, "Pils"), 277 | (1120, NS_CATEGORY, "Bad_Cheese"), 278 | (1122, NS_MAIN, "Toe_Cheese"), 279 | (2110, NS_CATEGORY, "Cheese") ] 280 | 281 | self.assertEquals(expected, a ) 282 | 283 | #----------------------------------------------------------- 284 | pages.dispose() 285 | 286 | 287 | def test_SubtractPageSet( self ) : 288 | self._makeWikiStructure() 289 | self.gp.add_arcs_from_category_structure() 290 | 291 | pages = PageSet(self.gp) 292 | pages.create_table() 293 | 294 | rpages = PageSet(self.gp) 295 | rpages.create_table() 296 | 297 | #----------------------------------------------------------- 298 | ok = pages.add_pages_in("topics", None, 5) 299 | ok = rpages.add_pages_in("Maintenance", None, 5) 300 | 301 | ok = pages.subtract_page_set( rpages ) 302 | self.assertTrue( ok ) 303 | 304 | a = pages.capture() 305 | expected = [ (110, NS_CATEGORY, "Topics"), 306 | (1110, NS_CATEGORY, "Beer"), 307 | (1111, NS_MAIN, "Lager"), 308 | (1112, NS_MAIN, "Pils"), 309 | (2110, NS_CATEGORY, "Cheese") ] 310 | 311 | self.assertEquals(expected, a ) 312 | 313 | #----------------------------------------------------------- 314 | pages.dispose() 315 | rpages.dispose() 316 | 317 | 318 | def test_RetainPageSet( self ) : 319 | self._makeWikiStructure() 320 | self.gp.add_arcs_from_category_structure() 321 | 322 | pages = PageSet(self.gp) 323 | pages.create_table() 324 | 325 | rpages = PageSet(self.gp) 326 | rpages.create_table() 327 | 328 | #----------------------------------------------------------- 329 | ok = pages.add_pages_in("topics", None, 5) 330 | ok = rpages.add_pages_in("Maintenance", None, 5) 331 | 332 | ok = pages.retain_page_set( rpages ) 333 | self.assertTrue( ok ) 334 | 335 | a = pages.capture() 336 | expected = [ (1120, NS_CATEGORY, "Bad_Cheese"), 337 | (1122, NS_MAIN, "Toe_Cheese") ] 338 | 339 | self.assertEquals(expected, a ) 340 | 341 | #----------------------------------------------------------- 342 | pages.dispose() 343 | rpages.dispose() 344 | 345 | 346 | def test_AddPageSet( self ) : 347 | self._makeWikiStructure() 348 | self.gp.add_arcs_from_category_structure() 349 | 350 | beer = PageSet(self.gp) 351 | beer.create_table() 352 | 353 | cheese = PageSet(self.gp) 354 | cheese.create_table() 355 | 356 | #----------------------------------------------------------- 357 | ok = cheese.add_pages_in("Cheese", None, 5) 358 | ok = beer.add_pages_in("Beer", None, 5) 359 | 360 | ok = cheese.add_page_set( beer ) 361 | self.assertTrue( ok ) 362 | 363 | a = cheese.capture() 364 | expected = [ (1110, NS_CATEGORY, "Beer"), 365 | (1111, NS_MAIN, "Lager"), 366 | (1112, NS_MAIN, "Pils"), 367 | (1120, NS_CATEGORY, "Bad_Cheese"), 368 | (1122, NS_MAIN, "Toe_Cheese"), 369 | (2110, NS_CATEGORY, "Cheese") ] 370 | 371 | self.assertEquals(expected, a ) 372 | 373 | #----------------------------------------------------------- 374 | beer.dispose() 375 | cheese.dispose() 376 | 377 | 378 | def test_DeleteWhere( self ) : 379 | self._makeWikiStructure() 380 | self.gp.add_arcs_from_category_structure() 381 | 382 | pages = PageSet(self.gp) 383 | pages.create_table() 384 | 385 | pages.add_pages_in("topics", None, 5) 386 | 387 | #----------------------------------------------------------- 388 | pages.delete_where( "where page_namespace = %i" % NS_CATEGORY ) 389 | 390 | a = pages.capture() 391 | expected = [ (1111, NS_MAIN, "Lager"), 392 | (1112, NS_MAIN, "Pils"), 393 | (1122, NS_MAIN, "Toe_Cheese") ] 394 | 395 | self.assertEquals(expected, a ) 396 | 397 | #----------------------------------------------------------- 398 | pages.dispose() 399 | 400 | 401 | def test_DeleteUsing( self ) : 402 | self._makeWikiStructure() 403 | self.gp.add_arcs_from_category_structure() 404 | 405 | pages = PageSet(self.gp) 406 | pages.create_table() 407 | 408 | pages.add_pages_in("topics", None, 5) 409 | 410 | #----------------------------------------------------------- 411 | sql = " JOIN " + self.gp.wiki_table("templatelinks") + " as X " 412 | sql += " ON T.page_id = X.tl_from " 413 | sql += " WHERE X.tl_namespace = %i" % NS_TEMPLATE 414 | sql += " AND X.tl_title = " + self.gp.quote_string("Yuck") 415 | 416 | pages.delete_using( sql ) 417 | 418 | a = pages.capture(NS_MAIN) 419 | expected = [ (1112, NS_MAIN, "Pils") ] 420 | 421 | self.assertEquals(expected, a ) 422 | 423 | #----------------------------------------------------------- 424 | pages.dispose() 425 | 426 | 427 | def test_StripNamespace( self ) : 428 | self._makeWikiStructure() 429 | self.gp.add_arcs_from_category_structure() 430 | 431 | pages = PageSet(self.gp) 432 | pages.create_table() 433 | 434 | #----------------------------------------------------------- 435 | pages.clear() 436 | pages.add_pages_in("topics", None, 5) 437 | pages.strip_namespace( NS_CATEGORY ) 438 | 439 | a = pages.capture() 440 | expected = [ (1111, NS_MAIN, "Lager"), 441 | (1112, NS_MAIN, "Pils"), 442 | (1122, NS_MAIN, "Toe_Cheese") ] 443 | 444 | self.assertEquals(expected, a ) 445 | 446 | #----------------------------------------------------------- 447 | pages.clear() 448 | pages.add_pages_in("Portals", None, 5) 449 | pages.strip_namespace( (NS_CATEGORY, NS_PROJECT) ) 450 | 451 | a = pages.capture() 452 | expected = [ (1, NS_MAIN, "Main_Page") ] 453 | 454 | self.assertEquals(expected, a ) 455 | 456 | #----------------------------------------------------------- 457 | pages.dispose() 458 | 459 | 460 | def test_RetainNamespace( self ) : 461 | self._makeWikiStructure() 462 | self.gp.add_arcs_from_category_structure() 463 | 464 | pages = PageSet(self.gp) 465 | pages.create_table() 466 | 467 | #----------------------------------------------------------- 468 | pages.clear() 469 | pages.add_pages_in("topics", None, 5) 470 | pages.retain_namespace( (NS_MAIN,) ) 471 | 472 | a = pages.capture() 473 | expected = [ (1111, NS_MAIN, "Lager"), 474 | (1112, NS_MAIN, "Pils"), 475 | (1122, NS_MAIN, "Toe_Cheese") ] 476 | 477 | self.assertEquals(expected, a ) 478 | 479 | #----------------------------------------------------------- 480 | pages.clear() 481 | pages.add_pages_in("Portals", None, 5) 482 | pages.retain_namespace( NS_MAIN ) 483 | 484 | a = pages.capture() 485 | expected = [ (1, NS_MAIN, "Main_Page") ] 486 | 487 | self.assertEquals(expected, a ) 488 | 489 | 490 | 491 | if __name__ == '__main__': 492 | unittest.main() 493 | 494 | 495 | --------------------------------------------------------------------------------