| {0} | ".format(h)) 14 | buf.write("
|---|
| {0} | ".format(c)) 28 | buf.write("
| {0} | ".format(c)) 35 | buf.write("
Rows: %d / Columns: %d
" % (len(rows), len(head))) 38 | return buf.getvalue() 39 | 40 | 41 | def memoize(obj): 42 | cache = obj.cache = {} 43 | 44 | @functools.wraps(obj) 45 | def memoizer(*args, **kwargs): 46 | key = str(args) + str(kwargs) 47 | if key not in cache: 48 | cache[key] = obj(*args, **kwargs) 49 | return cache[key] 50 | 51 | 52 | return memoizer 53 | -------------------------------------------------------------------------------- /pyxplorer/loader.py: -------------------------------------------------------------------------------- 1 | __author__ = 'grund' 2 | import re 3 | 4 | from snakebite.client import Client 5 | import pyhs2 6 | 7 | 8 | class Loader: 9 | """ 10 | The idea of the loader is to provide a convenient interface to create a new table 11 | based on some input files 12 | """ 13 | 14 | def __init__(self, path, name_node, hive_server, 15 | user="root", hive_db="default", password=None, nn_port=8020, hive_port=10000): 16 | 17 | # HDFS Connection 18 | self._client = Client(name_node, nn_port) 19 | 20 | self._db = hive_db 21 | 22 | # Hive Connection 23 | self._hive = pyhs2.connect(host=hive_server, 24 | port=hive_port, 25 | authMechanism="PLAIN", 26 | database=hive_db, 27 | user=user, 28 | password=password) 29 | self._path = path 30 | 31 | 32 | def load(self): 33 | # Check data to see which kind it is 34 | files = self._client.ls([self._path]) 35 | 36 | files = [f for f in files if f['file_type'] == 'f'] 37 | if len(files) == 0: 38 | raise Exception("Cannot load empty directory") 39 | 40 | # Pick the first file and assume that it has the same content as the others 41 | data = self.head(files[0]['path']) 42 | res = self.check_separator(data) 43 | if res == None: 44 | # We cant load the data and better abort here 45 | print("cant load data, cannot find a separator") 46 | return 47 | 48 | sep = res[0] 49 | num_cols = res[1] 50 | 51 | # Build table statement 52 | table_statement, table_name = self._create_table(self._path, sep, num_cols) 53 | cursor = self._hive.cursor() 54 | cursor.execute(table_statement) 55 | 56 | return self._db, table_name 57 | 58 | 59 | def _create_table(self, path, sep, count): 60 | buf = """CREATE EXTERNAL TABLE pyxplorer_data ( 61 | %s 62 | )ROW FORMAT DELIMITED FIELDS TERMINATED BY '%s' 63 | STORED AS TEXTFILE LOCATION '%s' 64 | """ % (",".join(["col_%d string" % x for x in range(count)]), sep, path) 65 | return buf, "pyxplorer_data" 66 | 67 | def check_separator(self, data): 68 | """ 69 | THis method evaluates a list of separators on the input data to check which one 70 | is correct. This is done by first splitting the input by newline and then 71 | checking if the split by separator is equal for each input row except the last 72 | that might be incomplete due to the limited input data 73 | 74 | :param data: input data to check 75 | :return: 76 | """ 77 | 78 | sep_list = [r'\t', r';', r',', r'|', r'\s+'] 79 | 80 | data_copy = data 81 | for sep in sep_list: 82 | # Check if the count matches each line 83 | splitted = data_copy.split("\n") 84 | parts = [len(re.split(sep, line)) for line in splitted] 85 | 86 | # If we did not split anything continue 87 | if sum(parts) == len(splitted): 88 | continue 89 | 90 | diff = 0 91 | 92 | for i in range(len(parts[1:-1])): 93 | diff += abs(parts[i] - parts[i + 1]) 94 | 95 | if diff == 0: 96 | return sep, parts[0] 97 | 98 | # If we reach this point we did not find a separator 99 | return None 100 | 101 | 102 | def head(self, file_path): 103 | """ 104 | Onlye read the first packets that come, try to max out at 1024kb 105 | 106 | :return: up to 1024b of the first block of the file 107 | """ 108 | processor = lambda path, node, tail_only=True, append=False: self._handle_head( 109 | path, node) 110 | 111 | # Find items and go 112 | for item in self._client._find_items([file_path], processor, 113 | include_toplevel=True, 114 | include_children=False, recurse=False): 115 | if item: 116 | return item 117 | 118 | def _handle_head(self, path, node, upper=1024 * 1024): 119 | data = '' 120 | for load in self._client._read_file(path, node, tail_only=False, 121 | check_crc=False): 122 | data += load 123 | if (len(data) > upper): 124 | return data 125 | 126 | return data 127 | -------------------------------------------------------------------------------- /pyxplorer/manager.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import sys 3 | 4 | import pandas as pd 5 | 6 | import types as t 7 | import helper as h 8 | 9 | 10 | class Database: 11 | def __init__(self, db, conn): 12 | self.db = db 13 | self.connection = conn 14 | 15 | def __getitem__(self, item): 16 | for x in self.tables(): 17 | if x.name() == item: 18 | return x 19 | raise KeyError(item) 20 | 21 | @h.memoize 22 | def tables(self): 23 | """ 24 | :return: all tables stored in this database 25 | """ 26 | cursor = self.connection.cursor() 27 | cursor.execute("show tables in %s" % self.db) 28 | self._tables = [t.Table(r[0], con=self.connection, db=self.db) for r in cursor.fetchall()] 29 | return self._tables 30 | 31 | def __len__(self): 32 | return len(self.tables()) 33 | 34 | @h.memoize 35 | def tcounts(self): 36 | """ 37 | :return: a data frame containing the names and sizes for all tables 38 | """ 39 | df = pd.DataFrame([[t.name(), t.size()] for t in self.tables()], columns=["name", "size"]) 40 | df.index = df.name 41 | return df 42 | 43 | @h.memoize 44 | def dcounts(self): 45 | """ 46 | :return: a data frame with names and distinct counts and fractions for all columns in the database 47 | """ 48 | print("WARNING: Distinct value count for all tables can take a long time...", file=sys.stderr) 49 | sys.stderr.flush() 50 | 51 | data = [] 52 | for t in self.tables(): 53 | for c in t.columns(): 54 | data.append([t.name(), c.name(), c.dcount(), t.size(), c.dcount() / float(t.size())]) 55 | df = pd.DataFrame(data, columns=["table", "column", "distinct", "size", "fraction"]) 56 | return df 57 | 58 | 59 | def _repr_html_(self): 60 | return h.render_table(["Name", "Size"], [[x.name(), x.size()] for x in self.tables()]) 61 | 62 | 63 | def num_tables(self): 64 | return len(self) 65 | 66 | def num_columns(self): 67 | return sum([len(x.columns()) for x in self.tables()]) 68 | 69 | def num_tuples(self): 70 | return sum([x.size() for x in self.tables()]) 71 | -------------------------------------------------------------------------------- /pyxplorer/types.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import pandas as pd 4 | import helper as h 5 | import sys 6 | 7 | 8 | class Column: 9 | """ 10 | Representation of a column and the profiling information 11 | """ 12 | 13 | def _qexec(self, fld, group=None, order=None): 14 | c = self._con.cursor() 15 | if not group is None: 16 | group = " group by %s" % group 17 | else: 18 | group = "" 19 | 20 | if not order is None: 21 | order = " order by %s" % order 22 | else: 23 | order = "" 24 | 25 | query = "select %s from `%s`.`%s` %s %s" % (fld, self._table.db(), self._table.name(), group, order) 26 | c.execute(query) 27 | return c.fetchall() 28 | 29 | def __init__(self, name, type_name, con, table): 30 | self._name = name 31 | self._type_name = type_name 32 | self._con = con 33 | self._table = table 34 | self._distribution = None 35 | self._min = None 36 | self._max = None 37 | self._dcount = None 38 | self._most_frequent = None 39 | self._most_frequent_count = None 40 | self._least_frequent = None 41 | self._least_frequent_count = None 42 | 43 | def __repr__(self): 44 | return self.name() 45 | 46 | def __str__(self): 47 | buf = "%s\n" % self.name() 48 | funs = [self.min, self.max, self.dcount, self.most_frequent, self.least_frequent] 49 | for x in funs: 50 | buf += "%s:\t%s\n" % (x.__name__, x()) 51 | return buf 52 | 53 | def name(self): 54 | return self._name 55 | 56 | @classmethod 57 | def build(cls, data, con, table): 58 | return Column(data[0], data[1], con, table) 59 | 60 | def __eq__(self, other): 61 | return self._name == other._name and self._type_name == other._type_name 62 | 63 | @h.memoize 64 | def min(self): 65 | """ 66 | :returns the minimum of the column 67 | """ 68 | res = self._qexec("min(%s)" % self._name) 69 | if len(res) > 0: 70 | self._min = res[0][0] 71 | return self._min 72 | 73 | @h.memoize 74 | def max(self): 75 | """ 76 | :returns the maximum of the column 77 | """ 78 | res = self._qexec("max(%s)" % self._name) 79 | if len(res) > 0: 80 | self._max = res[0][0] 81 | return self._max 82 | 83 | @h.memoize 84 | def dcount(self): 85 | res = self._qexec("count(distinct %s)" % self._name) 86 | if len(res) > 0: 87 | self._dcount = res[0][0] 88 | return self._dcount 89 | 90 | @h.memoize 91 | def distribution(self, limit=1024): 92 | """ 93 | Build the distribution of distinct values 94 | """ 95 | res = self._qexec("%s, count(*) as __cnt" % self.name(), group="%s" % self.name(), 96 | order="__cnt DESC LIMIT %d" % limit) 97 | dist = [] 98 | cnt = self._table.size() 99 | for i, r in enumerate(res): 100 | dist.append(list(r) + [i, r[1] / float(cnt)]) 101 | 102 | self._distribution = pd.DataFrame(dist, columns=["value", "cnt", "r", "fraction"]) 103 | self._distribution.index = self._distribution.r 104 | 105 | return self._distribution 106 | 107 | @h.memoize 108 | def most_frequent(self): 109 | res = self.n_most_frequent(1) 110 | self._most_frequent = res[0][0] 111 | self._most_frequent_count = res[0][1] 112 | return self._most_frequent, self._most_frequent_count 113 | 114 | @h.memoize 115 | def least_frequent(self): 116 | res = self.n_least_frequent(1) 117 | self._least_frequent = res[0][0] 118 | self._least_frequent_count = res[0][1] 119 | return self._least_frequent, self._least_frequent_count 120 | 121 | @h.memoize 122 | def n_most_frequent(self, limit=10): 123 | res = self._qexec("%s, count(*) as __cnt" % self.name(), group="%s" % self.name(), 124 | order="__cnt DESC LIMIT %d" % limit) 125 | return res 126 | 127 | @h.memoize 128 | def n_least_frequent(self, limit=10): 129 | res = self._qexec("%s, count(*) as cnt" % self.name(), group="%s" % self.name(), 130 | order="cnt ASC LIMIT %d" % limit) 131 | return res 132 | 133 | def size(self): 134 | return self._table.size() 135 | 136 | def uniqueness(self): 137 | return self.dcount() / float(self.size()) 138 | 139 | def constancy(self): 140 | tup = self.most_frequent() 141 | return tup[1] / float(self.size()) 142 | 143 | def _repr_html_(self): 144 | 145 | funs = [("Min", self.min), ("Max", self.max), ("#Distinct Values", self.dcount), 146 | ("Most Frequent", lambda: "{0} ({1})".format(*self.most_frequent())), 147 | ("Least Frequent", lambda: "{0} ({1})".format(*self.least_frequent())), 148 | ("Top 10 MF", lambda: ",".join(map(str, h.car(self.n_most_frequent())))), 149 | ("Top 10 LF", lambda: ", ".join(map(str, h.car(self.n_least_frequent())))), 150 | ("Uniqueness", self.uniqueness), 151 | ("Constancy", self.constancy), 152 | ] 153 | return h.render_table(["Name", "Value"], [[x[0], x[1]()] for x in funs]) 154 | 155 | 156 | class Table: 157 | """ 158 | Generic Table Object 159 | 160 | This class provides simple access to the columns of the table. Most of the methods that perform actual data access 161 | are cached to avoid costly lookups. 162 | 163 | 164 | """ 165 | 166 | def __init__(self, name, con, db="default"): 167 | self._cols = [] 168 | self._db = db 169 | self._name = name 170 | self._connection = con 171 | 172 | def name(self): 173 | """ 174 | :return: name of the table 175 | """ 176 | return self._name 177 | 178 | def db(self): 179 | """ 180 | :return: name of the database used 181 | """ 182 | return self._db 183 | 184 | def column(self, col): 185 | """ 186 | Given either a column index or name return the column structure 187 | :param col: either index or name 188 | :return: column data structure 189 | """ 190 | if type(col) is str: 191 | for c in self._cols: 192 | if c.name == col: 193 | return c 194 | else: 195 | return self._cols[col] 196 | 197 | @h.memoize 198 | def __len__(self): 199 | """ 200 | :return: number of rows in the table 201 | """ 202 | c = self._connection.cursor() 203 | c.execute("select count(*) from `%s`.`%s`" % (self._db, self._name)) 204 | self._count = c.fetchall()[0][0] 205 | return self._count 206 | 207 | def size(self): 208 | """ 209 | alias to __len__() 210 | :return: 211 | """ 212 | return len(self) 213 | 214 | @h.memoize 215 | def columns(self): 216 | """ 217 | :return: the list of column in this table 218 | """ 219 | c = self._connection.cursor() 220 | c.execute("describe `%s`.`%s`" % (self._db, self._name)) 221 | self._cols = [] 222 | for col in c.fetchall(): 223 | self._cols.append(Column.build(col, table=self, con=self._connection)) 224 | return self._cols 225 | 226 | def __getitem__(self, item): 227 | """ 228 | Subscript access to the tables by name 229 | :param item: 230 | :return: 231 | """ 232 | for x in self.columns(): 233 | if x.name() == item: 234 | return x 235 | raise KeyError(item) 236 | 237 | def __dir__(self): 238 | """ 239 | :return: an array of custom attributes, for code-completion in ipython 240 | """ 241 | return [x.name() for x in self.columns()] 242 | 243 | def __repr__(self): 244 | return "| Name | Size |
|---|---|
| customerp | 30000000 |
| districtp | 10000 |
| historyp | 30000000 |
| itemp | 100000 |
| new_orderp | 9000000 |
| oorderp | 30000000 |
| order_linep | 299991280 |
| stockp | 100000000 |
| warehousep | 1000 |
Rows: 9 / Columns: 2
" 100 | ], 101 | "metadata": {}, 102 | "output_type": "pyout", 103 | "prompt_number": 4, 104 | "text": [ 105 | "| Name | Value |
|---|---|
| Min | 1 |
| Max | 1000 |
| #Distinct Values | 1000 |
| Most Frequent | 109 (301593) |
| Least Frequent | 212 (298395) |
| Top 10 MF | 109,676,117,460,19,877,165,764,340,689 |
| Top 10 LF | 212, 405, 52, 284, 304, 769, 727, 665, 90, 163 |
| Uniqueness | 3.33343022504e-06 |
| Constancy | 0.00100533922186 |
Rows: 9 / Columns: 2
" 382 | ], 383 | "metadata": {}, 384 | "output_type": "pyout", 385 | "prompt_number": 20, 386 | "text": [ 387 | "ol_w_id" 388 | ] 389 | } 390 | ], 391 | "prompt_number": 20 392 | }, 393 | { 394 | "cell_type": "markdown", 395 | "metadata": {}, 396 | "source": [ 397 | "Its possible to access the column either using subscript notation or directly as an attribute of the object" 398 | ] 399 | }, 400 | { 401 | "cell_type": "code", 402 | "collapsed": false, 403 | "input": [ 404 | "tab.ol_w_id" 405 | ], 406 | "language": "python", 407 | "metadata": {}, 408 | "outputs": [ 409 | { 410 | "html": [ 411 | "| Name | Value |
|---|---|
| Min | 1 |
| Max | 1000 |
| #Distinct Values | 1000 |
| Most Frequent | 109 (301593) |
| Least Frequent | 212 (298395) |
| Top 10 MF | 109,676,117,460,19,877,165,764,340,689 |
| Top 10 LF | 212, 405, 52, 284, 304, 769, 727, 665, 90, 163 |
| Uniqueness | 3.33343022504e-06 |
| Constancy | 0.00100533922186 |
Rows: 9 / Columns: 2
" 412 | ], 413 | "metadata": {}, 414 | "output_type": "pyout", 415 | "prompt_number": 21, 416 | "text": [ 417 | "ol_w_id" 418 | ] 419 | } 420 | ], 421 | "prompt_number": 21 422 | }, 423 | { 424 | "cell_type": "markdown", 425 | "metadata": {}, 426 | "source": [ 427 | "Based on this information we can further deduct what role this column might have in the overall schema. For example, based on the uniqueness we can say that the column is not suitable to uniquely identify every row. In additiona, based on the constancy of the most frequent value and the spread between the most and least frequent value we can deduct that the data is almost uniformly distributed." 428 | ] 429 | }, 430 | { 431 | "cell_type": "markdown", 432 | "metadata": {}, 433 | "source": [ 434 | "## Distinct Values and Distinct Value Distribution\n", 435 | "\n", 436 | "One important feature ist to look at the distinct values and their distribution" 437 | ] 438 | }, 439 | { 440 | "cell_type": "code", 441 | "collapsed": false, 442 | "input": [ 443 | "tab.ol_w_id.dcount()" 444 | ], 445 | "language": "python", 446 | "metadata": {}, 447 | "outputs": [ 448 | { 449 | "metadata": {}, 450 | "output_type": "pyout", 451 | "prompt_number": 22, 452 | "text": [ 453 | "1000" 454 | ] 455 | } 456 | ], 457 | "prompt_number": 22 458 | }, 459 | { 460 | "cell_type": "code", 461 | "collapsed": false, 462 | "input": [ 463 | "dist = tab.ol_w_id.distribution(limit=10000)" 464 | ], 465 | "language": "python", 466 | "metadata": {}, 467 | "outputs": [], 468 | "prompt_number": 24 469 | }, 470 | { 471 | "cell_type": "code", 472 | "collapsed": false, 473 | "input": [ 474 | "%matplotlib inline\n", 475 | "dist.fraction.hist()" 476 | ], 477 | "language": "python", 478 | "metadata": {}, 479 | "outputs": [ 480 | { 481 | "metadata": {}, 482 | "output_type": "pyout", 483 | "prompt_number": 25, 484 | "text": [ 485 | "| \n", 572 | " | table | \n", 573 | "column | \n", 574 | "distinct | \n", 575 | "size | \n", 576 | "fraction | \n", 577 | "
|---|---|---|---|---|---|
| 0 | \n", 582 | "customerp | \n", 583 | "c_w_id | \n", 584 | "1000 | \n", 585 | "30000000 | \n", 586 | "3.333333e-05 | \n", 587 | "
| 1 | \n", 590 | "customerp | \n", 591 | "c_d_id | \n", 592 | "10 | \n", 593 | "30000000 | \n", 594 | "3.333333e-07 | \n", 595 | "
| 2 | \n", 598 | "customerp | \n", 599 | "c_id | \n", 600 | "3000 | \n", 601 | "30000000 | \n", 602 | "1.000000e-04 | \n", 603 | "
| 3 | \n", 606 | "customerp | \n", 607 | "c_discount | \n", 608 | "5000 | \n", 609 | "30000000 | \n", 610 | "1.666667e-04 | \n", 611 | "
| 4 | \n", 614 | "customerp | \n", 615 | "c_credit | \n", 616 | "2 | \n", 617 | "30000000 | \n", 618 | "6.666667e-08 | \n", 619 | "
| 5 | \n", 622 | "customerp | \n", 623 | "c_last | \n", 624 | "1000 | \n", 625 | "30000000 | \n", 626 | "3.333333e-05 | \n", 627 | "
| 6 | \n", 630 | "customerp | \n", 631 | "c_first | \n", 632 | "29999284 | \n", 633 | "30000000 | \n", 634 | "9.999761e-01 | \n", 635 | "
| 7 | \n", 638 | "customerp | \n", 639 | "c_credit_lim | \n", 640 | "1 | \n", 641 | "30000000 | \n", 642 | "3.333333e-08 | \n", 643 | "
| 8 | \n", 646 | "customerp | \n", 647 | "c_balance | \n", 648 | "1 | \n", 649 | "30000000 | \n", 650 | "3.333333e-08 | \n", 651 | "
| 9 | \n", 654 | "customerp | \n", 655 | "c_ytd_payment | \n", 656 | "1 | \n", 657 | "30000000 | \n", 658 | "3.333333e-08 | \n", 659 | "
| Name | Size |
|---|---|
| pyxplorer_data | 7 |
Rows: 1 / Columns: 2
" 756 | ], 757 | "metadata": {}, 758 | "output_type": "pyout", 759 | "prompt_number": 33, 760 | "text": [ 761 | "| Name | Value |
|---|---|
| Min | 100 |
| Max | distance |
| #Distinct Values | 7 |
| Most Frequent | distance (1) |
| Least Frequent | distance (1) |
| Top 10 MF | 100,129,148,distance,192,113,168 |
| Top 10 LF | 100, 129, 148, distance, 192, 113, 168 |
| Uniqueness | 1.0 |
| Constancy | 0.142857142857 |
Rows: 9 / Columns: 2
" 820 | ], 821 | "metadata": {}, 822 | "output_type": "pyout", 823 | "prompt_number": 39, 824 | "text": [ 825 | "col_0" 826 | ] 827 | } 828 | ], 829 | "prompt_number": 39 830 | } 831 | ], 832 | "metadata": {} 833 | } 834 | ] 835 | } -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | setup( 4 | name='pyxplorer', 5 | version='0.1.0', 6 | author='Martin Grund', 7 | author_email='grundprinzip+pip@gmail.com', 8 | packages=['pyxplorer'], 9 | url='http://github.com/grundprinzip/pyxplorer', 10 | license='LICENSE', 11 | description='Simple Big Data Profiling', 12 | long_description=open('README.rst').read(), 13 | install_requires=[ 14 | "snakebite", 15 | "pyhs2", 16 | "pandas" 17 | ], 18 | ) 19 | --------------------------------------------------------------------------------