"""
300 |
301 |
302 | for index, row in df.iterrows():
303 | table_scorecard += """
304 |
305 |
306 |
307 |
308 |
309 |
310 |
311 |
"""+human_format(row['ROW_COUNT'])+"""
312 |
Rows
313 |
314 |
"""+human_bytes(row['BYTES'])+"""
315 |
"""+human_bytes_text(row['BYTES'])+"""
316 |
317 |
"""+"{0:}".format(row['COLUMN_COUNT'])+"""
318 |
Columns
319 |
320 |
321 |
322 |
327 |
335 |
"""
336 |
337 | st.markdown(table_scorecard, unsafe_allow_html=True)
338 |
--------------------------------------------------------------------------------
/snowflake-table-catalog.py:
--------------------------------------------------------------------------------
1 | #from turtle import onclick
2 | import streamlit as st
3 | import snowflake.connector
4 | import pandas as pd
5 | #import streamlit.components.v1 as components
6 | st.set_page_config(layout="wide")
7 | # Initialize connection.
8 | # Uses st.experimental_singleton to only run once.
9 |
10 |
11 | @st.experimental_singleton
12 | def init_connection():
13 | return snowflake.connector.connect(**st.secrets["snowflake"])
14 |
15 |
16 | conn = init_connection()
17 | cur = conn.cursor()
18 |
19 | # Perform query.
20 | # Uses st.experimental_memo to only rerun when the query changes or after 10 min.
21 |
22 | @st.experimental_memo(ttl=600)
23 | def run_query(query):
24 | with conn.cursor() as cur:
25 | cur.execute(query)
26 |
27 | dat = cur.fetchall()
28 | df = pd.DataFrame(dat, columns=[col[0] for col in cur.description])
29 | return df
30 |
31 |
32 | df = run_query("""SELECT
33 | t.TABLE_ID,
34 | t.TABLE_CATALOG,
35 | t.CREATED,
36 | t.TABLE_NAME,
37 | t.TABLE_SCHEMA,
38 | t.TABLE_OWNER,
39 | t.TABLE_TYPE,
40 | t.IS_TRANSIENT,
41 | t.CLUSTERING_KEY,
42 | t.ROW_COUNT,
43 | t.BYTES,
44 | t.RETENTION_TIME,
45 | t.LAST_ALTERED,
46 | t.AUTO_CLUSTERING_ON,
47 | t.COMMENT,
48 | c.column_count
49 | from
50 | SNOWFLAKE.ACCOUNT_USAGE.TABLES t
51 | left join (
52 | select
53 | table_id,
54 | count(distinct column_id) column_count
55 | from
56 | SNOWFLAKE.ACCOUNT_USAGE.COLUMNS
57 | group by
58 | table_id
59 | ) c on c.table_id = t.table_id
60 | where t.table_schema not like '%ANON_HOL%' and deleted is null;""")
61 |
62 | df2 = df
63 | # if 'df' not in st.session_state:
64 | # st.session_state.df = df
65 |
66 | st.title('Snowflake Table Catalog')
67 |
68 | def human_bytes(B):
69 | """Return the given bytes as a human friendly KB, MB, GB, or TB string."""
70 | B = float(B)
71 | KB = float(1024)
72 | MB = float(KB ** 2) # 1,048,576
73 | GB = float(KB ** 3) # 1,073,741,824
74 | TB = float(KB ** 4) # 1,099,511,627,776
75 |
76 | if B < KB:
77 | return '{0} {1}'.format(B, '')
78 | elif KB <= B < MB:
79 | return '{0:.2f}'.format(B / KB)
80 | elif MB <= B < GB:
81 | return '{0:.2f}'.format(B / MB)
82 | elif GB <= B < TB:
83 | return '{0:.2f}'.format(B / GB)
84 | elif TB <= B:
85 | return '{0:.2f}'.format(B / TB)
86 |
87 |
88 | def human_bytes_text(B):
89 | """Return the given bytes as a human friendly KB, MB, GB, or TB string."""
90 | B = float(B)
91 | KB = float(1024)
92 | MB = float(KB ** 2) # 1,048,576
93 | GB = float(KB ** 3) # 1,073,741,824
94 | TB = float(KB ** 4) # 1,099,511,627,776
95 |
96 | if B < KB:
97 | return 'Bytes'
98 | elif KB <= B < MB:
99 | return 'KB'
100 | elif MB <= B < GB:
101 | return 'MB'
102 | elif GB <= B < TB:
103 | return 'GB'
104 | elif TB <= B:
105 | return 'TB'
106 |
107 |
108 | def human_format(num):
109 | magnitude = 0
110 | while abs(num) >= 1000:
111 | magnitude += 1
112 | num /= 1000.0
113 | # add more suffixes if you need them
114 | return ('%.2f%s' % (num, ['', 'K', 'M', 'G', 'T', 'P'][magnitude])).replace('.00', '')
115 |
116 |
117 | def local_css(file_name):
118 | with open(file_name) as f:
119 | st.markdown(f'', unsafe_allow_html=True)
120 |
121 |
122 | def remote_css(url):
123 | st.markdown(f'
',
124 | unsafe_allow_html=True)
125 |
126 |
127 | def header_bg(table_type):
128 | if table_type == "BASE TABLE":
129 | return "tablebackground"
130 | elif table_type == "VIEW":
131 | return "viewbackground"
132 | else:
133 | return "mvbackground"
134 |
135 |
136 | remote_css(
137 | "https://cdnjs.cloudflare.com/ajax/libs/semantic-ui/2.4.1/semantic.min.css")
138 |
139 |
140 | local_css("style.css")
141 | cb_view_details = st.sidebar.checkbox('View Details')
142 |
143 | if cb_view_details:
144 | view_details=""
145 | else:
146 | view_details="""style="display: none;" """
147 |
148 | selectbox_orderby = st.sidebar.selectbox("Order By", ('A → Z', 'Z → A', 'Data Size ↓', 'Data Size ↑',
149 | 'Rows ↓', 'Rows ↑', 'Date Created ↓', 'Date Created ↑', 'Date Altered ↓', 'Date Altered ↑'))
150 | #button_clicked = st.button("OK")
151 |
152 | all_option = pd.Series(['All'], index=[9999999])
153 |
154 | #TABLE_SCHEMA=TABLE_SCHEMA.append({'TABLE_SCHEMA':'All'},ignore_index = True)
155 |
156 | if 'selectbox_database_key' not in st.session_state:
157 | st.session_state.selectbox_database_key = 10
158 | st.session_state.selectbox_schema_key = 20
159 | st.session_state.selectbox_owner_key = 30
160 | st.session_state.selectbox_table_type_key = 40
161 | st.session_state.selectbox_max_rows_key = 50
162 | st.session_state.selectbox_data_size_key = 60
163 |
164 | # Table Catalog/Database
165 | fv_database = df['TABLE_CATALOG'].drop_duplicates()
166 | fv_database = fv_database.append(all_option)
167 |
168 | selectbox_database = st.sidebar.selectbox(
169 | 'Database', fv_database, index=len(fv_database)-1, key=st.session_state.selectbox_database_key)
170 |
171 | if selectbox_database != 'All':
172 | df = df.loc[df['TABLE_CATALOG'] == selectbox_database]
173 | else:
174 | df = df.loc[df['TABLE_CATALOG'].isin(fv_database)]
175 |
176 | # Table Schema
177 | fv_table_schema = df['TABLE_SCHEMA'].drop_duplicates()
178 | fv_table_schema = fv_table_schema.append(all_option)
179 |
180 | selectbox_schema = st.sidebar.selectbox(
181 | "Table Schema", fv_table_schema, len(fv_table_schema)-1, key=st.session_state.selectbox_schema_key)
182 |
183 | if selectbox_schema != 'All':
184 | df = df.loc[df['TABLE_SCHEMA'] == selectbox_schema]
185 | else:
186 | df = df.loc[df['TABLE_SCHEMA'].isin(fv_table_schema)]
187 |
188 | # Table Owner
189 | fv_owner = df['TABLE_OWNER'].drop_duplicates()
190 | fv_owner = fv_owner.append(all_option)
191 | selectbox_owner = st.sidebar.selectbox(
192 | "Table Owner", fv_owner, len(fv_owner)-1, key=st.session_state.selectbox_owner_key)
193 |
194 | if selectbox_owner != 'All':
195 | df = df.loc[df['TABLE_OWNER'] == selectbox_owner]
196 | else:
197 | df = df.loc[df['TABLE_OWNER'].isin(fv_owner)]
198 |
199 | # Table Type
200 | fv_table_type = df['TABLE_TYPE'].drop_duplicates()
201 | selectbox_table_type = st.sidebar.multiselect(
202 | 'Table Type', fv_table_type, fv_table_type, key=st.session_state.selectbox_table_type_key)
203 |
204 | if len(selectbox_table_type) > 0:
205 | df = df.loc[df['TABLE_TYPE'].isin(selectbox_table_type)]
206 | else:
207 | df = df.loc[df['TABLE_TYPE'].isin(fv_table_type)]
208 |
209 | # #!!! This part is disabled since sliders are causing performance issues with large datasets.!!!
210 | # # data size selection
211 | max_data_mb = int(df['BYTES'].max()/1048576)
212 | step_size = 1
213 |
214 | if max_data_mb>1000:
215 | step_size=10
216 | elif max_data_mb>1000000:
217 | step_size=100
218 | elif max_data_mb>1000000000:
219 | step_size=1000
220 | elif max_data_mb>1000000000000:
221 | step_size=10000
222 |
223 | data_size = st.sidebar.slider(
224 | 'Data Size (MB)', 0, max_data_mb+1, (0, max_data_mb+1), key=st.session_state.selectbox_data_size_key, step=step_size)
225 | df = df.loc[(df['BYTES'] >= data_size[0]*1048576) &
226 | (df['BYTES'] <= data_size[1]*1048576)]
227 |
228 | # rows selection
229 | max_rows = int(df['ROW_COUNT'].max())
230 | step_size = 10
231 |
232 | if max_rows>1000000:
233 | step_size=100
234 | elif max_rows>1000000000:
235 | step_size=1000
236 | elif max_rows>1000000000000:
237 | step_size=10000
238 |
239 | data_rows = st.sidebar.slider('Number of Rows', 0, max_rows+1,
240 | (0, max_rows+1), key=st.session_state.selectbox_max_rows_key, step=step_size)
241 | df = df.loc[(df['ROW_COUNT'] >= data_rows[0]) &
242 | (df['ROW_COUNT'] <= data_rows[1])]
243 |
244 |
245 | def reset_button():
246 | st.session_state.selectbox_database_key = st.session_state.selectbox_database_key+1
247 | st.session_state.selectbox_schema_key = st.session_state.selectbox_schema_key+1
248 | st.session_state.selectbox_owner_key = st.session_state.selectbox_owner_key+1
249 | st.session_state.selectbox_table_type_key = st.session_state.selectbox_table_type_key+1
250 | st.session_state.selectbox_max_rows_key = st.session_state.selectbox_max_rows_key+1
251 | st.session_state.selectbox_data_size_key = st.session_state.selectbox_data_size_key+1
252 |
253 |
254 | clear_button = st.sidebar.button(
255 | label='Clear Selections', on_click=reset_button)
256 |
257 | if clear_button:
258 | df = df2
259 |
260 | # Card order
261 | orderby_column = ''
262 | orderby_asc = True
263 |
264 |
265 | if selectbox_orderby == 'A → Z':
266 | orderby_column = 'TABLE_NAME'
267 | orderby_asc = True
268 | elif selectbox_orderby == 'Z → A':
269 | orderby_column = 'TABLE_NAME'
270 | orderby_asc = False
271 | elif selectbox_orderby == 'Data Size ↓':
272 | orderby_column = 'BYTES'
273 | orderby_asc = False
274 | elif selectbox_orderby == 'Data Size ↑':
275 | orderby_column = 'BYTES'
276 | orderby_asc = True
277 | elif selectbox_orderby == 'Rows ↓':
278 | orderby_column = 'ROW_COUNT'
279 | orderby_asc = False
280 | elif selectbox_orderby == 'Rows ↑':
281 | orderby_column = 'ROW_COUNT'
282 | orderby_asc = True
283 | elif selectbox_orderby == 'Date Created ↓':
284 | orderby_column = 'CREATED'
285 | orderby_asc = False
286 | elif selectbox_orderby == 'Date Created ↑':
287 | orderby_column = 'CREATED'
288 | orderby_asc = True
289 | elif selectbox_orderby == 'Date Altered ↓':
290 | orderby_column = 'LAST_ALTERED'
291 | orderby_asc = False
292 | elif selectbox_orderby == 'Date Altered ↑':
293 | orderby_column = 'LAST_ALTERED'
294 | orderby_asc = True
295 |
296 |
297 | df.sort_values(by=[orderby_column], inplace=True, ascending=orderby_asc)
298 |
299 |
300 |
301 | table_scorecard = """
302 |
303 |
304 |
"""+str(df[df['TABLE_TYPE'] == 'BASE TABLE']['TABLE_ID'].count())+"""
305 |
306 |
307 | Tables
308 |
309 |
310 |
311 |
"""+str(df[df['TABLE_TYPE'] == 'VIEW']['TABLE_ID'].count())+"""
312 |
313 |
314 | Views
315 |
316 |
317 |
318 |
"""+str(df[df['TABLE_TYPE'] == 'MATERIALIZED VIEW']['TABLE_ID'].count())+"""
319 |
320 |
321 | Materialized Views
322 |
323 |
324 |
325 |
326 | """+human_format(df['ROW_COUNT'].sum())+"""
327 |
328 |
329 | Rows
330 |
331 |
332 |
333 |
334 |
335 | """+human_bytes(df['BYTES'].sum())+" "+human_bytes_text(df['BYTES'].sum())+"""
336 |
337 |
338 | Data Size
339 |
340 |
341 |
"""
342 |
343 | table_scorecard += """
"""
344 |
345 |
346 | for index, row in df.iterrows():
347 | table_scorecard += """
348 |
349 |
350 |
351 |
352 |
353 |
354 |
355 |
"""+human_format(row['ROW_COUNT'])+"""
356 |
Rows
357 |
358 |
"""+human_bytes(row['BYTES'])+"""
359 |
"""+human_bytes_text(row['BYTES'])+"""
360 |
361 |
"""+"{0:}".format(row['COLUMN_COUNT'])+"""
362 |
Columns
363 |
364 |
365 |
366 |
371 |
379 |
"""
380 |
381 | st.markdown(table_scorecard, unsafe_allow_html=True)
382 |
--------------------------------------------------------------------------------
/style.css:
--------------------------------------------------------------------------------
1 | .column {
2 | float: left;
3 | width: 33.33%;
4 | }
5 |
6 | /* Clear floats after the columns */
7 | .row:after {
8 | content: "";
9 | display: table;
10 | clear: both;
11 | }
12 |
13 |
14 | .kpi.number {
15 | font-size: 2rem;
16 | text-align: center;
17 | color: rgb(90, 90, 90);
18 | }
19 |
20 | .kpi.text{
21 | font-size: 1rem;
22 | text-align: center;
23 | color: rgb(90, 90, 90);
24 | padding-top: 10px;
25 | }
26 |
27 | .tablebackground{
28 | background-color: #BAD2DE !important;
29 | }
30 |
31 |
32 | .viewbackground{
33 | background-color: #CBE2DA !important;
34 | }
35 |
36 |
37 | .mvbackground{
38 | background-color: #E5F0EC !important;
39 | }
40 |
41 |
42 | .smallheader{
43 | font-size: small !important;
44 | }
45 |
46 | #my-container {
47 | height: 100vh;
48 | width: 100%;
49 | }
--------------------------------------------------------------------------------