├── utils ├── __init__.py ├── intro.py └── ui.py ├── data_sources ├── __init__.py ├── supabase.py ├── google_sheet.py ├── snowflake.py ├── aws_s3_boto.py └── big_query.py ├── .gitignore ├── Pipfile ├── README.md └── streamlit_app.py /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data_sources/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Python / pyenv 2 | .python-version 3 | *.pyc 4 | .venv 5 | 6 | # Streamlit 7 | .streamlit 8 | 9 | # Mac stuff 10 | .DS_Store 11 | 12 | # Editors 13 | .vscode 14 | .vim 15 | -------------------------------------------------------------------------------- /data_sources/supabase.py: -------------------------------------------------------------------------------- 1 | from supabase_py import create_client, Client 2 | import streamlit as st 3 | 4 | supabase: Client = create_client(**st.secrets["supabase"]) 5 | data = supabase.table("users").select("*").execute() 6 | st.write(data) 7 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | google-cloud-bigquery = "==2.30.*" 8 | matplotlib = "*" 9 | streamlit-agraph = "*" 10 | snowflake-connector-python = "*" 11 | google-api-python-client = "*" 12 | gsheetsdb = "*" 13 | pyarrow = "*" 14 | boto3 = "*" 15 | websockets = ">=9.1" 16 | cffi = "==1.14.6" 17 | 18 | [dev-packages] 19 | black = "*" # Pretty formatting of Python code 20 | pynvim = "*" # Allows nvim users to use black formatting 21 | 22 | [pipenv] 23 | allow_prereleases = true 24 | -------------------------------------------------------------------------------- /utils/intro.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | 3 | TUTORIAL_URL = "https://docs.streamlit.io/en/latest/tutorial/databases.html" 4 | 5 | INTRO_IDENTIFIER = "—" 6 | 7 | HOME_PAGE_TEXT = f""" ## Welcome to the 🔌 Data Sources app! 8 | 9 | Congratulations, you have successfully forked and deployed this app 🎊 10 | 11 | We'll now help you: 12 | - Collect your credentials and safely add them to your 🔑 Streamlit Secrets 13 | - Get started with your own app by providing you with sufficient code 🚀 14 | 15 | **Ready?** 16 | 17 | 👈 Choose the data source you want to access! 18 | """ 19 | 20 | 21 | def load_keyboard_class(): 22 | st.write( 23 | """""", 38 | unsafe_allow_html=True, 39 | ) 40 | 41 | 42 | def app(): 43 | 44 | load_keyboard_class() 45 | 46 | st.write(HOME_PAGE_TEXT, unsafe_allow_html=True) 47 | -------------------------------------------------------------------------------- /utils/ui.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | import requests 3 | from PIL import Image 4 | from io import BytesIO 5 | 6 | 7 | def striken(text): 8 | return "".join(chr(822) + t for t in text) 9 | 10 | 11 | def to_do(st_commands, checkbox_id): 12 | cols = st.columns((1, 20)) 13 | done = cols[0].checkbox(" ", key=checkbox_id) 14 | if done: 15 | _, title, *_ = st_commands[0][1].split("**") 16 | cols[1].write(f" **{title}** ", unsafe_allow_html=True) 17 | else: 18 | for (cmd, *args) in st_commands: 19 | with cols[1]: 20 | if cmd == st.write: 21 | st.write(*args, unsafe_allow_html=True) 22 | else: 23 | 24 | cmd(*args) 25 | st.write("") 26 | return done 27 | 28 | 29 | def load_keyboard_class(): 30 | st.write( 31 | """""", 46 | unsafe_allow_html=True, 47 | ) 48 | 49 | 50 | def to_button(text): 51 | return f'{text}' 52 | 53 | 54 | @st.experimental_memo(ttl=60 * 60 * 24) 55 | def image_from_url(url): 56 | response = requests.get(url) 57 | return Image.open(BytesIO(response.content)) 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Streamlit demo: 🔌 The Data Sources app 2 | 3 | Streamlit makes it **super easy** to connect your apps to your data sources. With this app, we'll show you the way! 4 | 5 | ## Get started! 6 | 7 | **Let's deploy this app** in your Streamlit Cloud workspace 🎈 8 | 9 | 10 | 11 | 12 | 13 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 |
Start by forking this app's repository. 14 |
Now, visit your Streamlit Cloud dashboard and click on New App
Make sure that you have chosen the right Streamlit workspace. If you haven't, simply click on your current workspace in the upper right corner of the dashboard, and choose the one you want to deploy with!
26 | You can fill the form with: 27 |
    28 |
  • your repository name ({username}/data_sources_app)
    • 29 |
  • the branch name (main)
    • 30 |
  • the main file path (streamlit_app.py)
    • 31 |
32 |
Finally, click on Deploy! and watch your app launch.
It can take a minute or two!
🎊 Congrats, you're done with this part: your app is running on Streamlit Cloud!

Now stay on your app, choose a data source and follow the instructions there!
43 | 44 | 45 | ### Questions? Comments? 46 | 47 | Please ask in the [Streamlit community](https://discuss.streamlit.io). 48 | -------------------------------------------------------------------------------- /data_sources/google_sheet.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | from gsheetsdb import connect 3 | import toml 4 | 5 | from utils.ui import to_do, to_button, image_from_url 6 | 7 | INIT_GSHEET = f"""**If you don't have one yet, [create a new Google Sheet](https://sheets.new/)**. 8 | 9 | Give it any name, and fill it with mock data.""" 10 | 11 | MAKE_IT_PUBLIC = f"""**Make sure that your Sheet is public** 12 | 13 | Open your Sheet, click on {to_button("Share")} > {to_button("Share with ...")} and select {to_button("Anyone with the link can view")}.""" 14 | 15 | PASTE_INTO_SECRETS = f"""**Paste these TOML credentials into your Streamlit Secrets! ** 16 | 17 | To open your settings, click on {to_button("Manage app")} > {to_button("⋮")} > {to_button("⚙ Settings")} and then update {to_button("Sharing")} and {to_button("Secrets")}""" 18 | 19 | 20 | @st.experimental_singleton() 21 | def get_connector(): 22 | connector = connect() 23 | 24 | assert st.secrets["gsheets"]["public_gsheets_url"].startswith( 25 | "https://docs.google.com/" 26 | ), "Invalid URL, must start with https://docs.google.com" 27 | 28 | return connector 29 | 30 | 31 | def tutorial(): 32 | 33 | to_do( 34 | [ 35 | (st.write, INIT_GSHEET), 36 | ], 37 | "google_sheet_public_gsheet", 38 | ) 39 | 40 | to_do( 41 | [ 42 | (st.write, MAKE_IT_PUBLIC), 43 | ( 44 | st.image, 45 | image_from_url( 46 | "https://user-images.githubusercontent.com/7164864/143441230-0968925f-86a0-4bf1-bc89-c8403df6ef36.png" 47 | ), 48 | ), 49 | ], 50 | "make_it_public", 51 | ) 52 | 53 | def url_to_toml(): 54 | url_input_str = st.text_input("URL of the Google Sheet") 55 | convert = st.button("Create TOML credentials") 56 | if url_input_str or convert: 57 | if not url_input_str.startswith("https://docs.google.com/"): 58 | st.error( 59 | "Invalid URL! The URL must start with https://docs.google.com. Please retry!" 60 | ) 61 | else: 62 | toml_output = toml.dumps( 63 | {"gsheets": {"public_gsheets_url": url_input_str}} 64 | ) 65 | st.code(toml_output, "toml") 66 | 67 | to_do( 68 | [ 69 | ( 70 | st.write, 71 | """**Create TOML credentials**""", 72 | ), 73 | (url_to_toml,), 74 | ], 75 | "google_sheet_creds_formatted", 76 | ) 77 | 78 | to_do( 79 | [ 80 | (st.write, PASTE_INTO_SECRETS), 81 | ( 82 | st.image, 83 | image_from_url( 84 | "https://user-images.githubusercontent.com/7164864/143465207-fa7ddc5f-a396-4291-a08b-7d2ecc9512d2.png" 85 | ), 86 | ), 87 | ], 88 | "copy_pasted_secrets", 89 | ) 90 | 91 | 92 | def app(): 93 | import streamlit as st 94 | import pandas as pd 95 | from gsheetsdb import connect 96 | 97 | # Share the connector across all users connected to the app 98 | @st.experimental_singleton() 99 | def get_connector(): 100 | return connect() 101 | 102 | # Time to live: the maximum number of seconds to keep an entry in the cache 103 | TTL = 24 * 60 * 60 104 | 105 | # Using `experimental_memo()` to memoize function executions 106 | @st.experimental_memo(ttl=TTL) 107 | def query_to_dataframe(_connector, query: str) -> pd.DataFrame: 108 | rows = _connector.execute(query, headers=1) 109 | dataframe = pd.DataFrame(list(rows)) 110 | return dataframe 111 | 112 | @st.experimental_memo(ttl=600) 113 | def get_data(_connector, gsheets_url) -> pd.DataFrame: 114 | return query_to_dataframe(_connector, f'SELECT * FROM "{gsheets_url}"') 115 | 116 | st.markdown(f"## 📝 Connecting to a public Google Sheet") 117 | 118 | gsheet_connector = get_connector() 119 | gsheets_url = st.secrets["gsheets"]["public_gsheets_url"] 120 | 121 | data = get_data(gsheet_connector, gsheets_url) 122 | st.write("👇 Find below the data in the Google Sheet you provided in the secrets:") 123 | st.dataframe(data) 124 | -------------------------------------------------------------------------------- /data_sources/snowflake.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | from snowflake.connector import connect 3 | from snowflake.connector.connection import SnowflakeConnection 4 | import toml 5 | 6 | from utils.ui import to_do, to_button, image_from_url 7 | 8 | SIGN_UP_SNOWFLAKE = """**If you haven't already, [sign up for Snowflake](https://signup.snowflake.com/)**""" 9 | 10 | SIGN_IN_URL = "https://docs.snowflake.com/en/user-guide/connecting.html#logging-in-using-the-web-interface" 11 | SIGN_IN_SNOWFLAKE = f"""**Sign in to the [Snowflake web interface]({SIGN_IN_URL})** 12 | 13 | Don't forget to write down your username, password, account and warehouse identifiers, we need them right after!""" 14 | 15 | PASTE_INTO_SECRETS = f"""**Paste these TOML credentials into your Streamlit Secrets!** 16 | 17 | To open your settings, click on {to_button("Manage app")} > {to_button("⋮")} > {to_button("⚙ Settings")} and then update {to_button("Sharing")} and {to_button("Secrets")}""" 18 | 19 | 20 | @st.experimental_singleton() 21 | def get_connector() -> SnowflakeConnection: 22 | """Create a connector to SnowFlake using credentials filled in Streamlit secrets""" 23 | connector = connect(**st.secrets["snowflake"], client_session_keep_alive=True) 24 | return connector 25 | 26 | 27 | def tutorial(): 28 | 29 | to_do([(st.write, SIGN_UP_SNOWFLAKE)], "sign_up_snowflake") 30 | to_do([(st.write, SIGN_IN_SNOWFLAKE)], "sign_in_snowflake") 31 | 32 | def generate_credentials(): 33 | creds = st.form(key="creds") 34 | user = creds.text_input("User") 35 | password = creds.text_input("Password", type="password") 36 | account = creds.text_input("Account") 37 | warehouse = creds.text_input("Warehouse") 38 | button = creds.form_submit_button("Create TOML credentials") 39 | 40 | if button: 41 | toml_credentials = toml.dumps( 42 | { 43 | "snowflake": { 44 | "user": user, 45 | "password": password, 46 | "account": account, 47 | "warehouse": warehouse, 48 | } 49 | } 50 | ) 51 | st.write("""TOML output:""") 52 | st.caption( 53 | "(You can copy this TOML by hovering on the code box: a copy button will appear on the right)" 54 | ) 55 | st.code(toml_credentials, "toml") 56 | 57 | to_do( 58 | [ 59 | ( 60 | st.write, 61 | """**Fill in your Snowflake credentials and transform them to TOML:**""", 62 | ), 63 | (generate_credentials,), 64 | ], 65 | "snowflake_creds_formatted", 66 | ) 67 | 68 | to_do( 69 | [ 70 | (st.write, PASTE_INTO_SECRETS), 71 | ( 72 | st.image, 73 | image_from_url( 74 | "https://user-images.githubusercontent.com/7164864/143465207-fa7ddc5f-a396-4291-a08b-7d2ecc9512d2.png" 75 | ), 76 | ), 77 | ], 78 | "copy_pasted_secrets", 79 | ) 80 | 81 | 82 | def app(): 83 | import streamlit as st 84 | import pandas as pd 85 | from snowflake.connector import connect 86 | from snowflake.connector.connection import SnowflakeConnection 87 | 88 | # Share the connector across all users connected to the app 89 | @st.experimental_singleton() 90 | def get_connector() -> SnowflakeConnection: 91 | """Create a connector using credentials filled in Streamlit secrets""" 92 | connector = connect(**st.secrets["snowflake"], client_session_keep_alive=True) 93 | return connector 94 | 95 | # Time to live: the maximum number of seconds to keep an entry in the cache 96 | TTL = 24 * 60 * 60 97 | 98 | # Using `experimental_memo()` to memoize function executions 99 | @st.experimental_memo(ttl=TTL) 100 | def get_databases(_connector) -> pd.DataFrame: 101 | """Get all databases available in Snowflake""" 102 | return pd.read_sql("SHOW DATABASES;", _connector) 103 | 104 | @st.experimental_memo(ttl=TTL) 105 | def get_data(_connector, database) -> pd.DataFrame: 106 | """Get tables available in this database""" 107 | query = f"SELECT * FROM {database}.INFORMATION_SCHEMA.TABLES;" 108 | return pd.read_sql(query, _connector) 109 | 110 | st.markdown(f"## ❄️ Connecting to Snowflake") 111 | 112 | snowflake_connector = get_connector() 113 | 114 | databases = get_databases(snowflake_connector) 115 | database = st.selectbox("Choose a Snowflake database", databases.name) 116 | 117 | data = get_data(snowflake_connector, database) 118 | st.write(f"👇 Find below the available tables in database `{database}`") 119 | st.dataframe(data) 120 | -------------------------------------------------------------------------------- /data_sources/aws_s3_boto.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | import boto3 3 | 4 | from utils.ui import to_do, to_button, image_from_url 5 | 6 | SIGN_UP = """**[Sign up](https://aws.amazon.com/) for AWS or log in**""" 7 | 8 | CREATE_BUCKET = f"""**Create a bucket** 9 | 10 | Go to the [S3 console](https://s3.console.aws.amazon.com/s3/home), click on {to_button("Create bucket")} 11 | and create a bucket 12 | """ 13 | 14 | UPLOAD_FILE_IN_BUCKET = f"""**Upload one or more file(s) in the bucket** 15 | 16 | Click on one of your buckets and head to the upload section. Now upload one or more file(s) 17 | """ 18 | 19 | CREATE_ACCESS_KEYS = f"""**Create an access key** 20 | 21 | First, visit the [AWS console](https://console.aws.amazon.com/). Then: 22 | """ 23 | 24 | FORMAT_INTO_TML = f"""**Generate your TOML credentials for Streamlit** 25 | 26 | Paste your "Access Key ID" and "Secret Access Key" below to generate TOML credentials 27 | """ 28 | 29 | PASTE_INTO_SECRETS = f"""**Paste these TOML credentials into your Streamlit Secrets! ** 30 | 31 | To open your settings, click on {to_button("Manage app")} > {to_button("⋮")} > {to_button("⚙ Settings")} and then update {to_button("Sharing")} and {to_button("Secrets")}""" 32 | 33 | # @st.experimental_singleton() 34 | def get_connector(): 35 | """Create a connector to AWS S3""" 36 | 37 | connector = boto3.Session( 38 | aws_access_key_id=st.secrets.aws_s3.ACCESS_KEY_ID, 39 | aws_secret_access_key=st.secrets.aws_s3.SECRET_ACCESS_KEY, 40 | ).resource("s3") 41 | 42 | return connector 43 | 44 | 45 | def tutorial(): 46 | 47 | st.write( 48 | "(Feel free to skip steps if you already have an account, bucket or file!)" 49 | ) 50 | 51 | to_do( 52 | [(st.write, SIGN_UP)], 53 | "sign_up_or_log_in", 54 | ) 55 | 56 | to_do( 57 | [ 58 | (st.write, CREATE_BUCKET), 59 | ( 60 | st.image, 61 | image_from_url( 62 | "https://user-images.githubusercontent.com/7164864/143440317-7db1d5f9-7dc6-45c2-b637-3ec360e73a6d.png" 63 | ), 64 | ), 65 | ], 66 | "create_bucket", 67 | ) 68 | 69 | to_do( 70 | [ 71 | (st.write, UPLOAD_FILE_IN_BUCKET), 72 | ( 73 | st.image, 74 | image_from_url( 75 | "https://user-images.githubusercontent.com/7164864/143440405-aa572b34-b559-407c-97f5-ded4ad9f0495.png" 76 | ), 77 | ), 78 | ], 79 | "upload_file_in_bucket", 80 | ) 81 | 82 | to_do( 83 | [ 84 | (st.write, CREATE_ACCESS_KEYS), 85 | ( 86 | st.image, 87 | image_from_url( 88 | "https://user-images.githubusercontent.com/7164864/143440472-f0bf5bd3-4029-49ad-8732-49e45ebdeef8.png" 89 | ), 90 | ), 91 | ], 92 | "create_access_keys", 93 | ) 94 | 95 | def generate_toml(): 96 | import toml 97 | 98 | form = st.form(key="toml_form") 99 | access_key_id = form.text_input("Access Key ID") 100 | secret_access_key = form.text_input("Secret Access Key", type="password") 101 | submit = form.form_submit_button("Create TOML credentials") 102 | 103 | if submit: 104 | json_credentials = { 105 | "aws_s3": { 106 | "ACCESS_KEY_ID": access_key_id, 107 | "SECRET_ACCESS_KEY": secret_access_key, 108 | } 109 | } 110 | toml_credentials = toml.dumps(json_credentials) 111 | st.write("""TOML credentials:""") 112 | st.caption( 113 | "(You can copy this TOML by hovering on the code box: a copy button will appear on the right)" 114 | ) 115 | st.code(toml_credentials, "toml") 116 | 117 | to_do( 118 | [ 119 | (st.write, FORMAT_INTO_TML), 120 | (generate_toml,), 121 | ], 122 | "format_into_toml", 123 | ) 124 | 125 | to_do( 126 | [ 127 | (st.write, PASTE_INTO_SECRETS), 128 | ( 129 | st.image, 130 | image_from_url( 131 | "https://user-images.githubusercontent.com/7164864/143465207-fa7ddc5f-a396-4291-a08b-7d2ecc9512d2.png" 132 | ), 133 | ), 134 | ], 135 | "copy_pasted_secrets", 136 | ) 137 | 138 | 139 | def app(): 140 | import streamlit as st 141 | import pandas as pd 142 | import boto3 143 | 144 | @st.experimental_singleton() 145 | def get_connector(): 146 | """Create a connector to AWS S3""" 147 | connector = boto3.Session( 148 | aws_access_key_id=st.secrets.aws_s3.ACCESS_KEY_ID, 149 | aws_secret_access_key=st.secrets.aws_s3.SECRET_ACCESS_KEY, 150 | ).resource("s3") 151 | return connector 152 | 153 | # Time to live: the maximum number of seconds to keep an entry in the cache 154 | TTL = 24 * 60 * 60 155 | 156 | @st.experimental_memo(ttl=TTL) 157 | def get_buckets(_connector) -> list: 158 | return [bucket.name for bucket in list(_connector.buckets.all())] 159 | 160 | def to_tuple(s3_object): 161 | return ( 162 | s3_object.key, 163 | s3_object.last_modified, 164 | s3_object.size, 165 | s3_object.storage_class, 166 | ) 167 | 168 | @st.experimental_memo(ttl=TTL) 169 | def get_files(_connector, bucket) -> pd.DataFrame: 170 | files = list(s3.Bucket(name=bucket).objects.all()) 171 | if files: 172 | df = pd.DataFrame( 173 | pd.Series(files).apply(to_tuple).tolist(), 174 | columns=["key", "last_modified", "size", "storage_class"], 175 | ) 176 | return df 177 | 178 | st.markdown(f"## 📦 Connecting to AWS S3") 179 | 180 | s3 = get_connector() 181 | 182 | buckets = get_buckets(s3) 183 | if buckets: 184 | st.write(f"🎉 Found {len(buckets)} bucket(s)!") 185 | bucket = st.selectbox("Choose a bucket", buckets) 186 | files = get_files(s3, bucket) 187 | if isinstance(files, pd.DataFrame): 188 | st.write(f"📁 Found {len(files)} file(s) in this bucket:") 189 | st.dataframe(files) 190 | else: 191 | st.write(f"This bucket is empty!") 192 | else: 193 | st.write(f"Couldn't find any bucket. Make sure to create one!") 194 | -------------------------------------------------------------------------------- /streamlit_app.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import textwrap 3 | import streamlit as st 4 | from pathlib import Path 5 | 6 | import data_sources 7 | from data_sources import big_query, snowflake, aws_s3_boto, google_sheet 8 | 9 | from utils import ui, intro 10 | 11 | DATA_SOURCES = { 12 | intro.INTRO_IDENTIFIER: { 13 | "module": intro, 14 | "secret_key": None, 15 | "docs_url": None, 16 | "get_connector": None, 17 | }, 18 | "🔎 BigQuery": { 19 | "module": data_sources.big_query, 20 | "secret_key": "bigquery", 21 | "docs_url": "https://docs.streamlit.io/knowledge-base/tutorials/databases/bigquery", 22 | "get_connector": data_sources.big_query.get_connector, 23 | "tutorial": data_sources.big_query.tutorial, 24 | "tutorial_anchor": "#tutorial-connecting-to-bigquery", 25 | }, 26 | "❄️ Snowflake": { 27 | "module": data_sources.snowflake, 28 | "secret_key": "snowflake", 29 | "docs_url": "https://docs.streamlit.io/knowledge-base/tutorials/databases/snowflake", 30 | "get_connector": data_sources.snowflake.get_connector, 31 | "tutorial": data_sources.snowflake.tutorial, 32 | "tutorial_anchor": "#tutorial-connecting-to-snowflake", 33 | }, 34 | "📦 AWS S3": { 35 | "module": data_sources.aws_s3_boto, 36 | "secret_key": "aws_s3", 37 | "docs_url": "https://docs.streamlit.io/knowledge-base/tutorials/databases/aws-s3", 38 | "get_connector": data_sources.aws_s3_boto.get_connector, 39 | "tutorial": data_sources.aws_s3_boto.tutorial, 40 | "tutorial_anchor": "#tutorial-connecting-to-aws-s3", 41 | }, 42 | "📝 Google Sheet": { 43 | "module": data_sources.google_sheet, 44 | "secret_key": "gsheets", 45 | "docs_url": "https://docs.streamlit.io/en/latest/tutorial/public_gsheet.html#connect-streamlit-to-a-public-google-sheet", 46 | "get_connector": data_sources.google_sheet.get_connector, 47 | "tutorial": data_sources.google_sheet.tutorial, 48 | "tutorial_anchor": "#tutorial-connecting-to-google-sheet", 49 | }, 50 | } 51 | 52 | NO_CREDENTIALS_FOUND = """❌ **We couldn't find credentials for '`{}`' in your Streamlit Secrets.** 53 | Please follow our tutorial just below 👇""" 54 | 55 | CREDENTIALS_FOUND_BUT_ERROR = """**❌ Credentials were found but there is an error.** 56 | While you have successfully filled in Streamlit secrets for the key `{}`, 57 | we have not been able to connect to the data source. You might have forgotten some fields. 58 | 59 | Check the exception below 👇 60 | """ 61 | 62 | PIPFILE_URL = "https://github.com/streamlit/data_sources_app/blob/main/Pipfile" 63 | WHAT_NEXT = f"""## What next? 64 | 65 | 🚀 Kick-off your own app now! 66 | 67 | - Create a new repository 68 | - Paste the code above into a new file `streamlit_app.py` and use it as a starter! 69 | - Add your dependencies in a `requirements.txt` (take inspiration from our [`Pipfile`]({PIPFILE_URL})!) 70 | 71 | And the rest **you know already**: deploy, add credentials and you're ready to go! 72 | 73 | 🤔 Stuck? Check out our docs on [creating](https://docs.streamlit.io/library/get-started/create-an-app) 74 | and [deploying](https://docs.streamlit.io/streamlit-cloud/get-started/deploy-an-app) an app or reach out to 75 | support@streamlit.io! 76 | """ 77 | 78 | QUESTION_OR_FEEDBACK = """Questions? Comments? Please ask in the [Streamlit community](https://discuss.streamlit.io/).""" 79 | 80 | 81 | def has_data_source_key_in_secrets(data_source: str) -> bool: 82 | return DATA_SOURCES[data_source]["secret_key"] in st.secrets 83 | 84 | 85 | def show_success(data_source: str): 86 | st.success( 87 | f"""👏 Congrats! You have successfully filled in your Streamlit secrets.. 88 | Below, you'll find a sample app that connects to {data_source} and its associated [source code](#code).""" 89 | ) 90 | 91 | 92 | def show_error_when_not_connected(data_source: str): 93 | 94 | st.error( 95 | NO_CREDENTIALS_FOUND.format( 96 | DATA_SOURCES[data_source]["secret_key"], 97 | ) 98 | ) 99 | 100 | st.write(f"### Tutorial: connecting to {data_source}") 101 | ui.load_keyboard_class() 102 | DATA_SOURCES[data_source]["tutorial"]() 103 | 104 | 105 | def what_next(): 106 | st.write(WHAT_NEXT) 107 | 108 | 109 | def code(app): 110 | st.markdown("## Code") 111 | sourcelines, _ = inspect.getsourcelines(app) 112 | st.code(textwrap.dedent("".join(sourcelines[1:])), "python") 113 | 114 | 115 | def connect(data_source): 116 | """Try connecting to data source. 117 | Print exception should something wrong happen.""" 118 | 119 | try: 120 | get_connector = DATA_SOURCES[data_source]["get_connector"] 121 | connector = get_connector() 122 | return connector 123 | 124 | except Exception as e: 125 | 126 | st.sidebar.error("❌ Could not connect.") 127 | 128 | st.error( 129 | CREDENTIALS_FOUND_BUT_ERROR.format(DATA_SOURCES[data_source]["secret_key"]) 130 | ) 131 | 132 | st.exception(e) 133 | 134 | st.stop() 135 | 136 | 137 | # If viewer clicks on page selector: Update query params to point to this page. 138 | def change_page_url(): 139 | """Update query params to reflect the selected page.""" 140 | if st.session_state["page_selector"] == intro.INTRO_IDENTIFIER: 141 | st.experimental_set_query_params() 142 | else: 143 | st.experimental_set_query_params(data_source=st.session_state["page_selector"]) 144 | 145 | 146 | if __name__ == "__main__": 147 | 148 | st.set_page_config(page_title="Data Sources app", page_icon="🔌", layout="centered") 149 | 150 | # Infer selected page from query params. 151 | query_params = st.experimental_get_query_params() 152 | if "data_source" in query_params: 153 | page_url = query_params["data_source"][0] 154 | if page_url in DATA_SOURCES.keys(): 155 | st.session_state["page_selector"] = page_url 156 | 157 | data_source = st.sidebar.selectbox( 158 | "Choose a data source", 159 | list(DATA_SOURCES.keys()), 160 | index=0, 161 | key="page_selector", 162 | on_change=change_page_url, 163 | ) 164 | 165 | st.session_state.active_page = data_source 166 | if "data_sources_already_connected" not in st.session_state: 167 | st.session_state.data_sources_already_connected = list() 168 | 169 | if data_source == intro.INTRO_IDENTIFIER: 170 | show_code = False 171 | show_balloons = False 172 | 173 | else: 174 | show_code = True 175 | show_balloons = True 176 | 177 | # First, look for credentials in the secrets 178 | data_source_key_in_secrets = has_data_source_key_in_secrets(data_source) 179 | 180 | if data_source_key_in_secrets: 181 | connect(data_source) 182 | st.sidebar.success("✔ Connected!") 183 | show_success(data_source) 184 | else: 185 | st.sidebar.error("❌ Could not connect!") 186 | show_error_when_not_connected(data_source) 187 | st.caption(QUESTION_OR_FEEDBACK) 188 | st.stop() 189 | 190 | # Release balloons to celebrate (only upon first success) 191 | if ( 192 | show_balloons 193 | and data_source not in st.session_state.data_sources_already_connected 194 | ): 195 | st.session_state.data_sources_already_connected.append(data_source) 196 | show_balloons = False 197 | st.balloons() 198 | 199 | # Display data source app 200 | data_source_app = DATA_SOURCES[st.session_state["active_page"]]["module"].app 201 | data_source_app() 202 | 203 | # Show source code and what next 204 | if show_code: 205 | code(data_source_app) 206 | what_next() 207 | 208 | st.caption(QUESTION_OR_FEEDBACK) 209 | -------------------------------------------------------------------------------- /data_sources/big_query.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | from google.cloud import bigquery 3 | from google.oauth2.service_account import Credentials 4 | import json, toml 5 | from io import StringIO 6 | 7 | from utils.ui import to_do, to_button, image_from_url 8 | 9 | TUTORIAL_1 = """**Enable the BigQuery API.** 10 | 11 | Programmatic access to BigQuery is controlled through [Google 12 | Cloud Platform](https://cloud.google.com/). Create an account or sign in and head over to the [APIs 13 | & Services dashboard](https://console.cloud.google.com/apis/dashboard). If it's not already listed, 14 | search for the [BigQuery API](https://console.cloud.google.com/marketplace/product/google/bigquery.googleapis.com) 15 | and enable it.""" 16 | 17 | TUTORIAL_2_1 = """**Create a service account & key file.** 18 | 19 | To use the BigQuery API from Streamlit Cloud, 20 | you need a Google Cloud Platform service account (a special account type for programmatic 21 | data access). Go to the [Service Accounts](https://console.cloud.google.com/iam-admin/serviceaccounts) 22 | page, choose a project and click + CREATE SERVICE ACCOUNT .""" 23 | 24 | TUTORIAL_2_2 = """If that button is gray and unavailable, you don't have the correct 25 | permissions. Ask the admin of your Google Cloud project for help.""" 26 | 27 | TUTORIAL_2_3 = """After clicking on DONE 28 | , you should be back on the service accounts overview. Click on your service account, head over to 29 | Keys > Add a key > Create a key > JSON 30 | to create and download your service account file.""" 31 | 32 | TUTORIAL_3 = """**Convert your JSON service account to TOML.**""" 33 | 34 | PASTE_INTO_SECRETS = f"""**Paste these TOML credentials into your Streamlit Secrets! ** 35 | 36 | To open your settings, click on {to_button("Manage app")} > {to_button("⋮")} > {to_button("⚙ Settings")} and then update {to_button("Sharing")} and {to_button("Secrets")}""" 37 | 38 | 39 | @st.experimental_singleton() 40 | def get_connector(): 41 | """Create a connector to BigQuery using credentials filled in Streamlit secrets""" 42 | credentials = Credentials.from_service_account_info(st.secrets["bigquery"]) 43 | connector = bigquery.Client(credentials=credentials) 44 | return connector 45 | 46 | 47 | def tutorial(): 48 | st.write( 49 | """We assume that you have a BigQuery account already, and a database. 50 | If not, please follow [Google's quickstart guide](https://cloud.google.com/bigquery/docs/quickstarts/quickstart-web-ui). 51 | """ 52 | ) 53 | 54 | to_do( 55 | [ 56 | (st.write, TUTORIAL_1), 57 | ( 58 | st.image, 59 | image_from_url( 60 | "https://user-images.githubusercontent.com/7164864/143440812-fbff40a0-2c15-4ade-af74-48b4bdaa5700.png" 61 | ), 62 | ), 63 | ], 64 | "bigquery_enabled", 65 | ) 66 | 67 | to_do( 68 | [ 69 | (st.write, TUTORIAL_2_1), 70 | ( 71 | st.image, 72 | image_from_url( 73 | "https://user-images.githubusercontent.com/7164864/143441050-51754071-0463-4c0b-a733-78c6e9d73572.png" 74 | ), 75 | ), 76 | (st.caption, TUTORIAL_2_2), 77 | (st.write, TUTORIAL_2_3), 78 | ( 79 | st.image, 80 | image_from_url( 81 | "https://user-images.githubusercontent.com/7164864/143441099-8edeb680-c5c7-452f-aae9-30de5f8b700d.png" 82 | ), 83 | ), 84 | ], 85 | "service_account_created", 86 | ) 87 | 88 | def json_to_toml(): 89 | 90 | widget_choice = st.selectbox( 91 | "Choose how to insert your service account", 92 | ["Upload JSON file", "Paste raw JSON content"], 93 | index=0, 94 | ) 95 | 96 | service_account_str = None 97 | 98 | if widget_choice == "Upload JSON file": 99 | service_account_file = st.file_uploader( 100 | "Upload your JSON service account", 101 | accept_multiple_files=False, 102 | type="json", 103 | ) 104 | 105 | if service_account_file is not None: 106 | service_account_str = StringIO( 107 | service_account_file.getvalue().decode("utf-8") 108 | ).read() 109 | 110 | elif widget_choice == "Paste raw JSON content": 111 | service_account_str = st.text_area("Paste your JSON service account below") 112 | 113 | convert = st.button("Convert to TOML") 114 | 115 | if service_account_str or convert: 116 | try: 117 | input_json = json.loads(service_account_str) 118 | input_json = {"bigquery": input_json} 119 | toml_output = toml.dumps(input_json) 120 | st.write("""TOML output:""") 121 | st.caption( 122 | "(You can copy this TOML by hovering on the code box: a copy button will appear on the right)" 123 | ) 124 | st.code(toml_output, "toml") 125 | except Exception as e: 126 | st.error( 127 | "There has been a problem converting your JSON input to TOML. Please check that your JSON input is valid. More information below 👇" 128 | ) 129 | st.exception(e) 130 | 131 | to_do( 132 | [ 133 | (st.write, TUTORIAL_3), 134 | (json_to_toml,), 135 | ], 136 | "filled_in_secrets", 137 | ) 138 | 139 | to_do( 140 | [ 141 | (st.write, PASTE_INTO_SECRETS), 142 | ( 143 | st.image, 144 | image_from_url( 145 | "https://user-images.githubusercontent.com/7164864/143465207-fa7ddc5f-a396-4291-a08b-7d2ecc9512d2.png" 146 | ), 147 | ), 148 | ], 149 | "copy_pasted_secrets", 150 | ) 151 | 152 | 153 | def app(): 154 | import pandas as pd 155 | import streamlit as st 156 | from google.cloud import bigquery 157 | from google.oauth2.service_account import Credentials 158 | 159 | # Share the connector across all users connected to the app 160 | @st.experimental_singleton() 161 | def get_connector(): 162 | """Create a connector using credentials filled in Streamlit secrets""" 163 | credentials = Credentials.from_service_account_info(st.secrets["bigquery"]) 164 | connector = bigquery.Client(credentials=credentials) 165 | return connector 166 | 167 | # Time to live: the maximum number of seconds to keep an entry in the cache 168 | TTL = 24 * 60 * 60 169 | 170 | # Using `experimental_memo()` to memoize function executions 171 | @st.experimental_memo(ttl=TTL) 172 | def get_projects(_connector) -> list: 173 | """Get the list of projects available""" 174 | return [project.project_id for project in list(_connector.list_projects())] 175 | 176 | @st.experimental_memo(ttl=TTL) 177 | def get_data(_connector, project: str) -> pd.DataFrame: 178 | """Get schema data for a given project""" 179 | query = f"SELECT * FROM {project}.INFORMATION_SCHEMA.SCHEMATA;" 180 | return _connector.query(query).to_dataframe() 181 | 182 | st.markdown(f"## 🔎 BigQuery app") 183 | 184 | big_query_connector = get_connector() 185 | 186 | projects = get_projects(big_query_connector) 187 | project = st.selectbox("Choose a BigQuery project", projects) 188 | 189 | data = get_data(big_query_connector, project) 190 | st.write(f"👇 Find below the available schemas in project `{project}`!") 191 | st.dataframe(data) 192 | --------------------------------------------------------------------------------