├── .DS_Store
├── .devcontainer
    └── devcontainer.json
├── .gitignore
├── README.md
├── case_studies.md
├── deta_store.py
├── requirements.txt
├── scraper-embedder.py
└── streamlit_app.py


/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/cefege/seo-chat-bot/06ad5a99838901e8284b0f577481425812f5fc51/.DS_Store


--------------------------------------------------------------------------------
/.devcontainer/devcontainer.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "Python 3",
 3 |   // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
 4 |   "image": "mcr.microsoft.com/devcontainers/python:1-3.11-bullseye",
 5 |   "customizations": {
 6 |     "codespaces": {
 7 |       "openFiles": [
 8 |         "README.md",
 9 |         "streamlit_app.py"
10 |       ]
11 |     },
12 |     "vscode": {
13 |       "settings": {},
14 |       "extensions": [
15 |         "ms-python.python",
16 |         "ms-python.vscode-pylance"
17 |       ]
18 |     }
19 |   },
20 |   "updateContentCommand": "[ -f packages.txt ] && sudo apt update && sudo apt upgrade -y && sudo xargs apt install -y <packages.txt; [ -f requirements.txt ] && pip3 install --user -r requirements.txt; pip3 install --user streamlit; echo '✅ Packages installed and Requirements met'",
21 |   "postAttachCommand": {
22 |     "server": "streamlit run streamlit_app.py --server.enableCORS false --server.enableXsrfProtection false"
23 |   },
24 |   "portsAttributes": {
25 |     "8501": {
26 |       "label": "Application",
27 |       "onAutoForward": "openPreview"
28 |     }
29 |   },
30 |   "forwardPorts": [
31 |     8501
32 |   ]
33 | }


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | # Byte-compiled / optimized / DLL files
  2 | __pycache__/
  3 | *.py[cod]
  4 | *$py.class
  5 | 
  6 | # C extensions
  7 | *.so
  8 | 
  9 | # Distribution / packaging
 10 | .Python
 11 | build/
 12 | develop-eggs/
 13 | dist/
 14 | downloads/
 15 | eggs/
 16 | .eggs/
 17 | lib/
 18 | lib64/
 19 | parts/
 20 | sdist/
 21 | var/
 22 | wheels/
 23 | share/python-wheels/
 24 | *.egg-info/
 25 | .installed.cfg
 26 | *.egg
 27 | MANIFEST
 28 | 
 29 | # PyInstaller
 30 | #  Usually these files are written by a python script from a template
 31 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 32 | *.manifest
 33 | *.spec
 34 | 
 35 | # Installer logs
 36 | pip-log.txt
 37 | pip-delete-this-directory.txt
 38 | 
 39 | # Unit test / coverage reports
 40 | htmlcov/
 41 | .tox/
 42 | .nox/
 43 | .coverage
 44 | .coverage.*
 45 | .cache
 46 | nosetests.xml
 47 | coverage.xml
 48 | *.cover
 49 | *.py,cover
 50 | .hypothesis/
 51 | .pytest_cache/
 52 | cover/
 53 | 
 54 | # Translations
 55 | *.mo
 56 | *.pot
 57 | 
 58 | # Django stuff:
 59 | *.log
 60 | local_settings.py
 61 | db.sqlite3
 62 | db.sqlite3-journal
 63 | 
 64 | # Flask stuff:
 65 | instance/
 66 | .webassets-cache
 67 | 
 68 | # Scrapy stuff:
 69 | .scrapy
 70 | 
 71 | # Sphinx documentation
 72 | docs/_build/
 73 | 
 74 | # PyBuilder
 75 | .pybuilder/
 76 | target/
 77 | 
 78 | # Jupyter Notebook
 79 | .ipynb_checkpoints
 80 | 
 81 | # IPython
 82 | profile_default/
 83 | ipython_config.py
 84 | 
 85 | # pyenv
 86 | #   For a library or package, you might want to ignore these files since the code is
 87 | #   intended to run in multiple environments; otherwise, check them in:
 88 | # .python-version
 89 | 
 90 | # pipenv
 91 | #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
 92 | #   However, in case of collaboration, if having platform-specific dependencies or dependencies
 93 | #   having no cross-platform support, pipenv may install dependencies that don't work, or not
 94 | #   install all needed dependencies.
 95 | #Pipfile.lock
 96 | 
 97 | # poetry
 98 | #   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
 99 | #   This is especially recommended for binary packages to ensure reproducibility, and is more
100 | #   commonly ignored for libraries.
101 | #   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102 | #poetry.lock
103 | 
104 | # pdm
105 | #   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106 | #pdm.lock
107 | #   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108 | #   in version control.
109 | #   https://pdm.fming.dev/#use-with-ide
110 | .pdm.toml
111 | 
112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
113 | __pypackages__/
114 | 
115 | # Celery stuff
116 | celerybeat-schedule
117 | celerybeat.pid
118 | 
119 | # SageMath parsed files
120 | *.sage.py
121 | 
122 | # Environments
123 | .env
124 | .venv
125 | env/
126 | venv/
127 | ENV/
128 | env.bak/
129 | venv.bak/
130 | 
131 | # Spyder project settings
132 | .spyderproject
133 | .spyproject
134 | 
135 | # Rope project settings
136 | .ropeproject
137 | 
138 | # mkdocs documentation
139 | /site
140 | 
141 | # mypy
142 | .mypy_cache/
143 | .dmypy.json
144 | dmypy.json
145 | 
146 | # Pyre type checker
147 | .pyre/
148 | 
149 | # pytype static type analyzer
150 | .pytype/
151 | 
152 | # Cython debug symbols
153 | cython_debug/
154 | 
155 | # PyCharm
156 | #  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
157 | #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
158 | #  and can be added to the global gitignore or merged into this file.  For a more nuclear
159 | #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
160 | #.idea/
161 | 
162 | .vscode/
163 | .streamlit/
164 | article_gen.py
165 | .DS_Store


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # SEO Q&A Chatbot
 2 | 
 3 | This Python code implements a chatbot that answers questions related to SEO.  The chatbot is based on OpenAI's GPT-3.5 language model, which is a powerful and versatile natural language processing model.
 4 | Sources used:
 5 | - seobythesea.com
 6 | - holisticseo.digital
 7 | - guest posts published by Koray Tuğberk GÜBÜR on other publications.
 8 | 
 9 | ## Dependencies
10 | 
11 | This code uses the following libraries:
12 | - `streamlit`: for building the user interface.
13 | - `pinecone`: for retrieving relevant text chunks based on a user's question.
14 | - `openai`: for generating responses to user questions.
15 | - `streamlit_chat`: for displaying chat history in the user interface.
16 | 
17 | To install these libraries, use the following command:
18 | ```
19 | pip install -r requirements.txt
20 | ```
21 | 
22 | ## Usage
23 | 
24 | To run this code, first set the `PINECONE_API_KEY` and `OPEN_AI_API_KEY` environment variables with your Pinecone and OpenAI API keys, respectively. You can get an OpenAI API key by creating an account on the OpenAI website.
25 | 
26 | Then, run the following command:
27 | ```
28 | streamlit run streamlit_app.py
29 | ```
30 | 
31 | This will start the Streamlit server, and you can access the chatbot by opening a web browser and navigating to `http://localhost:8501`.
32 | 
33 | ## How it Works
34 | 
35 | The chatbot works as follows:
36 | 1. The user enters a question in the input field.
37 | 2. The chatbot retrieves relevant text chunks based on the user's question using the Pinecone similarity search service.
38 | 3. The chatbot adds the user's question to the retrieved text chunks to create an augmented query.
39 | 4. The chatbot generates a response to the augmented query using OpenAI's GPT-3.5 (Chat GPT) language model.
40 | 5. The chatbot displays the response to the user, along with the chat history.
41 | 
42 | The chat history is saved in the `st.session_state` dictionary, which is a dictionary that persists across Streamlit sessions. The `message` function from the `streamlit_chat` library is used to display the chat history in the user interface.
43 | 


--------------------------------------------------------------------------------
/case_studies.md:
--------------------------------------------------------------------------------
 1 | ### **Article SEO Case Studies**
 2 | 
 3 | 1. [Topical Authority](https://www.oncrawl.com/technical-seo/importance-topical-authority-semantic-seo/)
 4 | 2. [Semantic SEO and Topical Authority](https://www.holisticseo.digital/theoretical-seo/topical-authority/)
 5 | 3. [Importance of Entity-oriented Search](https://inlinks.net/p/case-studies/importance-of-entity-oriented-search-understanding-for-seo-beyond-strings/)
 6 | 4. [Holistic SEO Case Study](https://www.authoritas.com/blog/encazip-case-study/)
 7 | 5. [Creating Semantic Content Network](https://www.oncrawl.com/technical-seo/creating-semantic-content-networks-with-query-document-templates-case-study/)
 8 | 6. [How does Google Rank](https://www.holisticseo.digital/theoretical-seo/ranking/)
 9 | 7. [Multilingual SEO](https://www.oncrawl.com/technical-seo/multilingual-international-seo-guidelines-case-study/)
10 | 8. [Multilingual and Multiregional SEO](https://www.holisticseo.digital/technical-seo/multilingual-seo/)
11 | 9. [Broad Core Algorithm Updates for YMYL Websites](https://www.oncrawl.com/technical-seo/google-core-updates-effects-problems-and-solutions-for-ymyl-sites/)
12 | 10. [What to know for BCAU](https://www.oncrawl.com/technical-seo/how-to-become-a-winner-from-every-google-core-algorithm-update/)
13 | 11. [Ranking Signal Dilution](https://jetoctopus.com/importance-of-ranking-signal/)
14 | 12. [Technical SEO for YMYL Websites](https://serpstat.com/blog/the-importance-of-technical-seo-insights-of-technical-issues-of-unibaby-project/)
15 | 13. [Authoritative Content Marketing for SEO](https://serpstat.com/blog/user-experience-authoritative-content-and-page-speed-for-organic-traffic-growth/)
16 | 14. [Organic Traffic Growth and BCAU](https://serpstat.com/blog/how-a-web-entity-can-grow-traffic-and-lose-google-core-algorithm-update-at-the-same-time/)
17 | 15. [Contextual Search](https://www.rootandbranchgroup.com/importance-of-context-for-seo/)
18 | 16. [Data Science for SEO](https://www.holisticseo.digital/python-seo/data-science/)
19 | 17. [Google Author Authority](https://www.holisticseo.digital/theoretical-seo/google-author)
20 | 18. [Entity Identity Creation and Management](https://www.holisticseo.digital/seo-research-study/entity-identity-management)
21 | 19. [Holistic SEO Case Study](https://www.holisticseo.digital/seo-research-study/holistic-seo)
22 | 20. [Exact Matching Domain SEO Case Study](https://www.holisticseo.digital/seo-research-study/exact-matching-domain)
23 | 21. [SaaS SEO Case Study](https://www.holisticseo.digital/seo-research-study/saas)
24 | 22. [Semantic Search and Understanding Verbs](https://www.holisticseo.digital/seo-research-study/semantic-search)
25 | 23. [B2B SEO Case Study](https://www.holisticseo.digital/seo-research-study/b2b)
26 | 24. [Lexical Semantics SEO Case Study](https://www.holisticseo.digital/seo-research-study/lexical-semantics)
27 | 25. [Topical Map Expansion and Creation](https://www.holisticseo.digital/seo-research-study/topical-map)
28 | 26. [Entity Attribute Value (EAV) SEO Case Study](https://www.holisticseo.digital/seo-research-study/entity-attribute-value)
29 | 27. [Importance of Quality Thresholds](https://www.oncrawl.com/technical-seo/importance-quality-thresholds-predictive-ranking/)
30 | 28. [Entity Oriented Search](https://www.holisticseo.digital/seo-research-study/entity-seo)
31 | 
32 | ### **Interview References for Semantic SEO**
33 | 
34 | 1. [Koray Tuğberk Gübür talks with Jason Barnard about understanding semantic SEO.](https://www.youtube.com/watch?v=4o2KK0AqmY4&ab_channel=Kalicube)
35 | 2. [Focus on semantic SEO and natural language processing](https://www.youtube.com/watch?v=4KvEsr0x_LI&ab_channel=Majestic)
36 | 3. [With SEO Videoshow channel:](https://www.youtube.com/watch?v=kWEiIK90o1I&ab_channel=SEOVideoShow)
37 | 4. [An ınterview with Koray Tuğberk Gübür about SAAS Seo , sponsored by Ahref](https://www.youtube.com/watch?v=USCtz-OTHAs&ab_channel=KorayTu%C4%9FberkG%C3%9CB%C3%9CR)
38 | 5. [Interview with Koray Tuğberk Gübür for Semantic SEO “Semantic SEO Secrets with Koray Tuğberk Gübür” by Itamar Blauer:](https://www.youtube.com/watch?v=MUPWdpnVFHs)
39 | 6. [Interview with Legend Matt Diggity](https://www.youtube.com/watch?v=77jXj7gNpzY&ab_channel=MattDiggity) (A Breaktrough Moment for SEO with a Dear Friend)
40 | 
41 | ### **Video SEO Case Studies**
42 | 
43 | 1. [Entity Identity Creation and Management SEO Case Study](https://www.youtube.com/watch?v=2unr8p1gk3c)
44 | 2. [Exact Matching Domain SEO Case Study and Tutorial](https://www.youtube.com/watch?v=rsj17rExU_A)
45 | 3. [Entity Attribute Value (EAV) SEO Case Study](https://www.youtube.com/watch?v=fb5nXF6eu5U)
46 | 4. [SaaS SEO Case Study](https://www.youtube.com/watch?v=YEZPo6tyIh4)
47 | 5. [B2B SEO Case Study and Guide](https://www.youtube.com/watch?v=ZlDwGYl0LuI)
48 | 6. [Query Semantics SEO Case Study](https://www.youtube.com/watch?v=bv9Bzufci74)
49 | 7. [Holistic SEO Tutorial Step by Step and Case Study](https://www.youtube.com/watch?v=JNCGp7c-Yy0)
50 | 8. [Lexical Semantics and Relations for Semantic SEO](https://www.youtube.com/watch?v=XL2aL7P7ZUI)
51 | 9. [Entity-Oriented SEO Case Study by Exceeding 4 Millions Clicks a Month](https://www.youtube.com/watch?v=xeBW5GEIVI4)
52 | 10. [Keyword Difficulty: Quality Thresholds and Predictive Ranking of Search Engines - SEO Case Study](https://www.youtube.com/watch?v=n550vDnj0fU)
53 | 11. [Semantic SEO Strategy: Case Study](https://www.youtube.com/watch?v=CzMiZk-XZXU)
54 | 12. [Semantic SEO - An SEO Case Study with Google's Algorithm Analysis](https://www.youtube.com/watch?v=mwtV5ji7MF0)


--------------------------------------------------------------------------------
/deta_store.py:
--------------------------------------------------------------------------------
 1 | import pandas as pd
 2 | import streamlit as st
 3 | from deta import Deta
 4 | 
 5 | deta = Deta(st.secrets["API"]["DETA_KEY"])
 6 | db = deta.Base("topical_q_a")
 7 | 
 8 | 
 9 | def fetch_all_items(db):
10 |     res = db.fetch()
11 |     all_items = res.items
12 | 
13 |     # fetch until last is 'None'
14 |     while res.last:
15 |         res = db.fetch(last=res.last)
16 |         all_items += res.items
17 |         print(res)
18 | 
19 |     return all_items
20 | 
21 | 
22 | def check_dataframe(data):
23 |     if not isinstance(data, pd.DataFrame):
24 |         data = pd.DataFrame(data)
25 |     return data
26 | 
27 | 
28 | def convert_timestamp_to_time(df):
29 |     df["timestamp"] = pd.to_datetime(df["timestamp"])
30 |     return df
31 | 
32 | 
33 | def sort_by_timestamp(df):
34 |     df = df.sort_values(by="timestamp", ascending=False)
35 |     return df
36 | 
37 | 
38 | def app():
39 |     st.set_page_config(layout="wide")
40 |     st.title("Data Table")
41 |     data = fetch_all_items(db)
42 |     data = check_dataframe(data)
43 |     data = data.drop(columns=["key"])
44 |     data = convert_timestamp_to_time(data)
45 |     data = sort_by_timestamp(data)
46 |     if data is not None:
47 |         st.dataframe(data)
48 |         csv = data.to_csv(index=False)
49 |         st.download_button(
50 |             label="Download data as CSV",
51 |             data=csv,
52 |             file_name="q_data.csv",
53 |             mime="text/csv",
54 |         )
55 |     else:
56 |         st.warning("No data to display.")
57 | 
58 | 
59 | app()
60 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
  1 | advertools==0.14.2
  2 | aiohttp==3.9.3
  3 | aiosignal==1.3.1
  4 | altair==5.2.0
  5 | annotated-types==0.6.0
  6 | anyio==4.3.0
  7 | attrs==23.2.0
  8 | Automat==22.10.0
  9 | blinker==1.7.0
 10 | cachetools==5.3.3
 11 | certifi==2024.2.2
 12 | cffi==1.16.0
 13 | charset-normalizer==3.3.2
 14 | click==8.1.7
 15 | constantly==23.10.4
 16 | cryptography==42.0.5
 17 | cssselect==1.2.0
 18 | dataclasses-json==0.6.4
 19 | deta==1.2.0
 20 | distro==1.9.0
 21 | filelock==3.13.1
 22 | frozenlist==1.4.1
 23 | gitdb==4.0.11
 24 | GitPython==3.1.42
 25 | greenlet==3.0.3
 26 | h11==0.14.0
 27 | httpcore==1.0.4
 28 | httpx==0.27.0
 29 | hyperlink==21.0.0
 30 | idna==3.6
 31 | incremental==22.10.0
 32 | itemadapter==0.8.0
 33 | itemloaders==1.1.0
 34 | Jinja2==3.1.3
 35 | jmespath==1.0.1
 36 | jsonpatch==1.33
 37 | jsonpointer==2.4
 38 | jsonschema==4.21.1
 39 | jsonschema-specifications==2023.12.1
 40 | langchain==0.1.11
 41 | langchain-community==0.0.27
 42 | langchain-core==0.1.30
 43 | langchain-text-splitters==0.0.1
 44 | langsmith==0.1.23
 45 | lxml==5.1.0
 46 | markdown-it-py==3.0.0
 47 | MarkupSafe==2.1.5
 48 | marshmallow==3.21.1
 49 | mdurl==0.1.2
 50 | multidict==6.0.5
 51 | mypy-extensions==1.0.0
 52 | numpy==1.26.4
 53 | oauthlib==3.2.2
 54 | openai==1.13.3
 55 | orjson==3.9.15
 56 | packaging==23.2
 57 | pandas==2.2.1
 58 | parsel==1.8.1
 59 | pillow==10.2.0
 60 | pinecone-client==3.1.0
 61 | Protego==0.3.0
 62 | protobuf==4.25.3
 63 | pyarrow==15.0.1
 64 | pyasn1==0.5.1
 65 | pyasn1-modules==0.3.0
 66 | pycparser==2.21
 67 | pydantic==2.6.3
 68 | pydantic_core==2.16.3
 69 | pydeck==0.8.1b0
 70 | PyDispatcher==2.0.7
 71 | Pygments==2.17.2
 72 | pyOpenSSL==24.1.0
 73 | python-dateutil==2.9.0.post0
 74 | pytz==2024.1
 75 | PyYAML==6.0.1
 76 | queuelib==1.6.2
 77 | referencing==0.33.0
 78 | regex==2023.12.25
 79 | requests==2.31.0
 80 | requests-file==2.0.0
 81 | requests-oauthlib==1.3.1
 82 | rich==13.7.1
 83 | rpds-py==0.18.0
 84 | Scrapy==2.11.1
 85 | service-identity==24.1.0
 86 | six==1.16.0
 87 | smmap==5.0.1
 88 | sniffio==1.3.1
 89 | SQLAlchemy==2.0.28
 90 | streamlit==1.32.0
 91 | streamlit-chat==0.1.1
 92 | tenacity==8.2.3
 93 | tiktoken==0.6.0
 94 | tldextract==5.1.1
 95 | toml==0.10.2
 96 | toolz==0.12.1
 97 | tornado==6.4
 98 | tqdm==4.66.2
 99 | Twisted==24.3.0
100 | twython==3.9.1
101 | typing-inspect==0.9.0
102 | typing_extensions==4.10.0
103 | tzdata==2024.1
104 | urllib3==2.2.1
105 | w3lib==2.1.2
106 | watchdog==4.0.0
107 | yarl==1.9.4
108 | zope.interface==6.2
109 | 


--------------------------------------------------------------------------------
/scraper-embedder.py:
--------------------------------------------------------------------------------
  1 | import time
  2 | from uuid import uuid4
  3 | 
  4 | import advertools as adv
  5 | import pandas as pd
  6 | import pinecone
  7 | import tiktoken
  8 | import toml
  9 | from langchain.text_splitter import RecursiveCharacterTextSplitter
 10 | from openai import OpenAI
 11 | from tqdm.auto import tqdm
 12 | 
 13 | # Load the secrets.toml file
 14 | secrets = toml.load(".streamlit/secrets.toml")
 15 | 
 16 | # Retrieve the values
 17 | PINECONE_API_KEY = secrets["API"]["PINECONE_API_KEY"]
 18 | OPEN_AI_API_KEY = secrets["API"]["OPEN_AI_API_KEY"]
 19 | 
 20 | client = OpenAI(api_key=OPEN_AI_API_KEY)
 21 | 
 22 | sitemaps = [
 23 |     "https://www.seobythesea.com/post-sitemap.xml",
 24 |     "https://www.seobythesea.com/post-sitemap2.xml",
 25 |     "https://www.holisticseo.digital/post-sitemap1.xml",
 26 |     "https://www.holisticseo.digital/post-sitemap2.xml",
 27 |     "https://www.holisticseo.digital/post-sitemap3.xml",
 28 |     "https://www.holisticseo.digital/post-sitemap4.xml",
 29 | ]
 30 | 
 31 | additional_urls = [
 32 |     "https://www.oncrawl.com/technical-seo/importance-topical-authority-semantic-seo/",
 33 |     "https://www.holisticseo.digital/theoretical-seo/topical-authority/",
 34 |     "https://inlinks.net/p/case-studies/importance-of-entity-oriented-search-understanding-for-seo-beyond-strings/",
 35 |     "https://www.oncrawl.com/technical-seo/creating-semantic-content-networks-with-query-document-templates-case-study/",
 36 |     "https://www.holisticseo.digital/theoretical-seo/ranking/",
 37 |     "https://www.oncrawl.com/technical-seo/multilingual-international-seo-guidelines-case-study/",
 38 |     "https://www.holisticseo.digital/technical-seo/multilingual-seo/",
 39 |     "https://www.oncrawl.com/technical-seo/google-core-updates-effects-problems-and-solutions-for-ymyl-sites/",
 40 |     "https://www.oncrawl.com/technical-seo/how-to-become-a-winner-from-every-google-core-algorithm-update/",
 41 |     "https://jetoctopus.com/importance-of-ranking-signal/",
 42 |     "https://serpstat.com/blog/the-importance-of-technical-seo-insights-of-technical-issues-of-unibaby-project/",
 43 |     "https://serpstat.com/blog/user-experience-authoritative-content-and-page-speed-for-organic-traffic-growth/",
 44 |     "https://serpstat.com/blog/how-a-web-entity-can-grow-traffic-and-lose-google-core-algorithm-update-at-the-same-time/",
 45 |     "https://www.rootandbranchgroup.com/importance-of-context-for-seo/",
 46 |     "https://www.holisticseo.digital/python-seo/data-science/",
 47 |     "https://www.holisticseo.digital/theoretical-seo/google-author",
 48 |     "https://www.authoritas.com/blog/encazip-case-study/",
 49 | ]
 50 | 
 51 | 
 52 | def scrape_sitemaps(sitemaps):
 53 |     """Scrape a list of sitemaps and return a dataframe with the results"""
 54 |     df = pd.concat([adv.sitemap_to_df(sitemap) for sitemap in sitemaps])
 55 |     df.to_csv("sitemap.csv", index=False)
 56 | 
 57 |     return df
 58 | 
 59 | 
 60 | def convert_df_to_list(df):
 61 |     """Convert a dataframe to a list"""
 62 |     return df["loc"].tolist()
 63 | 
 64 | 
 65 | # merge url lists and remove duplicates
 66 | def merge_url_lists(url_list1, url_list2):
 67 |     return list(set(url_list1 + url_list2))
 68 | 
 69 | 
 70 | def scrape_pages(url_list):
 71 |     adv.crawl(url_list, output_file="page_data.jl", follow_links=False)
 72 |     # convert jl to csv
 73 |     df = pd.read_json("page_data.jl", lines=True)
 74 |     df = df[["url", "title", "body_text"]]
 75 |     df.to_json("page_data.json", orient="records", lines=True)
 76 |     df.to_csv("page_data.csv", index=False)
 77 |     return df
 78 | 
 79 | 
 80 | def convert_df_to_dict(df):
 81 |     df = pd.read_json(
 82 |         "page_data.json", lines=True, orient="records", encoding="utf-8", dtype=str
 83 |     )
 84 |     df = df.to_dict(orient="records")
 85 |     return df
 86 | 
 87 | 
 88 | # Step 1
 89 | # df = scrape_sitemaps(sitemaps)
 90 | # url_list_1 = convert_df_to_list(df)
 91 | # urls = merge_url_lists(url_list_1, additional_urls)
 92 | # scrape_pages(urls)
 93 | 
 94 | 
 95 | # create the length function
 96 | def tiktoken_len(text):
 97 |     tokenizer = tiktoken.get_encoding("p50k_base")
 98 |     tokens = tokenizer.encode(text, disallowed_special=())
 99 |     return len(tokens)
100 | 
101 | 
102 | def create_chunks(data):
103 |     text_splitter = RecursiveCharacterTextSplitter(
104 |         chunk_size=500,
105 |         chunk_overlap=20,
106 |         length_function=tiktoken_len,
107 |         separators=["\n\n", "\n", " ", ""],
108 |     )
109 |     chunks = []
110 |     for idx, record in enumerate(tqdm(data)):
111 |         texts = text_splitter.split_text(record["body_text"])
112 |         chunks.extend(
113 |             [
114 |                 {
115 |                     "id": str(uuid4()),
116 |                     "title": record["title"],
117 |                     "text": texts[i],
118 |                     "chunk": i,
119 |                     "url": record["url"],
120 |                 }
121 |                 for i in range(len(texts))
122 |             ]
123 |         )
124 |     return chunks
125 | 
126 | 
127 | def init_pinecone(api_key, environment):
128 |     """
129 |     Initializes a connection to the Pinecone service.
130 | 
131 |     Args:
132 |         api_key (str): The API key for the Pinecone service.
133 |         environment (str): The environment name for the Pinecone service.
134 | 
135 |     Returns:
136 |         None
137 |     """
138 |     pinecone.init(api_key=api_key, environment=environment)
139 | 
140 | 
141 | def create_index_if_not_exists(index_name, dimension, metric):
142 |     # Check if the index already exists
143 |     if index_name not in pinecone.list_indexes():
144 |         # If the index does not exist, create a new index
145 |         pinecone.create_index(index_name, dimension=dimension, metric=metric)
146 |         print(f"Index '{index_name}' created successfully.")
147 |     else:
148 |         print(f"Index '{index_name}' already exists.")
149 | 
150 | 
151 | def create_embeddings(chunks, embed_model, index, batch_size=100):
152 |     """
153 |     Create embeddings for text chunks and insert them into Pinecone index.
154 | 
155 |     Args:
156 |         chunks (list): List of text chunks to be embedded.
157 |         embed_model (str): Name of the OpenAI GPT model to use for embedding.
158 |         index (pinecone.Index): Pinecone index object to insert embeddings into.
159 |         batch_size (int): Number of embeddings to create and insert at once.
160 | 
161 |     Returns:
162 |         None
163 |     """
164 |     # Set up OpenAI API key
165 | 
166 |     for i in tqdm(range(0, len(chunks), batch_size)):
167 |         # find end of batch
168 |         i_end = min(len(chunks), i + batch_size)
169 |         meta_batch = chunks[i:i_end]
170 |         # get ids
171 |         ids_batch = [x["id"] for x in meta_batch]
172 |         # get texts to encode
173 |         texts = [x["text"] for x in meta_batch]
174 |         # create embeddings (try-except added to avoid RateLimitError)
175 |         try:
176 |             res = client.embeddings.create(input=texts, engine=embed_model)
177 |         except Exception:
178 |             done = False
179 |             while not done:
180 |                 time.sleep(5)
181 |                 try:
182 |                     res = client.embeddings.create(input=texts, engine=embed_model)
183 |                     done = True
184 |                 except Exception:
185 |                     pass
186 |         embeds = [record["embedding"] for record in res.data]
187 |         # cleanup metadata
188 |         meta_batch = [
189 |             {
190 |                 "title": x["title"],
191 |                 "text": x["text"],
192 |                 "chunk": x["chunk"],
193 |                 "url": x["url"],
194 |             }
195 |             for x in meta_batch
196 |         ]
197 |         to_upsert = list(zip(ids_batch, embeds, meta_batch))
198 |         # upsert to Pinecone
199 |         index.upsert(vectors=to_upsert)
200 | 
201 | 
202 | ### STEP 2 Embedding and indexing
203 | 
204 | data = pd.read_json(
205 |     "page_data.json", lines=True, orient="records", encoding="utf-8", dtype=str
206 | )
207 | # convert to list of dicts
208 | data = data.to_dict(orient="records")
209 | 
210 | chunks = create_chunks(data)
211 | 
212 | # # save chunks to json
213 | # pd.DataFrame(chunks).to_json("chunks.json", orient="records", lines=True)
214 | 
215 | ### STEP 3 Uploading to pinecone
216 | 
217 | PINECONE_INDEX_NAME = secrets["API"]["PINECONE_INDEX_NAME"]
218 | PINECONE_ENV = secrets["API"]["PINECONE_ENV"]
219 | # chunks = pd.read_json("chunks.json", lines=True, orient="records", dtype=str)
220 | 
221 | print("Initializing Pinecone...")
222 | print(f"API key: {PINECONE_API_KEY}")
223 | print(f"Environment: {PINECONE_ENV}")
224 | print(f"Index name: {PINECONE_INDEX_NAME}")
225 | 
226 | init_pinecone(api_key=PINECONE_API_KEY, environment=PINECONE_ENV)
227 | 
228 | create_index_if_not_exists(
229 |     index_name=PINECONE_INDEX_NAME, dimension=1536, metric="dotproduct"
230 | )
231 | 
232 | index = pinecone.GRPCIndex(PINECONE_INDEX_NAME)
233 | 
234 | create_embeddings(
235 |     chunks=chunks,
236 |     embed_model="text-embedding-ada-002",
237 |     index=index,
238 |     batch_size=100,
239 | )
240 | 


--------------------------------------------------------------------------------
/streamlit_app.py:
--------------------------------------------------------------------------------
  1 | import datetime
  2 | 
  3 | import streamlit as st
  4 | from deta import Deta
  5 | from openai import OpenAI
  6 | from pinecone import Pinecone, ServerlessSpec
  7 | 
  8 | # Initialize Pinecone client with the API key
  9 | pinecone_client = Pinecone(api_key=st.secrets["API"]["PINECONE_API_KEY"])
 10 | 
 11 | PINECONE_INDEX_NAME = st.secrets["API"]["PINECONE_INDEX_NAME"]
 12 | PINECONE_HOST = st.secrets["API"]["PINECONE_HOST"]
 13 | 
 14 | deta = Deta(st.secrets["API"]["DETA_KEY"])
 15 | 
 16 | 
 17 | client = OpenAI(api_key=st.secrets["API"]["OPEN_AI_API_KEY"])
 18 | 
 19 | 
 20 | def display_existing_messages():
 21 |     if "messages" not in st.session_state:
 22 |         st.session_state["messages"] = []
 23 |     for message in st.session_state["messages"]:
 24 |         with st.chat_message(message["role"]):
 25 |             st.markdown(message["content"])
 26 | 
 27 | 
 28 | def add_user_message_to_session(prompt):
 29 |     if prompt:
 30 |         st.session_state["messages"].append({"role": "user", "content": prompt})
 31 |         with st.chat_message("user"):
 32 |             st.markdown(prompt)
 33 | 
 34 | 
 35 | def generate_assistant_response(augmented_query):
 36 |     primer = """
 37 | Your task is to answer user questions based on the information given above each question.It is crucial to cite sources accurately by using the [[number](URL)] notation after the reference. Say "I don't know" if the information is missing and be as detailed as possible. End each sentence with a period. Please begin.
 38 |               """
 39 |     with st.chat_message("assistant"):
 40 |         message_placeholder = st.empty()
 41 |         full_response = ""
 42 |         for response in client.chat.completions.create(
 43 |             model="gpt-3.5-turbo",
 44 |             temperature=0,
 45 |             messages=[
 46 |                 {"role": "system", "content": primer},
 47 |                 {"role": "user", "content": augmented_query},
 48 |             ],
 49 |             stream=True,
 50 |         ):
 51 |             if partial_response := response.choices[0].delta.content:
 52 |                 full_response += partial_response
 53 |             message_placeholder.markdown(full_response + "▌")
 54 |         message_placeholder.markdown(full_response)
 55 | 
 56 |         st.session_state["messages"].append(
 57 |             {"role": "assistant", "content": full_response}
 58 |         )
 59 |     return full_response
 60 | 
 61 | 
 62 | def get_query_embedding(query):
 63 |     # Embed the query using OpenAI's text-embedding-ada-002 engine
 64 |     query_embedding = (
 65 |         client.embeddings.create(input=[query], model="text-embedding-ada-002")
 66 |         .data[0]
 67 |         .embedding
 68 |     )
 69 | 
 70 |     return query_embedding
 71 | 
 72 | 
 73 | def get_relevant_contexts(query_embedding, index_name):
 74 |     # Ensure the index exists or create it if it doesn't
 75 |     if index_name not in pinecone_client.list_indexes().names():
 76 |         pinecone_client.create_index(
 77 |             name=index_name,
 78 |             dimension=len(query_embedding),
 79 |             metric="cosine",
 80 |             spec=ServerlessSpec(
 81 |                 cloud="aws",
 82 |                 region="us-west-2",
 83 |             ),
 84 |         )
 85 | 
 86 |     # Connect to the existing index
 87 |     index = pinecone_client.Index(index_name=index_name, host=PINECONE_HOST)
 88 | 
 89 |     res = index.query(vector=query_embedding, top_k=6, include_metadata=True)
 90 |     contexts = []
 91 |     for item in res["matches"]:
 92 |         metadata = item["metadata"]
 93 |         text = metadata.get("text", "")
 94 |         url = metadata.get("url", "")
 95 |         title = metadata.get("title", "")
 96 |         relevance_score = item.get("score", "")
 97 |         context = {
 98 |             "search_results_text": text,
 99 |             "search_results_url": url,
100 |             "search_results_title": title,
101 |             "search_relevance_score": relevance_score,
102 |         }
103 |         contexts.append(context)
104 | 
105 |     contexts = str(contexts)
106 |     return contexts
107 | 
108 | 
109 | def augment_query(contexts, query):
110 |     augmented_query = (
111 |         f"###Search Results: \n{contexts} #End of Search Results\n\n-----\n\n {query}"
112 |     )
113 |     return augmented_query
114 | 
115 | 
116 | def add_to_database(query, response):
117 |     db = deta.Base("topical_q_a")
118 |     timestamp = datetime.datetime.now().strftime("%d-%m-%Y %H:%M:%S")
119 |     db.put({"query": query, "response": response, "timestamp": timestamp})
120 | 
121 | 
122 | def print_markdown_from_file(file_path):
123 |     with open(file_path, "r") as f:
124 |         markdown_content = f.read()
125 |         st.markdown(markdown_content)
126 | 
127 | 
128 | def hide_streamlit_header_footer():
129 |     hide_st_style = """
130 |             <style>
131 |             #MainMenu {visibility: hidden;}
132 |             footer {visibility: hidden;}
133 |             header {visibility: hidden;}
134 |             #root > div:nth-child(1) > div > div > div > div > section > div {padding-top: 0rem;}
135 |             </style>
136 |             """
137 |     st.markdown(hide_st_style, unsafe_allow_html=True)
138 | 
139 | 
140 | def main():
141 |     st.title("SEO Q&A Chatbot")
142 |     st.write(
143 |         "“The great aim of education is not knowledge but action.” ― Herbert Spencer"
144 |     )
145 | 
146 |     hide_streamlit_header_footer()
147 |     display_existing_messages()
148 | 
149 |     query = st.chat_input("Ask any question related to SEO")
150 |     if query:
151 |         add_user_message_to_session(query)
152 |         query_embedding = get_query_embedding(query)
153 |         contexts = get_relevant_contexts(
154 |             query_embedding, index_name=PINECONE_INDEX_NAME
155 |         )
156 |         augmented_query = augment_query(contexts, query)
157 |         response = generate_assistant_response(augmented_query)
158 |         add_to_database(query, response)
159 |     with st.sidebar:
160 |         print_markdown_from_file("case_studies.md")
161 | 
162 | 
163 | if __name__ == "__main__":
164 |     main()
165 | 


--------------------------------------------------------------------------------