├── .github └── workflows │ ├── main.yaml │ └── static.yml ├── .gitignore ├── CITATION.cff ├── README.md ├── _minted-main ├── 0B400837BE07578DE366DCD410AD7E392877B981C94AD6CEAF0075A5318FF017.pygtex ├── 0C985E5C55D91891120F152CE09600D75018845A16FEE2F871CFA2D2B2CFB1C6.pygtex ├── 32D5CF32604E0D919E7B4A991777294A2877B981C94AD6CEAF0075A5318FF017.pygtex ├── 38851A735B84BFD7FC57CBEDC1A8C54C5018845A16FEE2F871CFA2D2B2CFB1C6.pygtex ├── 3C3C80F0A97EC51AA116D2F587016AE85018845A16FEE2F871CFA2D2B2CFB1C6.pygtex ├── 50F353AB414E901AB3F8253F641CCD3D5018845A16FEE2F871CFA2D2B2CFB1C6.pygtex ├── 614143CE898A10CDD926BB56864F00B85018845A16FEE2F871CFA2D2B2CFB1C6.pygtex ├── 6E7EEB2C370D1F397DEAB0E58E0EDB462877B981C94AD6CEAF0075A5318FF017.pygtex ├── 7195AF5D847B6BCCB7A150D19F7ABA625018845A16FEE2F871CFA2D2B2CFB1C6.pygtex ├── 7D831596D43890A2BADD43064A830E6E5018845A16FEE2F871CFA2D2B2CFB1C6.pygtex ├── 824C4087A6F8A7A7A1B282559BD58B455018845A16FEE2F871CFA2D2B2CFB1C6.pygtex ├── 8E5087457994186475A82E6630AE60F05018845A16FEE2F871CFA2D2B2CFB1C6.pygtex ├── 9A8269913F182230BF3333652DD315692877B981C94AD6CEAF0075A5318FF017.pygtex ├── 9D8E8AD3D297B1AD40F60B8FF6CF54802877B981C94AD6CEAF0075A5318FF017.pygtex ├── A68336B712BE240AD81BE1B66DBA3B5A5018845A16FEE2F871CFA2D2B2CFB1C6.pygtex ├── B796679A805702BA76C4A53F8A0B63172877B981C94AD6CEAF0075A5318FF017.pygtex ├── BA7FD7AFD5C763308290D7AB2859D8E6360E91D77B40F6AB2A9DE4914DF5D2A9.pygtex ├── C7DD6291193B588922A017A6743B30C65018845A16FEE2F871CFA2D2B2CFB1C6.pygtex ├── DE71EB197869238CE0CF93F140D0B6012877B981C94AD6CEAF0075A5318FF017.pygtex ├── F559E8704ECF0DC32EFED27A75ED0EB35018845A16FEE2F871CFA2D2B2CFB1C6.pygtex ├── F5A0B26D022F8704EBD415636678FCE62877B981C94AD6CEAF0075A5318FF017.pygtex └── default.pygstyle ├── diazessay.cls ├── embeddings.bib ├── embeddings.pdf ├── embeddings.tex ├── figures ├── app_flow.png ├── bag_of_words.png ├── deepmodel.png ├── embed_flow.png ├── embedding_arc.png ├── embedding_pipeline.png ├── embeddings.png ├── embeddings1.png ├── embeddings2.png ├── embeddings_1.png ├── embeddings_arch.png ├── encoder_only.png ├── flutter.png ├── function.png ├── function@2x.png ├── home_timeline.png ├── kandinsky.jpg ├── kandinsky.png ├── ml-flow.png ├── ml_flow.png ├── ml_system.png ├── ml_workflow.png ├── mlnotml.png ├── mltree.png ├── model_cycle.png ├── paradigm.png ├── pca.png ├── pinnersage.png ├── pinnersage_model.png ├── pyramid.png ├── recsys.png ├── rules.png ├── rules_ml.png ├── slack_suggestions.png ├── solutions.png ├── swe_flow.png ├── timeline.png ├── timeline2.png ├── transformer.png ├── transformer_timeline.png ├── transformer_v.png ├── true_embeddings.png ├── twhin.png ├── twice.png ├── web_service.png ├── web_service_ml.png ├── wideanddeep.png ├── wideanddeep_arch.png ├── wideanddeep_ml_arch.png ├── word2vec.png ├── word2vecmodel.png ├── youtube.png ├── youtube_model.png └── youtube_recsys.png ├── kandinsky.png └── notebooks ├── README.md ├── fig_22_ordinal.ipynb ├── fig_24_tf_idf_from_scratch.ipynb ├── fig_29_cos_sim_from_scratch.ipynb ├── fig_2_embeddings_papers.ipynb ├── fig_40_word_2_vec.ipynb ├── fig_4_bert.ipynb └── requirements.txt /.github/workflows/main.yaml: -------------------------------------------------------------------------------- 1 | name: Build LaTeX document 2 | run-name: Deploy ${{ github.event.head_commit.message }} 3 | to main by @${{ github.actor }} 4 | on: 5 | push: 6 | branches: 7 | - main 8 | paths: 9 | - embeddings.bib 10 | - embeddings.tex 11 | pull_request: 12 | branches: 13 | - main 14 | 15 | 16 | jobs: 17 | build_latex: 18 | runs-on: ubuntu-latest 19 | steps: 20 | - name: Set up Git repository 21 | uses: actions/checkout@v3 22 | - name: Compile LaTeX document 23 | uses: xu-cheng/latex-action@v3 24 | with: 25 | root_file: embeddings.tex 26 | latexmk_shell_escape: true 27 | continue_on_error: true 28 | docker_image: ghcr.io/xu-cheng/texlive-full:20230701 29 | - name: Upload PDF file 30 | uses: actions/upload-artifact@v3 31 | with: 32 | name: PDF 33 | path: embeddings.pdf 34 | - name: Commit changes 35 | run: | 36 | git config --local user.email "vicki.boykis@gmail.com" 37 | git config --local user.name "Vicki Boykis" 38 | git fetch origin main 39 | git stash 40 | git rebase origin/main # keep all other changes 41 | git stash pop 42 | git add embeddings.pdf 43 | git commit -m "Regenerate PDF from ${{ github.sha }}" 44 | git push --force # overwrite old PDF 45 | if: github.event_name != 'pull_request' 46 | -------------------------------------------------------------------------------- /.github/workflows/static.yml: -------------------------------------------------------------------------------- 1 | # Simple workflow for deploying static content to GitHub Pages 2 | name: Deploy static content to Pages 3 | 4 | on: 5 | # Runs on pushes targeting the default branch 6 | push: 7 | branches: ["site"] 8 | 9 | # Allows you to run this workflow manually from the Actions tab 10 | workflow_dispatch: 11 | 12 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 13 | permissions: 14 | contents: read 15 | pages: write 16 | id-token: write 17 | 18 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. 19 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. 20 | concurrency: 21 | group: "pages" 22 | cancel-in-progress: false 23 | 24 | jobs: 25 | # Single deploy job since we're just deploying 26 | deploy: 27 | environment: 28 | name: github-pages 29 | url: ${{ steps.deployment.outputs.page_url }} 30 | runs-on: ubuntu-latest 31 | steps: 32 | - name: Checkout 33 | uses: actions/checkout@v3 34 | - name: Setup Pages 35 | uses: actions/configure-pages@v3 36 | - name: Upload artifact 37 | uses: actions/upload-pages-artifact@v1 38 | with: 39 | # Upload entire repository 40 | path: docs 41 | - name: Deploy to GitHub Pages 42 | id: deployment 43 | uses: actions/deploy-pages@v2 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Core latex/pdflatex auxiliary files: 2 | *.aux 3 | *.lof 4 | *.log 5 | *.lot 6 | *.fls 7 | *.out 8 | *.toc 9 | *.fmt 10 | *.fot 11 | *.cb 12 | *.cb2 13 | .*.lb 14 | *.gz 15 | 16 | ## Intermediate documents: 17 | *.dvi 18 | *.xdv 19 | *-converted-to.* 20 | 21 | .DS_Store 22 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | message: "If you use this software, please cite it as below." 3 | authors: 4 | - family-names: "Boykis" 5 | given-names: "Vicki" 6 | title: "What are embeddings?" 7 | version: 1.0.1 8 | doi: 10.5281/zenodo.8015029 9 | date-released: 2023-06-08 10 | url: "https://github.com/veekaybee/what_are_embeddings" 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 |

3 | 4 |

5 | 6 | 7 | # What are embeddings? 8 | This repository contains the generated LaTex document, website, and complementary notebook code for 9 | ["What are Embeddings".](https://vickiboykis.com/what_are_embeddings/) 10 | 11 | [![DOI](https://zenodo.org/badge/644343479.svg)](https://zenodo.org/badge/latestdoi/644343479) 12 | 13 | ## Abstract 14 | 15 | Over the past decade, embeddings --- numerical representations of non-tabular machine learning features used as input to deep learning models --- have become a foundational data structure in industrial machine learning systems. TF-IDF, PCA, and one-hot encoding have always been key tools in machine learning systems as ways to compress and make sense of large amounts of textual data. However, traditional approaches were limited in the amount of context they could reason about with increasing amounts of data. As the volume, velocity, and variety of data captured by modern applications has exploded, creating approaches specifically tailored to scale has become increasingly important. 16 | 17 | [Google's Word2Vec paper](https://arxiv.org/abs/1301.3781) made an important step in moving from simple statistical representations to semantic meaning of words. The subsequent rise of the [Transformer architecture](https://arxiv.org/abs/1706.03762) and transfer learning, as well as the latest surge in generative methods has enabled the growth of embeddings as a foundational machine learning data structure. This survey paper aims to provide a deep dive into what embeddings are, their history, and usage patterns in industry. 18 | 19 | ## Running 20 | 21 | The [LaTex document](https://github.com/veekaybee/what_are_embeddings/blob/main/.github/workflows/main.yaml) is written in Overleaf and deployed to GitHub, where it's compiled via Actions. The site is likewise generated via Actions from the `site` branch. The notebooks are flying fast and free and not under any kind of CI whatsoever. 22 | 23 | ## Contributing 24 | 25 | If you have any changes that you'd like to make to the document including clarification or typo fixes, you'll need to build the LaTeX artifact. I use GitHub to track issues and feature requests, as well as accept pull requests. Pull requests are the best way to propose changes to the codebase: 26 | 27 | 1. Fork the repo and create your branch from `main`. 28 | 2. Make your changes in your fork. 29 | 3. Make sure that your LaTeX document compiles. The GH action that triggers the PDF is set to run on PR into main. 30 | 4. Ensure that the document compiles to a PDF correctly and inspect the output. 31 | 5. Make sure your code lints. 32 | 6. Issue that pull request! 33 | 34 | 35 | ## Citing 36 | 37 | ```bibtex 38 | @software{Boykis_What_are_embeddings_2023, 39 | author = {Boykis, Vicki}, 40 | doi = {10.5281/zenodo.8015029}, 41 | month = jun, 42 | title = {{What are embeddings?}}, 43 | url = {https://github.com/veekaybee/what_are_embeddings}, 44 | version = {1.0.1}, 45 | year = {2023} 46 | } 47 | ``` 48 | -------------------------------------------------------------------------------- /_minted-main/0B400837BE07578DE366DCD410AD7E392877B981C94AD6CEAF0075A5318FF017.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{k+kn}{from} \PYG{n+nn}{sklearn.preprocessing} \PYG{k+kn}{import} \PYG{n}{OrdinalEncoder} 3 | \PYG{n+nb}{print}\PYG{p}{(}\PYG{n}{data}\PYG{p}{)} 4 | \PYG{p}{[[}\PYG{l+s+s1}{\PYGZsq{}US\PYGZsq{}}\PYG{p}{]} 5 | \PYG{p}{[}\PYG{l+s+s1}{\PYGZsq{}UK\PYGZsq{}}\PYG{p}{]} 6 | \PYG{p}{[}\PYG{l+s+s1}{\PYGZsq{}NZ\PYGZsq{}}\PYG{p}{]]} 7 | 8 | \PYG{c+c1}{\PYGZsh{} our label features} 9 | \PYG{n}{encoder} \PYG{o}{=} \PYG{n}{OrdinalEncoder}\PYG{p}{()} 10 | \PYG{n}{result} \PYG{o}{=} \PYG{n}{encoder}\PYG{o}{.}\PYG{n}{fit\PYGZus{}transform}\PYG{p}{(}\PYG{n}{data}\PYG{p}{)} 11 | \PYG{n+nb}{print}\PYG{p}{(}\PYG{n}{result}\PYG{p}{)} 12 | \PYG{p}{[[}\PYG{l+m+mf}{2.}\PYG{p}{]} 13 | \PYG{p}{[}\PYG{l+m+mf}{1.}\PYG{p}{]} 14 | \PYG{p}{[}\PYG{l+m+mf}{0.}\PYG{p}{]]} 15 | \end{Verbatim} 16 | -------------------------------------------------------------------------------- /_minted-main/0C985E5C55D91891120F152CE09600D75018845A16FEE2F871CFA2D2B2CFB1C6.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{n}{bird\PYGZus{}likes} 3 | \end{Verbatim} 4 | -------------------------------------------------------------------------------- /_minted-main/32D5CF32604E0D919E7B4A991777294A2877B981C94AD6CEAF0075A5318FF017.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{n}{In} \PYG{p}{[}\PYG{l+m+mi}{11}\PYG{p}{]:} \PYG{n}{data} \PYG{o}{=} \PYG{n}{np}\PYG{o}{.}\PYG{n}{asarray}\PYG{p}{([[}\PYG{l+s+s2}{\PYGZdq{}Hold fast to dreams, for if dreams die, life is a} 3 | \PYG{o}{...}\PYG{p}{:} \PYG{n}{broken}\PYG{o}{\PYGZhy{}}\PYG{n}{winged} \PYG{n}{bird} \PYG{n}{that} \PYG{n}{cannot} \PYG{n}{fly}\PYG{o}{.}\PYG{l+s+s2}{\PYGZdq{},} 4 | \PYG{o}{...}\PYG{p}{:} \PYG{l+s+s2}{\PYGZdq{}No bird soars too high if he soars with his own wings.\PYGZdq{}}\PYG{p}{,} 5 | \PYG{o}{...}\PYG{p}{:} \PYG{l+s+s2}{\PYGZdq{}A bird does not sing because it has an answer, it sings because it has} 6 | \PYG{o}{...}\PYG{p}{:} \PYG{n}{a} \PYG{n}{song}\PYG{o}{.}\PYG{l+s+s2}{\PYGZdq{}]]} 7 | \PYG{o}{...}\PYG{p}{:} \PYG{p}{)} 8 | 9 | \PYG{n}{In} \PYG{p}{[}\PYG{l+m+mi}{12}\PYG{p}{]:} \PYG{n}{enc}\PYG{o}{.}\PYG{n}{fit}\PYG{p}{(}\PYG{n}{data}\PYG{p}{)} 10 | \PYG{n}{Out}\PYG{p}{[}\PYG{l+m+mi}{12}\PYG{p}{]:} \PYG{n}{OneHotEncoder}\PYG{p}{(}\PYG{n}{handle\PYGZus{}unknown}\PYG{o}{=}\PYG{l+s+s1}{\PYGZsq{}ignore\PYGZsq{}}\PYG{p}{)} 11 | 12 | \PYG{n}{In} \PYG{p}{[}\PYG{l+m+mi}{13}\PYG{p}{]:} \PYG{n}{onehotlabels} \PYG{o}{=} \PYG{n}{enc}\PYG{o}{.}\PYG{n}{transform}\PYG{p}{(}\PYG{n}{data}\PYG{p}{)}\PYG{o}{.}\PYG{n}{toarray}\PYG{p}{()} 13 | 14 | \PYG{n}{In} \PYG{p}{[}\PYG{l+m+mi}{14}\PYG{p}{]:} \PYG{n}{onehotlabels} 15 | \PYG{n}{Out}\PYG{p}{[}\PYG{l+m+mi}{14}\PYG{p}{]:} \PYG{n}{array}\PYG{p}{([[}\PYG{l+m+mf}{1.}\PYG{p}{,} \PYG{l+m+mf}{1.}\PYG{p}{,} \PYG{l+m+mf}{1.}\PYG{p}{]])} 16 | \end{Verbatim} 17 | -------------------------------------------------------------------------------- /_minted-main/38851A735B84BFD7FC57CBEDC1A8C54C5018845A16FEE2F871CFA2D2B2CFB1C6.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{k+kn}{from} \PYG{n+nn}{sklearn.feature\PYGZus{}extraction.text} \PYG{k+kn}{import} \PYG{n}{TfidfVectorizer} 3 | \PYG{n}{corpus} \PYG{o}{=} \PYG{p}{[}\PYG{l+s+s2}{\PYGZdq{}Hold fast to dreams, for if dreams die, life is a broken\PYGZhy{}winged bird that cannot fly.\PYGZdq{}}\PYG{p}{,} 4 | \PYG{l+s+s2}{\PYGZdq{}No bird soars too high if he soars with his own wings.\PYGZdq{}}\PYG{p}{,} 5 | \PYG{l+s+s2}{\PYGZdq{}A bird does not sing because it has an answer, it sings because it has a song.\PYGZdq{}}\PYG{p}{]} 6 | 7 | \PYG{n}{vectorizer} \PYG{o}{=} \PYG{n}{TfidfVectorizer}\PYG{p}{()} 8 | \PYG{n}{X} \PYG{o}{=} \PYG{n}{vectorizer}\PYG{o}{.}\PYG{n}{fit\PYGZus{}transform}\PYG{p}{(}\PYG{n}{corpus}\PYG{p}{)} 9 | \PYG{n+nb}{dict}\PYG{p}{(}\PYG{n+nb}{zip}\PYG{p}{(}\PYG{n}{vectorizer}\PYG{o}{.}\PYG{n}{get\PYGZus{}feature\PYGZus{}name\PYGZus{}outs}\PYG{p}{(),} \PYG{n}{X}\PYG{o}{.}\PYG{n}{toarray}\PYG{p}{()[}\PYG{l+m+mi}{0}\PYG{p}{]))} 10 | 11 | \PYG{p}{\PYGZob{}}\PYG{l+s+s1}{\PYGZsq{}an\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{,} 12 | \PYG{l+s+s1}{\PYGZsq{}answer\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{,} 13 | \PYG{l+s+s1}{\PYGZsq{}because\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{,} 14 | \PYG{l+s+s1}{\PYGZsq{}bird\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.14355303576663192}\PYG{p}{,} 15 | \PYG{l+s+s1}{\PYGZsq{}broken\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.24305641776909384}\PYG{p}{,} 16 | \PYG{l+s+s1}{\PYGZsq{}cannot\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.24305641776909384}\PYG{p}{,} 17 | \PYG{l+s+s1}{\PYGZsq{}die\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.24305641776909384}\PYG{p}{,} 18 | \PYG{l+s+s1}{\PYGZsq{}does\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{,} 19 | \PYG{l+s+s1}{\PYGZsq{}dreams\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.4861128355381877}\PYG{p}{,} 20 | \PYG{l+s+s1}{\PYGZsq{}fast\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.24305641776909384}\PYG{p}{,} 21 | \PYG{l+s+s1}{\PYGZsq{}fly\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.24305641776909384}\PYG{p}{,} 22 | \PYG{l+s+s1}{\PYGZsq{}for\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.24305641776909384}\PYG{p}{,} 23 | \PYG{l+s+s1}{\PYGZsq{}has\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{,} 24 | \PYG{l+s+s1}{\PYGZsq{}he\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{,} 25 | \PYG{l+s+s1}{\PYGZsq{}high\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{,} 26 | \PYG{l+s+s1}{\PYGZsq{}his\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{,} 27 | \PYG{l+s+s1}{\PYGZsq{}hold\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.24305641776909384}\PYG{p}{,} 28 | \PYG{l+s+s1}{\PYGZsq{}if\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.1848506706027212}\PYG{p}{,} 29 | \PYG{l+s+s1}{\PYGZsq{}is\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.24305641776909384}\PYG{p}{,} 30 | \PYG{l+s+s1}{\PYGZsq{}it\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{,} 31 | \PYG{l+s+s1}{\PYGZsq{}life\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.24305641776909384}\PYG{p}{,} 32 | \PYG{l+s+s1}{\PYGZsq{}no\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{,} 33 | \PYG{l+s+s1}{\PYGZsq{}not\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{,} 34 | \PYG{l+s+s1}{\PYGZsq{}own\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{,} 35 | \PYG{l+s+s1}{\PYGZsq{}sing\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{,} 36 | \PYG{l+s+s1}{\PYGZsq{}sings\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{,} 37 | \PYG{l+s+s1}{\PYGZsq{}soars\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{,} 38 | \PYG{l+s+s1}{\PYGZsq{}song\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{,} 39 | \PYG{l+s+s1}{\PYGZsq{}that\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.24305641776909384}\PYG{p}{,} 40 | \PYG{l+s+s1}{\PYGZsq{}to\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.24305641776909384}\PYG{p}{,} 41 | \PYG{l+s+s1}{\PYGZsq{}too\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{,} 42 | \PYG{l+s+s1}{\PYGZsq{}winged\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.24305641776909384}\PYG{p}{,} 43 | \PYG{l+s+s1}{\PYGZsq{}wings\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{,} 44 | \PYG{l+s+s1}{\PYGZsq{}with\PYGZsq{}}\PYG{p}{:} \PYG{l+m+mf}{0.0}\PYG{p}{\PYGZcb{}} 45 | 46 | \end{Verbatim} 47 | -------------------------------------------------------------------------------- /_minted-main/3C3C80F0A97EC51AA116D2F587016AE85018845A16FEE2F871CFA2D2B2CFB1C6.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{k+kn}{from} \PYG{n+nn}{sklearn.feature\PYGZus{}extraction.text} \PYG{k+kn}{import} \PYG{n}{CountVectorizer} 3 | \PYG{k+kn}{import} \PYG{n+nn}{pandas} \PYG{k}{as} \PYG{n+nn}{pd} 4 | 5 | \PYG{n}{vect} \PYG{o}{=} \PYG{n}{CountVectorizer}\PYG{p}{()} 6 | 7 | \PYG{n}{responses} \PYG{o}{=} \PYG{p}{[}\PYG{l+s+s2}{\PYGZdq{}Hold fast to dreams, for if dreams die, }\PYG{l+s+se}{\PYGZbs{}} 8 | \PYG{l+s+s2}{life is a broken\PYGZhy{}winged bird that cannot fly.\PYGZdq{}}\PYG{p}{,} 9 | \PYG{l+s+s2}{\PYGZdq{}No bird soars too high if he soars with his own wings.\PYGZdq{}}\PYG{p}{,} 10 | \PYG{l+s+s2}{\PYGZdq{}A bird does not sing because it has an answer, it sings because it has a song.\PYGZdq{}}\PYG{p}{]} 11 | 12 | \PYG{n}{doc} \PYG{o}{=} \PYG{n}{pd}\PYG{o}{.}\PYG{n}{DataFrame}\PYG{p}{(}\PYG{n+nb}{list}\PYG{p}{(}\PYG{n+nb}{zip}\PYG{p}{(}\PYG{n}{responses}\PYG{p}{)))} 13 | 14 | \PYG{n}{td} \PYG{o}{=} \PYG{n}{pd}\PYG{o}{.}\PYG{n}{DataFrame}\PYG{p}{(}\PYG{n}{vects}\PYG{o}{.}\PYG{n}{todense}\PYG{p}{())}\PYG{o}{.}\PYG{n}{iloc}\PYG{p}{[:}\PYG{l+m+mi}{5}\PYG{p}{]} 15 | \PYG{n}{td}\PYG{o}{.}\PYG{n}{columns} \PYG{o}{=} \PYG{n}{vect}\PYG{o}{.}\PYG{n}{get\PYGZus{}feature\PYGZus{}names\PYGZus{}out}\PYG{p}{()} 16 | \PYG{n}{term\PYGZus{}document\PYGZus{}matrix} \PYG{o}{=} \PYG{n}{td}\PYG{o}{.}\PYG{n}{T} 17 | \PYG{n}{term\PYGZus{}document\PYGZus{}matrix}\PYG{o}{.}\PYG{n}{columns} \PYG{o}{=} \PYG{p}{[}\PYG{l+s+s1}{\PYGZsq{}Doc \PYGZsq{}}\PYG{o}{+}\PYG{n+nb}{str}\PYG{p}{(}\PYG{n}{i}\PYG{p}{)} \PYG{k}{for} \PYG{n}{i} \PYG{o+ow}{in} \PYG{n+nb}{range}\PYG{p}{(}\PYG{l+m+mi}{1}\PYG{p}{,} \PYG{l+m+mi}{4}\PYG{p}{)]} 18 | \PYG{n}{term\PYGZus{}document\PYGZus{}matrix}\PYG{p}{[}\PYG{l+s+s1}{\PYGZsq{}total\PYGZus{}count\PYGZsq{}}\PYG{p}{]} \PYG{o}{=} \PYG{n}{term\PYGZus{}document\PYGZus{}matrix}\PYG{o}{.}\PYG{n}{sum}\PYG{p}{(}\PYG{n}{axis}\PYG{o}{=}\PYG{l+m+mi}{1}\PYG{p}{)} 19 | 20 | \PYG{n+nb}{print}\PYG{p}{(}\PYG{n}{term\PYGZus{}document\PYGZus{}matrix}\PYG{o}{.}\PYG{n}{drop}\PYG{p}{(}\PYG{n}{columns}\PYG{o}{=}\PYG{p}{[}\PYG{l+s+s1}{\PYGZsq{}total\PYGZus{}count\PYGZsq{}}\PYG{p}{])}\PYG{o}{.}\PYG{n}{head}\PYG{p}{(}\PYG{l+m+mi}{10}\PYG{p}{))} 21 | 22 | \PYG{n}{Doc} \PYG{l+m+mi}{1} \PYG{n}{Doc} \PYG{l+m+mi}{2} \PYG{n}{Doc} \PYG{l+m+mi}{3} 23 | \PYG{n}{an} \PYG{l+m+mi}{0} \PYG{l+m+mi}{0} \PYG{l+m+mi}{1} 24 | \PYG{n}{answer} \PYG{l+m+mi}{0} \PYG{l+m+mi}{0} \PYG{l+m+mi}{1} 25 | \PYG{n}{because} \PYG{l+m+mi}{0} \PYG{l+m+mi}{0} \PYG{l+m+mi}{2} 26 | \PYG{n}{bird} \PYG{l+m+mi}{1} \PYG{l+m+mi}{1} \PYG{l+m+mi}{1} 27 | \PYG{n}{broken} \PYG{l+m+mi}{1} \PYG{l+m+mi}{0} \PYG{l+m+mi}{0} 28 | \PYG{n}{cannot} \PYG{l+m+mi}{1} \PYG{l+m+mi}{0} \PYG{l+m+mi}{0} 29 | \PYG{n}{die} \PYG{l+m+mi}{1} \PYG{l+m+mi}{0} \PYG{l+m+mi}{0} 30 | \PYG{n}{does} \PYG{l+m+mi}{0} \PYG{l+m+mi}{0} \PYG{l+m+mi}{1} 31 | \PYG{n}{dreams} \PYG{l+m+mi}{2} \PYG{l+m+mi}{0} \PYG{l+m+mi}{0} 32 | \PYG{n}{fast} \PYG{l+m+mi}{1} \PYG{l+m+mi}{0} \PYG{l+m+mi}{0} 33 | \end{Verbatim} 34 | -------------------------------------------------------------------------------- /_minted-main/50F353AB414E901AB3F8253F641CCD3D5018845A16FEE2F871CFA2D2B2CFB1C6.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{k+kn}{from} \PYG{n+nn}{sklearn.preprocessing} \PYG{k+kn}{import} \PYG{n}{OneHotEncoder} 3 | \PYG{k+kn}{import} \PYG{n+nn}{numpy} \PYG{k}{as} \PYG{n+nn}{np} 4 | 5 | \PYG{n}{enc} \PYG{o}{=} \PYG{n}{OneHotEncoder}\PYG{p}{(}\PYG{n}{handle\PYGZus{}unknown}\PYG{o}{=}\PYG{l+s+s1}{\PYGZsq{}ignore\PYGZsq{}}\PYG{p}{)} 6 | \PYG{n}{data} \PYG{o}{=} \PYG{n}{np}\PYG{o}{.}\PYG{n}{asarray}\PYG{p}{([[}\PYG{l+s+s1}{\PYGZsq{}US\PYGZsq{}}\PYG{p}{],} \PYG{p}{[}\PYG{l+s+s1}{\PYGZsq{}UK\PYGZsq{}}\PYG{p}{],} \PYG{p}{[}\PYG{l+s+s1}{\PYGZsq{}NZ\PYGZsq{}}\PYG{p}{]])} 7 | \PYG{n}{enc}\PYG{o}{.}\PYG{n}{fit}\PYG{p}{(}\PYG{n}{data}\PYG{p}{)} 8 | \PYG{n}{enc}\PYG{o}{.}\PYG{n}{categories\PYGZus{}} 9 | \PYG{c+c1}{\PYGZsh{} Result: [array([\PYGZsq{}NZ\PYGZsq{}, \PYGZsq{}UK\PYGZsq{}, \PYGZsq{}US\PYGZsq{}], dtype=\PYGZsq{}\PYGZlt{}U2\PYGZsq{})]} 10 | \PYG{n}{onehotlabels} \PYG{o}{=} \PYG{n}{enc}\PYG{o}{.}\PYG{n}{transform}\PYG{p}{(}\PYG{n}{data}\PYG{p}{)}\PYG{o}{.}\PYG{n}{toarray}\PYG{p}{()} 11 | \PYG{n}{onehotlabels} 12 | \PYG{c+c1}{\PYGZsh{} Result:} 13 | \PYG{n}{array}\PYG{p}{([[}\PYG{l+m+mf}{0.}\PYG{p}{,} \PYG{l+m+mf}{0.}\PYG{p}{,} \PYG{l+m+mf}{1.}\PYG{p}{],} 14 | \PYG{p}{[}\PYG{l+m+mf}{0.}\PYG{p}{,} \PYG{l+m+mf}{1.}\PYG{p}{,} \PYG{l+m+mf}{0.}\PYG{p}{],} 15 | \PYG{p}{[}\PYG{l+m+mf}{1.}\PYG{p}{,} \PYG{l+m+mf}{0.}\PYG{p}{,} \PYG{l+m+mf}{0.}\PYG{p}{]])} 16 | \end{Verbatim} 17 | -------------------------------------------------------------------------------- /_minted-main/614143CE898A10CDD926BB56864F00B85018845A16FEE2F871CFA2D2B2CFB1C6.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{n}{v1} \PYG{o}{=} \PYG{p}{[}\PYG{l+m+mi}{0}\PYG{p}{,}\PYG{l+m+mi}{3}\PYG{p}{,}\PYG{l+m+mi}{4}\PYG{p}{,}\PYG{l+m+mi}{5}\PYG{p}{,}\PYG{l+m+mi}{6}\PYG{p}{]} 3 | \PYG{n}{v2} \PYG{o}{=} \PYG{p}{[}\PYG{l+m+mi}{4}\PYG{p}{,}\PYG{l+m+mi}{5}\PYG{p}{,}\PYG{l+m+mi}{6}\PYG{p}{,}\PYG{l+m+mi}{7}\PYG{p}{,}\PYG{l+m+mi}{8}\PYG{p}{]} 4 | 5 | \PYG{k}{def} \PYG{n+nf}{dot}\PYG{p}{(}\PYG{n}{v1}\PYG{p}{,} \PYG{n}{v2}\PYG{p}{):} 6 | \PYG{n}{dot\PYGZus{}product} \PYG{o}{=} \PYG{n+nb}{sum}\PYG{p}{((}\PYG{n}{a} \PYG{o}{*} \PYG{n}{b}\PYG{p}{)} \PYG{k}{for} \PYG{n}{a}\PYG{p}{,}\PYG{n}{b} \PYG{o+ow}{in} \PYG{n+nb}{zip}\PYG{p}{(}\PYG{n}{v1}\PYG{p}{,}\PYG{n}{v2}\PYG{p}{))} 7 | \PYG{k}{return} \PYG{n}{dot\PYGZus{}product} 8 | 9 | \PYG{k}{def} \PYG{n+nf}{cosine\PYGZus{}similarity}\PYG{p}{(}\PYG{n}{v1}\PYG{p}{,} \PYG{n}{v2}\PYG{p}{):} 10 | \PYG{l+s+sd}{\PYGZsq{}\PYGZsq{}\PYGZsq{}} 11 | \PYG{l+s+sd}{ (v1 dot v2)/||v1|| *||v2||)} 12 | \PYG{l+s+sd}{ \PYGZsq{}\PYGZsq{}\PYGZsq{}} 13 | \PYG{n}{products} \PYG{o}{=} \PYG{n}{dot}\PYG{p}{(}\PYG{n}{v1}\PYG{p}{,}\PYG{n}{v2}\PYG{p}{)} 14 | \PYG{n}{denominator} \PYG{o}{=} \PYG{p}{(} \PYG{p}{(}\PYG{n}{dot}\PYG{p}{(}\PYG{n}{v1}\PYG{p}{,}\PYG{n}{v1}\PYG{p}{)} \PYG{o}{**}\PYG{l+m+mf}{.5}\PYG{p}{)} \PYG{o}{*} \PYG{p}{(}\PYG{n}{dot}\PYG{p}{(}\PYG{n}{v2}\PYG{p}{,}\PYG{n}{v2}\PYG{p}{)} \PYG{o}{**} \PYG{l+m+mf}{.5}\PYG{p}{)} \PYG{p}{)} 15 | \PYG{n}{similarity} \PYG{o}{=} \PYG{n}{products} \PYG{o}{/} \PYG{n}{denominator} 16 | \PYG{k}{return} \PYG{n}{similarity} 17 | 18 | \PYG{n+nb}{print}\PYG{p}{(}\PYG{n}{cosine\PYGZus{}similarity}\PYG{p}{(}\PYG{n}{v1}\PYG{p}{,} \PYG{n}{v2}\PYG{p}{))} 19 | \PYG{c+c1}{\PYGZsh{} 0.9544074144996451} 20 | \end{Verbatim} 21 | -------------------------------------------------------------------------------- /_minted-main/6E7EEB2C370D1F397DEAB0E58E0EDB462877B981C94AD6CEAF0075A5318FF017.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{k}{class} \PYG{n+nc}{CBOW}\PYG{p}{(}\PYG{n}{torch}\PYG{o}{.}\PYG{n}{nn}\PYG{o}{.}\PYG{n}{Module}\PYG{p}{):} 3 | \PYG{k}{def} \PYG{n+nf+fm}{\PYGZus{}\PYGZus{}init\PYGZus{}\PYGZus{}}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{p}{):} \PYG{c+c1}{\PYGZsh{} we pass in vocab\PYGZus{}size and embedding\PYGZus{}dim as hyperparams} 4 | \PYG{n+nb}{super}\PYG{p}{(}\PYG{n}{CBOW}\PYG{p}{,} \PYG{n+nb+bp}{self}\PYG{p}{)}\PYG{o}{.}\PYG{n+nf+fm}{\PYGZus{}\PYGZus{}init\PYGZus{}\PYGZus{}}\PYG{p}{()} 5 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{num\PYGZus{}epochs} \PYG{o}{=} \PYG{l+m+mi}{3} 6 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{context\PYGZus{}size} \PYG{o}{=} \PYG{l+m+mi}{2} \PYG{c+c1}{\PYGZsh{} 2 words to the left, 2 words to the right} 7 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{embedding\PYGZus{}dim} \PYG{o}{=} \PYG{l+m+mi}{100} \PYG{c+c1}{\PYGZsh{} Size of your embedding vector} 8 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{learning\PYGZus{}rate} \PYG{o}{=} \PYG{l+m+mf}{0.001} 9 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{device} \PYG{o}{=} \PYG{n}{torch}\PYG{o}{.}\PYG{n}{device}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}cuda\PYGZsq{}} \PYG{k}{if} \PYG{n}{torch}\PYG{o}{.}\PYG{n}{cuda}\PYG{o}{.}\PYG{n}{is\PYGZus{}available}\PYG{p}{()} \PYG{k}{else} \PYG{l+s+s1}{\PYGZsq{}cpu\PYGZsq{}}\PYG{p}{)} 10 | 11 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{vocab} \PYG{o}{=} \PYG{n}{TextPreProcessor}\PYG{p}{()}\PYG{o}{.}\PYG{n}{build\PYGZus{}vocab}\PYG{p}{()} 12 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{word\PYGZus{}to\PYGZus{}ix} \PYG{o}{=} \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{vocab}\PYG{o}{.}\PYG{n}{get\PYGZus{}stoi}\PYG{p}{()} 13 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{ix\PYGZus{}to\PYGZus{}word} \PYG{o}{=} \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{vocab}\PYG{o}{.}\PYG{n}{get\PYGZus{}itos}\PYG{p}{()} 14 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{vocab\PYGZus{}list} \PYG{o}{=} \PYG{n+nb}{list}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{vocab}\PYG{o}{.}\PYG{n}{get\PYGZus{}stoi}\PYG{p}{()}\PYG{o}{.}\PYG{n}{keys}\PYG{p}{())} 15 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{vocab\PYGZus{}size} \PYG{o}{=} \PYG{n+nb}{len}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{vocab}\PYG{p}{)} 16 | 17 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{model} \PYG{o}{=} \PYG{k+kc}{None} 18 | 19 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{model\PYGZus{}path} \PYG{o}{=} \PYG{l+s+s1}{\PYGZsq{}model.ckpt\PYGZsq{}} 20 | 21 | \PYG{c+c1}{\PYGZsh{} out: 1 x embedding\PYGZus{}dim} 22 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{embeddings} \PYG{o}{=} \PYG{n}{nn}\PYG{o}{.}\PYG{n}{Embedding}\PYG{p}{(} 23 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{vocab\PYGZus{}size}\PYG{p}{,} \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{embedding\PYGZus{}dim} 24 | \PYG{p}{)} \PYG{c+c1}{\PYGZsh{} initialize an Embedding matrix based on our inputs} 25 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{linear1} \PYG{o}{=} \PYG{n}{nn}\PYG{o}{.}\PYG{n}{Linear}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{embedding\PYGZus{}dim}\PYG{p}{,} \PYG{l+m+mi}{128}\PYG{p}{)} 26 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{activation\PYGZus{}function1} \PYG{o}{=} \PYG{n}{nn}\PYG{o}{.}\PYG{n}{ReLU}\PYG{p}{()} 27 | 28 | \PYG{c+c1}{\PYGZsh{} out: 1 x vocab\PYGZus{}size} 29 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{linear2} \PYG{o}{=} \PYG{n}{nn}\PYG{o}{.}\PYG{n}{Linear}\PYG{p}{(}\PYG{l+m+mi}{128}\PYG{p}{,} \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{vocab\PYGZus{}size}\PYG{p}{)} 30 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{activation\PYGZus{}function2} \PYG{o}{=} \PYG{n}{nn}\PYG{o}{.}\PYG{n}{LogSoftmax}\PYG{p}{(}\PYG{n}{dim}\PYG{o}{=\PYGZhy{}}\PYG{l+m+mi}{1}\PYG{p}{)} 31 | \end{Verbatim} 32 | -------------------------------------------------------------------------------- /_minted-main/7195AF5D847B6BCCB7A150D19F7ABA625018845A16FEE2F871CFA2D2B2CFB1C6.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{k+kn}{from} \PYG{n+nn}{sklearn.metrics} \PYG{k+kn}{import} \PYG{n}{pairwise} 3 | 4 | \PYG{n}{v1} \PYG{o}{=} \PYG{p}{[}\PYG{l+m+mi}{0}\PYG{p}{,}\PYG{l+m+mi}{3}\PYG{p}{,}\PYG{l+m+mi}{4}\PYG{p}{,}\PYG{l+m+mi}{5}\PYG{p}{,}\PYG{l+m+mi}{6}\PYG{p}{]} 5 | \PYG{n}{v2} \PYG{o}{=} \PYG{p}{[}\PYG{l+m+mi}{4}\PYG{p}{,}\PYG{l+m+mi}{5}\PYG{p}{,}\PYG{l+m+mi}{6}\PYG{p}{,}\PYG{l+m+mi}{7}\PYG{p}{,}\PYG{l+m+mi}{8}\PYG{p}{]} 6 | 7 | \PYG{c+c1}{\PYGZsh{} need to be in numpy data format} 8 | \PYG{n}{pairwise}\PYG{o}{.}\PYG{n}{cosine\PYGZus{}similarity}\PYG{p}{([}\PYG{n}{v1}\PYG{p}{],[}\PYG{n}{v2}\PYG{p}{])} 9 | \PYG{c+c1}{\PYGZsh{} array([[0.95440741]])} 10 | 11 | \end{Verbatim} 12 | -------------------------------------------------------------------------------- /_minted-main/7D831596D43890A2BADD43064A830E6E5018845A16FEE2F871CFA2D2B2CFB1C6.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{n}{bird\PYGZus{}posts} 3 | \end{Verbatim} 4 | -------------------------------------------------------------------------------- /_minted-main/824C4087A6F8A7A7A1B282559BD58B455018845A16FEE2F871CFA2D2B2CFB1C6.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{k+kn}{from} \PYG{n+nn}{sklearn.feature\PYGZus{}extraction.text} \PYG{k+kn}{import} \PYG{n}{CountVectorizer} 3 | \PYG{k+kn}{import} \PYG{n+nn}{pandas} \PYG{k}{as} \PYG{n+nn}{pd} 4 | \PYG{n}{vect} \PYG{o}{=} \PYG{n}{CountVectorizer}\PYG{p}{()} 5 | 6 | \PYG{n}{responses} \PYG{o}{=} \PYG{p}{[}\PYG{l+s+s2}{\PYGZdq{}Hold fast to dreams, for if dreams die, life is a broken\PYGZhy{}winged bird that cannot fly.\PYGZdq{}}\PYG{p}{,} \PYG{l+s+s2}{\PYGZdq{}No bird soars too high if he soars with his own wings.\PYGZdq{}}\PYG{p}{,} 7 | \PYG{l+s+s2}{\PYGZdq{}A bird does not sing because it has an answer, it sings because it has a song.\PYGZdq{}}\PYG{p}{]} 8 | 9 | \PYG{n}{doc} \PYG{o}{=} \PYG{n}{pd}\PYG{o}{.}\PYG{n}{DataFrame}\PYG{p}{(}\PYG{n+nb}{list}\PYG{p}{(}\PYG{n+nb}{zip}\PYG{p}{(}\PYG{n}{responses}\PYG{p}{)))} 10 | 11 | \PYG{n}{td} \PYG{o}{=} \PYG{n}{pd}\PYG{o}{.}\PYG{n}{DataFrame}\PYG{p}{(}\PYG{n}{vects}\PYG{o}{.}\PYG{n}{todense}\PYG{p}{())}\PYG{o}{.}\PYG{n}{iloc}\PYG{p}{[:}\PYG{l+m+mi}{5}\PYG{p}{]} 12 | \PYG{n}{td}\PYG{o}{.}\PYG{n}{columns} \PYG{o}{=} \PYG{n}{vect}\PYG{o}{.}\PYG{n}{get\PYGZus{}feature\PYGZus{}names\PYGZus{}out}\PYG{p}{()} 13 | \PYG{n}{term\PYGZus{}document\PYGZus{}matrix} \PYG{o}{=} \PYG{n}{td}\PYG{o}{.}\PYG{n}{T} 14 | \PYG{n}{term\PYGZus{}document\PYGZus{}matrix}\PYG{o}{.}\PYG{n}{columns} \PYG{o}{=} \PYG{p}{[}\PYG{l+s+s1}{\PYGZsq{}Doc \PYGZsq{}}\PYG{o}{+}\PYG{n+nb}{str}\PYG{p}{(}\PYG{n}{i}\PYG{p}{)} \PYG{k}{for} \PYG{n}{i} \PYG{o+ow}{in} \PYG{n+nb}{range}\PYG{p}{(}\PYG{l+m+mi}{1}\PYG{p}{,} \PYG{l+m+mi}{4}\PYG{p}{)]} 15 | \PYG{n}{term\PYGZus{}document\PYGZus{}matrix}\PYG{p}{[}\PYG{l+s+s1}{\PYGZsq{}total\PYGZus{}count\PYGZsq{}}\PYG{p}{]} \PYG{o}{=} \PYG{n}{term\PYGZus{}document\PYGZus{}matrix}\PYG{o}{.}\PYG{n}{sum}\PYG{p}{(}\PYG{n}{axis}\PYG{o}{=}\PYG{l+m+mi}{1}\PYG{p}{)} 16 | 17 | \PYG{n+nb}{print}\PYG{p}{(}\PYG{n}{term\PYGZus{}document\PYGZus{}matrix}\PYG{o}{.}\PYG{n}{drop}\PYG{p}{(}\PYG{n}{columns}\PYG{o}{=}\PYG{p}{[}\PYG{l+s+s1}{\PYGZsq{}total\PYGZus{}count\PYGZsq{}}\PYG{p}{])}\PYG{o}{.}\PYG{n}{head}\PYG{p}{(}\PYG{l+m+mi}{10}\PYG{p}{))} 18 | 19 | \PYG{n}{Doc} \PYG{l+m+mi}{1} \PYG{n}{Doc} \PYG{l+m+mi}{2} \PYG{n}{Doc} \PYG{l+m+mi}{3} 20 | \PYG{n}{an} \PYG{l+m+mi}{0} \PYG{l+m+mi}{0} \PYG{l+m+mi}{1} 21 | \PYG{n}{answer} \PYG{l+m+mi}{0} \PYG{l+m+mi}{0} \PYG{l+m+mi}{1} 22 | \PYG{n}{because} \PYG{l+m+mi}{0} \PYG{l+m+mi}{0} \PYG{l+m+mi}{2} 23 | \PYG{n}{bird} \PYG{l+m+mi}{1} \PYG{l+m+mi}{1} \PYG{l+m+mi}{1} 24 | \PYG{n}{broken} \PYG{l+m+mi}{1} \PYG{l+m+mi}{0} \PYG{l+m+mi}{0} 25 | \PYG{n}{cannot} \PYG{l+m+mi}{1} \PYG{l+m+mi}{0} \PYG{l+m+mi}{0} 26 | \PYG{n}{die} \PYG{l+m+mi}{1} \PYG{l+m+mi}{0} \PYG{l+m+mi}{0} 27 | \PYG{n}{does} \PYG{l+m+mi}{0} \PYG{l+m+mi}{0} \PYG{l+m+mi}{1} 28 | \PYG{n}{dreams} \PYG{l+m+mi}{2} \PYG{l+m+mi}{0} \PYG{l+m+mi}{0} 29 | \PYG{n}{fast} \PYG{l+m+mi}{1} \PYG{l+m+mi}{0} \PYG{l+m+mi}{0} 30 | \end{Verbatim} 31 | -------------------------------------------------------------------------------- /_minted-main/8E5087457994186475A82E6630AE60F05018845A16FEE2F871CFA2D2B2CFB1C6.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | 3 | \PYG{k+kn}{import} \PYG{n+nn}{math} 4 | 5 | \PYG{n}{documentA} \PYG{o}{=} \PYG{p}{[}\PYG{l+s+s1}{\PYGZsq{}Hold\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}fast\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}to\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}dreams,\PYGZsq{}}\PYG{p}{,} \PYG{l+s+s1}{\PYGZsq{}for\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}if\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}dreams\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}die,\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}life\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}is\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}a\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}broken\PYGZhy{}winged\PYGZsq{}}\PYG{p}{,} 6 | \PYG{l+s+s1}{\PYGZsq{}bird\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}that\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}cannot\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}fly.\PYGZsq{}}\PYG{p}{]} 7 | \PYG{n}{documentB} \PYG{o}{=} \PYG{p}{[}\PYG{l+s+s1}{\PYGZsq{}No\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}bird\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}soars\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}too\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}high\PYGZsq{}}\PYG{p}{,} \PYG{l+s+s1}{\PYGZsq{}if\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}he\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}soars\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}with\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}his\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}own\PYGZsq{}}\PYG{p}{,}\PYG{l+s+s1}{\PYGZsq{}wings.\PYGZsq{}}\PYG{p}{]} 8 | 9 | \PYG{k}{def} \PYG{n+nf}{tf}\PYG{p}{(}\PYG{n}{term}\PYG{p}{:} \PYG{n+nb}{str}\PYG{p}{,} \PYG{n}{document}\PYG{p}{:}\PYG{n+nb}{list}\PYG{p}{[}\PYG{n+nb}{str}\PYG{p}{])} \PYG{o}{\PYGZhy{}\PYGZgt{}} \PYG{n}{flota}\PYG{p}{:} 10 | \PYG{l+s+sd}{\PYGZsq{}\PYGZsq{}\PYGZsq{}Term frequency of a word in a document over total words in document\PYGZsq{}\PYGZsq{}\PYGZsq{}} 11 | 12 | \PYG{n}{term\PYGZus{}count} \PYG{o}{=} \PYG{l+m+mi}{0} 13 | \PYG{n}{total\PYGZus{}count} \PYG{o}{=} \PYG{l+m+mi}{0} 14 | 15 | \PYG{k}{for} \PYG{n}{word} \PYG{o+ow}{in} \PYG{n}{document}\PYG{p}{:} 16 | \PYG{n}{total\PYGZus{}count} \PYG{o}{+=}\PYG{l+m+mi}{1} 17 | \PYG{k}{if} \PYG{n}{word} \PYG{o}{==} \PYG{n}{term}\PYG{p}{:} 18 | \PYG{n}{term\PYGZus{}count} \PYG{o}{+=} \PYG{l+m+mi}{1} 19 | 20 | \PYG{k}{return} \PYG{p}{(}\PYG{n}{term\PYGZus{}count} \PYG{o}{/} \PYG{n}{total\PYGZus{}count}\PYG{p}{)} 21 | 22 | \PYG{k}{def} \PYG{n+nf}{idf}\PYG{p}{(}\PYG{n}{term}\PYG{p}{:}\PYG{n+nb}{str}\PYG{p}{,} \PYG{n}{doc\PYGZus{}list}\PYG{p}{:}\PYG{n+nb}{list}\PYG{p}{[}\PYG{n+nb}{str}\PYG{p}{])} \PYG{o}{\PYGZhy{}\PYGZgt{}} \PYG{n+nb}{float}\PYG{p}{:} 23 | \PYG{l+s+sd}{\PYGZsq{}\PYGZsq{}\PYGZsq{}Inverse frequency of term across a set of documents\PYGZsq{}\PYGZsq{}\PYGZsq{}} 24 | 25 | \PYG{n}{total\PYGZus{}docs} \PYG{o}{=} \PYG{l+m+mi}{0} 26 | \PYG{n}{total\PYGZus{}docs\PYGZus{}with\PYGZus{}term} \PYG{o}{=} \PYG{l+m+mi}{0} 27 | 28 | \PYG{k}{for} \PYG{n}{doc} \PYG{o+ow}{in} \PYG{n}{doc\PYGZus{}list}\PYG{p}{:} 29 | \PYG{n}{total\PYGZus{}docs} \PYG{o}{+=}\PYG{l+m+mi}{1} 30 | \PYG{k}{if} \PYG{n}{term} \PYG{o+ow}{in} \PYG{n}{doc}\PYG{p}{:} 31 | \PYG{n}{total\PYGZus{}docs\PYGZus{}with\PYGZus{}term} \PYG{o}{+=}\PYG{l+m+mi}{1} 32 | 33 | \PYG{n}{idf} \PYG{o}{=} \PYG{n}{math}\PYG{o}{.}\PYG{n}{log}\PYG{p}{(}\PYG{n}{total\PYGZus{}docs} \PYG{o}{/} \PYG{n}{total\PYGZus{}docs\PYGZus{}with\PYGZus{}term}\PYG{p}{)} 34 | \PYG{k}{return} \PYG{n}{idf} 35 | 36 | \PYG{k}{def} \PYG{n+nf}{tf\PYGZus{}idf}\PYG{p}{(}\PYG{n}{tf}\PYG{p}{:}\PYG{n+nb}{float}\PYG{p}{,} \PYG{n}{idf}\PYG{p}{:}\PYG{n+nb}{float}\PYG{p}{)} \PYG{o}{\PYGZhy{}\PYGZgt{}} \PYG{n+nb}{float}\PYG{p}{:} 37 | 38 | \PYG{n}{tfidf} \PYG{o}{=} \PYG{n}{tf}\PYG{o}{*}\PYG{n}{idf} 39 | \PYG{n+nb}{print}\PYG{p}{(}\PYG{l+s+s2}{\PYGZdq{}tf\PYGZhy{}idf:}\PYG{l+s+si}{\PYGZob{}:0.3f\PYGZcb{}}\PYG{l+s+s2}{\PYGZdq{}}\PYG{o}{.}\PYG{n}{format}\PYG{p}{(}\PYG{n}{tfidf}\PYG{p}{))} 40 | 41 | \PYG{n}{tf\PYGZus{}bird} \PYG{o}{=} \PYG{n}{tf}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}bird\PYGZsq{}}\PYG{p}{,} \PYG{n}{documentA}\PYG{p}{)} 42 | \PYG{n}{idf\PYGZus{}docs} \PYG{o}{=} \PYG{n}{idf}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}bird,[documentA, documentB])} 43 | 44 | \PYG{n}{tf\PYGZus{}idf}\PYG{p}{(}\PYG{n}{tf\PYGZus{}bird}\PYG{p}{,} \PYG{n}{idf\PYGZus{}docs}\PYG{p}{)} 45 | \end{Verbatim} 46 | -------------------------------------------------------------------------------- /_minted-main/9A8269913F182230BF3333652DD315692877B981C94AD6CEAF0075A5318FF017.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{k}{class} \PYG{n+nc}{EncoderDecoder}\PYG{p}{(}\PYG{n}{nn}\PYG{o}{.}\PYG{n}{Module}\PYG{p}{):} 3 | \PYG{l+s+sd}{\PYGZdq{}\PYGZdq{}\PYGZdq{}} 4 | \PYG{l+s+sd}{ Defining the encoder/decoder steps} 5 | \PYG{l+s+sd}{ \PYGZdq{}\PYGZdq{}\PYGZdq{}} 6 | \PYG{k}{def} \PYG{n+nf+fm}{\PYGZus{}\PYGZus{}init\PYGZus{}\PYGZus{}}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{p}{,} \PYG{n}{encoder}\PYG{p}{,} \PYG{n}{decoder}\PYG{p}{,} \PYG{n}{src\PYGZus{}embed}\PYG{p}{,} \PYG{n}{tgt\PYGZus{}embed}\PYG{p}{,} \PYG{n}{generator}\PYG{p}{):} 7 | \PYG{n+nb}{super}\PYG{p}{(}\PYG{n}{EncoderDecoder}\PYG{p}{,} \PYG{n+nb+bp}{self}\PYG{p}{)}\PYG{o}{.}\PYG{n+nf+fm}{\PYGZus{}\PYGZus{}init\PYGZus{}\PYGZus{}}\PYG{p}{()} 8 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{encoder} \PYG{o}{=} \PYG{n}{encoder} 9 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{decoder} \PYG{o}{=} \PYG{n}{decoder} 10 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{src\PYGZus{}embed} \PYG{o}{=} \PYG{n}{src\PYGZus{}embed} 11 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{tgt\PYGZus{}embed} \PYG{o}{=} \PYG{n}{tgt\PYGZus{}embed} 12 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{generator} \PYG{o}{=} \PYG{n}{generator} 13 | 14 | \PYG{k}{def} \PYG{n+nf}{forward}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{p}{,} \PYG{n}{src}\PYG{p}{,} \PYG{n}{tgt}\PYG{p}{,} \PYG{n}{src\PYGZus{}mask}\PYG{p}{,} \PYG{n}{tgt\PYGZus{}mask}\PYG{p}{):} 15 | \PYG{l+s+s2}{\PYGZdq{}Take in and process masked src and target sequences.\PYGZdq{}} 16 | \PYG{k}{return} \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{decode}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{encode}\PYG{p}{(}\PYG{n}{src}\PYG{p}{,} \PYG{n}{src\PYGZus{}mask}\PYG{p}{),} \PYG{n}{src\PYGZus{}mask}\PYG{p}{,} 17 | \PYG{n}{tgt}\PYG{p}{,} \PYG{n}{tgt\PYGZus{}mask}\PYG{p}{)} 18 | 19 | \PYG{k}{def} \PYG{n+nf}{encode}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{p}{,} \PYG{n}{src}\PYG{p}{,} \PYG{n}{src\PYGZus{}mask}\PYG{p}{):} 20 | \PYG{k}{return} \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{encoder}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{src\PYGZus{}embed}\PYG{p}{(}\PYG{n}{src}\PYG{p}{),} \PYG{n}{src\PYGZus{}mask}\PYG{p}{)} 21 | 22 | \PYG{k}{def} \PYG{n+nf}{decode}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{p}{,} \PYG{n}{memory}\PYG{p}{,} \PYG{n}{src\PYGZus{}mask}\PYG{p}{,} \PYG{n}{tgt}\PYG{p}{,} \PYG{n}{tgt\PYGZus{}mask}\PYG{p}{):} 23 | \PYG{k}{return} \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{decoder}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{tgt\PYGZus{}embed}\PYG{p}{(}\PYG{n}{tgt}\PYG{p}{),} \PYG{n}{memory}\PYG{p}{,} \PYG{n}{src\PYGZus{}mask}\PYG{p}{,} \PYG{n}{tgt\PYGZus{}mask}\PYG{p}{)} 24 | 25 | \PYG{k}{class} \PYG{n+nc}{Generator}\PYG{p}{(}\PYG{n}{nn}\PYG{o}{.}\PYG{n}{Module}\PYG{p}{):} 26 | \PYG{l+s+s2}{\PYGZdq{}Define standard linear + softmax generation step.\PYGZdq{}} 27 | \PYG{k}{def} \PYG{n+nf+fm}{\PYGZus{}\PYGZus{}init\PYGZus{}\PYGZus{}}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{p}{,} \PYG{n}{d\PYGZus{}model}\PYG{p}{,} \PYG{n}{vocab}\PYG{p}{):} 28 | \PYG{n+nb}{super}\PYG{p}{(}\PYG{n}{Generator}\PYG{p}{,} \PYG{n+nb+bp}{self}\PYG{p}{)}\PYG{o}{.}\PYG{n+nf+fm}{\PYGZus{}\PYGZus{}init\PYGZus{}\PYGZus{}}\PYG{p}{()} 29 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{proj} \PYG{o}{=} \PYG{n}{nn}\PYG{o}{.}\PYG{n}{Linear}\PYG{p}{(}\PYG{n}{d\PYGZus{}model}\PYG{p}{,} \PYG{n}{vocab}\PYG{p}{)} 30 | 31 | \PYG{k}{def} \PYG{n+nf}{forward}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{p}{,} \PYG{n}{x}\PYG{p}{):} 32 | \PYG{k}{return} \PYG{n}{F}\PYG{o}{.}\PYG{n}{log\PYGZus{}softmax}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{proj}\PYG{p}{(}\PYG{n}{x}\PYG{p}{),} \PYG{n}{dim}\PYG{o}{=\PYGZhy{}}\PYG{l+m+mi}{1}\PYG{p}{)} 33 | 34 | \end{Verbatim} 35 | -------------------------------------------------------------------------------- /_minted-main/9D8E8AD3D297B1AD40F60B8FF6CF54802877B981C94AD6CEAF0075A5318FF017.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | 3 | \PYG{k}{def} \PYG{n+nf}{make\PYGZus{}context\PYGZus{}vector}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{p}{,} \PYG{n}{context}\PYG{p}{,} \PYG{n}{word\PYGZus{}to\PYGZus{}ix}\PYG{p}{)} \PYG{o}{\PYGZhy{}\PYGZgt{}} \PYG{n}{torch}\PYG{o}{.}\PYG{n}{LongTensor}\PYG{p}{:} 4 | \PYG{l+s+sd}{\PYGZdq{}\PYGZdq{}\PYGZdq{}} 5 | \PYG{l+s+sd}{ For each word in the vocab, find sliding windows of [\PYGZhy{}2,1,0,1,2] indexes} 6 | \PYG{l+s+sd}{ relative to the position of the word} 7 | \PYG{l+s+sd}{ :param vocab: list of words in the vocab} 8 | \PYG{l+s+sd}{ :return: torch.LongTensor} 9 | \PYG{l+s+sd}{ \PYGZdq{}\PYGZdq{}\PYGZdq{}} 10 | \PYG{n}{idxs} \PYG{o}{=} \PYG{p}{[}\PYG{n}{word\PYGZus{}to\PYGZus{}ix}\PYG{p}{[}\PYG{n}{w}\PYG{p}{]} \PYG{k}{for} \PYG{n}{w} \PYG{o+ow}{in} \PYG{n}{context}\PYG{p}{]} 11 | \PYG{n}{tensor} \PYG{o}{=} \PYG{n}{torch}\PYG{o}{.}\PYG{n}{LongTensor}\PYG{p}{(}\PYG{n}{idxs}\PYG{p}{)} 12 | 13 | 14 | \PYG{k}{def} \PYG{n+nf}{train\PYGZus{}model}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{p}{):} 15 | 16 | \PYG{c+c1}{\PYGZsh{} Loss and optimizer} 17 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{model} \PYG{o}{=} \PYG{n}{CBOW}\PYG{p}{()}\PYG{o}{.}\PYG{n}{to}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{device}\PYG{p}{)} 18 | \PYG{n}{optimizer} \PYG{o}{=} \PYG{n}{optim}\PYG{o}{.}\PYG{n}{Adam}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{model}\PYG{o}{.}\PYG{n}{parameters}\PYG{p}{(),} \PYG{n}{lr}\PYG{o}{=}\PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{learning\PYGZus{}rate}\PYG{p}{)} 19 | \PYG{n}{loss\PYGZus{}function} \PYG{o}{=} \PYG{n}{nn}\PYG{o}{.}\PYG{n}{NLLLoss}\PYG{p}{()} 20 | 21 | \PYG{n}{logging}\PYG{o}{.}\PYG{n}{warning}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}Building training data\PYGZsq{}}\PYG{p}{)} 22 | \PYG{n}{data} \PYG{o}{=} \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{build\PYGZus{}training\PYGZus{}data}\PYG{p}{()} 23 | 24 | \PYG{n}{logging}\PYG{o}{.}\PYG{n}{warning}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}Starting forward pass\PYGZsq{}}\PYG{p}{)} 25 | \PYG{k}{for} \PYG{n}{epoch} \PYG{o+ow}{in} \PYG{n}{tqdm}\PYG{p}{(}\PYG{n+nb}{range}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{num\PYGZus{}epochs}\PYG{p}{)):} 26 | \PYG{c+c1}{\PYGZsh{} we start tracking how accurate our intial words are} 27 | \PYG{n}{total\PYGZus{}loss} \PYG{o}{=} \PYG{l+m+mi}{0} 28 | 29 | \PYG{c+c1}{\PYGZsh{} for the x, y in the training data:} 30 | \PYG{k}{for} \PYG{n}{context}\PYG{p}{,} \PYG{n}{target} \PYG{o+ow}{in} \PYG{n}{data}\PYG{p}{:} 31 | \PYG{n}{context\PYGZus{}vector} \PYG{o}{=} \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{make\PYGZus{}context\PYGZus{}vector}\PYG{p}{(}\PYG{n}{context}\PYG{p}{,} \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{word\PYGZus{}to\PYGZus{}ix}\PYG{p}{)} 32 | 33 | \PYG{c+c1}{\PYGZsh{} we look at loss} 34 | \PYG{n}{log\PYGZus{}probs} \PYG{o}{=} \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{model}\PYG{p}{(}\PYG{n}{context\PYGZus{}vector}\PYG{p}{)} 35 | 36 | \PYG{c+c1}{\PYGZsh{} we compare the loss from what the actual word is related to the probaility of the words} 37 | \PYG{n}{total\PYGZus{}loss} \PYG{o}{+=} \PYG{n}{loss\PYGZus{}function}\PYG{p}{(} 38 | \PYG{n}{log\PYGZus{}probs}\PYG{p}{,} \PYG{n}{torch}\PYG{o}{.}\PYG{n}{tensor}\PYG{p}{([}\PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{word\PYGZus{}to\PYGZus{}ix}\PYG{p}{[}\PYG{n}{target}\PYG{p}{]])} 39 | \PYG{p}{)} 40 | 41 | \PYG{c+c1}{\PYGZsh{} optimize at the end of each epoch} 42 | \PYG{n}{optimizer}\PYG{o}{.}\PYG{n}{zero\PYGZus{}grad}\PYG{p}{()} 43 | \PYG{n}{total\PYGZus{}loss}\PYG{o}{.}\PYG{n}{backward}\PYG{p}{()} 44 | \PYG{n}{optimizer}\PYG{o}{.}\PYG{n}{step}\PYG{p}{()} 45 | 46 | \PYG{c+c1}{\PYGZsh{} Log out some metrics to see if loss decreases} 47 | \PYG{n}{logging}\PYG{o}{.}\PYG{n}{warning}\PYG{p}{(}\PYG{l+s+s2}{\PYGZdq{}end of epoch }\PYG{l+s+si}{\PYGZob{}\PYGZcb{}}\PYG{l+s+s2}{ | loss }\PYG{l+s+si}{\PYGZob{}:2.3f\PYGZcb{}}\PYG{l+s+s2}{\PYGZdq{}}\PYG{o}{.}\PYG{n}{format}\PYG{p}{(}\PYG{n}{epoch}\PYG{p}{,} \PYG{n}{total\PYGZus{}loss}\PYG{p}{))} 48 | 49 | \PYG{n}{torch}\PYG{o}{.}\PYG{n}{save}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{model}\PYG{o}{.}\PYG{n}{state\PYGZus{}dict}\PYG{p}{(),} \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{model\PYGZus{}path}\PYG{p}{)} 50 | \PYG{n}{logging}\PYG{o}{.}\PYG{n}{warning}\PYG{p}{(}\PYG{l+s+sa}{f}\PYG{l+s+s1}{\PYGZsq{}Save model to }\PYG{l+s+si}{\PYGZob{}}\PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{model\PYGZus{}path}\PYG{l+s+si}{\PYGZcb{}}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{)} 51 | \end{Verbatim} 52 | -------------------------------------------------------------------------------- /_minted-main/A68336B712BE240AD81BE1B66DBA3B5A5018845A16FEE2F871CFA2D2B2CFB1C6.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{n}{bird\PYGZus{}geo} 3 | \end{Verbatim} 4 | -------------------------------------------------------------------------------- /_minted-main/B796679A805702BA76C4A53F8A0B63172877B981C94AD6CEAF0075A5318FF017.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{n}{responses} \PYG{o}{=} \PYG{p}{[}\PYG{l+s+s2}{\PYGZdq{}Hold fast to dreams, for if dreams die, life is a broken\PYGZhy{}winged bird that cannot fly.\PYGZdq{}}\PYG{p}{,} \PYG{l+s+s2}{\PYGZdq{}No bird soars too high if he soars with his own wings.\PYGZdq{}}\PYG{p}{,} 3 | \PYG{l+s+s2}{\PYGZdq{}A bird does not sing because it has an answer, it sings because it has a song.\PYGZdq{}}\PYG{p}{]} 4 | \end{Verbatim} 5 | -------------------------------------------------------------------------------- /_minted-main/BA7FD7AFD5C763308290D7AB2859D8E6360E91D77B40F6AB2A9DE4914DF5D2A9.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{n}{bird1} \PYG{o}{=} \PYG{p}{[}\PYG{l+m+mi}{012}\PYG{p}{,}\PYG{l+m+mi}{2}\PYG{p}{,}\PYG{l+s+s2}{\PYGZdq{}US\PYGZdq{}}\PYG{p}{,} \PYG{l+m+mi}{5}\PYG{p}{]} 3 | \end{Verbatim} 4 | -------------------------------------------------------------------------------- /_minted-main/C7DD6291193B588922A017A6743B30C65018845A16FEE2F871CFA2D2B2CFB1C6.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{n}{sparse\PYGZus{}vector} \PYG{o}{=} \PYG{p}{[}\PYG{l+m+mi}{1}\PYG{p}{,}\PYG{l+m+mi}{0}\PYG{p}{,}\PYG{l+m+mi}{0}\PYG{p}{,}\PYG{l+m+mi}{0}\PYG{p}{,}\PYG{l+m+mi}{0}\PYG{p}{,}\PYG{l+m+mi}{0}\PYG{p}{,}\PYG{l+m+mi}{0}\PYG{p}{,}\PYG{l+m+mi}{0}\PYG{p}{,}\PYG{l+m+mi}{0}\PYG{p}{,}\PYG{l+m+mi}{0}\PYG{p}{]} 3 | \PYG{n}{dense\PYGZus{}vector} \PYG{o}{=} \PYG{p}{[}\PYG{l+m+mi}{1}\PYG{p}{,}\PYG{l+m+mi}{2}\PYG{p}{,}\PYG{l+m+mi}{2}\PYG{p}{,}\PYG{l+m+mi}{3}\PYG{p}{,}\PYG{l+m+mi}{0}\PYG{p}{,}\PYG{l+m+mi}{4}\PYG{p}{,}\PYG{l+m+mi}{5}\PYG{p}{,}\PYG{l+m+mi}{8}\PYG{p}{,}\PYG{l+m+mi}{8}\PYG{p}{,}\PYG{l+m+mi}{5}\PYG{p}{]} 4 | \end{Verbatim} 5 | -------------------------------------------------------------------------------- /_minted-main/DE71EB197869238CE0CF93F140D0B6012877B981C94AD6CEAF0075A5318FF017.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{k+kn}{import} \PYG{n+nn}{torch} 3 | \PYG{k+kn}{from} \PYG{n+nn}{transformers} \PYG{k+kn}{import} \PYG{n}{BertTokenizer}\PYG{p}{,} \PYG{n}{BertModel} 4 | 5 | \PYG{c+c1}{\PYGZsh{} Load pre\PYGZhy{}trained model tokenizer (vocabulary)} 6 | \PYG{n}{tokenizer} \PYG{o}{=} \PYG{n}{BertTokenizer}\PYG{o}{.}\PYG{n}{from\PYGZus{}pretrained}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}bert\PYGZhy{}base\PYGZhy{}uncased\PYGZsq{}}\PYG{p}{)} 7 | 8 | \PYG{n}{text} \PYG{o}{=} \PYG{l+s+s2}{\PYGZdq{}\PYGZdq{}\PYGZdq{}Not to beat around the bush,} 9 | \PYG{l+s+s2}{but the butterfly bush is the best flower for hummingbirds.\PYGZdq{}\PYGZdq{}\PYGZdq{}} 10 | 11 | \PYG{c+c1}{\PYGZsh{} Lots of BERT code truncated for clarity of this example} 12 | 13 | \PYG{n+nb}{print} \PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}Shape is: }\PYG{l+s+si}{\PYGZpc{}d}\PYG{l+s+s1}{ x }\PYG{l+s+si}{\PYGZpc{}d}\PYG{l+s+s1}{\PYGZsq{}} \PYG{o}{\PYGZpc{}} \PYG{p}{(}\PYG{n+nb}{len}\PYG{p}{(}\PYG{n}{embeddings}\PYG{p}{),} \PYG{n+nb}{len}\PYG{p}{(}\PYG{n}{embeddings}\PYG{p}{[}\PYG{l+m+mi}{0}\PYG{p}{])))} 14 | 15 | \PYG{p}{[}\PYG{n}{tensor}\PYG{p}{([} \PYG{l+m+mf}{1.0265e\PYGZhy{}01}\PYG{p}{,} \PYG{o}{\PYGZhy{}}\PYG{l+m+mf}{2.2793e+00}\PYG{p}{,} \PYG{o}{\PYGZhy{}}\PYG{l+m+mf}{2.0930e\PYGZhy{}01}\PYG{p}{,} \PYG{l+m+mf}{2.9740e\PYGZhy{}01}\PYG{p}{,} \PYG{o}{\PYGZhy{}}\PYG{l+m+mf}{1.2967e+00}\PYG{p}{,} 16 | \PYG{o}{\PYGZhy{}}\PYG{l+m+mf}{4.9467e\PYGZhy{}01}\PYG{p}{,} \PYG{l+m+mf}{7.1440e\PYGZhy{}01}\PYG{p}{,} \PYG{l+m+mf}{2.1999e+00}\PYG{p}{,} \PYG{o}{\PYGZhy{}}\PYG{l+m+mf}{4.0721e\PYGZhy{}01}\PYG{p}{,} \PYG{o}{\PYGZhy{}}\PYG{l+m+mf}{4.2629e\PYGZhy{}01}\PYG{p}{,} \PYG{o}{...} 17 | \PYG{l+m+mf}{1.5620e+00}\PYG{p}{])} 18 | 19 | \end{Verbatim} 20 | -------------------------------------------------------------------------------- /_minted-main/F559E8704ECF0DC32EFED27A75ED0EB35018845A16FEE2F871CFA2D2B2CFB1C6.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{n}{array}\PYG{p}{([[}\PYG{l+m+mf}{0.}\PYG{p}{,} \PYG{l+m+mf}{0.}\PYG{p}{,} \PYG{l+m+mf}{1.}\PYG{p}{],} 3 | \PYG{p}{[}\PYG{l+m+mf}{0.}\PYG{p}{,} \PYG{l+m+mf}{1.}\PYG{p}{,} \PYG{l+m+mf}{0.}\PYG{p}{],} 4 | \PYG{p}{[}\PYG{l+m+mf}{1.}\PYG{p}{,} \PYG{l+m+mf}{0.}\PYG{p}{,} \PYG{l+m+mf}{0.}\PYG{p}{]])} 5 | \end{Verbatim} 6 | -------------------------------------------------------------------------------- /_minted-main/F5A0B26D022F8704EBD415636678FCE62877B981C94AD6CEAF0075A5318FF017.pygtex: -------------------------------------------------------------------------------- 1 | \begin{Verbatim}[commandchars=\\\{\}] 2 | \PYG{k}{class} \PYG{n+nc}{TextPreProcessor}\PYG{p}{:} 3 | \PYG{k}{def} \PYG{n+nf+fm}{\PYGZus{}\PYGZus{}init\PYGZus{}\PYGZus{}}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{p}{)} \PYG{o}{\PYGZhy{}\PYGZgt{}} \PYG{k+kc}{None}\PYG{p}{:} 4 | 5 | \PYG{c+c1}{\PYGZsh{} TODO: create utility class for reading relative paths across the project} 6 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{input\PYGZus{}file} \PYG{o}{=} \PYG{l+s+s2}{\PYGZdq{}word2vec\PYGZus{}input.csv\PYGZdq{}} 7 | 8 | \PYG{c+c1}{\PYGZsh{} TODO: split into training and test set} 9 | 10 | \PYG{k}{def} \PYG{n+nf}{generate\PYGZus{}tokens}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{p}{):} 11 | 12 | \PYG{k}{with} \PYG{n+nb}{open}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{input\PYGZus{}file}\PYG{p}{,} \PYG{n}{encoding}\PYG{o}{=}\PYG{l+s+s2}{\PYGZdq{}utf\PYGZhy{}8\PYGZdq{}}\PYG{p}{)} \PYG{k}{as} \PYG{n}{f}\PYG{p}{:} 13 | \PYG{k}{for} \PYG{n}{line} \PYG{o+ow}{in} \PYG{n}{f}\PYG{p}{:} 14 | \PYG{n}{line} \PYG{o}{=} \PYG{n}{line}\PYG{o}{.}\PYG{n}{replace}\PYG{p}{(}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+se}{\PYGZbs{}\PYGZbs{}}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,} \PYG{l+s+s2}{\PYGZdq{}\PYGZdq{}}\PYG{p}{)} \PYG{c+c1}{\PYGZsh{} Strip extra \PYGZbs{}\PYGZbs{} in input text} 15 | \PYG{k}{yield} \PYG{n}{line}\PYG{o}{.}\PYG{n}{strip}\PYG{p}{()}\PYG{o}{.}\PYG{n}{split}\PYG{p}{()} 16 | 17 | \PYG{k}{def} \PYG{n+nf}{build\PYGZus{}vocab}\PYG{p}{(}\PYG{n+nb+bp}{self}\PYG{p}{)} \PYG{o}{\PYGZhy{}\PYGZgt{}} \PYG{n}{Vocab}\PYG{p}{:} 18 | 19 | \PYG{n}{vocab} \PYG{o}{=} \PYG{n}{build\PYGZus{}vocab\PYGZus{}from\PYGZus{}iterator}\PYG{p}{(} 20 | \PYG{n+nb+bp}{self}\PYG{o}{.}\PYG{n}{generate\PYGZus{}tokens}\PYG{p}{(),} \PYG{n}{specials}\PYG{o}{=}\PYG{p}{[}\PYG{l+s+s2}{\PYGZdq{}\PYGZlt{}unk\PYGZgt{}\PYGZdq{}}\PYG{p}{],} \PYG{n}{min\PYGZus{}freq}\PYG{o}{=}\PYG{l+m+mi}{100} 21 | \PYG{p}{)} 22 | \PYG{k}{return} \PYG{n}{vocab} 23 | \end{Verbatim} 24 | -------------------------------------------------------------------------------- /_minted-main/default.pygstyle: -------------------------------------------------------------------------------- 1 | 2 | \makeatletter 3 | \def\PYG@reset{\let\PYG@it=\relax \let\PYG@bf=\relax% 4 | \let\PYG@ul=\relax \let\PYG@tc=\relax% 5 | \let\PYG@bc=\relax \let\PYG@ff=\relax} 6 | \def\PYG@tok#1{\csname PYG@tok@#1\endcsname} 7 | \def\PYG@toks#1+{\ifx\relax#1\empty\else% 8 | \PYG@tok{#1}\expandafter\PYG@toks\fi} 9 | \def\PYG@do#1{\PYG@bc{\PYG@tc{\PYG@ul{% 10 | \PYG@it{\PYG@bf{\PYG@ff{#1}}}}}}} 11 | \def\PYG#1#2{\PYG@reset\PYG@toks#1+\relax+\PYG@do{#2}} 12 | 13 | \@namedef{PYG@tok@w}{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}} 14 | \@namedef{PYG@tok@c}{\let\PYG@it=\textit\def\PYG@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}} 15 | \@namedef{PYG@tok@cp}{\def\PYG@tc##1{\textcolor[rgb]{0.74,0.48,0.00}{##1}}} 16 | \@namedef{PYG@tok@k}{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} 17 | \@namedef{PYG@tok@kp}{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} 18 | \@namedef{PYG@tok@kt}{\def\PYG@tc##1{\textcolor[rgb]{0.69,0.00,0.25}{##1}}} 19 | \@namedef{PYG@tok@o}{\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}} 20 | \@namedef{PYG@tok@ow}{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.67,0.13,1.00}{##1}}} 21 | \@namedef{PYG@tok@nb}{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} 22 | \@namedef{PYG@tok@nf}{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}} 23 | \@namedef{PYG@tok@nc}{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}} 24 | \@namedef{PYG@tok@nn}{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}} 25 | \@namedef{PYG@tok@ne}{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.82,0.25,0.23}{##1}}} 26 | \@namedef{PYG@tok@nv}{\def\PYG@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}} 27 | \@namedef{PYG@tok@no}{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}} 28 | \@namedef{PYG@tok@nl}{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.63,0.00}{##1}}} 29 | \@namedef{PYG@tok@ni}{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.60,0.60,0.60}{##1}}} 30 | \@namedef{PYG@tok@na}{\def\PYG@tc##1{\textcolor[rgb]{0.49,0.56,0.16}{##1}}} 31 | \@namedef{PYG@tok@nt}{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} 32 | \@namedef{PYG@tok@nd}{\def\PYG@tc##1{\textcolor[rgb]{0.67,0.13,1.00}{##1}}} 33 | \@namedef{PYG@tok@s}{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}} 34 | \@namedef{PYG@tok@sd}{\let\PYG@it=\textit\def\PYG@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}} 35 | \@namedef{PYG@tok@si}{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.73,0.40,0.53}{##1}}} 36 | \@namedef{PYG@tok@se}{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.73,0.40,0.13}{##1}}} 37 | \@namedef{PYG@tok@sr}{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.40,0.53}{##1}}} 38 | \@namedef{PYG@tok@ss}{\def\PYG@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}} 39 | \@namedef{PYG@tok@sx}{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} 40 | \@namedef{PYG@tok@m}{\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}} 41 | \@namedef{PYG@tok@gh}{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}} 42 | \@namedef{PYG@tok@gu}{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}} 43 | \@namedef{PYG@tok@gd}{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}} 44 | \@namedef{PYG@tok@gi}{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}} 45 | \@namedef{PYG@tok@gr}{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} 46 | \@namedef{PYG@tok@ge}{\let\PYG@it=\textit} 47 | \@namedef{PYG@tok@gs}{\let\PYG@bf=\textbf} 48 | \@namedef{PYG@tok@gp}{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}} 49 | \@namedef{PYG@tok@go}{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} 50 | \@namedef{PYG@tok@gt}{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} 51 | \@namedef{PYG@tok@err}{\def\PYG@bc##1{{\setlength{\fboxsep}{\string -\fboxrule}\fcolorbox[rgb]{1.00,0.00,0.00}{1,1,1}{\strut ##1}}}} 52 | \@namedef{PYG@tok@kc}{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} 53 | \@namedef{PYG@tok@kd}{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} 54 | \@namedef{PYG@tok@kn}{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} 55 | \@namedef{PYG@tok@kr}{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} 56 | \@namedef{PYG@tok@bp}{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} 57 | \@namedef{PYG@tok@fm}{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}} 58 | \@namedef{PYG@tok@vc}{\def\PYG@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}} 59 | \@namedef{PYG@tok@vg}{\def\PYG@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}} 60 | \@namedef{PYG@tok@vi}{\def\PYG@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}} 61 | \@namedef{PYG@tok@vm}{\def\PYG@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}} 62 | \@namedef{PYG@tok@sa}{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}} 63 | \@namedef{PYG@tok@sb}{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}} 64 | \@namedef{PYG@tok@sc}{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}} 65 | \@namedef{PYG@tok@dl}{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}} 66 | \@namedef{PYG@tok@s2}{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}} 67 | \@namedef{PYG@tok@sh}{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}} 68 | \@namedef{PYG@tok@s1}{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}} 69 | \@namedef{PYG@tok@mb}{\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}} 70 | \@namedef{PYG@tok@mf}{\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}} 71 | \@namedef{PYG@tok@mh}{\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}} 72 | \@namedef{PYG@tok@mi}{\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}} 73 | \@namedef{PYG@tok@il}{\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}} 74 | \@namedef{PYG@tok@mo}{\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}} 75 | \@namedef{PYG@tok@ch}{\let\PYG@it=\textit\def\PYG@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}} 76 | \@namedef{PYG@tok@cm}{\let\PYG@it=\textit\def\PYG@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}} 77 | \@namedef{PYG@tok@cpf}{\let\PYG@it=\textit\def\PYG@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}} 78 | \@namedef{PYG@tok@c1}{\let\PYG@it=\textit\def\PYG@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}} 79 | \@namedef{PYG@tok@cs}{\let\PYG@it=\textit\def\PYG@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}} 80 | 81 | \def\PYGZbs{\char`\\} 82 | \def\PYGZus{\char`\_} 83 | \def\PYGZob{\char`\{} 84 | \def\PYGZcb{\char`\}} 85 | \def\PYGZca{\char`\^} 86 | \def\PYGZam{\char`\&} 87 | \def\PYGZlt{\char`\<} 88 | \def\PYGZgt{\char`\>} 89 | \def\PYGZsh{\char`\#} 90 | \def\PYGZpc{\char`\%} 91 | \def\PYGZdl{\char`\$} 92 | \def\PYGZhy{\char`\-} 93 | \def\PYGZsq{\char`\'} 94 | \def\PYGZdq{\char`\"} 95 | \def\PYGZti{\char`\~} 96 | % for compatibility with earlier versions 97 | \def\PYGZat{@} 98 | \def\PYGZlb{[} 99 | \def\PYGZrb{]} 100 | \makeatother 101 | 102 | -------------------------------------------------------------------------------- /diazessay.cls: -------------------------------------------------------------------------------- 1 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 2 | % Diaz Essay 3 | % LaTeX Class 4 | % Version 1.0 (13/1/19) 5 | % 6 | % This class originates from: 7 | % http://www.LaTeXTemplates.com 8 | % 9 | % Authors: 10 | % Vel (vel@LaTeXTemplates.com) 11 | % Nicolas Diaz (nsdiaz@uc.cl) 12 | % 13 | % License: 14 | % CC BY-NC-SA 3.0 (http://creativecommons.org/licenses/by-nc-sa/3.0/) 15 | % 16 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 17 | 18 | %---------------------------------------------------------------------------------------- 19 | % CLASS CONFIGURATION 20 | %---------------------------------------------------------------------------------------- 21 | 22 | \NeedsTeXFormat{LaTeX2e} 23 | \ProvidesClass{diazessay}[2019/01/13 Diaz Essay class v1.0] 24 | 25 | \DeclareOption*{\PassOptionsToClass{\CurrentOption}{article}} % Pass through any options to the base class 26 | \ProcessOptions\relax % Process given options 27 | 28 | \LoadClass{article} % Load the base class 29 | 30 | %---------------------------------------------------------------------------------------- 31 | % PACKAGES AND OTHER DOCUMENT CONFIGURATIONS 32 | %---------------------------------------------------------------------------------------- 33 | 34 | 35 | \usepackage[english]{babel} % English language hyphenation 36 | 37 | \usepackage{graphicx} % Required for including images 38 | \graphicspath{{figures/}{./}} % Specifies where to look for included images (trailing slash required) 39 | 40 | \usepackage{wrapfig} % Allows in-line images 41 | 42 | \usepackage{booktabs} % Required for better horizontal rules in tables 43 | 44 | \usepackage{enumitem} % Required for list customisation 45 | \setlist{noitemsep} % No spacing between list items 46 | 47 | \makeatletter 48 | \renewcommand\@biblabel[1]{\textbf{#1.}} % Remove the square brackets from each bibliography item ('[1]' to '1.') 49 | \makeatother 50 | 51 | %---------------------------------------------------------------------------------------- 52 | % DOCUMENT MARGINS 53 | %---------------------------------------------------------------------------------------- 54 | 55 | \usepackage{geometry} % Required for adjusting page dimensions and margins 56 | 57 | \geometry{ 58 | paper=a4paper, % Paper size, change to letterpaper for US letter size 59 | top=2.5cm, % Top margin 60 | bottom=3cm, % Bottom margin 61 | left=4cm, % Left margin 62 | right=4cm, % Right margin 63 | headheight=0.75cm, % Header height 64 | footskip=1.5cm, % Space from the bottom margin to the baseline of the footer 65 | headsep=0.75cm, % Space from the top margin to the baseline of the header 66 | %showframe, % Uncomment to show how the type block is set on the page 67 | } 68 | 69 | %---------------------------------------------------------------------------------------- 70 | % FONTS 71 | %---------------------------------------------------------------------------------------- 72 | 73 | \usepackage[utf8]{inputenc} % Required for inputting international characters 74 | \usepackage[T1]{fontenc} % Use 8-bit encoding 75 | 76 | \usepackage{mathpazo} % Use the Palatino font 77 | 78 | \usepackage[protrusion=true, expansion=true]{microtype} % Better typography 79 | 80 | \linespread{1.05} % Increase line spacing slightly; Palatino benefits from a slight increase by default 81 | 82 | %---------------------------------------------------------------------------------------- 83 | % TITLE 84 | %---------------------------------------------------------------------------------------- 85 | 86 | \makeatletter 87 | \renewcommand{\maketitle}{ % Customise the title (do not edit title and author name here) 88 | \vspace*{30pt} % Vertical whitespace before the title 89 | \begin{flushright} % Right align 90 | {\LARGE\@title} % Increase the font size of the title 91 | 92 | \vspace{50pt} % Vertical whitespace between the title and author name 93 | 94 | {\large\@author}\\ % Author name 95 | \@date % Date 96 | 97 | \vspace{40pt} % Vertical whitespace between the author block and abstract 98 | \end{flushright} 99 | } 100 | \makeatother 101 | 102 | %---------------------------------------------------------------------------------------- 103 | % HEADERS AND FOOTERS 104 | %---------------------------------------------------------------------------------------- 105 | 106 | \usepackage{fancyhdr} % Required for customising headers and footers 107 | \pagestyle{fancy} % Enable custom headers and footers 108 | 109 | \renewcommand{\headrulewidth}{0pt} % Remove default top horizontal rule 110 | \lhead{} % Left header 111 | \chead{} % Centre header 112 | \rhead{} % Right header 113 | \lfoot{} % Left footer 114 | \cfoot{\thepage} % Centre footer 115 | \rfoot{} % Right footer 116 | -------------------------------------------------------------------------------- /embeddings.bib: -------------------------------------------------------------------------------- 1 | @book{abu2012learning, 2 | title = {Learning from data}, 3 | author = {Abu-Mostafa, Yaser S and Magdon-Ismail, Malik and Lin, Hsuan-Tien}, 4 | year = 2012, 5 | publisher = {AMLBook New York}, 6 | volume = 4 7 | } 8 | @misc{altay_2020, 9 | title = {Categorical variables in decision trees}, 10 | author = {Altay, Gabriel}, 11 | year = 2020, 12 | month = {Mar}, 13 | journal = {Kaggle}, 14 | publisher = {Kaggle}, 15 | url = {https://www.kaggle.com/code/gabrielaltay/categorical-variables-in-decision-trees} 16 | } 17 | @book{arpaci2018operating, 18 | title={Operating systems: Three easy pieces}, 19 | author={Arpaci-Dusseau, Remzi H and Arpaci-Dusseau, Andrea C}, 20 | year={2018}, 21 | publisher={Arpaci-Dusseau Books, LLC} 22 | } 23 | 24 | @inproceedings{arapakis2014impact, 25 | title = {Impact of response latency on user behavior in web search}, 26 | author = {Arapakis, Ioannis and Bai, Xiao and Cambazoglu, B Barla}, 27 | year = 2014, 28 | booktitle = {Proceedings of the 37th international ACM SIGIR conference on Research \& development in information retrieval}, 29 | pages = {103--112} 30 | } 31 | @article{cong2016novel, 32 | title={A novel alignment-free method for detection of lateral genetic transfer based on TF-IDF}, 33 | author={Cong, Yingnan and Chan, Yao-ban and Ragan, Mark A}, 34 | journal={Scientific reports}, 35 | volume={6}, 36 | number={1}, 37 | pages={1--13}, 38 | year={2016}, 39 | publisher={Springer} 40 | } 41 | @article{bengio2013representation, 42 | title={Representation learning: A review and new perspectives}, 43 | author={Bengio, Yoshua and Courville, Aaron and Vincent, Pascal}, 44 | journal={IEEE transactions on pattern analysis and machine intelligence}, 45 | volume={35}, 46 | number={8}, 47 | pages={1798--1828}, 48 | year={2013}, 49 | publisher={IEEE} 50 | } 51 | 52 | @article{gabin2023keyword, 53 | title={Keyword Embeddings for Query Suggestion}, 54 | author={Gab{\'\i}n, Jorge and Ares, M Eduardo and Parapar, Javier}, 55 | journal={arXiv preprint arXiv:2301.08006}, 56 | year={2023} 57 | } 58 | 59 | @article{castells2023recommender, 60 | title = {Recommender Systems: A Primer}, 61 | author = {Castells, Pablo and Jannach, Dietmar}, 62 | year = 2023, 63 | journal = {arXiv preprint arXiv:2302.02579} 64 | } 65 | @article{chandrasekaran2021evolution, 66 | title = {Evolution of semantic similarity—a survey}, 67 | author = {Chandrasekaran, Dhivya and Mago, Vijay}, 68 | year = 2021, 69 | journal = {ACM Computing Surveys (CSUR)}, 70 | publisher = {ACM New York, NY, USA}, 71 | volume = 54, 72 | number = 2, 73 | pages = {1--37} 74 | } 75 | @inproceedings{cheng2016wide, 76 | title={Wide \& deep learning for recommender systems}, 77 | author={Cheng, Heng-Tze and Koc, Levent and Harmsen, Jeremiah and Shaked, Tal and Chandra, Tushar and Aradhye, Hrishi and Anderson, Glen and Corrado, Greg and Chai, Wei and Ispir, Mustafa and others}, 78 | booktitle={Proceedings of the 1st workshop on deep learning for recommender systems}, 79 | pages={7--10}, 80 | year={2016} 81 | } 82 | @book{chollet2021deep, 83 | title = {Deep learning with Python}, 84 | author = {Chollet, Francois}, 85 | year = 2021, 86 | publisher = {Simon and Schuster} 87 | } 88 | @inproceedings{collobert2008unified, 89 | title={A unified architecture for natural language processing: Deep neural networks with multitask learning}, 90 | author={Collobert, Ronan and Weston, Jason}, 91 | booktitle={Proceedings of the 25th international conference on Machine learning}, 92 | pages={160--167}, 93 | year={2008} 94 | } 95 | @inproceedings{covington2016deep, 96 | title = {Deep neural networks for youtube recommendations}, 97 | author = {Covington, Paul and Adams, Jay and Sargin, Emre}, 98 | year = 2016, 99 | booktitle = {Proceedings of the 10th ACM conference on recommender systems}, 100 | pages = {191--198} 101 | } 102 | @inproceedings{cvitanic2016lda, 103 | title = {LDA v. LSA: A comparison of two computational text analysis tools for the functional categorization of patents}, 104 | author = {Cvitanic, Toni and Lee, Bumsoo and Song, Hyeon Ik and Fu, Katherine and Rosen, David}, 105 | year = 2016, 106 | booktitle = {International Conference on Case-Based Reasoning} 107 | } 108 | @article{devlin2018bert, 109 | title={Bert: Pre-training of deep bidirectional transformers for language understanding}, 110 | author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina}, 111 | journal={arXiv preprint arXiv:1810.04805}, 112 | year={2018} 113 | } 114 | @article{di2021considerations, 115 | title={Considerations about learning Word2Vec}, 116 | author={Di Gennaro, Giovanni and Buonanno, Amedeo and Palmieri, Francesco AN}, 117 | journal={The Journal of Supercomputing}, 118 | pages={1--16}, 119 | year={2021}, 120 | publisher={Springer} 121 | } 122 | @misc{doctorow_2023, 123 | title = {Pluralistic: Tiktok's enshittification (21 Jan 2023)}, 124 | author = {Doctorow, Author Cory}, 125 | year = 2023, 126 | month = {Feb}, 127 | journal = {Pluralistic Daily links from Cory Doctorow}, 128 | url = {https://pluralistic.net/2023/01/21/potemkin-ai/#hey-guys} 129 | } 130 | @misc{dye_ekandham_saluja_rastogi_2020, 131 | title = {Supporting content decision makers with machine learning}, 132 | author = {Dye, Melody and Ekandham, Chaitanya and Saluja, Avneesh and Rastogi, Ashish}, 133 | year = 2020, 134 | month = {Dec}, 135 | url = {https://netflixtechblog.com/supporting-content-decision-makers-with-machine-learning-995b7b76006f} 136 | } 137 | @article{ekstrand2019recommender, 138 | title = {Recommender systems notation: proposed common notation for teaching and research}, 139 | author = {Ekstrand, Michael D and Konstan, Joseph A}, 140 | year = 2019, 141 | journal = {arXiv preprint arXiv:1902.01348} 142 | } 143 | @inproceedings{el2022twhin, 144 | title={Twhin: Embedding the twitter heterogeneous information network for personalized recommendation}, 145 | author={El-Kishky, Ahmed and Markovich, Thomas and Park, Serim and Verma, Chetan and Kim, Baekjin and Eskander, Ramy and Malkov, Yury and Portman, Frank and Samaniego, Sof{\'\i}a and Xiao, Ying and others}, 146 | booktitle={Proceedings of the 28th ACM SIGKDD Conference on Knowledge Discovery and Data Mining}, 147 | pages={2842--2850}, 148 | year={2022} 149 | } 150 | @inproceedings{flach2013reducing, 151 | title={Reducing web latency: the virtue of gentle aggression}, 152 | author={Flach, Tobias and Dukkipati, Nandita and Terzis, Andreas and Raghavan, Barath and Cardwell, Neal and Cheng, Yuchung and Jain, Ankur and Hao, Shuai and Katz-Bassett, Ethan and Govindan, Ramesh}, 153 | booktitle={Proceedings of the ACM SIGCOMM 2013 conference on SIGCOMM}, 154 | pages={159--170}, 155 | year={2013} 156 | } 157 | @book{fowler2012patterns, 158 | title={Patterns of Enterprise Application Architecture: Pattern Enterpr Applica Arch}, 159 | author={Fowler, Martin}, 160 | year={2012}, 161 | publisher={Addison-Wesley} 162 | } 163 | 164 | @article{gerbrands1981relationships, 165 | title = {On the relationships between SVD, KLT and PCA}, 166 | author = {Gerbrands, Jan J}, 167 | year = 1981, 168 | journal = {Pattern recognition}, 169 | publisher = {Elsevier}, 170 | volume = 14, 171 | number = {1-6}, 172 | pages = {375--381} 173 | } 174 | @article{goldberg1992using, 175 | title = {Using collaborative filtering to weave an information tapestry}, 176 | author = {Goldberg, David and Nichols, David and Oki, Brian M and Terry, Douglas}, 177 | year = 1992, 178 | journal = {Communications of the ACM}, 179 | publisher = {ACM New York, NY, USA}, 180 | volume = 35, 181 | number = 12, 182 | pages = {61--70} 183 | } 184 | @article{goldberg2014word2vec, 185 | title={word2vec Explained: deriving Mikolov et al.'s negative-sampling word-embedding method}, 186 | author={Goldberg, Yoav and Levy, Omer}, 187 | journal={arXiv preprint arXiv:1402.3722}, 188 | year={2014} 189 | } 190 | @article{goldberg2017neural, 191 | title={Neural network methods for natural language processing}, 192 | author={Goldberg, Yoav}, 193 | journal={Synthesis lectures on human language technologies}, 194 | volume={10}, 195 | number={1}, 196 | pages={1--309}, 197 | year={2017}, 198 | publisher={Morgan \& Claypool Publishers} 199 | } 200 | @inproceedings{gomez2022learning, 201 | title={Learning Users’ Preferred Visual Styles in an Image Marketplace}, 202 | author={Gomez Bruballa, Raul and Burnham-King, Lauren and Sala, Alessandra}, 203 | booktitle={Proceedings of the 16th ACM Conference on Recommender Systems}, 204 | pages={466--468}, 205 | year={2022} 206 | } 207 | @book{gregg2014systems, 208 | title = {Systems performance: enterprise and the cloud}, 209 | author = {Gregg, Brendan}, 210 | year = 2014, 211 | publisher = {Pearson Education} 212 | } 213 | @inproceedings{grbovic2018real, 214 | title={Real-time personalization using embeddings for search ranking at airbnb}, 215 | author={Grbovic, Mihajlo and Cheng, Haibin}, 216 | booktitle={Proceedings of the 24th ACM SIGKDD international conference on knowledge discovery \& data mining}, 217 | pages={311--320}, 218 | year={2018} 219 | } 220 | 221 | @inproceedings{hansen2020contextual, 222 | title = {Contextual and sequential user embeddings for large-scale music recommendation}, 223 | author = {Hansen, Casper and Hansen, Christian and Maystre, Lucas and Mehrotra, Rishabh and Brost, Brian and Tomasi, Federico and Lalmas, Mounia}, 224 | year = 2020, 225 | booktitle = {Proceedings of the 14th ACM Conference on Recommender Systems}, 226 | pages = {53--62} 227 | } 228 | @inproceedings{he2017neural, 229 | title={Neural collaborative filtering}, 230 | author={He, Xiangnan and Liao, Lizi and Zhang, Hanwang and Nie, Liqiang and Hu, Xia and Chua, Tat-Seng}, 231 | booktitle={Proceedings of the 26th international conference on world wide web}, 232 | pages={173--182}, 233 | year={2017} 234 | } 235 | @article{hooker2021hardware, 236 | title={The hardware lottery}, 237 | author={Hooker, Sara}, 238 | journal={Communications of the ACM}, 239 | volume={64}, 240 | number={12}, 241 | pages={58--65}, 242 | year={2021}, 243 | publisher={ACM New York, NY, USA} 244 | } 245 | @inproceedings{hong2011predicting, 246 | title={Predicting popular messages in twitter}, 247 | author={Hong, Liangjie and Dan, Ovidiu and Davison, Brian D}, 248 | booktitle={Proceedings of the 20th international conference companion on World wide web}, 249 | pages={57--58}, 250 | year={2011} 251 | } 252 | @inproceedings{houle2010can, 253 | title = {Can shared-neighbor distances defeat the curse of dimensionality?}, 254 | author = {Houle, Michael E and Kriegel, Hans-Peter and Kr{\"o}ger, Peer and Schubert, Erich and Zimek, Arthur}, 255 | year = 2010, 256 | booktitle = {Scientific and Statistical Database Management: 22nd International Conference, SSDBM 2010, Heidelberg, Germany, June 30--July 2, 2010. Proceedings 22}, 257 | pages = {482--500}, 258 | organization = {Springer} 259 | } 260 | @book{jannach2010recommender, 261 | title = {Recommender systems: an introduction}, 262 | author = {Jannach, Dietmar and Zanker, Markus and Felfernig, Alexander and Friedrich, Gerhard}, 263 | year = 2010, 264 | publisher = {Cambridge University Press} 265 | } 266 | @inproceedings{jing2015visual, 267 | title = {Visual search at pinterest}, 268 | author = {Jing, Yushi and Liu, David and Kislyuk, Dmitry and Zhai, Andrew and Xu, Jiajing and Donahue, Jeff and Tavel, Sarah}, 269 | year = 2015, 270 | booktitle = {Proceedings of the 21th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining}, 271 | pages = {1889--1898} 272 | } 273 | @inproceedings{joachims2005text, 274 | title={Text categorization with support vector machines: Learning with many relevant features}, 275 | author={Joachims, Thorsten}, 276 | booktitle={Machine Learning: ECML-98: 10th European Conference on Machine Learning Chemnitz, Germany, April 21--23, 1998 Proceedings}, 277 | pages={137--142}, 278 | year={2005}, 279 | organization={Springer} 280 | } 281 | 282 | @misc{karpathy_2015, 283 | title = {The Unreasonable Effectiveness of Recurrent Neural Networks}, 284 | author = {Karpathy, Andrej}, 285 | year = 2015, 286 | month = {May}, 287 | journal = {The unreasonable effectiveness of recurrent neural networks}, 288 | url = {https://karpathy.github.io/2015/05/21/rnn-effectiveness/} 289 | } 290 | @book{klein2013coding, 291 | title={Coding the Matrix: Linear Algebra Through Applications to Computer Science}, 292 | author={Klein, P.N.}, 293 | isbn={9780615880990}, 294 | url={https://books.google.com/books?id=3AA4nwEACAAJ}, 295 | year={2013}, 296 | publisher={Newtonian Press} 297 | } 298 | @book{kleppmann2017designing, 299 | title={Designing data-intensive applications: The big ideas behind reliable, scalable, and maintainable systems}, 300 | author={Kleppmann, Martin}, 301 | year={2017}, 302 | publisher={" O'Reilly Media, Inc."} 303 | } 304 | @article{kvernadze2022two, 305 | title={Two Is Better Than One: Dual Embeddings for Complementary Product Recommendations}, 306 | author={Kvernadze, Giorgi and Sudyanti, Putu Ayu G and Subedi, Nishan and Hajiaghayi, Mohammad}, 307 | journal={arXiv preprint arXiv:2211.14982}, 308 | year={2022} 309 | } 310 | @book{kreps2014heart, 311 | title = {I heart logs: Event data, stream processing, and data integration}, 312 | author = {Kreps, Jay}, 313 | year = 2014, 314 | publisher = {" O'Reilly Media, Inc."} 315 | } 316 | @article{kreuzberger2022machine, 317 | title={Machine learning operations (mlops): Overview, definition, and architecture}, 318 | author={Kreuzberger, Dominik and K{\"u}hl, Niklas and Hirschl, Sebastian}, 319 | journal={arXiv preprint arXiv:2205.02302}, 320 | year={2022} 321 | } 322 | @book{lakshmanan2020machine, 323 | title = {Machine learning design patterns}, 324 | author = {Lakshmanan, Valliappa and Robinson, Sara and Munn, Michael}, 325 | year = 2020, 326 | publisher = {O'Reilly Media} 327 | } 328 | @article{lecun1998gradient, 329 | title={Gradient-based learning applied to document recognition}, 330 | author={LeCun, Yann and Bottou, L{\'e}on and Bengio, Yoshua and Haffner, Patrick}, 331 | journal={Proceedings of the IEEE}, 332 | volume={86}, 333 | number={11}, 334 | pages={2278--2324}, 335 | year={1998}, 336 | publisher={Ieee} 337 | } 338 | @article{lecun2015deep, 339 | title={Deep learning. nature, 521 (7553), 436-444}, 340 | author={LeCun, Yann and Bengio, Yoshua and Hinton, Geoffrey and others}, 341 | journal={Google Scholar Google Scholar Cross Ref Cross Ref}, 342 | pages={25}, 343 | year={2015} 344 | } 345 | @book{leskovec2020mining, 346 | title={Mining of massive data sets}, 347 | author={Leskovec, Jure and Rajaraman, Anand and Ullman, Jeffrey David}, 348 | year={2020}, 349 | publisher={Cambridge university press} 350 | } 351 | 352 | @article{levy2014neural, 353 | title={Neural word embedding as implicit matrix factorization}, 354 | author={Levy, Omer and Goldberg, Yoav}, 355 | journal={Advances in neural information processing systems}, 356 | volume={27}, 357 | year={2014} 358 | } 359 | @inproceedings{liu2022twice, 360 | title={Twice-Twitter Content Embeddings}, 361 | author={Liu, Xianjing and Golshan, Behzad and Leung, Kenny and Saini, Aman and Kulkarni, Vivek and Mollahosseini, Ali and Mo, Jeff}, 362 | booktitle = {CIKM 2022}, 363 | year={2022} 364 | } 365 | @inproceedings{lv2011documents, 366 | title = {When documents are very long, bm25 fails!}, 367 | author = {Lv, Yuanhua and Zhai, ChengXiang}, 368 | year = 2011, 369 | booktitle = {Proceedings of the 34th international ACM SIGIR conference on Research and development in Information Retrieval}, 370 | pages = {1103--1104} 371 | } 372 | @book{meadows2008thinking, 373 | title={Thinking in systems: A primer}, 374 | author={Meadows, Donella H}, 375 | year={2008}, 376 | publisher={chelsea green publishing} 377 | } 378 | @article{meil2023ai, 379 | title={AI in the Enterprise}, 380 | author={Meil, Doug}, 381 | journal={Communications of the ACM}, 382 | volume={66}, 383 | number={6}, 384 | pages={6--7}, 385 | year={2023}, 386 | publisher={ACM New York, NY, USA} 387 | } 388 | @article{mikolov2013efficient, 389 | title = {Efficient estimation of word representations in vector space}, 390 | author = {Mikolov, Tomas and Chen, Kai and Corrado, Greg and Dean, Jeffrey}, 391 | year = 2013, 392 | journal = {arXiv preprint arXiv:1301.3781} 393 | } 394 | @article{naseem2021comprehensive, 395 | title={A comprehensive survey on word representation models: From classical to state-of-the-art word representation language models}, 396 | author={Naseem, Usman and Razzak, Imran and Khan, Shah Khalid and Prasad, Mukesh}, 397 | journal={Transactions on Asian and Low-Resource Language Information Processing}, 398 | volume={20}, 399 | number={5}, 400 | pages={1--35}, 401 | year={2021}, 402 | publisher={ACM New York, NY} 403 | } 404 | @article{oancea2014gpgpu, 405 | title={Gpgpu computing}, 406 | author={Oancea, Bogdan and Andrei, Tudorel and Dragoescu, Raluca Mariana}, 407 | journal={arXiv preprint arXiv:1408.6923}, 408 | year={2014} 409 | } 410 | @inproceedings{pal2020pinnersage, 411 | title={Pinnersage: Multi-modal user embedding framework for recommendations at pinterest}, 412 | author={Pal, Aditya and Eksombatchai, Chantat and Zhou, Yitong and Zhao, Bo and Rosenberg, Charles and Leskovec, Jure}, 413 | booktitle={Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery \& Data Mining}, 414 | pages={2311--2320}, 415 | year={2020} 416 | } 417 | @book{rao2019natural, 418 | title={Natural language processing with PyTorch: build intelligent language applications using deep learning}, 419 | author={Rao, Delip and McMahan, Brian}, 420 | year={2019}, 421 | publisher={" O'Reilly Media, Inc."} 422 | } 423 | @article{raschka2018model, 424 | title={Model evaluation, model selection, and algorithm selection in machine learning}, 425 | author={Raschka, Sebastian}, 426 | journal={arXiv preprint arXiv:1811.12808}, 427 | year={2018} 428 | } 429 | @inproceedings{rendle2020neural, 430 | title={Neural collaborative filtering vs. matrix factorization revisited}, 431 | author={Rendle, Steffen and Krichene, Walid and Zhang, Li and Anderson, John}, 432 | booktitle={Proceedings of the 14th ACM Conference on Recommender Systems}, 433 | pages={240--248}, 434 | year={2020} 435 | } 436 | @article{rumelhart1986learning, 437 | title={Learning representations by back-propagating errors}, 438 | author={Rumelhart, David E and Hinton, Geoffrey E and Williams, Ronald J}, 439 | journal={nature}, 440 | volume={323}, 441 | number={6088}, 442 | pages={533--536}, 443 | year={1986}, 444 | publisher={Nature Publishing Group UK London} 445 | } 446 | @article{russakovsky2015imagenet, 447 | title={Imagenet large scale visual recognition challenge}, 448 | author={Russakovsky, Olga and Deng, Jia and Su, Hao and Krause, Jonathan and Satheesh, Sanjeev and Ma, Sean and Huang, Zhiheng and Karpathy, Andrej and Khosla, Aditya and Bernstein, Michael and others}, 449 | journal={International journal of computer vision}, 450 | volume={115}, 451 | pages={211--252}, 452 | year={2015}, 453 | publisher={Springer} 454 | } 455 | @inproceedings{rush2018annotated, 456 | title={The annotated transformer}, 457 | author={Rush, Alexander M}, 458 | booktitle={Proceedings of workshop for NLP open source software (NLP-OSS)}, 459 | pages={52--60}, 460 | year={2018} 461 | } 462 | @book{schutze2008introduction, 463 | title = {Introduction to information retrieval}, 464 | author = {Sch{\"u}tze, Hinrich and Manning, Christopher D and Raghavan, Prabhakar}, 465 | year = 2008, 466 | publisher = {Cambridge University Press Cambridge}, 467 | volume = 39 468 | } 469 | @misc{sculley2014machine, 470 | title={Machine learning: The high interest credit card of technical debt.(2014)}, 471 | author={Sculley, David and Holt, Gary and Golovin, Daniel and Davydov, Eugene and Phillips, Todd and Ebner, Dietmar and Chaudhary, Vinay and Young, Michael}, 472 | year={2014} 473 | } 474 | @book{seaver2022computing, 475 | title = {Computing Taste: Algorithms and the Makers of Music Recommendation}, 476 | author = {Seaver, Nick}, 477 | year = 2022, 478 | publisher = {University of Chicago Press} 479 | } 480 | @misc{Shabani_2023, title={How to train your own large language models}, url={https://blog.replit.com/llm-training}, journal={Replit Blog}, publisher={Replit}, author={Shabani, Reza}, year={2023}, month={Apr}} 481 | @article{sharir2020cost, 482 | title={The cost of training nlp models: A concise overview}, 483 | author={Sharir, Or and Peleg, Barak and Shoham, Yoav}, 484 | journal={arXiv preprint arXiv:2004.08900}, 485 | year={2020} 486 | } 487 | @article{shallue2018measuring, 488 | title={Measuring the effects of data parallelism on neural network training}, 489 | author={Shallue, Christopher J and Lee, Jaehoon and Antognini, Joseph and Sohl-Dickstein, Jascha and Frostig, Roy and Dahl, George E}, 490 | journal={arXiv preprint arXiv:1811.03600}, 491 | year={2018} 492 | } 493 | @article{shiebler2010making, 494 | title={Making machine learning easy with embeddings}, 495 | author={Shiebler, Dan and Tayal, Abhishek}, 496 | journal={SysML http://www.sysml.cc/doc/115.pdf}, 497 | year={2010} 498 | } 499 | @article{simhi2022interpreting, 500 | title={Interpreting Embedding Spaces by Conceptualization}, 501 | author={Simhi, Adi and Markovitch, Shaul}, 502 | journal={arXiv preprint arXiv:2209.00445}, 503 | year={2022} 504 | } 505 | @article{steck2021deep, 506 | title = {Deep learning for recommender systems: A Netflix case study}, 507 | author = {Steck, Harald and Baltrunas, Linas and Elahi, Ehtsham and Liang, Dawen and Raimond, Yves and Basilico, Justin}, 508 | year = 2021, 509 | journal = {AI Magazine}, 510 | volume = 42, 511 | number = 3, 512 | pages = {7--18} 513 | } 514 | @book{stevens2020deep, 515 | title = {Deep learning with PyTorch}, 516 | author = {Stevens, Eli and Antiga, Luca and Viehmann, Thomas}, 517 | year = 2020, 518 | publisher = {Manning Publications} 519 | } 520 | @inproceedings{svore2009machine, 521 | title = {A machine learning approach for improved BM25 retrieval}, 522 | author = {Svore, Krysta M and Burges, Christopher JC}, 523 | year = 2009, 524 | booktitle = {Proceedings of the 18th ACM conference on Information and knowledge management}, 525 | pages = {1811--1814} 526 | } 527 | @book{tunstall2022natural, 528 | title={Natural language processing with transformers}, 529 | author={Tunstall, Lewis and Von Werra, Leandro and Wolf, Thomas}, 530 | year={2022}, 531 | publisher={" O'Reilly Media, Inc."} 532 | } 533 | @inproceedings{ying2018graph, 534 | title={Graph convolutional neural networks for web-scale recommender systems}, 535 | author={Ying, Rex and He, Ruining and Chen, Kaifeng and Eksombatchai, Pong and Hamilton, William L and Leskovec, Jure}, 536 | booktitle={Proceedings of the 24th ACM SIGKDD international conference on knowledge discovery \& data mining}, 537 | pages={974--983}, 538 | year={2018} 539 | } 540 | @article{vaswani2017attention, 541 | title = {Attention is all you need}, 542 | author = {Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia}, 543 | year = 2017, 544 | journal = {Advances in neural information processing systems}, 545 | volume = 30 546 | } 547 | @article{Wang2020FromST, 548 | title={From static to dynamic word representations: a survey}, 549 | author={Yuxuan Wang and Yutai Hou and Wanxiang Che and Ting Liu}, 550 | journal={International Journal of Machine Learning and Cybernetics}, 551 | year={2020}, 552 | volume={11}, 553 | pages={1611-1630} 554 | } 555 | @article{wang2019evaluating, 556 | title={Evaluating word embedding models: Methods and experimental results}, 557 | author={Wang, Bin and Wang, Angela and Chen, Fenxiao and Wang, Yuncheng and Kuo, C-C Jay}, 558 | journal={APSIPA transactions on signal and information processing}, 559 | volume={8}, 560 | pages={e19}, 561 | year={2019}, 562 | publisher={Cambridge University Press} 563 | } 564 | 565 | @article{wewer2021updating, 566 | title={Updating embeddings for dynamic knowledge graphs}, 567 | author={Wewer, Christopher and Lemmerich, Florian and Cochez, Michael}, 568 | journal={arXiv preprint arXiv:2109.10896}, 569 | year={2021} 570 | } 571 | @article{wu2023bloomberggpt, 572 | title={Bloomberggpt: A large language model for finance}, 573 | author={Wu, Shijie and Irsoy, Ozan and Lu, Steven and Dabravolski, Vadim and Dredze, Mark and Gehrmann, Sebastian and Kambadur, Prabhanjan and Rosenberg, David and Mann, Gideon}, 574 | journal={arXiv preprint arXiv:2303.17564}, 575 | year={2023} 576 | } 577 | @article{xu2023multimodal, 578 | title={Multimodal learning with transformers: A survey}, 579 | author={Xu, Peng and Zhu, Xiatian and Clifton, David A}, 580 | journal={IEEE Transactions on Pattern Analysis and Machine Intelligence}, 581 | year={2023}, 582 | publisher={IEEE} 583 | } 584 | 585 | @article{zhang2019deep, 586 | title={Deep learning based recommender system: A survey and new perspectives}, 587 | author={Zhang, Shuai and Yao, Lina and Sun, Aixin and Tay, Yi}, 588 | journal={ACM computing surveys (CSUR)}, 589 | volume={52}, 590 | number={1}, 591 | pages={1--38}, 592 | year={2019}, 593 | publisher={ACM New York, NY, USA} 594 | } 595 | @article{zhang2020revisiting, 596 | title={Revisiting few-sample BERT fine-tuning}, 597 | author={Zhang, Tianyi and Wu, Felix and Katiyar, Arzoo and Weinberger, Kilian Q and Artzi, Yoav}, 598 | journal={arXiv preprint arXiv:2006.05987}, 599 | year={2020} 600 | } 601 | @article{zheng2017multi, 602 | title = {Multi-stakeholder recommendation: Applications and challenges}, 603 | author = {Zheng, Yong}, 604 | year = 2017, 605 | journal = {arXiv preprint arXiv:1707.08913} 606 | } 607 | @article{zinkevich2017rules, 608 | title = {Rules of machine learning: Best practices for ML engineering}, 609 | author = {Zinkevich, Martin}, 610 | year = 2017, 611 | journal = {URL: https://developers. google. com/machine-learning/guides/rules-of-ml} 612 | } 613 | -------------------------------------------------------------------------------- /embeddings.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/embeddings.pdf -------------------------------------------------------------------------------- /figures/app_flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/app_flow.png -------------------------------------------------------------------------------- /figures/bag_of_words.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/bag_of_words.png -------------------------------------------------------------------------------- /figures/deepmodel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/deepmodel.png -------------------------------------------------------------------------------- /figures/embed_flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/embed_flow.png -------------------------------------------------------------------------------- /figures/embedding_arc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/embedding_arc.png -------------------------------------------------------------------------------- /figures/embedding_pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/embedding_pipeline.png -------------------------------------------------------------------------------- /figures/embeddings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/embeddings.png -------------------------------------------------------------------------------- /figures/embeddings1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/embeddings1.png -------------------------------------------------------------------------------- /figures/embeddings2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/embeddings2.png -------------------------------------------------------------------------------- /figures/embeddings_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/embeddings_1.png -------------------------------------------------------------------------------- /figures/embeddings_arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/embeddings_arch.png -------------------------------------------------------------------------------- /figures/encoder_only.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/encoder_only.png -------------------------------------------------------------------------------- /figures/flutter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/flutter.png -------------------------------------------------------------------------------- /figures/function.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/function.png -------------------------------------------------------------------------------- /figures/function@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/function@2x.png -------------------------------------------------------------------------------- /figures/home_timeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/home_timeline.png -------------------------------------------------------------------------------- /figures/kandinsky.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/kandinsky.jpg -------------------------------------------------------------------------------- /figures/kandinsky.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/kandinsky.png -------------------------------------------------------------------------------- /figures/ml-flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/ml-flow.png -------------------------------------------------------------------------------- /figures/ml_flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/ml_flow.png -------------------------------------------------------------------------------- /figures/ml_system.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/ml_system.png -------------------------------------------------------------------------------- /figures/ml_workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/ml_workflow.png -------------------------------------------------------------------------------- /figures/mlnotml.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/mlnotml.png -------------------------------------------------------------------------------- /figures/mltree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/mltree.png -------------------------------------------------------------------------------- /figures/model_cycle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/model_cycle.png -------------------------------------------------------------------------------- /figures/paradigm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/paradigm.png -------------------------------------------------------------------------------- /figures/pca.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/pca.png -------------------------------------------------------------------------------- /figures/pinnersage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/pinnersage.png -------------------------------------------------------------------------------- /figures/pinnersage_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/pinnersage_model.png -------------------------------------------------------------------------------- /figures/pyramid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/pyramid.png -------------------------------------------------------------------------------- /figures/recsys.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/recsys.png -------------------------------------------------------------------------------- /figures/rules.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/rules.png -------------------------------------------------------------------------------- /figures/rules_ml.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/rules_ml.png -------------------------------------------------------------------------------- /figures/slack_suggestions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/slack_suggestions.png -------------------------------------------------------------------------------- /figures/solutions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/solutions.png -------------------------------------------------------------------------------- /figures/swe_flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/swe_flow.png -------------------------------------------------------------------------------- /figures/timeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/timeline.png -------------------------------------------------------------------------------- /figures/timeline2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/timeline2.png -------------------------------------------------------------------------------- /figures/transformer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/transformer.png -------------------------------------------------------------------------------- /figures/transformer_timeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/transformer_timeline.png -------------------------------------------------------------------------------- /figures/transformer_v.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/transformer_v.png -------------------------------------------------------------------------------- /figures/true_embeddings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/true_embeddings.png -------------------------------------------------------------------------------- /figures/twhin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/twhin.png -------------------------------------------------------------------------------- /figures/twice.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/twice.png -------------------------------------------------------------------------------- /figures/web_service.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/web_service.png -------------------------------------------------------------------------------- /figures/web_service_ml.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/web_service_ml.png -------------------------------------------------------------------------------- /figures/wideanddeep.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/wideanddeep.png -------------------------------------------------------------------------------- /figures/wideanddeep_arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/wideanddeep_arch.png -------------------------------------------------------------------------------- /figures/wideanddeep_ml_arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/wideanddeep_ml_arch.png -------------------------------------------------------------------------------- /figures/word2vec.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/word2vec.png -------------------------------------------------------------------------------- /figures/word2vecmodel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/word2vecmodel.png -------------------------------------------------------------------------------- /figures/youtube.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/youtube.png -------------------------------------------------------------------------------- /figures/youtube_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/youtube_model.png -------------------------------------------------------------------------------- /figures/youtube_recsys.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/figures/youtube_recsys.png -------------------------------------------------------------------------------- /kandinsky.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/what_are_embeddings/944077dfb29bfb44353c3f2177bd38cb04bc6b57/kandinsky.png -------------------------------------------------------------------------------- /notebooks/README.md: -------------------------------------------------------------------------------- 1 | # Code Implementations for the Embeddings Paper 2 | 3 | [To create a Jupyter environment from these requirements](https://vickiboykis.com/2020/02/18/lauching-a-jupyter-notebook-from-within-a-virtual-environment/), first [create a virtual environment](https://gifted-bohr-74bf66.netlify.app/). 4 | 5 | 1. `cd notebooks` 6 | 2. `virtualenv -p python3.10 embed` (you'll need to have Python 3.10 for this) 7 | 3. `source embed/bin/activate` 8 | 4. `pip install -r requirements.txt` 9 | 5. `python -m pip install ipykernel` # Manages kernel environments 10 | 6. `ipython kernel install --user --name=myproject` # myproject is the name of your environment 11 | 7. `jupyter notebook` (or `jupyter lab`) and pick `myproject` as the kernel for a new notebook 12 | 13 | 14 | -------------------------------------------------------------------------------- /notebooks/fig_22_ordinal.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "c5d19fd4", 6 | "metadata": {}, 7 | "source": [ 8 | "## Ordinal Encoding" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 4, 14 | "id": "83d4a565", 15 | "metadata": {}, 16 | "outputs": [ 17 | { 18 | "name": "stdout", 19 | "output_type": "stream", 20 | "text": [ 21 | "[['US'], ['UK'], ['NZ']]\n", 22 | "[[2.]\n", 23 | " [1.]\n", 24 | " [0.]]\n" 25 | ] 26 | } 27 | ], 28 | "source": [ 29 | "from sklearn.preprocessing import OrdinalEncoder\n", 30 | "\n", 31 | "\n", 32 | "data = [[\"US\"], [\"UK\"], [\"NZ\"]]\n", 33 | "print(data)\n", 34 | "\n", 35 | "\n", 36 | "# our label features\n", 37 | "encoder = OrdinalEncoder()\n", 38 | "result = encoder.fit_transform(data)\n", 39 | "print(result)" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 5, 45 | "id": "60bc735e", 46 | "metadata": {}, 47 | "outputs": [ 48 | { 49 | "name": "stdout", 50 | "output_type": "stream", 51 | "text": [ 52 | "[[2.]\n", 53 | " [1.]\n", 54 | " [0.]]\n" 55 | ] 56 | } 57 | ], 58 | "source": [ 59 | "encoder = OrdinalEncoder()\n", 60 | "result = encoder.fit_transform(data)\n", 61 | "print(result)" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "id": "00c87814", 67 | "metadata": {}, 68 | "source": [ 69 | "## One-Hot Encoding" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 7, 75 | "id": "13798a9d", 76 | "metadata": {}, 77 | "outputs": [ 78 | { 79 | "data": { 80 | "text/plain": [ 81 | "array([[0., 0., 1.],\n", 82 | " [0., 1., 0.],\n", 83 | " [1., 0., 0.]])" 84 | ] 85 | }, 86 | "execution_count": 7, 87 | "metadata": {}, 88 | "output_type": "execute_result" 89 | } 90 | ], 91 | "source": [ 92 | "from sklearn.preprocessing import OneHotEncoder\n", 93 | "import numpy as np\n", 94 | "\n", 95 | "enc = OneHotEncoder(handle_unknown=\"ignore\")\n", 96 | "data = np.asarray([[\"US\"], [\"UK\"], [\"NZ\"]])\n", 97 | "enc.fit(data)\n", 98 | "enc.categories_\n", 99 | "\n", 100 | "onehotlabels = enc.transform(data).toarray()\n", 101 | "onehotlabels" 102 | ] 103 | }, 104 | { 105 | "cell_type": "markdown", 106 | "id": "3675fff7", 107 | "metadata": {}, 108 | "source": [ 109 | "## Bag of Words\n", 110 | "[See more here](https://okan.cloud/posts/2021-04-08-text-vectorization-using-python-term-document-matrix/)" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": 22, 116 | "id": "92d35261", 117 | "metadata": {}, 118 | "outputs": [], 119 | "source": [ 120 | "from sklearn.feature_extraction.text import CountVectorizer\n", 121 | "import pandas as pd\n", 122 | "\n", 123 | "# Input data\n", 124 | "flits = [\n", 125 | " \"Hold fast to dreams, for if dreams die, life is a broken-winged bird that cannot fly.\",\n", 126 | " \"No bird soars too high if he soars with his own wings.\",\n", 127 | " \"A bird does not sing because it has an answer, it sings because it has a song.\",\n", 128 | "]" 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": 23, 134 | "id": "28e41cf3", 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "flits = [\n", 139 | " \"Hold fast to dreams, for if dreams die, life is a broken-winged bird that cannot fly.\",\n", 140 | " \"No bird soars too high if he soars with his own wings.\",\n", 141 | " \"A bird does not sing because it has an answer, it sings because it has a song.\",\n", 142 | "]" 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": 35, 148 | "id": "c7e95e19", 149 | "metadata": {}, 150 | "outputs": [], 151 | "source": [ 152 | "vect = CountVectorizer(binary=True)\n", 153 | "vects = vect.fit_transform(flits)" 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": 41, 159 | "id": "2ff5113d", 160 | "metadata": {}, 161 | "outputs": [], 162 | "source": [ 163 | "td = pd.DataFrame(vects.todense()).iloc[:5]\n", 164 | "td.columns = vect.get_feature_names_out()\n", 165 | "count_vect_df = td\n", 166 | "term_document_matrix = td.T\n", 167 | "term_document_matrix.columns = [\"flit_\" + str(i) for i in range(1, 4)]\n", 168 | "term_document_matrix[\"total_count\"] = term_document_matrix.sum(axis=1)" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": 42, 174 | "id": "7901c6ba", 175 | "metadata": {}, 176 | "outputs": [ 177 | { 178 | "name": "stdout", 179 | "output_type": "stream", 180 | "text": [ 181 | " flit_1 flit_2 flit_3\n", 182 | "an 0 0 1\n", 183 | "answer 0 0 1\n", 184 | "because 0 0 1\n", 185 | "bird 1 1 1\n", 186 | "broken 1 0 0\n", 187 | "cannot 1 0 0\n", 188 | "die 1 0 0\n", 189 | "does 0 0 1\n", 190 | "dreams 1 0 0\n", 191 | "fast 1 0 0\n" 192 | ] 193 | } 194 | ], 195 | "source": [ 196 | "print(term_document_matrix.drop(columns=[\"total_count\"]).head(10))" 197 | ] 198 | }, 199 | { 200 | "cell_type": "code", 201 | "execution_count": 17, 202 | "id": "6151a0d9", 203 | "metadata": {}, 204 | "outputs": [ 205 | { 206 | "data": { 207 | "text/html": [ 208 | "
\n", 209 | "\n", 222 | "\n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | " \n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | " \n", 250 | " \n", 251 | " \n", 252 | " \n", 253 | " \n", 254 | " \n", 255 | " \n", 256 | " \n", 257 | " \n", 258 | " \n", 259 | " \n", 260 | " \n", 261 | " \n", 262 | " \n", 263 | " \n", 264 | " \n", 265 | " \n", 266 | " \n", 267 | " \n", 268 | " \n", 269 | " \n", 270 | " \n", 271 | " \n", 272 | " \n", 273 | " \n", 274 | " \n", 275 | " \n", 276 | " \n", 277 | " \n", 278 | " \n", 279 | " \n", 280 | " \n", 281 | " \n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | " \n", 289 | " \n", 290 | " \n", 291 | " \n", 292 | " \n", 293 | " \n", 294 | " \n", 295 | " \n", 296 | " \n", 297 | " \n", 298 | " \n", 299 | " \n", 300 | " \n", 301 | " \n", 302 | " \n", 303 | " \n", 304 | " \n", 305 | " \n", 306 | " \n", 307 | " \n", 308 | " \n", 309 | " \n", 310 | " \n", 311 | " \n", 312 | " \n", 313 | " \n", 314 | " \n", 315 | " \n", 316 | " \n", 317 | " \n", 318 | " \n", 319 | " \n", 320 | " \n", 321 | " \n", 322 | " \n", 323 | "
ananswerbecausebirdbrokencannotdiedoesdreamsfast...singsingssoarssongthattotoowingedwingswith
00001111011...0000110100
10001000000...0010001011
21111000100...1101000000
\n", 324 | "

3 rows × 34 columns

\n", 325 | "
" 326 | ], 327 | "text/plain": [ 328 | " an answer because bird broken cannot die does dreams fast ... \n", 329 | "0 0 0 0 1 1 1 1 0 1 1 ... \\\n", 330 | "1 0 0 0 1 0 0 0 0 0 0 ... \n", 331 | "2 1 1 1 1 0 0 0 1 0 0 ... \n", 332 | "\n", 333 | " sing sings soars song that to too winged wings with \n", 334 | "0 0 0 0 0 1 1 0 1 0 0 \n", 335 | "1 0 0 1 0 0 0 1 0 1 1 \n", 336 | "2 1 1 0 1 0 0 0 0 0 0 \n", 337 | "\n", 338 | "[3 rows x 34 columns]" 339 | ] 340 | }, 341 | "execution_count": 17, 342 | "metadata": {}, 343 | "output_type": "execute_result" 344 | } 345 | ], 346 | "source": [ 347 | "count_vect_df" 348 | ] 349 | }, 350 | { 351 | "cell_type": "code", 352 | "execution_count": 18, 353 | "id": "bca7ca84", 354 | "metadata": {}, 355 | "outputs": [ 356 | { 357 | "data": { 358 | "text/html": [ 359 | "
\n", 360 | "\n", 373 | "\n", 374 | " \n", 375 | " \n", 376 | " \n", 377 | " \n", 378 | " \n", 379 | " \n", 380 | " \n", 381 | " \n", 382 | " \n", 383 | " \n", 384 | " \n", 385 | " \n", 386 | " \n", 387 | " \n", 388 | " \n", 389 | " \n", 390 | " \n", 391 | " \n", 392 | " \n", 393 | " \n", 394 | " \n", 395 | " \n", 396 | " \n", 397 | " \n", 398 | " \n", 399 | " \n", 400 | " \n", 401 | " \n", 402 | " \n", 403 | " \n", 404 | " \n", 405 | " \n", 406 | " \n", 407 | " \n", 408 | " \n", 409 | " \n", 410 | " \n", 411 | " \n", 412 | " \n", 413 | " \n", 414 | " \n", 415 | " \n", 416 | " \n", 417 | " \n", 418 | " \n", 419 | " \n", 420 | " \n", 421 | " \n", 422 | " \n", 423 | " \n", 424 | " \n", 425 | " \n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | " \n", 457 | " \n", 458 | " \n", 459 | " \n", 460 | " \n", 461 | " \n", 462 | " \n", 463 | " \n", 464 | " \n", 465 | " \n", 466 | " \n", 467 | " \n", 468 | " \n", 469 | " \n", 470 | " \n", 471 | " \n", 472 | " \n", 473 | " \n", 474 | "
0ananswerbecausebirdbrokencannotdiedoesdreams...singsingssoarssongthattotoowingedwingswith
0Hold fast to dreams, for if dreams die, life i...000111101...0000110100
1No bird soars too high if he soars with his ow...000100000...0010001011
2A bird does not sing because it has an answer,...111100010...1101000000
\n", 475 | "

3 rows × 35 columns

\n", 476 | "
" 477 | ], 478 | "text/plain": [ 479 | " 0 an answer because \n", 480 | "0 Hold fast to dreams, for if dreams die, life i... 0 0 0 \\\n", 481 | "1 No bird soars too high if he soars with his ow... 0 0 0 \n", 482 | "2 A bird does not sing because it has an answer,... 1 1 1 \n", 483 | "\n", 484 | " bird broken cannot die does dreams ... sing sings soars song \n", 485 | "0 1 1 1 1 0 1 ... 0 0 0 0 \\\n", 486 | "1 1 0 0 0 0 0 ... 0 0 1 0 \n", 487 | "2 1 0 0 0 1 0 ... 1 1 0 1 \n", 488 | "\n", 489 | " that to too winged wings with \n", 490 | "0 1 1 0 1 0 0 \n", 491 | "1 0 0 1 0 1 1 \n", 492 | "2 0 0 0 0 0 0 \n", 493 | "\n", 494 | "[3 rows x 35 columns]" 495 | ] 496 | }, 497 | "execution_count": 18, 498 | "metadata": {}, 499 | "output_type": "execute_result" 500 | } 501 | ], 502 | "source": [ 503 | "pd.concat([doc, count_vect_df], axis=1)" 504 | ] 505 | } 506 | ], 507 | "metadata": { 508 | "kernelspec": { 509 | "display_name": "embed", 510 | "language": "python", 511 | "name": "embed" 512 | }, 513 | "language_info": { 514 | "codemirror_mode": { 515 | "name": "ipython", 516 | "version": 3 517 | }, 518 | "file_extension": ".py", 519 | "mimetype": "text/x-python", 520 | "name": "python", 521 | "nbconvert_exporter": "python", 522 | "pygments_lexer": "ipython3", 523 | "version": "3.10.8" 524 | } 525 | }, 526 | "nbformat": 4, 527 | "nbformat_minor": 5 528 | } 529 | -------------------------------------------------------------------------------- /notebooks/fig_24_tf_idf_from_scratch.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 30, 6 | "id": "ba4628a8", 7 | "metadata": {}, 8 | "outputs": [ 9 | { 10 | "data": { 11 | "text/html": [ 12 | "\n", 13 | " \n", 26 | " " 27 | ], 28 | "text/plain": [ 29 | "" 30 | ] 31 | }, 32 | "metadata": {}, 33 | "output_type": "display_data" 34 | } 35 | ], 36 | "source": [ 37 | "# Lint notebook\n", 38 | "%load_ext jupyter_black" 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "id": "6f8a3a00", 44 | "metadata": {}, 45 | "source": [ 46 | "For more examples and intutition see [here](https://www.kaggle.com/code/yassinehamdaoui1/creating-tf-idf-model-from-scratch) and [here](https://www.capitalone.com/tech/machine-learning/understanding-tf-idf/) and [here](https://melaniewalsh.github.io/Intro-Cultural-Analytics/05-Text-Analysis/03-TF-IDF-Scikit-Learn.html)" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": 55, 52 | "id": "7ab14f29", 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "import math\n", 57 | "\n", 58 | "# Process documents into individual words\n", 59 | "document_a = [\n", 60 | " \"Hold\",\n", 61 | " \"fast\",\n", 62 | " \"to\",\n", 63 | " \"dreams\",\n", 64 | " \"for\",\n", 65 | " \"if\",\n", 66 | " \"dreams\",\n", 67 | " \"die\",\n", 68 | " \"life\",\n", 69 | " \"is\",\n", 70 | " \"a\",\n", 71 | " \"broken-winged\",\n", 72 | " \"bird\",\n", 73 | " \"that\",\n", 74 | " \"cannot\",\n", 75 | " \"fly\",\n", 76 | "]\n", 77 | "document_b = [\n", 78 | " \"No\",\n", 79 | " \"bird\",\n", 80 | " \"soars\",\n", 81 | " \"too\",\n", 82 | " \"high\",\n", 83 | " \"if\",\n", 84 | " \"he\",\n", 85 | " \"soars\",\n", 86 | " \"with\",\n", 87 | " \"his\",\n", 88 | " \"own\",\n", 89 | " \"wings\",\n", 90 | "]\n", 91 | "\n", 92 | "# Total set of words\n", 93 | "total_corpus = set(document_a).union(set(document_b))" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 60, 99 | "id": "149797a5", 100 | "metadata": {}, 101 | "outputs": [ 102 | { 103 | "data": { 104 | "text/html": [ 105 | "
\n", 106 | "\n", 119 | "\n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | " \n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | "
01
a10
dreams20
No01
Hold10
for10
die10
he01
fast10
his01
life10
that10
wings01
to10
with01
cannot10
fly10
soars02
is10
bird11
if11
own01
high01
broken-winged10
too01
\n", 250 | "
" 251 | ], 252 | "text/plain": [ 253 | " 0 1\n", 254 | "a 1 0\n", 255 | "dreams 2 0\n", 256 | "No 0 1\n", 257 | "Hold 1 0\n", 258 | "for 1 0\n", 259 | "die 1 0\n", 260 | "he 0 1\n", 261 | "fast 1 0\n", 262 | "his 0 1\n", 263 | "life 1 0\n", 264 | "that 1 0\n", 265 | "wings 0 1\n", 266 | "to 1 0\n", 267 | "with 0 1\n", 268 | "cannot 1 0\n", 269 | "fly 1 0\n", 270 | "soars 0 2\n", 271 | "is 1 0\n", 272 | "bird 1 1\n", 273 | "if 1 1\n", 274 | "own 0 1\n", 275 | "high 0 1\n", 276 | "broken-winged 1 0\n", 277 | "too 0 1" 278 | ] 279 | }, 280 | "execution_count": 60, 281 | "metadata": {}, 282 | "output_type": "execute_result" 283 | } 284 | ], 285 | "source": [ 286 | "# Simple frequency counts of words per document by initializing a dict\n", 287 | "import pandas as pd\n", 288 | "dict_a = dict.fromkeys(total_corpus, 0)\n", 289 | "dict_b = dict.fromkeys(total_corpus, 0)\n", 290 | "\n", 291 | "for word in document_a:\n", 292 | " dict_a[word] += 1\n", 293 | "\n", 294 | "for word in document_b:\n", 295 | " dict_b[word] += 1\n", 296 | "\n", 297 | "frequency = pd.DataFrame([dict_a, dict_b])\n", 298 | "frequency.T" 299 | ] 300 | }, 301 | { 302 | "cell_type": "code", 303 | "execution_count": 57, 304 | "id": "4408b987", 305 | "metadata": {}, 306 | "outputs": [ 307 | { 308 | "data": { 309 | "text/html": [ 310 | "
\n", 311 | "\n", 324 | "\n", 325 | " \n", 326 | " \n", 327 | " \n", 328 | " \n", 329 | " \n", 330 | " \n", 331 | " \n", 332 | " \n", 333 | " \n", 334 | " \n", 335 | " \n", 336 | " \n", 337 | " \n", 338 | " \n", 339 | " \n", 340 | " \n", 341 | " \n", 342 | " \n", 343 | " \n", 344 | " \n", 345 | " \n", 346 | " \n", 347 | " \n", 348 | " \n", 349 | " \n", 350 | " \n", 351 | " \n", 352 | " \n", 353 | " \n", 354 | " \n", 355 | " \n", 356 | " \n", 357 | " \n", 358 | " \n", 359 | " \n", 360 | " \n", 361 | " \n", 362 | " \n", 363 | " \n", 364 | " \n", 365 | " \n", 366 | " \n", 367 | " \n", 368 | " \n", 369 | " \n", 370 | " \n", 371 | " \n", 372 | " \n", 373 | " \n", 374 | " \n", 375 | " \n", 376 | " \n", 377 | " \n", 378 | " \n", 379 | " \n", 380 | " \n", 381 | " \n", 382 | " \n", 383 | " \n", 384 | " \n", 385 | " \n", 386 | " \n", 387 | " \n", 388 | " \n", 389 | " \n", 390 | " \n", 391 | " \n", 392 | " \n", 393 | " \n", 394 | " \n", 395 | " \n", 396 | " \n", 397 | " \n", 398 | " \n", 399 | " \n", 400 | " \n", 401 | " \n", 402 | " \n", 403 | " \n", 404 | " \n", 405 | " \n", 406 | " \n", 407 | " \n", 408 | " \n", 409 | " \n", 410 | " \n", 411 | " \n", 412 | " \n", 413 | " \n", 414 | " \n", 415 | " \n", 416 | " \n", 417 | " \n", 418 | " \n", 419 | " \n", 420 | " \n", 421 | " \n", 422 | " \n", 423 | " \n", 424 | " \n", 425 | " \n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | "
01
a0.0188140.000000
dreams0.0376290.000000
No0.0000000.025086
Hold0.0188140.000000
for0.0188140.000000
die0.0188140.000000
he0.0000000.025086
fast0.0188140.000000
his0.0000000.025086
life0.0188140.000000
that0.0188140.000000
wings0.0000000.025086
to0.0188140.000000
with0.0000000.025086
cannot0.0188140.000000
fly0.0188140.000000
soars0.0000000.050172
is0.0188140.000000
bird0.0188140.025086
if0.0188140.025086
own0.0000000.025086
high0.0000000.025086
broken-winged0.0188140.000000
too0.0000000.025086
\n", 455 | "
" 456 | ], 457 | "text/plain": [ 458 | " 0 1\n", 459 | "a 0.018814 0.000000\n", 460 | "dreams 0.037629 0.000000\n", 461 | "No 0.000000 0.025086\n", 462 | "Hold 0.018814 0.000000\n", 463 | "for 0.018814 0.000000\n", 464 | "die 0.018814 0.000000\n", 465 | "he 0.000000 0.025086\n", 466 | "fast 0.018814 0.000000\n", 467 | "his 0.000000 0.025086\n", 468 | "life 0.018814 0.000000\n", 469 | "that 0.018814 0.000000\n", 470 | "wings 0.000000 0.025086\n", 471 | "to 0.018814 0.000000\n", 472 | "with 0.000000 0.025086\n", 473 | "cannot 0.018814 0.000000\n", 474 | "fly 0.018814 0.000000\n", 475 | "soars 0.000000 0.050172\n", 476 | "is 0.018814 0.000000\n", 477 | "bird 0.018814 0.025086\n", 478 | "if 0.018814 0.025086\n", 479 | "own 0.000000 0.025086\n", 480 | "high 0.000000 0.025086\n", 481 | "broken-winged 0.018814 0.000000\n", 482 | "too 0.000000 0.025086" 483 | ] 484 | }, 485 | "execution_count": 57, 486 | "metadata": {}, 487 | "output_type": "execute_result" 488 | } 489 | ], 490 | "source": [ 491 | "def tf(doc_dict: dict, doc_elements: list[str]) -> dict:\n", 492 | " \"\"\"Term frequency of a word in a document over total words in document\"\"\"\n", 493 | " tf_dict = {}\n", 494 | " corpus_count = len(doc_elements)\n", 495 | " \n", 496 | " for word, count in doc_dict.items():\n", 497 | " tf_dict[word] = count / float(corpus_count)\n", 498 | " \n", 499 | " return tf_dict\n", 500 | "\n", 501 | "\n", 502 | "def idf(doc_list: list[dict[str, int]]) -> dict:\n", 503 | " \"\"\"Given N documents (doc_list), the number of documents in which the term appears per a term\"\"\"\n", 504 | " idf_dict = {}\n", 505 | " N = len(doc_list)\n", 506 | "\n", 507 | " idf_dict = dict.fromkeys(doc_list[0].keys(), 0)\n", 508 | "\n", 509 | " for word in idf_dict.keys():\n", 510 | " idf_dict[word] = sum(doc[word] > 0 for doc in doc_list)\n", 511 | "\n", 512 | " for word, val in idf_dict.items():\n", 513 | " idf_dict[word] = math.log10((N + 1.0) / (val + 1.0))\n", 514 | "\n", 515 | " return idf_dict\n", 516 | "\n", 517 | "# All inverse document frequencies for all words \n", 518 | "idfs = idf([dict_a, dict_b])\n", 519 | "\n", 520 | "\n", 521 | "def tfidf(doc_elements: dict[str, int], idfs: dict[str, int])-> dict:\n", 522 | " \"\"\"TF * IDF per word given a single word in a single document and number of docs the term appears in\"\"\"\n", 523 | " tfidf_dict = {}\n", 524 | " \n", 525 | " for word, val in doc_elements.items():\n", 526 | " tfidf_dict[word] = val * idfs[word]\n", 527 | "\n", 528 | " return tfidf_dict\n", 529 | "\n", 530 | "# Calculate the term frequency for each document individually\n", 531 | "tf_a = tf(dict_a, document_a)\n", 532 | "tf_b = tf(dict_b, document_b)\n", 533 | "\n", 534 | "# Calculate the inverse document frequency given each term frequency\n", 535 | "tfidf_a = tfidf(tf_a, idfs)\n", 536 | "tfidf_b = tfidf(tf_b, idfs)\n", 537 | "\n", 538 | "# Return weight of each word in each document wrt to the total corpus\n", 539 | "document_tfidf = pd.DataFrame([tfidf_a, tfidf_b])\n", 540 | "document_tfidf.T" 541 | ] 542 | }, 543 | { 544 | "cell_type": "code", 545 | "execution_count": 6, 546 | "id": "d368b9a6", 547 | "metadata": {}, 548 | "outputs": [], 549 | "source": [ 550 | "from sklearn.feature_extraction.text import TfidfVectorizer\n", 551 | "\n", 552 | "corpus = [\n", 553 | " \"Hold fast to dreams, for if dreams die, life is a broken-winged bird that cannot fly.\",\n", 554 | " \"No bird soars too high if he soars with his own wings.\",\n", 555 | "]\n", 556 | "\n", 557 | "text_titles = [\"quote_langstonhughes\", \"quote_william_blake\"]\n", 558 | "\n", 559 | "vectorizer = TfidfVectorizer()\n", 560 | "vector = vectorizer.fit_transform(corpus)\n", 561 | "dict(zip(vectorizer.get_feature_names_out(), vector.toarray()[0]))\n", 562 | "\n", 563 | "tfidf_df = pd.DataFrame(vector.toarray(), index=text_titles, columns=vectorizer.get_feature_names_out())" 564 | ] 565 | }, 566 | { 567 | "cell_type": "code", 568 | "execution_count": 8, 569 | "id": "e74435ab", 570 | "metadata": {}, 571 | "outputs": [], 572 | "source": [ 573 | "tfidf_df.loc['00_Document Frequency'] = (tfidf_df > 0).sum()" 574 | ] 575 | }, 576 | { 577 | "cell_type": "code", 578 | "execution_count": 59, 579 | "id": "224355e4", 580 | "metadata": {}, 581 | "outputs": [ 582 | { 583 | "data": { 584 | "text/html": [ 585 | "
\n", 586 | "\n", 599 | "\n", 600 | " \n", 601 | " \n", 602 | " \n", 603 | " \n", 604 | " \n", 605 | " \n", 606 | " \n", 607 | " \n", 608 | " \n", 609 | " \n", 610 | " \n", 611 | " \n", 612 | " \n", 613 | " \n", 614 | " \n", 615 | " \n", 616 | " \n", 617 | " \n", 618 | " \n", 619 | " \n", 620 | " \n", 621 | " \n", 622 | " \n", 623 | " \n", 624 | " \n", 625 | " \n", 626 | " \n", 627 | " \n", 628 | " \n", 629 | " \n", 630 | " \n", 631 | " \n", 632 | " \n", 633 | " \n", 634 | " \n", 635 | " \n", 636 | " \n", 637 | " \n", 638 | " \n", 639 | " \n", 640 | " \n", 641 | " \n", 642 | " \n", 643 | " \n", 644 | " \n", 645 | " \n", 646 | " \n", 647 | " \n", 648 | " \n", 649 | " \n", 650 | " \n", 651 | " \n", 652 | " \n", 653 | " \n", 654 | " \n", 655 | " \n", 656 | " \n", 657 | " \n", 658 | " \n", 659 | " \n", 660 | " \n", 661 | " \n", 662 | " \n", 663 | " \n", 664 | " \n", 665 | " \n", 666 | " \n", 667 | " \n", 668 | " \n", 669 | " \n", 670 | " \n", 671 | " \n", 672 | " \n", 673 | " \n", 674 | " \n", 675 | " \n", 676 | " \n", 677 | " \n", 678 | " \n", 679 | " \n", 680 | " \n", 681 | " \n", 682 | " \n", 683 | " \n", 684 | " \n", 685 | " \n", 686 | " \n", 687 | " \n", 688 | " \n", 689 | " \n", 690 | " \n", 691 | " \n", 692 | " \n", 693 | " \n", 694 | " \n", 695 | " \n", 696 | " \n", 697 | " \n", 698 | " \n", 699 | " \n", 700 | " \n", 701 | " \n", 702 | " \n", 703 | " \n", 704 | " \n", 705 | " \n", 706 | " \n", 707 | " \n", 708 | " \n", 709 | " \n", 710 | " \n", 711 | " \n", 712 | " \n", 713 | " \n", 714 | " \n", 715 | " \n", 716 | " \n", 717 | " \n", 718 | " \n", 719 | " \n", 720 | " \n", 721 | " \n", 722 | " \n", 723 | " \n", 724 | " \n", 725 | " \n", 726 | " \n", 727 | " \n", 728 | " \n", 729 | " \n", 730 | " \n", 731 | " \n", 732 | " \n", 733 | " \n", 734 | " \n", 735 | " \n", 736 | " \n", 737 | " \n", 738 | " \n", 739 | " \n", 740 | " \n", 741 | " \n", 742 | " \n", 743 | " \n", 744 | " \n", 745 | " \n", 746 | " \n", 747 | " \n", 748 | " \n", 749 | " \n", 750 | " \n", 751 | " \n", 752 | " \n", 753 | " \n", 754 | "
dreams_langstonhughesquote_william_blake00_Document Frequency
bird0.1725030.1972422.0
broken0.2424470.0000001.0
cannot0.2424470.0000001.0
die0.2424470.0000001.0
dreams0.4848930.0000001.0
fast0.2424470.0000001.0
fly0.2424470.0000001.0
for0.2424470.0000001.0
he0.0000000.2772171.0
high0.0000000.2772171.0
his0.0000000.2772171.0
hold0.2424470.0000001.0
if0.1725030.1972422.0
is0.2424470.0000001.0
life0.2424470.0000001.0
no0.0000000.2772171.0
own0.0000000.2772171.0
soars0.0000000.5544341.0
that0.2424470.0000001.0
to0.2424470.0000001.0
too0.0000000.2772171.0
winged0.2424470.0000001.0
wings0.0000000.2772171.0
with0.0000000.2772171.0
\n", 755 | "
" 756 | ], 757 | "text/plain": [ 758 | " quote_langstonhughes quote_william_blake 00_Document Frequency\n", 759 | "bird 0.172503 0.197242 2.0\n", 760 | "broken 0.242447 0.000000 1.0\n", 761 | "cannot 0.242447 0.000000 1.0\n", 762 | "die 0.242447 0.000000 1.0\n", 763 | "dreams 0.484893 0.000000 1.0\n", 764 | "fast 0.242447 0.000000 1.0\n", 765 | "fly 0.242447 0.000000 1.0\n", 766 | "for 0.242447 0.000000 1.0\n", 767 | "he 0.000000 0.277217 1.0\n", 768 | "high 0.000000 0.277217 1.0\n", 769 | "his 0.000000 0.277217 1.0\n", 770 | "hold 0.242447 0.000000 1.0\n", 771 | "if 0.172503 0.197242 2.0\n", 772 | "is 0.242447 0.000000 1.0\n", 773 | "life 0.242447 0.000000 1.0\n", 774 | "no 0.000000 0.277217 1.0\n", 775 | "own 0.000000 0.277217 1.0\n", 776 | "soars 0.000000 0.554434 1.0\n", 777 | "that 0.242447 0.000000 1.0\n", 778 | "to 0.242447 0.000000 1.0\n", 779 | "too 0.000000 0.277217 1.0\n", 780 | "winged 0.242447 0.000000 1.0\n", 781 | "wings 0.000000 0.277217 1.0\n", 782 | "with 0.000000 0.277217 1.0" 783 | ] 784 | }, 785 | "execution_count": 59, 786 | "metadata": {}, 787 | "output_type": "execute_result" 788 | } 789 | ], 790 | "source": [ 791 | "tfidf_df.T" 792 | ] 793 | } 794 | ], 795 | "metadata": { 796 | "celltoolbar": "Raw Cell Format", 797 | "kernelspec": { 798 | "display_name": "embed", 799 | "language": "python", 800 | "name": "embed" 801 | }, 802 | "language_info": { 803 | "codemirror_mode": { 804 | "name": "ipython", 805 | "version": 3 806 | }, 807 | "file_extension": ".py", 808 | "mimetype": "text/x-python", 809 | "name": "python", 810 | "nbconvert_exporter": "python", 811 | "pygments_lexer": "ipython3", 812 | "version": "3.10.8" 813 | } 814 | }, 815 | "nbformat": 4, 816 | "nbformat_minor": 5 817 | } 818 | -------------------------------------------------------------------------------- /notebooks/fig_29_cos_sim_from_scratch.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "871c1ae1", 7 | "metadata": {}, 8 | "outputs": [ 9 | { 10 | "data": { 11 | "text/html": [ 12 | "\n", 13 | " \n", 26 | " " 27 | ], 28 | "text/plain": [ 29 | "" 30 | ] 31 | }, 32 | "metadata": {}, 33 | "output_type": "display_data" 34 | } 35 | ], 36 | "source": [ 37 | "%load_ext jupyter_black" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": 1, 43 | "id": "21275a04", 44 | "metadata": {}, 45 | "outputs": [ 46 | { 47 | "name": "stdout", 48 | "output_type": "stream", 49 | "text": [ 50 | "0.9544074144996451\n" 51 | ] 52 | } 53 | ], 54 | "source": [ 55 | "v1 = [0, 3, 4, 5, 6]\n", 56 | "v2 = [4, 5, 6, 7, 8]\n", 57 | "\n", 58 | "\n", 59 | "def dot(v1, v2):\n", 60 | " dot_product = sum((a * b) for a, b in zip(v1, v2))\n", 61 | " return dot_product\n", 62 | "\n", 63 | "\n", 64 | "def cosine_similarity(v1, v2):\n", 65 | " \"\"\"\n", 66 | " (v1 dot v2)/||v1|| *||v2||)\n", 67 | " \"\"\"\n", 68 | " products = dot(v1, v2)\n", 69 | " denominator = (dot(v1, v1) ** 0.5) * (dot(v2, v2) ** 0.5)\n", 70 | " similarity = products / denominator\n", 71 | " return similarity\n", 72 | "\n", 73 | "\n", 74 | "print(cosine_similarity(v1, v2))" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 2, 80 | "id": "74f41cf3", 81 | "metadata": {}, 82 | "outputs": [ 83 | { 84 | "data": { 85 | "text/plain": [ 86 | "array([[0.95440741]])" 87 | ] 88 | }, 89 | "execution_count": 2, 90 | "metadata": {}, 91 | "output_type": "execute_result" 92 | } 93 | ], 94 | "source": [ 95 | "from sklearn.metrics import pairwise\n", 96 | "\n", 97 | "v1 = [0, 3, 4, 5, 6]\n", 98 | "v2 = [4, 5, 6, 7, 8]\n", 99 | "\n", 100 | "# need to be in numpy data format\n", 101 | "pairwise.cosine_similarity([v1], [v2])\n", 102 | "# array([[0.95440741]])" 103 | ] 104 | } 105 | ], 106 | "metadata": { 107 | "kernelspec": { 108 | "display_name": "embed", 109 | "language": "python", 110 | "name": "embed" 111 | }, 112 | "language_info": { 113 | "codemirror_mode": { 114 | "name": "ipython", 115 | "version": 3 116 | }, 117 | "file_extension": ".py", 118 | "mimetype": "text/x-python", 119 | "name": "python", 120 | "nbconvert_exporter": "python", 121 | "pygments_lexer": "ipython3", 122 | "version": "3.10.8" 123 | } 124 | }, 125 | "nbformat": 4, 126 | "nbformat_minor": 5 127 | } 128 | -------------------------------------------------------------------------------- /notebooks/fig_2_embeddings_papers.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "9705ffc4", 6 | "metadata": {}, 7 | "source": [ 8 | "Analyzing papers about \"embeddings\" from CS Arxiv over the past 5 years to see if their occurrence has increased using [ArXiv API Wraper](https://pypi.org/project/arxiv/)" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": null, 14 | "id": "ffdb427c", 15 | "metadata": { 16 | "scrolled": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "%matplotlib inline\n", 21 | "\n", 22 | "import requests\n", 23 | "import numpy as np\n", 24 | "import pylab\n", 25 | "import matplotlib.pyplot as plt\n", 26 | "import pandas as pd\n", 27 | "\n", 28 | "import tqdm\n", 29 | "\n", 30 | "from arxiv import SortCriterion , Search, SortOrder\n", 31 | "import typing\n", 32 | "from collections import defaultdict" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 3, 38 | "id": "e31b78d4", 39 | "metadata": { 40 | "scrolled": true 41 | }, 42 | "outputs": [], 43 | "source": [ 44 | "# Arxiv API Search\n", 45 | "embedding_search = Search(\n", 46 | " query = \"embeddings\",\n", 47 | " max_results = float('inf'), # limit of 300k results from the API total\n", 48 | ")" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": null, 54 | "id": "53e6261f", 55 | "metadata": { 56 | "scrolled": true 57 | }, 58 | "outputs": [], 59 | "source": [ 60 | "# Preparing results for Pandas import\n", 61 | "embedding_dict = defaultdict(list, title=[],date=[],category=[])\n", 62 | "\n", 63 | "results = embedding_search.results()\n", 64 | "\n", 65 | "for result in tqdm.tqdm(results):\n", 66 | " embedding_dict['title'].append(result.title)\n", 67 | " embedding_dict['date'].append(result.published)\n", 68 | " embedding_dict['category'].append(result.primary_category)\n" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 10, 74 | "id": "29d00179", 75 | "metadata": { 76 | "scrolled": true 77 | }, 78 | "outputs": [], 79 | "source": [ 80 | "# Get default dict into Pandas for graphing\n", 81 | "embedding_df = pd.DataFrame.from_dict(embedding_dict)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "id": "732b7ca4", 88 | "metadata": { 89 | "scrolled": true 90 | }, 91 | "outputs": [], 92 | "source": [ 93 | "embedding_df.to_csv('embeddings.csv', index=False) " 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 10, 99 | "id": "187a4ea2", 100 | "metadata": { 101 | "scrolled": true 102 | }, 103 | "outputs": [], 104 | "source": [ 105 | "embedding_df = pd.read_csv(\"embeddings.csv\")" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": 11, 111 | "id": "d8e5185c", 112 | "metadata": { 113 | "scrolled": true 114 | }, 115 | "outputs": [ 116 | { 117 | "data": { 118 | "text/html": [ 119 | "
\n", 120 | "\n", 133 | "\n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | "
titledatecategory
0Frustratingly Easy Meta-Embedding -- Computing...2018-04-14 18:30:01+00:00cs.CL
1Learning Meta-Embeddings by Using Ensembles of...2015-08-18 09:29:22+00:00cs.CL
2Discrete Word Embedding for Logical Natural La...2020-08-26 16:15:18+00:00cs.CL
3Hash Embeddings for Efficient Word Representat...2017-09-12 16:13:10+00:00cs.CL
4Gender Bias in Meta-Embeddings2022-05-19 21:20:47+00:00cs.CL
............
16195ArchNet: Data Hiding Model in Distributed Mach...2020-04-23 04:59:05+00:00cs.LG
16196On the Role of Visual Cues in Audiovisual Spee...2020-04-25 01:00:03+00:00cs.LG
16197Multilayer Structures of Graphene and Pt Nanop...2020-04-25 10:35:45+00:00physics.app-ph
16198Gravitational couplings in ${\\cal N}=2$ hetero...2020-04-25 13:06:51+00:00hep-th
16199GLUECoS : An Evaluation Benchmark for Code-Swi...2020-04-26 13:28:34+00:00cs.CL
\n", 211 | "

16200 rows × 3 columns

\n", 212 | "
" 213 | ], 214 | "text/plain": [ 215 | " title \\\n", 216 | "0 Frustratingly Easy Meta-Embedding -- Computing... \n", 217 | "1 Learning Meta-Embeddings by Using Ensembles of... \n", 218 | "2 Discrete Word Embedding for Logical Natural La... \n", 219 | "3 Hash Embeddings for Efficient Word Representat... \n", 220 | "4 Gender Bias in Meta-Embeddings \n", 221 | "... ... \n", 222 | "16195 ArchNet: Data Hiding Model in Distributed Mach... \n", 223 | "16196 On the Role of Visual Cues in Audiovisual Spee... \n", 224 | "16197 Multilayer Structures of Graphene and Pt Nanop... \n", 225 | "16198 Gravitational couplings in ${\\cal N}=2$ hetero... \n", 226 | "16199 GLUECoS : An Evaluation Benchmark for Code-Swi... \n", 227 | "\n", 228 | " date category \n", 229 | "0 2018-04-14 18:30:01+00:00 cs.CL \n", 230 | "1 2015-08-18 09:29:22+00:00 cs.CL \n", 231 | "2 2020-08-26 16:15:18+00:00 cs.CL \n", 232 | "3 2017-09-12 16:13:10+00:00 cs.CL \n", 233 | "4 2022-05-19 21:20:47+00:00 cs.CL \n", 234 | "... ... ... \n", 235 | "16195 2020-04-23 04:59:05+00:00 cs.LG \n", 236 | "16196 2020-04-25 01:00:03+00:00 cs.LG \n", 237 | "16197 2020-04-25 10:35:45+00:00 physics.app-ph \n", 238 | "16198 2020-04-25 13:06:51+00:00 hep-th \n", 239 | "16199 2020-04-26 13:28:34+00:00 cs.CL \n", 240 | "\n", 241 | "[16200 rows x 3 columns]" 242 | ] 243 | }, 244 | "execution_count": 11, 245 | "metadata": {}, 246 | "output_type": "execute_result" 247 | } 248 | ], 249 | "source": [ 250 | "embedding_df" 251 | ] 252 | }, 253 | { 254 | "cell_type": "code", 255 | "execution_count": 13, 256 | "id": "8851ed79", 257 | "metadata": { 258 | "scrolled": true 259 | }, 260 | "outputs": [ 261 | { 262 | "data": { 263 | "text/html": [ 264 | "
\n", 265 | "\n", 278 | "\n", 279 | " \n", 280 | " \n", 281 | " \n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | " \n", 289 | " \n", 290 | " \n", 291 | " \n", 292 | " \n", 293 | " \n", 294 | " \n", 295 | " \n", 296 | " \n", 297 | " \n", 298 | " \n", 299 | " \n", 300 | " \n", 301 | " \n", 302 | " \n", 303 | " \n", 304 | " \n", 305 | " \n", 306 | " \n", 307 | " \n", 308 | " \n", 309 | " \n", 310 | " \n", 311 | " \n", 312 | " \n", 313 | " \n", 314 | " \n", 315 | " \n", 316 | " \n", 317 | " \n", 318 | " \n", 319 | " \n", 320 | " \n", 321 | " \n", 322 | " \n", 323 | " \n", 324 | " \n", 325 | " \n", 326 | " \n", 327 | " \n", 328 | " \n", 329 | " \n", 330 | " \n", 331 | " \n", 332 | " \n", 333 | " \n", 334 | " \n", 335 | " \n", 336 | " \n", 337 | " \n", 338 | " \n", 339 | " \n", 340 | " \n", 341 | " \n", 342 | " \n", 343 | " \n", 344 | " \n", 345 | " \n", 346 | " \n", 347 | " \n", 348 | " \n", 349 | " \n", 350 | " \n", 351 | " \n", 352 | " \n", 353 | " \n", 354 | " \n", 355 | " \n", 356 | " \n", 357 | " \n", 358 | " \n", 359 | " \n", 360 | " \n", 361 | " \n", 362 | " \n", 363 | " \n", 364 | " \n", 365 | " \n", 366 | " \n", 367 | "
titledatecategorycreated_month
0Frustratingly Easy Meta-Embedding -- Computing...2018-04-14 18:30:01+00:00cs.CL2018-04
1Learning Meta-Embeddings by Using Ensembles of...2015-08-18 09:29:22+00:00cs.CL2015-08
2Discrete Word Embedding for Logical Natural La...2020-08-26 16:15:18+00:00cs.CL2020-08
3Hash Embeddings for Efficient Word Representat...2017-09-12 16:13:10+00:00cs.CL2017-09
4Gender Bias in Meta-Embeddings2022-05-19 21:20:47+00:00cs.CL2022-05
...............
16193A novel embedded min-max approach for feature ...2020-04-21 09:40:38+00:00cs.LG2020-04
16194TAEN: Temporal Aware Embedding Network for Few...2020-04-21 16:32:10+00:00cs.CV2020-04
16195ArchNet: Data Hiding Model in Distributed Mach...2020-04-23 04:59:05+00:00cs.LG2020-04
16196On the Role of Visual Cues in Audiovisual Spee...2020-04-25 01:00:03+00:00cs.LG2020-04
16199GLUECoS : An Evaluation Benchmark for Code-Swi...2020-04-26 13:28:34+00:00cs.CL2020-04
\n", 368 | "

7499 rows × 4 columns

\n", 369 | "
" 370 | ], 371 | "text/plain": [ 372 | " title \\\n", 373 | "0 Frustratingly Easy Meta-Embedding -- Computing... \n", 374 | "1 Learning Meta-Embeddings by Using Ensembles of... \n", 375 | "2 Discrete Word Embedding for Logical Natural La... \n", 376 | "3 Hash Embeddings for Efficient Word Representat... \n", 377 | "4 Gender Bias in Meta-Embeddings \n", 378 | "... ... \n", 379 | "16193 A novel embedded min-max approach for feature ... \n", 380 | "16194 TAEN: Temporal Aware Embedding Network for Few... \n", 381 | "16195 ArchNet: Data Hiding Model in Distributed Mach... \n", 382 | "16196 On the Role of Visual Cues in Audiovisual Spee... \n", 383 | "16199 GLUECoS : An Evaluation Benchmark for Code-Swi... \n", 384 | "\n", 385 | " date category created_month \n", 386 | "0 2018-04-14 18:30:01+00:00 cs.CL 2018-04 \n", 387 | "1 2015-08-18 09:29:22+00:00 cs.CL 2015-08 \n", 388 | "2 2020-08-26 16:15:18+00:00 cs.CL 2020-08 \n", 389 | "3 2017-09-12 16:13:10+00:00 cs.CL 2017-09 \n", 390 | "4 2022-05-19 21:20:47+00:00 cs.CL 2022-05 \n", 391 | "... ... ... ... \n", 392 | "16193 2020-04-21 09:40:38+00:00 cs.LG 2020-04 \n", 393 | "16194 2020-04-21 16:32:10+00:00 cs.CV 2020-04 \n", 394 | "16195 2020-04-23 04:59:05+00:00 cs.LG 2020-04 \n", 395 | "16196 2020-04-25 01:00:03+00:00 cs.LG 2020-04 \n", 396 | "16199 2020-04-26 13:28:34+00:00 cs.CL 2020-04 \n", 397 | "\n", 398 | "[7499 rows x 4 columns]" 399 | ] 400 | }, 401 | "execution_count": 13, 402 | "metadata": {}, 403 | "output_type": "execute_result" 404 | } 405 | ], 406 | "source": [ 407 | "# Only CS papers (maybe filter further to IR?)\n", 408 | "embedding_df = embedding_df.loc[embedding_df[\"category\"].str.startswith('cs')]\n", 409 | "# Compress to months\n", 410 | "embedding_df['created_month'] = pd.to_datetime(embedding_df['date']).dt.strftime('%Y-%m')\n", 411 | "embedding_df" 412 | ] 413 | }, 414 | { 415 | "cell_type": "code", 416 | "execution_count": 35, 417 | "id": "cf34c817", 418 | "metadata": { 419 | "scrolled": true 420 | }, 421 | "outputs": [ 422 | { 423 | "data": { 424 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAn8AAAG0CAYAAABDiUERAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACiZElEQVR4nOzdd3hUZfr/8fdk0nsvJKSQkBAgQICgAgJiWxsq9rKiiH5tu669rT/BXVa36Np3WSvqquCuImvFrtgICTUEQkI6SUjvbTLn98eQISHJkEyGzJPM/bouL5OZMzP3nM+c4c4pz6PTNE1DCCGEEEI4BCd7FyCEEEIIIUaONH9CCCGEEA5Emj8hhBBCCAcizZ8QQgghhAOR5k8IIYQQwoFI8yeEEEII4UCk+RNCCCGEcCDS/AkhhBBCOBBp/oQQQgghHIg0f2JUWrlyJTqdjm+++cYur19QUIBOp+Paa68d9GNee+01dDodr732Wq/bY2NjiY2NtWl9QgyWfP6G79prr0Wn01FQUGDvUsaUgb4zxfBJ8ycs6m5yVq5c2ev27i87S/8NpTESaututof6j1tzczNPPfUUixcvJjQ0FFdXV/z9/ZkzZw4PPfQQBw4c6POY9PR0rrrqKmJiYnBzc8PX15f4+HjOO+88/vKXv9Dc3Dykmnv+5+HhQWJiIrfeeislJSVDei/C9hYtWmTO5pVXXhlwuVWrVtn1e8Xef2zay+rVq83rfd++fVY/z8svv4xOp+Oss84acJlzzjkHnU7HCy+8YPXriMFztncBYnQ7//zzmTFjRr/3DXS76O3LL7+0dwnHxc8//8zFF19MaWkpUVFRnH322YwbN47m5ma2bdvGn//8Z/7617/y888/M3PmTADefPNNli1bhqZpLF68mAsvvBAPDw8KCwvZvHkzH374IUuXLiUhIWHQdSxcuJBFixYBUFVVxaZNm3jhhRdYv349P//8M/Hx8cfj7YshcHZ25qWXXmL58uV97jMajbzyyis4OztjMBjsUJ1j0jSNl156CZ1Oh6ZpvPjii/ztb3+z6rmuv/56Nm7cyMaNG3n++ee59dZbe93/j3/8g48//pizzjqLW265xXz7hRdeyIknnkhERMSw3ovoS5o/MSwXXHCB7OEbprHYfOzdu5czzzyTpqYmHn/8ce666y6cnXt/3eTn53PffffR0NAAQEtLC7feeis6nY5NmzZx6qmn9nneH3/8keDg4CHVsmjRol57rjs7OznrrLP48ssv+eMf/8irr7469DcobOrcc89lw4YNZGVlMWXKlF73ffbZZxQVFXHhhRfy/vvv26lCx7Np0yYKCgq49tpr+fTTT1m7di1/+tOfcHV1ter5XnzxRX766SfuvfdeTjvtNJKSkgDIycnh7rvvJigoqM/eXz8/P/z8/Ib9XkRfcthXjIie58jl5eVx8cUXExQUhI+PD2eccQa7d+8GoLKykhtvvJGIiAjc3d1JS0vj66+/tvjca9euJTU1FQ8PD0JDQ1m+fDnl5eX9LltTU8MDDzxAcnIyHh4e+Pn5ceqpp7Jp06Z+l29sbOTOO+8kKioKd3d3Jk2axJNPPonRaBywntzcXC655BICAgLw8vJi7ty5fPTRRwMu3985Vz3Pdfn6669ZtGgRPj4++Pr6cs4555Cdnd3vc+Xk5HDRRRf1ee2Bzp3ZuXMnV1xxBbGxsbi5uRESEsLMmTP53e9+R2dn54A1H8tvfvMbGhoauO+++7jvvvv6NH4AcXFxrF+/npNOOgmA3bt309DQwNSpU/tt/ADmzp2Lv7+/1XUBuLi4cOONNwKwZcsWAA4ePMijjz7KvHnzCA8Px9XVlXHjxnHllVeyZ8+ePs/R8/O8d+9eLrjgAgIDA/Hy8mL+/PkDfp4A3n77bU455RT8/f1xd3cnOTmZP/7xj7S3t/dZVqfTsWjRIsrLy1mxYgWRkZHo9XpzjhUVFdx9990kJSXh5eWFv78/SUlJXHvttf0eUrekvr6e2267jcjISNzd3Zk8eTLPPPMMmqaZl9m7dy86nY5TTjllwOdJSUnBxcWFsrKyQb/2ihUrAFODcLQXX3wRT09PrrrqKou1P/DAAyQlJeHu7k5AQABnnnkmX3zxRZ9lv/nmG/OpLNu3b+ecc87B398fT09PFi5cyI8//thr+djYWFatWgXAKaec0us0gv6sWbOGlJQU3N3dCQsL48Ybb6S+vn7Q6+J4v5/B6s7ihhtu4KqrrqKqqmrA5rvnYfG33nqLE044AW9v717fa6Ghobz44ou0tLRw9dVXYzAYMBgMXH311bS0tPCvf/2L8PDwXs979PdWW1sb/v7+hIaGDrgX+Oabb0an0/Hhhx9a9b4dhiaEBfn5+RqgPfLII71uX7ZsmQZor7766pCeZ+HChVpQUJA2f/587c4779SWLl2q6XQ6LSgoSMvJydEmTJigzZgxQ7v99tu1X//615qLi4vm5uamFRYW9nq+Rx55RAO0JUuWaO7u7tqyZcu0+++/X5s/f74GaHFxcdqhQ4d6PaagoECLjY3VAO3kk0/Wfve732k33HCDFhERoel0Ou1f//pXr+Xb2tq0tLQ0DdCmT5+u3XvvvdqNN96o+fv7a0uWLNEAbdmyZb0ek5OTowUFBWmAdtZZZ2kPPPCAdskll2jOzs7aeeed1+86i4mJ0WJiYnrd9uqrr2qAdtFFF5kfe/fdd2tnn322BmghISFaZWVlr8dkZ2drAQEBGqCdc8452gMPPKBddtllmouLi3b++ef3ee0dO3Zo7u7umoeHh3bZZZdp999/v3bLLbdoZ5xxhubi4qI1Njb2Wd/5+fnHzPrAgQMaoLm7u2u1tbXHXL5bbm6u+b01NTUN+nED6a756M+upmnaO++8owHalClTNE3TtLffflvz8PDQzj77bO2WW27R7r33Xu3CCy/UXFxcNC8vL2379u29Ht/9eV6wYIHm7++vnXzyydr999+vLVu2THN3d9ecnJy0d955p8/rXnfddRqgRUVFacuXL9fuvPNObe7cuRqgLVq0SOvs7Oy1PKClpKRoMTEx2pQpU7TbbrtN++1vf6t9/PHHWnNzsxYfH68B2umnn67ddddd2p133qlddNFFmr+/v/a///1vUOspJiZGi4iI0GbPnq0lJCRod955p3bbbbdpERERGqDdcsstvZY/5ZRTNEDbt29fn+f64YcfzJ/bwVi4cKEGaPv379cWLFigBQUFaW1tbeb7y8rKNGdnZ23ZsmXa559/3u82V1tbq02ePFkDtLS0NO2+++7Trr/+es3Hx0fT6XTaP//5z17Lf/311+ZtxMPDQ1u8eLF21113aZdcconm5OSkubu7a3v37jUv//e//91c57Jly7RHHnnE/F+37u/DSy65RPP19dWuuuoq7c4779RSU1M1QDvllFMGtT5G4v0MRnl5uebi4qIlJiZqmqZpu3bt0gBt8eLF/S7fva2de+65mpubm3bxxRdr9913n3bTTTf1WXb58uUaoD388MPaww8/rAHatdde2+/zdn8P9vzeuvHGGzVA27hxY5/l29ratICAAC0sLKzPtiR6k+ZPWKX7y+7888/v9WXY87/s7Gzz8t3/WALaH//4x17P9eijj2qAFhAQoP3f//2f1tXVZb7v9ddf1wDtd7/7Xa/HdH/ZuLi4aJmZmb3u+93vfqcB2vLly3vdvnDhQk2n02lvv/12r9tra2u16dOna+7u7lp5ebn59tWrV2uAtnTp0l41HThwwNxkHf0P0emnn64B2lNPPdXr9g0bNpjf/1CaP71er33xxRe97rv//vs1QPvzn//c6/bFixdrgPbCCy/0uv3jjz/u97XvvPNODdA2bNigHa2mpqbXex6K7szmzZs3pMcZjcZezfZzzz2nZWZmau3t7VbVMVDz19nZaV5X3Z+RiooKraGhoc9zbN++XfPy8tJ+9atf9bq95+f57rvv7nVfenq65uzsrPn7+2v19fXm27szvfDCC7WWlpZ+az36c9P9Gr/+9a/7/GO2cePGfrcNTdO09vb2ft9Pf2JiYsx59Wy8qqurtQkTJmiA9u2335pvf/fddzVAu+uuu/o8V/f3wqZNmwb12j2bvzfeeEMDtLfeest8/2OPPaYB2ubNmwds/rqbgRtvvFEzGo3m23NycjRfX1/N1dW11x8t3c1Sf9viP//5Tw3Qbr755l63d+fz9ddf9/s+ut/3+PHje/2h2tnZqZ188skaoP3yyy+DWicj8X6OpXu9/+lPfzLfNmvWLE2n02n79+/vs3z3+vH09OzzfXy0hoYGLS4uTtPr9Zper9diY2MH/Kz21/z9+OOPA/6BsX79eg3Q7rzzzkG+U8clzZ+wSveXnaX/3n//ffPy3f9YxsbGagaDoddzFRYWmr84jv4SMBgMmrOzs7Zo0aJet3d/2Rzd4GmaptXV1Wl+fn6au7u7+R+z7du3a4B28cUX9/t+upuz559/3nxbQkKC5uTkpOXm5vZZvvv1e/5DVFxcbN7rePR71LQj/9ANpfm76qqr+jxP9561nl9+RUVFGqAlJCT027SddtppAzZ/n332WZ/lh+PPf/6zBmiXXXbZkB9bWFioLVq0qNfnyMXFRZszZ472+OOP92qmjqU7o4ULF5r/ILntttu0iRMnaoAWHBys5eXlHfN5zjvvPM3NzU3r6Ogw39b9efbz8+v3H67u7eO1114z3zZjxgzN2dm5372hBoNBCwoK0tLS0nrdDmiurq5aRUVFn8d0N38PPPDAMd+DJd3N33fffdfnvu7PYc89M52dnVpERESfvXS1tbWah4eHFh8f36tpsaRn89fa2qoFBASY95IZjUYtPj5eS05O1jRN67f5a29v1zw9PTVvb2+turq6z/P//ve/1wBt1apV5tu6m6X+/jjp6OjQnJ2dtVmzZvW6fbDN34svvtjnvldeeUUDtGefffaY62Ok3o8l3evdyclJKykpMd/+7LPPaoB277339nlM9/rp7w+R/nR/rgDtk08+OeZyR39nJiYmaq6urn3W0TnnnKMB2o4dOwZVhyOTc/7EsLz66qtopj8i+vx3wQUX9Fl+xowZ6PX6XreNGzcOgMTERHx8fHrdp9frCQsLG3BYjoULF/a5zc/PjxkzZtDW1mY+N+6nn34CTOfSrFy5ss9/3edodS/f2NhIbm4ukZGR/V6Q0X31aE/btm0DYP78+X3e40CPOZbZs2f3uW38+PEA1NbWmm/bvn07ACeddBJOTn036/nz5/e57bLLLkOv13PBBRdwzTXX8Prrr5OXlzfkGm0pOjqar7/+mj179vD000/z61//mgkTJrBlyxbuv/9+UlJSyM/PH9Jzfvvtt6xatYpVq1bxr3/9C6PRyE033URmZiYTJkwwL/fRRx9x3nnnERERgYuLi/m8rv/973+0t7dTVVXV57lnzpzZ5zMLR7Lu/ky0tLSwY8cOAgICeOqpp/p8/v7whz/g5ubW77mcsbGxhIaG9rl94cKFREZG8vjjj/OrX/2KZ555hoyMDLq6uoa0fsB0te3cuXOP+T66l73hhhuorq7mv//9r/n2N954g9bWVm688cYBz4ezxN3dnauvvppvvvmG3NxcvvrqK/Ly8rjhhhsGfMy+fftoaWlh+vTpBAYG9rl/8eLFferv1t+25eLiQlhYWK9taygGu70ORIX3073eTz/9dCIjI823X3nllbi6uvLaa68NeD7wnDlzjvn8ra2t/PnPfzb//u677w66tm7Lli2jo6ODd955x3xbRUUFn332GampqUybNm3Iz+lo5GpfMaL6u3Kr+2KAga7qcnZ2HvDLJiwsrN/bu08c7j7Rurq6GoDPP/+czz//fMD6mpqaej3uWM/fkzWPOZb+Lm7oXl89/5E/1mv3d/ucOXP4/vvvWb16Nf/5z3944403AEhKSuKRRx7hiiuuGHK9gHlYhtLSUqseD5CcnExycrL5971797J8+XJ++ukn7rjjDjZs2DDo53rkkUf6jFN5tKeffprf/e53BAQEcPrppxMdHY2npyc6nY4NGzawY8eOfi/IGOznr7a2Fk3TqKysNF88MFgDfW58fX35+eefeeSRR9i4cSOfffYZAMHBwdxyyy38/ve/x8XFZVCvERwc3O8fLEe/j2433ngjq1evZs2aNVx55ZUA/Otf/8LV1ZXrrrtu0O/taDfccAPPPvssL7/8Mvn5+bi5uXHNNdcMuHx3XQMNBdJ9e11dXZ/7BrpwyNnZ2aoGeqDn7G97HYgK7+df//oXQJ9RHAIDAznvvPP473//ywcffMDFF1/c57GD+Y6799572bt3L7fffjvffvstr7zyChdccAHnnXfeoGu85pprePjhh1m7dq15aJh///vfGAwGli1bNujncWTS/IlRraKiot/bu6/27W4ou///9NNP89vf/vaYz9u9/LGef7iPsRVfX1+Lrz3Q7SeddBIffvgh7e3tZGRk8Omnn/Lss89y5ZVXEhISwmmnnTbkWrr3Mm7dupX6+nqbDNUwadIk3njjDRISEvjqq6+G/Xw9GQwGVq5cSXh4OJmZmX3+4e3ea9yfoX7+UlNTyczMHFJ9lvaiRUVF8fLLL6NpGnv27OGrr77i+eef59FHH8VoNPKHP/xhUK9RVVVFV1dXnwbw6PfRLTIykiVLlvD++++zd+9eampq2L17N5dddhkhISFDen89paSkcOKJJ/Lyyy9TX1/PRRddRFBQ0IDLd9c10LbVfcXxaBkuxN7vp7Ky0vyH1RVXXDHgH4D/+te/+m3+jrXHd9OmTTz//POkpKTw5z//mf379zN79mxuuOEGdu/ePehhnKKioli8eDFffPEFe/fuZdKkSaxduxYXFxfzHyPCMjnsK0a1b7/9ts9t9fX1bN++3TyMBsCJJ54IwPfffz+o5/Xx8SEhIYHS0tJ+D4X2N9J/amoqAJs3b+73L+3jOTtA94DaP/30U7/D0GzevNni493c3Jg7dy6PPvoozzzzDAAffPCBVbXExcVx2mmn0dbWxl//+tdjLt/fHrX+dB9e1XoMPWILVVVV1NXVMXfu3D6NX1NTk8VmLTMzk8bGxj63d2fd/Znw9vZmypQpZGVlUVNTY7viD9PpdEyZMoXf/OY35j3bQ9k7ajAY+h0S5Oj30VP3Hpc1a9aY9xb93//93xAr7+uGG26gsrKSjo4Oi4d8wbSX2tPTkx07dvS7N6x7mKjuQcSt1d0UW7tHcLBG6v0MZO3atXR0dDBr1iyuv/76fv8LCQnhiy++GPLpFzU1NVx33XW4uLjw5ptv4ubmxtSpU/nDH/5ARUUFN99885Cer3vP5Nq1a9m+fTs7d+7krLPOGtYfH45Emj8xqr3xxht9zn9ZuXIl9fX1XHHFFbi5uQGm82FOPvlk3nvvvQGnkdq1axeHDh0y/37ddddhNBq57777ejVU+fn55gapp6ioKE4//XTy8/N57rnnet33wQcf9Nuo2kp0dDSLFi0iNzeXNWvW9Lrv008/7Xd8sB9//JHW1tY+t3fvzfL09LS6nmeeeQZfX18ee+wxnnjiiX7H5CoqKuLyyy8371nrXq/9jYmmaRqrV68GYMGCBVbX1Z/Q0FA8PT3JyMgwH/YH02DQt99+e7/n+nWrr6/n0Ucf7XXb1q1b+fe//42fnx8XXnih+fY777yTjo4Oli9f3u8/7LW1tUPaK5iVldXvnkdr83vggQd6NeI1NTX88Y9/BOj3UO6pp55KYmIia9euZf369SQlJVkc/2+wLr/8ct5//30++OCDY54n6+rqylVXXUVjYyMPP/xwr/vy8vJ45plncHFx4de//vWwaure+1hUVDSs5zmWkXo/A+ke2++FF17gpZde6ve///u//zPP/jEUN910EwcPHuSPf/xjr3Py7rrrLk4++WT+85//8Oabbw76+ZYuXYqvry9vvvmmeRxAmXBg8OSwrxiWDRs2DDjfa2xs7HHfGM866yzmzZvHpZdeSkREBJs3b2bz5s3Exsby+OOP91r2rbfeYvHixVx//fU888wznHDCCfj7+1NSUsLOnTvZvXs3P/30k/nk+rvuuosNGzbw3//+l5kzZ3LmmWdSV1fH+vXrWbBgARs3buxTz/PPP89JJ53E7373OzZt2sT06dPJzc3l/fff57zzzuN///vfcVsXzz//PPPmzeOWW27h448/Ztq0aRw4cID//ve/nH/++XzwwQe9Lgb5y1/+wldffcXJJ59MXFwc3t7eZGVl8cknnxAQEGAeCNkaycnJfPbZZ1x88cXcfffdPP3005x66qnm6d127NjBDz/8gE6n47777gNMjdTtt9/OPffcw7x585g6dSo+Pj4cOnSIr776igMHDhAaGsoTTzwx7HXVk5OTE7/97W95/PHHSUlJ4fzzz6ejo4Ovv/6ampoaTjnllAEHGl+wYAEvvfQSv/zyC/PmzaOsrIx169ZhNBpZs2aN+XA8wPLly8nIyOCFF14gPj6eM888k+joaGpqasjPz+e7777juuuu45///Oeg6v7888+55557OOmkk0hMTCQ0NJSSkhJzzvfcc8+g10FERATt7e1MnTqVJUuW0NnZyX/+8x/Kysq45ZZb+m24dTodN910E3feeSfAsD4vPXl6evZ7sdhAHn/8cb7//nuee+450tPTOeWUU6iqqmL9+vU0Njby3HPPERcXN6yaTjnlFJycnHjggQfYvXs3AQEBAPz+978f1vP2ZyTeT3+++eYbcnJySElJsXjhxvXXX8/q1at59dVXWbVqVb8DuB/tjTfe4N1332XBggXcddddve5zcnJi7dq1TJs2jd/85jcsWrSIqKioYz6nh4cHl1xyCS+//DIvvPACQUFBnHPOOcd+o8LETlcZi1FuMEO9LFy40Lx899AYR4/R1e3o5XvqbyiUnkMvvPrqq+Zx+oKDg7Vrr71WO3jwYL/P1dDQoK1evVqbOXOm5uXlpbm7u2uxsbHa2Wefra1Zs6bP4ML19fXaHXfcoY0bN05zc3PTkpKStL/97W9aXl7egO9n//792kUXXaT5+flpnp6e2oknnqh9+OGHAw5bYGmol4EG0R5ofWVnZ2sXXnhhn9f+61//2mf4nc8++0y79tprteTkZM3X11fz9PTUEhMTtd/85jdaQUFBv687VI2NjdqTTz6pLVq0SAsJCdGcnZ01X19fbebMmdr999+vHThwwLxsW1ub9v7772s333yzlpqaqoWGhvZa/qGHHuozcLcllgZ5PlpnZ6f2xBNPaMnJyZq7u7sWFhamXX311VpBQYH5s95zbLWen+c9e/ZoS5Ys0fz9/TUPDw9t7ty52qeffjrga/3vf//TzjnnHC0kJERzcXHRwsLCtLS0NO2hhx7qNTamplneLvbs2aPdcccd2qxZs7Tg4GDN1dVVi4mJ0S666CLthx9+GMwq0jTtyOevrq5Ou+WWW7Rx48Zprq6u2qRJk7Snn37a4rAtNTU15oGEq6qqBv2a3XoO9XIsA43zp2mmYWbuvfdeLSEhQXN1ddX8/Py00047rd9hjLqHRhnoc9Hf9qhpmvbGG2+Yv2e6v+O69fcZGezr9Wck3s/RrrzySg3Qnn766WMu2z2e6XvvvadpmuWhcAoLCzU/Pz/N19fX4vfKiy++qIFpwPLuz9yxvge///57cxa33XbbMesWR+g0zcYn0AghlHPVVVfx1ltvsXfvXvOcmsJ6BQUFxMXFsWzZsj5T5jmSb775hlNOOYWrr77afLW4EEJ9cs6fEGOE0Wjs9yrBL7/8knXr1jF58mRp/IRN/eUvfwHgtttus3MlQoihkHP+hBgjOjo6GD9+PKeccgqTJk3C2dmZrKwsPv/8c1xdXXn++eftXaIYA3bt2sWHH35IRkYGn3zyCeeeey4nnHCCvcsSQgyBNH9CjBEuLi7cdNNNfPXVV/zyyy+0tLQQHBzMJZdcwv3339/vcB1CDFVGRgYPPvggvr6+XHLJJbzwwgv2LkkIMURyzp8QQgghhAORc/6EEEIIIRyINH9CCCGEEA5Ezvk7ho6ODqqrq/Hw8Og1QK4QQgghhEqMRiOtra0EBQXh6uo64HLS/B1DdXU1e/bssXcZQgghhBCDMnny5D5zlfckzd8xuLu7A6YV6eXlNSKvmZ2dTXJy8oi8lhg6yUdtko/6JCO1ST5qs5RPc3Mze/bsMfcuA5Hm7xj0ej0AXl5evebpPJ50Ot2IvZYYOslHbZKP+iQjtUk+ahtMPt29y0DkJDYFhYSE2LsEYYHkozbJR32SkdokH7XZIh9p/hQUEBBg7xKEBZKP2iQf9UlGapN81GaLfKT5U1BOTo69SxAWSD5qk3zUJxmpTfJRmy3ykeZPCCGEEMKBSPOnoMTERHuXICyQfNQm+ahPMlKb5KM2W+QjzZ+Camtr7V2CsEDyUZvkoz7JSG2Sj9pskY80fwqqrKy0dwnCAslHbZKP+iQjtUk+arNFPtL8KUin09m7BGGB5KM2yUd9kpHaJB+12SIfnaZpmg1qGbMaGhpIT08nLS1NBr0UQgghhLIG27PInj8FZWZm2rsEYYHkozbJR32SkdokH7XZIh9p/hRkMBjsXYKwQPJRm+SjPslIbT3z2XWwnm/2V9Ju6LJjRaInW2w/MrevggIDA+1dgrBA8lGb5KM+yUht3flsKazhN+9ux6hBiLcry06I4aIZkTg7yX4je7LF9iMJKigsLMzeJQgLJB+1ST7qk4zUFhYWhqHLyN++zMF4+KqAyqYO/vblfv7x/QH7Fidssv1I86eg7Oxse5cgLJB81Cb5qE8yUlt2djbrt5WQX90CgJ/7kYOE3+VW2asscZgtth9p/oQQQghh1thh5F8/5AOgA56+eAYpEaYrRwtqWqhr6bBjdcIWpPlTUHx8vL1LEBZIPmqTfNQnGamtSAugucN0gceSlAimRPgyLdLPfP+Og/X2Kk1gm+1Hmj8FNTc327sEYYHkozbJR32Skdq2ldSZf75oRiQAM6L8zbftKJHmz55ssf1I86eg8vJye5cgLJB81Cb5qE8yUpemaeyuMJ3r5+WqJzHUB4Bp447s+dteWmeP0sRhtth+pPkTQgghBGA6p6+x03SJ74wof/ROpqnEAr1ciQ7wACC7vJG2Thn3bzST5k9BaWlp9i5BWCD5qE3yUZ9kpK7M4jrzzzN7HOoFmB5p+t1g1MiuaBy5okQvtth+pPlT0M6dO+1dgrBA8lGb5KM+yUhdmcW15p9njvfvdd/0nhd9yKFfu7HF9iPNn4La29vtXYKwQPJRm+SjPslITZqmmff8ebjomRTm0+v+6VFHmr8fD1Rj1LSRLE8cZovtR5o/Bfn7+9u7BGGB5KM2yUd9kpGaiutaqWo2jeE3PdIPZ33vFiEmwJNQbzcAtpXU8/ev97P/UBP/3HyAL/cdGvF6HZUtth+Z21dBkZGR9i5BWCD5qE3yUZ9kpJ7Wji7eySg2/370IV8AnU7Hfacncc+GnRg1eCejhHcySkz3AR/ceBIRfh4jVLHjssX2I3v+FJSVlWXvEoQFko/aJB/1SUZq+T6viiX/+pF3t5Wab5s1PqDfZRckBPPQmZP63K4BeyuajleJogdbbD/S/AkhhBAOKru8gQc27qautRMw7cGbF+FMyjjfAR+zJGUcd586ETdnJwI9Xcy3F9TI4N2jhRz2VVBcXJy9SxAWSD5qk3zUJxmpobq5g7s37KLdYARgblwQty9KwEdrQafTWXzsZTPHc2lqFPsrm7hqbToAhTUtx71mYZvtR/b8KUiuhFOb5KM2yUd9kpEa/vhZNocaTVmkRPjy1wtSmBDsNeh8dDod0QGedLeJBdL8jQi52neMOnjwoL1LEBZIPmqTfNQnGdlfS4eBHw9UAxDo6cpfLkjB1dnUEgwlH3cXPRF+7oBpz58mw78cd7bYfqT5E0IIIRxMdnkjxsN92qKJwQQfHsLFGtEBngA0tRuoPjxUjFCbNH8Kmjlzpr1LEBZIPmqTfNQnGdnf7rIG889TInpf3DHUfGKDPM0/y3l/x58tth9p/hSUnZ1t7xKEBZKP2iQf9UlG9tez+Zsa4dfrvqHmExvoZf5Zzvs7/myx/Ujzp6DW1lZ7lyAskHzUJvmoTzKyv6yyegC8XPW99tzB0POJCZQ9fyPJFtuPNH8K8vUdeHwlYX+Sj9okH/VJRvZV0dhGZZPp3LzkcF+cjhrWZaj5xPZo/mSsv+PPFtuPNH8KiomJsXcJwgLJR22Sj/okI/vKOtjzkG/fRmKo+QR5ueLlqgdkz99IsMX2I82fgnbt2mXvEoQFko/aJB/1SUb21ft8v77N31Dz0el05kO/ZfVttHV2Da9AYZEtth9p/oQQQggHYulKX2t1X/ShAcW1ck6n6qT5U5AcElGb5KM2yUd9kpH9GIxGsitMzV+4r1u/4/tZk0/Pi0by5by/40oO+45RXV2yy1xlko/aJB/1SUb2U1TTSlunaS7fo4d46WZNPhOCjwz3sqfHnkVhe7bYfqT5U1BJSYm9SxAWSD5qk3zUJxnZT371kb1y8T0atp6syWfauCON5I7S+qEXJgbNFtuPNH9CCCGEgzjQo/mbENR/82eNAE9X85Av2RWNctGH4qT5U9CMGTPsXYKwQPJRm+SjPsnIfnru+YsbYM+ftflMjzLt/esyauaLSrq6JxAWNmOL7UeaPwXt37/f3iUICyQftUk+6pOM7Keg2jQOn95Jx3h/j36XsTaf1Eh/88/bSup4+MMsTn7qGzbuOmjV84n+2WL7keZPQc3NcqWUyiQftUk+6pOM7MNgNJoHYY4O8MBZ338LYG0+06P8zT//Z1sJn2ZX0NmlsS5TzvG0JVtsP842qGPE7Nmzh40bN5Kfn09tbS133303c+bMMd+vaRrr16/nyy+/pLm5mUmTJrFixQoiIiLMyzQ1NfHKK6+QkZGBTqfjhBNO4LrrrsPd3d0eb6lf3t7e9i5BWCD5qE3yUZ9kZB8H69vo6DJd6ds9Ll9/rM0n0s+dYC9Xqpo7qGnpNN9eWNOCUdP6TCMnrGOL7WdU7flrb28nNjaW66+/vt/7P/jgAz755BNuuOEG/vSnP+Hm5sbq1avp6OgwL/PMM89QXFzM73//e+6//36ys7NZs2bNSL2FQYmPj7d3CcICyUdtko/6JCP7GMz5fmB9Pjqdjhk99v51azcYqWhos+o5RV+22H5GVfOXmprK5Zdf3mtvXzdN0/j4449ZunQpaWlpxMTEcNttt1FbW0t6ejpgujx6+/bt3HTTTUycOJFJkyaxfPlyfvzxR2pqakb67Qxox44d9i5BWCD5qE3yUZ9kZB/5va709RxwueHkMz2y/7EDC2TOX5uxxfYzqpo/Sw4dOkRdXR3Tpk0z3+bp6UlCQgI5OTkA5OTk4OXl1atrTklJQafTkZubO+I1CyGEECOl+2IPsHzYdzhSe+z50zsdOcybXy3Nn0pG1Tl/ltTV1QHg59f7rw4/Pz/zfXV1dfj69p7HUK/X4+3tbV5mIFlZWej1embOnEl2djatra34+PgQFxfHzp07AYiOjkbTNIqLiwGYPn06eXl5NDU14eXlxcSJE9m+fTsAUVFR6PV6CgsLAVMTWlhYSENDA0ajEYPBQGZmJgDjxo3Dzc2N/Px8AKZMmUJpaSl1dXW4ubkxbdo0897N8PBwvLy8yMvLAyA5OZmKigpqampwdnZm5syZpKeno2kaISEhBAQEmJvjxMREamtrqaysxMnJidmzZ5OZmYnBYCAoKIiQkBD27t0LQEJCAo2NjVRUVAAwZ84ctm/fTkdHBwEBAURERLBnzx4AJkyYQGtrK2VlZQDMmjWLrKws2tra8PPzY/z48ezevRuA2NhYOjs7KS0tBUx7e/ft20dLSwve3t7Ex8eb/+oZP348QL/r29PTk6SkJLZt2wZAZGQkLi4uFBQUADB16lSKi4upr6/H3d2dKVOmkJGRAUBERAQeHh4cOHAAgMmTJ1NWVkZtbS2urq6MHz+eLVu2ABAWFoaPj4/5j4dJkyZRWVlJdXW1eX1v3boVo9FocX3rdDrS0tLM6zswMJCwsDCys7MB027+5uZmysvLAUhLS2Pnzp20t7fj7+9PZGQkWVlZAMTFxdHe3s7Bg6Yr7Hp+Zn19fYmJiTFPDB4TE0NXV5d50NAZM2awf/9+mpub+13fOp2OoqIiAKZNm0Z+fj6NjY14eHiQnJxs/sz2t75LSkrMn9mUlBS2bt16zM+si4sLqamp5vUdGhqKn5+f+Wq3pKQkqqurqaqqQq/XM2vWLDo6OtiyZQvBwcEEBQWxb98+ACZOnEh9fT2HDh0yf2a3bdtGZ2fnMdf37Nmz2bVrl3l9R0VFDfiZHYnviKPX92j7jnB2dqapqWnMfkfMmDFDye+IXUWmz74OcG6tZcuW7D6fWV9fX8LCwsz1D/U7wk+n41fRLhxoMHLx7Dj+9IXpfW/LLebSGRFKfEdkZGTQ1dU1ar8j2tra2LJlS7/fEd3r81h0mqaNykF4Lr300l4XfOzbt4+HH36YNWvWEBAQYF7uySefRKfTcccdd/Dee+/x7bff8vTTT/d6rhUrVnDppZdyxhln9HmdhoYG0tPTSUtL69M4Hi/l5eWEh4ePyGuJoZN81Cb5qE8yGnlGTWPR09/R2tlFpJ87G26cO+Cytsqnsa2Txc9+D8DM8f6suXzmsJ9TWM5nsD3LmDns6+/vD0B9fe9pZerr6833+fv709DQe87Brq4umpqazMuooHvPilCT5KM2yUd9ktHIO9TYTuvhWTcsXewBtsvHx92FIC9XAAqqZXgfW7FFPmOm+QsNDcXf3998OAugpaWF3NxcEhMTAdMu9ObmZvOueoDdu3ejaRoJCQkjXrMQQggxEnpO6xZ3nM7360/3lG81LZ3Ut3YeY2kxUkZV89fW1kZBQYH5mPahQ4coKCigqqoKnU7H2WefzXvvvcfWrVspKiriueeeIyAggLS0NMB0Ds2MGTNYs2YNubm57N27l1deeYW5c+cSGBhox3fWW8+LVoR6JB+1ST7qk4xGXlGPq21jLVzpC7bNp+eFJYVyxa9N2CKfUXXBR15eHqtWrTL//vrrrwOwcOFCbr31Vs4//3za29tZs2YNLS0tTJo0iQcffBBXV1fzY37729/y8ssv8+ijj5oHeV6+fPmIvxdL8vPzSU5OtncZYgCSj9okH/VJRiOvZ+MVE2C5+bNlPj0bzYKaZqYNMBSMGDxb5DOqmr8pU6awfv36Ae/X6XRcdtllXHbZZQMu4+3tze233348yrOZxsZGe5cgLJB81Cb5qE8yGnlFtUeav+hAy82fLfOJ7fFa3UPNaJrGxl1leLrqOS0pFJ3M/DEktsjH6uavpaWFTZs2kZWVRX19PTfeeCMJCQk0NTXxzTffMHv2bLmay0oeHv1Pti3UIPmoTfJRn2Q08rr3/Pm6O+Pv4WJxWVvmExt05LBv90DPb2cU8/evTUPANHd0ccG0cTZ7PUdgi3ysOuevurqa++67j3Xr1lFdXU1hYSFtbaapW7y9vfn888/55JNPhl2co5LDIWqTfNQm+ahPMhpZbZ1dVDS2AxAd4HnMPW22zCfUxw0PFz0AB6qaaO3o4rVfCs33P/X1fsrqW4f8vG2dXfxj8wF+998dvWYucQS2yMeq5u+NN96gtbWVv/71r6xcubLP/Wlpab2uuhVD0z0IplCT5KM2yUd9ktHI6nnIN+YYh3zBtvk46XQkHB5aprS+jVvXb6O25chVv80dXfzh070MZcjhvRWNXPNGOq/8VMAPB6p55lvHmqHLFvlY1fzt3LmTs846i6ioqH7/gggLC6O6unrYxQkhhBBieIpqj+xZiz7GxR7Hw7ITY8w/7yozjbWrAwI9TRdjphfVsnrTXtoNXcd8roP1rfzf25m9potLL6ylrfPYjxVHWNX8dXR0WBw5urV16LtwxRGRkZH2LkFYIPmoTfJRn2Q0sgprjhwWHcyeP1vnszAhhCtnje912xnJYaw6+8jhyw92lnHDW5nUtnRYfK7P9x6i5XCj53x47uB2g5H0otpey+VWNlF3jOcarWyRj1XNX1RUlHl+u/6kp6cTGxtrbU0Oz8XF8sm4wr4kH7VJPuqTjEZWUc3Q9vwdj3xuWxjPlAjTTiO9TseKk2I5MS6IlWcl4+ZsakWyKxp58qv9Fp9nS2GN+edbF8Sbf96cVwWAwWjkj59lc8VrW7hy7RYa2sbewNK2yMeq5u/ss8/mhx9+YMOGDbS0mHa9Go1GysvLefbZZ8nJyeGcc84ZdnGOarATMwv7kHzUJvmoTzIaWYU9zvkbH3DsK0WPRz4ueieeumg6N8yN5amLppmvAj5nagSvXDULP3fT4COfZVew/1ATJXWt/Du9qNf5im2dXWwvMU3hGu7rxkXTI3HVm9qYzQeqaTd08eDGLD7YWQZAZVMHX+w9ZPP3Ym+2yMeqoV4WLFhAVVUV69at45133gHgT3/6E5qm4eTkxBVXXMGcOXOGXZwQQgghrKdpmnl2j3BfN9wPX3lrD/4eLtw4b0Kf2xNDfbjuxFie+iYXDfjDZ9kU17bS1G7gnz8c4OFfJXPGpDB2Hqyno8sIwJyYQDxc9cyODuDH/GoONbbz69e39rny95M95SydIacZHM3qcf6WLl3KggUL+PnnnykvL0fTNMLCwjjhhBMICwuzZY0OZ+rUqfYuQVgg+ahN8lGfZDRyals6aWw3AMee2aObPfK5ODWStzKKOdTYTnb5kUGM2zqNPPS/LHIrmzD2uCB4TkwAAPMmBPFjvukC0+7Gz8NFj7ebnsqmDraX1lNa10qk/9gZW9IW+Vh12LeqqoqOjg6Cg4M599xzWbFiBTfccANLliwhLCyMjo4Oqqqqhl2coyopKbF3CcICyUdtko/6JKORM5SZPbrZIx83Zz03zo3rdVuYj5v551d/LuQ/247UNTs6EID58UG9HhPu68aay1O5JDXKfNtn2RXHo2S7sUU+VjV/t956K1u2bBnw/q1bt3LrrbdaXZSjq6urs3cJwgLJR22Sj/oko5HTa07fQTZ/9srnnKnhpBy+KOSMSaG8t+Ik7jhlovn+5g7TVb4TQ7wJ8jINEzPOz8PcAKZFB/D6r9NIDvflV8lHjkB+sqd8SOMIqs4W+RyXuX0NBgNOTlb1lQJwc3M79kLCbiQftUk+6pOMRk7OoSbzz4M97GuvfJydnPjn5TM51NRO1OHDtFfOHk9+dTMbdh40L5d2+JBvt79dMI2yhjYi/dzNYw9H+Hkwc7w/mcV1FNS0sLeikeTwgYeoG01skc+gm7+Wlhbzlb1gmli4v0O7zc3N/Pjjj/j7+w+7OEeVkpJi7xKEBZKP2iQf9UlGI2dbSR1gGlR56ji/QT3Gnvm4OjuZG79udy2eyO6D9eRWmc7pOyE2sNf9eiddn8cAnJkcRmZxHQA/5deMmebPFvkMuvn76KOP+M9//mP+/bXXXuO1114bcPnLLrtsWIU5sq1bt8rV0gqTfNQm+ahPMhoZjW2d5Faa9vxNDPXG221w/+Srlo+7i56/XTiNv32VQ6SfBycd1fwNZE7MkeW6m+CxwBb5DLr5mz59Ou7u7miaxr///W/mzZtHXFzvkzN1Oh1ubm5MmDCB+Pj4AZ5JCCGEEMfbjtJ6us90S43yt2cpwxbp78Hfl04f2mP83AnxdqWyqYOdpfUYjEac5ZQ0YAjNX2JiIomJiQC0t7dzwgknEB0dfdwKc2Th4eH2LkFYIPmoTfJRn2Q0Mnru7RpK8zdW8tHpdMyI8jdPCZdzqInJRx36/TG/mg07DnLu1AgWJATbqdKhsUU+Vl3wcckllwz7hcXAvLy87F2CsEDyUZvkoz7JaGR0z4YBMGMIzd9Yyif1cPMHsL2kDhcnJ/YdaiQuyIsthTX84/sDaMDPBTV8fPO8QR8atydb5DOsd7l3717y8/NpaWnp9zLqiy++eDhP77Dy8vIICgo69oLCLiQftUk+6pOMjr+2zi72lDcApvl8u4dGGYyxlE/PPZ4fZZXzwvcHaDcY+yzX2tnFh7vLuHzW+BGszjq2yMeq5q+pqYnHHnuM3Nxci8tJ8yeEEEKMvKyyBgyHp8RIHT+4q3zHognBXvi6O9PQZug17E03HZjPi3x3WymXzozC6fBwMWOZVWc+vvHGGxQVFXH77bfz7LPPAvDQQw/x9NNPc/rppxMbG8uaNWtsWqgjSU5OtncJwgLJR22Sj/oko+Mvs+f5fpH+Q3rsWMrHSadj+lHvP8TblXOnhnNCTABPLp3GrPGm+4tqW0gvrB35IofIFvlY1fxt27aN0047jblz5+LhYRpbR6fTER4ezooVKwgJCbE4DIywrKJibE1FM9ZIPmqTfNQnGR1/WwpqzD8P5Xw/GHv5pEb13vP524UJPHLWZJ67NJX58cFc2mMquPXb1J960Bb5WNX8NTc3M3686bi4u7s7AG1tbeb7p02bxo4dO4ZdnKOqqak59kLCbiQftUk+6pOMjq9Dje3sKDVd7BEb6ElkPwMgWzLW8pkVfWRGkJQIX87sMfUbwIKJwYR6m2bN+C63ir9+kUO7oWtEaxwKW+RjVfMXGBhonlvOxcUFX19fCgsLexWmc4Bj5seLi4uLvUsQFkg+apN81CcZHV9f5hwyn8d2+qTQIT9+rOUzOdyX60+KZW5cEI+eM7lPf+Ls5MQ1JxwZum79thJueCuTtk41G0Bb5GPVBR/Jycns3LmTpUuXAjB37lw++OADnJycMBqNfPzxx0yfPrTBGMURqamp9i5BWCD5qE3yUZ9kdHx9cXhoE4DTksIsLNm/sZjPTfMnWLz/0lTThR5Pf5NLu8FIdkUjm/ZWsCRl3AhVOHi2yMeqPX/nnnsus2fPprOzEzCN+5eYmMi6det49913mTBhAsuXLx92cY5qy5Yt9i5BWCD5qE3yUZ9kdPyUN7Sx86DpkG98sBcTgoc+Jpwj5qPT6bgkNYonLpxmvm1zXnW/y3YZNXaU1FHX2jlS5fVii3ys2vMXHR3da3YPb29vHn74YZqbm3FycjJfBCKEEEKIkfNVTs+9fkM/5OvoZkcH4O/hQl1rJ78U1NBhMOLq3Hs/2Ys/5vPyTwVE+rmzfvmJfe4fDWxasZeXlzR+NhAaKhusyiQftUk+6pOMjp+vcirNP59mxfl+4Nj56J10zJtgGkC5pbOr1xR5AB0GI/85fEVwaX0bPxX0v3fweLJFPoPa8/ftt99a9eQLFy606nGOzs/PcQfkHA0kH7VJPuqTjI6PLqPG3opGAKL8PYgNtG4aMEfPZ358MB9llQPwfV4VJ8QGmu/bnFdFfZvB/PsXew+xMCFkROuzRT6Dav5eeOEFq55cmj/r7N+/nzlz5ti7DDEAyUdtko/6JKPj42B9q3nqsoQQb6ufx9HzOTE2EL2Tji6jxvd5Vdy1eKL5CuEPDzeF3b7LraKtswt3F/2I1WeLfAbV/D333HO9fm9ubub555/H09OTs846i3HjTFfDlJaW8umnn9La2sqtt946rMKEEEIIMXgHqprNP8dbcaGHMPF2cyY1yp+tRbUcrG+joKaFuCAvqps7+PFA78O8LZ1d/FxQw6KJpr1/Te0Gqps7iAn0tEfpgzaoc/5CQkJ6/ffxxx/j6+vLypUrOfHEE80XgJx00kk88sgj+Pj48NFHHx3v2sespKQke5cgLJB81Cb5qE8yOj7ybNT8ST5wcnyQ+ef/7SoD4LPscro00wiKk8N9zPd/sc90kU1Tu4Flb2zl4pd/Np8XeDzYIh+rLvhIT09nzpw5/Q7k7OTkxJw5c0hPTx92cY6qunrkTyAVgyf5qE3yUZ9kdHzkVTWZf7ZmiJdukg+cMjEEZydTj/NWRjEZRbW8k3GkoXv4V8n4upsOnn6XW0VrRxf/3V5KUW0LAK/8XIDBaDwutdkiH6uaP03TKC0tHfD+khL158ZTWVVVlb1LEBZIPmqTfNQnGR0f3Xv+nJ10RAdYf9hR8oEIPw+WnRADmC6kuXndNsoaTNPYzhzvT0KIN6ccPtTb2tnF09/m8tbWIvPjK5s6+PHA8Zkmzxb5WNX8paWl8fnnn/Phhx/S3t5uvr29vZ3//e9/fPHFF8yePXvYxTkqvX7kThwVQyf5qE3yUZ9kZHuGLiOFNaa9TjGBnrjorR/JTfIxue7EGHMT3T1dXqi3G6vOngzA1WnRuB5ez//dXkpNS+9Bn9/fOfBOsuGwRT5WDfJ83XXXcejQId544w3eeustAgJMkybX1tbS1dVFUlIS11577bCLc1SzZs2ydwnCAslHbZKP+iQj2yuqbcVgNLUowznkC5JPNzdnPQ+ekcRN67YB4OWq5+mLpxPu6w5AbJAXN82fwDPf5vZ6nK+7Mw1tBn48UE1FYxthPu42rcsW+Vj1p4GnpyerVq3i7rvvZtGiRURGRhIZGcmiRYu45557WLVqFV5ecqWRtTIyMuxdgrBA8lGb5KM+ycj2ep7vN9wrfSWfI2ZFB3DvaYmcFBfIs5fM6DOEzpWzxzNt3JFx9+bHB3HZzCgAjBo8vmmfzaeBs0U+Vu3565aWlkZaWtqwixC9dXV12bsEYYHkozbJR32Ske31vtLX+jH+QPI52iWpUVySGtXvfXonHSvPTuaW9dto6eji1pPj8XZzZu0vRXR0Gdl8oJorXv2Fx89PYXqkbQbPtkU+o29COgcQHBxs7xKEBZKP2iQf9UlGttez+RvuYV/JZ2jGB3jy3oqT2HTrySSEeBPu685jS6bid/hq4KrmDu77YBetHbZpqm2RjzR/CgoKCjr2QsJuJB+1ST7qk4xsr3uAZzdnJyL9PIb1XJLP0LnondA7HRn+bkFCMG9fd4L5kHB1cwdvZRQN9PAhsUU+0vwpaN++ffYuQVgg+ahN8lGfZGRbzR0GSupMV/rGBXn1akKsIfnYRoi3Gw+fNQn94TGR39hSRG1Lx7Cf1xb5SPMnhBBCjGKb86o4fKEvUyN87VuM6CU20Ivzp0UA0NzRxcs/Fdi3oMOk+VPQxIkT7V2CsEDyUZvkoz7JyLa+2HvI/PNpSaHDfj7Jx7ZumBuHu4sTJ8UFsiQlYtjPZ4t8rLrad8OGDSxYsIDAwMBhF2BrRqOR9evX8/3331NXV0dgYCALFy7koosuMk9Hp2ka69ev58svv6S5uZlJkyaxYsUKIiKGH4ot1NfXm8dOFOqRfNQm+ahPMrKdpnYDP+abZpII8nJlRpT/sJ9T8rGtYG831l93AhHDPBezmy3ysWrP3zvvvMMtt9zCqlWr+Prrr2ltbR1WEba0YcMGPv/8c66//nr+/ve/c9VVV7Fx40Y++eQT8zIffPABn3zyCTfccAN/+tOfcHNzY/Xq1XR0DP9YvC0cOnTo2AsJu5F81Cb5qE8ysp3v86ro6DLNIXtqYuiwz/cDyed4sFXjB7bJx6rm74UXXuDKK6+kqamJf/7zn9x444089dRTZGZmYjxOExkPVk5ODrNnz2bmzJmEhoZy4oknMm3aNHJzTSNwa5rGxx9/zNKlS0lLSyMmJobbbruN2tpa0tPT7Vq7EEIIMRRf7OtxyHfS8A/5Csdg1WHfwMBAlixZwpIlSygqKmLz5s388MMP/PTTT/j4+DB37lxOPvlku5w3kJiYyJdffsnBgwcZN24cBQUF7Nu3j2uuuQYwdcx1dXVMmzbN/BhPT08SEhLIyclh3rx5I17z0ebMmWPvEoQFko/aJB/1SUa20dRu4Kf8agCCvVxtNoiw5KM2W+QzrBk+AKKjo7nyyiu58soryc7O5qOPPuKzzz7js88+Izw8nAULFnDaaafh52ebD+WxXHDBBbS2tnLHHXfg5OSE0Wjk8ssv5+STTwagrq4OoE89fn5+5vv6k5WVhV6vZ+bMmWRnZ9Pa2oqPjw9xcXHs3LkTMK0LTdMoLi4GYPr06eTl5dHU1ISXlxcTJ05k+/btAERFRaHX6yksLAQgJSWFwsJCGhoaaGpqYsGCBWRmZgIwbtw43NzcyM/PB2DKlCmUlpZSV1eHm5sb06ZNM++1DA8Px8vLi7y8PACSk5OpqKigpqYGZ2dnZs6cSXp6OpqmERISQkBAADk5OYCpca6traWyshInJydmz55NZmYmBoOBoKAgQkJC2Lt3LwAJCQk0NjZSUVEBmD6M27dvp6Ojg4CAACIiItizZw8AEyZMoLW1lbKyMsA0L2FWVhZtbW34+fkxfvx4du/eDUBsbCydnZ2UlpomxE5NTWXfvn20tLTg7e1NfHw8O3bsAGD8+PEA/a5vT09PkpKS2LbNNCdjZGQkLi4uFBQUADB16lSKi4upr6/H3d2dKVOmmKfMiYiIwMPDgwMHDgAwefJkysrKqK2txdXVFU3T6Ow0TdcTFhaGj4+Pec/ypEmTqKyspLq62ry+t27ditFotLi+dTodaWlp5vUdGBhIWFgY2dnZAMTHx9Pc3Ex5eTlgml1n586dtLe34+/vT2RkJFlZWQDExcXR3t7OwYMHAXp9Zn19fYmJiWHXrl0AxMTE0NXVRUlJCQAzZsxg//79NDc397u+dTodRUWmsaqmTZtGfn4+jY2NeHh4kJycbP7M9re+S0pKzJ/ZlJQUtm7deszPrIuLC6mpqWzZsgWA0NBQ/Pz82L9/PwBJSUlUV1dTVVWFXq9n1qxZfPHFF/j6+hIcHExQUJB5WISJEydSX19vPmQyZ84ctm3bRmdn5zHX9+zZs9m1a5d5fUdFRQ34mR2J74ij1/do+45obW1l1qxZY/Y7YsaMGebP7PH8jjjQ5kZnl+ky35OifSgpLrbJd0RjYyOaZnresfgdkZGRQVdX16j9jvj222/x8/Pr9zuie30ei07rTngYOjo6SE9P5/vvvzd/CKZPn46zszOZmZk4Oztz2223jchfEz/88ANvvvkmV199NePHj6egoIDXXnuNa665hkWLFrFv3z4efvhh1qxZ0+uEySeffBKdTscdd9zR6/kaGhpIT08nLS0NX9+RuYR+y5Yt8peXwiQftUk+6pOMbOMf3+fxys+mf/gfWzLVJlf6guSjOkv5DLZnsXrPn6Zp7Ny5k++//5709HTa2tqIjY3l6quvZv78+eY9a7W1tTz99NO8/vrrI/JhevPNNzn//PPNh2+jo6OprKxkw4YNLFq0CH9/f6Dv1TL19fXExsYe9/oGQ8WrqMURko/aJB/1SUa2setgg/lnW47vJ/mozRb5WNX8vfbaa/z000/U1dUREBDA6aefzsKFC8272HsKCAhg8eLFPP/888MudjDa29txcup9HYuTk5N5F3ZoaCj+/v7s2rXL3Oy1tLSQm5vLGWecMSI1HktYWJi9SxAWSD5qk3zUJxkNX5dRY0+5qfkL8XYlzMfNZs8t+ajNFvlYdbXvl19+ydSpU3nooYf4xz/+YT7EOpBJkyZx8803W13kUMyaNYv33nuPzMxMDh06xJYtW/jwww9JS0sDQKfTcfbZZ/Pee++xdetWioqKeO655wgICDAvY2/d5xQINUk+apN81CcZDV9+dTPNHV0ATI3wM49jawuSj9pskY9Ve/5efPFF3N3dB718aGgooaEjcwn68uXLWbduHS+99BL19fUEBgZy+umnc/HFF5uXOf/882lvb2fNmjW0tLQwadIkHnzwQVxdXUekRiGEEGI4dpcdOeSbMk6mdBNDY1Xz1934GY1GDhw4YL4yJjQ0lAkTJvQ57DqSPDw8uPbaa7n22msHXEan03HZZZdx2WWXjVxhQxAfH2/vEoQFko/aJB/1SUbDt/tgvfnnqeNsO5qG5KM2W+Rj9QUf33zzDW+99Rb19fW9bvf19eWKK65g8eLFwy7OUTU3NxMUFGTvMsQAJB+1ST7qk4yGb9fhPX96Jx3JYT42fW7JR222yMeq5u/zzz/npZdeIjY2lksuucQ8J+7Bgwf54osvWLNmDQaDQZkLKEab8vJyoqOj7V2GGIDkozbJR32S0fA0tRvIr2oGIDHEG3cXvU2fX/JRmy3ysar5++CDD5g0aRIPP/wwzs5HnmLq1KksXryYRx99lI0bN0rzJ4QQQtjYzoP1dA/QO1XO9xNWsOrkvLq6Ok466aRejV83Z2dn5s6d2+dwsBi82bNn27sEYYHkozbJR32SkfVaOgz8/av95t9tNaVbT5KP2myRj1XNX1xcnHkanv6UlZUpM2DyaNQ99ZZQk+SjNslHfZKRdTRN40+b9lFQ0wLAxBBvFk0MsfnrSD5qs0U+VjV/1113HT/99BMff/wxHR0d5ts7Ojr48MMP+emnn1i+fPmwi3NU7e3t9i5BWCD5qE3yUZ9kZJ2vcir5LNs0V7KXq57Hz5+Km7Ntz/cDyUd1tshnUOf83X333X1uc3JyYu3atbz55pvmadJqa2vp6uoiICCAF154gb/+9a/DLtARdU9BJ9Qk+ahN8lGfZDR0XUaNNT8cMP/+4JmTiA7wPC6vJfmozRb5DKr58/b27jN6uI+Pj/kq324jNZDzWBcVFWXvEoQFko/aJB/1SUZD98W+CvKrTYd7p0f6cXrS8fv3VvJRmy3yGVTzt3LlymG/kBi83bt3M2fOHHuXIQYg+ahN8lGfZDQ0XUaNl34sMP9+47w4m07ndjTJR222yMd+U3EIIYQQ4pg+31thvsgjNcqPtOgAO1ckRjtp/hQkV0qrTfJRm+SjPsloaD7eU27++Ya5x3evH0g+qrNFPtL8Kaizs9PeJQgLJB+1ST7qk4wGr93QRWZxHQAh3q7MHoG9fpKP2myRjzR/CiotLbV3CcICyUdtko/6JKPB21FaT7vBCMAJsYHHfa8fSD6qs0U+0vwJIYQQivqloMb88wkxgXasRIwl0vwpaObMmfYuQVgg+ahN8lGfZDR4PZu/OSPU/Ek+arNFPtL8KSg7O9veJQgLJB+1ST7qk4wGp7alg32HmgBICvUm0Mt1RF5X8lGbLfIZ1Dh/l112mVVPvm7dOqse5+haW1vtXYKwQPJRm+SjPslocLYU9jjkGztyh3wlH7XZIp9BNX8XXXRRn5NMt2zZQklJCdOnT2fcuHGA6STEnTt3Mn78eNLS0oZdnKPy8fGxdwnCAslHbZKP+iSjwfmloNb880g2f5KP2myRz6Cav0svvbTX71988QUNDQ088cQT5savW0lJCY8++qh5vl8xdHFxcfYuQVgg+ahN8lGfZHRsmqaZz/dzc3ZieqTfiL225KM2W+Rj1Tl/Gzdu5Mwzz+zT+IFpzrlf/epXbNy4cdjFOaqdO3fauwRhgeSjNslHfZLRsRXUtHCoqR2A1Ch/3Jz1I/bako/abJGPVc1fdXU1zs4D7zTU6/VUV1dbXZQQQgjhyH7ucZXviSN4yFc4Bquav+joaD777DNqamr63FddXc2mTZuIjo4ednGOStad2iQftUk+6pOMjm1LgX0u9gDJR3W2yGdQ5/wdbdmyZaxevZrbb7+dtLQ0wsPDASgvLyc9PR1N0/jNb34z7OIclaZp9i5BWCD5qE3yUZ9kZFmHwcjWYtPFHkFersQHe43o60s+arNFPlY1f5MmTWL16tWsW7eO9PR0Ojo6AHB1dWX69Olceuml8pfDMBQXFxMREWHvMsQAJB+1ST7qk4ws23WwnrZO05RuJ47QlG49ST5qs0U+VjV/YNrteM8992A0GmloaADA19cXJycZN1oIIYSw1i+FIz+rh3AsVjd/3ZycnHB1dcXd3V0aPxuZPn26vUsQFkg+apN81CcZWfZzryndRn7YNMlHbbbIx+puLS8vj9WrV3P11VezfPly9uzZA0BDQwN/+ctfyMrKGnZxjiovL8/eJQgLJB+1ST7qk4wGVtXUzt7yRgAmhngT7O024jVIPmqzRT5WNX/79u3j//2//0d5eTknn3xyr5MPfX19aWlp4fPPPx92cY6qqanJ3iUICyQftUk+6pOMBvb1/kq6/0VdkBBslxokH7XZIh+rmr+3336byMhInnzySa644oo+90+ZMoXc3NxhF+eovLxG9souMTSSj9okH/VJRgP7KueQ+edTE0PtUoPkozZb5GNV85eXl8eiRYtwcXHp9yqkwMBA6urqhlubw5o4caK9SxAWSD5qk3zUJxn1r7alg8ziOgDG+3uQEGKfJkzyUZst8rGq+dPr9RbHmampqcHd3d3qohzd9u3b7V2CsEDyUZvkoz7JqH/f5lZhPPxP6ymJISM+xEs3yUdttsjHquZv4sSJ/Pzzz/3e19bWxjfffMPkyZOHVZgQQgjhSL7cd+SQ72I7HfIVjsGq5u/SSy/lwIEDPPbYY2zbtg2AgoICvvzyS+6//34aGhq46KKLbFqoI4mKirJ3CcICyUdtko/6JKO+apo7SC8yzeoR7uvG5HAfu9Ui+ajNFvlYNc7fxIkTeeCBB3jxxRd5/vnnAXjjjTcACAsL44EHHiAmJmbYxTkqvV5v7xKEBZKP2iQf9UlGfT3zbS5dh4/5npoYardDviD5qM4W+Vg9yPPUqVN5+umnKSgooKysDE3TCAsLY8KECXb90I4FhYWFhIWF2bsMMQDJR22Sj/oko962FtXyUVY5AD5uzvx6jn13nkg+arNFPlY1f99++y3JycmEhoYSGxtLbGxsr/sPHTpEdnY2CxcuHFZxQgghxFjWYTDy2KZ95t9vWxBPkJerHSsSjsCqc/5eeOEFcnJyBrw/NzeXF154weqiHF1KSoq9SxAWSD5qk3zUJxkd8fqWQopqWwBIGefLBdPH2bkiyUd1tsjnuEzG29bWJucMDENhYaG9SxAWSD5qk3zUJxmZFNW28OrPpnWh1+l44PRJOClw2pTkozZb5DPow76FhYUUFBSYf8/Ozqarq6vPcs3NzXz++edEREQMuzhH1dDQYO8ShAWSj9okH/VJRqBpGn/+fB8dXUYArpw9nomh3nauykTyUZst8hl087dlyxb+85//mH//4osv+OKLL/pd1tPTk9tuu23YxTkqDw8Pe5cgLJB81Cb5qE8ygq9zKtlSeGRolxvmxtm5oiMkH7XZIp9BN3+nnXYas2bNQtM0HnzwQS699FJSU1P7LOfu7k5YWJgc9h2G5ORke5cgLJB81Cb5qE8ygk17K8w/37U4EQ9Xdf7NlHzUZot8Bt38BQQEEBAQAMAjjzxCVFQUvr6+wy7geKipqeHNN99k+/bttLe3Ex4ezi233EJ8fDxg2t2+fv16vvzyS5qbm5k0aRIrVqxQ5lB1ZmYmc+bMsXcZYgCSj9okH/U5ekaappFxeA5fbzdnTo4Ptm9BR3H0fFRni3yOy9W+GRkZdjvs29TUxMMPP4yzszMPPvggf//737nmmmvw8joyQfYHH3zAJ598wg033MCf/vQn3NzcWL16NR0dHXapWQghhOPIq2qmrrUTgNQof/RO9r/IQzgWq5q/yspK2traBry/ra2NyspKq4sajg8++ICgoCBuueUWEhISCA0NZfr06YSHhwOmv7g+/vhjli5dSlpaGjExMdx2223U1taSnp5ul5qPNm6c/S/1FwOTfNQm+ajP0TPKPLzXD2B2tL/d6hiIo+ejOlvkY/UMH5bk5eX12tM2krZu3cr06dN58skn2bNnD4GBgZxxxhmcdtppgGkA6rq6OqZNm2Z+jKenJwkJCeTk5DBv3jy71N2Tm5ubvUsQFkg+apN81OfoGW09PIcvwKzxAXaspH+Ono/qbJHPoJu/jz/+mI8//tj8+9q1a3nnnXf6LNfS0kJzczPz588fdnHWOHToEJ9//jnnnHMOF154IXl5ebz66qs4OzuzaNEi6urqAPDz8+v1OD8/P/N9/cnKykKv1zNz5kyys7NpbW3Fx8eHuLg4du7cCUB0dDSaplFcXAzA9OnTycvLo6mpCS8vLyZOnMj27dsB08TMer3ePF5PSkoKhYWFNDQ00NDQwKJFi8jMzARMXb6bmxv5+fkATJkyhdLSUurq6nBzc2PatGnmvZbh4eF4eXmRl5cHmE4MraiooKamBmdnZ2bOnEl6ejqaphESEkJAQID5EH5iYiK1tbVUVlbi5OTE7NmzyczMxGAwEBQUREhICHv37gUgISGBxsZGKipMJy3PmTOH7du309HRQUBAABEREezZsweACRMm0NraSllZGQCzZs0iKyuLtrY2/Pz8GD9+PLt37wYgNjaWzs5OSktLAUhNTWXfvn20tLTg7e1NfHw8O3bsAGD8+PEA/a5vT09PkpKS2LZtGwCRkZG4uLiYhyuaOnUqxcXF1NfX4+7uzpQpU8jIyAAgIiICDw8PDhw4AMDkyZMpKyujtrYWV1dXOjo6zFmEhYXh4+NDbm4uAJMmTaKyspLq6mrz+t66dStGo9Hi+tbpdKSlpZnXd2BgIGFhYWRnZwMQHx9Pc3Mz5eWmKaDS0tLYuXMn7e3t+Pv7ExkZSVZWFgBxcXG0t7dz8OBBgF6fWV9fX2JiYti1axcAMTExdHV1UVJSAsCMGTPYv38/zc3N/a5vnU5HUVERANOmTSM/P5/GxkY8PDxITk42f2b7W98lJSXmz2xKSgpbt2495mfWxcWF1NRUtmzZAkBoaCh+fn7s378fgKSkJKqrq6mqqkKv1zNr1iy2bduGv78/wcHBBAUFsW+fafaEiRMnUl9fz6FDh8yf2W3bttHZ2XnM9T179mx27dplXt9RUVEDfmZH4jvi6PU92r4jmpubSUtLG7PfETNmzDB/Zo/+jkhMSmJrYTUAns46JoZ6K/cdUVtba/4sjcXviIyMDLq6ukbtd0RmZiYBAQH9fkf0HJLPEp2madpgFty8eTObN28GYNu2bcTGxpovADE/mU6Hm5sbEyZM4Mwzz7TLXw9XXHEF8fHx/PGPfzTf9sorr5CXl8fq1avZt28fDz/8MGvWrOlV/5NPPolOp+OOO+7o9XwNDQ2kp6eTlpY2Yhe4bNmyRU62VZjkozbJR32OnNH+Q01cudbUqCxMCOZvF047xiNGniPnMxpYymewPcug9/zNnz/fvDdv1apVLF26VMkpYLq74Z6ioqL45ZdfAPD39wegvr6+V/NXX1/fZ45ie5kyZYq9SxAWSD5qk3zU58gZZRT3OOQbrd4hX3DsfEYDW+Rj1QUfjzzyiJKNH5h28Xbvyu528OBBQkJCANMuYX9/f/NhLzAdqs7NzSUxMXFEax1I965hoSbJR22Sj/ocNaM95Q28nVFs/n22guf7gePmM1rYIp9B7fnrPi9j8uTJvX4/lu7lR9I555zDww8/zHvvvcfcuXPJzc3lyy+/5MYbbwRMh6bPPvts3nvvPSIiIggNDeWdd94hICCAtLS0Ea+3P5bOPRT2J/moTfJRnyNmtGHnQR7/fB9dRtOZVnFBnsSH2OfCyGNxxHxGE1vkM6jmb9WqVQD8+9//xtnZ2fz7saxbt876yqyUkJDA3XffzVtvvcV///tfQkNDWbZsGSeffLJ5mfPPP5/29nbWrFlDS0sLkyZN4sEHH8TV1XXE6+2PXGmlNslHbZKP+hwto30Vjb0av8nhPqw+bypOOjXH93O0fEYbW+QzqAs+RtOeP1uzxwUfmqahU/RLQUg+qpN81OdIGXV2GVn2xlb2VzYBcOG0cdx7WiLOeqvOuhoRjpTPaGQpH5te8HF0EzcWmjqVpaeny5VWCpN81Cb5qM+RMnr5pwJz4zcxxJt7FG/8wLHyGY1skY/an0AhhBBilKpv7WTtL6Yx2PROOh45KxkXxRs/4RisnuGjsrKSb7/9loqKCpqbmzn66LFOp+Pee+8ddoGOqHsqOqEmyUdtko/6HCWj3WUNGA6f53fBtHEkhfnYuaLBcZR8Ritb5GNV87d582aef/55jEYjnp6eeHp69llGzhewnr2mxhODI/moTfJRn6NklF3eYP55ZpS//QoZIkfJZ7SyRT5WNX9vv/02kZGR3HnnnTIB9HGQl5dHUFCQvcsQA5B81Cb5qM9RMsquaDT/nBw+Ovb6gePkM1rZIh+rTj5oaGjg9NNPl8ZPCCGEGED3nj9vN2ei/D3sXI0QR1jV/E2cOJGqqipb1yIOS05OtncJwgLJR22Sj/ocIaPKpnYqmzoASA7zGVWnQjlCPqOZLfKxqvm79tpr+f777/n555+HXYDoq6Kiwt4lCAskH7VJPupzhIyyy0fnIV9wjHxGM1vkY9U5f9HR0Vx++eU89dRTuLm5ERQUhJNT7z5Sp9Px17/+ddgFOqKamhp7lyAskHzUJvmozxEy6nmxx+TwkZkgwFYcIZ/RzBb5WNX8ffbZZ7zyyiu4uroSHh7e79W+wnrOzlaPwCNGgOSjNslHfY6Q0Wi92AMcI5/RzBb5WPUM77//PklJSdx///3S+B0HM2fOtHcJwgLJR22Sj/rGekaappn3/Pl5uBDh627nioZmrOcz2tkiH6vO+WtpaWH+/PnS+B0n6enp9i5BWCD5qE3yUd9Yz6iisZ2alk5g9F3sAWM/n9HOFvlY1fxNnjyZoqKiYb+46N/Rs6UItUg+apN81DfWM9pSeOScrMmj7JAvjP18Rjtb5GNV87dixQqys7P54IMPaGxsPPYDxJCEhITYuwRhgeSjNslHfWM9o8/3HjL/vCBh9L3XsZ7PaGeLfKw65+/OO+9E0zTeeust3nrrLVxdXftc7Quwdu3aYRfoiAICAuxdgrBA8lGb5KO+sZxRXWsn6YW1AET4uo/KPX9jOZ+xwBb5WNX8nXDCCaPuHIbRJCcnhzlz5ti7DDEAyUdtko/6xnJG3+yvpOvwYbnTkkJH5b+VYzmfscAW+VjV/N16663DelEhhBBiLPpi75EBeE+bFGrHSoQYmFXn/InjKzEx0d4lCAskH7VJPuobqxnVtnSwtagOgHF+7iSHjb5DvjB28xkrbJGP1SMFNjQ0sGHDBrZt20ZlZSVgOgkxNTWVJUuW4O/vP+ziHFVtba2sP4VJPmqTfNQ3VjMaC4d8YezmM1bYIh+r9vwVFxdz11138dFHH+Hp6cmJJ57IiSeeiKenJx999BH33HOPDAUzDN3NtFCT5KM2yUd9YzWjHw5Um39enDh6D/mO1XzGClvkY9Wev5dffhmj0cjq1atJSEjodV9ubi6PPfYYr776Ko888siwC3RE/V05LdQh+ahN8lHfWMyos8tovso3wNNl1E3p1tNYzGcssUU+Vj1Dbm4uZ599dp/GDyAhIYGzzjqL/fv3D7s4RzV79mx7lyAskHzUJvmobyxmtKO0npbOLgBOjA3EaZQe8oWxmc9YYot8rGr+/Pz8cHFxGfB+V1dX/Pz8rC7K0WVmZtq7BGGB5KM2yUd9YzGjH3sc8p0bF2THSoZvLOYzltgiH6uav7PPPpvPP/+curq6PvfV1NSwadMmzj777OHW5rAMBoO9SxAWSD5qk3zUNxYz+jHf1PzpMO35G83GYj5jiS3yGdQ5fx9++GGf29zd3fnNb37DnDlzCA8PB6CsrIz09HTCw8NlbsBhCAoa3X81jnWSj9okH/WNtYzKG9rIq2oGYEqEL/6ernauaHjGWj5jjS3yGVTz98Ybbwx43+bNm/vcVlRUxBtvvMG5555rfWUOTOZVVJvkozbJR31jLaNvc49cfTnaD/nC2MtnrBmxuX2fe+65Yb+QGLy9e/fK1DoKk3zUJvmobyxl1GEw8saWI0ObLUgItmM1tjGW8hmLbJHPoJo/+StACCGE6GvjroNUNLYDMG9CEEmjdFYP4VhkMB8F9TeEjlCH5KM2yUd9YyWjDoORV38pNP9+w9w4O1ZjO2Mln7HKFvlYNcjzrbfeesxpa3Q6Hc8++6xVRTm6xsZGAgNH99ViY5nkozbJR31jIaPyhjae+y6PQ4f3+s2PD2JKhK+dq7KNsZDPWGaLfKxq/iZPntyn+TMajVRWVrJv3z7Gjx9PXNzY+AvIHioqKoiJibF3GWIAko/aJB/1jfaM1mUW89TXuRiMR0a1uHGM7PWD0Z/PWGeLfKze8zeQgoICVq9ezfz5860uSgghhFBRVlkDT361n+6+z9NFz28WxpMcPjb2+gnHoNOOw4B869evJyMjgz//+c+2fuoR19DQQHp6Omlpafj6ysYthBCOqrPLyDWvp5N7eEy/86dFcNuCBPw9Bp7xSoiRNNie5bhc8OHn50dJScnxeGqHsH37dnuXICyQfNQm+ahvtGb02i+F5sYvMdSb+09LGpON32jNx1HYIh+bN3+NjY189dVXMkL4MHR0dNi7BGGB5KM2yUd9ozGj0rpWXv25AAC9Tsf/+1UyzvqxOWDGaMzHkdgiH6vO+Vu1alW/t7e0tFBaWorBYOC2224bVmGOLCAgwN4lCAskH7VJPuobjRk9910enV2ms6SuShs/psfzG435OBJb5GNV86dpWr9DvYSEhJCSksIpp5xCZGTksItzVBEREfYuQVgg+ahN8lHfaMtoZ2k9X+w7BECgpwvXnRhr34KOs9GWj6OxRT5WNX8rV64c9guLge3Zs0em1lGY5KM2yUd9oykjTdN46uv95t9vnDcBbzer/ukcNUZTPo7IFvmMzRMWhBBCCBv4qaCGXWUNAMQFeXH+NNkrJkY/q/582bVrF/n5+SxZssR821dffcW7776LwWBg3rx5XHPNNTg5SW9pjQkTJti7BGGB5KM2yUd9oymj97aXmn++aV4czg7w79poyscR2SIfqz7F7777LgUFBebfi4qKePHFF/H19WXy5Ml88sknbNy4cdjFOarW1lZ7lyAskHzUJvmob7RkVNHYxvd5VQCEeLuyYGKwnSsaGaMlH0dli3ysav5KS0uJj483//7dd9/h4eHBo48+yh133MGpp57Kd999N+ziHFVZWZm9SxAWSD5qk3zUN1oy+mDnQfNMHhdMG+cQe/1g9OTjqGyRj1Wf5La2Njw8PMy/b9++nRkzZuDm5gZAQkIClZWVwy5uuDZs2MCll17Ka6+9Zr6to6ODl156ieXLl/PrX/+av/3tb9TV1dmtRiGEEOoxGI1s2HkQMI3rd8E0GcFCjB1WNX/BwcHk5eUBUF5eTnFxMdOmTTPf39TUhIuLfUc9z83N5fPPP+8z+fHatWvJyMjgzjvvZNWqVdTW1vLEE0/Yqcr+zZo1y94lCAskH7VJPuobDRn9lF9DZZNpMN2T44MI9XGzc0UjZzTk48hskY9Vzd/8+fP54osv+POf/8zq1avx8vIiLS3NfP+BAwfsOk5QW1sbzz77LP/3f/+Hl5eX+faWlha++uorli1bxtSpU5kwYQK33HIL+/btIycnx271Hi0rK8veJQgLJB+1ST7qGw0ZpRfWmn8+Z6pjXeE7GvJxZLbIx6rmb+nSpVxwwQVUV1cTHBzMPffcY26ympqayMrKYvbs2cMuzlovvfQSqampvfZGgqkp7erqIiUlxXxbZGQkwcHBSjV/bW1t9i5BWCD5qE3yUd9oyGh7aZ3555nj/e1Whz2MhnwcmS3ysWqoF71ezxVXXMEVV1zR5z5vb29efPHFYRdmrR9++IH8/Hwee+yxPvfV1dXh7Ozca28ggJ+f3zHP+8vKykKv1zNz5kyys7NpbW3Fx8eHuLg4du7cCUB0dDSaplFcXAzA9OnTycvLo6mpCS8vLyZOnGiekDkqKgq9Xk9hYSEAKSkpFBYW0tDQQFtbGwaDgczMTADGjRuHm5sb+fn5AEyZMoXS0lLq6upwc3Nj2rRppKenAxAeHo6Xl5f5sHxycjIVFRXU1NTg7OzMzJkzSU9PR9M0QkJCCAgIMDe+iYmJ1NbWUllZiZOTE7NnzyYzMxODwUBQUBAhISHs3bsXMJ3X2djYSEVFBQBz5sxh+/btdHR0EBAQQEREBHv27AFMl6W3traaT1KdNWsWWVlZtLW14efnx/jx49m9ezcAsbGxdHZ2UlpqGl4hNTWVffv20dLSgre3N/Hx8ezYsQOA8ePHA/S7vj09PUlKSmLbtm2Aqcl3cXExX6U+depUiouLqa+vx93dnSlTppCRkQGYRk/38PDgwIEDAEyePJmysjJqa2txdXXFz8+PLVu2ABAWFoaPjw+5ubkATJo0icrKSqqrq83re+vWrRiNRovrW6fTkZaWZl7fgYGBhIWFkZ2dDUB8fDzNzc2Ul5cDkJaWxs6dO2lvb8ff35/IyEjzX4NxcXG0t7dz8KDpfKWen1lfX19iYmLYtWsXADExMXR1dVFSUgLAjBkz2L9/P83Nzf2ub51OR1FREQDTpk0jPz+fxsZGPDw8SE5ONn9m+1vfJSUl5s9sSkoKW7duPeZn1sXFhdTUVPP6Dg0Nxc/Pj/37TYPuJiUlUV1dTVVVFXq9nlmzZtHc3MyWLVsIDg4mKCiIffv2ATBx4kTq6+s5dOiQ+TO7bds2Ojs7j7m+Z8+eza5du8zrOyoqasDP7Eh8Rxy9vkfbd0RXVxdNTU3Kfke0dWnkVDSb1q2XEwU52UP6jpgxY8ao/o7w9PQ01z8WvyMyMjLo6uoatd8RTU1NbNmypd/viJ4jsVii0zRNG9SSo0BVVRUPPPAAv//9783n+q1cuZLY2FiuvfZaNm/ezAsvvMBbb73V63EPPPAAU6ZM4eqrr+7znA0NDaSnp5OWloavr++IvI+WlhY8PT1H5LXE0Ek+apN81Kd6RlsKa7h1/XYALpoeyf1nJNm3oBGmej6OzlI+g+1ZxtR16wcOHKC+vp777ruPyy+/nMsvv5w9e/bwySefcPnll+Pn54fBYKC5ubnX4+rr6/H397dP0f3o/mtBqEnyUZvkoz7VM9pRUm/+eXqUnx0rsQ/V83F0tshnTE1QmJKSwt/+9rdet/3jH/9g3LhxnH/++QQHB6PX69m1axcnnngiAAcPHqSqqorExER7lCyEEEIxPc/3mxHpeM2fGPvGVPPn4eFBdHR0r9vc3Nzw8fEx37548WJef/11vL298fT05JVXXiExMVGp5i82NtbeJQgLJB+1ST7qUzkjg9HIroOmuXxDfdwI93W3c0UjT+V8hG3yGVPN32AsW7YMnU7HE088gcFgYPr06axYscLeZfXS2dlp7xKEBZKP2iQf9amYkaZpHKxvo6CmmdbOLsC010+n09m5spGnYj7iCFvkM+abv5UrV/b63dXVlRUrVijX8PVUWlpKZKSMJq8qyUdtko/6VMzoqW9yeWtrca/bpkf626cYO1MxH3GELfKxqvnbtWsX+fn5LFmyxHzbV199xbvvvovBYGDevHlcc801ODnIPIhCCCFGr6qmdtZllvS53dHG9xOOw6rm79133yU4ONj8e1FRES+++CLR0dGEh4fzySef4O/vzwUXXGCrOh1KamqqvUsQFkg+apN81KdaRu/vOEiX0TTq2cQQb7xc9cyPDyYhxNvOldmHavmI3myRj1W75kpLS4mPjzf//t133+Hh4cGjjz7KHXfcwamnnsp333037OIcVfeAk0JNko/aJB/1qZRRZ5eR/+4wDb7rpIMnl07jxStnseyEmGM8cuxSKR/Rly3ysar5a2trw8PDw/z79u3bmTFjBm5upomvExISqKysHHZxjqqlpcXeJQgLJB+1ST7qUymjr3IOUd3cAcDChBCHvLr3aCrlI/qyRT5WNX/BwcHmaVbKy8spLi7uNY9uU1MTLi4uwy7OUXl7O+ahhtFC8lGb5KM+VTIyalqvizwumxllx2rUoUo+on+2yMeqc/7mz5/Pf/7zH2pqaigpKcHLy4u0tDTz/QcOHCAiImLYxTmqnofUhXokH7VJPupTJaN3MkrYU94IQHywl1zgcZgq+Yj+2SIfq/b8LV26lAsuuIDq6mqCg4O555578PLyAkx7/bKyspg9e/awi3NU3ZNkCzVJPmqTfNSnQkYF1c288H2e+fe7Fk90yDH9+qNCPmJgtshnyHv+NE2jvb2diy66iCuuuKLP/d7e3rz44ovDLkwIIYQ4HgxGI498vId2gxEwHe5Niwm0c1VCjJwh7/kzGAwsX76cTz/99HjUI4Dx48fbuwRhgeSjNslHffbO6LWfC82He6MDPLltgRzm7Mne+QjLbJHPkJs/FxcX/P39cXYe85ODCCGEGGOyyxt46acCAPQ6HY+eMxl3F719ixJihFl1zt+iRYv49ttvMRgMtq5HAMXFxcdeSNiN5KM2yUd99sqow2Bk5SfZ5gGdl50Yw5QIX7vUojLZhtRmi3ys2n0XHR1Neno6d955J4sWLSIkJARXV9c+y51wwgnDLlAIIYSwhQ+zyjhQ1QxAUqg3K06KtW9BQtiJVc3f008/bf553bp1Ay5n6T4xsOnTp9u7BGGB5KM2yUd99shI0zTeyTiyx+SBMybhopf55/sj25DabJGPVc3fI488MuwXFgPLy8tj8uTJ9i5DDEDyUZvkoz57ZPRzQQ351aaZEVKj/ORwrwWyDanNFvlY1fzJh+L4ampqsncJwgLJR22Sj/rskdHbPfb6XT5Lrma1RLYhtdkin2FdstvZ2Ul+fj719fUkJSXh6yt/SdmCp6envUsQFkg+apN81DfSGe0pb+Cn/BoAxvm5szAhZERff7SRbUhttsjH6ubv448/5t133zVPMPzwww8zdepUGhoauOOOO7jqqqtYvHjxsAt0RElJSfYuQVgg+ahN8lHfSGVk1DT+nV7EPzfnm2+7bGYUeieZycMS2YbUZot8rDrb9euvv2bt2rXMmDGDm2++udd9vr6+TJkyhR9//HHYxTmqbdu22bsEYYHkozbJR30jldG/fsjnmW/z6OgyzeSREOzF+dPGjchrj2ayDanNFvlYtefvww8/ZPbs2dx+++00Njb2uX/ChAl88sknwy5OCCGEsEZGcS2vHB7MWQdcMXs8N8+fIAM6C4GVe/7Ky8tJTU0d8H5vb285YXQYIiMj7V2CsEDyUZvko77jnVF9ayf/76M9aId/v2XBBO44ZaI0foMk25DabJGPVc2fp6cnDQ0NA95fUlKCv7+/tTU5PBcXF3uXICyQfNQm+ajveGbU2tHF/Rt3caixHYDZ0f78Oi3muL3eWCTbkNpskY9VzV9qaipffvklzc3Nfe4rLi7myy+/ZNasWcMuzlEVFBTYuwRhgeSjNslHfccro9aOLu54bwdbi+oA8HN3ZtXZU+QCjyGSbUhttsjHqnP+Lr/8ch566CHuuusuc5P3zTff8NVXX/HLL78QEBDAxRdfPOzihBBCiMGoamrn3g92seug6aiUl6uepy6aTqiPm50rE0I9Ok3TtGMv1ld9fT1vv/02v/zyi3m4F3d3d0444QSuuuoq/Pz8bFqovTQ0NJCenk5aWtqIjWPY0tIi4ywpTPJRm+SjPltnlFXWwD0bdlLZ1AGAt5szz10yQ2bxsJJsQ2qzlM9gexarx/nz8/Pjpptu4qabbqKhoQGj0Yivry9OTjJX4nAVFxfLOEsKk3zUJvmoz5YZ/W93GY9t2ktnl2k/RqiPG09eOI2kMB+bPL8jkm1IbbbIZ1gzfIBpsmxN09DpdOh0cl6FLdTX19u7BGGB5KM2yUd9tsjIqGn8/ev9vJNRYr4tNcqPx5akEOTlOuznd2SyDanNFvlY3fyVlJSwbt06duzYQXu76aoqNzc3pk+fziWXXEJ0dPSwi3NU7u7u9i5BWCD5qE3yUZ8tMnono6RX43dJaiR3nDIRF70cfRou2YbUZot8rGr+srOz+dOf/oSmacyePZtx40wjph88eJCtW7eyfft2HnzwQZKTk4ddoCOaMmWKvUsQFkg+apN81DfcjHIrm3j+uzzz7w+ekcSF02VsOluRbUhttsjHquZv7dq1+Pn5sXLlSoKDg3vdV1VVxSOPPMLrr7/OY489NuwCHVFGRgZz5syxdxliAJKP2iQf9VmT0fd5VTz6STZuzk4YNc08ZduVs8dL42djsg2pzRb5WLV/vLi4mDPOOKNP4wcQHBzMGWecQXFx8bAKE0IIIQAONbbzyEd7qGvtpKKx3XxV74RgL245eYKdqxNi9LGq+QsJCcFgMAx4v8FgICgoyOqiHF1ERIS9SxAWSD5qk3zUN5SMNE3jD59m09hu+jdHf/jCQm83Z/5wzmTcnGXKNluTbUhttsjHqsO+F198MWvXrmXmzJnExsb2ui8/P59PP/2UZcuWDbs4R+Xh4WHvEoQFko/aJB/1DSWj93Yc5OeCGgBCvF1569oTaGrrxMfdBT8PmYbseJBtSG22yMeq5i8nJwc/Pz/uu+8+kpKSCA8PB6CsrIycnByio6PJyckhJyfH/BidTsd111037IIdwYEDB/o9pC7UIPmoTfJR32AzKqlt4elvcs2///5Xyfh7uOAvTd9xJduQ2myRj1XN32effWb+ed++fezbt6/X/UVFRRQVFfV5nDR/QgghBqPLqLHyk2xaO7sAWDp9HHPj5HQiIWzBquZv3bp1tq5D9DB58mR7lyAskHzUJvmobzAZ/Tu9iB2lpsFsI/3cuX1RwvEuSxwm25DabJGPjIapoLKyMnuXICyQfNQm+ajvWBmV1bfyrx/zAdABj5w9GU/XYU9IJQZJtiG12SIfaf4UVFtba+8ShAWSj9okH/UdK6Pnvz9Au8E0jt/ls8aTGuU/AlWJbrINqc0W+Vj9p9S2bdv48MMPyc/Pp6WlBU3T+iwjh4et4+oq81KqTPJRm+SjPksZZZU18Fl2BQB+Hi7cMDd2hKoS3WQbUpst8rFqz9/PP//M448/Tn19PXPnzkXTNObNm8e8efNwdXUlJiaGiy++eNjFOaoZM2bYuwRhgeSjNslHfQNlpGkaT3293/z7jXPj8HGXK3tHmmxDarNFPlY1fxs2bCAhIYG//OUvXHrppQAsXryY3/72tzzxxBPU1tYSGho67OIc1ZYtW+xdgrBA8lGb5KO+gTLatLeC7Ycv8ogJ9GTp9HEjWZY4TLYhtdkiH6uav5KSEubNm4eTkxN6vWl09e4ZP0JDQznzzDP54IMPhl2cEEIIx9DUbuDvXx8Z0+93ixJw1stp6UIcD1ZtWW5ubjg7m04X9PLywtnZmbq6OvP9fn5+HDp0yCYFOqKwsDB7lyAskHzUJvmor7+M/rn5ANXNpjl7FyYEMz9eBhm2F9mG1GaLfKy64GPcuHGUlJSYf4+NjeW7777j5JNPpquri82bN9ttdPD333+fLVu2UFpaiqurK4mJiVx99dWMG3fk8EFHRwevv/46P/74I52dnUyfPp0VK1bg7+9vl5qP5uPjY+8ShAWSj9okH/UdndHHWWW8u830b4q7ixN3LZ5oj7LEYbINqc0W+Vi15y8tLY309HQ6OzsBWLp0KVlZWVx77bWsWLGCvXv3csEFFwy7OGvs2bOHM888k9WrV/P73/+erq4u/vjHP9LW1mZeZu3atWRkZHDnnXeyatUqamtreeKJJ+xSb39yc3OPvZCwG8lHbZKP+rozMhiN/O3LHB75OBvj4QEjrj8plgg/mVvWnmQbUpst8rFqz9+SJUtYsmSJ+fdZs2axcuVKfvnlF5ycnJg5cyZTp04ddnHWeOihh3r9fuutt7JixQoOHDjA5MmTaWlp4auvvuL2228313jLLbdwxx13kJOTQ2Jioj3KFkIIh/PCdwdYl3nkKNKF08bx67QYO1YkhGMYUvPX0dHB1q1bOXToED4+PsycOZOAgAAAkpOTSU5OPi5FDkdLSwsA3t7egGlC5K6uLlJSUszLREZGEhwcrEzzN2nSJHuXICyQfNQm+ahv0qRJ7Kto5K2txQA4O+m497RELpweaefKBMg2pDpb5DPo5q++vp7f//73vS7kcHV15Z577mHatGnDLuR4MBqNvPbaayQlJREdHQ1AXV0dzs7OeHl59VrWz8+v10UrR8vKykKv1zNz5kyys7NpbW3Fx8eHuLg4du7cCUB0dDSaplFcbPpCmz59Onl5eTQ1NeHl5cXEiRPZvn07AFFRUej1egoLCwFISUmhsLCQhoYGOjo6OPHEE8nMzARM51i6ubmRn2+a7mjKlCmUlpZSV1eHm5sb06ZNIz09HYDw8HC8vLzIy8sDTE15RUUFNTU1ODs7M3PmTNLT09E0jZCQEAICAsjJyQEgMTGR2tpaKisrcXJyYvbs2WRmZmIwGAgKCiIkJIS9e/cCkJCQQGNjIxUVpsFY58yZw/bt2+no6CAgIICIiAj27NkDwIQJE2htbTVPSTNr1iyysrJoa2vDz8+P8ePHs3v3bsB0/mhnZyelpaUApKamsm/fPlpaWvD29iY+Pp4dO3YAMH78eIB+17enpydJSUls27YNMDX4Li4uFBQUADB16lSKi4upr6/H3d2dKVOmkJGRAUBERAQeHh4cOHAAMM2jWFZWRm1tLa6urvj4+JjXQ1hYGD4+Pubd8JMmTaKyspLq6mrz+t66dStGo9Hi+tbpdKSlpZnXd2BgIGFhYWRnZwMQHx9Pc3Mz5eXlgOnUi507d9Le3o6/vz+RkZFkZWUBEBcXR3t7OwcPHgTo9Zn19fUlJiaGXbt2ARATE0NXV5f5HN4ZM2awf/9+mpub+13fOp2OoqIiAKZNm0Z+fj6NjY14eHiQnJxs/sz2t75LSkrMn9mUlBS2bt16zM+si4sLqamp5qENQkND8fPzY/9+01hwSUlJVFdXU1VVhV6vZ9asWWzbtg0PDw+Cg4MJCgpi3759AEycOJH6+nrzd9icOXPYtm0bnZ2dx1zfs2fPZteuXeb1HRUVNeBndiS+I45e36PtO6LLaOT5vTq6Dk8OcOWMUGYHGs05j/bviBkzZpjfy2j8jgDMOY7F74iMjAy6urpG7XdERkYGXl5e/X5HdK/PY9Fp/U3N0Y9XXnmFTZs2cfbZZzN16lTKy8v573//i6enJ88+++ygXmykvfjii2zfvp1HH32UoKAgADZv3swLL7zAW2+91WvZBx54gClTpnD11Vf3ur2hoYH09HTS0tLMG8XxtmXLFubMmTMiryWGTvJRm+Sjrs4uI1/sO8Ta7/eS12Cavm1CsBdvXpOGiwzrogzZhtRmKZ/B9iyD3vO3Y8cOFixYwDXXXGO+zd/fn6effpqDBw/2uppWBS+//DKZmZmsWrXK3PiBqWaDwUBzc3OvvX/19fXKXO3bPYyOUJPkozbJR025lU08/GEWuVXNvW5/8PQkafwUI9uQ2myRz6C3uKqqqj7Hmbt/t3S4dKRpmsbLL7/Mli1b+H//7//1mWlkwoQJ6PV682EvgIMHD1JVVaXE+X5g2iUs1CX5qE3yUYtR03g7o5hlb2zt1fjFBHry+JKpTI/yt19xol+yDanNFvkMuvkzGAx9JhN2cTHNuWg0GoddiK28/PLLfP/999x+++14eHhQV1dHXV0dHR2mwUM9PT1ZvHgxr7/+Ort37+bAgQO88MILJCYmKtP8dZ/nINQk+ahN8lFHZVM7v313O09+tZ+OLtO/E/HBXtwxw5N3l5/AqUkyDaiKZBtSmy3yGdK+w0OHDplPcIUjV9KWlZXh6enZZ/kJEyYMs7yh27RpEwArV67sdfstt9zCokWLAFi2bBk6nY4nnngCg8FgHuRZFSo106IvyUdtko8a6lo7ufaNrRxqajffduWs8dyyYAI7MjPQ6XR2rE5YItuQ2myRz5Cav3Xr1rFu3bo+t7/00ksDLj/S1q9ff8xlXF1dWbFihVINX08hISH2LkFYIPmoTfJRw7uZJebGL8TblUfOmswJsYGm3yUjpUk+arNFPoNu/m6++eZhv5gYnO6xE4WaJB+1ST7219bZxfrD07XpdTpevGIWkf5HZu2QjNQm+ajNFvkMuvnrPmQqjr+cnBy5zF5hko/aJB/7+9/uMupaTdN/njYptFfjB5KR6iQftdkiH7meWwghhE3UtXZSWtdqnrkD4Ndp0XasSAjRH2n+FKTKVceif5KP2iSfkZdf3czLPxWwKbuCnrMGzIkJICnMp8/ykpHaJB+12SIfGVlTQbW1tfYuQVgg+ahN8hlZ6zNLuOyVX/jsqMYPYPmJsf0+RjJSm+SjNlvkI82fgiorK+1dgrBA8lGb5DNyDlQ18/ev95ubPn8PFy6aHsktJ0/gn5elMiu6/xPTJSO1ST5qs0U+cthXQTL+ldokH7VJPiPDqGk8tmkvBqOp9btg2jjuOCUBT9dj/7MiGalN8lGbLfKR5k9BaWlp9i5BWCD5qE3yGRkbdh5ke2k9AOP9Pbhr8UTcXfSDeqxkpDbJR222yEcO+yooMzPT3iUICyQftUk+x9+3uZU88eV+8+/3n5E06MYPJCPVST5qs0U+sudPQQaDwd4lCAskH7VJPsfXR7vL+MOne+nSTId7l6REMCcmcEjPIRmpTfJRmy3ykeZPQYGBQ/siFSNL8lGb5HP8fJ9XxapPss0XeJyZHMYDpycN+XkkI7VJPmqzRT7S/CkoLCzM3iUICyQftUk+x8f+Q038/n9Z5sbvktRI7j41EScrTj6XjNQm+ajNFvnIOX8Kys7OtncJwgLJR22Sj21pmsa3uZX87r0dtHR2AXBaUqjVjR9IRqqTfNRmi3xkz58QQoh+FVQ3s/KTbLLKGsy3TQ734ZGzkq1u/IQQ9ifNn4Li4+PtXYKwQPJRm+RjG9/sr2Tlx3to7ugy3zY1wpe/XpAypCt7+yMZqU3yUZst8pHmT0HNzc0EBQXZuwwxAMlHbZLP8BxqbGfNDwfYuKvMfFtsoCe3LYhnQUKwTQaYlYzUJvmozRb5yDl/CiovL7d3CcICyUdtko91uowab6YXsfSln3o1fmdMCuX1X6excGKIzWZ+kIzUJvmozRb5yJ4/IYRwIF1GjQ07D7KtpA4AZycdQV6u7DrYYL4NwNvNmZvmx3FpapRM9yXEGKPTNE079mKOq6GhgfT0dNLS0vD19R2R19Q0Tb5sFSb5qE3yGVi7oYv/99EevsoZeGJ4HXDpzChWzI3D38PluNQhGalN8lGbpXwG27PIYV8F7dy5094lCAskH7VJPv0rqm3htvXbLTZ+4b5u/OPyVO4+NfG4NX4gGalO8lGbLfKRw74Kam9vt3cJwgLJR22OnE+Hwcg3+yvJq2pifIAnMYGelNS18nN+DZ9lV5inZPNw0fPoOZOZGOJNu8FIdXM7nUaN1Eh/PFyHdyXvYDhyRqOB5KM2W+QjzZ+C/P397V2CsEDyUdtI5HOgqpldB+tZnBiCj7tpD5k9D5XVtXbydkYx7+8opbal0+KywV6uPLl0GsnhRw4JTQj2Ot4l9iLbkNokH7XZIh9p/hQUGRlp7xKEBZKP2o5nPl1GjbW/FPKvH/PpMmq8vqWQJy6cxn+2l/LejlImhfpwwfRxnJ4Udtz3oBk1jayyBr7cd4j3dxw0z74xEB83Z65Oi+ayWVF4udr3q1+2IbVJPmqzRT7S/CkoKyuLOXPm2LsMMQDJR23HK5/SulYe+XgPO0rrzbcV1bZy6Su/mOe73VXWwK6yBv7yRQ5pMQEsTgzltMTQYTeChi4ju8oaaO3sorndwM8FNWzOq6ampaPXcs5OOk5JDGFxYigH61opqm1hnJ8Hk8J9mB7pZ/emr5tsQ2qTfNRmi3zU+CYQQggFaZpGaV0rPxXU8Px3eebZLpx0EOztxqHGdnPjpwPzz+0GI5vzqtmcV80TX+ZwZnIYp0wMYUaUP03tBurbOvFzdyHA0xW9U99DxW2dXRTXtqJh2rv32i+FHKxvG7BOF72O81PGseyEGMJ93W27EoQQY440fwqKi4uzdwnCAslHbbbIp7nDwLqMEtZlFlNz1Dl04/zcefTsycQGeXHPhp1sK6kn0s+dP5+fQmeXkQ93l/NtbiVVzR2Hn6uL93Yc5L0dB/u8jpMOXPSmQRfigrw4b2oE9W2dvLW1mKZ2g8Ua3V2cODEmkJMTgpk/IZhAL9dhv++RItuQ2iQftdkiH2n+FCRXWqlN8lHbcPIprWvlg10HeX/HQepa+144cc6UcO4+NRFvN9NX5z8vn0luZRNxQV7mJm7qOD/uPT2RXQcb+N+ug3y2t4K2TmO/r2fUTHsJAfZWNLK3onHA2k6MDWTaOD90OpgU5sPs6IBhz7FrL7INqU3yUZtc7TtGHTx4kKioKHuXIQYg+ajNmnzKG9p4+ptcvth3qNftep2OE+MCmRzuQ1pMIKlR/r3ud9LpSAz16fN8Tjod0yP9mB7pxx2LJ/Jzfg3f5lZSXNtKgKcrvu7O1Ld1Ut3cgaFLo93QRVFta6/XXTQxGD8PF9yd9Zw2KZSUcX5Dek8qk21IbZKP2myRjzR/QgiHVdnUzn+3l/LvrUW99s7pnXSclhTKDXPjiAn0HNZreLk6c2pSKKcmhVpcbv+hJj7MKsPQpXH5rCjGBwzvdYUQYiAyvdsx2GN6N4PBgLOz9OWqknzUNph89h9q4pWfC/g6p9I88DFAgKcLV8waz3lTIwj2djvepTos2YbUJvmozVI+Mr3bKJadnW3vEoQFko/1NE3D0NX/+W+2Yimf4toW7t2wiyvXbuGLfYfMjZ9ep+OymVH89/oTue7EWGn8jjPZhtQm+ajNFvlIa6+g1tbWYy8k7MaR82k3dLGztB5XvRMpkX449ZjRoqndwFc5h9hT1khuVRNers4kh/tg1DT2VjRSXNtKZVM77QYjvu7OhPu6Mzs6gJPjg0kI9sLPw6XPDBlN7Qb2VjRS09LBhCAvYoM8cXay/Ddrf/m0dXax9pdCXt9SREeP5jPQ05ULpo/jwmnjZIiUEeTI29BoIPmozRb5SPOnoJE6vCys40j5aJpGTUsnP+ZX811uFb8U1NB6eCaJiSHeXDl7PAGeruyraOTN9CIajxqe5Mf86n6ft6HNQENbEzmHmnhrazFgGqvOVd+7seseV6+bu4sTZ0wK4/qTYhnn59Hvc/v6+lJQ08x3+6uIDvTEqGk8/U1ur3Hygr1cWXZCDBdMGzdqr5gdzRxpGxqNJB+12SIfaf4UFBMTY+8ShAXDzSe3som9FY34ebgQ5uNGQoh3rz1o9mbUNP67vZT/bC+lrL7N3OwdbX9lE6s+GfzhBy9XPaE+7ni76alu7qC8oQ1jjzOOO7s0OrssT1HW1mlk464yPs4qJ9Lf1PwFe7mSHO7LjCg/5sUFcUjnx8Ovb+13ujO9k46rZo/n+pNi8VRktgtHJN9xapN81GaLfOTbT0G7du2SqXUUNph8NE2jurmDnwpq+PFANW2dXQR7u5FX2cSusoZeyy5ODOHxJVP7HPK0h4rGNv7wSTa/FNb2e3+Apwtz44IoqGkh66j34aSDs6eEc+6UCJLCfGho62RvRSNOOh1Jod6E+7r3eo/1rZ38XFDN1qI6KhrbqGrqwGDsfT6gt6szSWE+hHi7kVvVxE/5NTS1GzAYNQprWgAorGkho7iON9NNh3HrWjt6NZXdZkf7c+9pScQFeQ1zLYnhku84tUk+arNFPtL8CWGBpmm0dRpp7jDQ0tlFS0cXZc1GGts68XZz7tXMFNe28N6Og3y7v5KKxvZe55ZZ8lVOJe9uK+XSmfYbV6u2pYP1mSW8edSQJzGBnoR4uzElwpeT44OZGuGL3kmHpmlsKaxld1k9Rg3c9E4snBjSa1gUbzfnAQ/NAvh5uHBmcjhnJocPus7Gtk7+vbWYD3eX0dLRhVHTeh0a7jnX7fz4IOKDvalqamfehCBOSwpVosEWQgh7k+ZPQbLL3b7yq5v599YisssbOVDVjKGf3Uh/3Po97i5OhHi54e6ip7q5vc80YAOZGOLNGcmh1LcaeDO9CICnv8lleqQfSWE+aJpGY7uBlqPOd3N20uHv6XLMCx4s0TSNnENN/HCgmuK6FqqaOiioaaa8ofeI8SHervy/XyVzYlxQv8+j0+k4ITaQE2IDra7FGj7uLtw0fwI3zZ9gvq2qqZ3dZQ18uqecb3Kr6DJqXDQjkntOTex33lxhf/IdpzbJR21y2HeM6jrGeU/i+Cmrb+XaN/o/X+xobZ1Giuv6XnXlotcRHeBJsJcrSWE+LEgIIdLPnermDlydnYgN9DTvgTIYjbyTUUJHl5GrX0/Hy1VPZ5c24F5DJx0EebmSGuXPwoQQJoZ6E+zl2msvZE1zB9/lVfFDXhXZFY0Ye4xj12EwUt828JyxeicdF8+I5Ia5cfh5uBxzHagg2NuNRRNDWDQxhLqWDnKKSpkzSeYmVZl8x6lN8lGbLfKR5k9BJSUljBs3zt5lOBxN0/jTpn3mxs9JBzGBXgR5ueLpqsfTRY+nq57S8kMY3Xyoam6nsqmD1s4ugr1cCfVxY1FCCOelRBDg6drn+fsbO+62BfFkFNWxv7IJ6Ht169GMGlQ2dbBp7yE27T0yFVm4rxsnxwdT3dzBt4f3fg2Wp4uepDBvJof7snRGJNGjeGYJf09XaKgEpPlTmXzHqU3yUZst8pHmTyjPqGm0d/beE6bTgZuz07DO4ersMmLo0tAwnTf25b5D/FxQA0Cotxv/vnYO/v3s/dqypYE5c1Ktft2e3Jz1PLl0Gi/+mE9JXStVTe246J367M0DaDcYqWpup6impU+TWN7QzrvbSvs8v4+bM56uR4Yy0ekgNtCLhQnBzIjyJ9jbDT93ZzkXTgghHIhM73YMIzW926bsCtKLTFdYGo1dODkde+wxHeDhosfDVY+Xqx5PV2dc9P3/I+6k0xHo5UqwlyseLnpc9E6H/9Ph4uSEi7MOvU43pCagrqWD7IpG9lY0UtnUTqCnK4Ferjj3OM9K00xNS3OHgeaOLlo6DHR29f7IdXYZqW7uoKalA2OPPVYapkF+q5o7+t2T5eGiJ8TbFS9XZ1ydnXBzdsLNWX/4/04463Xo0PV5nermDiqb2qlrHfgcvb8vncb8+OB+7+vo6MDVte+evZFi6DKyraSOLYW1HGpsp6yhjV0H683nJgZ6unLu1HAWJ4aSHO6j1DAyI8He+Yhjk4zUJvmozVI+g+1ZHHbP36effsr//vc/6urqiImJYfny5SQkJNitnp0H69mw86DdXh9MzaSrsxPOTjpcnZ1MTaFe16tRdNWbmqrSujbKGtqO+ZzHU2tnF0W1th+J/szksAEbP4D9+/czZcoUm7/uYDnrnUiLCSQt5sjFFk3tBjKKa3HVO5EWHYCz3nFnbrR3PuLYJCO1ST5qs0U+Dtn8/fjjj7z++uvccMMNTJw4kY8++ojVq1fz1FNP4efnZ+/y7EbDtJeunWOfe3Y86J10ffZcerroCfJyw8/DGXrsxesyatS0dFDV1E5bp9E8R+tguOh1BHu5Eezlitvh2R08XPQEe7kyPsCTS1IjLT6+ubl58G9qhHi7ObMwIcTeZShBxXxEb5KR2iQftdkiH4ds/j788ENOPfVUTjnlFABuuOEGMjMz+frrr7ngggvsUlP3VFMAeXl5xMfHH/MxRk2jtdNIS4dpWJDmDsOAJ/p3dmlUt3RQ3dxOh8F4eDYFIx1dR37u7O9no5EOgxGDUaPDYKT72T1c9EwK8yY5zJdJ4T5E+nlQ29pBXUtnr6tLwXRem+mwtOnQtKveiZ5HIrsPSQ/n3DNDl5E2g6nWdkMX7Ydr7skWrwPg7e1t9WPF8Sf5qE8yUpvkozZb5ONwzZ/BYODAgQO9mjwnJydSUlLIycmxW10h3m6EHL4adLzvJNzc+l4ZqgKD0UinQcPNxUmpc8mc9U54651gBFbbYBpzYT+Sj/okI7VJPmqzRT4Od2JQQ0MDRqMRf3//Xrf7+/tTV1dnl5qOtmPHDnuXMCBnJyc8XPVKNX4jTeV8hOQzGkhGapN81GaLfBxuz5+1srKy0Ov1zJw5k+zsbFpbW/Hx8SEuLo6dO3cCEB0djaZpFBcXAzB9+nTy8vJoamrCy8uLiRMnsn37dgCioqLQ6/UUFhYCkJKSQmFhIQ0NDTQ0NGAwGMjMzARg3LhxuLm5kZ+fD8CUKVMoLS2lrq4ONzc3pk2bRnp6OgDh4eF4eXmRl5cHQHJyMhUVFdTU1ODs7MzMmTNJT09H0zRCQkIICAgw7/FMTEyktraWyspKnJycmD17NpmZmRgMBoKCgggJCWHv3r0AJCQk0NjYSEVFBQBz5sxh+/btdHR0EBAQQEREBHv27AFgwoQJtLa2UlZWBsCsWbPIysqira0NPz8/xo8fz+7duwGIjY2ls7OT0lLTsCWpqans27ePlpYWvL29iY+PN3/wx48fD9Dv+vb09CQpKYlt27YBEBkZiYuLCwUFBQBMnTqV4uJi6uvrcXd3Z8qUKWRkZAAQERGBh4cHBw4cAGDy5MmUlZVRW1trvsJqy5YtAISFheHj40Nubi4AkyZNorKykurqavP63rp1K0aj0eL61ul0pKWlmdd3YGAgYWFhZGdnA6a/9JqbmykvLwcgLS2NnTt30t7ejr+/P5GRkWRlZQEQFxdHe3s7Bw+aLiDq+Zn19fUlJiaGXbt2AaaR4ru6uigpKQFgxowZ7N+/n+bm5n7Xt06no6jINCvJtGnTyM/Pp7GxEQ8PD5KTk82f2f7Wd0lJifkzm5KSwtatW4/5mXVxcSE1NdW8vkNDQ/Hz82P//v0AJCUlUV1dTVVVFXq9nlmzZlFXV8eWLVsIDg4mKCiIffv2ATBx4kTq6+s5dOiQ+TO7bds2Ojs7j7m+Z8+eza5du8zrOyoqasDP7Eh8Rxy9vkfbd0RzczNNTU1j9jtixowZo/o7oqury1z/WPyOyMjIoKura9R+R9TW1rJly5Z+vyO61+exONxQLwaDgauvvpo777yz18TIzz33HC0tLdx77729lh+poV56KisrIyIiYkReSwyd5KM2yUd9kpHaJB+1WcpnsD2Lwx32dXZ2ZsKECeaOHMBoNLJ7924SExPtWNkRMuCu2iQftUk+6pOM1Cb5qM0W+Thc8wdw7rnn8uWXX/LNN99QUlLCSy+9RHt7O4sWLbJ3aQDmXeZCTZKP2iQf9UlGapN81GaLfBzynL+5c+fS0NDA+vXrqaurIzY2lgcffLDPRSBCCCGEEGONw53zN1T2OOevra0Nd3f3EXktMXSSj9okH/VJRmqTfNRmKR85528U675iT6hJ8lGb5KM+yUhtko/abJGPNH8KamxstHcJwgLJR22Sj/okI7VJPmqzRT7S/CnIw8PD3iUICyQftUk+6pOM1Cb5qM0W+Ujzp6Dk5GR7lyAskHzUJvmoTzJSm+SjNlvkI82fgrpHQBdqknzUJvmoTzJSm+SjNlvk45BDvQxFV1cXAM3NzSP6mg0NDSP2emJoJB+1ST7qk4zUJvmozVI+3b1Kd+8yEBnq5RjKysrM808KIYQQQqhu8uTJFqfok+bvGDo6OqiursbDwwMnJzlKLoQQQgg1GY1GWltbCQoKwtXVdcDlpPkTQgghhHAgsitLCCGEEMKBSPMnhBBCCOFApPkTQgghhHAg0vwJIYQQQjgQaf5GUGVlJdXV1YDpihyhloaGBpqbm83ZSEZq6ejosHcJwoKysjI2btzIwYMH7V2KGIBsQ2obyR5BBnkeIenp6fztb39j1qxZ3HvvvTJsjEIMBgOvvPIK2dnZeHl5ERERwc033ywZKcJgMPDmm29SVVWFu7s7p556qkw/pRCj0cgrr7zC119/zfz580lMTGTcuHH2Lkv0YDAYePXVV6msrMTX15czzjiDiRMnotPp7F2aOGykewT5122E5ObmkpCQQHV1NT///DMge5ZUUF5ezgMPPEBZWRnXX389qamp5OTksHHjRnuXJoAtW7bwm9/8hsLCQqZMmUJBQQFvvfWWeRsS9vfhhx9SWFjIypUrufnmm5k0aRIAMoqYGurq6njooYcoKipi1qxZFBYW8uKLL5q/4+TfITWMdI8ge/6OM6PRiJOTEy0tLcTHx/P/27v3oCjLxo3jX5Zd2RB0JQQ0MARRFEVJQq1UVGwqpXAy7ew0OdiZapo0G8umfLVxqMmamk42aVpmamOTJqCWliZmJp3QSMpDAjGcQVgX9veHv32SV19LXdht9/r81bDLet9dsw/X3s/9PGu329m4cSOpqamYzWacTqc+fXnQ3r17sVqtzJ49G6vVSmJiIj///DPBwcGeHprfKysrY/v27YwbN45p06YBcOWVV/Liiy9SVlbm4dGJ0+mkpaWFwsJCxo0bR0JCAgcOHODQoUNER0cTGxuL1Wr19DD9XnFxMQ6Hg9mzZxMWFsbo0aP59NNP+fDDD7nsssuIiYnR3yEP8lRH0MpfBzOZTDidTsrKyhgzZgxpaWnU19eTl5cH/P3370nHcH2iqq+vp6amxvgjVVNTQ2NjI0FBQRw9etSTQ/RbrhUjh8PBpZdeSnp6OnAys27dumEymVT+vEBAQADV1dWUl5czbNgwli1bRm5uLl988QW5ubksXryYpqYmTw/Tb7mOcXV1dTQ0NBAWFgZAcHAwEydOJDExkTfeeANAxc+DPNURtPLnRgUFBVitVvr160dUVBTwV6s3mUycOHGChIQE0tLS2Lp1KyUlJfTp04dJkyZhsVg8PHrf58onPj7e+M7D2NhYNm/ezIIFCwgNDWXnzp3069ePDRs2UFNTw/Tp0xk/frw+GXcC12kP1//r6Ohopk6dajxuMplwOBzY7Xb69+/vwZH6J1c+rmMawMUXX0xoaCgffPABzc3NzJs3j+7du1NdXc3TTz/N2rVrue222/Te6SRff/01wcHBxMTE0KNHD+Dk+8Zms/Hzzz8be2VtNhtZWVksXLiQoqIikpOTdYzrBGfKx1MdQeXPDb777jteeeUVwsLCaGxsxGw2c/XVVzNp0iRMJhMNDQ2UlpaSkJCA2WympaWFP/74g2PHjjF58mQVvw52pnwmTpzI5MmTSU1NJTw8nCNHjrBmzRpycnIYOXIkjY2NbNq0iRUrVjB27FgCAwM9PQ2fVVhYyNtvv43D4WDhwoVERES0Kxin/lFyOBzU1dURExPjySH7lbPl4yriO3fuZMiQIcaFHqGhodxxxx0sW7aMadOmnfU7RuXCbdu2jeXLl9OzZ08qKiro1asXkydPZsSIEcTHx7Nhwwb2799v/A0CiImJYdiwYWzbto3k5GQVvw50pnwyMzNJS0vzWEdQ+XODLVu2kJaWRnZ2NseOHeObb75h2bJl9OrVi2HDhtHa2kpiYiK7du1i3bp1VFdXM2TIEMrLy9vdVkRXl3aMM+WzfPlyevXqRUpKCnFxcRQXF9O1a1dGjhyJ0+mka9euDBw4kHXr1lFSUsKAAQM8PQ2ftH37djZs2MDAgQOpqqri448/Jjs7u9174dQ/SsXFxTQ3Nxsrt3DyVL3NZuvMYfuNv8snJCSEwYMHs3v3buMY5jqmxcTEYDabOXLkCHFxcZ6chs9qbW1l06ZN5Ofnc8sttzBmzBh+/fVX8vPz2bJlCykpKfTt25fExEQKCwtJSEggKSkJOLn6FxgYqNLXgc6Wz+bNm0lJScFisWC32xk0aFCndgS1jfPk2pdUUVHB999/T1paGoDR6K+88kqWL19OTU0Nra2t7Nq1i1deeYWBAweyZMkSbr/9dnr27MmyZcsAVPzc7J/k895771FRUQHAiRMn6NatG01NTcbBsLi4mLi4OGJjYz0yB1/mOqBFRUUxZMgQbr/9dlJTU/npp5/48ccf2z3nVIWFhQwaNIiQkBBKS0t55plneOutt3TFopv9k3wcDgcAqampjB49mj179lBUVGQcy4qLi4mNjdX7pwO1tLRQV1fH2LFjSU9Px2w2M2DAAKKjo2lqajIymjZtGq2trRQUFFBVVWX8vt1uJyQkxFPD93l/l49rP19bWxs7d+7s1I6glb9zdOzYMaKiooyCYLPZMJvNVFZWAicPiGazmZkzZ5Kdnc22bdvIysriwQcfJCoqin79+gHQtWtXLr/8co4fP24UFX0Cu3Dnks+sWbPYtWsXN9xwAzabjdraWl544QUyMjLYu3cve/bsYfr06QQFBXlySj7FlY/rQJaQkEBcXByBgYGkpKRQXFzM+vXrSUpKMjZCu7Jsa2ujpqaGxMREli5dyqZNmxg9ejT33HOPPjy5ybnkYzabaWtrw2q1ct1119HQ0MDixYsZOnQoFouFffv2ceutt56Wo1yYU49xwcHBjBw5kj59+hirriaTifDwcFpaWozT7TabjSlTprBx40bmzZvHtddey2+//cbBgweZMmWKh2fkW84lH9cp+PDwcHJycoiIiOi0jhDg1M2Y/pEdO3awYsUKLBYLwcHBZGRkMH78eJqbm3nrrbeora1l9uzZmM1mo2CsXLmSL7/8kldffbXda7kOhDrV6z7uyGf79u0UFBTgdDq56KKLmDFjhm5W6yb/Kx9ov6dv69atfPLJJ2RmZjJu3Lh275HKykruv/9+APr378+sWbOIjo72zIR8zPnm09ra2m4/bH5+PuXl5dTV1ZGVlaX3jxudLSNof1pwyZIlmM1m7rvvPuN4B1BVVcVHH31EbW0tDodDxzg3ckc+Lp3REbTy9w8UFRWxYsUKrr/+eiIjIykqKuLNN9+kra2NjIwMBg8ezMaNG/n888/JyMgwDpQjRoxgy5YtHDx4sN2eF9fjKn7ucaH5uK5iHD16NFdccQX19fXaQ+ZGZ8tnzJgxdOnSxSgRQ4cOZf/+/eTl5TFq1CisVqtxcDx+/DijRo1i/PjxJCcne3paPsNd+QBMnDjRw7PxTf8ko4CAAJxOJydOnODw4cNkZmYCtCsWYWFhZGdnY7fbdRGOG7krH1fZ64yOoPJ3Fq72feDAAUJDQ5kwYQJms5lhw4Zht9vJz8+nZ8+ejBgxgqKiIr744guSk5OJiIgAoLy8nMDAQO2p6CDuyqdbt27GawYGBqr4ucnf5bN582a6detGWlqasXoUFhZGWloav//+O+vXr2fEiBG8//77zJw5k5iYGB5++GHPTsqHuDuf8PBwD8/I95xLRq7C0NDQQFNTEwkJCcDJ05B5eXnMmDHDeF0VP/dwdz6duSCkpaezcIV15MgRIiMjjVOGADfffDNBQUFs374dk8nENddcQ0BAAC+99BL79++nsrKSvXv3EhcXpzLRQZSPd/u7fCwWC7t376ampgb46yKDpKQk4uPjWbNmDXPmzKG1tZXu3bt7ZA6+TPl4v3PNCOD7778nPDycHj168M477/Doo4/y559/4nA49JV7bvZvzkcrf6coKirim2++ITIykgEDBhgbLwcPHszy5ctpa2szwg0JCWHMmDGsX7+e0tJSEhMTueeee1iyZAmvvfYajY2NhIWFkZOTo09ZbqJ8vNv55PPJJ5/wxx9/YLPZMJlMNDc3s3nzZgoKChg0aBB33XUXffr08fDMfIPy8X7nm9HRo0ex2Ww4nU727NnDoUOHuP/++7HZbDz33HPEx8d7eGa+wZfy0cofUF1dzaJFi3j55ZdpaGhg69atPPfcc5SUlAAwaNAgLrroIlavXt3u9zIyMmhubjae17t3b+bPn8/cuXN5/PHHef7557WZ1g2Uj3e7kHyOHz9OaWmp8bPKykp27NjBfffdx9NPP61i4QbKx/tdaEa//fYbcPLWLXa7HavVyt13301ubq6Knxv4Yj5+v/LX0tLCypUrsVqtLFiwwNgPNnfuXPLy8ujXrx89evTg6quvZu3atUyYMIHw8HDjXH/v3r05cuSI8XpBQUFEREQYryMXRvl4N3fkc/jwYeP1oqOjWbBggaem43OUj/dzZ0ZBQUFMmzZNN9V2I1/Nx+9X/oKCgrBYLKSnpxMREWHcdDElJYWjR48at/246qqr6Nu3Ly+++CJ//vknAQEBVFZWUltba9xAGHSvPndTPt7N3fmIeykf7+fujLyhWPgSX81H9/mDdrcqcF1qvWTJEoKCgpg1a5bxvKqqKubPn09rayvx8fHs37+fSy65hIceekgXDXQg5ePdlI93Uz7eTxl5N1/MR+Xvf5g3bx4TJkwgPT3duMrNZDJRVlbGwYMH+eWXX7j00ktJT0/37ED9lPLxbsrHuykf76eMvNu/PR+/3/N3JuXl5ZSVlRmbmU0mEw6HA5PJRFRUFFFRUVxxxRUeHqX/Uj7eTfl4N+Xj/ZSRd/OFfPx+z9+pXIugxcXFWK1W49z86tWreeedd6itrfXk8Pye8vFuyse7KR/vp4y8my/lo5W/U7guBigpKTG+FeL111/HbrfzwAMP6EamHqZ8vJvy8W7Kx/spI+/mS/mo/P0Xu93Ovn37KC8vZ+PGjdx0001kZWV5eljy/5SPd1M+3k35eD9l5N18JR+Vv//SpUsXevbsSXJyMnfeeae+/cHLKB/vpny8m/LxfsrIu/lKPrra9wxcl3KLd1I+3k35eDfl4/2UkXfzhXxU/kRERET8yL+7uoqIiIjIOVH5ExEREfEjKn8iIiIifkTlT0RERMSPqPyJiIiI+BGVPxERERE/ovInIiIi4kdU/kRERET8iMqfiIiIiB9R+RMRERHxIyp/IiIiIn5E5U9ERETEj6j8iYiIiPgRlT8RERERP6LyJyIiIuJHVP5ERM7Df/7zH+666y5qampOe6ypqYns7Gzmzp1LW1tb5w9OROQsVP5ERM7DzJkzcTgcvPvuu6c9tnLlSurr68nOzsZk0mFWRLyLjkoiIuchIiKCqVOn8tVXX7Fv3z7j5yUlJeTn5zNp0iRiY2M7dAx2u10riyJyzgKcTqfT04MQEfk3am1tZc6cOTQ3N5Obm4vZbOaJJ56gsbGR3NxcKisr+eCDD/jhhx+w2+3ExMQwdepUUlNTjddoaGhg7dq17Nu3j4qKCkwmEwMGDODWW29tVx5//PFHnnnmGXJycjh8+DBbt26lpqaGpUuX0rVrVw/MXkT+rbTyJyJyngIDA8nOzqaiooI1a9bw2WefUVpaysyZM6moqODJJ5/k6NGjZGVlcccddxAUFMTixYspLCw0XqO8vJzdu3czfPhwZsyYQWZmJocOHWL+/PlUVVWd9m+uWbOGb7/9lszMTG655RbMZnNnTllEfIBW/kRELtDSpUvJz8/HYrEwfPhwcnJyePbZZ6mtrWXhwoVYLBYAnE4nTz31FHV1dbz00ksAnDhxgsDAwHZ7AysqKnjkkUeYMmUKU6dOBf5a+YuMjCQ3N5cuXbp0/kRFxCdo5U9E5ALdfPPNhIaGEhAQwIwZM2hoaOCHH35g1KhRHD9+nLq6Ourq6qivr2fo0KEcO3bMWNWzWCxG8Wtra6O+vh6r1Urv3r0pLS097d8aO3asip+IXBCdLxARuUDBwcH07t2b+vp6bDYbJSUlOJ1OVq1axapVq874O7W1tYSFhdHW1saGDRvIy8ujoqKi3QUcISEhp/1eREREh81DRPyDyp+IiJu5ClxmZiZDhw4943OioqIAWLduHatWrWLcuHFMnz6dkJAQAgICePfddznTrhyt+onIhVL5ExFxs8jISODkBSHJyclnfe7XX39NUlIS9957b7ufNzY2Ehoa2mFjFBH/pT1/IiJu1r17d5KSkigoKKC6uvq0x+vq6oz/PtNNoHfu3HnGK31FRNxBK38iIh3g7rvvZt68eTz22GNMmDCBiIgIamtrOXDgAFVVVSxevBiA4cOH89FHH/Hqq6/Sv39/Dh06xJdffmmsHoqIuJvKn4hIB4iOjmbRokWsXr2azz//nPr6erp3705sbCw33nij8bwpU6bQ3NzMV199xY4dO+jbty9z5sxh5cqVHhy9iPgy3edPRERExI9oz5+IiIiIH1H5ExEREfEjKn8iIiIifkTlT0RERMSPqPyJiIiI+BGVPxERERE/ovInIiIi4kdU/kRERET8iMqfiIiIiB9R+RMRERHxIyp/IiIiIn5E5U9ERETEj6j8iYiIiPiR/wNWPWdXICIVPAAAAABJRU5ErkJggg==", 425 | "text/plain": [ 426 | "
" 427 | ] 428 | }, 429 | "metadata": {}, 430 | "output_type": "display_data" 431 | } 432 | ], 433 | "source": [ 434 | "import matplotlib.dates as mdates\n", 435 | "plt.style.use('bmh')\n", 436 | "\n", 437 | "histo = embedding_df.groupby('created_month').size().reset_index()\n", 438 | "histo_filtered = histo\n", 439 | "histo_filtered.columns = ['created_month','published_articles']\n", 440 | "\n", 441 | "# Average Smoothing\n", 442 | "window = 12\n", 443 | "average_y = []\n", 444 | "for ind in range(len(histo['published_articles']) - window + 1):\n", 445 | " average_y.append(np.mean(histo['published_articles'][ind:ind+window]))\n", 446 | "\n", 447 | "for ind in range(window - 1):\n", 448 | " average_y.insert(0, np.nan)\n", 449 | "\n", 450 | "histo_filtered['average_articles'] = average_y\n", 451 | "histo_filtered['created_month'] = pd.to_datetime(histo_filtered['created_month'])\n", 452 | "histo_smoothed = histo_filtered[['created_month','average_articles']]\n", 453 | "histo_smoothed.head(50)\n", 454 | "\n", 455 | "\n", 456 | "fig,ax = plt.subplots()\n", 457 | "ax.set_facecolor('white')\n", 458 | "histo_smoothed.plot(x=\"created_month\",x_compat=True,ax=ax);\n", 459 | "\n", 460 | "\n", 461 | "plt.title(\"'Embeddings' CS Papers by Month on ArXiv\")\n", 462 | "plt.ylabel(\"Papers submitted by date\",labelpad=10);\n", 463 | "plt.xlabel(\"Year\",labelpad=10);\n", 464 | "\n", 465 | "ax.get_legend().remove()\n", 466 | "\n", 467 | "plt.show()" 468 | ] 469 | } 470 | ], 471 | "metadata": { 472 | "kernelspec": { 473 | "display_name": "Python 3 (ipykernel)", 474 | "language": "python", 475 | "name": "python3" 476 | }, 477 | "language_info": { 478 | "codemirror_mode": { 479 | "name": "ipython", 480 | "version": 3 481 | }, 482 | "file_extension": ".py", 483 | "mimetype": "text/x-python", 484 | "name": "python", 485 | "nbconvert_exporter": "python", 486 | "pygments_lexer": "ipython3", 487 | "version": "3.10.8" 488 | }, 489 | "vscode": { 490 | "interpreter": { 491 | "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49" 492 | } 493 | } 494 | }, 495 | "nbformat": 4, 496 | "nbformat_minor": 5 497 | } 498 | -------------------------------------------------------------------------------- /notebooks/requirements.txt: -------------------------------------------------------------------------------- 1 | anyio==3.6.2 2 | appnope==0.1.3 3 | argon2-cffi==21.3.0 4 | argon2-cffi-bindings==21.2.0 5 | arrow==1.2.3 6 | arxiv==1.4.7 7 | asttokens==2.2.1 8 | attrs==23.1.0 9 | backcall==0.2.0 10 | beautifulsoup4==4.12.2 11 | bleach==6.0.0 12 | cachetools==5.3.0 13 | certifi==2023.5.7 14 | cffi==1.15.1 15 | chardet==3.0.4 16 | comm==0.1.3 17 | contourpy==1.0.7 18 | cycler==0.11.0 19 | debugpy==1.6.7 20 | decorator==5.1.1 21 | defusedxml==0.7.1 22 | entrypoints==0.4 23 | executing==1.2.0 24 | fastjsonschema==2.17.1 25 | feedparser==6.0.10 26 | fonttools==4.39.4 27 | fqdn==1.5.1 28 | idna==2.8 29 | ipykernel==6.23.1 30 | ipython==8.13.2 31 | ipython-genutils==0.2.0 32 | ipywidgets==8.0.6 33 | isoduration==20.11.0 34 | jedi==0.18.2 35 | Jinja2==3.1.2 36 | joblib==1.2.0 37 | jsonpointer==2.3 38 | jsonschema==4.17.3 39 | jupyter==1.0.0 40 | jupyter-console==6.6.3 41 | jupyter-events==0.6.3 42 | jupyter_client==8.2.0 43 | jupyter_core==5.3.0 44 | jupyter_server==2.5.0 45 | jupyter_server_terminals==0.4.4 46 | jupyterlab-pygments==0.2.2 47 | jupyterlab-widgets==3.0.7 48 | kiwisolver==1.4.4 49 | MarkupSafe==2.1.2 50 | matplotlib==3.7.1 51 | matplotlib-inline==0.1.6 52 | mistune==2.0.5 53 | nbclassic==1.0.0 54 | nbclient==0.7.4 55 | nbconvert==7.4.0 56 | nbformat==5.8.0 57 | nest-asyncio==1.5.6 58 | notebook==6.5.4 59 | notebook_shim==0.2.3 60 | numpy==1.24.3 61 | packaging==23.1 62 | pandas==2.0.1 63 | pandocfilters==1.5.0 64 | parso==0.8.3 65 | pexpect==4.8.0 66 | pickleshare==0.7.5 67 | Pillow==9.5.0 68 | platformdirs==3.5.1 69 | portpicker==1.2.0 70 | prometheus-client==0.16.0 71 | prompt-toolkit==3.0.38 72 | psutil==5.9.5 73 | ptyprocess==0.7.0 74 | pure-eval==0.2.2 75 | pyasn1==0.5.0 76 | pyasn1-modules==0.3.0 77 | pycparser==2.21 78 | Pygments==2.15.1 79 | pyparsing==3.0.9 80 | pyrsistent==0.19.3 81 | python-dateutil==2.8.2 82 | python-json-logger==2.0.7 83 | pytz==2023.3 84 | PyYAML==6.0 85 | pyzmq==25.0.2 86 | qtconsole==5.4.3 87 | QtPy==2.3.1 88 | requests==2.21.0 89 | rfc3339-validator==0.1.4 90 | rfc3986-validator==0.1.1 91 | rsa==4.9 92 | scikit-learn==1.2.2 93 | scipy==1.10.1 94 | Send2Trash==1.8.2 95 | sgmllib3k==1.0.0 96 | simplegeneric==0.8.1 97 | six==1.12.0 98 | sniffio==1.3.0 99 | soupsieve==2.4.1 100 | stack-data==0.6.2 101 | terminado==0.13.3 102 | threadpoolctl==3.1.0 103 | tinycss2==1.2.1 104 | tornado==6.3.2 105 | tqdm==4.65.0 106 | traitlets==5.9.0 107 | tzdata==2023.3 108 | uri-template==1.2.0 109 | urllib3==1.24.3 110 | wcwidth==0.2.6 111 | webcolors==1.13 112 | webencodings==0.5.1 113 | websocket-client==1.5.2 114 | widgetsnbextension==4.0.7 115 | --------------------------------------------------------------------------------