├── BLOOM_api_example.ipynb ├── BLOOM_local_example.ipynb └── LICENSE /BLOOM_api_example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "# requirements:\n", 10 | "! pip install huggingface_hub\n", 11 | "! git config --global credential.helper store" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "metadata": {}, 18 | "outputs": [ 19 | { 20 | "name": "stdout", 21 | "output_type": "stream", 22 | "text": [ 23 | "Login successful\n", 24 | "Your token has been saved to /home/h/.huggingface/token\n" 25 | ] 26 | } 27 | ], 28 | "source": [ 29 | "from huggingface_hub import notebook_login\n", 30 | "from huggingface_hub import HfFolder\n", 31 | "\n", 32 | "\n", 33 | "#enter your API key, you can make one for free on HF\n", 34 | "notebook_login()" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 2, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "from huggingface_hub import InferenceApi\n", 44 | "\n", 45 | "inference = InferenceApi(\"bigscience/bloom\",token=HfFolder.get_token())" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 5, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "import time\n", 55 | "\n", 56 | "def infer(prompt,\n", 57 | " max_length = 32,\n", 58 | " top_k = 0,\n", 59 | " num_beams = 0,\n", 60 | " no_repeat_ngram_size = 2,\n", 61 | " top_p = 0.9,\n", 62 | " seed=42,\n", 63 | " temperature=0.7,\n", 64 | " greedy_decoding = False,\n", 65 | " return_full_text = False):\n", 66 | " \n", 67 | "\n", 68 | " top_k = None if top_k == 0 else top_k\n", 69 | " do_sample = False if num_beams > 0 else not greedy_decoding\n", 70 | " num_beams = None if (greedy_decoding or num_beams == 0) else num_beams\n", 71 | " no_repeat_ngram_size = None if num_beams is None else no_repeat_ngram_size\n", 72 | " top_p = None if num_beams else top_p\n", 73 | " early_stopping = None if num_beams is None else num_beams > 0\n", 74 | "\n", 75 | " params = {\n", 76 | " \"max_new_tokens\": max_length,\n", 77 | " \"top_k\": top_k,\n", 78 | " \"top_p\": top_p,\n", 79 | " \"temperature\": temperature,\n", 80 | " \"do_sample\": do_sample,\n", 81 | " \"seed\": seed,\n", 82 | " \"early_stopping\":early_stopping,\n", 83 | " \"no_repeat_ngram_size\":no_repeat_ngram_size,\n", 84 | " \"num_beams\":num_beams,\n", 85 | " \"return_full_text\":return_full_text\n", 86 | " }\n", 87 | " \n", 88 | " s = time.time()\n", 89 | " response = inference(prompt, params=params)\n", 90 | " #print(response)\n", 91 | " proc_time = time.time()-s\n", 92 | " #print(f\"Processing time was {proc_time} seconds\")\n", 93 | " return response" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 6, 99 | "metadata": {}, 100 | "outputs": [ 101 | { 102 | "data": { 103 | "text/plain": [ 104 | "[{'generated_text': 'The thing that makes large language models interesting is that they can be used to encode language at a deeper level than a bag-of-words approach, which is the basis for most search engines. In the case of'}]" 105 | ] 106 | }, 107 | "execution_count": 6, 108 | "metadata": {}, 109 | "output_type": "execute_result" 110 | } 111 | ], 112 | "source": [ 113 | "prompt = \"The thing that makes large language models interesting is\"\n", 114 | "resp = infer(prompt)\n", 115 | "\n", 116 | "resp" 117 | ] 118 | } 119 | ], 120 | "metadata": { 121 | "kernelspec": { 122 | "display_name": "Python 3.8.10 64-bit", 123 | "language": "python", 124 | "name": "python3" 125 | }, 126 | "language_info": { 127 | "codemirror_mode": { 128 | "name": "ipython", 129 | "version": 3 130 | }, 131 | "file_extension": ".py", 132 | "mimetype": "text/x-python", 133 | "name": "python", 134 | "nbconvert_exporter": "python", 135 | "pygments_lexer": "ipython3", 136 | "version": "3.8.10" 137 | }, 138 | "orig_nbformat": 4, 139 | "vscode": { 140 | "interpreter": { 141 | "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a" 142 | } 143 | } 144 | }, 145 | "nbformat": 4, 146 | "nbformat_minor": 2 147 | } 148 | -------------------------------------------------------------------------------- /BLOOM_local_example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "# Download the model (350ish GB)\n", 10 | "from transformers import AutoTokenizer, AutoModel\n", 11 | "tokenizer = AutoTokenizer.from_pretrained(\"bigscience/bloom\")\n", 12 | "model = AutoModel.from_pretrained(\"bigscience/bloom\")" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": null, 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "from transformers import pipeline\n", 22 | "import torch\n", 23 | "import time\n", 24 | "\n", 25 | "\n", 26 | "s = time.time()\n", 27 | "pipe = pipeline(model=\"bigscience/bloom\", torch_dtype=torch.bfloat16)\n", 28 | "print(f\"Time to load model: {time.time()-s}\")" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": null, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "# src: https://stackoverflow.com/questions/16816013/is-it-possible-to-print-using-different-colors-in-ipythons-notebook\n", 38 | "from IPython.display import HTML as html_print\n", 39 | "\n", 40 | "def cstr(s, color='black'):\n", 41 | " #return \"{}\".format(color, s)\n", 42 | " return \"{}\".format(color, s.replace('\\n', '
'))\n", 43 | "\n", 44 | "def cstr_with_newlines(s, color='black'):\n", 45 | " return \"{}\".format(color, s.replace('\\n', '
'))" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "def local_inf(prompt, temperature=0.7, top_p=None, max_new_tokens=32, repetition_penalty=None, do_sample=False, num_return_sequences=1): \n", 55 | " response = pipe(f\"{prompt}\", \n", 56 | " temperature = temperature, # 0 to 1\n", 57 | " top_p = top_p, # None, 0-1\n", 58 | " max_new_tokens = max_new_tokens, # up to 2047 theoretically\n", 59 | " return_full_text = False, # include prompt or not.\n", 60 | " repetition_penalty = repetition_penalty, # None, 0-100 (penalty for repeat tokens.\n", 61 | " do_sample = do_sample, # True: use sampling, False: Greedy decoding.\n", 62 | " num_return_sequences = num_return_sequences\n", 63 | " )\n", 64 | " return html_print(cstr(prompt, color='#f1f1c7') + cstr(response[0]['generated_text'], color='#a1d8eb')), response[0]['generated_text']" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": null, 70 | "metadata": {}, 71 | "outputs": [], 72 | "source": [ 73 | "inp = \"\"\"# Use OpenCV in Python\"\"\"\n", 74 | "color_resp, resp = local_inf(inp, max_new_tokens=64)\n", 75 | "color_resp" 76 | ] 77 | } 78 | ], 79 | "metadata": { 80 | "kernelspec": { 81 | "display_name": "Python 3.8.10 64-bit", 82 | "language": "python", 83 | "name": "python3" 84 | }, 85 | "language_info": { 86 | "name": "python", 87 | "version": "3.8.10" 88 | }, 89 | "orig_nbformat": 4, 90 | "vscode": { 91 | "interpreter": { 92 | "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a" 93 | } 94 | } 95 | }, 96 | "nbformat": 4, 97 | "nbformat_minor": 2 98 | } 99 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Harrison 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | --------------------------------------------------------------------------------