├── .gitignore ├── 2024_01_30_LLM_Workshop.Rproj ├── README.md ├── demo.qmd ├── media ├── JBGruber.jpg ├── cluster_1 │ ├── 1.png │ ├── 2.png │ ├── 3.png │ └── 4.png ├── cluster_10 │ ├── 1.png │ ├── 2.png │ ├── 3.png │ └── 4.png ├── crack-dolphin-censored.png ├── crack-dolphin-censored.png~ ├── crack-dolphin.png ├── crack-gpt4.png ├── gllm.png ├── gpt3-locally.png ├── max.png ├── notes.txt ├── ollama-r.png ├── ollama.png ├── qr-code-gist.png ├── qr-code-rollama.png ├── raspi.jpg ├── raspi.jpg~ ├── rollama_hex.png ├── so-easy.png └── surprised.png ├── references.bib ├── slides.html └── slides.qmd /.gitignore: -------------------------------------------------------------------------------- 1 | .Rhistory 2 | prep.sh 3 | .Rproj.user 4 | data/* 5 | -------------------------------------------------------------------------------- /2024_01_30_LLM_Workshop.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | 3 | RestoreWorkspace: Default 4 | SaveWorkspace: Default 5 | AlwaysSaveHistory: Default 6 | 7 | EnableCodeIndexing: Yes 8 | UseSpacesForTab: Yes 9 | NumSpacesForTab: 2 10 | Encoding: UTF-8 11 | 12 | RnwWeave: Sweave 13 | LaTeX: pdfLaTeX 14 | 15 | AutoAppendNewline: Yes 16 | StripTrailingWhitespace: Yes 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Ollama presentation 2 | 3 | My presentation about running generative large language models with [Ollama](https://ollama.ai/) and [(r)ollama](https://jbgruber.github.io/rollama/) @ [Societal Analytics Lab](https://www.societal-analytics.nl) of the Faculty of Social Science at the Vrije Universiteit Amsterdam. Date: 2024-01-30 4 | 5 | I also filmed it again for YouTube: 6 | 7 | [![Get up and running with local ChatGPT/gLLMs with Ollama in R](https://img.youtube.com/vi/N-k3RZqiSZY/0.jpg)](https://www.youtube.com/watch?v=N-k3RZqiSZY) 8 | -------------------------------------------------------------------------------- /demo.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "(r)ollama demo" 3 | author: "JBGruber" 4 | format: html 5 | editor_options: 6 | chunk_output_type: console 7 | --- 8 | 9 | ```{r setup} 10 | rlang::check_installed(c("rollama", 11 | "tidyverse")) 12 | library(rollama) 13 | library(tidyverse) 14 | ``` 15 | 16 | Is Ollama running? 17 | 18 | ```{r} 19 | ping_ollama() 20 | ``` 21 | 22 | ```{r} 23 | query("Who are you?", model = "orca-mini") # this one forgets the conversation 24 | chat("What is 1 + 1?", model = "orca-mini") # this one retains the conversation 25 | chat("Now take the result and add 1. What is it?", model = "orca-mini") 26 | new_chat() 27 | ``` 28 | 29 | # configure 30 | 31 | ```{r} 32 | # use desktop PC instead 33 | options(rollama_server = "http://192.168.2.29:11434") 34 | ping_ollama() 35 | ``` 36 | 37 | ```{r} 38 | query("why is the sky blue?") 39 | ``` 40 | 41 | ```{r} 42 | options(rollama_config = "You make answers understandable to a 5 year old") 43 | ``` 44 | 45 | ```{r} 46 | query("why is the sky blue?") 47 | ``` 48 | 49 | ```{r} 50 | options(rollama_model = "mistral") 51 | ``` 52 | 53 | ```{r} 54 | query("why is the sky blue?") 55 | ``` 56 | 57 | ```{r} 58 | options(rollama_config = NULL, rollama_model = NULL) 59 | ``` 60 | 61 | ```{r} 62 | query("why is the sky blue?", 63 | model_params = list( 64 | num_keep = 5, 65 | seed = 42, # for reproducibility 66 | num_predict = 100, 67 | top_k = 20, 68 | top_p = 0.9, 69 | tfs_z = 0.5, 70 | typical_p = 0.7, 71 | repeat_last_n = 33, 72 | temperature = 0, # less random answers 73 | repeat_penalty = 1.2, 74 | presence_penalty = 1.5, 75 | frequency_penalty = 1.0, 76 | mirostat = 1, 77 | mirostat_tau = 0.8, 78 | mirostat_eta = 0.6, 79 | penalize_newline = TRUE, 80 | numa = FALSE, 81 | num_ctx = 1024, 82 | num_batch = 2, 83 | num_gqa = 1, 84 | num_gpu = 0, # turn GPU on and off 85 | main_gpu = 0, 86 | low_vram = FALSE, 87 | f16_kv = TRUE, 88 | vocab_only = FALSE, 89 | use_mmap = TRUE, 90 | use_mlock = FALSE, 91 | rope_frequency_base = 1.1, 92 | rope_frequency_scale = 0.8, 93 | num_thread = 8 94 | )) 95 | ``` 96 | 97 | # models 98 | 99 | ```{r} 100 | list_models() 101 | ``` 102 | 103 | ```{r} 104 | model_info <- show_model("mistral") 105 | View(model_info) 106 | ``` 107 | 108 | ```{r} 109 | # get this model at https://openwebui.com/m/hub/based-dolphin-mixtral:latest 110 | model_info <- show_model("based-dolphin-mixtral") |> 111 | cli::bg_br_green() |> 112 | cli::cat_line() 113 | ``` 114 | 115 | ```{r} 116 | modelfile <- system.file("extdata", "modelfile.txt", package = "rollama") 117 | readLines(modelfile) |> 118 | cli::bg_br_green() |> 119 | cli::cat_line() 120 | ``` 121 | 122 | ```{r} 123 | create_model("mario", modelfile = modelfile) 124 | ``` 125 | 126 | ```{r} 127 | chat("who are you?", model = "mario") 128 | ``` 129 | 130 | # annotation 131 | 132 | (see ) 133 | 134 | ```{r} 135 | q <- tribble( 136 | ~role, ~content, 137 | "system", "You assign texts into categories. Answer with just the correct category.", 138 | "user", "text: the pizza tastes terrible\ncategories: positive, neutral, negative", 139 | "assistant", "{'Category':'Negative','Confidence':'100%','Important':'terrible'}", 140 | "user", "text: the service is great\ncategories: positive, neutral, negative" 141 | ) 142 | answer <- query(q) 143 | ``` 144 | 145 | ```{r} 146 | # Create an example dataframe with 5 movie reviews 147 | movie_reviews <- tibble( 148 | review_id = 1:5, 149 | review = c("A stunning visual spectacle with a gripping storyline.", 150 | "The plot was predictable, but the acting was superb.", 151 | "An overrated film with underwhelming performances.", 152 | "A beautiful tale of love and adventure, beautifully shot.", 153 | "The movie lacked depth, but the special effects were incredible.") 154 | ) 155 | # Print the initial dataframe 156 | movie_reviews 157 | ``` 158 | 159 | ```{r} 160 | movie_reviews_annotated <- movie_reviews |> 161 | mutate(annotation = map_chr(review, function(r) { 162 | q <- tribble( 163 | ~role, ~content, 164 | "system", "You assign texts into categories. Answer with just the correct category.", 165 | "user", "text: the pizza tastes terrible\ncategories: positive, neutral, negative", 166 | "assistant", "{'Category':'Negative','Confidence':'100%','Important':'terrible'}", 167 | "user", glue::glue("text: {r}\ncategories: positive, neutral, negative") 168 | ) 169 | query(q) |> 170 | pluck("message", "content") 171 | })) 172 | 173 | movie_reviews_annotated 174 | ``` 175 | 176 | ```{r} 177 | q <- tribble( 178 | ~role, ~content, 179 | "system", "You are a helpful text annotator that annotates opinions in language. An opinion consists of an **evaluation** (e.g., good or bad) and a **target** (e.g., a movie, a restaurant or a policy). If either one is missing, say NONE, do not make things up.", 180 | "user", "the pizza at this restaurant tastes terrible", 181 | "assistant", "{'Target':'Pizza at restaurant','Evaluation':'bad taste','Important words':'terrible'}", 182 | "user", "the service at Corte & Cata is great" 183 | ) 184 | answer <- query(q) 185 | ``` 186 | 187 | ```{r} 188 | f <- "data/wilson_speech.txt" 189 | speech <- readChar(f, nchars = file.size(f)) 190 | cat(speech) 191 | ``` 192 | 193 | ```{r} 194 | q <- tribble( 195 | ~role, ~content, 196 | "system", "You are a helpful text annotator that annotates opinions in language. An opinion consists of an **evaluation** (e.g., good or bad) and a **target** (e.g., a movie, a restaurant or a policy). If either one is missing, say NONE, do not make things up.", 197 | "user", "the pizza at this restaurant tastes terrible", 198 | "assistant", "{'Target':'Pizza at restaurant','Evaluation':'bad taste','Important words':'terrible'}", 199 | "user", speech 200 | ) 201 | query(q, model = "llama3:8b", model_params = list(seed = 42, temperature = 0)) 202 | ``` 203 | 204 | ```{r} 205 | q <- tribble( 206 | ~role, ~content, 207 | "system", "You are a helpful text annotator that annotates opinions in language. An opinion consists of an **evaluation** (e.g., good or bad) and a **target** (e.g., a movie, a restaurant or a policy). If either one is missing, say NONE, do not make things up.", 208 | "user", "the pizza at this restaurant tastes terrible", 209 | "assistant", "{'Target':'Pizza at restaurant','Evaluation':'bad taste','Important words':'terrible'}", 210 | "user", "YES! 211 | There are signs of a small surprise in the European elections in the Netherlands. According to post-election polls, the red-green alliance of Social Democrats and Greens is just ahead of the party of right-wing populist Geert Wilders, which had been ahead in the polls." 212 | ) 213 | query(q, model = "llama3:8b", model_params = list(seed = 42, temperature = 0)) 214 | ``` 215 | 216 | ```{r} 217 | df <- paperboy::pb_deliver("https://www.theguardian.com/commentisfree/article/2024/jun/09/the-observer-view-on-d-day-furore-rishi-sunak-is-driving-tories-over-an-electoral-cliff-edge") 218 | article <- paste(df$headline[1], "\n\n", df$text[1]) 219 | cat(article) 220 | q <- tribble( 221 | ~role, ~content, 222 | "system", "You are a helpful text annotator that annotates opinions in language. An opinion consists of an **evaluation** (e.g., good or bad) and a **target** (e.g., a movie, a restaurant or a policy). If either one is missing, say NONE, do not make things up.", 223 | "user", "the pizza at this restaurant tastes terrible", 224 | "assistant", "{'Target':'Pizza at restaurant','Evaluation':'bad taste','Important words':'terrible'}", 225 | "user", article 226 | ) 227 | query(q, model = "llama3:8b", model_params = list(seed = 42, temperature = 0)) 228 | ``` 229 | 230 | ```{r} 231 | df <- paperboy::pb_deliver("https://www.theguardian.com/commentisfree/article/2024/jun/09/the-guardian-view-on-frances-olympic-summer-searching-for-the-feelgood-factor") 232 | article <- paste(df$headline[1], "\n\n", df$text[1]) 233 | cat(article) 234 | q <- tribble( 235 | ~role, ~content, 236 | "system", "You are a helpful text annotator that annotates opinions in language. An opinion consists of an **evaluation** (e.g., good or bad) and a **target** (e.g., a movie, a restaurant or a policy). If either one is missing, say NONE, do not make things up.", 237 | "user", "the pizza at this restaurant tastes terrible", 238 | "assistant", "{'Target':'Pizza at restaurant','Evaluation':'bad taste','Important words':'terrible'}", 239 | "user", article 240 | ) 241 | query(q, model = "llama3:8b", model_params = list(seed = 42, temperature = 0)) 242 | ``` 243 | 244 | # image annotation 245 | 246 | (see ) 247 | 248 | ![](https://raw.githubusercontent.com/JBGruber/rollama/master/man/figures/logo.png) 249 | 250 | ```{r} 251 | query("Excitedly desscribe this logo", 252 | model = "llava", 253 | images = "https://raw.githubusercontent.com/JBGruber/rollama/master/man/figures/logo.png") 254 | ``` 255 | 256 | ![](media/JBGruber.jpg) 257 | 258 | ```{r} 259 | query("Who do you see on this picture: a student or a professor. Assign probabilities to each category. Think before you answer", 260 | model = "llava", 261 | images = "media/JBGruber.jpg") 262 | ``` 263 | -------------------------------------------------------------------------------- /media/JBGruber.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/JBGruber.jpg -------------------------------------------------------------------------------- /media/cluster_1/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/cluster_1/1.png -------------------------------------------------------------------------------- /media/cluster_1/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/cluster_1/2.png -------------------------------------------------------------------------------- /media/cluster_1/3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/cluster_1/3.png -------------------------------------------------------------------------------- /media/cluster_1/4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/cluster_1/4.png -------------------------------------------------------------------------------- /media/cluster_10/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/cluster_10/1.png -------------------------------------------------------------------------------- /media/cluster_10/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/cluster_10/2.png -------------------------------------------------------------------------------- /media/cluster_10/3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/cluster_10/3.png -------------------------------------------------------------------------------- /media/cluster_10/4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/cluster_10/4.png -------------------------------------------------------------------------------- /media/crack-dolphin-censored.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/crack-dolphin-censored.png -------------------------------------------------------------------------------- /media/crack-dolphin-censored.png~: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/crack-dolphin-censored.png~ -------------------------------------------------------------------------------- /media/crack-dolphin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/crack-dolphin.png -------------------------------------------------------------------------------- /media/crack-gpt4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/crack-gpt4.png -------------------------------------------------------------------------------- /media/gllm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/gllm.png -------------------------------------------------------------------------------- /media/gpt3-locally.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/gpt3-locally.png -------------------------------------------------------------------------------- /media/max.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/max.png -------------------------------------------------------------------------------- /media/notes.txt: -------------------------------------------------------------------------------- 1 | ![https://github.com/openai/gpt-3/issues/1#issuecomment-636008427](gpt3-locally.png) 2 | ![https://appian.com/blog/acp/process-automation/generative-ai-vs-large-language-models.html](gllm.png) 3 | 4 | 5 | # Demo 6 | 7 | Show how to install on laptop (pre-download container) 8 | 9 | 10 | -------------------------------------------------------------------------------- /media/ollama-r.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/ollama-r.png -------------------------------------------------------------------------------- /media/ollama.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/ollama.png -------------------------------------------------------------------------------- /media/qr-code-gist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/qr-code-gist.png -------------------------------------------------------------------------------- /media/qr-code-rollama.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/qr-code-rollama.png -------------------------------------------------------------------------------- /media/raspi.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/raspi.jpg -------------------------------------------------------------------------------- /media/raspi.jpg~: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/raspi.jpg~ -------------------------------------------------------------------------------- /media/rollama_hex.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/rollama_hex.png -------------------------------------------------------------------------------- /media/so-easy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/so-easy.png -------------------------------------------------------------------------------- /media/surprised.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JBGruber/rollama-presentation/fb9eec7b3ab63e8362f133154800aa20cc2905c5/media/surprised.png -------------------------------------------------------------------------------- /references.bib: -------------------------------------------------------------------------------- 1 | @misc{weber2023evaluation, 2 | title = {Evaluation is all you need. Prompting Generative Large Language Models for Annotation Tasks in the Social Sciences. A Primer using Open Models}, 3 | author = {Maximilian Weber and Merle Reichardt}, 4 | year = 2023, 5 | eprint = {2401.00284}, 6 | archiveprefix = {arXiv}, 7 | primaryclass = {cs.CL} 8 | } 9 | @misc{Chae_Davidson_2023, 10 | title = {Large Language Models for Text Classification: From Zero-Shot Learning to Fine-Tuning}, 11 | author = {Chae, Youngjin (YJ) and Davidson, Thomas}, 12 | year = 2023, 13 | publisher = {OSF}, 14 | doi = {10.31235/osf.io/sthwk}, 15 | url = {https://osf.io/sthwk}, 16 | language = {en-us} 17 | } 18 | @misc{Davidson_2023, 19 | title = {Start Generating: Harnessing Generative Artificial Intelligence for Sociological Research}, 20 | author = {Davidson, Thomas}, 21 | year = 2023, 22 | publisher = {OSF}, 23 | doi = {10.31235/osf.io/u9nft}, 24 | url = {https://osf.io/u9nft}, 25 | language = {en-us} 26 | } 27 | @article{GilardiChatGPT2023, 28 | title = {ChatGPT outperforms crowd workers for text-annotation tasks}, 29 | author = {Gilardi, Fabrizio and Alizadeh, Meysam and Kubli, Maël}, 30 | year = 2023, 31 | journal = {Proceedings of the National Academy of Sciences}, 32 | publisher = {Proceedings of the National Academy of Sciences}, 33 | volume = 120, 34 | number = 30, 35 | doi = {10.1073/pnas.2305016120}, 36 | url = {https://www.pnas.org/doi/10.1073/pnas.2305016120} 37 | } 38 | @misc{He_Lin_et_al._2023, 39 | title = {AnnoLLM: Making Large Language Models to Be Better Crowdsourced Annotators}, 40 | author = {He, Xingwei and Lin, Zhenghao and Gong, Yeyun and Jin, A.-Long and Zhang, Hang and Lin, Chen and Jiao, Jian and Yiu, Siu Ming and Duan, Nan and Chen, Weizhu}, 41 | year = 2023, 42 | publisher = {arXiv}, 43 | url = {http://arxiv.org/abs/2303.16854} 44 | } 45 | @misc{Heseltine_Hohenberg_2023, 46 | title = {Large Language Models as a Substitute for Human Experts in Annotating Political Text}, 47 | author = {Heseltine, Michael and Hohenberg, Bernhard Clemm von}, 48 | year = 2023, 49 | publisher = {OSF}, 50 | doi = {10.31219/osf.io/cx752}, 51 | url = {https://osf.io/cx752}, 52 | language = {en-us} 53 | } 54 | @misc{Kheiri_Karimi_2023, 55 | title = {SentimentGPT: Exploiting GPT for Advanced Sentiment Analysis and its Departure from Current Machine Learning}, 56 | author = {Kheiri, Kiana and Karimi, Hamid}, 57 | year = 2023, 58 | publisher = {arXiv}, 59 | url = {http://arxiv.org/abs/2307.10234}, 60 | language = {en} 61 | } 62 | @misc{Kojima_Gu_et_al._2023, 63 | title = {Large Language Models are Zero-Shot Reasoners}, 64 | author = {Kojima, Takeshi and Gu, Shixiang Shane and Reid, Machel and Matsuo, Yutaka and Iwasawa, Yusuke}, 65 | year = 2023, 66 | publisher = {arXiv}, 67 | url = {http://arxiv.org/abs/2205.11916} 68 | } 69 | @misc{Luccioni_Jernite_et_al._2023, 70 | title = {Power Hungry Processing: Watts Driving the Cost of AI Deployment?}, 71 | author = {Luccioni, Alexandra Sasha and Jernite, Yacine and Strubell, Emma}, 72 | year = 2023, 73 | publisher = {arXiv}, 74 | url = {http://arxiv.org/abs/2311.16863} 75 | } 76 | @misc{Møller_Dalsgaard_et_al._2023, 77 | title = {Is a prompt and a few samples all you need? Using GPT-4 for data augmentation in low-resource classification tasks}, 78 | author = {Møller, Anders Giovanni and Dalsgaard, Jacob Aarup and Pera, Arianna and Aiello, Luca Maria}, 79 | year = 2023, 80 | publisher = {arXiv}, 81 | url = {http://arxiv.org/abs/2304.13861} 82 | } 83 | @misc{Ollion_Shen_et_al._2023, 84 | title = {ChatGPT for Text Annotation? Mind the Hype!}, 85 | author = {Ollion, Etienne and Shen, Rubing and Macanovic, Ana and Chatelain, Arnault}, 86 | year = 2023, 87 | publisher = {OSF}, 88 | doi = {10.31235/osf.io/x58kn}, 89 | url = {https://osf.io/x58kn}, 90 | language = {en-us} 91 | } 92 | @misc{Pangakis_Wolken_et_al._2023, 93 | title = {Automated Annotation with Generative AI Requires Validation}, 94 | author = {Pangakis, Nicholas and Wolken, Samuel and Fasching, Neil}, 95 | year = 2023, 96 | publisher = {arXiv}, 97 | url = {http://arxiv.org/abs/2306.00176} 98 | } 99 | @misc{Plaza-del-Arco_Nozza_et_al._2023, 100 | title = {Leveraging Label Variation in Large Language Models for Zero-Shot Text Classification}, 101 | author = {Plaza-del-Arco, Flor Miriam and Nozza, Debora and Hovy, Dirk}, 102 | year = 2023, 103 | publisher = {arXiv}, 104 | url = {http://arxiv.org/abs/2307.12973} 105 | } 106 | @inproceedings{Rosenthal_Farra_et_al._2017, 107 | title = {SemEval-2017 Task 4: Sentiment Analysis in Twitter}, 108 | author = {Rosenthal, Sara and Farra, Noura and Nakov, Preslav}, 109 | year = 2017, 110 | booktitle = {Proceedings of the 11th International Workshop on Semantic Evaluation (SemEval-2017)}, 111 | publisher = {Association for Computational Linguistics}, 112 | address = {Vancouver, Canada}, 113 | pages = {502–518}, 114 | doi = {10.18653/v1/S17-2088}, 115 | url = {https://aclanthology.org/S17-2088} 116 | } 117 | @article{Spirling_2023, 118 | title = {Why open-source generative AI models are an ethical way forward for science}, 119 | author = {Spirling, Arthur}, 120 | year = 2023, 121 | journal = {Nature}, 122 | volume = 616, 123 | number = 7957, 124 | pages = {413–413}, 125 | doi = {10.1038/d41586-023-01295-4}, 126 | rights = {2023 Springer Nature Limited}, 127 | language = {en} 128 | } 129 | @misc{Törnberg_2023, 130 | title = {ChatGPT-4 Outperforms Experts and Crowd Workers in Annotating Political Twitter Messages with Zero-Shot Learning}, 131 | author = {Törnberg, Petter}, 132 | year = 2023, 133 | publisher = {arXiv}, 134 | url = {http://arxiv.org/abs/2304.06588} 135 | } 136 | @misc{Wang_Wei_et_al._2023, 137 | title = {Self-Consistency Improves Chain of Thought Reasoning in Language Models}, 138 | author = {Wang, Xuezhi and Wei, Jason and Schuurmans, Dale and Le, Quoc and Chi, Ed and Narang, Sharan and Chowdhery, Aakanksha and Zhou, Denny}, 139 | year = 2023, 140 | publisher = {arXiv}, 141 | url = {http://arxiv.org/abs/2203.11171} 142 | } 143 | @misc{Wei_Wang_et_al._2023, 144 | title = {Chain-of-Thought Prompting Elicits Reasoning in Large Language Models}, 145 | author = {Wei, Jason and Wang, Xuezhi and Schuurmans, Dale and Bosma, Maarten and Ichter, Brian and Xia, Fei and Chi, Ed and Le, Quoc and Zhou, Denny}, 146 | year = 2023, 147 | publisher = {arXiv}, 148 | url = {http://arxiv.org/abs/2201.11903} 149 | } 150 | @misc{White_Fu_et_al._2023, 151 | title = {A Prompt Pattern Catalog to Enhance Prompt Engineering with ChatGPT}, 152 | author = {White, Jules and Fu, Quchen and Hays, Sam and Sandborn, Michael and Olea, Carlos and Gilbert, Henry and Elnashar, Ashraf and Spencer-Smith, Jesse and Schmidt, Douglas C.}, 153 | year = 2023, 154 | publisher = {arXiv}, 155 | url = {http://arxiv.org/abs/2302.11382} 156 | } 157 | @misc{Zhu_Zhang_et_al._2023, 158 | title = {Can ChatGPT Reproduce Human-Generated Labels? A Study of Social Computing Tasks}, 159 | author = {Zhu, Yiming and Zhang, Peixian and Haq, Ehsan-Ul and Hui, Pan and Tyson, Gareth}, 160 | year = 2023, 161 | publisher = {arXiv}, 162 | url = {http://arxiv.org/abs/2304.10145} 163 | } 164 | @misc{Ziems_Held_et_al._2023, 165 | title = {Can Large Language Models Transform Computational Social Science?}, 166 | author = {Ziems, Caleb and Held, William and Shaikh, Omar and Chen, Jiaao and Zhang, Zhehao and Yang, Diyi}, 167 | year = 2023, 168 | publisher = {arXiv}, 169 | url = {http://arxiv.org/abs/2305.03514} 170 | } 171 | @inproceedings{Huang_Kwak_et_al._2023, 172 | title = {Is ChatGPT better than Human Annotators? Potential and Limitations of ChatGPT in Explaining Implicit Hate Speech}, 173 | author = {Huang, Fan and Kwak, Haewoon and An, Jisun}, 174 | year = 2023, 175 | booktitle = {Companion Proceedings of the ACM Web Conference 2023}, 176 | pages = {294–297}, 177 | doi = {10.1145/3543873.3587368}, 178 | url = {http://arxiv.org/abs/2302.07736} 179 | } 180 | @misc{Zhong_ChatGPT_2023, 181 | title = {Can ChatGPT Understand Too? A Comparative Study on ChatGPT and Fine-tuned BERT}, 182 | author = {Zhong, Qihuang and Ding, Liang and Liu, Juhua and Du, Bo and Tao, Dacheng}, 183 | year = 2023, 184 | publisher = {arXiv}, 185 | url = {http://arxiv.org/abs/2302.10198} 186 | } 187 | @misc{Reiss_2023, 188 | title = {Testing the Reliability of ChatGPT for Text Annotation and Classification: A Cautionary Remark}, 189 | author = {Reiss, Michael V.}, 190 | year = 2023, 191 | publisher = {arXiv}, 192 | url = {http://arxiv.org/abs/2304.11085} 193 | } 194 | @article{Linegar_Kocielnik_et_al._2023, 195 | title = {Large language models and political science}, 196 | author = {Linegar, Mitchell and Kocielnik, Rafal and Alvarez, R. Michael}, 197 | year = 2023, 198 | journal = {Frontiers in Political Science}, 199 | volume = 5, 200 | issn = {2673-3145}, 201 | url = {https://www.frontiersin.org/articles/10.3389/fpos.2023.1257092} 202 | } 203 | @misc{Thalken_Stiglitz_Mimno_Wilkens_2023, 204 | title = {Modeling Legal Reasoning: LM Annotation at the Edge of Human Agreement}, 205 | author = {Thalken, Rosamond and Stiglitz, Edward H. and Mimno, David and Wilkens, Matthew}, 206 | year = 2023, 207 | publisher = {arXiv}, 208 | number = {arXiv:2310.18440}, 209 | url = {http://arxiv.org/abs/2310.18440}, 210 | note = {arXiv:2310.18440 [cs]} 211 | } 212 | @article{Wankmüller_2022, 213 | title = {Introduction to Neural Transfer Learning With Transformers for Social Science Text Analysis}, 214 | author = {Wankmüller, Sandra}, 215 | year = 2022, 216 | journal = {Sociological Methods \& Research}, 217 | publisher = {SAGE Publications Inc}, 218 | pages = {1--77}, 219 | doi = {10.1177/00491241221134527}, 220 | issn = {0049-1241}, 221 | abstractnote = {Transformer-based models for transfer learning have the potential to achieve high prediction accuracies on text-based supervised learning tasks with relatively few training data instances. These models are thus likely to benefit social scientists that seek to have as accurate as possible text-based measures, but only have limited resources for annotating training data. To enable social scientists to leverage these potential benefits for their research, this article explains how these methods work, why they might be advantageous, and what their limitations are. Additionally, three Transformer-based models for transfer learning, BERT, RoBERTa, and the Longformer, are compared to conventional machine learning algorithms on three applications. Across all evaluated tasks, textual styles, and training data set sizes, the conventional models are consistently outperformed by transfer learning with Transformers, thereby demonstrating the benefits these models can bring to text-based social science research.}, 222 | language = {en} 223 | } 224 | @misc{UKDA-SN-5790-2, 225 | title = {National Child Development Study: Age 11, Sweep 2, Sample of Essays, 1969}, 226 | author = {{University of London, Institute of Education, Centre for Longitudinal Studies}}, 227 | year = 2023, 228 | publisher = {UK Data Service}, 229 | doi = {10.5255/UKDA-SN-5790-2}, 230 | url = {http://doi.org/10.5255/UKDA-SN-5790-2}, 231 | note = {SN: 5790} 232 | } 233 | 234 | @misc{alizadeh2023opensource, 235 | title = {Open-Source Large Language Models Outperform Crowd Workers and Approach ChatGPT in Text-Annotation Tasks}, 236 | author = {Meysam Alizadeh and Maël Kubli and Zeynab Samei and Shirin Dehghani and Juan Diego Bermeo and Maria Korobeynikova and Fabrizio Gilardi}, 237 | year = 2023, 238 | eprint = {2307.02179}, 239 | archiveprefix = {arXiv}, 240 | primaryclass = {cs.CL} 241 | } 242 | -------------------------------------------------------------------------------- /slides.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Ollama" 3 | subtitle: "*So simple school kids can use it!*" 4 | author: "Johannes B. Gruber // Societal Analytics Lab" 5 | format: 6 | revealjs: 7 | embed-resources: true 8 | smaller: true 9 | scrollable: true 10 | bibliography: references.bib 11 | title-slide-attributes: 12 | data-background-image: media/ollama-r.png 13 | data-background-size: "350px" 14 | data-background-position: bottom -15px right -15px 15 | --- 16 | 17 | # Late 2022 18 | 19 | ```{r setup} 20 | #| include: false 21 | library(tidyverse) 22 | knitr::opts_chunk$set(eval = FALSE, echo = TRUE) 23 | ``` 24 | 25 | Question: How can you run generative large language models (gLLM; aka GPT, aka ChatGPT-like, aka GPT-like, aka generative AI, AI text bots) locally? 26 | 27 | [![](media/gpt3-locally.png)](https://github.com/openai/gpt-3/issues/1#issuecomment-636008427) 28 | 29 | # Late 2023 30 | 31 | Question: How can you run generative large language models (gLLM; aka GPT, aka ChatGPT-like, aka GPT-like, aka generative AI, AI text bots) locally? 32 | 33 | [![](media/so-easy.png)](https://www.reddit.com/r/LocalLLaMA/comments/177tryq/ollama_is_making_entry_into_the_llm_world_so/) 34 | ![](media/raspi.jpg){.absolute width=300 right=1 top=450} 35 | 36 | # What is Ollama? 37 | 38 | ## Ollama: a tool/framework 39 | 40 | - Ollama is a tool for running large language models (LLMs) locally 41 | - It optimizes setup and configuration details, including GPU usage 42 | - Is easy to install and run 43 | - Easy to extend 44 | - MIT license (aka. open-source)! 45 | 46 | 47 | ## Ollama models {.smaller .scrollable} 48 | 49 | - Ollama bundles model weights, configuration, and data into a single package, defined by a Modelfile 50 | - A curated model library is available at 51 | - Custom models can be up/downloaded from 52 | - Huggingface models can be imported (Llama, Mistral, Falcon, RW and BigCode architecture supported atm) 53 | 54 | ::: {style="font-size: 70%;"} 55 | | Model | Parameters | Size | 56 | | ------------------ | ---------- | ----- | 57 | | Llama 2 | 7B | 3.8GB | 58 | | Mistral | 7B | 4.1GB | 59 | | Dolphin Phi | 2.7B | 1.6GB | 60 | | Phi-2 | 2.7B | 1.7GB | 61 | | Neural Chat | 7B | 4.1GB | 62 | | Starling | 7B | 4.1GB | 63 | | Code Llama | 7B | 3.8GB | 64 | | Llama 2 Uncensored | 7B | 3.8GB | 65 | | Llama 2 13B | 13B | 7.3GB | 66 | | Llama 2 70B | 70B | 39GB | 67 | | Orca Mini | 3B | 1.9GB | 68 | | Vicuna | 7B | 3.8GB | 69 | | LLaVA | 7B | 4.5GB | 70 | ::: 71 | 72 | # Why use gLLMs? 73 | 74 | :::{.smaller} 75 | - cheaper to use 💸 (experts > research assistants > crowd workers > gLLMs) 76 | - more accurate 🧠 (experts > research assistants > gLLMs > crowd workers) 77 | - feasible to make large training set (for SML) or code entire corpus 78 | - multilingual^[but still need of validation!] 79 | - better at annotate complex categories 🤔 (human* > gLLM > LLM > SML) 80 | - can use contextual information (human* > gLLM > LLM > ❌ SML) 81 | - easier to use: plain English prompting 82 | 83 | [e.g., @GilardiChatGPT2023;@Heseltine_Hohenberg_2023;@Zhong_ChatGPT_2023;@Törnberg_2023] 84 | ::: 85 | 86 | # Why is local important? 87 | ## Why should we run gLLMs locally? 88 | 89 | - No guardrails 🤨 90 | 91 | :::: {.columns} 92 | 93 | ::: {.column width="50%" .absolute top=200 left=40} 94 | ![](media/crack-gpt4.png) 95 | ::: 96 | 97 | ::: {.column width="50%" .absolute top=200 right=40 .fragment .fade-in} 98 | ![](media/crack-dolphin-censored.png) 99 | ::: 100 | 101 | :::: 102 | 103 | ![](media/surprised.png){.absolute width="150px" top=100 right=150 .fragment} 104 | 105 | ## Why should we run gLLMs locally? 106 | 107 | - No guardrails 🤨 108 | 109 | :::: {.columns} 110 | 111 | ::: {.column width="50%" .absolute top=200 left=40} 112 | ![](media/crack-gpt4.png) 113 | ::: 114 | 115 | ::: {.column width="50%" .absolute top=200 right=40} 116 | ![](media/crack-dolphin.png) 117 | ::: 118 | 119 | :::: 120 | 121 | ![](media/surprised.png){.absolute width="150px" top=100 right=150} 122 | 123 | ## Why should we run gLLMs locally? 124 | 125 | - privacy (nothing is uploaded anywhere) 126 | - cost (cheap and reliable pricing) 127 | - long-term reproducible 128 | - other ethical concerns (?) 129 | - transparency (?) 130 | 131 | [@weber2023evaluation; @Spirling_2023] 132 | 133 | ![](media/max.png){.absolute top=100 right=0 .fragment .fade-up} 134 | 135 | 136 | # Examples 137 | ## Classification 138 | ## Classification--Strategies 139 | 140 | ::: {style="font-size: 50%;"} 141 | | Prompting Strategy | Example Structure | 142 | |--------------------|--------------------| 143 | | Zero-shot | `{"role": "system", "content": "Text of System Prompt"},`
`{"role": "user", "content": "(Text to classify) + classification question"}` | 144 | | One-shot | `{"role": "system", "content": "Text of System Prompt"},`
`{"role": "user", "content": "(Example text) + classification question"},`
`{"role": "assistant", "content": "Example classification"},`
`{"role": "user", "content": "(Text to classify) + classification question"}` | 145 | | Few-shot | `{"role": "system", "content": "Text of System Prompt"},`
`{"role": "user", "content": "(Example text) + classification question"},`
`{"role": "assistant", "content": "Example classification"},`
`{"role": "user", "content": "(Example text) + classification question"},`
`{"role": "assistant", "content": "Example classification"},`
`. . . more examples`
`{"role": "user", "content": "(Text to classify) + classification question"}` | 146 | | Chain-of-Thought | `{"role": "system", "content": "Text of System Prompt"},`
`{"role": "user", "content": "(Text to classify) + reasoning question"},`
`{"role": "assistant", "content": "Reasoning"},`
`{"role": "user", "content": "Classification question"}` | 147 | : Prompting strategies {.striped .hover tbl-colwidths="[15,85]"} 148 | ::: 149 | 150 | See: @weber2023evaluation 151 | 152 | 153 | ## Classification--Zero-shot {.scrollable} 154 | 155 | ```{r} 156 | #| echo: true 157 | #| eval: false 158 | library(rollama) 159 | library(tibble) 160 | library(purrr) 161 | q <- tribble( 162 | ~role, ~content, 163 | "system", "You assign texts into categories. Answer with just the correct category.", 164 | "user", "text: the pizza tastes terrible\ncategories: positive, neutral, negative" 165 | ) 166 | query(q) 167 | #> 168 | #> ── Answer ──────────────────────────────────────────────────────── 169 | #> Category: Negative 170 | ``` 171 | 172 | ```{r} 173 | #| message: false 174 | #| include: false 175 | if (reticulate::virtualenv_exists("r-ollama")) { 176 | reticulate::virtualenv_create("r-ollama", packages = c( 177 | "ollama" 178 | )) 179 | } 180 | reticulate::use_virtualenv("r-ollama") 181 | library(reticulate) 182 | ``` 183 | 184 | :::{.fragment} 185 | 186 | Also packages for Dart, Swift, C#, Java, PHP, Rust etc. 187 | 188 | Python: 189 | 190 | ```{python} 191 | #| echo: true 192 | #| eval: false 193 | import ollama 194 | response = ollama.chat(model='llama2', messages=[ 195 | { 196 | 'role': 'system', 197 | 'content': 'You assign texts into categories. Answer with just the correct category.', 198 | }, 199 | { 200 | 'role': 'user', 201 | 'content': 'text: the pizza tastes terrible\ncategories: positive, neutral, negative', 202 | }, 203 | ]) 204 | print(response['message']['content']) 205 | #> Category: Negative 206 | ``` 207 | 208 | JavaScript: 209 | 210 | ```{js} 211 | #| echo: true 212 | #| eval: false 213 | import ollama from 'ollama' 214 | 215 | const response = await ollama.chat({ 216 | model: 'llama2', 217 | messages: [ 218 | { 219 | 'role': 'system', 220 | 'content': 'You assign texts into categories. Answer with just the correct category.', 221 | }, 222 | { 223 | 'role': 'user', 224 | 'content': 'text: the pizza tastes terrible\ncategories: positive, neutral, negative', 225 | }, 226 | ], 227 | }) 228 | console.log(response.message.content) 229 | #> Category: Negative 230 | ``` 231 | 232 | 233 | ::: 234 | 235 | ## Classification--One-shot {.scrollable} 236 | 237 | ```{r} 238 | #| echo: true 239 | #| eval: false 240 | q <- tribble( 241 | ~role, ~content, 242 | "system", "You assign texts into categories. Answer with just the correct category.", 243 | "user", "text: the pizza tastes terrible\ncategories: positive, neutral, negative", 244 | "assistant", "Category: Negative", 245 | "user", "text: the service is great\ncategories: positive, neutral, negative" 246 | ) 247 | query(q) 248 | #> 249 | #> ── Answer ──────────────────────────────────────────────────────── 250 | #> Category: Positive 251 | ``` 252 | 253 | :::{.fragment} 254 | 255 | Neat effect: change the output structure 256 | 257 | ```{r} 258 | #| echo: true 259 | #| eval: false 260 | #| classes: .fragement .fade-in 261 | #| code-line-numbers: "5,11" 262 | q <- tribble( 263 | ~role, ~content, 264 | "system", "You assign texts into categories. Answer with just the correct category.", 265 | "user", "text: the pizza tastes terrible\ncategories: positive, neutral, negative", 266 | "assistant", "{'Category':'Negative','Confidence':'100%','Important':'terrible'}", 267 | "user", "text: the service is great\ncategories: positive, neutral, negative" 268 | ) 269 | answer <- query(q) 270 | #> 271 | #> ── Answer ──────────────────────────────────────────────────────── 272 | #> {'Category':'Positive','Confidence':'100%','Important':'great'} 273 | ``` 274 | 275 | ::: 276 | 277 | ## Classification--Few-shot 278 | 279 | 280 | ```{r} 281 | q <- tribble( 282 | ~role, ~content, 283 | "system", "You assign texts into categories. Answer with just the correct category.", 284 | "user", "text: the pizza tastes terrible\ncategories: positive, neutral, negative", 285 | "assistant", "Category: Negative", 286 | "user", "text: the service is great\ncategories: positive, neutral, negative", 287 | "assistant", "Category: Positive", 288 | "user", "text: I once came here with my wife\ncategories: positive, neutral, negative", 289 | "assistant", "Category: Neutral", 290 | "user", "text: I once ate pizza\ncategories: positive, neutral, negative" 291 | ) 292 | query(q) 293 | #> 294 | #> ── Answer ──────────────────────────────────────────────────────── 295 | #> Category: Neutral 296 | ``` 297 | 298 | ## Classification--Chain-of-Thought {.scrollable} 299 | 300 | ```{r} 301 | #| echo: true 302 | #| eval: false 303 | #| code-line-numbers: "4,11-12" 304 | q_thought <- tribble( 305 | ~role, ~content, 306 | "system", "You assign texts into categories. ", 307 | "user", "text: the pizza tastes terrible\nWhat sentiment (positive, neutral, or negative) would you assign? Provide some thoughts." 308 | ) 309 | output_thought <- query(q_thought) 310 | #> 311 | #> ── Answer ──────────────────────────────────────────────────────── 312 | #> 313 | #> Based on the given text, I would assign it to the category of "negative 314 | #> sentiment." The use of the word "terrible" and the tone of the sentence convey 315 | #> a strong negative emotion towards the taste of the pizza. It's unlikely that 316 | #> someone would describe something as "terrible" if they were neutral or 317 | #> positive about it. 318 | ``` 319 | 320 | :::{.fragment} 321 | Now we can use these thoughts in classification 322 | 323 | ```{r} 324 | #| echo: true 325 | #| eval: false 326 | #| code-line-numbers: "6" 327 | q <- tribble( 328 | ~role, ~content, 329 | "system", "You assign texts into categories. ", 330 | "user", "text: the pizza tastes terrible\nWhat sentiment (positive, neutral, or negative) would you assign? Provide some thoughts.", 331 | "assistant", pluck(output_thought, "message", "content"), 332 | "user", "Now answer with just the correct category (positive, neutral, or negative)" 333 | ) 334 | query(q) 335 | #> 336 | #> ── Answer ──────────────────────────────────────────────────────── 337 | #> 338 | #> Negative 339 | ``` 340 | 341 | Full tutorial (inlcuding how to batch annotate): 342 | ::: 343 | 344 | ## Annotating news events {.scrollable .smaller} 345 | 346 | ::: {style="font-size: 60%;"} 347 | ```{r} 348 | #| echo: false 349 | #| eval: true 350 | eval_set_df <- readRDS("data/eval_set_df.rds") 351 | eval_set_df_annotated <- rio::import("data/eval_set.csv") 352 | eval_set_df |> 353 | arrange(cluster) |> 354 | select(-name, -text) |> 355 | head(10) |> 356 | mutate(url = str_trunc(url, 24)) |> 357 | mutate(across(where(is.character), \(s) str_trunc(s, 120))) |> 358 | tinytable::tt(theme = "striped") 359 | ``` 360 | ::: 361 | 362 | ```{r} 363 | #| echo: true 364 | #| code-fold: true 365 | #| code-summary: "Annotation" 366 | #| code-line-numbers: "1-30|13|15|17|9,27,29" 367 | library(rollama) 368 | eval_set_df_annotated <- eval_set_df |> 369 | group_by(cluster) |> 370 | summarise(headlines = paste0(title, collapse = "\n\n"), 371 | n = length(title)) |> 372 | filter(n > 1) |> 373 | mutate(label = NA) 374 | 375 | for (i in seq_len(nrow(eval_set_df_annotated))) { 376 | 377 | q <- tibble::tribble( 378 | ~role, ~content, 379 | "system", "You assign short highly descriptive topic labels to a collection of headlines about the same topic. Answer with just the topic label.", 380 | 381 | "user", "headlines: \nGuns, ammunition, drugs and cash seized as part of investigation into organised crime in Tipperary as...\n\nFirearms, ammunition, machete, crack cocaine, heroin and cash seized in garda operation - Irish Mirror Online\n\nMan arrested after gardaí seize weapons and drugs in Clonmel", 382 | 383 | "assistant", "blow against organised crime", 384 | 385 | "user", "headlines: Taoiseach Leo Varadkar stung by Russian pranksters in hoax diplomatic call\n\nTaoiseach Leo Varadkar ‘pranked’ in phone call by pro-Kremlin Russian ‘comedians’ pretending to be African diplomats\n\nGovernment changes meetings protocol after Taoiseach duped in Russian hoax call\n\nTaoiseach Leo Varadkar ‘pranked’ in phone call by pro-Kremlin Russian ‘comedians’ pretending to be African diplomats\n\nLeo Varadkar falls victim to a prank by two Russian comedians\n\nLeo Varadkar insists he 'played along' with Russian pranksters on hoax video call - Irish Mirror Online\n\nLeo Varadkar falls victim to prank call by pair of Russian comedians - Irish Mirror Online\n\nTaoiseach was 'suspicious immediately' of prank by two Russian comedians but had to play along\n\nRed-faced Leo Varadkar makes urgent change to Government video call security protocols after embarrassing...\n\nI played along with prank call by Russian comedians – Taoiseach\n\nTaoiseach Leo Varadkar ‘played along’ with Russian comedians’ prank phone call after he became suspicious\n\nTaoiseach Leo Varadkar ‘played along’ with Russian comedians’ prank phone call after he became suspicious\n\nLeo Varadkar “pranked” by pro-Kremlin Russian “comedians”", 386 | 387 | "assistant", "Taoiseach pranked by Russians", 388 | 389 | "user", "headlines: Social media platform X back online after global outage\n\nSocial media platform X back online after global outage\n\nX is down: Thousands of users experiencing outages on platform formerly known as Twitter\n\nTwitter down: X website owned by Elon Musk suffers outages as thousands of users report problems...\n\nElon Musk’s X back online after global outage\n\nSocial media platform X back online after global outage · TheJournal.ie\n\nX users experiencing widespread outages", 390 | 391 | "assistant", "platform X outage", 392 | 393 | "user", glue::glue("headlines: {eval_set_df_annotated$headlines[i]}") 394 | ) 395 | eval_set_df_annotated$label[i] <- query(q, screen = FALSE) |> pluck("message", "content") 396 | } 397 | ``` 398 | 399 | 400 | ## Annotating news events {.smaller} 401 | 402 | ## Classification--Cluster 1: *Super League* 403 | 404 | ![](media/cluster_1/1.png){.absolute width="45%"} 405 | ![](media/cluster_1/2.png){.absolute width="45%" left="35%"} 406 | ![](media/cluster_1/4.png){.absolute width="45%" top=300 left="50%"} 407 | ![](media/cluster_1/3.png){.absolute width="45%" top=500} 408 | 409 | ## Annotating news events {.smaller} 410 | 411 | ### Cluster 10: *Trump banned from Colorado ballot* 412 | 413 | ![](media/cluster_10/1.png){.absolute width="45%"} 414 | ![](media/cluster_10/2.png){.absolute width="45%" left="35%"} 415 | ![](media/cluster_10/3.png){.absolute width="45%" top=500} 416 | ![](media/cluster_10/4.png){.absolute width="45%" top=350 left="50%"} 417 | 418 | ```{r} 419 | #| echo: false 420 | eval_set_df |> 421 | filter(cluster == 10) |> 422 | pull(url) |> 423 | sapply(browseURL) 424 | # webshot2::webshot(file = "media/cluster_1/1.png") 425 | ``` 426 | 427 | ## Annotating news events {.smaller .scrollable} 428 | 429 | ```{r} 430 | #| eval: true 431 | #| echo: false 432 | eval_set_df_annotated |> 433 | slice(1:30) |> 434 | mutate(across(where(is.character), \(s) str_trunc(s, 50))) |> 435 | mutate(good = str_replace(good, "^v$", cli::symbol$tick)) |> 436 | tinytable::tt(theme = "striped") |> 437 | tinytable::style_tt(fontsize = 0.7) |> 438 | tinytable::style_tt(i = which(eval_set_df_annotated$good == "v"), j = 5, background = "#D2D9A6") |> 439 | tinytable::style_tt(i = which(eval_set_df_annotated$good != "v"), j = 5, background = "#C40E19") 440 | ``` 441 | 442 | ## Text embedding 443 | 444 | ```r 445 | embed_text(text = reviews$full_text[1:3]) 446 | #> ✔ embedded 3 texts [4s] 447 | #> # A tibble: 3 × 4,096 448 | #> dim_1 dim_2 dim_3 dim_4 dim_5 dim_6 dim_7 dim_8 dim_9 dim_10 449 | #> 450 | #> 1 1.85 -1.71 1.47 0.478 -1.75 0.771 3.01 0.961 1.65 0.569 451 | #> 2 1.14 -3.61 2.10 -0.385 -4.11 -3.09 0.990 -1.06 2.55 1.84 452 | #> 3 -3.35 0.172 -3.49 -0.569 -3.14 1.25 -0.102 1.15 0.575 -2.33 453 | #> # ℹ 4,086 more variables: dim_11 , dim_12 , 454 | #> # dim_13 , dim_14 , dim_15 , dim_16 , 455 | #> # dim_17 , dim_18 , dim_19 , dim_20 , 456 | #> # dim_21 , dim_22 , dim_23 , dim_24 , 457 | #> # dim_25 , dim_26 , dim_27 , dim_28 , 458 | #> # dim_29 , dim_30 , dim_31 , dim_32 , 459 | #> # dim_33 , dim_34 , dim_35 , dim_36 , … 460 | ``` 461 | 462 | Tutorial for SML: 463 | 464 | # Demo 465 | 466 | ## What I will use here {.scrollable .smaller} 467 | 468 | :::: {.columns} 469 | 470 | ::: {.column width="70%"} 471 | ```yaml 472 | version: '3.6' 473 | 474 | services: 475 | 476 | # ollama and API 477 | ollama: 478 | image: ollama/ollama:latest 479 | container_name: ollama 480 | pull_policy: missing 481 | tty: true 482 | restart: unless-stopped 483 | # Expose Ollama API outside the container stack 484 | ports: 485 | - 11434:11434 486 | volumes: 487 | - ollama:/root/.ollama 488 | # GPU support (turn off by commenting with # if you don't have an nvidia gpu) 489 | deploy: 490 | resources: 491 | reservations: 492 | devices: 493 | - driver: nvidia 494 | count: 1 495 | capabilities: 496 | - gpu 497 | 498 | # webui, nagivate to http://localhost:3000/ to use 499 | ollama-webui: 500 | image: ghcr.io/ollama-webui/ollama-webui:main 501 | container_name: ollama-webui 502 | pull_policy: missing 503 | volumes: 504 | - ollama-webui:/app/backend/data 505 | depends_on: 506 | - ollama 507 | ports: 508 | - 3000:8080 509 | environment: 510 | - "OLLAMA_API_BASE_URL=http://ollama:11434/api" 511 | extra_hosts: 512 | - host.docker.internal:host-gateway 513 | restart: unless-stopped 514 | 515 | volumes: 516 | ollama: {} 517 | ollama-webui: {} 518 | ``` 519 | ::: 520 | 521 | ::: {.column width="30%"} 522 | [![](media/qr-code-gist.png)](https://gist.github.com/JBGruber/73f9f49f833c6171b8607b976abc0ddc){target="_blank"} 523 | 524 | tinyurl.com/ollama-gist 525 | ::: 526 | 527 | :::: 528 | 529 | ## What I will use here {.scrollable .smaller} 530 | 531 | :::: {.columns} 532 | 533 | ::: {.column width="40%"} 534 | ![](media/rollama_hex.png) 535 | ::: 536 | 537 | ::: {.column width="40%"} 538 | 539 | 540 | [![](media/qr-code-rollama.png)](https://tinyurl.com/r-ollama){target="_blank"} 541 | 542 | tinyurl.com/r-ollama 543 | ::: 544 | 545 | :::: 546 | 547 | 548 | # Issues 549 | 550 | Methodological: 551 | 552 | - field is still an academic Wild West -- lack of established standards for the reliable, reproducible and ethical use of gLLMs (see Törnberg draft) 553 | - open models still lack behind proprietary ones (but not as far as we though a year ago!) 554 | - consistent bias (as opposed to sporadic bias with a group of human annotators) 555 | 556 | Technical: 557 | 558 | - Small-ish hurdle to configure GPU, especially with Docker (and cost to get (NVIDIA*) GPU) 559 | - Replies are not consistent despite seed 560 | - Mixtral is currently slow 561 | - Some API parameters don't work 562 | - In short: it's still work in progress (but impressive nonetheless!) 563 | 564 | # References 565 | --------------------------------------------------------------------------------