├── LICENSE ├── README.md ├── backend.py ├── blog ├── 2021-01-20-understand-your-unstructured-data │ └── index.mdx ├── 2021-01-25-history-of-weaviate │ └── index.mdx ├── 2021-01-27-graphql-api-design │ └── index.mdx ├── 2021-02-15-crud-support-in-weaviate │ └── index.mdx ├── 2021-03-30-weaviate-1-2-transformer-models │ └── index.mdx ├── 2021-05-19-docker-and-containers-with-weaviate │ └── index.mdx ├── 2021-11-25-semantic-search-with-wikipedia-and-weaviate │ └── index.mdx ├── 2022-06-23-the-ai-first-database-ecosystem │ └── index.mdx ├── 2022-07-04-weaviate-1-14-release │ └── index.mdx ├── 2022-08-09-using-cross-encoders-as-reranker-in-multistage-vector-search │ └── index.mdx ├── 2022-08-15-gomemlimit-a-came-changer-for-high-memory-applications │ └── index.mdx ├── 2022-08-30-research-insights-spider │ └── index.mdx ├── 2022-09-01-weaviate-cloud-service │ └── index.mdx ├── 2022-09-07-weaviate-1-15-release │ └── index.mdx ├── 2022-09-13-why-is-vector-search-so-fast │ └── index.mdx ├── 2022-09-21-weaviate-1-15-1-release │ └── index.mdx ├── 2022-09-27-hugging-face-inference-api-in-weaviate │ └── index.mdx ├── 2022-10-04-how-to-choose-a-sentence-transformer-from-hugging-face │ └── index.mdx ├── 2022-10-11-ann-algorithms-vamana-vs-hnsw │ └── index.mdx ├── 2022-10-18-how-to-build-an-image-search-application-with-weaviate │ └── index.mdx ├── 2022-10-25-lock-striping-pattern │ └── index.mdx ├── 2022-11-01-weaviate-1-16-release │ └── index.mdx ├── 2022-11-15-tutorial-backup-and-restore-in-weaviate │ └── index.mdx ├── 2022-11-23-ref2vec-centroid │ └── index.mdx ├── 2022-12-01-vector-library-vs-vector-database │ └── index.mdx ├── 2022-12-06-sphere-dataset-in-weaviate │ └── index.mdx ├── 2022-12-12-cohere-multilingual-with-weaviate │ └── index.mdx ├── 2022-12-20-weaviate-1-17-release │ ├── _core-1-17-include.mdx │ └── index.mdx ├── 2022-12-27-details-behind-the-sphere-dataset-in-weaviate │ └── index.mdx ├── 2023-01-03-hybrid-search-explained │ └── index.mdx ├── 2023-01-10-pulling-back-the-curtains-on-text2vec │ └── index.mdx ├── 2023-01-16-vector-embeddings-explained │ └── index.mdx ├── 2023-01-24-how-ai-creates-art │ └── index.mdx ├── 2023-01-31-weaviate-podcast-search │ └── index.mdx ├── 2023-02-07-generative-search │ └── _index.md ├── 2023-02-14-what-to-expect-from-weaviate-in-2023 │ └── index.mdx ├── 2023-02-21-combining-langchain-and-weaviate │ └── index.mdx ├── 2023-02-28-solution-to-tl-drs │ └── index.mdx ├── 2023-03-07-weaviate-1-18-release │ └── index.mdx ├── 2023-03-14-ann-algorithms-hnsw-pq │ └── index.mdx ├── 2023-03-21-ann-algorithms-tiles-encoder │ └── index.mdx ├── 2023-03-23-what-are-llms │ └── index.mdx ├── 2023-03-28-monitoring-weaviate-in-production │ └── index.mdx ├── 2023-04-04-weaviate-retrieval-plugin │ └── index.mdx ├── 2023-04-11-ranking-models │ └── index.mdx ├── 2023-04-18-autogpt-and-weaviate │ └── index.mdx ├── 2023-04-25-auth-in-weaviate │ └── index.mdx ├── 2023-04-27-how-to-chatgpt-plugin │ └── index.mdx ├── 2023-05-02-wcs-public-beta │ └── index.mdx ├── 2023-05-04-weaviate-1-19-release │ └── index.mdx ├── 2023-05-05-generative-feedback-loops │ └── index.mdx ├── 2023-05-10-announcing-palm-modules │ └── index.mdx ├── 2023-05-23-pdfs-to-weaviate │ └── index.mdx ├── 2023-05-30-private-LLM │ └── index.mdx ├── 2023-06-06-embedded-weaviate │ └── index.mdx ├── 2023-06-13-llms-and-search │ └── index.mdx ├── 2023-06-15-multi-tenancy │ └── index.mdx ├── 2023-06-22-llamaindex-and-weaviate │ └── index.mdx ├── 2023-06-27-multimodal-models │ └── index.mdx ├── 2023-07-11-weaviate-1-20-release │ └── index.mdx ├── 2023-07-18-automated-testing │ └── index.mdx ├── 2023-07-26-healthsearch-blogpost │ └── index.mdx ├── 2023-08-01-vector-database │ └── index.mdx ├── 2023-08-15-distance-metrics-in-vector-search │ └── index.mdx ├── 2023-08-22-weaviate-1-21-release │ └── _core-1-21-include.mdx ├── 2023-08-29-hybrid-search-fusion │ └── index.mdx ├── 2023-09-11-weaviate-gorilla-part-1 │ └── index.mdx ├── 2023-09-19-pq-rescoring │ └── index.mdx ├── 2023-09-26-collections-python-client │ └── index.mdx ├── 2023-09-26-confluent-and-weaviate │ └── index.mdx ├── 2023-10-02-hacktoberfest-2023 │ └── index.mdx ├── 2023-10-31-weaviate-1-22-release │ └── index.mdx ├── 2023-11-15-moonsift-story │ └── index.mdx ├── 2023-11-21-rag-evaluation │ └── index.mdx ├── 2023-11-30-zero-downtime-upgrades │ └── index.mdx ├── 2023-12-05-multimodal-RAG │ └── index.mdx ├── 2023-12-12-aws-reinvent │ └── index.mdx ├── 2023-12-19-weaviate-1-23-release │ └── index.mdx ├── 2023-12-26-weaviate-recap │ └── index.mdx ├── 2024-01-23-typescript-multimodal-search │ └── index.mdx ├── 2024-01-30-weaviate-non-english-unicode │ └── index.mdx ├── 2024-02-01-py-client-v4-release │ └── index.mdx ├── 2024-02-08-weaviate-and-spcs │ └── index.mdx ├── 2024-02-13-fine-tuning-cohere-reranker │ └── index.mdx ├── 2024-02-20-grpc-performance-improvements │ └── index.mdx ├── 2024-02-27-weaviate-1-24-release │ └── _core-1-24-include.mdx ├── 2024-03-07-verba-open-source-rag-app │ └── index.mdx ├── 2024-03-12-enterprise-use-cases-weaviate │ └── index.mdx ├── 2024-03-13-weaviate-champions │ └── index.mdx └── 2024-03-19-hurricane-blog-gfls │ └── index.mdx ├── blog_schema.py ├── docker-compose.yml ├── gpt4_compiled_hurricane.json ├── hurricane-frontend ├── README.md ├── package.json ├── public │ ├── favicon.ico │ ├── index.html │ ├── logo192.png │ ├── logo512.png │ ├── manifest.json │ └── robots.txt └── src │ ├── App.css │ ├── App.js │ ├── App.test.js │ ├── Blogpost.js │ ├── assets │ └── bg-light-3.png │ ├── index.css │ ├── index.js │ ├── logo.svg │ ├── reportWebVitals.js │ └── setupTests.js ├── hurricane_program.py ├── hurricane_signatures.py ├── import_blogs.py ├── init_schema.py └── utils.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Weaviate Tutorials 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Hurricane 🌀 2 | Writing Blog Posts with Generative Feedback Loops! 3 | 4 | To run, follow these steps: 5 | 1. Spin up Weaviate with: (add your OpenAI API key) 6 | ```bash 7 | docker-compose up -d 8 | ``` 9 | 2. Insert the current Weaviate blogs into Weaviate by running: 10 | ```bash 11 | python3 import_blogs.py 12 | ``` 13 | 3. Start the backend with: (add your OpenAI and You API keys) 14 | ```bash 15 | uvicorn backend:app --reload 16 | ``` 17 | 4. Start the frontend with: 18 | ```bash 19 | cd hurricane-frontend 20 | npm install 21 | npm start 22 | ``` 23 | -------------------------------------------------------------------------------- /blog/2021-01-20-understand-your-unstructured-data/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: What if you could understand your unstructured data? 4 | slug: understand-your-unstructured-data 5 | authors: [laura] 6 | date: 2021-01-20 7 | tags: ['concepts'] 8 | image: ./img/hero.png 9 | # canonical-url: https://medium.com/semi-technologies/what-if-you-could-understand-your-unstructured-data-92f0861e016 10 | # canonical-name: Medium 11 | description: "Learn how the AI-first vector database Weaviate unlocks the potential of unstructured data and why this is important." 12 | --- 13 | ![What if you could understand your unstructured data?](./img/hero.png) 14 | 15 | 16 | 17 | ## Intro 18 | These days, more and more organizations are adopting a data-driven culture. Business processes and customer experience benefit from good data collection, management and analysis. But in order to really benefit from available data, it is essential to also understand the unstructured data, like free text in PDF documents, emails, invoices or voice transcriptions. Unstructured data is especially hard to index, manage and understand. Since around [80% of all data is unstructured](https://www.forbes.com/sites/forbestechcouncil/2019/01/29/the-80-blind-spot-are-you-ignoring-unstructured-organizational-data/), it is hard to actually search and retrieve insights from most of the data. 19 | 20 | The Weaviate vector database unlocks the potential of unstructured data. Searching by fuzzy terms and classification of rich data like free text becomes possible with Weaviate. It uses AI-driven indexing and search technologies to enable real-time text processing. With machine learning methods, Weaviate automatically classifies texts. Finding information you are looking for and providing recommendations is possible because knowledge and information is placed in context. 21 | 22 | ## How does Weaviate work? 23 | Data usually refers to something in the real world. Traditional databases and search engines often struggle with understanding the context of data, a situation or a search query. The context of data is important, because it enables storage and search by fuzzy concepts instead of exact matching keywords. Weaviate automatically places concepts and words in relation to its context by using machine learning. It understands that the Eiffel Tower is a landmark in Paris, and will place these concepts closely together in the database model. Showing [orange tompouces](https://en.wikipedia.org/wiki/Tompouce) when looking for treats around King's day in the Netherlands is an example of finding concepts related to the context. 24 | 25 | ## Example use cases 26 | Now let's get more serious than landmarks and holiday treats (although we should agree that tompouces are essential during King's day), and find out how also your business could benefit from search and classification of unstructured data. 27 | 28 | ## Automatic classification 29 | Data classification is often done manually or by expensive external data science teams. ERP (enterprise resource planning) and supply chain management systems rely on correct classification of for example employees' travel expenses or ingredient labeling of product packages. Automatic classification by Weaviate reduces manual classification errors and eliminates expensive, single-to-use classification applications. Automatic classification can also be applied to cybersecurity and event management, where intervention on threats and riskful events should be taken real-time. Current platforms deal with very large amounts of streaming data, consisting of mostly free text. Weaviate can automatically recognize threats in free text, and classify if and which mitigation should be taken. With Weaviate this can be achieved near real-time, even on a very large scale. 30 | 31 | ## Semantic search 32 | Traditional search systems return items that have an exact match with the search terms. This means that searching through your emails for ‘reimbursement' will skip all results about ‘compensation'. Searching for ‘science fiction' in a traditional book search engine, might lead to anything related to ‘science' or ‘fiction' so this would include ‘neuroscience' but not ‘a book about the future'. Weaviate uses semantic search, and matches search results to the user's intent through contextual meaning. For e-commerce websites this means that people searching for ‘jumpers' will also see ‘sweatshirts' simply because their contextual meaning is the same although the keywords do not match. 33 | 34 | See how automatically placing orange tompouces in the context of King's day might thus actually lead to upselling your products and happier customers? Now there's certainly also potential hidden in the majority of your data. Think about how an AI-driven vector-search can unlock this from your organization's data! 35 | 36 | 37 | import WhatNext from '/_includes/what-next.mdx' 38 | 39 | 40 | -------------------------------------------------------------------------------- /blog/2021-01-25-history-of-weaviate/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: The History of Weaviate 4 | slug: history-of-weaviate 5 | authors: [bob] 6 | date: 2021-01-25 7 | tags: ['concepts'] 8 | image: ./img/hero.png 9 | # canonical-url: https://medium.com/semi-technologies/the-history-of-the-weaviate-vector-search-engine-17a274f36178 10 | # canonical-name: Medium 11 | description: "Weaviate is an open-source vector database with a built-in NLP model called the Contextionary. Learn what makes Weaviate unique." 12 | --- 13 | ![The History of Weaviate](./img/hero.png) 14 | 15 | 16 | 17 | In this article, I want to share the history of Weaviate, how the concept was born, and where we are heading towards in the near future. 18 | 19 | ## A World of Wonders called Natural Language Processing 20 | Somewhere early 2015 I was introduced to the concept of word embeddings through the publication of an article that contained a machine-learning algorithm to turn individual words into embeddings called [GloVe](https://nlp.stanford.edu/projects/glove/). 21 | 22 | ### Example of an embedding 23 | 24 | ```txt 25 | squarepants 0.27442 -0.25977 -0.17438 0.18573 0.6309 0.77326 -0.50925 -1.8926 26 | 0.72604 0.54436 -0.2705 1.1534 0.20972 1.2629 1.2796 -0.12663 0.58185 0.4805 27 | -0.51054 0.026454 0.20253 0.32844 0.72568 1.23 0.90203 1.1607 -0.39605 0.80305 28 | 0.16373 0.053425 -0.65308 1.0607 -0.29405 0.42172 -0.45183 0.57945 0.20217 29 | -1.3342 -0.71227 -0.6081 -0.3061 0.96214 -1.1097 -0.6499 -1.1147 0.4808 0.29857 30 | -0.30444 1.3929 0.088861 31 | ``` 32 | 33 | If you are new to the world of word embeddings, a metaphor to understand them is in the form of a supermarket. The supermarket itself is a space in which products are stored based on the category they are in. Inside, you can find an apple by moving your shopping cart to the right coordinates in the fruit section and when you look around you, you'll find similar products like oranges, limes, bananas, etcetera, and you also know that a cucumber will be closer by than washing powder. 34 | 35 | 36 | This is the same way a word embedding is structured. All the coordinates combined represent a multidimensional hyperspace (often around 300 dimensions) and words that have a similar meaning are more closely related to each other, like similar products in the store. 37 | 38 | Being able to represent words in a space gives you a superpower, because it allows you to calculate with language! Instead of creating algorithms to understand language, it is now possible to simply look up what is in the neighborhood in the space. 39 | 40 | ## How to semantically store data objects 41 | While working on software projects in my day-to-day life, I noticed that one of the most recurring challenges presented itself in naming and searching. How would we call certain objects and how could we find data that was structured in different ways? I fell in love with the semantic web but the challenge I saw there, was the need to have people agree on naming conventions and standards. 42 | 43 | This made me wonder, what if we wouldn't have to agree on this any more? What if we could just store data and have the machine figure out the concept that your data represents? 44 | 45 | The validation of the concept was chunked up into three main sections that were validated one by one. 46 | 47 | 1. Can one get more context around a word by moving through the hyperspace? If so; 48 | 1. Can one keep semantic meaning by calculating a centroid of a group of words (e.g., "thanks for the sushi last week")? If so; 49 | 1. Can this be done fast without retraining the ML model? 50 | 51 | Finding more context around a word has to do with a concept called disambiguation. Take for example the word "apple". In the hyperspace, if you look for other words in the neighborhood, you will find words related to apple the fruit (e.g., other fruits, juices, etcetera) but also Apple the company (iPhone, Macintosh, and other concepts related to the company). 52 | 53 | To validate if we could disambiguate the word "apple" the following simple step was taken. What if we looked for all the words that are in the neighborhood of the space in between "apple" and "fruit"? Turns out the results are way better! We can disambiguate by moving through the hyperspace while using individual words as beacons to navigate. 54 | 55 | ![Vector Space](./img/vector-space.jpeg) 56 | 57 | In the next step, the goal was to validate if we could keep semantic meaning when storing a data object in the hyperspace by calculating the centroid using the individual words as beacons. We did that as follows, the title of this Vogue article: "Louis Vuitton's New Capsule with League of Legends Brings French High Fashion to Online Gaming — and Vice Versa". 58 | 59 | If we look up the vector positions for the individual words (Louis, Vuitton, new, capsule, etc.) and place a new beacon in the center of the space of those vector positions, can we find the article by searching for "haute couture"? This turns out to work as well! Of course, through time, the centroid calculation algorithm in Weaviate has become way more sophisticated, but the overall concept is still the same. 60 | 61 | > By validating the above two assumptions, we knew that we could almost instantly store data objects in a semantic space rather than a more traditional row-column structure or graph. This allows us to index data based on its meaning. 62 | 63 | Although we had validated the assumptions of the semantic concepts, it was not enough to create an actual semantic search engine. Weaviate also needed a data model to represent these results. 64 | 65 | ## Things Rather Than Strings 66 | In September 2017 I wrote a [blog post](https://medium.com/bob-van-luijt/semantic-internet-of-things-42811e1ca7a7) about the overlap between the internet of things and the semantic web. IoT focuses on physical objects (i.e., "things") and the semantic web focuses on the mapping of data objects that represent something (a product, transaction, car, person, etcetera) which at the highest level of abstraction, are also "things". 67 | 68 | I wrote this article because, in January 2016, I was invited as part of the Google Developer Expert program in San Francisco, to visit the Ubiquity Conference. A conference where, back then, Google's Weave and Brillo were introduced to the public. 69 | 70 | Weave was the cloud application built around Brillo, but it piqued my interest because it focused on "things", how you defined them, and actions that you could execute around them. The very first iteration of Weaviate focused on exactly this: "Could Weave be used to define other things than IoT devices like transactions, or cars, or any other things?". In 2017 Google deprecated Weave and renamed Brillo to Android Things but the concept for Weaviate stayed. 71 | 72 | From the get-go, I knew that the "things" in Weaviate should be connected to each other in graph format because I wanted to be able to represent the relationships between the data objects (rather than flat, row-based information) in an as simple and straightforward manner as possible. 73 | 74 | > This search led to the RDF structure used by schema.org, which functioned as an inspiration for how to represent Weaviate's data objects. 75 | 76 | Weaviate is not per se RDF- or schema.org-based, but is definitely inspired by it. One of the most important upsides of this approach was that we could use GraphQL (the graph query language which was entering the software stage through Facebook open-sourcing it) to represent the data inside Weaviate. 77 | 78 | With the concept of realtime vectorization of data objects and RDF-like representation of Weaviate objects in GraphQL, all the ingredients to turn Weaviate into the search graph that it currently is, were present. 79 | 80 | ## The Birth of the Weaviate Search Graph 81 | By the end of 2018, I entered Weaviate into a startup accelerator program in The Netherlands. A place where I had the time to build a team around Weaviate that could help get the software to production and create a business model around the open-source project (the startup became: SeMI Technologies, which is short for Semantic Machine Insights). 82 | 83 | When the team started, Weaviate was more of a traditional graph where the semantic (NLP) element was a feature rather than the core architecture. But when we started to learn how powerful the out-of-the-box semantic search was and the role that embeddings play in day-to-day software development (i.e., a lot of machine learning models create embeddings as output), the team decided that we wanted to double down on the NLP part and vector storage, creating a unique open-source project, which could be used as a semantic search engine. The Weaviate Search Graph was born. 84 | 85 | ## How people use it today 86 | One of the coolest things about an open-source community and users of the software is to see how people use it and what trends you can see emerge around implementations. The core features of Weaviate are the semantic search element and the semantic classification, which are used in a variety of ways and industries. 87 | 88 | Examples of implementations include: classification of invoices into categories, searching through documents for specific concepts rather than keywords, site search, product knowledge graphs, and many other things. 89 | 90 | ## The Future 91 | Weaviate will stay fully open source for the community to use. The team is growing to accelerate building Weaviate and supporting users. We are releasing new features very frequently, like new vector indices and search pipeline features, and Weaviate Cloud Services and Weaviate Console. 92 | 93 | ![Weaviate Console](./img/weaviate-console.jpeg) 94 | 95 | ## Listen to Bob's story at the Open Source Data 96 | Want to learn more about the background of Vector Search and how the ecosystem is developing? [Listen to Sam Ramji from DataStax interviewing Bob van Luijt about Vector Search and the AI stack.](https://www.datastax.com/resources/podcast/vector-search-ai-stack-more-bob-van-luijt). 97 | 98 | 99 | import WhatNext from '/_includes/what-next.mdx' 100 | 101 | 102 | -------------------------------------------------------------------------------- /blog/2021-01-27-graphql-api-design/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: How Weaviate's GraphQL API was designed 3 | slug: graphql-api-design 4 | authors: [laura] 5 | date: 2021-01-27 6 | tags: ['concepts'] 7 | image: ./img/hero.png 8 | # canonical-url: https://medium.com/semi-technologies/how-weaviates-graphql-api-was-designed-b38885aa9cee 9 | # canonical-name: Medium 10 | description: "Any kind of data storage architecture needs an API. Learn how and why Weaviate picked GraphQL." 11 | --- 12 | ![How Weaviate's GraphQL API was designed](./img/hero.png) 13 | 14 | 15 | 16 | Choosing a good API, its design and development, is a crucial but time-consuming process, especially if you want to develop one in an ongoing software development project. 17 | 18 | Weaviate uses the API query language [GraphQL](https://graphql.org/). GraphQL enables efficient development and provides high user experience (UX) for data interaction. 19 | 20 | In this article we explain how the use of GraphQL leverages the UX of Weaviate, and how we approach the design of this API. 21 | 22 | ![Demo of Weaviate](./img/weaviate-demo.gif?i=8) 23 | 24 | ## What is Weaviate and why does it need an API? 25 | Weaviate is an [open-source](https://github.com/weaviate/weaviate) Vector Database: for understandable knowledge representation, enabling semantic search and automatic classification. Weaviate does not only store data, but also its (automatically derived) context, consisting of linguistic information and relations to other concepts. The result is a network of knowledge, or a graph of data. 26 | 27 | One core question is: How do we interact with this new type of data storage? Interacting with big data, enriched with contextual information, might sound even more overwhelming than interacting with a traditional, relational database. 28 | 29 | Data needs to be added, retrieved and manipulated, all controlled by the user but enabled by the underlying database interface. Here's where APIs jump in. Because of Weaviate's graph-based architecture, an alternative to traditional RESTful APIs was what we were looking for. 30 | 31 | ## What is GraphQL, and why does Weaviate use it? 32 | [GraphQL](https://graphql.org/) is an API query language, which was open-sourced by Facebook in 2012 and currently maintained by the Linux Foundation. The core strength of GraphQL is that the client is fully in control of what data is returned. This means that over-fetching is never a problem and the need for multiple requests is removed. In addition, using GraphQL allows fast API development and easy API evolvement over time, because of its powerful development tools and single evolving versioning. 33 | 34 | All in all, many reasons to adopt GraphQL for the graph-like structure of Weaviate. GraphQL seems the perfect solution for intuitive database interaction and efficient development. Weaviate still uses traditional RESTful endpoints (using OpenAPI/Swagger) to add data, but the main interaction for data consumption goes via GraphQL.) 35 | 36 | GraphQL still follows the same constraints as REST APIs, but data is organized into a graph using only one interface. Thus instead of endpoints, the API is organized in terms of types and fields. The fields of interest are specified by the client, chosen from a schema defined by the server. This requires a well formulated schema. Let's dive into how we approach the design of Weaviate's GraphQL API. 37 | 38 | ## Human-Centered API Design 39 | So, how does our GraphQL design, development and implementation cycle look like? We approach this very carefully, since we are talking about a design for end-users, made by developers, two sides that differ in many ways. Instead of a data-driven approach, we took a user point of view. Humans tend to talk in interactions; we ask questions when we want to retrieve something, and use verbs or even commands if we want someone or something to do something for us. So why not also apply the same behaviour to interaction with data? So try to think about what you would want to get out of the database in different use cases (as a user), and how this could be done most efficiently. What naturally follows is the words that are frequently used in these interactions. We used these words in the design of our GraphQL schema. 40 | 41 | Let's look at an example. Consider you (digitally) store supermarket items in Weaviate, and now want to know which items are low in storage. This could be achieved by the following query: 42 | 43 | ```graphql 44 | { 45 | Get { 46 | Product( 47 | where: { 48 | path: ["inStock"], 49 | operator: LessThan, 50 | valueInt: 100 51 | }) { 52 | name 53 | inStock 54 | } 55 | } 56 | } 57 | ``` 58 | 59 | Which may return: 60 | 61 | ```json 62 | { 63 | "data": { 64 | "Get": { 65 | "Product": [ 66 | { 67 | "name": "Baguette", 68 | "inStock": 8 69 | }, 70 | { 71 | "name": "Banana", 72 | "inStock": 46 73 | }, 74 | { 75 | "name": "Pineapple", 76 | "inStock": 17 77 | }, 78 | { 79 | "name": "Toilet paper", 80 | "inStock": 38 81 | } 82 | ] 83 | } 84 | } 85 | } 86 | ``` 87 | 88 | Here's how this query is structured. "Get" is a high level schema item that we defined. "Get" is a verb that we use when we want to retrieve something. What comes next is defined by the data schema of the user. This supermarket use case is storing "Products", with at least the properties "name", and "inStock". The user might also store properties like "barcode" and "expirationDate", but chose not to retrieve those in this request. The goal of this query is to Get all the Products where the Stock is less than 100. See how the query is formed? 89 | 90 | The following query shows the automatic contextualization of concepts in Weaviate. Let's say we are only interested in the fruit items that are low in stock, but we never explicitly stored the supermarket products by category like ‘fruits', ‘dairy' and ‘household'. We can then make the following query, where we want to Get all the Products where the Stock is less than 100, which are related to the text concept "fruits". Now, only the Banana and Pineapple will be returned because these items are most related to fruits. If we are looking for more results, then Baguette will be shown earlier in the list than Toilet paper, because it is semantically (and in a real supermarket) closer to fruits.) 91 | 92 | ```graphql 93 | { 94 | Get { 95 | Product( 96 | where: { 97 | path: ["inStock"], 98 | operator: LessThan, 99 | valueInt: 100 100 | }, 101 | nearText: { 102 | concepts: ["fruit"], 103 | certainty: 0.7 104 | } 105 | ) { 106 | name 107 | inStock 108 | } 109 | } 110 | } 111 | ``` 112 | 113 | ## Iterative Prototyping and Feedback 114 | In the end, developers and UX designers need to collaborate in this requirement engineering process. The challenge is to match the user needs with technical possibilities, while supporting all features the underlying database has to offer. For the GraphQL API design of Weaviate, an iterative approach was taken. Quick prototypes of new ideas are made all the time, which are passed in a circle of UX designers, core developers and end-users, continuously gathering and implementing constructive feedback. Only when everyone is happy with the proposal (thus not only the developers), a final ‘pass' is made to the core developers to implement it in the software. 115 | 116 | ## In conclusion 117 | With Weaviate, we want to allow the end-user to be focused on their data, and thus not spend too much effort on thinking about how to actually access and interact with it. Only then we can ensure high user experience, and open doors to new insights. With GraphQL, users can count on predictable, fast and stable results, because they are in control of the data interaction. Using our human-centered, iterative design process, with a continuous prototyping and feedback loop, we enable efficient and effective data interaction. 118 | 119 | [Click here](/developers/weaviate/api/graphql) to learn more about the complete GraphQL structure. 120 | 121 | 122 | import WhatNext from '/_includes/what-next.mdx' 123 | 124 | 125 | -------------------------------------------------------------------------------- /blog/2021-03-30-weaviate-1-2-transformer-models/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Weaviate 1.2 release - transformer models 3 | slug: weaviate-1-2-transformer-models 4 | authors: [etienne] 5 | date: 2021-03-30 6 | tags: ['release'] 7 | image: ./img/hero.png 8 | # canonical-url: https://medium.com/semi-technologies/weaviate-version-1-2-x-now-supports-transformer-models-4a12d858cce3 9 | # canonical-name: Medium 10 | description: "Weaviate v1.2 introduced support for transformers (DistilBERT, BERT, RoBERTa, Sentence-BERT, etc) to vectorize and semantically search through your data." 11 | --- 12 | ![Weaviate 1.2 release - transformer models](./img/hero.png) 13 | 14 | In the v1.0 release of Weaviate ([docs](/developers/weaviate/) — [GitHub](https://github.com/weaviate/weaviate)) we introduced the concept of [modules](/developers/weaviate/concepts/modules). Weaviate modules are used to extend the vector database with vectorizers or functionality that can be used to query your dataset. With the release of Weaviate v1.2, we have introduced the use of transformers ([DistilBERT](https://arxiv.org/abs/1910.01108), [BERT](https://github.com/google-research/bert), [RoBERTa](https://arxiv.org/abs/1907.11692), Sentence-[BERT](https://arxiv.org/abs/1908.10084), etc) to vectorize and semantically search through your data. 15 | 16 | 17 | 18 | ### Weaviate v1.2 introduction video 19 | 20 |
21 | 22 |
23 | 24 | ## What are transformers? 25 | A [transformer](https://en.wikipedia.org/wiki/Transformer_(machine_learning_model)) (e.g., [BERT](https://en.wikipedia.org/wiki/BERT_(language_model))) is a deep learning model that is used for NLP tasks. Within Weaviate the transformer module can be used to vectorize and query your data. 26 | 27 | ## Getting started with out-of-the-box transformers in Weaviate 28 | By selecting the text-module in the [Weaviate configuration tool](/developers/weaviate/installation/docker-compose#configurator), you can run Weaviate with transformers in one command. You can learn more about the Weaviate transformer module [here](/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-transformers). 29 | 30 | ![Weaviate configurator — selecting the Transformers module](./img/configurator-demo.gif) 31 | *Weaviate configurator — selecting the Transformers module* 32 | 33 | ## Custom transformer models 34 | You can also use custom transformer models that are compatible with Hugging Face's `AutoModel` and `AutoTokenzier`. Learn more about using custom models in Weaviate [here](/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-transformers). 35 | 36 | ## Q&A style questions on your own dataset answered in milliseconds 37 | Weaviate now allows you to get to sub-50ms results by using transformers on your own data. You can learn more about Weaviate’s speed in combination with transformers in [this article](https://towardsdatascience.com/a-sub-50ms-neural-search-with-distilbert-and-weaviate-4857ae390154). 38 | 39 | 40 | import WhatNext from '/_includes/what-next.mdx' 41 | 42 | 43 | -------------------------------------------------------------------------------- /blog/2021-05-19-docker-and-containers-with-weaviate/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Docker and Containers with Weaviate 3 | slug: docker-and-containers-with-weaviate 4 | authors: [etienne] 5 | date: 2021-05-19 6 | tags: ['concepts'] 7 | image: ./img/hero.png 8 | # canonical-url: https://medium.com/semi-technologies/what-weaviate-users-should-know-about-docker-containers-1601c6afa079 9 | # canonical-name: Medium 10 | description: "What Weaviate users should know about Docker & Containers." 11 | --- 12 | ![Docker and Containers with Weaviate](./img/hero.png) 13 | 14 | 15 | 16 | Weaviate attracts different users with various backgrounds. Some have been working with containers for years, but we understand that not everyone has. Inspired by a few recent questions and comments about Docker on the Weaviate Slack, I've set out to write an article to provide a better background on Docker and containers in general. After reading this article, your most common questions about these technologies should be answered and there should be nothing in the way of building amazing use cases with Weaviate. 17 | 18 | In this short overview, we will: 19 | 20 | * look at what "Docker" and "Docker Compose" is, 21 | * why Weaviate relies on Docker, 22 | * what you need to do to get started, 23 | * … and answer a few more container-related questions that we encounter regularly. 24 | 25 | ## What is Docker? 26 | > Docker is a […] [set of products] that use OS-level virtualization to deliver software in packages called containers.
27 | > [Wikipedia](https://en.wikipedia.org/wiki/Docker_(software)) 28 | 29 | You can think of docker containers as lightweight virtual machines. Each container has its own file system, own OS libraries and software. This provides a lot of isolation. At the same time, containers share the host system's kernel, so the overhead is much smaller than with regular virtual machines. 30 | 31 | The best practice around docker containers dictates that one container only contains a single application. Typically the lifecycle of the container depends on this one process. If the process exits, the container is stopped. 32 | 33 | ## What is Docker Compose? 34 | Because one container contains only a single app, we need a way to "tie many containers together". [Docker Compose](https://docs.docker.com/compose/) is a tool that can run custom manifests, so called "Docker Compose files", together. Instead of starting or stopping a single container, you can start all the containers in one go. This is very helpful to conveniently tie individual services together, for example when an application's architecture consists of individual micro services. This is the case with Weaviate — especially when multiple modules are involved. 35 | 36 | ## Why does Weaviate use Docker & Docker Compose? 37 | There are many arguments for or against containerization, we have decided to go with Docker & Docker Compose in the Weaviate stack for the following reasons: 38 | 39 | * **Portability**: Whether you're running on Windows, Linux, macOS or yet another OS, Weaviate is only compiled for a single Linux distribution. The Docker container makes sure that from Weaviate's perspective it's always running on a well-known Linux — even if your Host machine is running a completely different OS. 40 | * **Isolation & Predictability**: Weaviate is a compiled binary that has zero runtime dependencies, but if you use Weaviate modules which rely on third-party tools, such as Hugging Face Transformers, PyTorch or Tensorflow, they have a set of runtime dependencies. Thanks to containers, all runtime dependencies (Python version, python packages, OS libraries, etc.) are guaranteed to be present and at the right version in each container. 41 | * **Distribution via Docker Hub**: Docker Hub makes it very easy to distribute public images. With a single `docker pull semitechnologies/weaviate:latest` command you can always obtain the latest version — without any prior configuration. Additionally, each container can have an exact version number, making it easy to switch between versions or upgrade your setup. Instead of a lengthy and risky update process, you can simply exchange the entire container for that of a newer version. 42 | 43 | ## What do I need to install before running Weaviate? 44 | You need to make sure that you have both the `docker` and the `docker compose CLI` tools installed. Depending on your operating system, these might be two separate installation process. Here are the guides for the most common operating systems: 45 | 46 | * [Docker Desktop for Mac](https://docs.docker.com/desktop/install/mac-install/) (both `docker` and `docker compose` ) 47 | * [Docker Desktop for Windows](https://docs.docker.com/desktop/install/windows-install/) (both `docker` and `docker compose` ) 48 | * [Docker for Ubuntu Linux](https://docs.docker.com/engine/install/ubuntu/) (only `docker` and its dependencies) 49 | * [Docker Compose for (Ubuntu) Linux](https://docs.docker.com/compose/install/) 50 | 51 | ## How do I obtain Weaviate Docker images? 52 | Usually, there is no need to obtain individual images, as we distribute entire `docker-compose.yml` files. See next step. 53 | 54 | ## How do I obtain a Weaviate Docker Compose file? 55 | ![Weaviate Configuration Tool](./img/weaviate-configuration-tool.png) 56 | *An example of the Weaviate Customizer which produces a complete Docker Compose file — tailored to your needs.* 57 | 58 | The easiest way to obtain a `docker-compose.yml` is to use the [Weaviate configuration tool](/developers/weaviate/installation/docker-compose#configurator) on our website. It will ask you a few questions about what you plan to do with Weaviate and customize a Compose file (including all module containers, if you select any) in the last step. 59 | 60 | If you don't want to use the customizer you can also copy/paste one of [the example files from the documentation](/developers/weaviate/installation/docker-compose). 61 | 62 | ## How do I start up the Docker Compose file I just obtained? 63 | Make sure that: 64 | 65 | * You are in the same folder you downloaded the file to 66 | * The file you downloaded is named exactly docker-compose.yml 67 | 68 | You can now start up with the whole setup by running: 69 | 70 | ```bash 71 | docker compose up -d 72 | ``` 73 | 74 | The `-d` means "detach" and means that your terminal will not attach to the log outputs of all the containers. To attach to the logs of a specific containers you can run: 75 | 76 | ```bash 77 | docker compose logs -f 78 | ``` 79 | 80 | for example: 81 | 82 | ```bash 83 | docker compose logs -f weaviate 84 | ``` 85 | 86 | You can also omit the -d option and keep your terminal attached to all logs. 87 | 88 | To see the names and status of each container you can run: 89 | 90 | ```bash 91 | docker compose ps 92 | ``` 93 | 94 | ## How do I know when Weaviate is up and ready? 95 | Weaviate implements a readiness check at `GET /v1/.well-known/ready`. It will return a `2xx` HTTP status code once everything is ready. 96 | 97 | To check for the readiness programmatically you can use `curl` in this simple `bash` loop: 98 | 99 | ```bash 100 | until curl --fail -s localhost:8080/v1/.well-known/ready; do 101 | sleep 1 102 | done 103 | ``` 104 | 105 | Make sure to adjust localhost:8080 if you are running on a different bind address. 106 | 107 | ## Can I run Weaviate without Docker / Containerization? 108 | We do not officially support running Weaviate in a non-containerized fashion, but it is certainly possible. To do so, you have to compile Weaviate for your desired target OS and architecture. Weaviate is written in Golang, so you need to have a working Go runtime. See the [official documentation for more details](https://golang.org/doc/install/source#environment). 109 | 110 | ## Can Weaviate run on Kubernetes? Is Helm supported? 111 | Yes, see next step. 112 | 113 | ## When should or shouldn't I use Docker Compose? 114 | Docker Compose is quick, easy and convenient, but there are situations that it isn't suited for. We recommend to use a Docker Compose setup for trying out or evaluating Weaviate and when developing with Weaviate locally. 115 | 116 | However, for a stable production environment, we [recommend to run Weaviate on Kubernetes](/developers/weaviate/installation/kubernetes). You can use Helm, there is an [officially supported Weaviate Helm Chart](https://github.com/weaviate/weaviate-helm). 117 | 118 | ## Where to go from here? 119 | Congratulations, you now have a lot of background on Docker and containerization in general! Now there's nothing in the way of [Getting Started with Weaviate](/developers/weaviate/quickstart) and begin building amazing apps with Weaviate. 120 | 121 | You can reach out to us on [Slack](https://weaviate.io/slack) or [Twitter](https://twitter.com/weaviate_io), or [join the community forum](https://forum.weaviate.io). 122 | 123 | import Ending from '/_includes/blog-end-oss-comment.md' 124 | 125 | 126 | -------------------------------------------------------------------------------- /blog/2022-06-23-the-ai-first-database-ecosystem/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: The AI-First Database Ecosystem 3 | slug: the-ai-first-database-ecosystem 4 | authors: [bob] 5 | date: 2022-06-23 6 | tags: ['concepts'] 7 | image: ./img/hero.jpeg 8 | # canonical-url: https://www.forbes.com/sites/forbestechcouncil/2022/06/23/the-ai-first-database-ecosystem/ 9 | # canonical-name: Forbes 10 | description: "Learn about the vision of the AI-First Database Ecosystem, which drives the R&D of the databases of the future." 11 | --- 12 | ![The AI-First Database Ecosystem](./img/hero.jpeg) 13 | 14 | 15 | 16 | A new ecosystem of smaller companies is ushering in a "third wave" of AI-first database technology. New search engines and databases brilliantly answer queries posed in natural language, but their machine-learning models are not limited to text searches. The same approach can also be used to search anything from images to DNA. 17 | 18 | Much of the software involved is open source, so it functions transparently and users can customize it to meet their specific needs. Clients can retain control of their data, keeping it safely behind their own firewalls. 19 | 20 | ## How We Got Here 21 | First-wave database technology is often called by the acronym SQL—the initials of a ubiquitous query language used to manage relational databases, which are conceptually similar to spreadsheets or tables. Throughout the 1980s, this technology was dominated by companies like Oracle and Microsoft. 22 | 23 | The second wave of databases is called "NoSQL". These are the domain of companies like MongoDB. They store data in different ways, for example, key-value stores, document stores, wide-column stores and graph databases, but what they all have in common is that they're not relational tables. Among a long list of capabilities, first- and second-wave databases have their strengths. For example, some are very good at finding every instance of a certain value in a database, and others are very good at storing time sequences. 24 | 25 | The third wave of database technologies focuses on data that is processed by a machine learning model first, where the AI models help in processing, storing and searching through the data as opposed to traditional ways. 26 | 27 | To better understand the concept, think of a supermarket with 50,000 items. Items on display are not organized alphabetically or by price, the way you'd expect a structured, digital system to do it; they're placed in context. You find things in the supermarket by understanding how they relate to each other. So if the store gets a new product—say, guavas—you know to look near the apples and bananas, not near garbage bags or other things that happen to also cost $1.98/lb. 28 | 29 | A key early milestone in the third wave happened in 2015 when Google changed its search algorithm from one based on page rankings to one based on a machine learning model that it dubbed RankBrain. Before then, Google's search engine was essentially a high-powered keyword search that ranked websites by the number of other sites that linked back to them. Essentially, Google trusted rankings to the collective users of the Internet. 30 | 31 | That "wisdom of the crowd" approach worked, but to improve the quality of the results it returned, Google needed RankBrain to "understand" the text it searched through. So, it used machine learning to vectorize (the process happening inside machine learning models such as transformers) text on sites and in links. 32 | 33 | Returning to the grocery store for a moment, the challenge comes because a grocery store is a three-dimensional space, but every significant word in unstructured text data needs to be related to hundreds of other words that it is frequently associated with. So, machine learning systems automatically classify text in hypervolumes—imaginary spaces with hundreds or even thousands of dimensions. For any given item in a database, those vectors form what's known as a "representation" of the item. 34 | 35 | Since it conveys both content and context, such a representation obviously presents a more complete and nuanced data picture. The challenge comes from searching through myriad dimensions. Initially, this was done with a brute force approach, looking at every vector associated with every entry. Needless to say, that approach didn't scale. 36 | 37 | One breakthrough that helped third-wave vector databases to scale was an approach called "approximate nearest neighbor" (ANN) search. If we make one final trip to the supermarket to understand what a guava is, we could look at the things around it—other fruit. Somewhat further away we might find guava juice or tinned guavas, but there's really no reason to look four aisles over for guava-flavored cat food. 38 | 39 | Using ANN allowed high-dimensional searches with near-perfect accuracy to be returned in milliseconds instead of hours. To be practical, vector databases also needed something prosaically called CRUD support. That stands for "create, read, update and delete," and solving that technical challenge meant that the complex process of indexing the database could be done once, rather than being repeated from scratch whenever the database was updated. 40 | 41 | This leads to the simplest definition of a third database wave: A vector database that stores data indexed by machine learning models. Different types of databases (e.g., vector search engines) allow users to search through these vectorized datasets, and others (e.g., feature stores) allow users to store vectors at a large scale for later use. 42 | 43 | ## We're awash in unstructured data. 44 | We're living in a time of massive data accumulation and much, if not most of it, is unstructured: text, photos, video, audio files, as well as other things such as genetic information. Vector search is particularly good at extracting value from such data. 45 | 46 | Tech giants like Google, AWS, or Microsoft Azure offer their vector-search capabilities to customers willing to upload their data. But there's now an ecosystem of newer companies with AI-first specific (often open-source) solutions and vector-search capabilities that customers can run on a SaaS basis or on their own systems. 47 | 48 | ## The AI-first Database Ecosystem 49 | The companies that make up this ecosystem provide specialized services that overlap to various degrees. Combined, four sub-groups make up the ecosystem. 50 | 51 | 1. Embedding providers (e.g., Hugging Face or OpenAI) 52 | 1. Neural framework (e.g., deepset or Jina) 53 | 1. Feature stores (e.g., FeatureBase, FeatureForm or Tecton) 54 | 1. Vector databases (e.g., Weaviate or Vertex) 55 | 56 | While the number of data that companies are collecting in their data warehouses keeps growing, the need for better, more efficient searches keeps growing too. The more data we collect, the more complex searching through it becomes. Thanks to the advances in machine learning in the past decade and the commoditization of AI-first database technologies, you can start using it in your business tomorrow. 57 | 58 | 59 | import WhatNext from '/_includes/what-next.mdx' 60 | 61 | 62 | -------------------------------------------------------------------------------- /blog/2022-08-09-using-cross-encoders-as-reranker-in-multistage-vector-search/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Using Cross-Encoders as reranker in multistage vector search 3 | slug: cross-encoders-as-reranker 4 | authors: [laura] 5 | date: 2022-08-09 6 | tags: ['search'] 7 | image: ./img/hero.png 8 | description: "Learn about bi-encoder and cross-encoder machine learning models, and why combining them could improve the vector search experience." 9 | --- 10 | ![Using Cross-Encoders as reranker in multistage vector search](./img/hero.png) 11 | 12 | 13 | 14 | Semantic search overcomes the shortcomings of keyword-based search. If you search for "programming for data scientists", a keyword search will find "An intro to programming for data scientists", but it won't return "Python for Machine Learning use-cases". 15 | 16 | Vector databases can use machine learning to capture the context and semantics of data and search queries. Usually, machine learning models have a tradeoff between high accuracy and speed. The higher the accuracy, the more computationally expensive the model is. If we talk about search or Information Retrieval, we would like to see **accurate search results**, encompassing **wide-range queries**, whilst retaining **high speed**. 17 | 18 | In search, or semantic matching of sentences, we can see this tradeoff in *Bi-Encoder* models compared with *Cross-Encoder* models. *Bi-Encoder* models are **fast**, but **less accurate**, while *Cross-Encoders* are **more accurate**, but **slow**. Luckily, we can combine them in a search pipeline to benefit from both models! 19 | 20 | In this article, you will learn about these two machine learning models, and why combining them could improve the vector search experience. 21 | 22 | ## Bi-Encoder models 23 | 24 | Vector databases like [Weaviate](/developers/weaviate/) use Bi-Encoder machine learning models to compute the similarity between search queries and data in a vector space. Those machine learning models are trained to represent data (like text, images, videos, audio, etc) as **vectors**, which capture the context and semantics of the data. Similarly, search queries can also be represented by a vector embedding. A vector database can then perform a similarity search to retrieve the vectors closest to the query embedding. 25 | 26 | The models that compute **dense vector embeddings for data** (which later can be used for **search queries**) are so-called *Bi-Encoder models*. Data vectors and query vectors can be compared by computing the similarity (for example cosine similarity) between vectors. All data items in the database can be represented by a vector embedding using a Bi-Encoder model. 27 | 28 | > Note. Vector databases **compute** dense vector embeddings **during import**. Calculating vectors up-front coupled with [ANN algorithms](/developers/weaviate/concepts/vector-index) (for faster retrieval) makes working with Bi-Encoder highly efficient. 29 | 30 | ![Vector Database](./img/vector-database.png) 31 | 32 | *Figure 1 - Representation of a Vector Database* 33 | 34 | ![Bi-Encoder](./img/bi-encoder.png) 35 | 36 | *Figure 2 - Representation of a Bi-Encoder model* 37 | 38 | ## Cross-Encoder models 39 | 40 | Bi-Encoder models are not the only possible way of scoring pairs of data on similarity. A different strategy is using *Cross-Encoders*. Cross-Encoder models do not produce vector embeddings for data, but use a classification mechanism for data pairs instead. The input of the model always consists of a **data pair**, for example two sentences, and outputs a value between 0 and 1 indicating the similarity between these two sentences (Figure 3). So, individual sentences cannot be passed to a Cross-Encoder model, it always needs a **pair** of "items". In terms of search, you need to use the Cross-Encoder with **each** data item and the search query, to calculate the similarity between the query and data object. 41 | 42 | ![Cross-Encoder](./img/cross-encoder.png) 43 | 44 | *Figure 3 - Representation of a Cross-Encoder model* 45 | 46 | 47 | If a Cross-Encoder model is trained on a representative training set, it [achieves higher accuracy than Bi-Encoders](https://arxiv.org/abs/1908.10084). However, since you need to use the Cross-Encoder model during a search for every single data item in combination with the query, this method is very inefficient. For a real-life semantic search application – with thousands or millions of objects – this would be impractical, as it would take "forever" to perform the search. 48 | 49 | ## Combining Bi-Encoders and Cross-Encoders 50 | 51 | We can combine the two methods to benefit from the strong points of both models! I'd like to illustrate this idea with an example. Imagine you are a fisherman who is looking for salmon in a sea full of fish of numerous species. First, you sail to the waters rich in fish, and you use a big net with the optimal bait for salmon to catch a lot of fish. You hope to find some salmon, but your net also catches other kinds of fish that you are not interested in. Next, you sort the catch by keeping the salmon and throwing the other fish back into the sea. This sorting step is time-consuming and expensive, but very effective. 52 | 53 | ![Fisherman as multistage search](./img/fisherman.jpg) 54 | 55 | *Figure 4 - Fisherman as multistage search* 56 | 57 | Catching fish with the big net represents how Bi-Encoders work. Bi-Encoders are fast, but are not as accurate as the expensive fisherman aka the Cross-Encoder. Cross-Encoders are time-consuming, like the fisherman who would need to limit the number of fishing rounds they could do. So we can chain those two methods behind each other (see Figure 5). First, you use a Bi-Encoder to retrieve a *list of result candidates*, then you use a Cross-Encoder on this list of candidates to pick out (or rerank) the most relevant results. This way, you benefit from the efficient retrieval method using Bi-Encoders and the high accuracy of the Cross-Encoder, so you can use this on large scale datasets! 58 | 59 | ![Multistage search pipeline](./img/weaviate-pipeline-long.png) 60 | 61 | *Figure 5 - Multistage search pipeline using Weaviate* 62 | 63 | ## Pre-trained Cross-Encoder models 64 | 65 | As noted, Cross-Encoders can achieve high *in-domain* accuracy. To get the most accurate results, we train our models on datasets representative of their specialized use case. 66 | 67 | Pretrained Sentence Transformers Cross-Encoder models are [available on HuggingFace](https://huggingface.co/Cross-Encoder). There you can find many different models, for example a model trained on MS Marco, which you could use in a search application with general natural language data (the model is trained on Bing search queries). If you have a search task for a dataset that is *out-of-domain*, you should train or fine-tune a model, see [here](https://www.sbert.net/examples/training/cross-encoder/README.html) for examples. 68 | 69 | ## Conclusion 70 | 71 | We can combine the fast Bi-Encoders and accurate Cross-Encoders in a search pipeline to improve the search experience. With a vector database like [Weaviate](/), you can store and retrieve vectors and data efficiently using Bi-Encoder models to encode data and queries. A search pipeline can then continue with a Cross-Encoder model which reranks a list of retrieved search result candidates. 72 | 73 | This blog post was inspired by [Nils Reimer's work on Bi-Encoders and Cross-Encoders](https://www.sbert.net/examples/applications/cross-encoder/README.html). 74 | 75 | 76 | import WhatNext from '/_includes/what-next.mdx' 77 | 78 | 79 | -------------------------------------------------------------------------------- /blog/2022-09-01-weaviate-cloud-service/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Weaviate introduces an all-new business model for vector search 3 | slug: weaviate-cloud-services 4 | authors: [bob] 5 | date: 2022-09-01 6 | tags: ['release'] 7 | image: ./img/hero.png 8 | description: "Press Release: Pay-as-you-grow comes to Vector Search." 9 | --- 10 | ![Weaviate introduces an all-new business model for vector search](./img/hero.png) 11 | 12 | 13 | 14 | **Amsterdam NL - September 1, 2022** 15 | 16 | Weaviate, the leader in open-source vector database technology, announces an [entirely new pricing model](/pricing) for its Weaviate Cloud Services. 17 | 18 | Inspired by "pay-as-you-grow" pricing used in cloud storage, Weaviate has introduced a new business model that makes it easier and more affordable for enterprises of any scale to take advantage of its Weaviate vector database. Beginning today, users have the option of paying a usage-based rate for search on a per-dimension basis. Pricing begins at $0.05 per million dimensions. (No, that's not a typo; the rate is five cents per million.) 19 | 20 | > "At this point, as we've reached 1,500,000 downloads, a lot of people know and love our open-source software, but they want us to run it for them," says Weaviate's co-founder Bob van Luijt." So, we've created a ‘NoOps' option that allows them to pay only for what they use on our optimized Weaviate Cloud Services." 21 | 22 | In addition to remarkably convenient access to the latest vector database capabilities, Weaviate Cloud Services customers get dedicated customer support on a private Slack channel. However, consistent with Weaviate's commitment to creating truly open-source software, customers using the free service will always be able to access all of the Weaviate's vector database capabilities. 23 | 24 | Weaviate vector database is an example of a "third wave" database technology. Data is processed by a machine learning model first, and AI models help in processing, storing, and searching through the data. As a result, Weaviate excels at answering questions in natural language, but it is not limited to language; it is as adaptable to searching images or even genetic information. 25 | 26 | > "Depending on the machine-learning model used, a "document"—basically a data object—in a vector database typically has anywhere from 120 to 12,800 dimensions," van Luijt explains. "Since vector dimensions are the lowest common denominator, it makes sense for vector dimensions to be the cost basis, as opposed to API calls. We feel that this very transparent and predictable pricing model is consistent with our open-source philosophy." 27 | 28 | For more information about Weaviate an open-source vector database, visit: [https://weaviate.io](/) 29 | 30 | For more information about the company visit: [about-us](/company/about-us). 31 | 32 | Customers interested in the Weaviate Cloud Services can purchase service for $0.05 per million dimensions. Or, spend an entire dollar and get 20 million dimensions. -------------------------------------------------------------------------------- /blog/2022-09-21-weaviate-1-15-1-release/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Weaviate 1.15.1 patch release 3 | slug: weaviate-1-15-1-release 4 | authors: [etienne] 5 | date: 2022-09-21 6 | tags: ['release'] 7 | image: ./img/hero.png 8 | description: "Two weeks after the 1.15 release, we have a patch (v1.15.1) release for you, which brings 15 bug fixes and 2 UX improvements." 9 | --- 10 | ![Weaviate 1.15.1 patch release](./img/hero.png) 11 | 12 | 13 | 14 | We usually wouldn't write a whole blog post about a patch release. But when I chatted with Sebastian (the regular author of our "big" release blog posts series), about the contents of Weaviate `v1.15.1` patch, we quickly realized that this release is too important to end up as a [side note](https://github.com/weaviate/weaviate/releases/tag/v1.15.1) somewhere. 15 | 16 | So, instead, I have the pleasure of introducing you to the latest patch release. Two weeks after the [v1.15.0 release, which introduced backups and more](/blog/weaviate-1-15-release), we fixed over **15** bugs. 17 | 18 | In summary, this release addresses fixes and improvements around: 19 | 20 | * [Indexing](#indexing-bug-fixes) – for vector and object/inverted stores 21 | * [Sorting](#sorting) 22 | * [Aggregation](#aggregation) 23 | * [Stability](#stability) 24 | * [UX](#ux-fixes) 25 | 26 | ## Indexing bug fixes 27 | This started as a single bug investigation, but quickly this led to a discovery of five others. There was an issue when trying to update objects without vectors, but then you would add a vector later. That could lead to cryptic error messages like **"incompatible vector dimensions: 0 vs. 128"**. We were able to narrow this down to an issue in Weaviate's LSM store implementation, but we found five other bugs when we fixed the problem. Among them: 28 | 29 | * an issue where a PATCH update could break the inverted index or geo props 30 | * an issue where concurrent updates and deletes on the HNSW index would lead to errors 31 | * an issue where the user would receive a cryptic error message "entry point deleted - will be cleaned up in the next cycle" 32 | * out-of-index panics in the HNSW index 33 | 34 | In short, these fixes make indexing more stable, prevent degrading both the vector and inverted indexes over time, and ensure updates are reflected correctly. If you want to dig deeper, here is the [actual changelist](https://github.com/weaviate/weaviate/pull/2191). 35 | 36 | ## Sorting 37 | We added support for sorting in Weaviate `v1.13`, which has been a popular feature. Instead of just sorting by one property, Weaviate lets you sort by any number of properties. The idea is that if the values for the first property are identical, Weaviate uses the second property to determine the order. There was a bug that occurred when you would specify multiple sorting properties in combination with a limit. You could end up in a situation where some results were missing. The new release fixes the sorting implementation and ensures that results are in the correct order and the limit is applied at the very end. This way, no more results can be missing because of sorting. While we were at it, we added plenty of new tests to prevent future regressions. 38 | 39 | 👏Big thanks to our community user **Patrice Bourgougnon** – the creator of the [WPSolr plugin](https://www.wpsolr.com/) – for spotting this issue. 40 | 41 | ## Aggregation 42 | The previous release already contained substantial performance improvements for (filtered) aggregations. This patch release goes one step further. We fixed seven issues where an aggregation would return incorrect results or an error. We have fixed calculation mode and median, cross-reference aggregations, filtered-date aggregations, and issues with multi-shard aggregations. If you want to dig deeper, here is the [pull request that introduces all seven fixes](https://github.com/weaviate/weaviate/pull/2192). 43 | 44 | ## Stability 45 | A rare but critical bug occurred when running specific filtered vector searches while running updates. This bug turned out to be a hidden data race between the filtering logic and a compaction process due to unsafe usage of data. The fix was simple; the affected byte array needed to be copied instead of referenced. We added [a new stress test pipeline](https://github.com/weaviate/weaviate-chaos-engineering/pull/11) to prove that the bug is fixed and prevent future regressions. 46 | 47 | 👏Big thanks to our community user **Juraj Bezdek** – who builds [labelator.io](https://www.labelator.io/), which uses Weaviate under the hood. He initially spotted this issue and helped us narrow it down. It was a great example of collaboration with our community, leading to a much faster fix for this critical issue. 48 | 49 | ## UX fixes 50 | This patch release also introduces a couple of great UX fixes, which are sure you will appreciate. 51 | 52 | We improved what happens when you delete a class that had incoming references from other classes. Previously a long and cryptic error message was displayed, and unless you also deleted the referenced classes, the GraphQL API would not have been functional. [The fix](https://github.com/weaviate/weaviate/pull/2189) handles this situation much more gracefully. The cryptic error is gone and the GraphQL API stays functional. The invalid reference prop is simply skipped. 53 | 54 | When importing null properties into Weaviate before this release, it would make a difference whether you would leave out a property or explicitly set it to null. That is now fixed, and both situations behave the same. 55 | 56 | Last but not least, there were two situations where Weaviate did not properly propagate error messages to the user in specific cases; in the GCP backup backend and the new Hugging Face module. We have fixed them both. 57 | 58 | ## Conclusion: Now is the best time to upgrade 59 | Even just two weeks after the `v1.15.0` release, this new `v1.15.1` release is full of improvements. There is something for everyone, whether for increased stability, improved behavior, or better UX. The latest release contains over 15 bug fixes compared to the previous one. Now I believe that's a great reason to update right away. 60 | 61 | ## Enjoy 62 | We hope you enjoy the `v1.15.1` patch release, with bug fixes and UX improvements making this the best Weaviate release yet!🔥 63 | 64 | import ShareFeedback from '/_includes/share-feedback.md'; 65 | 66 | 67 | -------------------------------------------------------------------------------- /blog/2022-09-27-hugging-face-inference-api-in-weaviate/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Support for Hugging Face Inference API in Weaviate 3 | slug: hugging-face-inference-api-in-weaviate 4 | authors: [sebastian] 5 | date: 2022-09-27 6 | tags: ['integrations'] 7 | image: ./img/hero.png 8 | description: "Running ML Model Inference in production is hard. You can use Weaviate – a vector database – with Hugging Face Inference module to delegate the heavy lifting." 9 | --- 10 | ![Support for Hugging Face Inference API in Weaviate](./img/hero.png) 11 | 12 | 13 | 14 | Vector databases use Machine Learning models to offer incredible functionality to operate on your data. We are looking at anything from **summarizers** (that can summarize any text into a short) sentence), through **auto-labelers** (that can classify your data tokens), to **transformers** and **vectorizers** (that can convert any data – text, image, audio, etc. – into vectors and use that for context-based queries) and many more use cases. 15 | 16 | All of these use cases require `Machine Learning model inference` – a process of running data through an ML model and calculating an output (e.g. take a paragraph, and summarize into to a short sentence) – which is a compute-heavy process. 17 | 18 | ### The elephant in the room 19 | Running model inference in production is hard. 20 | * It requires expensive specialized hardware. 21 | * You need a lot more computing power during the initial data import. 22 | * Hardware tends to be underutilized once the bulk of the heavy work is done. 23 | * Sharing and prioritizing resources with other teams is hard. 24 | 25 | The good news is, there are companies – like Hugging Face, OpenAI, and Cohere – that offer running model inference as a service. 26 | 27 | > "Running model inference in production is hard, 28 | let them do it for you." 29 | 30 | ## Support for Hugging Face Inference API in Weaviate 31 | Starting from Weaviate `v1.15`, Weaviate includes a Hugging Face module, which provides support for Hugging Face Inference straight from the vector database. 32 | 33 | The Hugging Face module, allows you to use the [Hugging Face Inference service](https://huggingface.co/inference-api#pricing) with sentence similarity models, to vectorize and query your data, straight from Weaviate. No need to run the Inference API yourself. 34 | 35 | > You can choose between `text2vec-huggingface` (Hugging Face) and `text2vec-openai` (OpenAI) modules to delegate your model inference tasks.
36 | > Both modules are enabled by default in the [Weaviate Cloud Services](/pricing). 37 | 38 | ## Overview 39 | ![Overview](./img/hugging-face-module-overview.png) 40 | 41 | The Hugging Face module is quite incredible, for many reasons. 42 | 43 | ### Public models 44 | You get access to over 1600 pre-trained [sentence similarity models](https://huggingface.co/models?pipeline_tag=sentence-similarity). No need to train your own models, if there is already one that works well for your use case. 45 | 46 | In case you struggle with picking the right model, see our blog post on [choosing a sentence transformer from Hugging Face](/blog/how-to-choose-a-sentence-transformer-from-hugging-face). 47 | 48 | ### Private models 49 | If you have your own models, trained specially for your data, then you can upload them to Hugging Face (as private modules), and use them in Weaviate. 50 | 51 | 52 | *We are working on an article that will guide you on how to create your own model and upload it to Hugging Face.* 53 | 54 | ### Fully automated and optimized 55 | Weaviate manages the whole process for you. From the perspective of writing your code – once you have your schema configuration – you can almost forget that Hugging Face is involved at all. 56 | 57 | For example, when you import data into Weaviate, Weaviate will automatically extract the relevant text fields, send them Hugging Face to vectorize, and store the data with the new vectors in the database. 58 | 59 | ### Ready to use with a minimum of fuss 60 | Every new Weaviate instance created with the [Weaviate Cloud Services](/pricing) has the Hugging Face module enabled out of the box. You don't need to update any configs or anything, it is there ready and waiting. 61 | 62 | On the other hand, to use the Hugging Face module in Weaviate open source (`v1.15` or newer), you only need to set `text2vec-huggingface` as the default vectorizer. Like this: 63 | 64 | ```yaml 65 | DEFAULT_VECTORIZER_MODULE: text2vec-huggingface 66 | ENABLE_MODULES: text2vec-huggingface 67 | ``` 68 | 69 | ## How to get started 70 | 71 | :::note 72 | This article is not meant as a hands-on tutorial. 73 | For more detailed instructions please check the [documentation](/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-huggingface). 74 | ::: 75 | 76 | The overall process to use a Hugging Face module with Weaviate is fairly straightforward. 77 | 78 | ![Recipe for using the Hugging Face module](./img/how-to-get-started-recipe.png) 79 | If this was a cooking class and you were following a recipe. 80 | 81 | You would need the following ingredients: 82 | * Raw Data 83 | * Hugging Face API token – which you can request from [their website](https://huggingface.co/settings/tokens) 84 | * A working Weaviate instance with the `text2vec-huggingface` enabled 85 | 86 | Then you would follow these steps. 87 | 88 | ### Step 1 – initial preparation – create schema and select the hf models 89 | Once you have a Weaviate instance up and running. 90 | Define your schema (standard stuff – pick a class name, select properties, and data types). As a part of the schema definition, you also need to provide, which Hugging Face model you want to use for each schema class. 91 | 92 | 93 | This is done by adding a `moduleConfig` property with the `model` name, to the schema definition, like this: 94 | ```javascript 95 | { 96 | "class": "Notes", 97 | "moduleConfig": { 98 | "text2vec-huggingface": { 99 | "model": "sentence-transformers/all-MiniLM-L6-v2", # model name 100 | ... 101 | } 102 | }, 103 | "vectorizer": "text2vec-huggingface", # vectorizer for hugging face 104 | ... 105 | } 106 | ``` 107 | 108 | *If you are wondering, yes, you can use a different model for each class.* 109 | 110 | ### Step 2 – cook for some time – import data 111 | Start importing data into Weaviate. 112 | 113 | For this, you need your Hugging Face API token, which is used to authorize all calls with 🤗. 114 | 115 | Add your token, to a Weaviate client configuration. For example in Python, you do it like this: 116 | 117 | ```javascript 118 | client = weaviate.Client( 119 | url='http://localhost:8080', 120 | additional_headers={ 121 | 'X-HuggingFace-Api-Key': 'YOUR-HUGGINGFACE-API-KEY' 122 | } 123 | ) 124 | ``` 125 | Then import the data the same way as always. And Weaviate will handle all the communication with Hugging Face. 126 | 127 | ### Step 3 – serving portions – querying data 128 | Once, you imported some or all of the data, you can start running queries. 129 | (yes, you can start querying your database even during the import). 130 | 131 | Running queries also requires the same token. 132 | But you can reuse the same client, so you are good to go. 133 | 134 | Then, you just run the queries, as per usual: 135 | ```javascript 136 | nearText = { 137 | "concepts": ["How to use Hugging Face modules with Weaviate?"], 138 | "distance": 0.6, 139 | } 140 | 141 | result = ( 142 | client.query 143 | .get("Notes", [ 144 | "name", 145 | "comment", 146 | "_additional {certainty distance} "]) 147 | .with_near_text(nearText) 148 | .do() 149 | ) 150 | ``` 151 | 152 | ## Summary 153 | > Now you can use [Hugging Face](/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-huggingface) or [OpenAI](/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-openai) modules in Weaviate to delegate model inference out. 154 | 155 | Just pick the model, provide your API key and start working with your data. 156 | 157 | Weaviate optimizes the communication process with the Inference API for you, so that you can focus on the challenges and requirements of your applications. No need to run the Inference API yourself. 158 | 159 | ## What next 160 | Check out the [text2vec-huggingface](/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-huggingface) documentation to learn more about the new module. 161 | 162 | If you are new to Weaviate, check out the [getting started guide](/developers/weaviate/quickstart). 163 | 164 | Let us know if you found this article interesting or useful. We are always happy to receive constructive feedback. 😀 165 | 166 | We are always working on new articles and are looking for new topics. Let us know if there is anything that you would like us to write about. 🤗 167 | 168 | import ShareFeedback from '/_includes/share-feedback.md'; 169 | 170 | 171 | -------------------------------------------------------------------------------- /blog/2022-10-25-lock-striping-pattern/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: How we solved a race condition with the Lock Striping pattern 3 | slug: Lock-striping-pattern 4 | authors: [dirk] 5 | date: 2022-10-25 6 | tags: ['engineering'] 7 | image: ./img/hero.png 8 | description: "The Lock Striping pattern is a great way to solve race conditions without sacrificing performance. Lean about Weaviate's improvements." 9 | --- 10 | ![How we solved a race condition with the Lock Striping pattern](./img/hero.png) 11 | 12 | 13 | 14 | ## Lock striping in database design 15 | Database design comes with interesting challenges. Like, dealing with race conditions when importing data in parallel streams. But for every new challenge, there is a clever solution. One of those clever solutions is Lock striping. It refers to an arrangement where locking occurs on multiple buckets or 'stripes'. 16 | 17 | Are you curious about, the challenge that we faced, which solutions we considered, and what was our final solution? Read on 😀. 18 | 19 | ## Background 20 | Databases must be able to import data quickly and reliably while maintaining data integrity and reducing time overhead. Weaviate is no exception to this! Given that our users populate Weaviate with hundreds of millions of data objects (if not more), we appreciate that import performance is of the highest ... *import-ance* (sorry) 🥁. 21 | 22 | Weaviate offers and strongly recommends the [batch import feature](/developers/weaviate/tutorials/import#import-setup) for adding data objects in bulk. To further speed up the import process, you can use parallelization, which lets you run multiple batches concurrently. Each object in these batches is then checked for duplicates and assigned a unique internal DocID used by Weaviate to access objects. 23 | 24 | We uncovered that there could be a race condition in this process. Sometimes when multiple batches contained identical objects with the same UUID, they could be added more than once to Weaviate, each time with different DocIDs. This, in turn, could cause issues within Weaviate. 25 | 26 | Luckily, we've addressed this issue without sacrificing performance (yay!🥳). Here's our journey that got us to the current solution. 27 | 28 | ## Our initial solutions 29 | In the initial solution, we added a lock (sync.Mutex in Go), so that now only a single goroutine can hold the lock, check for duplicate UUIDs, and assign DocIDs. This lock makes sure that the race does not occur anymore, but as an unintended side-effect the import time increased by ~20% due to lock-congestion. 30 | 31 | Upon further consideration, our team concluded that while using a single lock is effective, it's also overkill. Almost all objects are unique and it is not a problem to process those concurrently. We found that what we really needed was just a lock for each unique UUID. Cleverly, this approach would ensure that only one object per UUID is handled at each point in time, so that Weaviate cannot add multiple instances of objects with the same UUID. Meanwhile, it would still allow full parallelization of import processes to maximize performance. 32 | 33 | ![Single-lock solution](./img/single-lock-solution.png) 34 | 35 | As it often happens, implementing a lock-per-key solution created a different issue. Due to the large dataset size mentioned earlier, there can be millions or even billions of objects with unique UUIDs in Weaviate, and creating a lock for each of them would require a lot of memory. We found an elegant solution that is in-between both of the solutions above - a **lock striping** pattern. 36 | 37 | ## Solving both challenges 38 | Based on the UUID we assign each object to one of the 128 locks. This process is deterministic so objects with an identical UUID will always use the same lock. This gives us the best of both worlds: we have a small, fixed number of locks, but it still guarantees that two objects with the same UUID are never processed concurrently. With 128 locks, we only have 1/128th of the congestion of a single lock while still only using 128 * 8B = 1KB of memory. With the **lock striping** pattern, the import time is the same as without a lock, and we fixed the race condition without any negative performance impact. 39 | 40 | ![lock striping solution](./img/lock-striping-solution.png) 41 | 42 | We are very pleased to introduce this solution, which should eliminate the above issues that can be caused by data duplication at import. Additionally, we are also very happy to have arrived at a solution that comes with no data import performance penalty, having seen the mammoth datasets that our users often deal with. 43 | 44 | ## Update Weaviate 45 | The **lock striping** pattern was introduced in Weaviate `v1.15.4`. So if you are a Weaviate user, we encourage you to update Weaviate to the latest release to take advantage of this improvement as well as many others. 46 | 47 | Thank you for reading, and see you next time! 48 | 49 | 50 | import WhatNext from '/_includes/what-next.mdx' 51 | 52 | 53 | -------------------------------------------------------------------------------- /blog/2022-11-01-weaviate-1-16-release/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Weaviate 1.16 release 3 | slug: weaviate-1-16-release 4 | authors: [connor, jp, erika, sebastian, zain] 5 | date: 2022-11-01 6 | tags: ['release'] 7 | image: ./img/hero.png 8 | description: "Weaviate 1.16 introduces New Filter Operators, Distributed Backups, Centroid Module, Node Status API, Azure-based OIDC, and more. Lear all about it." 9 | --- 10 | 11 | import Core116 from './_core-1-16-include.mdx' ; 12 | 13 | 14 | 15 | import WhatsNext from '/_includes/what-next.mdx' 16 | 17 | 18 | 19 | import Ending from '/_includes/blog-end-oss-comment.md' ; 20 | 21 | 22 | -------------------------------------------------------------------------------- /blog/2022-12-06-sphere-dataset-in-weaviate/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: The Sphere Dataset in Weaviate 3 | slug: sphere-dataset-in-weaviate 4 | authors: [zain] 5 | date: 2022-12-6 6 | tags: ['how-to'] 7 | image: ./img/hero.png 8 | description: "Learn how to import and query the Sphere dataset in Weaviate!" 9 | --- 10 | ![The Sphere Dataset in Weaviate](./img/hero.png) 11 | 12 | 13 | 14 | ## What is Sphere? 15 | [Sphere](https://github.com/facebookresearch/sphere) is an open-source dataset recently [released](https://ai.facebook.com/blog/introducing-sphere-meta-ais-web-scale-corpus-for-better-knowledge-intensive-nlp/) by Meta. It is a collection of 134 million documents (broken up into 906 million 100-word snippets). It is one of the largest knowledge bases that can help solve knowledge-intensive natural language tasks such as question-answering, fact-checking, and much more. 16 | 17 | Simply stated, Sphere aims to act as a "universal, uncurated and unstructured source of knowledge." This means that the next time you have a question like: "Was McDonald's, the food chain, founded by the same Ol' McDonald who had a farm?" Sphere will have the relevant knowledge to answer your question and point you toward a relevant article. The potential for this large of a dataset is awe-inspiring and the applications one can dream up are limitless - from combating fake news on social media platforms to helping locate your next dream vacation spot. 18 | 19 | Additionally, Sphere is ideal for hybrid vector search at scale since it is one of the few large scale datasets where vectors are provided in addition to their corresponding text fields. You can learn more about the model that is used to generate these vectors [here](https://huggingface.co/facebook/dpr-question_encoder-single-nq-base). For all of these reasons we wanted to make this resource as accessible to the community as possible. 20 | 21 | ## The Challenges of using Sphere 22 | 23 | The only limitation of this dataset is how difficult it is to access and use for the average developer. In this regard the enormity of the dataset ends up as a double-edged sword. It is challenging to use Sphere in its current open-source format for anyone other than large industry and academic labs - even for them the UX leaves a lot to be desired. 24 | 25 | *Don't believe me?* 26 | 27 | Try following the [readme](https://github.com/facebookresearch/sphere) to get Sphere set up and usable on your machine. The first limitation you'll run into is that the smallest open-sourced sparse Sphere indices file is a whopping 833 Gigabytes in its compressed format. Once you get past that hurdle, to start using the Sphere dataset for its intended purpose of hybrid search testing and benchmarking, it requires another herculean effort. 28 | 29 | ## The Sphere Dataset in Weaviate 30 | In an effort to make this powerful resource accessible to everyone, we are happy to announce that Sphere is now **available** not only in **Weaviate** but also as **JSON** or **Parquet files**. The dataset can be easily imported with Python and Spark! You can import **large vectorized chunks** of Sphere (or the whole thing if you want!) and start searching through it in just a few lines of code! 31 | 32 | The power of the sun in the palm of your hand! *ensues evil maniacal laughter* 33 | 34 | import sphereVideo from './img/joke2.mp4'; 35 | 36 | 40 | 41 | 42 | Get it? ... It's a Sphere 🥁ba dum tsss🥁 I'll show myself out… 43 | 44 | 45 | There are two ways to import the Sphere dataset into Weaviate. You can use the Python client (less than 75 lines of code) or the Weaviate Spark connector. 46 | 47 | ### Importing Sphere with Python 48 | The setup is quite straightforward, all you need is the [Weaviate Client](/developers/weaviate/client-libraries). We provide an example that uses the Python Client and the dpr-question_encoder-single-nq-base model (i.e., the module that is used to vectorize objects in Sphere). 49 | 50 | We have prepared files ranging from 100K data points all the way up to the entire Sphere dataset, which consists of 899 million lines. You can download them from here: 51 | * [100k lines](https://storage.googleapis.com/sphere-demo/sphere.100k.jsonl.tar.gz) 52 | * [1M lines](https://storage.googleapis.com/sphere-demo/sphere.1M.jsonl.tar.gz) 53 | * [10M lines](https://storage.googleapis.com/sphere-demo/sphere.10M.jsonl.tar.gz) 54 | * [100M lines](https://storage.googleapis.com/sphere-demo/sphere.100M.jsonl.tar.gz) 55 | * [899M lines](https://storage.googleapis.com/sphere-demo/sphere.899M.jsonl.tar.gz) 56 | 57 | Once you have the dataset file downloaded and unpacked, the next step is to import the dataset into Weaviate with Python: 58 | 59 | ```python 60 | import sys 61 | import os 62 | import time 63 | import json 64 | import weaviate 65 | 66 | # Variables 67 | WEAVIATE_URL = 'https://loadtest.weaviate.network/' 68 | BATCH_SIZE = 100 69 | SPHERE_DATASET = 'sphere.100k.jsonl' # update to match your filename 70 | 71 | client = weaviate.Client( 72 | url=WEAVIATE_URL, 73 | timeout_config=600 74 | ) 75 | 76 | client.batch.configure( 77 | batch_size=BATCH_SIZE, 78 | dynamic=True, 79 | num_workers=os.cpu_count() 80 | ) 81 | 82 | # Set DPR model used for the Page class 83 | client.schema.create_class({ 84 | "class": "Page", 85 | "vectorizer": "text2vec-huggingface", 86 | "moduleConfig": { 87 | "passageModel": "sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base", 88 | "queryModel": "sentence-transformers/facebook-dpr-question_encoder-single-nq-base", 89 | "options": { 90 | "waitForModel": True, 91 | "useGPU": True, 92 | "useCache": True 93 | } 94 | }, 95 | "properties": [] 96 | }) 97 | 98 | # Import the data, Weaviate will use the auto-schema function to 99 | # create the other properties and other default settings. 100 | start = time.time() 101 | c = 0 102 | with open(SPHERE_DATASET) as jsonl_file: 103 | with client.batch as batch: 104 | for jsonl in jsonl_file: 105 | json_parsed = json.loads(jsonl) 106 | batch.add_data_object({ 107 | 'url': json_parsed['url'], 108 | 'title': json_parsed['title'], 109 | 'raw': json_parsed['raw'], 110 | 'sha': json_parsed['sha'] 111 | }, 112 | 'Page', 113 | json_parsed['id'], 114 | vector=json_parsed['vector'] 115 | ) 116 | c += 1 117 | if (c % (BATCH_SIZE * 10)) == 0: 118 | print('Imported', c) 119 | 120 | end = time.time() 121 | print('Done in', end - start) 122 | ``` 123 | *Make sure to update the SPHERE_DATASET property to correctly match your `.jsonl` filename.* 124 | 125 | ### Importing Sphere with Spark 126 | If you want to start training large language models for knowledge-intensive tasks on Sphere, then you might want to leverage big data frameworks. This is where Apache Spark enters the picture! 127 | 128 | To process Sphere with Spark, you can use [PySpark](https://spark.apache.org/docs/latest/api/python/) and [Weaviate's Python Client](/developers/weaviate/client-libraries/python). The setup is slightly more difficult than simply importing the dataset with python; however, once you do have it setup, it is lightning fast! ⚡ 129 | 130 | You can see the step-by-step instructions detailed in this [tutorial](/developers/weaviate/tutorials/spark-connector). The tutorial demonstrates how to get Sphere into a Spark dataframe, import it into Weaviate, and conduct queries. Once you have Sphere imported into your Spark instance you can leverage Spark functionality to start training powerful models. In this particular example we are using the [Weaviate Spark connector](https://github.com/weaviate/weaviate-spark-connector) making it easy to load data from Spark to Weaviate. 131 | 132 | We have also prepared two Parquet files one with 1M data points and another with the entire Sphere dataset, which consists of 899 million lines. You can download them into a dataframe as follows: 133 | 134 | ``` 135 | df = spark.read.parquet("gs://sphere-demo/parquet/sphere.1M.parquet") 136 | df = spark.read.parquet("gs://sphere-demo/parquet/sphere.899M.parquet") 137 | ``` 138 | ## Searching through Sphere with Weaviate 139 | Now that the instructions are out of the way lets have some fun and show you the combined power of Sphere on Weaviate! We imported the entire Sphere dataset into Weaviate - yes, all ~899 million objects, see below for proof! 140 | 141 | ![import proof](./img/image4.png) 142 | 143 | Once the Sphere dataset is in Weaviate we can use it in conjunction with all of the functionality that comes with Weaviate. 144 | 145 | Since the primary usage of Sphere is around conducting large scale hybrid search below you can see an example of where we leverage vector search to find out what Sphere says is the best food to eat in Italy while simultaneously using conventional word matching to ensure the returned objects are from a credible source, the New York Times in this case. 146 | 147 | ![query](./img/image1.png) 148 | 149 | And that's all folks! Now with the Sphere dataset readily available and easy to import into Weaviate anyone can start to build with this powerful tool in conjunction with the loads of awesome features that we already offer in Weaviate. Happy coding! 150 | 151 | 152 | import StayConnected from '/_includes/stay-connected.mdx' 153 | 154 | 155 | -------------------------------------------------------------------------------- /blog/2022-12-20-weaviate-1-17-release/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Weaviate 1.17 release 3 | slug: weaviate-1-17-release 4 | authors: [erika, zain, laura, dan] 5 | date: 2022-12-20 6 | tags: ['release'] 7 | image: ./img/hero.png 8 | description: "Weaviate 1.17 introduces Replication, Hybrid Search, BM25, Faster Startup and Imports and more. Learn all about it." 9 | --- 10 | 11 | import Core117 from './_core-1-17-include.mdx' ; 12 | 13 | 14 | 15 | import WhatsNext from '/_includes/what-next.mdx' 16 | 17 | 18 | 19 | import Ending from '/_includes/blog-end-oss-comment.md' ; 20 | 21 | 22 | -------------------------------------------------------------------------------- /blog/2023-01-03-hybrid-search-explained/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Hybrid Search Explained 3 | slug: hybrid-search-explained 4 | authors: [erika] 5 | date: 2023-01-03 6 | tags: ['concepts', 'search'] 7 | image: ./img/hero.png 8 | description: "Learn about the new hybrid search feature that enables you to combine dense and sparse vectors to deliver the best of both search methods." 9 | --- 10 | ![Hybrid search](./img/hero.png) 11 | 12 | 13 | 14 | Hybrid search is a technique that **combines** multiple search algorithms to improve the accuracy and relevance of search results. It uses the best features of both keyword-based search algorithms with vector search techniques. By leveraging the strengths of different algorithms, it provides a more effective search experience for users. 15 | 16 | The [Hybrid search](/developers/weaviate/api/graphql/search-operators#hybrid) feature was introduced in Weaviate 1.17. It uses sparse and dense vectors to represent the meaning and context of search queries and documents. 17 | 18 | In this blog post, you will learn about the implementation of hybrid search in Weaviate and how to use it. 19 | 20 | ## Sparse and Dense Vectors 21 | Sparse and dense vectors are calculated with distinct algorithms. Sparse vectors have mostly zero values with only a few non-zero values, while dense vectors mostly contain non-zero values. Sparse embeddings are generated from algorithms like [BM25](https://en.wikipedia.org/wiki/Okapi_BM25) and [SPLADE](https://arxiv.org/abs/2107.05720). Dense embeddings are generated from machine learning models like 22 | [GloVe](https://text2vec.org/glove.html) and [Transformers](https://huggingface.co/docs/transformers/index). 23 | 24 | Note, the current implementation of hybrid search in Weaviate uses BM25/BM25F and vector search. 25 | 26 | If you’re interested to learn about how dense vector indexes are built and optimized in Weaviate, check out this [article](/blog/why-is-vector-search-so-fast). 27 | 28 | ### BM25 29 | BM25 builds on the keyword scoring method [TF-IDF](https://en.wikipedia.org/wiki/Tf%E2%80%93idf) (Term-Frequency Inverse-Document Frequency) by taking the [Binary Independence Model](https://en.wikipedia.org/wiki/Binary_Independence_Model) from the IDF calculation and adding a normalization penalty that weighs a document’s length relative to the average length of all the documents in the database. 30 | 31 | The image below presents the scoring calculation of BM25: 32 | ![BM25 calculation](./img/BM25-calculation.png)
Source: Wikipedia page on Okapi BM25
33 | 34 | The score of the document, query pair is determined by weighing the uniqueness of each keyword in the query relative to the collection of texts. BM25 contains additional static parameters, k1 and b that may help calibrate performance to particular datasets. 35 | 36 | ### BM25F 37 | BM25F was also implemented in Weaviate `1.17`. BM25F is a variant of BM25 that allows multiple text fields per object to be given different weights in the ranking calculation. These weights are important for when fields in a document are more important than others. For example, a title may be given more weight than the abstract, since the title is sometimes more informative and concise. This type of weighting makes BM25F more flexible and customizable than BM25. 38 | 39 | ### Dense Vectors 40 | Dense vectors represent information stored in a database; this includes text, images, and other types of data. These embeddings are generated from machine learning models that convert data to vectors. The vectors are densely packed with information and are mostly made up of non-zero values. The meaning of each value in the vector depends on the machine learning model that you used. 41 | 42 | Vector databases, like [Weaviate](/developers/weaviate/), store these embeddings and calculate the distance between the two vectors. [Distance metrics](/blog/distance-metrics-in-vector-search) show how similar or dissimilar two vector embeddings are. The search query is converted to a vector, similar to the data vectors, and the distance value determines how close the vectors are. 43 | 44 | ![Hybrid Search](./img/hybrid-search-explained.png) 45 | 46 | ## Hybrid Search Explained 47 | Hybrid search merges dense and sparse vectors together to deliver the best of both search methods. Generally speaking, dense vectors excel at understanding the context of the query, whereas sparse vectors excel at keyword matches. Consider the query, “How to catch an Alaskan Pollock”. The dense vector representation is able to disambiguate “catch” as meaning fishing rather than baseball or sickness. The sparse vector search will match the phrase “Alaskan Pollock” only. This example query shows where hybrid search combines the best of both sparse and dense vectors. 48 | 49 | 50 | ## Reciprocal Rank Fusion (RRF) 51 | 52 | While researching hybrid search, we needed a way to combine the results of BM25 and dense vector search into a single ranked list. We came across a paper from Benham and Culpepper exploring rank fusion techniques. This paper analyzed seven strategies for combining the ranked results of two lists into a single ranking. We decided to start with the RRF score. The RRF score is calculated by taking the sum of the reciprocal rankings that is given from each list. By putting the rank of the document in the denominator, it penalizes the documents that are ranked lower in the list. 53 | 54 |

RRF Calculation

55 |
Source: Benham and Culpepper 2017
56 | 57 | 58 | Let's look at an example of this. We have three documents labeled `A`, `B`, and `C` and have run a BM25 and Dense search. In this example, we have set the constant *k* to 0. 59 | 60 | | BM25 Ranking | Dense Ranking | Results 61 | | --- | --- | --- | 62 | | A | B | A: 1/1 + 1/3 = 1.3 | 63 | | B | C | B: 1/2 + 1/1 = 1.5 | 64 | | C | A | C: 1/3 + 1/2 = 0.83| 65 | 66 | The above table shows the ranked order of the BM25 and Dense search. To fuse the two lists together, we need to take the sum of the reciprocal ranks. Based on the results, the top document is `Document B` with a ranking of 1.5, then `Document A` at 1.3, and `Document C` at 0.83. 67 | 68 | Re-ranking is a necessary step when implementing hybrid search. The alpha parameter dictates the weighting of each algorithm and determines the re-ranking of the results. 69 | 70 | 71 | ## Weaviate UX 72 | To use hybrid search in Weaviate, you only need to confirm that you’re using Weaviate `v1.17` or a later version. You can run the hybrid queries in GraphQL or the other various client programming languages. 73 | 74 | There are five parameters needed to run the hybrid search query (some are optional): 75 | * `hybrid`: shows that you want to use hybrid search 76 | * `query`: search query 77 | * `alpha`(optional): weighting for each search algorithm (alpha = 0 (sparse), alpha = 1 (dense), alpha = 0.5 (equal weight for sparse and dense)) 78 | * `vector` (optional): optional to supply your own vector 79 | * `score`(optional): additional information on how much the sparse and dense method contributed to the result 80 | 81 | With just a few lines of code, you can start using hybrid search. You can run a test query in the [Weaviate console](https://link.weaviate.io/3JyBYZR) using GraphQL. The query is, “Fisherman that catches salmon” (similar to the example above). When we set the alpha to 0.5 it is equally weighing the dense and sparse vector results. 82 | 83 | ``` 84 | { 85 | Get { 86 | Article ( 87 | hybrid: { 88 | query: "Fisherman that catches salmon" 89 | alpha: 0.5 90 | }) 91 | { 92 | title 93 | summary 94 | _additional {score} 95 | } 96 | } 97 | } 98 | ``` 99 | 100 | Check out the [documentation](/developers/weaviate/api/graphql/search-operators#hybrid) for more information on hybrid search! 101 | 102 | 103 | import WhatNext from '/_includes/what-next.mdx' 104 | 105 | 106 | -------------------------------------------------------------------------------- /blog/2023-02-14-what-to-expect-from-weaviate-in-2023/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: What to expect from Weaviate in 2023 3 | slug: what-to-expect-from-weaviate-in-2023 4 | authors: [etienne] 5 | date: 2023-02-14 6 | tags: ['engineering'] 7 | image: ./img/hero.png 8 | description: "Learn about the six pillars outlining how Weaviate will evolve in 2023." 9 | --- 10 | ![What to expect from Weaviate in 2023](./img/hero.png) 11 | 12 | Without a doubt, 2022 has been the most exciting year for Weaviate so far. The company and the product have grown tremendously, and we are incredibly excited about 2023. Weaviate’s usage numbers are through the roof, and so are your feedback and requests for what you’re still missing from Weaviate. 13 | 14 | 15 | 16 | In this blog post, I will introduce you to the six pillars outlining how Weaviate will get even better in 2023. Weaviate development is highly dynamic – we don’t waterfall-plan for the entire year – but nevertheless, we want to give you the best possible overview of what to expect in the coming year. 17 | 18 | ## The Six Pillars for 2023 19 | 20 | ### Ingestion and Search Pipelines 21 | ![Ingestion and Search Pipelines](./img/search-pipeline.png) 22 | Weaviate’s strong and growing module ecosystem gives you plenty of flexibility. Whether you use Weaviate as a pure vector search engine or with the addition of vectorizer, reader, and generator modules, you can always configure it to your liking. In early 2023 we even saw the addition of the [`generative-openai`](/developers/weaviate/modules/reader-generator-modules/generative-openai) module (with other generative modules to come). 23 | 24 | We want to give you even more flexibility in combining these steps this year. You can control arbitrary querying steps through the [proposed Pipe API](https://github.com/weaviate/weaviate/issues/2560), such as reading, re-ranking, summarizing, generating, and others. Similarly, we want to give you more flexibility during ingestion time: how about [extracting PDFs](https://github.com/weaviate/weaviate/issues/2509) or applying [stemming](https://github.com/weaviate/weaviate/issues/2439) to your BM25 and hybrid search? 25 | 26 |

27 | 28 | ### Beyond Billion Scale: Large-Scale Performance 29 | ![Billion Scale](./img/billion-scale.png) 30 | 31 | In 2022, we published the [Sphere Demo Dataset for Weaviate](/blog/sphere-dataset-in-weaviate). This marked the first time (to our knowledge) that more than a billion objects and vectors were imported into Weaviate. Dealing with ever-growing datasets is not only about being able to handle their size. Our users run complex queries in production and often have strict latency requirements. This pillar is all about performance. The first big step will be the move towards a [Native Roaring Bitmap Index](https://github.com/weaviate/weaviate/issues/2511). In the most extreme case, this new index time can speed up filtered vector search [by a factor of 1000](https://twitter.com/etiennedi/status/1621180981519458305). But it doesn’t stop there; we are already thinking about the next steps. Whether you want faster aggregations or new types of specialized indexes, we will ensure you can hit all your p99 latency targets with Weaviate. 32 | 33 |

34 | 35 | ### Cloud Operations & Scaling 36 | ![cloud operations scaling](./img/cloud-operations-scaling.png) 37 | 38 | When we introduced [Replication](/developers/weaviate/concepts/replication-architecture) to Weaviate in late 2022, we celebrated a significant milestone. It’s never been easier to achieve a highly available setup, and you can even dynamically scale your setup to increase throughput. 2023 is all about improving your cloud and operations experience. We will give you more control over [how to structure your workloads](https://github.com/weaviate/weaviate/issues/2586) in a distributed setup and more [flexibility to adapt to your ever-changing needs](https://github.com/weaviate/weaviate/issues/2228). And, of course, we’re constantly working on making your distributed cluster [even more resilient](https://github.com/weaviate/weaviate/issues/2405). 39 | 40 | Speaking of Cloud, arguably the easiest way to spin up a new use case with Weaviate is through the [Weaviate Cloud Services](/pricing). 41 | 42 |

43 | 44 | ### New Vector Indexes 45 | ![vector indexes](./img/vector-indexes.png) 46 | 47 | Last year we gave you a sneak peek into our [Vector Indexing Research](/blog/ann-algorithms-vamana-vs-hnsw), and this year you will be able to try out new vector indexes for yourself. Since the beginning, Weaviate has supported vector indexing with [HNSW](/developers/weaviate/concepts/vector-index), which leads to [best-in-class query times](/developers/weaviate/benchmarks/ann). But not every use case requires single-digit millisecond latencies. Instead, some prefer cost-effectiveness. Due to its relatively high memory footprint, HNSW is only cost-efficient in high-throughput scenarios. However, HNSW is inherently optimized for in-memory access. Simply storing the index or vectors on disk or memory-mapping the index kills performance. 48 | 49 | This is why we will offer you not just one but two memory-saving options to index your vectors without sacrificing latency and throughput. In early 2023, you will be able to use Product Quantization, a vector compression algorithm, in Weaviate for the first time. We are already working on a fully disk-based solution which we will release in late 2023. 50 | 51 |

52 | 53 | ### Improving our Client and Module Ecosystem 54 | ![client modules](./img/client-modules.png) 55 | 56 | So far, we have only discussed features related to Weaviate Core, the server in your setup. But the Weaviate experience is more than that. Modules allow you to integrate seamlessly with various embedding providers, and our language clients make Weaviate accessible right from your application. This year, we will further improve both. This includes [improved APIs](https://github.com/weaviate/weaviate-python-client/issues/205) on the client side, new modules, for example, for [generative search](/developers/weaviate/modules/reader-generator-modules/generative-openai), and improvements to our existing modules. 57 | 58 |

59 | 60 | ### Community 61 | ![community](./img/community.png) 62 | 63 | The most important pillar is all of you – our community. This includes both free, open-source users that self-host their Weaviate setup, as well as paid enterprise users and anyone using our Weaviate-as-a-Service offerings. We value your feedback and love that you are part of shaping our future. 64 | 65 | Last year we introduced our [dynamic roadmap page](/developers/weaviate/roadmap) that allows you to create and upvote your favorite feature requests. This way, you can make sure that your voice is heard, and we can see what all of you need the most. 66 | 67 |

68 | 69 | ## Conclusion: Proud of how far we’ve come, excited about the future 70 | In the beginning, I mentioned that not just the product but also the company grew significantly last year. I am incredibly proud of what we have achieved – both overall and in the past year. This wouldn’t have been possible without an absolutely fantastic team. Everyone working on Weaviate – whether a full-time employee or open-source contributor – is doing a fantastic job. I am proud of you all and highly excited about the future. Thank you all, and let’s make 2023 the most exciting year for Weaviate users so far! 71 | 72 |

73 | 74 | import WhatNext from '/_includes/what-next.mdx' 75 | 76 | 77 | -------------------------------------------------------------------------------- /blog/2023-03-07-weaviate-1-18-release/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Weaviate 1.18 release 3 | slug: weaviate-1-18-release 4 | authors: [jp, erika, zain, dan] 5 | date: 2023-03-07 6 | tags: ['release'] 7 | image: ./img/hero.png 8 | description: "Weaviate 1.18 introduces Faster Filtering through Bitmap Indexing, HNSW-PQ, Cursor API, and more! Learn all about it." 9 | --- 10 | 11 | import Core118 from './_core-1-18-include.mdx' ; 12 | 13 | 14 | 15 | import WhatsNext from '/_includes/what-next.mdx' 16 | 17 | 18 | 19 | import Ending from '/_includes/blog-end-oss-comment.md' ; 20 | 21 | 22 | -------------------------------------------------------------------------------- /blog/2023-03-28-monitoring-weaviate-in-production/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Monitoring Weaviate in Production 3 | slug: monitoring-weaviate-in-production 4 | authors: john 5 | date: 2023-03-28 6 | image: ./img/hero.png 7 | tags: ['concepts', 'engineering'] 8 | description: "Learn about how to monitor Weaviate in production and observe key metrics." 9 | 10 | --- 11 | ![Monitoring Weaviate in Production](./img/hero.png) 12 | 13 | 14 | 15 | Weaviate is designed to be easy to monitor and observe by following a cloud native approach. To do this Weaviate supports the following features 16 | 17 | 1. Publishing of Prometheus metrics to the standard `/metrics` endpoint 18 | 19 | 2. Use of built-in Kubernetes [liveness and readiness](/developers/weaviate/api/rest/well-known) checks 20 | 21 | 3. Configuration of settings via environment variables 22 | 23 | 4. Simplified deployment via [helm charts](https://github.com/weaviate/weaviate-helm) 24 | 25 | There is existing documentation on [the exported metrics](/developers/weaviate/configuration/monitoring) which also has [an example](https://github.com/weaviate/weaviate-examples/tree/main/monitoring-prometheus-grafana) for how to use a Prometheus instance for metrics. 26 | 27 | One common question though is: How can I integrate Weaviate with my existing observability stack? 28 | 29 | This article describes two approaches using either Grafana agent or Datadog agent to scrape these metrics. It also provides a list of important metrics to monitor. 30 | 31 | ## Prerequisites 32 | 33 | It is assumed that you have already deployed Weaviate. By default Prometheus monitoring is disabled, so you can enable it with this environment setting: 34 | 35 | ```sh 36 | PROMETHEUS_MONITORING_ENABLED=true 37 | ``` 38 | 39 | Weaviate will then publish Prometheus metrics on port `2112`. 40 | 41 | ::::note 42 | If you are using Weaviate `1.17` or lower, you may want to upgrade to `1.18` before enabling Prometheus metrics. The reason being Weaviate previously published many histograms which has since been [replaced](https://github.com/weaviate/weaviate/pull/2605) by summaries for performance reasons. Additionally, be careful enabling Prometheus metrics if you have many thousands of classes as you may end up with high cardinality labels due to some metrics being produced per class. 43 | :::: 44 | 45 | ## Grafana Agent 46 | ![monitoring weaviate in production](./img/Weaviate-monitoring-weaviate-in-prod-light.png#gh-light-mode-only) 47 | ![monitoring weaviate in production](./img/Weaviate-monitoring-weaviate-in-prod-dark.png#gh-dark-mode-only) 48 | 49 | For the first approach we will use the open-source [Grafana agent](https://grafana.com/docs/agent/latest/). In this case, we will show writing to Grafana Cloud for hosted metrics. This is configurable via the remote write section if you alternatively want to write to a self-hosted Mimir or Prometheus instance. 50 | 51 | 52 | ### Steps to Install 53 | 54 | 1\. Install Grafana agent in your target environment following the [set-up guide](https://grafana.com/docs/agent/latest/). 55 | 56 | 2\. Configure the Grafana `agent.yaml` to include a scrape job called `weaviate`. This will autodiscover Weaviate pods 57 | in Kubernetes. The `app=weaviate` label is automatically added by the Weaviate helm chart which makes autodiscovery easy. 58 | 59 | ```yaml 60 | metrics: 61 | configs: 62 | - name: weaviate 63 | # reference https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config 64 | scrape_configs: 65 | - job_name: weaviate 66 | scrape_interval: 30s 67 | scheme: http 68 | metrics_path: /metrics 69 | kubernetes_sd_configs: 70 | - role: pod 71 | selectors: 72 | - role: "pod" 73 | label: "app=weaviate" 74 | remote_write: 75 | - url: 76 | basic_auth: 77 | username: 78 | password: 79 | ``` 80 | 81 | 3\. Validate that you are receiving data by going to explore and running the following PromQL query in Grafana. 82 | 83 | ``` 84 | go_memstats_heap_inuse_bytes{job="weaviate"} 85 | ``` 86 | ### Dashboards 87 | 88 | One benefit of this approach is that you can now reuse the existing Weaviate Grafana dashboards. 89 | 90 | Steps to import these dashboards: 91 | 92 | 1\. Download and import the [preexisting dashboards](https://github.com/weaviate/weaviate/tree/master/tools/dev/grafana/dashboards). 93 | 94 | 2\. If you're using Grafana Cloud hosted Prometheus you will need to patch the dashboards to change the datasource uid to be `grafanacloud-prom` as below. 95 | 96 | ```sh 97 | sed 's/"uid": "Prometheus"/"uid": "grafanacloud-prom"/g' querying.json > querying-patched.json 98 | ``` 99 | 100 | The dashboards should now be visible! 101 | 102 | ![query latency](./img/query-latency.png) 103 | 104 | ## Datadog 105 | 106 | [Datadog](https://www.datadoghq.com/) is another popular solution for observability, and the Datadog agent has support for scraping Prometheus metrics. 107 | 108 | ### Steps to Install 109 | 110 | 1\. Install the datadog agent. For this example, installation was done using their [Helm](https://docs.datadoghq.com/containers/kubernetes/installation/?tab=helm) charts. 111 | 112 | 2\. Provide a `datadog-values.yml` config including the below. You can also capture Weaviate logs using the method. 113 | 114 | ```yaml 115 | datadog: 116 | # Note DD_KUBELET_TLS_VERIFY only needs to be set if running a local docker kubernetes cluster 117 | # env: 118 | # - name: DD_KUBELET_TLS_VERIFY 119 | # value: "false" 120 | clusterName: weaviate-deployment 121 | prometheusScrape: 122 | enabled: true 123 | serviceEndpoints: true 124 | additionalConfigs: 125 | - configurations: 126 | - max_returned_metrics: 20000 127 | min_collection_interval: 30 128 | ``` 129 | 130 | 3\. Customize the Weaviate [helm chart](https://github.com/weaviate/weaviate-helm/blob/80346f0f1e1f22ad84a899b5f9e12f44be3ee809/weaviate/values.yaml#L730) to have annotations `prometheus.io/scrape` and `prometheus.io/port` 131 | 132 | ```yaml 133 | # Pass any annotations to Weaviate pods 134 | annotations: 135 | prometheus.io/scrape: "true" 136 | prometheus.io/port: "2112" 137 | ``` 138 | 139 | 4\. Validate metrics are available. `go_memstats_heap_inuse_bytes` should always be present even with an empty schema. 140 | 141 | ![datadog summary](./img/datadog-summary.png) 142 | 143 | 144 | ## Key Metrics 145 | 146 | Below are some key Weaviate metrics to monitor. Standard CPU, Disk, Network metrics are also useful as are [Kubernetes 147 | events](https://grafana.com/blog/2023/01/23/how-to-use-kubernetes-events-for-effective-alerting-and-monitoring/). 148 | Note that some Weaviate metrics will not appear until an operation has occurred (for instance batch operations). 149 | 150 | ### Heap Usage 151 | 152 | For heap usage, the expectation is the memory will have a standard jagged pattern underload but that memory will drop periodically 153 | due to the Go garbage collection. If memory is not dropping and is staying very close to the [GOMEMLIMIT](/blog/gomemlimit-a-game-changer-for-high-memory-applications), you may need to increase resources. 154 | 155 | ``` 156 | go_memstats_heap_inuse_bytes 157 | ``` 158 | 159 | ### Batch Latency 160 | 161 | Batch latency is important as batch operations are the most efficient way to write data to 162 | Weaviate. Monitoring this can give an indication if there is a problem with indexing data. This metric has a label `operation` which 163 | allows you to see how long objects, vectors, and inverted index sub operations take. If you are using a [vectorizer module](/developers/weaviate/modules/retriever-vectorizer-modules) you will see additional latency due to the overhead of sending data to the module. 164 | 165 | ``` 166 | rate(batch_durations_ms_sum[30s])/rate(batch_durations_ms_count[30s]) 167 | ``` 168 | 169 | For batch deletes the corresponding `batch_delete_durations_ms` metric will also be useful. 170 | 171 | ### Object Latency 172 | 173 | Generally, batch indexing is recommended but there are situations where you would do single `PUT` or `DELETE` operations 174 | such as handling live changes from a user in an application. In this case you will want to monitor the object latency 175 | instead. 176 | 177 | ``` 178 | rate(objects_durations_ms_sum{operation="put"}[30s])/rate(objects_durations_ms_latency{operation="put"}[30s]) 179 | ``` 180 | 181 | ### Query Latency and Rate 182 | 183 | The latency and number of queries per second are also important, particularly for monitoring usage patterns. 184 | 185 | ``` 186 | rate(queries_durations_ms_sum[30s])/rate(queries_durations_ms_count[30s]) 187 | rate(queries_durations_ms_count)[30s] 188 | ``` 189 | 190 | ## Other Integrations 191 | 192 | Many other solutions that have integrations for Prometheus that can also be used: 193 | 194 | * [Elastic](https://docs.elastic.co/integrations/prometheus) 195 | * [Splunk](https://www.splunk.com/en_us/blog/devops/metrics-from-prometheus-exporters-are-now-available-with-the-sfx-smart-agent.html) 196 | * [New Relic](https://docs.newrelic.com/docs/infrastructure/prometheus-integrations/get-started/send-prometheus-metric-data-new-relic/) 197 | 198 | 199 | import StayConnected from '/_includes/stay-connected.mdx' 200 | 201 | 202 | -------------------------------------------------------------------------------- /blog/2023-04-04-weaviate-retrieval-plugin/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: The ChatGPT Retrieval Plugin - Weaviate as a Long-term Memory Store for Generative AI 3 | slug: weaviate-retrieval-plugin 4 | authors: zain 5 | date: 2023-04-04 6 | image: ./img/hero.png 7 | tags: ['integrations'] 8 | description: "Learn how you can connect Weaviate to ChatGPT to generate customized responses." 9 | 10 | --- 11 | ![Weaviate Retrieval Plugin](./img/hero.png) 12 | 13 | 14 | 15 | ## Overview of ChatGPT plugins 16 | 17 | Last week OpenAI announced plugins for ChatGPT and all of a sudden everyone’s favorite chatbot can now interact with the world of other apps we use in our everyday lives! The plugins announcement is a game-changer, arguably even more important than the 4.0 upgrade to the underlying GPT model itself. 18 | 19 | So what are ChatGPT plugins? Some are explaining plugins as the “eyes and ears” of LLMs to help products like ChatGPT interact more naturally with users by knowing of and integrating with other apps such as Zapier, Uber Eats, Duolingo, and many more. I would argue that plugins are much more than just the “eyes and ears” of ChatGPT. Hear me out. 20 | 21 | Until now ChatGPT was a tool that we could converse with and it would “imagine” creative stories, help us learn, summarize for us, write for us, and explain to us. The interface ChatGPT could use to interact with us was limited to a tab on our browser. But now, imbued with the power of plugins, ChatGPT can actually do much more than just “talk”, it can actually “do” things in the real world! 22 | 23 | For example, if you have the Instacart plugin enabled, not only can you ask ChatGPT to generate recipes, but also order groceries to be delivered to your door to help put together your next meal! Or if you don't feel like cooking, it’ll book you a seat at a restaurant that serves similar food using OpenTable! People were astounded by ChatGPT’s ability to put together great vacations, but now through plugin integrations with Expedia, it can actually buy your tickets for you, book your hotels and even book you a ride to and from the airport! Though we can still only interact with ChatGPT through a tab on our browser, it can now interact with us through any one of the thousands of integrations, from Google Sheets to Salesforce, that the [Zapier plugin](https://zapier.com/blog/announcing-zapier-chatgpt-plugin/) connects to. 24 | 25 | In short, plugins give ChatGPT the ability to leverage third-party resources to act upon the conversations that you have with it! Not only can ChatGPT call upon the wealth of Apps currently available to actually perform and impact things in the real world but also use the strengths of this App network to make up for some of its own shortcomings. 26 | 27 | What we’re most excited about is **the Retrieval Plugin**, which allows you to use ChatGPT with a [Vector Database](https://weaviate.io/blog/what-is-a-vector-database) like Weaviate to overcome one of the biggest challenges that ChatGPT has struggled with: its lack of long-term memory, as well as its inability to provide responses based on internal documents and data. 🧠🔌 28 | 29 | This has the potential to be the most widely used plugin as it will allow users to create customized versions of ChatGPT catered to their own data. Don’t just take it from me, take it [from Greg](https://twitter.com/gdb/status/1640006228930867200), the Co-founder of OpenAI and the fact that the plugin was trending on Github! 30 | 31 | ![gdbtweet](./img/gdbtweet.png) 32 | 33 | ::::note 34 | Plugins for ChatGPT, including the Weaviate Retrieval Plugin, are currently in Alpha and thus not available to the public. 35 | :::: 36 | 37 | ## The ChatGPT Retrieval Plugin 38 | 39 | ![plugin](./img/plugin-light.png#gh-light-mode-only) 40 | ![plugin](./img/plugin-dark.png#gh-dark-mode-only) 41 | 42 | The [ChatGPT retrieval plugin](https://github.com/openai/chatgpt-retrieval-plugin) allows you to connect ChatGPT to an instance of a [vector database](/developers/weaviate). Anything stored in a connected vector database can then be used to answer questions to prompts, thus allowing ChatGPT to “know” your custom data and answer questions based on the details stored in the vector database! Not only that but the vector database can also be used as a *long-term storage* of memory for ChatGPT to persist and store portions of your conversations from the short-lived memory of a tab to the long-term storage capabilities of a vector database. A vector database once connected via the retrieval plugin acts as the cortex of ChatGPT! 🧠 43 | 44 | The retrieval plugin can be used to get ChatGPT to accomplish tasks grounded in the context of knowledge stored in a connected vector database such as Weaviate. This process consists of two steps: first, we prompt ChatGPT with a question that it uses to craft a query that can be sent to the vector database as demonstrated below. 45 | 46 | ![diagram](./img/diagram-light.png#gh-light-mode-only) 47 | ![diagram](./img/diagram-dark.png#gh-dark-mode-only) 48 | 49 | This is what would happen if you used ChatGPT without the Weaviate Retrieval Plugin. 50 | 51 | ![pic1](./img/pic1.png) 52 | 53 | As expected, the model does not have any specific information with which to answer the question! Now let's say we enable the Weaviate Retrieval Plugin as shown below: 54 | 55 | ![pic2](./img/pic2.png) 56 | 57 | The example below demonstrates the plugin in action. Notice how the prompt `What is my cat’s name?` is used to create the query `cat’s name`, which is then sent to the vector database to conduct a search over the stored documents. 58 | 59 | ![pic3](./img/pic3.png) 60 | 61 | The connected vector database then responds with relevant information and context by searching and filtering a subset of your data. This is shown below: 62 | 63 | ![pic4](./img/pic4.png) 64 | 65 | The second step involves this information being passed to ChatGPT so that it can formulate its answer to accomplish the task specified in the prompt. This step is shown below: 66 | 67 | ![pic5](./img/pic5.png) 68 | 69 | In short, you can update the knowledge that ChatGPT knows about without having to go through the costly and time-consuming process of retraining the weights. The main functionality this plugin enables is: 70 | 71 | - It allows you to attach a vector database, to ChatGPT, that has proprietary data that can be used by ChatGPT when answering very specific questions 72 | - It allows you to persist personal documents and details to give ChatGPT a personal touch so that answers are not just general but can be catered to you (based on details specified in the vector database) 73 | - You can persist conversations with ChatGPT in the attached vector database to carry on conversations between turning on and shutting off ChatGPT tabs 74 | 75 | The advantage is that we can now schedule small and regular updates to content stored in connected vector databases to give the model awareness of new information and reducing the need to retrain the LLM model and update frequently - this ensures the LLM provides robust and up-to-date custom answers even in-between time-consuming retrained weight updates. 76 | 77 | We’ll now provide some more examples of how you can leverage the ChatGPT retrieval plugin with Weaviate - an open-source vector database. So without further ado, let's get into it! 78 | 79 | ## The Weaviate Retrieval Plugin in Action 80 | 81 | ### Using ChatGPT on Proprietary Company Documents 82 | 83 | In order to benefit from the capabilities of LLMs like ChatGPT's GPT4.0 in real-life use cases, it would be ideal if we could apply its generative power to new or custom data. For example, this would enable a private customized version of ChatGPT that's been trained on your company's internal documents and could act as a human resources chatbot. Wondering what the onboarding process for new employees looks like or how you can sign up for health benefits? You can simply ask your customized ChatGPT! The applications of a customized ChatGPT are limitless and quite exciting! 84 | 85 | Below I show you what this HR chatbot could look like! 86 | 87 | ![pic6](./img/pic6.png) 88 | ![pic7](./img/pic7.png) 89 | 90 | TaDaaaa!🎉 91 | 92 | This is just one example where I packaged a company policy and benefits documents and stored them in Weaviate and then used ChatGPT to conduct generative search over these documents. The possibilities are endless! 93 | 94 | ### Personalizing ChatGPT 95 | 96 | Another great application that the Weaviate Retrieval Plugin enables is customization of ChatGPT around your personal details. 97 | 98 | Below I add details about my friends into Weaviate and get it to plan the menu for my upcoming party! 99 | 100 | ![pic8](./img/pic8.png) 101 | ![pic9](./img/pic9.png) 102 | 103 | Another way you could personalize ChatGPT to yourself is by giving it an understanding of who you are. For example, below I tell ChatGPT the languages I understand and customize its responses around its knowledge of me. 104 | 105 | ![pic10](./img/pic10.png) 106 | ![pic11](./img/pic11.png) 107 | 108 | ### Helping ChatGPT Remember 109 | 110 | For the last example I'll share what I think is one of the most exciting and scary applications of the Weaviate Retrieval Plugin: **to use it to persist the memory of previous conversations you have with ChatGPT into Weaviate**, so that it can refer to them in a later conversation. Currently it is not possible to tell ChatGPT to persist something in the attached vector database through the chat UI, so I have replicated this by manually adding the chat content to Weaviate in between conversations. 111 | 112 | First I launched a chat session and gave ChatGPT these details about my pets: 113 | 114 | ![pic12](./img/pic12.png) 115 | 116 | Then I completely close ChatGPT, insert the contents of the chat into Weaviate and then initialize a completely new chat and ask: 117 | 118 | ![pic13](./img/pic13.png) 119 | 120 | Voilà! ChatGPT now remembers its previous conversations with you - persisting them in the attached instance of Weaviate! 121 | 122 | 123 | import StayConnected from '/_includes/stay-connected.mdx' 124 | 125 | 126 | -------------------------------------------------------------------------------- /blog/2023-04-18-autogpt-and-weaviate/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Giving Auto-GPT Long-Term Memory with Weaviate 3 | slug: autogpt-and-weaviate 4 | authors: [erika, jp] 5 | date: 2023-04-18 6 | image: ./img/hero.png 7 | tags: ['integrations'] 8 | description: "Learn about Auto-GPT and how to give it long-term memory with Weaviate!" 9 | 10 | --- 11 | 12 | ![autogpt and weaviate](./img/hero.png) 13 | 14 | 15 | 16 | :::info Auto-GPT being re-factored 17 | Edit (5/Jun/2023): Auto-GPT has [temporarily removed support for external vector stores as they refactor their code](https://github.com/Significant-Gravitas/Auto-GPT/blob/60ac0c4da15930d5e40af87fba6248ec37a951ee/BULLETIN.md?plain=1#L27). 18 | 19 | We are working on re-introducing the integration. For now, please use this version (https://github.com/Significant-Gravitas/Auto-GPT/tree/v0.3.1) to use Auto-GPT with Weaviate. 20 | ::: 21 | 22 | As generative language models such as GPT-4 continue to push the boundaries of what AI can do, the excitement surrounding its potential is spreading quickly. Many applications and projects are built on top of GPT-4 to extend its capabilities and features. Additionally, many tools were created in order to interact with large language models, like [LangChain](https://python.langchain.com/en/latest/) as an example. [Auto-GPT](https://github.com/Torantulino/Auto-GPT) is one of the fastest rising open-source python projects harnessing the power of GPT-4! 23 | 24 | ## What is Auto-GPT? 25 | Auto-GPT has gotten a lot of attention lately, with the amount of stars jumping from 20k to 80k in a matter of days. Auto-GPT chains together “thoughts” and completes various tasks or assignments **autonomously**. It takes GPT-4 one step further by enabling the model to run iteratively and complete various tasks in a siloed fashion. It can [write code and execute python scripts](https://twitter.com/SigGravitas/status/1642181498278408193?ref_src=twsrc%5Etfw%7Ctwcamp%5Etweetembed%7Ctwterm%5E1642181498278408193%7Ctwgr%5Eaa2e51a2fc46f95cf982d6baa333a4ea14d1b264%7Ctwcon%5Es1_&ref_url=https%3A%2F%2Fcdn.embedly.com%2Fwidgets%2Fmedia.html%3Ftype%3Dtext2Fhtmlkey%3Da19fcc184b9711e1b4764040d3dc5c07schema%3Dtwitterurl%3Dhttps3A%2F%2Ftwitter.com%2FSigGravitas%2Fstatus%2F1642181498278408193image%3Dhttps3A%2F%2Fi.embed.ly%2F1%2Fimage3Furl3Dhttps253A252F252Fabs.twimg.com252Ferrors252Flogo46x38.png26key3Da19fcc184b9711e1b4764040d3dc5c07), [conduct market research](https://twitter.com/BoucherNicolas/status/1646250166834307072), and even [order pizza](https://twitter.com/Saboo_Shubham_/status/1646739277328314368). 26 | 27 | ChatGPT requires humans to prompt the large language model (LLM) by developing and refining the text prompt. Meaning you need to build out your request step-by-step in order for the LLM to “understand”. Auto-GPT on the other hand, is able to independently define objectives needed to complete the assigned task **without (or with reduced) human feedback and intervention**. This is because of its ability to chain together thoughts. 28 | 29 | Chain of thought is a method that is used to help language models improve their reasoning. It breaks down tasks into the intermediate steps that are needed in order to complete it. The program will then run continuously until it completes these tasks. For example, if it is working on a coding project, it will debug the code as it goes. 30 | 31 | Let’s dig a little deeper into how Auto-GPT works under the hood. At the time of writing, Auto-GPT uses GPT-4 (or optionally, GPT-3.5) for text generation and GPT-3.5 for file storage and summarization. At configuration, Auto-GPT is given a list of tools such as a code executor, google search API, or a calculator. Additionally, it is possible to give Auto-GPT access to long-term memory via a [vector database](https://weaviate.io/blog/what-is-a-vector-database), such as [Weaviate](https://weaviate.io/). Auto-GPT also has access to `skills` which are manifested as pre-configured prompts such as summarization. 32 | 33 | Armed with these tools, Auto-GPT begins with a user query. For example, “Please write out a grocery list and create a recipe using each ingredient.” Auto-GPT takes this task and proposes an action plan to achieve the task such as: 34 | 35 | Task: Write out a grocery list based on past items 36 | 37 | Plan: 38 | 1. Use the long-term memory from the Weaviate database to curate the list from last week 39 | 2. Order the groceries 40 | 3. Use the internet to look up different recipes 41 | 42 | Action: 43 | 1. View previous grocery lists 44 | 2. Order the groceries online 45 | 3. Use the internet to gather recipes 46 | 47 | Since Auto-GPT is able to form these action plans autonomously, it is important that it confirms each action was completed. It shouldn’t jump to step 2, if step 1 isn’t finished. It will do this by reasoning with its actions. From there it will review the results and make a refined plan. The ability to reason and refine its actions is what makes Auto-GPT so clever. 48 | 49 | ## Examples of Auto-GPT 50 | People all over Twitter have shared multiple demos of what they’ve built with Auto-GPT. The possibilities are endless! In this section, we will cover a few popular examples. 51 | 52 | [Sully](https://twitter.com/SullyOmarr/status/1645205292756418562) shared a thread on using Auto-GPT to conduct market research. The task is to understand the space of waterproof shoes and find 5 competitors. It will then need to report the pros and cons of each. 53 | 54 | [Varun Mayya](https://twitter.com/VarunMayya/status/1643902198164717569) pointed out the opportunity of having Auto-GPT running autonomously. It was tasked to create an app, however, it noticed Node wasn’t installed so it took it upon itself to download it by finding a Stack Overflow link. 55 | 56 | [Shubham Saboo](https://twitter.com/Saboo_Shubham_/status/1646739277328314368) shared this video of Auto-GPT ordering pizza on Domino's website. 57 | 58 | ![autogpt and weaviate](./img/Weaviate-auto-gpt-dark.png#gh-dark-mode-only) 59 | ![autogpt and weaviate](./img/Weaviate-auto-gpt-light.png#gh-light-mode-only) 60 | 61 | ## How to use it in Weaviate 62 | 63 | Auto-GPT has both a short-term and long-term memory. By connecting to a vector database, like Weaviate, you enable the application to retrieve specific data. This extension is great if you’re asking Auto-GPT to complete a task that it wasn’t trained on. For example, if you have data about your customer base in Boston and would like to form an ad, Auto-GPT wouldn’t be able to do that because this wasn’t included in the training data. The solution to this problem is to connect to your Weaviate instance so that Auto-GPT can fetch the necessary information. Additionally, Auto-GPT can save and recall its actions for future use. 64 | 65 | Here is the [codebase](https://github.com/Significant-Gravitas/Auto-GPT/blob/v0.3.1/autogpt/memory/weaviate.py) to see how Weaviate is integrated in Auto-GPT. 66 | 67 | The easiest way to use Weaviate with Auto-GPT is with a [WCS](https://console.weaviate.cloud) instance. Create a Sandbox instance by following [these steps](https://weaviate.io/developers/wcs/quickstart#create-a-weaviate-cluster), and install Auto-GPT using the latest instructions from the repo, with the following notes in mind. 68 | 69 | Notes: 70 | - At the time of writing, it suggested using the [latest stable release](https://github.com/Significant-Gravitas/Auto-GPT/releases/latest) only, rather than the `master` branch. 71 | - We recommend running Auto-GPT in a Docker container so that it runs in a safer, sandboxed environment. 72 | - If you intend to run Auto-GPT directly from your device, rather than in a Docker container, we suggest installing the required libraries into a virtual environment, rather than to your system Python. 73 | 74 | During installation, edit the below variables in the Auto-GPT `.env` file based on the below: 75 | 76 | ``` 77 | MEMORY_BACKEND=weaviate # Change from `local` 78 | … 79 | 80 | WEAVIATE_HOST=your-endpoint.weaviate.network # URL to your Weaviate instance (without “https://”) 81 | WEAVIATE_PROTOCOL=https # http if deploying Weaviate locally with Docker 82 | WEAVIATE_API_KEY= # Add the API key here if you have authentication enabled 83 | ``` 84 | 85 | And then, you can start Auto-GPT. You can run it directly on your device, or within a Docker container. To run it directly, run: 86 | `python -m autogpt` 87 | 88 | Or run the following to start it within a Docker container: 89 | 90 | ``` 91 | docker build -t autogpt . 92 | docker run -it --env-file=./.env -v $PWD/auto_gpt_workspace:/app/auto_gpt_workspace autogpt 93 | ``` 94 | 95 | The above steps will allow Auto-GPT to use the WCS instance as the memory backend, allowing it to store and retrieve information as required. 96 | 97 | We note using a local instance of Weaviate with Docker is a little more complicated. You will need to modify the `Docker Compose` file such that the Weaviate container and the Auto-GPT container are able to communicate with each other. 98 | 99 | ## Proceed with Caution 100 | 101 | Although Auto-GPT is a great project to try, it is best to test it out with caution. It is currently in the experimental phase, so it is best to not use it in production. Additionally, allowing Auto-GPT to run for a long period of time could get very expensive, and keep in mind that it may affect your device. Such as by writing or modifying files, and installing dependencies for example. If your task requires multiple steps, it is best to keep an eye on your OpenAI API usage, such as by setting a limit on your spend. Check out the disclaimer [here](https://github.com/Significant-Gravitas/Auto-GPT#-disclaimer). 102 | 103 | 104 | import WhatNext from '/_includes/what-next.mdx' 105 | 106 | 107 | -------------------------------------------------------------------------------- /blog/2023-04-25-auth-in-weaviate/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Authentication in Weaviate (videos) 3 | slug: authentication-in-weaviate 4 | authors: [jp] 5 | date: 2023-04-25 6 | image: ./img/hero.png 7 | tags: ['concepts'] 8 | description: "Videos on authentication: an overview, how to log in, how to set it up, and core concepts - including recommendations." 9 | 10 | --- 11 | 12 | ![Authentication in Weaviate](./img/hero.png) 13 | 14 | 15 | 16 | import ReactPlayer from 'react-player/lazy' 17 | 18 | ## Overview 19 | 20 | Authentication is one of those topics that we get quite a few questions about. And we can see why. It's a big, complex topic, and even within Weaviate, there are many options available which can make it seem quite confusing. 21 | 22 | The core concept of authentication is relatively simple. When a client (e.g. a Weaviate client) sends a request to a server (e.g. a Weaviate database), it includes a "secret" that provides some assurances to Weaviate as to who that request is coming from, so that it can operate on that information. 23 | 24 | ![Conceptual diagram of sending a request with authentication credentials](./img/auth_light.png#gh-light-mode-only) 25 | ![Conceptual diagram of sending a request with authentication credentials](./img/auth_dark.png#gh-dark-mode-only) 26 | 27 | In other words, the server can provide as much access as the particular user is allowed. 28 | 29 | But balancing security with usability can be a tricky line to draw, as everybody has different needs and often use different systems. 30 | 31 | So, we thought that this might be a good time to provide an overview of all things authentication in Weaviate. Also, we've recently introduced an API key-based authentication method, which we think might be a good balance of security and usability for many of you. Please check them out below. 32 | 33 | :::infoEach video is only 3-6 minutes, and you do *not* need to watch them in order. 34 | ::: 35 | 36 | :::note Tell us what you think! 37 | We would love to know what you think of video content. Would you like to see more of them? Are there any particular types of videos you'd like more of? 38 | 39 | Please let us know below or on YouTube, and we'll do our best to listen. 😁 40 | ::: 41 | 42 | ## Videos 43 | 44 | ### Introduction to authentication in Weaviate 45 | 46 | 47 |
48 | 49 | #### Timestamps 50 | 51 | - 0:00 Overview 52 | - 0:13 What is authentication? & Key considerations 53 | - 0:58 Overview of available authentication options in Weaviate (anonymous / API key authentication / OIDC authentication) 54 | - 1:54 General Recommendations 55 | 56 | ### Authentication: A client-side perspective 57 | 58 | 59 |
60 | 61 | #### Timestamps 62 | 63 | - 0:00 Overview 64 | - 0:28 Getting authentication information from WCS 65 | - 2:10 Authenticating against Weaviate 66 | - 2:28 Anonymous access 67 | - 3:01 API key authentication 68 | - 3:45 OIDC (username+password) authentication 69 | - 4:21 Read-only key 70 | - 4:38 Authentication in action 71 | - 5:36 Wrap-up 72 | 73 | ### Authentication: Key concepts 74 | 75 | 76 |
77 | 78 | #### Timestamps 79 | 80 | - 0:00 Overview 81 | - 0:31 Anonymous access 82 | - 0:46 Authentication 83 | - 0:58 API key authentication 84 | - 1:04 OIDC authentication 85 | - 1:36 Authentication & Authorization 86 | - 1:52 A comparison of options 87 | - 2:09 Additional complexities in OIDC 88 | - 2:54 summary 89 | 90 | ### Authentication: A server-side perspective 91 | 92 | 93 |
94 | 95 | #### Timestamps 96 | 97 | - 0:00 Overview 98 | - 0:35 Weaviate without authentication 99 | - 1:39 Setting up API key access 100 | - 2:32 Enabling authorization (tiered access) 101 | - 3:46 Setting up OIDC access 102 | - 5:30 Enabling authorization with OIDC 103 | - 5:54 Summary 104 | - 6:02 Relevant environment variables 105 | 106 | ## Read more: 107 | 108 | - [How to configure authentication](/developers/weaviate/configuration/authentication) 109 | - [How to configure authorization](/developers/weaviate/configuration/authorization) 110 | - [References: Environment variables](/developers/weaviate/config-refs/env-vars) 111 | - [Weaviate clients](/developers/weaviate/client-libraries/) 112 | 113 | import WhatNext from '/_includes/what-next.mdx' 114 | 115 | 116 | -------------------------------------------------------------------------------- /blog/2023-05-02-wcs-public-beta/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Weaviate Cloud Service Public Beta - Open Now! 3 | slug: wcs-public-beta 4 | authors: [pete] 5 | date: 2023-05-02 6 | image: ./img/hero.png 7 | tags: ['release'] 8 | description: "The Weaviate Cloud Service is the easiest way to get a Weaviate cluster up and running within minutes. Learn more here." 9 | 10 | --- 11 | ![Weaviate Cloud Service Public Beta - Open Now!](./img/hero.png) 12 | 13 | 14 | 15 | import ReactPlayer from 'react-player/lazy' 16 | 17 | Last November, we quietly introduced the private beta of the [Weaviate Cloud Service](https://weaviate.io/pricing), a fully managed [vector database](https://weaviate.io/blog/what-is-a-vector-database) as a service. Today we are announcing a big upgrade to the Weaviate Cloud and availability of the public beta release of the service. 18 | 19 | 20 | 21 | The Weaviate Cloud enables AI application developers to use the [open source Weaviate vector database](https://weaviate.io/developers/weaviate) as a fully managed cloud service. It’s the easiest way to get a vector database cluster up and running within minutes, so you can get right to loading, vectorizing, and searching your data. 22 | 23 | ## Thanks for all the early feedback! 24 | 25 | During the Weaviate Cloud private beta, we had several thousand people create free Weaviate Cloud sandboxes and try it out. We’d like to thank everyone who tried it and gave us feedback; it was extremely helpful. We look forward to hearing about your experience with the new (and future versions), which you can still [try for free here](https://console.weaviate.cloud/). 26 | 27 | ## The Weaviate Cloud Service Public Beta 28 | Here’s a quick overview of what’s new and what’s coming next. 29 | 30 | ### Help yourself - Self-Service Provisioning Improvements 31 | One of the biggest changes from the private beta to today’s public beta release is automated Weviate database cluster provisioning. Immediately after signing up for a Weaviate Cloud account, you can begin creating, querying, securing, and controlling one or more databases via the Weaviate Cloud console. As a result of the automation improvements, newly created Weaviate database clusters will be ready for use within a minute or two. 32 | 33 | ![Weaviate Cloud Service Create Cluster Page ](./img/wcs-options.png) 34 | 35 | ### Weaviate Cloud Service runs on Google Cloud (other clouds & on-premises are options too) 36 | The Weaviate Cloud Service runs on Google Cloud, and by default, new Weaviate database clusters are provisioned to run on Google Cloud. When you create a new cluster, you can choose a specific Google Cloud location (region) on which to run your database cluster. We do plan to make Weaviate Cloud available on AWS and Azure; stay tuned for news on that! 37 | 38 | ### Hybrid SaaS Option - Fully Managed, on Your Cloud 39 | If you’d like your database hosted on a dedicated private cloud, you can contact us and [request a Hybrid-SaaS cluster](https://weaviate.io/pricing). With the Hybrid-SaaS option, Weaviate will set up and manage your database cluster on a dedicated private cloud system on the cloud platform of your choice or on premises in your data center. 40 | 41 | ## The Full Power of the Weaviate Vector Database 42 | The Weaviate Cloud is always updated with the latest [Weaviate open source database](https://github.com/weaviate/weaviate) version and functionality. If you’ve been using Weaviate Cloud since its release, you’ve seen it enhanced with new core Weaviate database functionality, such as: 43 | 44 | - [ChatGPT plug-in](https://weaviate.io/blog/weaviate-retrieval-plugin) 45 | - [Bitmap indexing](https://weaviate.io/blog/weaviate-1-18-release#faster-filtering-with-roaring-bitmaps) for faster filtered search 46 | - [HNSW-PQ index and data compression](https://weaviate.io/blog/weaviate-1-18-release#hnsw-pq) for greater cost efficiency 47 | - [Database replication](https://weaviate.io/blog/weaviate-1-18-release#replication) 48 | - [WAND and BM25 search](https://weaviate.io/blog/weaviate-1-18-release#improvements-to-bm25-and-hybrid-search) 49 | - [Cursor API](https://weaviate.io/blog/weaviate-1-18-release#cursor-api) for processing query results 50 | 51 | ## Develop More Productively - Interactive Admin & Query Console 52 | We care a lot about the Weaviate user experience, so developer productivity is a top design priority. Hence, we implemented many new features in the Weaviate Cloud Console, which now allows you to administer your Weaviate clusters - create, delete, restore, monitor your clusters and activate high-availability settings. 53 | 54 | ![Weaviate Cloud Service Console ](./img/wcs-connected-instances.png) 55 | 56 | The console also features a GraphQL IDE that allows you to interactively query your Weaviate databases. The GraphQL query editor features: 57 | 58 | - Syntax highlighting 59 | - Intelligent type ahead of fields, arguments, types, and more 60 | - Real-time error highlighting and reporting for queries and variables 61 | - Automatic query and variables completion 62 | 63 | 64 | ![Weaviate Cloud Service GraphQL IDE ](./img/wcs-console-example-query.png) 65 | 66 | ## Control Access to Weaviate - API Key Management 67 | To help make it easy to control access to your vector databases, all Weaviate Cloud instances can generate API keys with full admin (read and write) access. Read-only API keys are also supported for paid clusters. 68 | 69 | 70 | ![API Key Management in Weaviate Cloud Service](./img/wcs-get-key.png) 71 | 72 | ## Next Stop, GA… 73 | The Weaviate Cloud is in public beta. General Availability is planned for release later in 2023, after we’ve addressed feedback from public beta users. 74 | 75 | If you’d like to try Weaviate Cloud, you can create a [free Weaviate Cloud Sandbox here](https://console.weaviate.cloud/). 76 | 77 | And if you have questions or feedback, please join the [Weaviate community on Slack](https://weaviate.io/slack). We look forward to hearing from you! 78 | 79 | import WhatNext from '/_includes/what-next.mdx' 80 | 81 | 82 | -------------------------------------------------------------------------------- /blog/2023-05-04-weaviate-1-19-release/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Weaviate 1.19 Release 3 | slug: weaviate-1-19-release 4 | authors: [jp,zain,erika] 5 | date: 2023-05-04 6 | image: ./img/hero.png 7 | tags: ['release'] 8 | description: "Weaviate 1.19 introduces generative cohere module, gRPC API support, improved data types, and more." 9 | 10 | --- 11 | 12 | import Core119 from './_core-1-19-include.mdx' ; 13 | 14 | 15 | 16 | import WhatsNext from '/_includes/what-next.mdx' 17 | 18 | 19 | 20 | import Ending from '/_includes/blog-end-oss-comment.md' ; 21 | 22 | 23 | -------------------------------------------------------------------------------- /blog/2023-05-10-announcing-palm-modules/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Try Google's newly announced PaLM API with Weaviate 3 | slug: announcing-palm-modules 4 | authors: [jp] 5 | date: 2023-05-10 6 | image: ./img/hero.png 7 | tags: ['integrations'] 8 | description: "Weaviate now supports the PaLM models for embeddings and generative search through two new modules." 9 | 10 | --- 11 | 12 | ![Weaviate v1.19.1 introduces modules for use with Google's Vertex PALM API](./img/hero.png) 13 | 14 | 15 | 16 | We are thrilled to announce two brand new Weaviate modules that will help you to get the most out Google's new PaLM large language model (LLM). 17 | 18 | 19 | 22 | 23 | These new modules are: 24 | - [`text2vec-palm`](/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-palm) for generating vector embeddings and running semantic (`nearText`) queries, and 25 | - [`generative-palm`](/developers/weaviate/modules/reader-generator-modules/generative-palm) for generative search. 26 | 27 | 28 | These modules are available to all Weaviate users as of **today**, with the release of Weaviate version `v1.19.1`. They integrate the new Vertex PaLM API with Weaviate, allowing you to hit the ground running straight away with the latest in LLM and [vector database](https://weaviate.io/blog/what-is-a-vector-database) technologies. 29 | 30 | ## What is PaLM? 31 | 32 | The Pathways Language Model (or `PaLM`) is Google’s own LLM. According to Google, PaLM was designed to generalize across domains and tasks while being highly efficient. 33 | 34 | You can read more about in [this blog](https://ai.googleblog.com/2022/04/pathways-language-model-palm-scaling-to.html) or [this paper](https://arxiv.org/abs/2204.02311), but some of the highlights from these articles are that PaLM: 35 | - Contains 540-billion 🤯 parameters, 36 | - Is a dense decoder-only Transformer model, and 37 | - Was trained using a combination of English and multilingual datasets. 38 | 39 | Don't take our word for it - take a look at these [demo snippets from Google's blog](https://ai.googleblog.com/2022/04/pathways-language-model-palm-scaling-to.html). 40 | 41 | ![Image](https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhLmQjS3gOQ2x7ru3xovYjVw-Yr2fKDCqhDHByQZitD92Yu4L-v2BBa5f_VMfpWM4D0930Dmk35EY1TqGrYUtMQqJO41hkLqXuu51eOpXZ3PvYPSjf5stfEJNJn2idWnRYCCEgBiJuLDTXX5Fgt-Mk13kCKdO12JShGvDO_cArtLKv8U8obJaHiL5ASQg/s16000/Big%20Bench%20Sped%20Up%20Cropped.gif) 42 | 43 | > Edit: We now have more details on the PaLM 2 family of models, which the API is based on! [Read about it here](https://blog.google/technology/ai/google-palm-2-ai-large-language-model/). In the blog, Google especially highlights improvements in multilingual, reasoning and coding capabilities. 44 | 45 | ### What can the Vertex PaLM API do? 46 | 47 | Being an LLM, PaLM can perform numerous tasks, from question-answering, sentence-completion, reading comprehension tasks, common-sense reasoning tasks and more. 48 | 49 | Additionally, it can perform multi-lingual NLP tasks reflecting its dataset. What's more, you can use Google's Vertex AI PaLM API to test, customize, and deploy instances of Google's LLMs. 50 | 51 | In other words, you can not only use PaLM off-the-shelf, but also customize the foundation model to suit **your** needs. 52 | 53 | ## PaLM Weaviate 54 | 55 | ![You can use Weaviate with the Vertex PALM API](./img/highlight.png) 56 | 57 | And we are excited to tell you about the ['text2vec-palm`](/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-palm) and ['generative-palm'](/developers/weaviate/modules/reader-generator-modules/generative-palm) modules. These modules will help you to bring the power of the newest LLM on the block to the, *ahem*, `PaLM` of your hand - or at least, to your Weaviate instance. 58 | 59 | ### Embeddings with `text2vec-palm` 60 | 61 | The [`text2vec-palm`](/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-palm) module can use a PaLM model to convert text to vector embeddings that represent their meaning. 62 | 63 | So, you can use the `text2vec-palm` module to build a Weaviate database using PaLM-derived vectors for text objects, and perform lightning-fast searches. 64 | 65 | The PaLM model used, `textembedding-gecko-001`, takes a maximum of 3,072 input tokens, and outputs 768-dimensional vector embeddings. 66 | 67 | Please note that according to Google, the embedding endpoint does *not* support fine-tuning. 68 | 69 | ### Generative search with `generative-palm` 70 | 71 | The [`generative-palm`](/developers/weaviate/modules/reader-generator-modules/generative-palm) module is another exciting development for Weaviate and Google Cloud / Vertex AI users. 72 | 73 | Our generative search modules work in two stages. First, a search is performed in Weaviate, and then a generative model such as PaLM is called to action, whereby the results are sent to prompt the LLM (PaLM in this case). 74 | 75 | In other words - Weaviate can not only find the right data object for you, it can find it, and send it to PaLM to answer questions about it, expand upon it, summarize it, or otherwise transform it as you see fit. 76 | 77 | And remember that the Vertex PaLM API allows further fine-tuning of your PaLM model? It means you can do all this with an LLM that has been tuned to your use case, such as your task, or domain. 78 | 79 | The base PaLM model for generative search is `text-bison-001`, which has the following properties: 80 | 81 | - Max input token: 8,192 82 | - Max output tokens: 1,024 83 | - Training data: Up to Feb 2023 84 | 85 | ## Want to try it out? 86 | 87 | You can read more about how to use these modules in our documentation! See the pages for: 88 | 89 | - [`text2vec-palm`](/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-palm), and 90 | - [`generative-palm`](/developers/weaviate/modules/reader-generator-modules/generative-palm). 91 | 92 | import WhatNext from '/_includes/what-next.mdx' 93 | 94 | 95 | 96 | -------------------------------------------------------------------------------- /blog/2023-06-06-embedded-weaviate/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: How to run an embedded vector database in 10 lines of code 3 | slug: embedded-local-weaviate 4 | authors: [dan] 5 | date: 2023-06-06 6 | image: ./img/hero.png 7 | tags: ['how-to'] 8 | description: "The Weaviate server can be run locally directly from client code" 9 | 10 | --- 11 | 12 | import Tabs from '@theme/Tabs'; 13 | import TabItem from '@theme/TabItem'; 14 | import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; 15 | import PyCode from '!!raw-loader!/_includes/code/embedded.py'; 16 | import TSCode from '!!raw-loader!/_includes/code/embedded.ts'; 17 | 18 | Yes, 10 Python lines of code, generously formatted with whitespace. Or 14 for TypeScript. Oh, and all your data stays private locally, and we don't charge you anything. We're also going to build a useful example, illustrating a testing scenario. Here's how. 19 | 20 | 21 | 22 | 23 | ## 1. Install the client library 24 | 25 | The Python and TypeScript client libraries support running Weaviate embedded on Linux, and starting with versions 3.21.0 and 1.2.0 respectively, on macOS as well. 26 | 27 | 28 | 29 | 30 | ```bash 31 | pip install weaviate-client --upgrade 32 | ``` 33 | 34 | 35 | 36 | 37 | 38 | ```bash 39 | npm install weaviate-ts-embedded typescript ts-node jest # also install support for TypeScript and Jest testing 40 | ``` 41 | 42 | 43 | 44 | 45 | 46 | ## 2. Run the code 47 | 48 | 49 | 50 | 51 | Save as `embedded.py` and run `python embedded.py`: 52 |
53 | 54 | 60 | 61 | 62 | 63 | 64 | Save as `embedded.ts` and run `node --loader=ts-node/esm embedded.ts`: 65 |
66 | 67 | 73 | 74 | 75 | 76 | 77 | ## How does this work? 78 | 79 | Essentially, what happens behind the scenes is that the client library downloads the server binary, spawns it in a separate process, connects to it, then terminates it on exit. The data is persisted, so you can use it from future invocations, or you can [transfer it to another instance](/developers/weaviate/manage-data/read-all-objects/#restore-to-a-target-instance). 80 | 81 | You can learn more about running Weaviate locally from client code on the [Embedded Weaviate](/developers/weaviate/installation/embedded/) page. 82 | 83 | 84 | ## Use cases 85 | 86 | What can you do with Embedded Weaviate? Quite a few things! 87 | 88 | First off, you can get started very quickly with Weaviate on your local machine, without having to explicitly download, install or instantiate a server. 89 | 90 | 91 | ### Jupyter notebooks 92 | 93 | You can also use Embedded Weaviate from Jupyter notebooks, including on Google Colaboratory. Here is the example above as [Google Colab notebook](https://colab.research.google.com/drive/1XAJc9OvkKhsJRmheqWZmjYU707dqEIl8?usp=sharing). 94 | 95 | ![Colab screenshot](./img/colab.png) 96 | 97 | ### Use Weaviate in CI/CD pipelines 98 | 99 | You can use Embedded Weaviate in automated tests, where you can run integration tests without having to manage a separate server instance. Here is the example above slightly modified to perform similarity search and test that the added object was found. 100 | 101 | 102 | 103 | 104 | Save as `embedded_test.py` and run `pytest`. (If you don't have pytest, run `pip install pytest`.) 105 |
106 | 107 | 113 | 114 | 115 | 116 | 117 | Save as `embedded_test.ts` and run `npx jest`: 118 |
119 | 120 | 126 | 127 | 128 | 129 | 130 | Have you found other use cases for embedded Weaviate? Let us know in the comments below! 131 | 132 | 133 | import WhatNext from '/_includes/what-next.mdx' 134 | 135 | 136 | -------------------------------------------------------------------------------- /blog/2023-06-22-llamaindex-and-weaviate/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: LlamaIndex and Weaviate 3 | slug: llamaindex-and-weaviate 4 | authors: [jerry] 5 | date: 2023-06-22 6 | image: ./img/hero.png 7 | tags: ['integrations', 'how-to'] 8 | description: "An introductory overview of LlamaIndex, the LLM framework" 9 | 10 | --- 11 | 12 | ![LlamaIndex and Weaviate](./img/hero.png) 13 | 14 | 15 | 16 | While large language models (LLMs) like GPT-4 have impressive capabilities in generation and reasoning, they have limitations in terms of their ability to access and retrieve specific facts, figures, or contextually relevant information. A popular solution to this problem is setting up a retrieval-augmented generation (RAG) system: combine the language model with an external storage provider, and create an overall software system that can orchestrate the interactions with and between these components in order to create a “chat with your data” experience. 17 | 18 | The combination of Weaviate and LlamaIndex provide the critical components needed to easily setup a powerful and reliable RAG stack, so that you can easily deliver powerful LLM-enabled experiences over your data, such as search engines, chatbots, and more. First, we can use Weaviate as the [vector database](https://weaviate.io/blog/what-is-a-vector-database) that acts as the external storage provider. Next, we can use a powerful data framework such as LlamaIndex to help with data management and orchestration around Weaviate when building the LLM app. 19 | 20 | In this blog post, we walk through an overview of LlamaIndex and some of the core data management and query modules. We then go through an initial demo notebook. 21 | 22 | We’re kicking off a new series to guide you on how to use LlamaIndex and Weaviate for your LLM applications. 23 | 24 | ## An Introduction to LlamaIndex 25 | LlamaIndex is a data framework for building LLM applications. It provides a comprehensive toolkit for ingestion, management, and querying of your external data so that you can use it with your LLM app. 26 | 27 | ### Data Ingestion 28 | On data ingestion, LlamaIndex offers connectors to 100+ data sources, ranging from different file formats (.pdf, .docx, .pptx) to APIs (Notion, Slack, Discord, etc.) to web scrapers (Beautiful Soup, Readability, etc.). These data connectors are primarily hosted on [LlamaHub](https://llamahub.ai/). This makes it easy for users to integrate data from their existing files and applications. 29 | 30 | ### Data Indexing 31 | Once the data is loaded, LlamaIndex offers the ability to index this data with a wide variety of data structures and storage integration options (including Weaviate). LlamaIndex supports indexing unstructured, semi-structured, and structured data. A standard way to index unstructured data is to split the source documents into text “chunks”, embed each chunk, and store each chunk/embedding in a vector database. 32 | 33 | ### Data Querying 34 | Once your data is ingested/stored, LlamaIndex provides the tools to define an advanced retrieval / query “engine” over your data. Our retriever constructs allow you to retrieve data from your knowledge base given an input prompt. A query engine construct allows you to define an interface that can take in an input prompt, and output a knowledge-augmented response - it can use retrieval and synthesis (LLM) modules under the hood. 35 | 36 | Some examples of query engine “tasks” are given below, in rough order from easy to advanced: 37 | * Semantic Search: Retrieve the top-k most similar items from the knowledge corpus by embedding similarity to the query, and synthesize a response over these contexts. 38 | 39 | * Structured Analytics: Convert natural language to a SQL query that can be executed 40 | 41 | * Query Decomposition over Documents: Break down a query into sub-questions, each over a subset of underlying documents. Each sub-question can be executed against its own query engine. 42 | 43 | ## Demo Notebook Walkthrough 44 | 45 | Let’s walk through a simple example of how LlamaIndex can be used with Weaviate to build a simple Question-Answering (QA) system over the Weaviate blogs! 46 | 47 | The full code can be found in the [Weaviate recipes repo](https://github.com/weaviate/recipes/tree/main/integrations/llamaindex/simple-query-engine). 48 | 49 | The first step is to setup your Weaviate client. In this example, we connect to an [Embedded Weaviate](https://weaviate.io/developers/weaviate/installation/embedded) instance. 50 | 51 | ```python 52 | import os 53 | import weaviate 54 | 55 | # connect to your weaviate instance 56 | client = weaviate.Client(embedded_options=weaviate.embedded.EmbeddedOptions(), additional_headers={ 'X-OpenAI-Api-Key': os.environ["OPENAI_API_KEY"]}) 57 | ``` 58 | 59 | Ensure you have an Open AI API Key set up in your environment variables for use by the Weaviate client. 60 | 61 | The next step is to ingest the Weaviate documentation and parse the documents into chunks. You can choose to use one of our many web page readers to scrape any website yourself - but luckily, the downloaded files are already readily available in the recipes repo. 62 | 63 | ```python 64 | from llama_index import SimpleDirectoryReader 65 | from llama_index.node_parser import SimpleNodeParser 66 | 67 | # load the blogs in using the reader 68 | blogs = SimpleDirectoryReader('./data').load_data() 69 | 70 | # chunk up the blog posts into nodes 71 | parser = SimpleNodeParser.from_defaults(chunk_size=1024, chunk_overlap=20) 72 | nodes = parser.get_nodes_from_documents(blogs) 73 | ``` 74 | 75 | Here, we use the SimpleDirectoryReader to load in all documents from a given directory. We then use our `SimpleNodeParser` to chunk up the source documents into Node objects (text chunks). 76 | 77 | The next step is to 1) define a `WeaviateVectorStore`, and 2) build a vector index over this vector store using LlamaIndex. 78 | 79 | ```python 80 | from llama_index.vector_stores import WeaviateVectorStore 81 | from llama_index import VectorStoreIndex, StorageContext 82 | 83 | # construct vector store 84 | vector_store = WeaviateVectorStore(weaviate_client = client, index_name="BlogPost", text_key="content") 85 | 86 | # setting up the storage for the embeddings 87 | storage_context = StorageContext.from_defaults(vector_store = vector_store) 88 | 89 | # set up the index 90 | index = VectorStoreIndex(nodes, storage_context = storage_context) 91 | ``` 92 | 93 | Our WeaviateVectorStore abstraction creates a central interface between our data abstractions and the Weaviate service. Note that the `VectorStoreIndex` is initialized from both the nodes and the storage context object containing the Weaviate vector store. During the initialization phase, the nodes are loaded into the vector store. 94 | 95 | Finally, we can define a query engine on top of our index. This query engine will perform semantic search and response synthesis, and output an answer. 96 | 97 | ```python 98 | 99 | ​​query_engine = index.as_query_engine() 100 | response = query_engine.query("What is the intersection between LLMs and search?") 101 | print(response) 102 | ``` 103 | 104 | You should get an answer like the following: 105 | 106 | ``` 107 | The intersection between LLMs and search is the ability to use LLMs to improve search capabilities, such as retrieval-augmented generation, query understanding, index construction, LLMs in re-ranking, and search result compression. LLMs can also be used to manage document updates, rank search results, and compress search results. LLMs can be used to prompt the language model to extract or formulate a question based on the prompt and then send that question to the search engine, or to prompt the model with a description of the search engine tool and how to use it with a special `[SEARCH]` token. LLMs can also be used to prompt the language model to rank search results according to their relevance with the query, and to classify the most likely answer span given a question and text passage as input. 108 | ``` 109 | 110 | ## Next Up in this Series 111 | 112 | This blog post shared an initial overview of the LlamaIndex and Weaviate integration. We covered an introduction to the toolkits offered in LlamaIndex and a notebook on how to build a simple QA engine over Weaviate’s blog posts. Now that we have a baseline understanding, we will build on this by sharing more advanced guides soon. Stay tuned! 113 | 114 | import WhatNext from '/_includes/what-next.mdx' 115 | 116 | 117 | -------------------------------------------------------------------------------- /blog/2023-07-11-weaviate-1-20-release/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Weaviate 1.20 Release 3 | slug: weaviate-1-20-release 4 | authors: [dan,erika,jp,zain] 5 | date: 2023-07-11 6 | image: ./img/hero.png 7 | tags: ['release', 'engineering'] 8 | description: "Weaviate 1.20 released with multi-tenancy, PQ, search re-ranking, autocut, hybrid fusion algorithm ... take a look!" 9 | 10 | --- 11 | 12 | import Core120 from './_core-1-20-include.mdx' ; 13 | 14 | 15 | 16 | import WhatsNext from '/_includes/what-next.mdx' 17 | 18 | 19 | 20 | import Ending from '/_includes/blog-end-oss-comment.md' ; 21 | 22 | 23 | -------------------------------------------------------------------------------- /blog/2023-10-02-hacktoberfest-2023/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Hacktoberfest 2023 - Celebrating Open Source with Weaviate 3 | slug: hacktoberfest-2023 4 | authors: [leonie] 5 | date: 2023-10-02 6 | tags: [] 7 | image: ./img/weaviate-hacktoberfest-2023.png 8 | description: "Join us in celebrating Hacktoberfest, a month-long celebration of open source!" 9 | --- 10 | import hacktober_demo from './img/hacktoberfest_2023_demo.mp4'; 11 | 12 | ![Celebrating Hacktoberfest 2023 with Weaviate](./img/weaviate-hacktoberfest-2023.png) 13 | 14 | At [Weaviate](https://weaviate.io/), we love open source! Join us in celebrating [Hacktoberfest](https://hacktoberfest.com/), a month-long celebration of open source! 15 | Participants with four pull requests (PR) accepted between **October 1 - 31, 2023** will receive a unique digital reward [from Hacktoberfest](https://hacktoberfest.com/participation/). 16 | 17 | 18 | ## The Task 19 | 20 | Welcome to our demo sprint! 21 | 22 | It is aimed at onboarding engineers and machine learning practitioners to open-source. In our [example use cases and demos](https://weaviate.io/developers/weaviate/more-resources/example-use-cases) page, we showcase what you can do with a [vector database](https://weaviate.io/blog/what-is-a-vector-database). Some of these demos have received more love than others recently, and we want to give them a little polish. 23 | 24 | ![Untitled](./img/weaviate-demos.png) 25 | 26 | The gallery contains demos using Weaviate in different states. [Here](https://github.com/databyjp/distyll) is an example of a demo project in a good state. While some may only need a little polish of the description (README.md file), others are e.g., a little older or can use a little makeover for the user interface. 27 | 28 | The steps to your first Hacktoberfest PR are simple: 29 | 30 | 1. Find an issue you're interested in: Go to the [example use cases and demos](https://weaviate.io/developers/weaviate/more-resources/example-use-cases) page and select a project marked for Hacktober with a 🎃 sign. Click on “code” to get to the repository and have a look at its issues. 31 | 32 | 36 | 37 | 38 | 2. Ping us on the project’s issue, saying you're interested and which parts of the issue you would like to contribute to. 39 | 3. Open the PR as instructed in the [Weaviate Contributor Guide](https://weaviate.io/developers/contributor-guide). We will review and help you out in the process. 💚 40 | 41 | You can also contribute by adding your own Weaviate examples. If you have other great ideas for contributions, let us know on our [Discourse](https://forum.weaviate.io/) and [Slack](https://weaviate.slack.com/) channels, and we will figure out how to highlight it in Hacktoberfest. 42 | 43 | You don't need to be an expert to contribute to these demo projects! 44 | 45 | 46 | ## Resources to Get Started 47 | 48 | We're thrilled to help you make your first open-source contribution! Here are some helpful resources to kickstart your journey: 49 | 50 | What is Open Source, and how do you contribute to it? 51 | 52 | - 🎯 [What is Open Source](https://www.digitalocean.com/community/tutorials/what-is-open-source) 53 | - 🎯 [Introduction to GitHub and Open-Source Projects](https://www.digitalocean.com/community/tutorial_series/an-introduction-to-open-source) 54 | - 🎯 [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/) 55 | - 🎯 [GitHub Contribution Guide by Hugging Face](https://www.notion.so/Contribution-Guide-19411c29298644df8e9656af45a7686d?pvs=21) 56 | - 🎯 [How to Use Git](https://www.digitalocean.com/community/cheatsheets/how-to-use-git-a-reference-guide) 57 | - 🎯 [Weaviate Contributor Guide](https://weaviate.io/developers/contributor-guide) 58 | 59 | If you're new to Weaviate, get up and running quickly with these beginner-friendly guides: 60 | 61 | - [Quickstart Guide](https://weaviate.io/developers/weaviate/quickstart) 🚀 62 | - [Weaviate Academy](https://weaviate.io/developers/academy) 🎓 63 | 64 | Dive deeper into specific topics with these detailed guides: 65 | 66 | - [How-to Search Guides](https://weaviate.io/developers/weaviate/search) 🔍 67 | - [Keyword, Vector, Hybrid, and Generative Search](https://github.com/weaviate-tutorials/generative-search/blob/main/GenerativeSearchDemo.ipynb) 🔍 68 | - [How-to Manage Data (CRUD Operations)](https://weaviate.io/developers/weaviate/manage-data) 💾 69 | - [Tutorial: Importing Data with Your Own Vectors](https://weaviate.io/developers/weaviate/tutorials/wikipedia) 📊 70 | - [Weaviate Architecture Concepts](https://weaviate.io/developers/weaviate/concepts#weaviate-architecture) 🏛️ 71 | 72 | Join one of our [workshops](https://weaviate.io/learn/workshops) for an introduction to Weaviate. 73 | 74 | - 4th October 2023 (20:00-21:00 UTC+2): [Introduction to Weaviate](https://form.jotform.com/232574048361254) with Zain Hasan, Senior Dev Advocate @ Weaviate 75 | - 5th October 2023 (17:00-18:00 UTC+2): [NEW Python `Collections` Client API Preview](https://form.jotform.com/232683153137859) with JP Hwang - Educator @ Weaviate 76 | - 18th October 2023 (15:00-16:00 UTC+2): [Introduction to Weaviate](https://form.jotform.com/232602295283859) with JP Hwang - Educator @ Weaviate 77 | 78 | Expand your knowledge with these supplementary resources: 79 | 80 | - [Weaviate YouTube Channel](https://www.youtube.com/@Weaviate) 📺 81 | - [Weaviate Blog](https://weaviate.io/blog) 📰 82 | - [Weaviate Recipes](https://github.com/weaviate/recipes) 🍳 83 | 84 | 85 | ## What's in it for you? 86 | 87 | The repositories you contribute to participate in [Hacktoberfest](https://hacktoberfest.com/) and are a great opportunity for your first-ever Hacktoberfest PR. We will also assist you throughout the whole process. 88 | 89 | You might even receive some swag in the end. 90 | 91 | 92 | ## Connect with the Weaviate Community! 93 | 94 | To make your Hacktoberfest experience successful, connect with the Weaviate community for collaboration and assistance. You can reach us through: 95 | 96 | - Join our dedicated Hacktoberfest channel in our [Discourse community forum](https://forum.weaviate.io/), where we're ready to answer your questions. 97 | - Join our dedicated #hacktoberfest channel in our [Weaviate Community Slack](https://weaviate.slack.com/) channel to stay connected and receive real-time support. 98 | - Join our [newsletter](https://newsletter.weaviate.io/) and follow us on [Twitter](https://twitter.com/weaviate_io) and [Linkedin](https://www.linkedin.com/company/weaviate-io/mycompany/verification/) to receive updates. 99 | - Stay up to date with Weaviate's development by exploring the [Weaviate GitHub Repository](https://github.com/weaviate/weaviate). Don’t forget to give us a ⭐️ while you are there! 100 | 101 | :::info Pro Tip 102 | Share your process online and tag us on [Twitter](https://twitter.com/weaviate_io) and [LinkedIn](https://nl.linkedin.com/company/weaviate-io). Use the hashtag #hacktoberfest2023 for increased visibility. 103 | ::: 104 | 105 | 106 | ## FAQ 107 | 108 | - **Will this count towards Hacktoberfest?** Yes, it definitely does! If your PR/MR is created between **October 1** and **October 31** (in any time zone, UTC-12 thru UTC+14), we will add the "HACKTOBERFEST-ACCEPTED" label to it. 109 | - **Where do I get help?** For any questions or assistance, contact us on our [Discourse](https://forum.weaviate.io/) and [Slack](https://weaviate.slack.com/) channels. 110 | - **I have a cool contribution idea. Can I still participate?** Awesome! Connect with us on our [Discourse](https://forum.weaviate.io/) or [Slack](https://weaviate.slack.com/) channels and we will figure it out. 111 | - **I don’t know how to write code. Can I still contribute?** Yes, of course! You can make no-code contributions, e.g., by updating the README.md files. If you want to learn how to write code with a concrete example, we can help you find a good issue. Just ping us on our [Discourse](https://forum.weaviate.io/) or [Slack](https://weaviate.slack.com/) channels. 112 | 113 | --- 114 | 115 | Happy hacking, and let's make Hacktoberfest 2023 a memorable one together! 🚀 116 | 117 | Jump right in and have a look at our [example use cases and demos](https://weaviate.io/developers/weaviate/more-resources/example-use-cases) page. 118 | -------------------------------------------------------------------------------- /blog/2023-10-31-weaviate-1-22-release/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Weaviate 1.22 Release 3 | slug: weaviate-1-22-release 4 | authors: [jp] 5 | date: 2023-10-23 6 | image: ./img/hero.png 7 | tags: ['release', 'engineering'] 8 | description: "Weaviate 1.22 released with nest object storage, async indexing, further gRPC support, and more!" 9 | 10 | --- 11 | 12 | import Core122 from './_core-1-22-include.mdx' ; 13 | 14 | 15 | 16 | import WhatsNext from '/_includes/what-next.mdx' 17 | 18 | 19 | 20 | import Ending from '/_includes/blog-end-oss-comment.md' ; 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /blog/2023-11-15-moonsift-story/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Building an AI-Powered Shopping Copilot with Weaviate 3 | slug: moonsift-story 4 | authors: [alea, zain] 5 | date: 2023-11-15 6 | tags: [] 7 | image: ./img/hero.png 8 | description: "UK-based startup Moonsift is harnessing the power of AI with Weaviate." 9 | --- 10 | ![hero](img/hero.png) 11 | 12 | UK-based startup Moonsift is harnessing the power of AI—using machine learning models and Weaviate’s vector database—to help online shoppers discover the products they love. 13 | 14 | 15 | 16 | [Moonsift](https://www.moonsift.com/) offers an ecommerce browser extension for users to curate shoppable boards with products from across the internet. Stylists and curators use Moonsift to create collections, registries, and wish lists that can be shared and shopped with a simple link. While thousands of customers add products from tens of thousands of retailers per month to Moonsift, co-founders David Wood and Alex Reed have a bigger vision for improving product discoverability for online shoppers. 17 | 18 | With combined experience in natural language processing (NLP), data science, and consulting for retail brands, Wood and Reed saw how retailers unknowingly restrict their own discoverability by tailoring keywords for search engines rather than users. How even with the most illustrious descriptions, shoppers struggle to find the products they’re looking for. They knew that retail discoverability wasn’t what it could be, and built Moonsift to improve it. 19 | 20 | > From the beginning, Wood and Reed’s vision was to use the power of AI to help shoppers more easily discover the products they love. 21 | 22 | ## Collecting data to enable understanding 23 | The first generation of Moonsift has been a stepping stone toward this vision. In order to create a comprehensive AI shopping copilot, they needed product data from retailers along with cross-retailer shopping data to enable understanding of the discovery process and train machine learning models. And that’s exactly what they’ve been collecting. Usage of their browser extension over the last few years has enabled Moonsift to explore discovery and train models on over 60M products, 250M interactions, and 40K retailers across the internet. Now, Moonsift has the data they need (which is growing every day) to improve product discovery with AI. 24 | 25 | ## Building the discovery engine 26 | The Moonsift team brought on Marcel Marais as lead machine learning engineer to build a system that could harness the data they’ve gathered to take their product to the next level. At first, Marais looked at improving discovery through a keyword-based search system using BM25 and re-ranking, but he quickly assessed that would not be sufficient to power the type of recommendation engine they needed. He needed to implement semantic search, a system that could interpret a user’s intent rather than rely on exact keyword matches. And they needed a system that could index and search multimodal (text and image) data across millions of objects. Based on prior experience with semantic search and vector embeddings, Marais decided a vector database was the tool for the job. 27 | 28 | Marais quickly put together a checklist of priorities for their vector database selection. Moving fast was important, so ease of use was at the top of the list. The company preferred an open source platform, and since they would be indexing millions of products, they needed a solution that was both high-performing and cost-efficient at scale. 29 | 30 | ## Selecting Weaviate as the vector database of choice 31 | After a thorough evaluation of a handful of open and closed-source vector databases, the team decided that Weaviate was the best-fit solution for their needs. 32 | 33 | They cited the following reasons for choosing Weaviate: 34 | 35 | * Open source, with an active community and managed cloud offering. 36 | * Comprehensive documentation and strong support for popular LLMs and multi-modal models. 37 | * Direct integration of machine learning models using a module system, with the ability to easily swap out and experiment with different models. 38 | * Advanced monitoring and replication capabilities. 39 | * High query throughput at scale. 40 | * Unique search features to drive performance and efficiency (using important aspects of both keyword and vector search). 41 | 42 | > “Weaviate was exactly what we needed. Within a couple of weeks we had a production-ready AI-powered search engine. We’re confident in knowing we don’t have to trade performance for scale.” 43 | 44 | – David Wood, CTO and co-founder, Moonsift 45 | 46 | ## Next steps 47 | Moonsift is now getting ready to launch their [AI Copilot](https://www.moonsift.com/copilot) to the world. They’re seeing early results of the power of its ability to understand user intent and serve intelligent results. Some fun examples include “shirt that looks like a Caipirinha” or “skirt with a pattern inspired by ocean waves”. 48 | 49 | ![image](img/image1.png) 50 | 51 | As Moonsift prepares for the public launch of its shopping Copilot, the team is continuing to explore ways to optimize the cost, performance, and scale of their system. They are looking into Weaviate’s new feature, [Product Quantization (PQ)](/blog/pq-rescoring), which helps reduce the memory footprint of their system, by compressing vectors and performing rescoring, while retaining search relevance. Along with PQ they are also exploring [multi-tenancy](/blog/multi-tenancy-vector-search) that will allow them to scale and perform personalized vector search for millions of customers. 52 | 53 | 54 | [See a demo](https://www.youtube.com/watch?v=hOsBxvV9rvI) of Moonsift’s shopping Copilot, and [sign up for early access today](https://www.moonsift.com/copilot)! 55 | 56 | 57 | ## What's next 58 | Interested in using Weaviate to power your search and AI applications? [Give our fully managed cloud offering a spin](https://console.weaviate.cloud/) for free and check out our [Quickstart guide](/developers/weaviate/quickstart). 59 | 60 | You can reach out to us on [Slack](https://weaviate.io/slack) or [Twitter](https://twitter.com/weaviate_io), or [join the community forum](https://forum.weaviate.io/). 61 | 62 | -------------------------------------------------------------------------------- /blog/2023-12-12-aws-reinvent/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Weaviate’s re:Invent 2023 recap 3 | slug: reinvent-2023 4 | authors: [ieva, alea, erika] 5 | date: 2023-12-05 6 | image: ./img/hero.png 7 | tags: [] 8 | description: "Recap the first Weaviate visit to Amazon re:Invent in Las Vegas" 9 | --- 10 | 11 | ![AWS re:Invent](./img/hero.png) 12 | 13 | This November the Weaviate team went to our first ever AWS re:Invent! Over 50,000 people from around the world came together to learn about the latest developments in cloud technology and to share what they’ve been building with the AWS developer community. Let’s dive into our highlight reel! 14 | 15 | ## GenAI is (still) heating up 16 | 17 | This year’s re:Invent focus theme was Generative AI and everyone seemed to be talking about it. In the sessions our team attended, we noticed a particular emphasis on [Retrieval Augmented Generation (RAG)](https://weaviate.io/blog/rag-evaluation). 18 | 19 | Initially when people came to our booth on Monday, roughly 40% of people knew about RAG. By the last day, nearly everyone that talked to us knew what it was because so many different sessions had covered the topic. 20 | 21 |
22 | 23 | !["Bob talks about feedback loops"](img/bob-gen-ai.jpg) 24 | ##### Our CEO Bob talking about the next level of vector database capabilities - Generative Feedback Loops 25 | 26 |
27 | 28 | We also got tons of great feedback on our open-source RAG app, Verba, which we used to demo Weaviate. It’s something you can use to quickly build RAG apps that fit your use case. Check out [our intro video](https://youtu.be/IiNDCPwmqF8?si=cZxfsnY6vW75c4BD) or dive right in to [our GitHub repo](https://youtu.be/IiNDCPwmqF8?si=cZxfsnY6vW75c4BD)! 29 | 30 |
31 | 32 | !["Adam demos Verba"](img/adam-verba.jpg) 33 | ##### Adam from our Developer Growth team demoing Verba, powered by Weaviate! 34 | 35 |
36 | 37 | It was great hearing about the different use cases and industries in which folks are building AI apps – these included chatbots and search applications for things like customer support, document search, image search, and even video search. A common pattern we saw was that organizations of all sizes are evaluating vector database platforms and want guidance on how to find the right database for them. 38 | 39 | One of our recommendations is to look for a vector database purpose-built for AI. These databases (like Weaviate) were created specifically to store, manage, and retrieve vector embeddings, and do so faster and more efficiently than traditional databases. While vector search is becoming a popular offering in the database world, many “latched on” solutions don’t meet the requirements of modern AI-native development, from reduced developer experience to problems with speed and accuracy. 40 | 41 | Which brings us to our next point… 42 | 43 | ## The market is confusing 44 | 45 | The AI space is moving fast. And with so many new technologies and vendors, it’s no wonder re:Invent attendees were telling us that they were getting overwhelmed with the options. 46 | 47 | Luckily, your friendly neighborhood Weaviate crew is here to help. Check out one of our free weekly workshops to help you understand what vector databases are and how they can help you build production-ready AI apps quickly and easily. 48 | 49 | If you’re curious, here are some of the most commonly asked questions we encountered: 50 | 51 | **What’s the difference between a vector database and a graph or relational database?** 52 | 53 | Graph databases are used to identify relationships between objects, and vector databases are used to find objects 54 | Relational databases store the relations between tables and build indexes for the fast lookup of joined tables. Vector databases, on the other hand, store the embeddings of structured and unstructured data for the quick retrieval of the objects. 55 | 56 | **What features does Weaviate offer to help me protect data privacy?** 57 | 58 | Weaviate is designed with robust security measures to ensure it meets the requirements of enterprise environments. Weaviate has achieved SOC 2 certification and is encrypted in transit and at rest. In addition, our code undergoes constant security scanning and we conduct regular penetration testing ensuring our defenses are always up to date against evolving security threats. 59 | 60 | Since Weaviate is open-source technology that can be installed in any environment, it enables customers to keep all data locally, without having to send to a third party over the internet. 61 | 62 | **How does Weaviate compare to AWS?** 63 | 64 | Weaviate and AWS compliment each other nicely. You can use Weaviate as your vector database with Amazon Bedrock. Start building with the [`text2vec-aws` module](/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-aws) today! 65 | 66 | **What sets Weaviate apart from other vector databases?** 67 | 68 | Weaviate is an open-source vector database, built from the ground up for vector search. This means we deliver best-in-class features including: 69 | 70 | - **Native multi-tenancy support**: ensure your users’ data doesn’t merge together. 71 | - **Built-in hybrid search**: customize the weighting of the bm25 and vector search method with the `alpha` value. 72 | - **Robust integration library**: connect Weaviate to your AI stack of choice including providers like Amazon Sagemaker, OpenAI, Cohere, and Hugging Face through our modular architecture. 73 | - **Flexible deployment options**: run Weaviate locally or as a managed service (Serverless, Enterprise Dedicated, or Bring Your Own Cloud) depending on your business needs. 74 | 75 | **What is Weaviate Vector Club?** 76 | 77 | A great question, which brings us to our final point (and some behind-the-scenes moments with the Weaviate crew)… 78 | 79 | ## Weaviate Vector Club is here! 80 | 81 |
82 | 83 | !["Weaviate Vector Club"](img/vector-club..001.jpg)!["Cool sweatshirt"](img/vector-club..002.jpg) 84 | ##### Weaviate Vector Club is all about our community 85 | 86 |
87 | 88 | Our new line of merch originated during a brainstorming session by the sea in Croatia. What slogan could capture the Stranger Things-esque nerd camaraderie and quick iterating you need to keep up in the AI space? No slogan could. 89 | 90 | So, with our re:Invent t-shirt deadline looming over our heads, we scribbled a doodle instead! And somehow, it was just the right amount of weird and fun. We knew we hit something good when our CEO Bob, our resident style icon, said “I like it.” 91 | 92 | We made a bunch of t-shirts, stickers, and temporary tattoos and shared them with hundreds of our open-source users who came by our booth. And every evening, a few minutes before the expo closed, a huge crowd gathered to see what we were raffling – from clicky-clacky Keychron keyboards to Lego PacMan, and the prize of all prizes – a now discontinued Lego Bowser set! 93 | 94 |
95 | 96 | !["Erika picks the winners"](img/erika-raffle..001.jpg) 97 | ##### Erika, our Chief Raffle Officer, hand picked all the lucky winners! 98 | 99 |
100 | 101 | We even had a chance to visit the brand new Sphere with some of our customers and our partners over at Innovative Solutions – if you want to learn more about how we work together, check out [our upcoming webinar](https://webinars.devops.com/how-to-build-an-ai-native-foundation-for-enterprise-apps) about building an AI native foundation for enterprise applications. 102 | 103 |
104 | 105 | !["Team at the sphere"](img/go-to-market-team..001.jpg) 106 | ##### Part of our go-to-market team at The Sphere. 107 | 108 |
109 | 110 | And of course, no trip to Las Vegas could have been complete for our crew without a stop over at Meow Wolf, an immersive art installation at Area 15. 111 | 112 |
113 | 114 | !["Area 15"](img/area-15..001.jpg) 115 | ##### Meow Wolf at Area 15 116 | 117 |
118 | 119 | Remember how we said Bob was a style icon? We decided to pay homage to one of his most recent outfits featured in a Dutch documentary about AI. Now the question is – who wore it best? 120 | 121 |
122 | 123 | !["Copying Bob"](img/bobs..002.jpg)!["Bob"](img/bob..002.jpg) 124 | ##### Who wore it best? 125 | 126 |
127 | 128 | We hope you enjoyed our recap of AWS re:Invent 2023! -------------------------------------------------------------------------------- /blog/2023-12-19-weaviate-1-23-release/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Weaviate 1.23 Release 3 | slug: weaviate-1-23-release 4 | authors: [jp, dave] 5 | date: 2023-12-19 6 | image: ./img/hero.png 7 | tags: ['release', 'engineering'] 8 | description: "Weaviate 1.23 released with AutoPQ, flat indexing + Binary Quantization, OSS LLM support through Anyscale, and more!" 9 | 10 | --- 11 | 12 | import Core123 from './_core-1-23-include.mdx' ; 13 | 14 | 15 | 16 | import WhatsNext from '/_includes/what-next.mdx' 17 | 18 | 19 | 20 | import Ending from '/_includes/blog-end-oss-comment.md' ; 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /blog/2024-02-27-weaviate-1-24-release/_core-1-24-include.mdx: -------------------------------------------------------------------------------- 1 | Weaviate `1.24` is here! 2 | 3 | Here are the release ⭐️*highlights*⭐️! 4 | 5 | ![Weaviate 1.24](./img/hero.png) 6 | 7 | - [**Named vectors.**](/blog/weaviate-1-24-release#named-vectors) A single object can have multiple vectors. Create vectors for properties, use different vectorization models, and apply different metrics to fine tune interactions with your data. 8 | - [**HNSW and binary quantization (BQ)**](/blog/weaviate-1-24-release#hnsw-and-binary-quantization) HNSW indexes and BQ combine for serious compression and blazing speed. 9 | - [**Simplified Docker configuration.**](/blog/weaviate-1-24-release#simplified-docker-configuration) A new Docker image that needs no configuration. 10 | - [**Backend improvements.**](/blog/weaviate-1-24-release#backend-improvements) Numerous improvements to make updates, imports, and deletions faster. 11 | - [**Python client update.**](/blog/weaviate-1-24-release#python-client) General availability, support for all the latest features. 12 | 13 | ## Named vectors 14 | 15 | ![Named vectors](./img/multi-vector.png) 16 | 17 | Named vectors make your collections richer and more versatile. 18 | 19 | Starting in 1.24, collections can have multiple named vectors. Each vector is independent. Each vector has its own index, its own compression, and its own vectorizer. This means you can create vectors for properties, use different vectorization models, and apply different metrics to fine tune interactions with your data. 20 | 21 | In earlier versions, objects in your collections are limited to one vector. Now, you can vectorize meta data like titles and descriptions so that it is available for vector search. You can group things that are logically related, like a music video and a lyric sheet, in the same object. You don't have to create [cross-references](/developers/weaviate/manage-data/cross-references) or manage shared metadata. A single object in a collection can have multiple named vectors. 22 | 23 | You don't have to use multiple vectors in your collections, but if you do, adjust your queries to specify which vector you want to use. 24 | 25 | For details, see [Multiple vectors](/developers/weaviate/config-refs/schema/multi-vector). 26 | 27 | ## HNSW and binary quantization 28 | 29 | ![HNSW and binary quantization](./img/hnsw-and-bq.png) 30 | 31 | Weaviate improves [binary quantization (BQ)](/developers/weaviate/configuration/bq-compression) in 1.24 to be faster, more memory efficient, and more cost-effective. Use BQ vector compression with [HNSW indexes](/developers/weaviate/concepts/vector-index#what-is-hierarchical-navigable-small-world-hnsw) to dramatically improve your query speed. 32 | 33 | BQ compresses vector representations while preserving essential information. Uncompressed, Weaviate uses a `float32` to store each dimension. BQ uses one bit per dimension to encode Vector directionality. This means BQ compresses vectors from 32 bits per dimension to 1 bit per dimension - a savings of 32 times the space. This compression significantly reduces storage requirements. 34 | 35 | Comparing BQ compressed vectors is fast. To calculate the distance between to vectors, compare their dimensional agreement. The bitwise comparison ensures optimal resource utilization without compromising accuracy. For instance, consider two vectors encoded as [1,1,0,1,0,1,0,1] and [1,1,0,0,1,1,0,1]. Their distance is 2. 36 | 37 |
38 | 39 | Bitwise comparison details 40 | 41 | 42 | Compare each value in the first vector one with the corresponding value in the second vector. If they are the same, score `0`. If they are different, score `1`. The distance between the vectors is the sum of the bitwise comparisons. 43 | 44 | ``` 45 | [1,1,0,1,0,1,0,1] 46 | [1,1,0,0,1,1,0,1] 47 | ----------------- 48 | 0+0+0+1+1+0+0+0 = 2 49 | ``` 50 | 51 |
52 | 53 | Efficient BQ processing compliments our HNSW index to enable mind dizzyingly fast vector search and throughput! 54 | 55 | Let’s talk numbers! You might know from our v1.23 release that combining a [flat index](/developers/weaviate/concepts/vector-index#flat-index) with BQ enables real-time search capabilities. Even so, a brute force a search on 1 million vectors, where each vector has 768 dimensions, takes about 23 milliseconds with a flat index. When you use HNSW and BQ, even with vectors six times as big (4608 dimensions), the same brute force search only takes about 90 milliseconds. 56 | 57 | A flat index combined with BQ is fast, but compare a HNSW index combined with BQ. Together HNSW and BQ enable vector search capabilities that reach nearly 10,000 queries per second at 85% recall! 58 | 59 | Keep an eye out for an upcoming blog post with the nitty-gritty details of how HNSW+BQ compares to other index and compression algorithms. 60 | 61 | ## Simplified Docker configuration 62 | 63 | Starting in v1.24, there is a new Docker image that runs using common, default values. This image lets you start development and exploration quickly. Just enter a simple command line: 64 | 65 | ```bash 66 | docker run -p 8080:8080 -p 50051:50051 cr.weaviate.io/semitechnologies/weaviate:1.24.1 67 | ``` 68 | 69 | Alternatively, you can edit the `docker-compose.yml` file to customize your instance. 70 | 71 | For details, see [Docker configuration](/developers/weaviate/installation/docker-compose) 72 | 73 | ## Backend improvements 74 | 75 | Weaviate is a robust platform that handles many kinds of workloads. This release brings major improvements for imports, updates, and deletions. You don't have to do anything to enable these fixes, but you may notice improved upload times, especially if you make frequent changes to your collections. 76 | 77 | ## Python client 78 | 79 | The new Weaviate [Python client v4](/developers/weaviate/client-libraries/python) is available. The latest release adds support for named vectors and brings numerous other updates. If you have feedback on the new client, please let us know. 80 | 81 | - To learn more about the new client, read the [client release blog post](/blog/py-client-v4-release). 82 | - To learn how to upgrade your code, see the [v3 to v4 migration guide](/developers/weaviate/client-libraries/python/v3_v4_migration). 83 | 84 | ## Summary 85 | 86 | Enjoy the new features and improvements in Weaviate `1.24`. This release is available as a docker image and on [WCS](https://console.weaviate.cloud/). Try it out in a free, WCS sandbox, or download a copy and try it out locally. 87 | 88 | Thanks for reading, see you next time 👋! 89 | -------------------------------------------------------------------------------- /blog_schema.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | from pydantic import BaseModel 3 | 4 | class BlogPost(BaseModel): 5 | question: Optional[str] = None 6 | title: Optional[str] = None 7 | introduction_paragraph: Optional[str] = None 8 | outline: Optional[str] = None 9 | evidence_paragraphs: Optional[List[str]] = [] 10 | bold_prediction: Optional[str] = None 11 | weaviate_relevance: Optional[str] = None 12 | takeaways: Optional[List[str]] = [] -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.4' 3 | services: 4 | weaviate: 5 | command: 6 | - --host 7 | - 0.0.0.0 8 | - --port 9 | - '8080' 10 | - --scheme 11 | - http 12 | image: semitechnologies/weaviate:1.23.7 13 | ports: 14 | - 8080:8080 15 | - 50051:50051 16 | restart: on-failure:0 17 | environment: 18 | OPENAI_APIKEY: 'sk-foobar' 19 | QUERY_DEFAULTS_LIMIT: 25 20 | AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' 21 | PERSISTENCE_DATA_PATH: '/var/lib/weaviate' 22 | DEFAULT_VECTORIZER_MODULE: 'text2vec-openai' 23 | ENABLE_MODULES: 'text2vec-openai' 24 | CLUSTER_HOSTNAME: 'node1' 25 | ... -------------------------------------------------------------------------------- /hurricane-frontend/README.md: -------------------------------------------------------------------------------- 1 | # Getting Started with Create React App 2 | 3 | This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app). 4 | 5 | ## Available Scripts 6 | 7 | In the project directory, you can run: 8 | 9 | ## `npm install` 10 | 11 | Install the node_modules needed to run Hurricane. 12 | 13 | ### `npm start` 14 | 15 | Runs the app in the development mode.\ 16 | Open [http://localhost:3000](http://localhost:3000) to view it in your browser. 17 | 18 | The page will reload when you make changes.\ 19 | You may also see any lint errors in the console. 20 | 21 | ### `npm test` 22 | 23 | Launches the test runner in the interactive watch mode.\ 24 | See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information. 25 | 26 | ### `npm run build` 27 | 28 | Builds the app for production to the `build` folder.\ 29 | It correctly bundles React in production mode and optimizes the build for the best performance. 30 | 31 | The build is minified and the filenames include the hashes.\ 32 | Your app is ready to be deployed! 33 | 34 | See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information. 35 | 36 | ### `npm run eject` 37 | 38 | **Note: this is a one-way operation. Once you `eject`, you can't go back!** 39 | 40 | If you aren't satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project. 41 | 42 | Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you're on your own. 43 | 44 | You don't have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn't feel obligated to use this feature. However we understand that this tool wouldn't be useful if you couldn't customize it when you are ready for it. 45 | 46 | ## Learn More 47 | 48 | You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started). 49 | 50 | To learn React, check out the [React documentation](https://reactjs.org/). 51 | 52 | ### Code Splitting 53 | 54 | This section has moved here: [https://facebook.github.io/create-react-app/docs/code-splitting](https://facebook.github.io/create-react-app/docs/code-splitting) 55 | 56 | ### Analyzing the Bundle Size 57 | 58 | This section has moved here: [https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size](https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size) 59 | 60 | ### Making a Progressive Web App 61 | 62 | This section has moved here: [https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app](https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app) 63 | 64 | ### Advanced Configuration 65 | 66 | This section has moved here: [https://facebook.github.io/create-react-app/docs/advanced-configuration](https://facebook.github.io/create-react-app/docs/advanced-configuration) 67 | 68 | ### Deployment 69 | 70 | This section has moved here: [https://facebook.github.io/create-react-app/docs/deployment](https://facebook.github.io/create-react-app/docs/deployment) 71 | 72 | ### `npm run build` fails to minify 73 | 74 | This section has moved here: [https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify](https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify) 75 | -------------------------------------------------------------------------------- /hurricane-frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hurricane-frontend", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "@testing-library/jest-dom": "^5.17.0", 7 | "@testing-library/react": "^13.4.0", 8 | "@testing-library/user-event": "^13.5.0", 9 | "axios": "^1.6.7", 10 | "react": "^18.2.0", 11 | "react-dom": "^18.2.0", 12 | "react-scripts": "5.0.1", 13 | "react-transition-group": "^4.4.5", 14 | "web-vitals": "^2.1.4" 15 | }, 16 | "scripts": { 17 | "start": "react-scripts start", 18 | "build": "react-scripts build", 19 | "test": "react-scripts test", 20 | "eject": "react-scripts eject" 21 | }, 22 | "eslintConfig": { 23 | "extends": [ 24 | "react-app", 25 | "react-app/jest" 26 | ] 27 | }, 28 | "browserslist": { 29 | "production": [ 30 | ">0.2%", 31 | "not dead", 32 | "not op_mini all" 33 | ], 34 | "development": [ 35 | "last 1 chrome version", 36 | "last 1 firefox version", 37 | "last 1 safari version" 38 | ] 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /hurricane-frontend/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/Hurricane/e6a9daf82bda9b388854a3e7d407e1b924954da3/hurricane-frontend/public/favicon.ico -------------------------------------------------------------------------------- /hurricane-frontend/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 27 | React App 28 | 29 | 30 | 31 |
32 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /hurricane-frontend/public/logo192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/Hurricane/e6a9daf82bda9b388854a3e7d407e1b924954da3/hurricane-frontend/public/logo192.png -------------------------------------------------------------------------------- /hurricane-frontend/public/logo512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/Hurricane/e6a9daf82bda9b388854a3e7d407e1b924954da3/hurricane-frontend/public/logo512.png -------------------------------------------------------------------------------- /hurricane-frontend/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | }, 10 | { 11 | "src": "logo192.png", 12 | "type": "image/png", 13 | "sizes": "192x192" 14 | }, 15 | { 16 | "src": "logo512.png", 17 | "type": "image/png", 18 | "sizes": "512x512" 19 | } 20 | ], 21 | "start_url": ".", 22 | "display": "standalone", 23 | "theme_color": "#000000", 24 | "background_color": "#ffffff" 25 | } 26 | -------------------------------------------------------------------------------- /hurricane-frontend/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /hurricane-frontend/src/App.css: -------------------------------------------------------------------------------- 1 | /* App.css */ 2 | .app { 3 | margin: -2vh auto; 4 | padding: 0.5vh 10vh; /* height width */ 5 | text-align: center; 6 | height: 400vh; 7 | } 8 | 9 | h1 { 10 | text-align: center; 11 | font-size: 7vh; 12 | font-family: 'Montserrat'; 13 | color: #092844; 14 | } 15 | 16 | .input-container { 17 | display: flex; 18 | justify-content: space-between; 19 | margin-bottom: 20px; 20 | } 21 | 22 | input { 23 | margin-top: -1vh; 24 | flex: 1; 25 | height: 4vh; 26 | padding: 10px; 27 | font-size: 3vh; 28 | font-family: 'Montserrat'; 29 | color: #092844; 30 | } 31 | 32 | button { 33 | margin-top: -1vh; 34 | padding: 10px 20px; 35 | font-size: 3vh; 36 | font-family: 'Montserrat'; 37 | background-color: #092844; 38 | color: white; 39 | font-weight: bold; 40 | border: none; 41 | box-shadow: 0.5vh 0.5vh 0.5vh #092844; 42 | } 43 | 44 | button:hover { 45 | color: #092844; 46 | background-color: white; 47 | transition: 0.2s; 48 | cursor: pointer; 49 | box-shadow: 0 0 0 white; 50 | } 51 | 52 | .content-container { 53 | display: flex; 54 | } 55 | 56 | .blog-container h2 { 57 | text-align: center; 58 | } 59 | 60 | .blog-container h3 { 61 | text-align: left; 62 | } 63 | 64 | .blog-container { 65 | flex: 1; 66 | padding-right: 20px; 67 | text-align: left; 68 | } 69 | .blog-container h2 { 70 | font-size: 3vh; 71 | font-family: 'Montserrat'; 72 | color: #092844; 73 | } 74 | 75 | .thoughts-container h2 { 76 | font-size: 3vh; 77 | font-family: 'Montserrat'; 78 | color: #092844; 79 | } 80 | 81 | .blog { 82 | border: 1px solid #ccc; 83 | padding: 10px; 84 | } 85 | 86 | .blogstyling:hover { 87 | color: #092844; 88 | background-color: white; 89 | } 90 | 91 | .blogstyling { 92 | border: 0.2vh solid #092844; 93 | padding: 0vh 2vh; 94 | background-color: white; 95 | } 96 | 97 | .blogstyling h1 { 98 | font-size: 3vh; 99 | } 100 | 101 | .right-container { 102 | flex: 1; 103 | } 104 | 105 | .thoughts-container { 106 | margin-bottom: 20px; 107 | } 108 | 109 | .thought { 110 | border: 1px solid #ccc; 111 | text-align: left; 112 | padding: 10px; 113 | margin-bottom: 10px; 114 | font-family: 'Montserrat'; 115 | color: #092844; 116 | font-size: 3vh; 117 | background-color: white; 118 | border: solid 0.2vh #092844; 119 | transition: 0.5s; 120 | } 121 | 122 | .thought:hover { 123 | background-color: #092844; 124 | color: white; 125 | cursor: pointer; 126 | } 127 | 128 | .status-container { 129 | margin-top: -4vh; 130 | padding: 10px; 131 | } 132 | 133 | .status-container h2 { 134 | color: #092844; 135 | font-size: 3.4vh; 136 | } 137 | 138 | .WeaviateStatus { 139 | display: flex; 140 | flex-direction: row; 141 | justify-content: center; 142 | } 143 | 144 | .DataTable h2 { 145 | color: #092844; 146 | } 147 | 148 | .DataTable { 149 | text-align: center; 150 | margin: 0vh 2vw; 151 | background-color: white; 152 | font-size: 2vh; 153 | padding: 0 2vh; 154 | color: #092844; 155 | border-radius: 2vh; 156 | border: solid 0.2vh #092844; 157 | transition: 0.5s; 158 | } 159 | 160 | .DataTable:hover { 161 | background-color: #092844; 162 | color: white; 163 | cursor: pointer; 164 | } 165 | 166 | .DataTable h3 { 167 | text-align: left; 168 | } 169 | 170 | .fade-enter { 171 | opacity: 0; 172 | } 173 | 174 | .fade-enter-active { 175 | opacity: 1; 176 | transition: opacity 800ms ease-in; 177 | } 178 | 179 | .fade-exit { 180 | opacity: 1; 181 | } 182 | 183 | .fade-exit-active { 184 | opacity: 0; 185 | transition: opacity 800ms ease-in; 186 | } -------------------------------------------------------------------------------- /hurricane-frontend/src/App.js: -------------------------------------------------------------------------------- 1 | // App.js 2 | import React, { useState, useRef } from 'react'; 3 | import axios from 'axios'; 4 | import backgroundImage from './assets/bg-light-3.png'; 5 | import { CSSTransition, TransitionGroup } from 'react-transition-group'; 6 | import './App.css'; 7 | import BlogPost from './Blogpost'; 8 | 9 | const App = () => { 10 | const [thoughts, setThoughts] = useState(["Waiting for a question or topic."]); 11 | const [blog, setBlog] = useState(''); 12 | const [inputValue, setInputValue] = useState(''); 13 | const blogRef = useRef(null); 14 | 15 | const handleClick = async () => { 16 | try { 17 | // Call the /writeblog endpoint 18 | setThoughts(prevThoughts => ["Received question: " + String(inputValue)]); 19 | // Create Session 20 | const createSessionResponse = await axios.post('http://localhost:8000/create-session'); 21 | const sessionId = String(createSessionResponse.data.session_id); 22 | 23 | console.log(inputValue); 24 | console.log(sessionId); 25 | const writeBlogResponse = await axios.post('http://localhost:8000/question-to-blog', { 26 | question: inputValue, 27 | sessionId: sessionId 28 | }); 29 | setThoughts(prevThoughts => [...prevThoughts, writeBlogResponse.data.thoughts]); 30 | 31 | // Call the /researchtopic endpoint 32 | const researchTopicResponse = await axios.post('http://localhost:8000/topic-to-paragraph', { 33 | sessionId: sessionId 34 | }); 35 | setThoughts(prevThoughts => [...prevThoughts, researchTopicResponse.data.thoughts]); 36 | 37 | // Call the /finishtask endpoint 38 | const boldPrediction = await axios.post('http://localhost:8000/bold-prediction', { 39 | sessionId: sessionId 40 | }); 41 | setThoughts(prevThoughts => [...prevThoughts, boldPrediction.data.thoughts]); 42 | 43 | const weaviateRelevance = await axios.post('http://localhost:8000/weaviate-relevance', { 44 | sessionId: sessionId 45 | }); 46 | setThoughts(prevThoughts => [...prevThoughts, weaviateRelevance.data.thoughts]); 47 | 48 | const finishTaskResponse = await axios.post('http://localhost:8000/finish-blog', { 49 | sessionId: sessionId 50 | }); 51 | setBlog(finishTaskResponse.data.blog); 52 | } catch (error) { 53 | console.error('Error:', error); 54 | } 55 | }; 56 | 57 | return ( 58 |
59 |

Hurricane

60 |
61 | setInputValue(e.target.value)} 65 | placeholder="Enter a question or topic..." 66 | /> 67 | 68 |
69 |
70 |
71 |

Generated Blog

72 | 73 | 74 | 75 |
76 |
77 |
78 |

Hurricane Updates

79 | 80 | {thoughts.slice().reverse().map((thought, index) => ( 81 | 82 |
{thought}
83 | 84 | ))} 85 | 86 |
87 |
88 |

Database Status

89 |
90 |
91 |

Original Blog Count

92 |

79 Blogs

93 |

1182 Blog Chunks

94 |
95 |
96 |

Generated Blog Count

97 |

0 Blogs

98 |

0 Blog Chunks

99 |
100 |
101 |
102 |
103 |
104 |
105 | ); 106 | }; 107 | 108 | export default App; -------------------------------------------------------------------------------- /hurricane-frontend/src/App.test.js: -------------------------------------------------------------------------------- 1 | import { render, screen } from '@testing-library/react'; 2 | import App from './App'; 3 | 4 | test('renders learn react link', () => { 5 | render(); 6 | const linkElement = screen.getByText(/learn react/i); 7 | expect(linkElement).toBeInTheDocument(); 8 | }); 9 | -------------------------------------------------------------------------------- /hurricane-frontend/src/Blogpost.js: -------------------------------------------------------------------------------- 1 | // BlogPost.js 2 | import React from 'react'; 3 | 4 | const BlogPost = ({ blogPost }) => { 5 | if (!blogPost) { 6 | return null; 7 | } 8 | 9 | return ( 10 |
11 | {blogPost.title &&

{blogPost.title}

} 12 | {blogPost.introduction_paragraph &&

{blogPost.introduction_paragraph}

} 13 | {blogPost.evidence_paragraphs && blogPost.evidence_paragraphs.length > 0 && ( 14 |
15 | {blogPost.evidence_paragraphs.map((paragraph, index) => ( 16 |

{paragraph}

17 | ))} 18 |
19 | )} 20 | {blogPost.bold_prediction && ( 21 |
22 |

Bold Prediction

23 |

{blogPost.bold_prediction}

24 |
25 | )} 26 | {blogPost.weaviate_relevance && ( 27 |
28 |

Relevance to Weaviate

29 |

{blogPost.weaviate_relevance}

30 |
31 | )} 32 | {blogPost.takeaways && blogPost.takeaways.length > 0 && ( 33 |
34 |

Takeaways from {blogPost.title}:

35 |
    36 | {blogPost.takeaways.map((takeaway, index) => ( 37 |
  • {takeaway}
    • 38 | ))} 39 |
40 |
41 | )} 42 |
43 | ); 44 | }; 45 | 46 | export default BlogPost; -------------------------------------------------------------------------------- /hurricane-frontend/src/assets/bg-light-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaviate-tutorials/Hurricane/e6a9daf82bda9b388854a3e7d407e1b924954da3/hurricane-frontend/src/assets/bg-light-3.png -------------------------------------------------------------------------------- /hurricane-frontend/src/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 4 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', 5 | sans-serif; 6 | -webkit-font-smoothing: antialiased; 7 | -moz-osx-font-smoothing: grayscale; 8 | } 9 | 10 | code { 11 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', 12 | monospace; 13 | } 14 | -------------------------------------------------------------------------------- /hurricane-frontend/src/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom/client'; 3 | import './index.css'; 4 | import App from './App'; 5 | import reportWebVitals from './reportWebVitals'; 6 | 7 | const root = ReactDOM.createRoot(document.getElementById('root')); 8 | root.render( 9 | 10 | 11 | 12 | ); 13 | 14 | // If you want to start measuring performance in your app, pass a function 15 | // to log results (for example: reportWebVitals(console.log)) 16 | // or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals 17 | reportWebVitals(); 18 | -------------------------------------------------------------------------------- /hurricane-frontend/src/logo.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /hurricane-frontend/src/reportWebVitals.js: -------------------------------------------------------------------------------- 1 | const reportWebVitals = onPerfEntry => { 2 | if (onPerfEntry && onPerfEntry instanceof Function) { 3 | import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { 4 | getCLS(onPerfEntry); 5 | getFID(onPerfEntry); 6 | getFCP(onPerfEntry); 7 | getLCP(onPerfEntry); 8 | getTTFB(onPerfEntry); 9 | }); 10 | } 11 | }; 12 | 13 | export default reportWebVitals; 14 | -------------------------------------------------------------------------------- /hurricane-frontend/src/setupTests.js: -------------------------------------------------------------------------------- 1 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 2 | // allows you to do things like: 3 | // expect(element).toHaveTextContent(/react/i) 4 | // learn more: https://github.com/testing-library/jest-dom 5 | import '@testing-library/jest-dom'; 6 | -------------------------------------------------------------------------------- /hurricane_program.py: -------------------------------------------------------------------------------- 1 | import dspy 2 | from hurricane_signatures import Question2BlogOutline, Topic2Paragraph, BoldPrediction, WeaviateRelevance, TitleAndTakeaways 3 | from utils import format_weaviate_and_you_contexts 4 | from utils import format_blog_draft, format_blog_post, BlogPost 5 | 6 | class Hurricane(dspy.Module): 7 | def __init__(self, you_rm): 8 | # 5 LLM Layers (Question2BlogOutline, Topic2Paragraph, BoldPrediction, WeaviateRelevance, TitleAndTakeaways) 9 | # 2 Retrieval Engines (Weaviate and You) 10 | 11 | self.question_to_blog_outline = dspy.ChainOfThought(Question2BlogOutline) 12 | self.topic_to_paragraph = dspy.ChainOfThought(Topic2Paragraph) 13 | self.bold_prediction = dspy.ChainOfThought(BoldPrediction) 14 | self.weaviate_relevance = dspy.ChainOfThought(WeaviateRelevance) 15 | self.title_and_key_takeaways = dspy.ChainOfThought(TitleAndTakeaways) 16 | self.you_rm = you_rm 17 | 18 | def forward(self, question): 19 | blog_container = BlogPost() 20 | blog_contexts = dspy.Retrieve(k=5)(question).passages 21 | web_contexts = self.you_rm(question) 22 | blog_contexts, web_contexts = format_weaviate_and_you_contexts(blog_contexts, web_contexts) 23 | question_to_blog_outline_outputs = self.question_to_blog_outline(question=question, blog_context=blog_contexts, web_context=web_contexts) 24 | blog_container.outline = question_to_blog_outline_outputs.blog_outline 25 | parsed_blog_outline = blog_container.outline.split(",") 26 | blog_container.introduction_paragraph = question_to_blog_outline_outputs.introduction_paragraph 27 | for topic in parsed_blog_outline: 28 | blog_contexts = dspy.Retrieve(k=5)(topic).passages 29 | web_contexts = self.you_rm(topic) 30 | blog_contexts, web_contexts = format_weaviate_and_you_contexts(blog_contexts, web_contexts) 31 | blog_container.evidence_paragraphs.append(self.topic_to_paragraph(topic=topic, original_question=question, web_contexts=web_contexts, blog_contexts=blog_contexts).paragraph) 32 | blog = format_blog_draft(blog_container) 33 | blog_container.bold_prediction = self.bold_prediction(blog=blog).bold_prediction 34 | blog_contexts = dspy.Retrieve(k=8)("What technology does Weaviate build?").passages 35 | blog_contexts = "".join(blog_contexts) 36 | blog_container.weaviate_relevance = self.weaviate_relevance(blog_contexts=blog_contexts, blog_post=blog).weaviate_relevance 37 | title_and_takeaways = self.title_and_key_takeaways(blog=blog, original_question=question) 38 | blog_container.title = title_and_takeaways.title 39 | blog_container.takeaways = title_and_takeaways.key_takeaways 40 | 41 | final_blog = format_blog_post(blog_container) 42 | return dspy.Prediction(blog=final_blog) -------------------------------------------------------------------------------- /hurricane_signatures.py: -------------------------------------------------------------------------------- 1 | import dspy 2 | 3 | class Question2BlogOutline(dspy.Signature): 4 | """Your task is to write a Weaviate blog post that will help answer the given question.\nPlease use the contexts from a web search and published Weaviate blog posts to evaluate the structure of the blog post.""" 5 | 6 | question = dspy.InputField() 7 | blog_context = dspy.InputField() 8 | web_context = dspy.InputField() 9 | blog_outline = dspy.OutputField(desc="A list of topics the blog will cover. IMPORTANT!! This must follow a comma separated list of values!") 10 | introduction_paragraph = dspy.OutputField(desc="An introduction overview of the blog post that previews the topics and presents the question the blog post seeks to answer.") 11 | 12 | class Topic2Paragraph(dspy.Signature): 13 | """Please write a paragraph that explains a topic based on contexts fom a web search and blog posts from an authoritative source. You are also given the original question that inspired research into this topic, please try to connect your review of the topic to the original question.""" 14 | 15 | topic = dspy.InputField(desc="A topic to write a paragraph about based on the information in the contexts.") 16 | original_question = dspy.InputField(desc="The original question that inspired research into this topic.") 17 | web_contexts = dspy.InputField(desc="Contains relevant information abuot the topic from a web search.") 18 | blog_contexts = dspy.InputField(desc="Contains relevant information about the topic from the published Weaviate blogs.") 19 | paragraph = dspy.OutputField() 20 | 21 | class BoldPrediction(dspy.Signature): 22 | """Please review this blog post and propose a bold prediction about it's content.""" 23 | 24 | blog = dspy.InputField() 25 | bold_prediction = dspy.OutputField(desc="A bold prediction about it's content.") 26 | 27 | class WeaviateRelevance(dspy.Signature): 28 | """Please review this blog post and describe why it's content and the claims it's making are relevant for the development of the Weaviate Vector Database. You are given additional contexts describing what Weaviate is and some aspects of it's technology.""" 29 | 30 | blog_contexts = dspy.InputField(desc="Content describing what Weaviate is") 31 | blog_post = dspy.InputField() 32 | weaviate_relevance = dspy.OutputField() 33 | 34 | class TitleAndTakeaways(dspy.Signature): 35 | """Write a title and key takeaways for a blog post given the blog post and the original question it sought to answer as input and a bold prediction the author discovered after conducting initial research on the topic.""" 36 | 37 | blog = dspy.InputField() 38 | original_question = dspy.InputField() 39 | title = dspy.OutputField() 40 | key_takeaways = dspy.OutputField(desc="A list of key takeaways from the blog. IMPORTANT!! This must follow a comma separated list of values!") -------------------------------------------------------------------------------- /import_blogs.py: -------------------------------------------------------------------------------- 1 | # Import Weaviate and Connect to Client 2 | import weaviate 3 | client = weaviate.Client("http://localhost:8080") 4 | 5 | # Create Schema 6 | schema = { 7 | "classes": [ 8 | { 9 | "class": "WeaviateBlogChunk", 10 | "description": "A snippet from a Weaviate blogpost.", 11 | "moduleConfig": { 12 | "text2vec-openai": { 13 | "skip": False, 14 | "vectorizeClassName": False, 15 | "vectorizePropertyName": False 16 | }, 17 | "generative-openai": { 18 | "model": "gpt-3.5-turbo" 19 | } 20 | }, 21 | "vectorIndexType": "hnsw", 22 | "vectorizer": "text2vec-openai", 23 | "properties": [ 24 | { 25 | "name": "content", 26 | "dataType": ["text"], 27 | "description": "The text content of the podcast clip", 28 | "moduleConfig": { 29 | "text2vec-transformers": { 30 | "skip": False, 31 | "vectorizePropertyName": False, 32 | "vectorizeClassName": False 33 | } 34 | } 35 | }, 36 | { 37 | "name": "author", 38 | "dataType": ["text"], 39 | "description": "The author of the blog post.", 40 | "moduleConfig": { 41 | "text2vec-openai": { 42 | "skip": True, 43 | "vectorizePropertyName": False, 44 | "vectorizeClassName": False 45 | } 46 | } 47 | } 48 | ] 49 | } 50 | ] 51 | } 52 | 53 | client.schema.create(schema) 54 | 55 | import os 56 | import re 57 | 58 | def chunk_list(lst, chunk_size): 59 | """Break a list into chunks of the specified size.""" 60 | return [lst[i:i + chunk_size] for i in range(0, len(lst), chunk_size)] 61 | 62 | def split_into_sentences(text): 63 | """Split text into sentences using regular expressions.""" 64 | sentences = re.split(r'(?= 3.6.0 100 | callback=weaviate.util.check_batch_result, 101 | ) 102 | 103 | from weaviate.util import get_valid_uuid 104 | from uuid import uuid4 105 | import time 106 | start = time.time() 107 | for idx, blog_chunk in enumerate(blog_chunks): 108 | data_properties = { 109 | "content": blog_chunk 110 | } 111 | id = get_valid_uuid(uuid4()) 112 | with client.batch as batch: 113 | batch.add_data_object( 114 | data_properties, 115 | "WeaviateBlogChunk" 116 | ) 117 | ''' 118 | client.data_object.create( 119 | data_object = data_properties, 120 | class_name = "WeaviateBlogChunk", 121 | uuid=id 122 | ) 123 | ''' 124 | 125 | print(f"Uploaded {idx} documents in {time.time() - start} seconds.") 126 | -------------------------------------------------------------------------------- /init_schema.py: -------------------------------------------------------------------------------- 1 | import weaviate 2 | import weaviate.classes as wvc 3 | 4 | client = weaviate.connect_to_local() 5 | 6 | client.collections.create( 7 | name="Blog", 8 | description="An AI-generated blog post by Hurricane.", 9 | properties=[ 10 | wvc.config.Property( 11 | data_type=wvc.config.DataType.TEXT, 12 | description="Title of the Article", 13 | name="title", 14 | ), 15 | wvc.config.Property( 16 | data_type=wvc.config.DataType.TEXT, 17 | description="Initial question that inspired the blog post.", 18 | name="question", 19 | ), 20 | wvc.config.Property( 21 | data_type=wvc.config.DataType.TEXT, 22 | description="The introduction paragraph", 23 | name="introduction_paragraph", 24 | ), 25 | wvc.config.Property( 26 | data_type=wvc.config.DataType.TEXT, 27 | description="The outline for the blog.", 28 | name="outline", 29 | ), 30 | wvc.config.Property( 31 | data_type=wvc.config.DataType.TEXT_ARRAY, 32 | description="Evidence paragraphs that support the arguments in the blog.", 33 | name="evidence_paragraphs" 34 | ), 35 | wvc.config.Property( 36 | data_type=wvc.config.DataType.TEXT, 37 | description="A bold prediciton based on the content of the blog post.", 38 | name="bold_prediction" 39 | ), 40 | wvc.config.Property( 41 | data_type=wvc.config.DataType.TEXT, 42 | description="The relevance of the blog post to the Weaviate Vector Database.", 43 | name="weaviate_relevance", 44 | ), 45 | wvc.config.Property( 46 | data_type=wvc.config.DataType.TEXT_ARRAY, 47 | description="Takeaways from the blog post.", 48 | name="takeaways" 49 | ), 50 | ] 51 | ) 52 | 53 | client.close() -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | def format_weaviate_and_you_contexts(weaviateRM_output, youRM_output): 2 | weaviateRM_output = "".join(weaviateRM_output) 3 | youRM_output = [d['long_text'] for d in youRM_output] 4 | youRM_output = "".join(youRM_output) 5 | return weaviateRM_output, youRM_output 6 | 7 | from typing import List, Optional 8 | from pydantic import BaseModel 9 | 10 | class BlogPost(BaseModel): 11 | question: Optional[str] = None 12 | title: Optional[str] = None 13 | introduction_paragraph: Optional[str] = None 14 | outline: Optional[str] = None 15 | evidence_paragraphs: Optional[List[str]] = [] 16 | bold_prediction: Optional[str] = None 17 | weaviate_relevance: Optional[str] = None 18 | takeaways: Optional[List[str]] = [] 19 | 20 | def format_blog_draft(blog_post: BlogPost) -> str: 21 | blog_draft = "" 22 | for evidence_paragraph in blog_post.evidence_paragraphs: 23 | blog_draft += evidence_paragraph 24 | blog_draft += "\n" 25 | return blog_draft 26 | 27 | 28 | def format_blog_post(blog_post: BlogPost) -> str: 29 | formatted_blog = f"{blog_post.title}\n\n" 30 | formatted_blog += f"{blog_post.introduction_paragraph}\n\n" 31 | 32 | formatted_blog += "Outline:\n" 33 | for i, point in enumerate(blog_post.outline, start=1): 34 | formatted_blog += f"{i}. {point}\n" 35 | formatted_blog += "\n" 36 | 37 | formatted_blog += "Evidence Paragraphs:\n" 38 | for paragraph in blog_post.evidence_paragraphs: 39 | formatted_blog += f"{paragraph}\n\n" 40 | 41 | formatted_blog += f"Bold Prediction: {blog_post.bold_prediction}\n\n" 42 | formatted_blog += f"Weaviate Relevance: {blog_post.weaviate_relevance}\n\n" 43 | 44 | formatted_blog += "Takeaways:\n" 45 | for i, takeaway in enumerate(blog_post.takeaways, start=1): 46 | formatted_blog += f"{i}. {takeaway}\n" 47 | 48 | return formatted_blog --------------------------------------------------------------------------------