├── requirements.txt ├── embedding_create.py ├── astra_create.py ├── doc_chunker.py ├── astra_insert.py ├── .gitignore ├── astra_query.py ├── README.MD └── towns └── shadowfen.txt /requirements.txt: -------------------------------------------------------------------------------- 1 | astrapy>=2.0,<3.0 2 | transformers>=4.51 3 | torch>=2.6.0 4 | -------------------------------------------------------------------------------- /embedding_create.py: -------------------------------------------------------------------------------- 1 | from transformers import AutoModel 2 | 3 | 4 | def create_embeddings(paragraphs): 5 | model = AutoModel.from_pretrained( 6 | "jinaai/jina-embeddings-v2-base-en", trust_remote_code=True 7 | ) 8 | 9 | embeddings = model.encode(paragraphs) 10 | return embeddings 11 | -------------------------------------------------------------------------------- /astra_create.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from astrapy import DataAPIClient 4 | from astrapy.info import CollectionDefinition 5 | 6 | # Fetching necessary environment variables for AstraDB configuration 7 | ASTRA_DB_APPLICATION_TOKEN = os.environ["ASTRA_DB_APPLICATION_TOKEN"] 8 | ASTRA_DB_API_ENDPOINT = os.environ["ASTRA_DB_API_ENDPOINT"] 9 | ASTRA_DB_KEYSPACE = os.environ.get("ASTRA_DB_KEYSPACE") 10 | COLLECTION_NAME = "town_content" 11 | 12 | # Initialize connection to Astra DB 13 | client = DataAPIClient() 14 | db = client.get_database( 15 | ASTRA_DB_API_ENDPOINT, 16 | token=ASTRA_DB_APPLICATION_TOKEN, 17 | keyspace=ASTRA_DB_KEYSPACE, 18 | ) 19 | 20 | # Create collection 21 | db.create_collection( 22 | COLLECTION_NAME, 23 | definition=(CollectionDefinition.builder().set_vector_dimension(768).build()), 24 | ) 25 | -------------------------------------------------------------------------------- /doc_chunker.py: -------------------------------------------------------------------------------- 1 | def read_file(file_path): 2 | """ 3 | Reads the entire content of a file and returns it as a string. 4 | """ 5 | with open(file_path, "r") as file: 6 | return file.read() 7 | 8 | 9 | def chunk_text_by_paragraphs(text): 10 | """ 11 | This function takes a string of text and returns a list of paragraphs. 12 | A paragraph is defined as a string of text that is separated by two or more newlines. 13 | """ 14 | # Split the text by two or more newlines to separate paragraphs 15 | paragraphs = text.split("\n\n") 16 | 17 | # Filter out any empty strings that may result from extra newlines 18 | paragraphs = [para.strip() for para in paragraphs if para.strip()] 19 | 20 | return paragraphs 21 | 22 | 23 | def chunk_file(file_path): 24 | """ 25 | This function takes a file path, reads the file, and chunks it into paragraphs. 26 | It returns an array of paragraphs. 27 | """ 28 | # Read the content of the file 29 | text_content = read_file(file_path) 30 | 31 | # Chunk the text by paragraphs 32 | paragraphs = chunk_text_by_paragraphs(text_content) 33 | 34 | return paragraphs 35 | -------------------------------------------------------------------------------- /astra_insert.py: -------------------------------------------------------------------------------- 1 | import os 2 | import uuid 3 | 4 | from astrapy import DataAPIClient 5 | 6 | from doc_chunker import chunk_file 7 | from embedding_create import create_embeddings 8 | 9 | # Fetching necessary environment variables for AstraDB configuration 10 | ASTRA_DB_APPLICATION_TOKEN = os.environ["ASTRA_DB_APPLICATION_TOKEN"] 11 | ASTRA_DB_API_ENDPOINT = os.environ["ASTRA_DB_API_ENDPOINT"] 12 | ASTRA_DB_KEYSPACE = os.environ.get("ASTRA_DB_KEYSPACE") 13 | COLLECTION_NAME = "town_content" 14 | 15 | 16 | # Initialize connection to Astra DB 17 | client = DataAPIClient() 18 | db = client.get_database( 19 | ASTRA_DB_API_ENDPOINT, 20 | token=ASTRA_DB_APPLICATION_TOKEN, 21 | keyspace=ASTRA_DB_KEYSPACE, 22 | ) 23 | 24 | # Chunk the sample file into paragraphs 25 | paragraphs = chunk_file("./towns/shadowfen.txt") 26 | 27 | # Create embeddings for each paragraph 28 | embeddings_list = create_embeddings(paragraphs) 29 | 30 | documents = [] # Initialize an empty list to hold document dictionaries 31 | 32 | for embeddings, paragraph in zip(embeddings_list, paragraphs): 33 | # Create a dictionary for the current document 34 | document = { 35 | "_id": uuid.uuid4(), 36 | "text": paragraph, 37 | "$vector": embeddings.tolist(), 38 | } 39 | 40 | # Append the document dictionary to the list 41 | documents.append(document) 42 | 43 | # Get (an astrapy reference to) the db collection 44 | collection = db.get_collection(name=COLLECTION_NAME) 45 | 46 | # Insert the documents 47 | res = collection.insert_many(documents=documents) 48 | 49 | print(f"Inserted {len(res.inserted_ids)} chunks.") 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | junit/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # pyenv 81 | .python-version 82 | 83 | # celery beat schedule file 84 | celerybeat-schedule 85 | 86 | # SageMath parsed files 87 | *.sage.py 88 | 89 | # Environments 90 | .env 91 | .venv 92 | env/ 93 | venv/ 94 | ENV/ 95 | env.bak/ 96 | venv.bak/ 97 | myenv/ 98 | 99 | # Spyder project settings 100 | .spyderproject 101 | .spyproject 102 | 103 | # Rope project settings 104 | .ropeproject 105 | 106 | # mkdocs documentation 107 | /site 108 | 109 | # mypy 110 | .mypy_cache/ 111 | .dmypy.json 112 | dmypy.json 113 | 114 | # Pyre type checker 115 | .pyre/ 116 | 117 | # pytype static type analyzer 118 | .pytype/ 119 | 120 | # Cython debug symbols 121 | cython_debug/ 122 | 123 | # VSCode 124 | .vscode/ 125 | *.code-workspace 126 | 127 | # Jupyter 128 | .notebooks/ 129 | -------------------------------------------------------------------------------- /astra_query.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from astrapy import DataAPIClient 4 | 5 | from embedding_create import create_embeddings 6 | 7 | # Fetching necessary environment variables for AstraDB configuration 8 | ASTRA_DB_APPLICATION_TOKEN = os.environ["ASTRA_DB_APPLICATION_TOKEN"] 9 | ASTRA_DB_API_ENDPOINT = os.environ["ASTRA_DB_API_ENDPOINT"] 10 | ASTRA_DB_KEYSPACE = os.environ.get("ASTRA_DB_KEYSPACE") 11 | COLLECTION_NAME = "town_content" 12 | 13 | # Preparing a list of queries about the town Shadowfen 14 | queries = [ 15 | "What are the locations within Shadowfen?", 16 | "Who is Eldermarsh Thorne?", 17 | "Who is Brom Stoutfist?", 18 | "What is The Gloomwater Brewery?", 19 | "What is the terrain like surrounding Shadowfen?", 20 | "Who created Shadowfen?", 21 | "What is the climate of Shadowfen?", 22 | "What is the population of Shadowfen?", 23 | "What is the history of Shadowfen?", 24 | "Where can I get a drink in Shadowfen?", 25 | ] 26 | 27 | # Generating embeddings for each query using a custom embedding creation function 28 | embedding_list = create_embeddings(queries) 29 | 30 | # Establishing a connection to Astra DB with the provided credentials and keyspace 31 | client = DataAPIClient() 32 | db = client.get_database( 33 | ASTRA_DB_API_ENDPOINT, 34 | token=ASTRA_DB_APPLICATION_TOKEN, 35 | keyspace=ASTRA_DB_KEYSPACE, 36 | ) 37 | 38 | # Get (an astrapy reference to) the db collection 39 | collection = db.get_collection(name=COLLECTION_NAME) 40 | 41 | # Iterating through each query to perform a similarity search in the database 42 | for embedding, query in zip(embedding_list, queries): 43 | # Executing the find operation on the collection with the specified parameters 44 | search_results = collection.find( 45 | sort={"$vector": embedding.tolist()}, 46 | limit=2, 47 | projection={"text": True}, 48 | include_similarity=True, 49 | ) 50 | 51 | print("=" * 30) 52 | print(f"QUESTION: {query}") 53 | print("-" * 30) 54 | # Iterating through the retrieved documents to print their content 55 | for document in search_results: 56 | print(document["text"]) 57 | print(f" [Similarity: {document['$similarity']:.4f}]") 58 | print("\n") 59 | 60 | print("\n\n") 61 | -------------------------------------------------------------------------------- /README.MD: -------------------------------------------------------------------------------- 1 | # Astra Vector Similarity Search 2 | 3 | This is a simple example of vector similarity search using DataStax Astra DB. 4 | 5 | This repo aims to walk the line between providing a simplified example that is not overwhelmingly complex, but still illustrates key steps you'll need to take to solve real world vector similarity use cases. There are four key use cases that this repo will illustrate: 6 | 7 | - How to create a vector-enabled collection in Astra DB using `astrapy`; 8 | - How to generate vector embeddings using the HuggingFace transformers library and the `jinaai/jina-embeddings-v2-base-en` model; 9 | - How to insert vector embeddings and corresponding text documents in the collection; 10 | - How to run a vector similarity search and manipulate the documents returned from the search. 11 | 12 | # Retrieval Augmented Generation - RAG 13 | 14 | While this demo does not use the Open AI libraries to call an LLM, the patterns here are applicable for building RAG use cases. 15 | 16 | As such, the content that you are chunking and building embeddings for is contained in a text document located in `towns/shadowfen.txt`. This text file was generated by ChatGPT and describes many aspects of a fictional town in a fantasy setting. 17 | 18 | An advantage of this kinf of autogenerated content is that it's fictional and not something that ChatGPT has been trained on: if you ask ChatGPT about the fictional town of Shadowfen, it will tell you it's not a real place and that it doesn't have any information about it. 19 | 20 | It is easy for you to leverage the content in this repository and extend it to build a RAG application if that is your goal. The output from `astra_query.py` is a set of questions about Shadowfen and the most relevant chunks of content from the text file, so you can easily copy and paste the output directly into ChatGPT to get an idea of how well the content helps answer questions and then move on to an API based implementation if you so choose. 21 | 22 | # Prerequisites 23 | 24 | - Create a DataStax Astra account - https://astra.datastax.com 25 | - Create a vector database within Astra 26 | - Get a database accesss token for your database using the Astra UI 27 | - Get the API endpoint for your database (should have a form like: `https://{uuid}-{region}.apps.astra.datastax.com/api/json`) 28 | 29 | See [Create an Astra DB Serverless database](https://docs.datastax.com/en/astra-db-serverless/databases/create-database.html) for more information and documentation on Astra DB. 30 | 31 | # Setup 32 | 33 | Clone the repo, create a virtual environment with Python 3.9+, and activate it. 34 | 35 | In the virtual environment, install the required dependencies: 36 | 37 | ``` 38 | pip install -r requirements.txt 39 | ``` 40 | 41 | Set the following 3 environment variables: 42 | 43 | ``` 44 | export ASTRA_DB_API_ENDPOINT={Replace with your Astra DB API endpoint} 45 | export ASTRA_DB_APPLICATION_TOKEN={Replace with your database token} 46 | export ASTRA_DB_KEYSPACE={Keyspace to use, if omitted uses a default} 47 | ``` 48 | 49 | # Create a collection to store documents with vector embeddings 50 | 51 | From the root directory of the repo, start by executing: 52 | 53 | ``` 54 | python astra_create.py 55 | ``` 56 | 57 | > You can now see the collection in the "Data Explorer" of your [Astra UI](https://astra.datastax.com/). 58 | 59 | # Generate embeddings, populate the collection 60 | 61 | At this point you have a collection called `town_content`: it's time to chunk up our content text file and generate embeddings of each chunk using HuggingFace, then insert them into our collection. 62 | 63 | To do this, run the `astra_insert.py` script from the root directory of the repo: 64 | 65 | ``` 66 | python astra_insert.py 67 | ``` 68 | 69 | If this script completes successfully, it will print the amount of text chunks inserted into the collection along with their embedding vector. 70 | 71 | > You can now peek at the inserted documents with the "Data Explorer" of your [Astra UI](https://astra.datastax.com/). 72 | 73 | # Perform a similarity search 74 | 75 | At this point, you have several dozen items in the collection, each with its embedding vector. 76 | 77 | The `astra_query.py` script has an array of several queries about Shadowfen and will retrieve the most relevant results based on a similarity search of each query. 78 | 79 | You can modify this script to ask different questions and see which document chunks are returned. 80 | 81 | Note that the chunking algorithm used here is fairly naive: it just works at a paragraph level. As an improvement, you may want to consider changing the chunking algorithm to use a recursive strategy or add an overlap with adjacent sentences, to see how the results change. _(This is out of scope for this tutorial, just be aware that there are various chunking strategies which affect the accuracy of the retrieval.)_ 82 | 83 | To run the similarity search, run the `astra_query.py` script from the root directory of the repo: 84 | 85 | ``` 86 | python ./astra_query.py 87 | ``` 88 | 89 | If this runs successfully, you'll see each query printed to the console with the two best-match documents, along with their similarity score: 90 | 91 | ``` 92 | ============================== 93 | QUESTION: Who created Shadowfen? 94 | ------------------------------ 95 | Shadowfen, nestled in the crooked embrace of the Mirewood Forest, has a history [...] 96 | [Similarity: 0.9337] 97 | 98 | 99 | Shadowfen, with its deep connection to ancient magic and the mysterious swamp, [...] 100 | [Similarity: 0.9190] 101 | 102 | ``` 103 | 104 | Feel free to edit the `queries` list in `astra_query.py` if you want to try new questions. 105 | 106 | > You may want to check the contents of [`towns/shadowfen.txt`](towns/shadowfen.txt) to cross-check the results. 107 | -------------------------------------------------------------------------------- /towns/shadowfen.txt: -------------------------------------------------------------------------------- 1 | Shadowfen, nestled in the crooked embrace of the Mirewood Forest, has a history as dark and rich as the murky waters of its renowned swamps. Founded over five centuries ago by a clan of refugees fleeing a war-torn land, the town has grown in both size and legend. 2 | 3 | Originally a scattering of makeshift huts and tents, the refugees named it Shadowfen for the thick fog that clung to the swampy land like a shroud. As the years wore on, the refugees became settlers, their temporary homes transformed into sturdy, permanent structures made of wood from the Mirewood trees and stone quarried from the nearby Draken Hills. 4 | 5 | The town's early economy relied on the bountiful yet dangerous natural resources the swamp provided. Trappers and hunters ventured into the mists to collect the hides of the shadowgators, large reptilian creatures with scales as dark as the heart of the swamp. Alchemists and herbalists sought the rare fungi and plants that grew only in the deep fen, ingredients for powerful potions and poisons. 6 | 7 | But Shadowfen was not without its troubles. The swamp was a living entity, home to more than just animals and plants. Spirits of the water, the Naiads, and creatures of the earth, the Gloom Sprites, watched the encroaching humans with wary eyes. A delicate truce was struck with the swamp's denizens through the guidance of the town's druid circle, the Marshbinders. They served as intermediaries, ensuring that the natural balance was maintained. 8 | 9 | The greatest challenge came during the third century of Shadowfen's existence, when a necromancer, drawn by the potent magical energies that seeped from the swamp, sought to bind the town and its surrounding lands to his will. His undead armies rose from the swamp's depths, and it took the combined might of the Marshbinders and a band of heroic adventurers to banish the necromancer and his minions, an event that is celebrated annually in Shadowfen as the Festival of Lights, where lanterns are floated on the swamp waters to honor the fallen and to ward off dark magic. 10 | 11 | As time marched on, Shadowfen's reputation as a place of eerie beauty and potent magic grew, drawing in not only those seeking fortune but also those with a scholarly interest in the occult. The town established the Library of Whispers, a repository of arcane knowledge and forbidden lore. It is rumored that the library holds the key to ancient secrets and powers that the world is not yet ready to handle. 12 | 13 | In recent times, Shadowfen has become a haven for those seeking refuge from the outside world's judgment, a place where the misunderstood and the outcast can find a community. Governed by a council of the town's eldest and wisest, including descendants of the original refugees, members of the Marshbinders, and elected representatives of the various guilds and factions, Shadowfen continues to thrive. 14 | 15 | Despite its prosperity, the town never forgets the dark embrace of the swamp that cradles it. The Marshbinders and the druids still perform their rites to maintain the balance, the Naiads and Gloom Sprites continue to watch from the shadows, and the townsfolk of Shadowfen remain ever respectful of the ancient pact between land and people. Thus, the story of Shadowfen weaves on, a tale of survival, of darkness and light, and of the unyielding spirit of those who call the swamp their home. 16 | 17 | Shadowfen, shrouded in mystery and marshland mists, boasts a number of notable locations and characters that contribute to its unique charm and treacherous allure. 18 | 19 | **Notable Places:** 20 | 21 | 1. **The Marshbinders' Circle**: At the heart of Shadowfen lies an ancient stone circle, where the Marshbinders, a coven of druids, conduct their enigmatic rituals to maintain the natural order. This verdant enclave is a nexus of ley lines, and the air thrums with arcane energy, making it a sacred space where the veil between worlds grows thin. 22 | 23 | 2. **The Library of Whispers**: A towering edifice of stone and wood, filled with musty tomes and ancient scrolls. Overseen by the enigmatic Archivist Lyron, who is said to have not aged a day in a century, the library is both a haven for scholars and a vault of dangerous knowledge. 24 | 25 | 3. **The Gloomwater Brewery**: Run by a jovial dwarf named Brom Stoutfist, this brewery is famous for its 'Bog Ale', a drink with a taste as murky as the waters it's named after. The brewery is a bustling hub of activity and a favorite gathering spot for both locals and travelers. 26 | 27 | 4. **The Sable Market**: Shadowfen’s market is a labyrinth of stalls and shops, selling everything from exotic swamp herbs to rare artifacts dredged from the swamp. It's overseen by the hawk-eyed market mistress, Zara the Fair, who ensures that all transactions are on the level—though rumors persist of a thriving black market under her very nose. 28 | 29 | 5. **The Whispering Inn**: This creaky, lantern-lit inn is the first stop for many visitors. Managed by a charismatic halfling couple, Milo and Tilla Underfoot, the inn is known for its warm beds, hot meals, and the occasional secret being shared in hushed tones by the fireplace. 30 | 31 | **Notable People:** 32 | 33 | 1. **Eldermarsh Thorne**: The current High Druid of the Marshbinders, Thorne is a figure of awe and a little fear, given his reputed ability to converse with the swamp itself. With his staff carved from a blackened root and his cloak of woven reeds, he is the embodiment of the swamp's enigmatic nature. 34 | 35 | 2. **Archivist Lyron**: The keeper of the Library of Whispers, Lyron is a figure shrouded in as much secrecy as the tomes he guards. Pale and slender, with eyes that gleam with hidden knowledge, he aids those who seek the library's wisdom, though always at a price. 36 | 37 | 3. **Brom Stoutfist**: The boisterous master brewer of Gloomwater Brewery, Brom's robust laughter and hearty ales are almost as famous as his tales of adventure from his younger days exploring the wilds beyond Shadowfen. 38 | 39 | 4. **Zara the Fair**: As mistress of the Sable Market, Zara is a shrewd woman known for her fairness in trade and her network of informants. She has her finger on the pulse of the town's commerce, and nothing escapes her keen eyes. 40 | 41 | 5. **Milo and Tilla Underfoot**: This halfling couple are the beloved proprietors of the Whispering Inn. Their establishment is known far and wide for its hospitality, and they themselves are considered the unofficial chroniclers of the town's history and gossip. 42 | 43 | The interplay of these locations and characters makes Shadowfen not just a geographical location but a tapestry of stories and legends, where every corner holds a secret, and every face in the crowd could be a friend or foe. The town thrives on the balance of nature and civilization, magic and mundane, creating a symphony of existence that is as perilous as it is picturesque. 44 | 45 | The government of Shadowfen is a unique blend of traditional authority and the influence of magical and natural forces, reflective of its diverse inhabitants and their deep connection to the surrounding swamp. 46 | 47 | **Council of Elders:** 48 | At the top of Shadowfen's governance structure is the Council of Elders, a body comprising the most respected and influential members of the community. This council is responsible for making decisions on matters of policy, law, and the overall welfare of the town. The composition of the Council of Elders is as follows: 49 | 50 | 1. **High Druid of the Marshbinders**: The leader of the druid circle, currently Eldermarsh Thorne, holds a permanent seat on the council. They represent the interests of the natural world and ensure that the town's actions are in harmony with the swamp. 51 | 52 | 2. **Elected Representatives**: These are individuals chosen by the town's various factions and guilds, representing the diverse interests of Shadowfen's populace. They are elected for a term of three years, and their primary role is to voice the concerns and needs of their respective constituencies. 53 | 54 | 3. **Archivist of the Library of Whispers**: Given the significance of magic and ancient knowledge in Shadowfen, the Archivist, currently Lyron, is a key advisor on matters arcane and historical. 55 | 56 | 4. **Captain of the Guard**: Responsible for the town's safety and enforcement of its laws, the Captain of the Guard serves on the council to provide insights on security and public order. 57 | 58 | 5. **Master of the Market**: This role, currently held by Zara the Fair, ensures that the economic interests of Shadowfen are represented, especially concerning trade and commerce. 59 | 60 | **Advisory Bodies:** 61 | In addition to the Council of Elders, there are several advisory bodies that play a crucial role in the town's governance: 62 | 63 | 1. **Circle of Mages**: A group of skilled magic users who advise the council on matters of the arcane and mystical threats. 64 | 65 | 2. **Guildmasters' Assembly**: Representatives from the various trade and craft guilds meet regularly to discuss economic issues and advise the council on trade policies. 66 | 67 | 3. **Marshbinders' Enclave**: A group of druids and rangers who work closely with the High Druid to maintain the balance between Shadowfen and the natural world, providing counsel on environmental matters. 68 | 69 | **Judiciary and Enforcement:** 70 | For matters of law, Shadowfen has a small judiciary headed by a Magistrate, who is appointed by the council. The town guard, led by the Captain of the Guard, is responsible for law enforcement and maintaining peace. 71 | 72 | This governance structure allows Shadowfen to function effectively, balancing the needs of its diverse population with the unique challenges of its environment. The Council of Elders, with its mix of elected and appointed members, ensures a wide range of perspectives is considered in decision-making, while the advisory bodies provide specialized knowledge and expertise. This system has enabled Shadowfen to navigate the complexities of swamp politics, ancient magics, and the day-to-day needs of its people. 73 | 74 | Shadowfen, with its unique location and history, is a melting pot of various races and cultures, each contributing to the rich tapestry of life in the town. The demographics of Shadowfen reflect its status as a place of refuge, mystery, and arcane knowledge, attracting a wide array of individuals. 75 | 76 | **Major Races and Ethnicities:** 77 | 78 | 1. **Humans**: The largest demographic in Shadowfen, humans are mostly descendants of the original settlers. They are involved in various trades and professions, from farming and hunting to scholarly pursuits. 79 | 80 | 2. **Halflings**: Known for their resourcefulness and cheer, halflings form a significant portion of the population. Many are involved in the hospitality industry, like the Underfoot couple who run the Whispering Inn, or in the trade and crafts seen in the Sable Market. 81 | 82 | 3. **Dwarves**: Dwarves, often skilled artisans and brewers, contribute to Shadowfen’s reputation for exquisite craftsmanship and robust ales, like those brewed by Brom Stoutfist at the Gloomwater Brewery. 83 | 84 | 4. **Elves**: A smaller but notable group, elves in Shadowfen are often drawn to the arcane and natural aspects of the town. Some are scholars at the Library of Whispers, while others join the ranks of the Marshbinders. 85 | 86 | 5. **Half-Elves**: Common in a place like Shadowfen, half-elves often find it easier to blend into the diverse community, participating in various aspects of town life. 87 | 88 | 6. **Tieflings**: Their unique heritage makes tieflings stand out, but in Shadowfen, they find a haven from the prejudice they might face elsewhere. Their keen insights and talents, especially in the arcane arts, are highly valued. 89 | 90 | 7. **Other Races**: Given its nature as a refuge and center for arcane study, Shadowfen also has small populations of other races like gnomes, half-orcs, and even a few more exotic beings who are drawn to the town’s peculiar charm. 91 | 92 | **Cultural Dynamics:** 93 | 94 | - **Trade and Craftsmanship**: Shadowfen’s economy is robust, thanks to its diverse population. Each race brings its unique skills and traditions, contributing to a thriving market of goods and services. 95 | 96 | - **Magic and Scholarship**: The presence of the Library of Whispers and the natural magic of the swamp attract scholars, mages, and researchers, creating a community that values knowledge and the arcane. 97 | 98 | - **Harmony with Nature**: The influence of the Marshbinders ensures that all races in Shadowfen respect the delicate balance of nature, leading to a community that lives in harmony with its environment. 99 | 100 | - **Refuge and Acceptance**: Many who come to Shadowfen are seeking refuge from persecution or conflict. This shared experience has fostered a culture of acceptance and camaraderie among the town’s inhabitants. 101 | 102 | The demographic composition of Shadowfen is not just a statistic; it is a testament to the town's ability to unite disparate elements into a cohesive, functioning community. The diversity of its people is mirrored in the eclectic architecture, varied cuisine, and the myriad of festivals and traditions that mark the town's calendar, making Shadowfen a vibrant and unique place to live. 103 | 104 | Shadowfen, while a unique and vibrant community, faces several challenges stemming from its environment, diverse population, and the mystical elements that pervade its existence. These challenges often give rise to political debates and arguments among its inhabitants and governing bodies. 105 | 106 | **Key Challenges and Political Debates:** 107 | 108 | 1. **Environmental Balance**: One of the most pressing issues in Shadowfen is maintaining the delicate balance between the town's growth and the preservation of the swamp's ecosystem. The Marshbinders advocate for strict limitations on expansion and resource extraction to protect the swamp. However, this often conflicts with the ambitions of traders and builders who seek to expand and develop. Debates around sustainable use of the swamp's resources are frequent and heated. 109 | 110 | 2. **Magic Regulation**: Given the town's proximity to powerful ley lines and its history with dark magic, the regulation of magical practices is a topic of ongoing debate. While the Circle of Mages and the Archivist of the Library of Whispers push for careful oversight of magical research and practice, some mages and scholars argue for greater freedom in their arcane pursuits, citing the need for academic liberty and defense against potential threats. 111 | 112 | 3. **Trade and Commerce**: The economic interests of Shadowfen, represented by figures like Zara the Fair, often clash with regulatory efforts by the Council of Elders. The market's expansion, trade agreements with external entities, and the potential for exploitation of the swamp's unique resources are subjects of contention. There's a delicate balance between fostering economic growth and protecting Shadowfen's interests against external exploitation. 113 | 114 | 4. **Cultural Integration**: As a melting pot of various races and cultures, Shadowfen sometimes struggles with integrating its diverse population. Issues around cultural representation, equal rights, and racial tensions occasionally surface, prompting discussions on how to foster a more inclusive community. 115 | 116 | 5. **Defense and Security**: The Captain of the Guard and the town's militia face the challenge of defending Shadowfen against external threats like bandits or monstrous creatures from the swamp, as well as internal threats like criminal activity. Debates often arise over the allocation of resources for defense, the extent of the guard's authority, and the balance between security and personal freedoms. 117 | 118 | 6. **Refugee and Outsider Policy**: Given its history as a refuge, Shadowfen continues to attract those seeking sanctuary. This influx of refugees and outsiders leads to discussions about the town's capacity to accommodate new arrivals, the impact on local resources, and the process of integration into Shadowfen’s society. 119 | 120 | 7. **Supernatural Incidents**: The occasional surge of supernatural occurrences, such as apparitions, uncontrolled magic, or sightings of rare swamp creatures, raises debates about how to handle these phenomena. While some call for a more proactive approach in studying and potentially utilizing these occurrences, others, particularly the Marshbinders, advocate for caution and respect for the mystical forces at play. 121 | 122 | These challenges and debates are integral to Shadowfen’s character, reflecting the complexity and depth of a town that stands at the crossroads of the natural and supernatural, tradition and progress, unity and diversity. The Council of Elders, along with the town’s various factions and guilds, continuously navigate these issues, striving to maintain the harmony and prosperity of Shadowfen amidst the ever-present mysteries of the swamp. 123 | 124 | Crime in Shadowfen, much like in any community, is a reality that the town must contend with. However, given Shadowfen's unique setting and diverse population, the nature of crime and those involved in it can be quite distinctive. 125 | 126 | **Types of Crime in Shadowfen:** 127 | 128 | 1. **Theft and Smuggling**: Given Shadowfen's position as a trade hub with a bustling market, theft is a common crime. Smuggling is also prevalent, especially given the demand for rare swamp resources and magical artifacts. These crimes are often committed by individuals or small groups looking to make a quick profit. 129 | 130 | 2. **Black Market Activities**: The existence of a black market dealing in forbidden magical artifacts, rare swamp materials, and sometimes illicit substances is an open secret in Shadowfen. This market is typically run by more organized and secretive groups who have connections both within and outside of the town. 131 | 132 | 3. **Scams and Fraud**: Con artists and swindlers find Shadowfen's diverse and transient population to be ripe for exploitation. These criminals often prey on the unsuspecting, especially newcomers or those seeking magical solutions to their problems. 133 | 134 | 4. **Assault and Intimidation**: Occasional incidents of violence, often related to bar brawls or disputes among the rougher elements of the population, do occur. These are generally isolated incidents but can escalate if left unchecked. 135 | 136 | 5. **Dark Magic Practices**: Unlawful use of magic, especially that which delves into the darker and more dangerous aspects, is a crime taken very seriously in Shadowfen. These crimes are rare but are treated with the utmost severity due to their potential to cause widespread harm. 137 | 138 | **Demographic Makeup of Criminals:** 139 | 140 | Shadowfen's crime does not predominantly belong to any single race or ethnicity. Criminals come from various backgrounds, reflecting the town's diverse demographic makeup. However, certain patterns can be observed: 141 | 142 | - **Transient Populations**: Given the town's nature as a haven for refugees and travelers, some crimes are committed by those who are just passing through or who have not fully integrated into the community. 143 | 144 | - **Disenfranchised Individuals**: Those who feel marginalized or left out of Shadowfen’s prosperity may turn to crime, whether out of desperation or as a means of asserting themselves. 145 | 146 | - **Opportunists**: Individuals from various races and backgrounds, drawn by the prospect of easy wealth, might engage in smuggling or black market activities. 147 | 148 | - **Dark Magic Practitioners**: These are usually solitary figures or small, secretive groups who delve into forbidden magics. They could come from any race but are often those with some knowledge or background in the arcane. 149 | 150 | The town guard and the Council of Elders, particularly the Captain of the Guard and the representatives from the Circle of Mages, work diligently to address crime in Shadowfen. Their efforts are focused not only on law enforcement but also on addressing the underlying issues that lead to criminal behavior, such as economic disparity and social integration challenges. This balanced approach aims to maintain order while ensuring that the diverse fabric of Shadowfen's society remains intact. 151 | 152 | Shadowfen, with its rich history, diverse culture, and mystic ambiance, offers a variety of interesting spots that would intrigue any tourist. Here are some must-visit locations: 153 | 154 | 1. **The Marshbinders' Circle**: A visit to Shadowfen would be incomplete without seeing the ancient stone circle of the Marshbinders. This sacred site, pulsating with natural magic, offers a glimpse into the town's deep connection with the swamp and its mystical forces. The best time to visit is during one of the druidic festivals, where one can witness captivating rituals and celebrations. 155 | 156 | 2. **The Library of Whispers**: A treasure trove for the intellectually curious, this library houses an impressive collection of arcane tomes, historical scrolls, and rare manuscripts. Guided tours, often led by knowledgeable acolytes, provide insights into Shadowfen’s rich history and the mysteries of the swamp. 157 | 158 | 3. **Gloomwater Brewery**: For those who enjoy a good ale, the Gloomwater Brewery is a must. Here, visitors can taste the famous 'Bog Ale' and other unique brews, learn about the brewing process, and even hear a tale or two from Brom Stoutfist, the charismatic master brewer. 159 | 160 | 4. **The Sable Market**: This bustling marketplace is a sensory overload with its array of exotic goods, from enchanted trinkets to rare swamp herbs. The market is not just a place for commerce but a cultural experience, offering a taste of the diverse culinary delights and crafts of Shadowfen. 161 | 162 | 5. **The Whispering Inn**: A perfect spot to unwind, this inn offers not just comfortable lodgings and delicious food, but also a chance to mingle with locals and hear stories of Shadowfen’s past and present. The innkeepers, Milo and Tilla Underfoot, are known for their hospitable nature and wealth of local knowledge. 163 | 164 | 6. **Swamp Tours**: For the more adventurous, guided tours into the swamp are available. These tours offer a chance to experience the unique flora and fauna of the swamp, with opportunities to spot rare wildlife and learn about the medicinal and magical properties of various swamp plants. 165 | 166 | 7. **The Festivals of Shadowfen**: Timing a visit during one of Shadowfen’s many festivals, like the Festival of Lights or the Harvest Moon Celebration, provides a unique experience. These festivals are a vibrant display of local customs, music, dance, and culinary delights. 167 | 168 | 8. **The Naiad Pools**: Located at the edge of the town, these crystal-clear pools are said to be inhabited by water spirits. The serene beauty of the pools, coupled with the legends surrounding them, make for a mystical and tranquil experience. 169 | 170 | Each of these locations offers a different facet of Shadowfen’s rich tapestry, ensuring that visitors have a memorable and varied experience. Whether one is seeking knowledge, adventure, cultural experiences, or simply relaxation, Shadowfen has something to offer to every traveler. 171 | 172 | Shadowfen, with its deep connection to ancient magic and the mysterious swamp, is ripe with urban legends, folklore, and conspiracy theories. These tales, passed down through generations, add to the enigmatic charm of the town. 173 | 174 | **Urban Legends and Folklore:** 175 | 176 | 1. **The Whispering Trees of Mirewood**: It is said that the trees in the deepest part of Mirewood Forest whisper secrets of the past and future to those who listen carefully. Some claim to have heard voices of long-lost loved ones or forebodings of events yet to come. 177 | 178 | 2. **The Ghost of the Swamp**: A spectral figure that is said to roam the swamp at night. Legend has it that this ghost is the spirit of the first High Druid of Shadowfen, who now protects the town from malevolent forces. 179 | 180 | 3. **The Naiads' Curse**: According to local lore, the Naiad pools are home to water spirits who can bestow blessings or curses. There's a tale of a villager who wronged a Naiad and was cursed with eternal misfortune, serving as a cautionary tale about respecting the swamp's inhabitants. 181 | 182 | 4. **The Hidden Chamber of the Library of Whispers**: Many believe that there is a secret chamber within the Library of Whispers that holds forbidden knowledge and spells powerful enough to control or destroy the world. 183 | 184 | 5. **The Swamp Lights**: Mysterious lights are often seen dancing over the swamp at night. While some say these are just swamp gases, others believe them to be the spirits of the swamp, leading or misleading travelers. 185 | 186 | **Legends:** 187 | 188 | 1. **The Battle of the Necromancer**: A historical event that has taken on legendary status, this battle saw the townspeople, led by the Marshbinders and a band of heroes, defeat a powerful necromancer who sought to enslave Shadowfen. 189 | 190 | 2. **The Eternal Archivist**: The legend of Archivist Lyron, rumored to have lived for centuries without aging, is a popular tale. Many speculate about the source of his longevity, with theories ranging from a pact with a deity to the possession of an ancient artifact. 191 | 192 | **Conspiracy Theories:** 193 | 194 | 1. **The Council's Secret Pact**: There's a theory that the Council of Elders has a secret pact with an unknown, possibly malevolent, entity that helps keep peace in Shadowfen in exchange for undisclosed sacrifices or rituals. 195 | 196 | 2. **The Black Market's True Rulers**: Some believe that the black market in Shadowfen is controlled by a cabal of powerful figures from outside the town, who use it to manipulate local politics and economics for their own gain. 197 | 198 | 3. **The Swamp's True Nature**: A few conspiracy theorists argue that the swamp itself is a sentient being, or perhaps a gateway to another world, and that the town's leadership is aware of and actively concealing this fact. 199 | 200 | These tales and theories are a testament to Shadowfen's rich cultural tapestry, blending the mystical and the mundane. They serve not only as entertainment but also as a reflection of the fears, hopes, and beliefs of the townspeople, adding layers of depth to the already intriguing character of Shadowfen. 201 | 202 | In a town like Shadowfen, brimming with diverse characters and steeped in mystery, gossip is as common as the morning mist. The latest town gossip often revolves around recent events, intriguing newcomers, or the mysterious happenings that are all too common in a place like this. Here are some current topics that might be buzzing around the Whispering Inn or the Sable Market: 203 | 204 | 1. **A Mysterious Visitor**: There's talk of a stranger who arrived in town recently, cloaked in a dark mantle, asking questions about the Library of Whispers and the Marshbinders' Circle. Some say they might be a scout for a powerful mage or an agent of a distant kingdom. 205 | 206 | 2. **The Underfoot's Secret Recipe**: Rumor has it that Milo and Tilla Underfoot, the innkeepers, have discovered an ancient recipe in an old tome that could revolutionize their menu. This has piqued the interest of both locals and visitors, eager to taste this new culinary creation. 207 | 208 | 3. **Unusual Activity in the Swamp**: Several locals have reported seeing strange lights and hearing eerie sounds from deep within the swamp. This has led to speculation about an awakening of an ancient spirit or the return of a long-dormant magical force. 209 | 210 | 4. **The Archivist's New Apprentice**: Whispers are circulating about a young prodigy taken in by Archivist Lyron as an apprentice. Some speculate about the youngster's origins and the potential knowledge they might access in the Library of Whispers. 211 | 212 | 5. **A Discovery in the Market**: A rare artifact was reportedly found by a stall owner in the Sable Market. The object's nature and origins are the subject of much speculation, with some suggesting it might have magical properties. 213 | 214 | 6. **Expansion Plans**: There’s talk that the Council of Elders is considering expanding the town’s boundaries to accommodate the growing population. This has sparked debate among the townsfolk about the impact on the swamp and the balance with nature. 215 | 216 | 7. **The Brewer’s New Brew**: Brom Stoutfist is rumored to be experimenting with a new brew that uses a rare herb from the swamp. The anticipation is building, and there’s already a list of volunteers ready to taste it. 217 | 218 | 8. **The Marshbinders' Ritual**: The druids were seen performing an unusual ritual recently, more elaborate than their regular practices. This has led to speculation about what prompted this ritual – is it a response to a natural threat, or something more arcane? 219 | 220 | This gossip, while often light-hearted and speculative, reflects the dynamic nature of Shadowfen and the curiosity of its inhabitants. It’s through these stories and rumors that the community stays connected, sharing in the mysteries and adventures that make their home so unique. 221 | 222 | Shadowfen, with its blend of mysticism, diverse populace, and unique geographical features, naturally presents a variety of opportunities for adventurers seeking work and profit. Here are some potential situations where adventurers might find themselves in demand: 223 | 224 | 1. **Swamp Expeditions**: The treacherous and uncharted regions of the surrounding swamp are ripe for exploration. Adventurers might be hired to map unexplored areas, retrieve rare herbs and materials, or investigate the source of mysterious phenomena reported by locals. 225 | 226 | 2. **Artifact Recovery**: With the presence of the Library of Whispers, there's always a demand for rare artifacts and ancient tomes. Scholars and mages often hire adventurers to retrieve these items from forgotten ruins or dangerous locales within the swamp. 227 | 228 | 3. **Protecting Trade Caravans**: As a trading hub, Shadowfen sees its fair share of caravans and traders passing through. These groups often seek out adventurers for protection against bandits, swamp creatures, or other hazards that lurk along the trade routes. 229 | 230 | 4. **Monster Hunting**: The swamp and its surroundings are home to a variety of creatures, some of which pose a threat to the town. Adventurers could be commissioned to hunt down a particularly troublesome beast, like a rogue shadowgator or a pack of swamp wolves. 231 | 232 | 5. **Mystery Solving**: Shadowfen is a place of secrets and mysteries. Adventurers might find work in solving strange occurrences or investigating paranormal events that baffle the locals, such as the appearance of ghostly figures or unexplained magical anomalies. 233 | 234 | 6. **Rescue Missions**: The dangerous terrain of the swamp often leads to people getting lost or trapped. Adventurers could be employed for rescue missions, requiring not only combat prowess but also survival skills and knowledge of the swamp. 235 | 236 | 7. **Guarding the Town**: With its proximity to wild and magical territories, Shadowfen sometimes faces threats from external forces. Adventurers could be hired to bolster the town's defenses, either in response to a specific threat or as part of a regular guard force. 237 | 238 | 8. **Mediating Disputes**: In a town with such a diverse population, disputes are inevitable. Adventurers with diplomatic skills might find opportunities in mediating conflicts, whether between rival business interests, feuding families, or different racial groups within the town. 239 | 240 | 9. **Arcane Investigations**: For those with knowledge of the arcane, assisting in investigations related to magical disturbances, uncontrolled enchantments, or dark magic practices could be both profitable and crucial for the town’s safety. 241 | 242 | 10. **Underworld Dealings**: Adventurers willing to skirt the edges of legality might find opportunities in dealing with the black market or the seedier elements of Shadowfen. These jobs would be risky but potentially lucrative. 243 | 244 | Each of these opportunities not only offers a chance for profit but also the potential to become entwined in the complex tapestry of Shadowfen’s stories and intrigues, making any adventurer’s stay in the town both profitable and eventful. --------------------------------------------------------------------------------