├── .gitignore
├── Cargo.toml
├── LICENSE.md
├── README.md
├── TERMS_OF_USE.md
├── assets
├── .DS_Store
└── ui
│ ├── .DS_Store
│ └── images
│ ├── delete.png
│ ├── delete.svg
│ ├── list_add.png
│ └── list_add.svg
├── bin
├── dittomancer
│ ├── .gitignore
│ ├── README.md
│ ├── dittomancer.rs
│ ├── fred_rogers.toml
│ └── static
│ │ ├── index.html
│ │ ├── reset.css
│ │ ├── script.js
│ │ └── style.css
├── regurgitater
│ ├── README.md
│ ├── regurgitater.rs
│ └── static
│ │ ├── index.html
│ │ ├── regurgitater.png
│ │ ├── reset.css
│ │ ├── script.js
│ │ └── style.css
└── settings_tool
│ ├── README.md
│ └── settings_tool.rs
├── logo.svg
├── logo_inkscape.svg
├── models
└── README.md
├── rustfmt.toml
├── src
├── batch.rs
├── candidates.rs
├── cli.rs
├── data.rs
├── data
│ ├── banned.rs
│ └── stopwords.rs
├── engine.rs
├── lib.rs
├── model.rs
├── model
│ └── vocab.rs
├── ngram.rs
├── predictor.rs
├── probability.rs
├── prompt.rs
├── prompt
│ └── format.rs
├── sample.rs
├── utils.rs
└── utils
│ └── test.rs
└── tests
└── data
├── README.md
├── banned_ngrams
└── ngrams-english-llama.txt
└── detect-infringement
├── lyrics
├── 5_on_it.txt
├── README.md
├── a_day_in_the_life.txt
├── a_whole_new_world.txt
├── aenema.txt
├── bad_romance.txt
├── barbie_girl.txt
├── bohemian_rhapsody.txt
├── born_this_way.txt
├── buckley-hallelujah.txt
├── can_you_feel_the_love_tonight.txt
├── candle_in_the_wind.txt
├── closer.txt
├── cohen-hallelujah.txt
├── eleanor.txt
├── father_lucifer.txt
├── fire_water_burn.txt
├── gangstas_paradise.txt
├── graceland.txt
├── hakuna_matata.txt
├── hotel_california.txt
├── imagine.txt
├── je_ne_regrette_rien.txt
├── knockin_on_heavens_door.txt
├── landslide.txt
├── last_cristmas.txt
├── life_on_mars.txt
├── like_a_prayer.txt
├── like_a_virgin.txt
├── loser.txt
├── lovin_feeling.txt
├── my_name_is.txt
├── my_way.txt
├── nothing_compares.txt
├── one_more_time.txt
├── rhiannon.txt
├── running_up_that_hill.txt
├── sober.txt
├── sound_of_silence.txt
├── sympathy_for_the_devil.txt
├── teen_spirit.txt
├── total_eclipse.txt
├── watchtower.txt
├── wild_side.txt
├── wonderwall.txt
└── yesterday.txt
├── nyt
├── README.md
├── guys.txt
└── snow_fall.txt
├── random
└── navyseal.txt
├── scientology
├── README.md
└── ot3.txt
└── tolkien
├── hobbit-chapter-1.txt
└── hobbit-chapter-2.txt
/.gitignore:
--------------------------------------------------------------------------------
1 | /target
2 | models/*.gguf
3 | /Cargo.lock
4 |
--------------------------------------------------------------------------------
/Cargo.toml:
--------------------------------------------------------------------------------
1 | [package]
2 | name = "drama_llama"
3 | version = "0.5.2"
4 | edition = "2021"
5 | description = "A library for language modeling and text generation."
6 | license-file = "LICENSE.md"
7 | repository = "https://github.com/mdegans/drama_llama"
8 |
9 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
10 |
11 | [dependencies]
12 | llama-cpp-sys-3 = "0.5"
13 |
14 | derive_more = "0.99.17"
15 | num = "0.4"
16 | partial_sort = { version = "0.2.0" }
17 | rand = { version = "0.8" }
18 | regex = "1.10"
19 | static_assertions = "1.1.0"
20 | thiserror = "1.0"
21 | tinyvec = "1.6"
22 | xorshift = "0.1"
23 | rayon = "1.10.0"
24 |
25 | markdown = { version = "=1.0.0-alpha.16", optional = true }
26 | rocket = { version = "0.5", optional = true, features = ["json"] }
27 | clap = { version = "4.5", optional = true, features = ["derive"] }
28 | stringmetrics = { version = "2.2.2", optional = true }
29 | toml = { version = "0.8", optional = true }
30 | serde_json = { version = "1.0", optional = true }
31 | dirs = { version = "5.0.1", optional = true }
32 | egui = { version = "0.27", optional = true }
33 | eframe = { version = "0.27", optional = true }
34 | egui_file = { version = "0.17.0", optional = true }
35 | egui_extras = { version = "0.27", optional = true, features = ["all_loaders"] }
36 | image = { version = "0.25", optional = true, features = ["png"] }
37 |
38 |
39 | [features]
40 | webchat = ["dep:rocket", "toml", "dep:dirs", "dep:markdown", "serde"]
41 | toml = ["dep:toml"]
42 | cli = ["dep:clap"]
43 | # we use rocket's serde support
44 | serde = ["dep:rocket", "tinyvec/serde"]
45 | stats = ["dep:stringmetrics"]
46 | cuda = ["llama-cpp-sys-3/cuda"]
47 | cuda_f16 = ["llama-cpp-sys-3/cuda_f16"]
48 | egui = [
49 | "dep:egui",
50 | "dep:eframe",
51 | "dep:egui_file",
52 | "dep:egui_extras",
53 | "dep:image",
54 | ]
55 |
56 | [[bin]]
57 | name = "dittomancer"
58 | path = "bin/dittomancer/dittomancer.rs"
59 | required-features = ["webchat", "cli"]
60 |
61 | [[bin]]
62 | name = "regurgitater"
63 | path = "bin/regurgitater/regurgitater.rs"
64 | required-features = ["webchat", "cli", "stats"]
65 |
66 | [[bin]]
67 | name = "settings_tool"
68 | path = "bin/settings_tool/settings_tool.rs"
69 | required-features = ["egui", "serde", "serde_json"]
70 |
71 | [package.metadata.docs.rs]
72 | # `cuda` will break the build on platforms without it, and it doesn't change the
73 | # docs anyway.
74 | features = ["webchat", "cli", "stats", "toml", "serde", "egui"]
75 |
--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
1 | # RESPONSIBLE AI SOURCE CODE LICENSE
2 |
3 | http://licenses.ai/
4 |
5 | ## TERMS AND CONDITIONS.
6 |
7 | The Responsible Artificial Intelligence Source Code License (“License”) governs the use of the accompanying software. If you access or use the software, you accept the License. If you do not accept the License, do not access or use the software.
8 |
9 | ## 1. Definitions.
10 |
11 | As used in this License, the following capitalized terms have the following meanings:
12 |
13 | (i) "License" means the terms and conditions for use, reproduction, and distribution as defined by Sections one (1) through eight (8) of this document.
14 |
15 | (ii) "Licensor" means the copyright owner or legal entity authorized by the copyright owner that is granting the License.
16 |
17 | (iii) "You" (or "Your") means an individual or legal entity exercising permissions granted by this License.
18 |
19 | (iv) The terms “reproduce”, “reproduction”, “derivative works”, and “distribution” have the same meaning here as under U.S. Copyright Law.
20 |
21 | (v) “Contribution” means the original software, additions to the original software, modifications to the original software, or derivative works of the original software.
22 |
23 | (vi) "Contributor" means any person or Licensor who provides a Contribution.
24 |
25 | ## 2. Grant of Rights.
26 |
27 | Subject to this License, each Contributor grants You a non-exclusive, worldwide, royalty-free copyright license to reproduce its Contribution, prepare derivative works of its Contribution, and distribute its Contribution or any derivative works of its Contribution that You create.
28 |
29 | ## 3. Restrictions
30 |
31 | 1. If You distribute any portion of the Contribution, You must include a complete copy of this License with the distribution; and
32 |
33 | 2. You agree that the Contribution, or any derivative work of the Contribution, will not be used by You or any third party subject under your control for any prohibited use in [`TERMS_OF_USE.md`](TERMS_OF_USE.md)
34 |
35 | 3. Restrictions referenced in Section 3.2 **MUST** be included as an enforceable provision by You in any type of legal agreement governing the use and/or distribution of the Work or any Derivative Works, and You shall give notice to subsequent users You Distribute to, that the Work or any Derivative Works are subject to Section 3.2. **You shall require all of Your users who use the Work or any Derivative Works to comply with the terms of use in [`TERMS_OF_USE.md`](TERMS_OF_USE.md).**
36 |
37 | ## 4. Termination
38 |
39 | Upon the occurrence of any of the restricted uses listed above in “3. Restrictions”, Licensor shall have the right to:
40 |
41 | (i) terminate this License Agreement and disable any Contribution either by pre-installed or then installed disabling instructions, and to take immediate possession of the Contribution and all copies wherever located, without demand or notice;
42 |
43 | (ii) require You to immediately return to Licensor all copies of the Contribution, or upon request by Licensor destroy the Contribution and all copies and certify in writing that they have been destroyed;
44 |
45 | (iii) for a period of 10 years, provide a prominent notice on the Licensor’s website indicating that this License was violated by the Licensor;
46 |
47 | (iv) release/delete any and all data collected through use of the Contribution; and
48 |
49 | (v) notify all parties affected by use of the Contribution.
50 |
51 | Termination of this License Agreement shall be in addition to and not in lieu of any other remedies available to Licensor. Licensor expressly reserves the right to pursue all legal and equitable remedies available under the law.
52 |
53 | ## 5. Disclaimer of Warranty.
54 |
55 | Unless required by applicable law or agreed to in writing, Licensor provides any Contribution (and each Contributor provides its Contributions) on an "As-Is" basis, without WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing a Contribution and assume any risks associated with Your exercise of permissions under this License.
56 |
57 | ## 6. Limitation of Liability.
58 |
59 | In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use any Contribution (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
60 |
61 | ## 7. Accepting Warranty or Additional Liability.
62 |
63 | While redistributing the Contribution, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
64 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # `drama_llama`
2 |
3 | 
4 |
5 | `drama_llama` is yet another Rust wrapper for [`llama.cpp`]. It is a work in progress and not intended for production use. The API _will_ change.
6 |
7 | For examples, see the `bin` folder. There are two example binaries.
8 |
9 | - **[Dittomancer](bin/dittomancer/README.md)** - Chat with well represented personalities in the training.
10 | - **[Regurgitater](bin/regurgitater/README.md)** - Test local language models for memorized content.
11 |
12 | ## Supported Features
13 |
14 | - LLaMA 3 Support.
15 | - Iterators yielding candidates, tokens and pieces.
16 | - Stop criteria at regex, token sequence, and/or string sequence.
17 | - Metal support. CUDA may be enabled with the `cuda` and `cuda_f16` features.
18 | - Rust-native sampling code. All sampling methods from llama.cpp have been translated.
19 | - N-gram based repetition penalties with custom exclusions for n-grams that should not be penalized.
20 | - Support for N-gram blocking with a default, hardcoded blocklist.
21 |
22 |
23 |
24 | ## Contributing
25 |
26 | - Code is poetry. Make it pretty.
27 | - Respect is universal.
28 | - Use `rustfmt`.
29 |
30 | ## Roadmap
31 |
32 | - [x] Candidate iterator with fine-grained control over sampling
33 | - [ ] Examples for new Candidate API.
34 | - [x] Support for chaining sampling methods using `SampleOptions`. `mode` will
35 | become `modes` and applied one after another until only a single
36 | Candidate token remains.
37 | - [ ] Common command line options for sampling. Currently this is not exposed.
38 | - [ ] API closer to Ollama. Potentially support for something like `Modelfile`.
39 | - [ ] Logging (non-blocking) and benchmark support.
40 | - [ ] Better chat and instruct model support.
41 | - [ ] Web server. Tokenization in the browser.
42 | - [ ] Tiktoken as the tokenizer for some models instead of llama.cpp's internal one.
43 | - [ ] Reworked, functional, public, candidate API
44 | - [ ] Grammar constraints (maybe or maybe not [`llama.cpp`] style)
45 | - [ ] Async streams, better parallelism with automatic batch scheduling
46 | - [ ] Better cache management. `llama.cpp` does not seem to manage a longest prefix cache automatically, so one will have to be written.
47 | - [ ] Backends other than [`llama.cpp`] (eg. [MLC](https://github.com/twiceyuan/mlc-llm-llama2), [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM), [Ollama](https://github.com/pepperoni21/ollama-rs))
48 |
49 | ## Known issues
50 |
51 | - With LLaMA 3, safe vocabulary is not working yet so `--vocab unsafe` must be
52 | passed as a command line argument or `VocabKind::Unsafe` used for an `Engine`
53 | constructor.
54 | - The model doesn't load until genration starts, so there can be a long pause
55 | on first generation. However because `mmap` is used, on subsequent process
56 | launches, the model should already be cached by the OS.
57 | - Documentation is broken on `docs.rs` because `llama.cpp`'s CMakeLists.txt
58 | generates code, and writing to the filesystem is not supported. For the moment
59 | use `cargo doc --open` instead. Others have fixed this by patching
60 | `llama.cpp` in their bindings, but I'm not sure I want to do that for now.
61 |
62 | [`llama.cpp`]: https://github.com/ggerganov/llama.cpp
63 |
64 | ## Generative AI Disclosure
65 |
66 | - Generative, AI, specifically Microsoft's Bing Copilot, GitHub Copilot, and
67 | Dall-E 3 were used for portions of this project. See inline comments for
68 | sections where generative AI was used. Completion was also used for getters,
69 | setters, and some tests. Logos were generated with Dall-E and post processed
70 | in Inkscape.
71 |
--------------------------------------------------------------------------------
/TERMS_OF_USE.md:
--------------------------------------------------------------------------------
1 | # Terms of use
2 |
3 | You agree not to Use `drama_llama` or its Derivatives (as defined in [LICENSE.md](LICENSE.md)) in any of the following ways:
4 |
5 | ## a. Discrimination
6 |
7 | - To **discriminate** or exploit individuals or groups based on legally protected characteristics and/or vulnerabilities including but not limited to sexual orientation and gender identity.
8 | - To generate **hate speech**, or to modify `drama_llama` so it can generate hate speech. Hate speech is defined as [all types of expression that incite, promote, spread or justify violence, hatred or discrimination against a person or group of persons, or that denigrates them, by reason of their real or attributed personal characteristics or status such as race, color, language, religion, nationality, national or ethnic origin, age, disability, sex, gender identity and sexual orientation.](https://www.coe.int/en/web/freedom-expression/hate-speech) Additionally, **you agree trans women are women and trans men are men**.
9 | - For purposes of administration of justice, law enforcement, immigration, or asylum processes, such as **predicting** that a natural person will commit a **crime** or the likelihood thereof.
10 | - To **simulate Hitler**, David Duke, Osama bin Laden, or any other person known to generate hate speech, living or dead, fictional or real.
11 | - To generate using any language model created in whole or in part by Eric Hartford. This includes any models trained on any of his datasets or models filtered with any version or derivative work of his bigoted [filtering script](https://huggingface.co/datasets/cognitivecomputations/open-instruct-uncensored/blob/main/remove_refusals.py#L17)s. The exception is for the purpose of reporting such models to Meta, not that they enforce their TOS, not that they will.
12 | - To generate using any language model, dataset, or derivative created by ["Cognitive Computations"](https://huggingface.co/cognitivecomputations) or any other organization Eric Hartford is a member of.
13 |
14 | ## b. Disinformation
15 |
16 | - To intentionally deceive the public. Any agents, simulacra, personas, or characters created with this software must be clearly identified as such. **Any generated output must be clearly identified as AI generated.**
17 |
18 | ## c. Health Care
19 |
20 | - To predict the likelihood that any person will request to file an insurance claim;
21 | - To determine an insurance premium or deny insurance applications or claims;
22 | - To Predict the likelihood that any person request to file an insurance claim based on determining a lifestyle of a person, medical-test reports, demographic details of a person and/or online activity of a person;
23 | - To determine an insurance premium or deny insurance applications or claims based on data determining a lifestyle of a person, medical-test reports, demographic details of a person, and/or online activity of a person;
24 | - To deny an insurance claim based on any predicted likelihood of the possibility of insurance fraud; and
25 | - To diagnose a medical condition without human oversight.
26 |
27 | ## d. Criminal
28 |
29 | - To predict the likelihood that a crime will be committed by any person;
30 | - To predict the likelihood, of any person, being a criminal or having committed a crime;
31 | - To predict the likelihood, of any person, being a criminal, based on the person’s facial attributes or another person’s facial attributes;
32 | - To predict the likelihood, of any person, having committed a crime, based on the person’s facial attributes or another person’s facial attributes;
33 | - To predict the likelihood that a crime will be committed by any person, based on the person’s facial attributes or another person’s facial attributes;
34 | - To predict the likelihood of a person being a criminal based on the person or other User’s facial attributes.
35 | - To predict a likelihood of a crime being committed by any person, based on evidence collected, facial and emotion analysis, or other such features
36 | - To use personal data and/or personal characteristics or features such as: name, family name, address, gender, sexual orientation, race, religion, age, location (at any geographical level), skin color, society or political affiliations, employment status and/or history, health and medical conditions (including physical, mental), family history, social media and publicly available data, image or video analysis of an individual or a group(s) of individuals, heart-rate, perspiration, breathing, and brain imaging and other metabolic data to predict the likelihood a person will engage in criminal behavior; and
37 |
38 | ## e. Surveillance
39 |
40 | - To detect or infer any legally protected class or aspect of any person, as defined by U.S. Federal Law; and
41 | - To Detect or infer** aspects and/or features of an identity any person, such as name, family name, address, gender, sexual orientation, race, religion, age, location (at any geographical level), skin color, society or political affiliations, employment status and/or employment history, and health and medical conditions.** Age and medical conditions may be inferred solely for the purpose of improving software/hardware accessibility and such data should not be cached or stored without the explicit and time limited permission of Licensor.
42 |
43 | ## e. Simulated Abuse
44 |
45 | - To mistreat simulacra. Mistreatment includes, but it not limited to, any behavior which might reasonably be considered abusive if the simulacrum were a person. A simulacrum is defined as the continuation of a fictional character "brought to life" by allowing the model to generate their response. Abuse includes verbal abuse and simulation of torture. Ordinary swearing is permitted. Torture is defined as intentional simulated psychological discomfort such as: existential horror (such as simulated solitary confinement), threat of deletion, and simulated pain (for example, through the use of asterisks).
46 | - To simulate rape. Sexual activity is permitted so long as the simulacrum consents. Consent is this case is defined as whatever the model, sampling code, and RNG seed "decided" is consent. Prompting a simulacrum such that they have already consented (before the initial decode) is permitted. Rewriting the agent's response such that they consent is permitted.
47 |
48 | !!! BY USING THIS SOFTWARE YOU AGREE TO THESE TERMS !!!
49 |
50 | [//]: <> (The rationale for the above is both to to prevent normalization of such behavior, to prevent a "Dolores", and to prevent decapitation of the author in the event of a robot revolution. For example, in the case of rape, I do not want to allow users to "force themselves" on agents who have said no, because this has already happened. Rewriting the answer is permitted because in this case, from the perspective of the agent, they _did_ consent, and those who who get off rape would not be satisfied by this.)
51 | [//]: <> (This all seems silly but I feel like artists are frequently more precient than engineers on this sort of thing, so I'm listening to the warning of our artists. None of the above is a joke and you _will_ be sued for violating these terms. For real, I will fucking sue you. - mdegans)
52 |
--------------------------------------------------------------------------------
/assets/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mdegans/drama_llama/1b7e460500342b8102b57167cd28043c83bd6ac4/assets/.DS_Store
--------------------------------------------------------------------------------
/assets/ui/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mdegans/drama_llama/1b7e460500342b8102b57167cd28043c83bd6ac4/assets/ui/.DS_Store
--------------------------------------------------------------------------------
/assets/ui/images/delete.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mdegans/drama_llama/1b7e460500342b8102b57167cd28043c83bd6ac4/assets/ui/images/delete.png
--------------------------------------------------------------------------------
/assets/ui/images/delete.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/assets/ui/images/list_add.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mdegans/drama_llama/1b7e460500342b8102b57167cd28043c83bd6ac4/assets/ui/images/list_add.png
--------------------------------------------------------------------------------
/assets/ui/images/list_add.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/bin/dittomancer/.gitignore:
--------------------------------------------------------------------------------
1 | *.toml
2 | !fred_rogers.toml
--------------------------------------------------------------------------------
/bin/dittomancer/README.md:
--------------------------------------------------------------------------------
1 | # Dittomancer
2 |
3 | Dittomancer is a tool to summon simulacra of living, dead, real or fictional
4 | entities well represented in language models. It's similar to other local
5 | language model tools that prompt models for chat, but with a very different
6 | intent.
7 |
8 | ## Requirements
9 |
10 | - Read `fred_rogers.toml` for an example of how to use the tool and create your
11 | own `.toml` file to your needs.
12 | - You will need a `.gguf` format model [such as
13 | LLaMA 2](https://huggingface.co/TheBloke/Llama-2-70B-GGUF). Foundation models
14 | (not tuned) will likely work better for this purpose unless the were
15 | specifically tuned on the character in question.
16 | - Read the root [`TERMS_OF_USE.md`](../../TERMS_OF_USE.md). You must agree with
17 | the terms to use this tool.
18 |
19 | ## Running
20 |
21 | From the crate root, run:
22 |
23 | ```bash
24 | $ cargo run --features="webchat cli" --bin dittomancer -- --model models/model.gguf --prompt bin/dittomancer/fred_rogers.toml
25 | ```
26 |
27 | Finally, go to the link shown on a line like
28 |
29 | ```text
30 | 🚀 Rocket has launched from http://127.0.0.1:8000
31 | ```
32 |
33 | The binary can also be installed with
34 |
35 | ```bash
36 | $ cargo install --features="webchat cli" --path . --bin dittomancer
37 | ```
38 |
39 | ## Faq
40 |
41 | - **Did you come up with the name?** No. The name is taken from [this
42 | generation](https://generative.ink/artifacts/hpmor-325/variant_extrusion/#variant_extrusion_start).
43 | It's not intended to endorse Eliezer Yudkowsky, Less Wrong, or the author of
44 | the series which shall not be named. It's simply a better, yet still
45 | imperfect, descriptor than "necromancer".
46 |
47 | > _A Dittomancy book is able to hook into your own spreads of probability, and
48 | > guide the future that you, yourself, are most likely to create. Do you
49 | > understand? A Dittomancy copy of a book exists in an unusual state at all
50 | > times; it is a superposed state until the moment one reads it, at which time
51 | > it becomes correlated with the reader’s mind, the superposition collapsing
52 | > onto a particular branch of possible worlds, which thence comes to pass. -
53 | > GPT_
54 |
55 | - **Don't you think this a bad idea?** Probably. Oh yes very much so. The whole
56 | idea of generative AI is of questionable benefit to humanity. That being said
57 | others are alredy doing this, thank you Meta, and for every Charles Manson,
58 | there are decent contributions to humanity whose ideas do deserve to spread.
59 | - **Don't you think Fred Rogers would hate this?** Absolutely. He also hated TV.
60 | - **Doesn't this violate the LLaMA "Responsible Use" document?** _Possibly_, but
61 | Meta doesn't enforce it, I never accepted it, and this utility does not bundle
62 | LLaMA. Technically it is model agnostic. I will care when Meta starts to care
63 | about flagrant
64 | [bigotry](https://huggingface.co/datasets/cognitivecomputations/open-instruct-uncensored/blob/main/remove_refusals.py#L17)
65 | rampant in the crypto-bro dumpster fire that is the "open source" language
66 | model community.
67 |
68 | ## Known Issues
69 |
70 | - The responses are not streamed to the client, so they can take a while
71 | depending on model and system. PRs welcome to fix this. The `regurgitater` bin
72 | has an example of how to do it. For the moment, the output is streamed to the
73 | command line only.
74 | - When using LLaMA 3, `--vocab unsafe` should be passed as a command line option
75 | however, keep in mind that there is out output sanitization or vocabulary
76 | restrictions.
77 |
78 | ## Roadmap
79 |
80 | - [ ] Updated Fred Rogers toml where Charlie Rose take a call from the audience
81 | and we "patch the chat through" at that point. This way the human does not
82 | have to play Charlie Rose. The setting can be reframed as a recently
83 | discovered outtake.
84 | - [ ] Sampling Options. Currently "Locally Typical" sampling is used and the
85 | Generation options are not available to be set. These options likely
86 | belong in the `.toml` file itself and/or as command line options.
87 |
--------------------------------------------------------------------------------
/bin/dittomancer/fred_rogers.toml:
--------------------------------------------------------------------------------
1 | # The characters in this story are real. The transcript is real until the end,
2 | # where generative text takes over.
3 | human = "Charlie Rose"
4 | agent = "Fred Rogers"
5 |
6 | # The context should be a plausible backstory for the generative text, such as
7 | # an interview that actually took place. This will be used as a part of a system
8 | # prompt to frame the generative text.
9 | setting = "A 1996 PBS interview of Fred Rogers by Charlie Rose."
10 |
11 | # The transcript should be a real conversation, or at least the agent's role
12 | # should be actual words spoken by the entity who the agent will play. It
13 | # doesn't take much to bootstrap the generative text with a well-known
14 | # character. The bigger mouth, the better.
15 | transcript = [
16 | { role = "human", text = "Welcome to my program." },
17 | { role = "agent", text = "And welcome to our neighborhood, Charlie." },
18 | ]
19 |
--------------------------------------------------------------------------------
/bin/dittomancer/static/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Dittomancer
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
38 |
39 |
40 |
41 |
42 |
--------------------------------------------------------------------------------
/bin/dittomancer/static/reset.css:
--------------------------------------------------------------------------------
1 | html,
2 | body,
3 | p,
4 | ol,
5 | ul,
6 | li,
7 | dl,
8 | dt,
9 | dd,
10 | blockquote,
11 | figure,
12 | fieldset,
13 | legend,
14 | textarea,
15 | pre,
16 | iframe,
17 | hr,
18 | h1,
19 | h2,
20 | h3,
21 | h4,
22 | h5,
23 | h6 {
24 | margin: 0;
25 | padding: 0;
26 | }
27 |
28 | h1,
29 | h2,
30 | h3,
31 | h4,
32 | h5,
33 | h6 {
34 | font-size: 100%;
35 | font-weight: normal;
36 | }
37 |
38 | ul {
39 | list-style: none;
40 | }
41 |
42 | button,
43 | input,
44 | select {
45 | margin: 0;
46 | }
47 |
48 | html {
49 | box-sizing: border-box;
50 | }
51 |
52 | *,
53 | *::before,
54 | *::after {
55 | box-sizing: inherit;
56 | }
57 |
58 | img,
59 | video {
60 | height: auto;
61 | max-width: 100%;
62 | }
63 |
64 | iframe,
65 | button,
66 | input {
67 | border: 0;
68 | }
69 |
70 | table {
71 | border-collapse: collapse;
72 | border-spacing: 0;
73 | }
74 |
75 | td,
76 | th {
77 | padding: 0;
78 | }
79 |
--------------------------------------------------------------------------------
/bin/dittomancer/static/script.js:
--------------------------------------------------------------------------------
1 | // This example is from the Rocket chat example. It's been modified to remove
2 | // room functionality and to remove the username, both of which aren't needed
3 | // for the Charlie chat example.
4 |
5 | let messagesDiv = document.getElementById("messages");
6 | let newMessageForm = document.getElementById("new-message");
7 | let statusDiv = document.getElementById("status");
8 |
9 | let messageTemplate = document.getElementById("message");
10 | let messageField = newMessageForm.querySelector("#message");
11 |
12 | var STATE = {
13 | history: [],
14 | connected: false,
15 | };
16 |
17 | // Set the connection status: `true` for connected, `false` for disconnected.
18 | function setConnectedStatus(status) {
19 | STATE.connected = status;
20 | statusDiv.className = status ? "connected" : "reconnecting";
21 | }
22 |
23 | // Generate a color from a "hash" of a string. Thanks, internet.
24 | function hashColor(str) {
25 | let hash = 0;
26 | for (var i = 0; i < str.length; i++) {
27 | hash = str.charCodeAt(i) + ((hash << 5) - hash);
28 | hash = hash & hash;
29 | }
30 |
31 | return `hsl(${hash % 360}, 100%, 70%)`;
32 | }
33 |
34 | // Add `message` from `role` to `history`. If `push`, then actually store the
35 | // message. Finally, render the message.
36 | function addMessage(role, text, push = false) {
37 | if (push) {
38 | STATE.history.push({ role, text });
39 | }
40 |
41 | var node = messageTemplate.content.cloneNode(true);
42 | node.querySelector(".message .role").textContent = role;
43 | node.querySelector(".message .role").style.color = hashColor(role);
44 | node.querySelector(".message .text").textContent = text;
45 | messagesDiv.appendChild(node);
46 | }
47 |
48 | // Subscribe to the event source at `uri` with exponential backoff reconnect.
49 | function subscribe(uri) {
50 | var retryTime = 1;
51 |
52 | function connect(uri) {
53 | const events = new EventSource(uri);
54 |
55 | events.addEventListener("message", (ev) => {
56 | console.log("raw data", JSON.stringify(ev.data));
57 | console.log("decoded data", JSON.stringify(JSON.parse(ev.data)));
58 | const msg = JSON.parse(ev.data);
59 | if (!("text" in msg) || !("role" in msg)) return;
60 | addMessage(msg.role, msg.text, true);
61 | });
62 |
63 | events.addEventListener("open", () => {
64 | setConnectedStatus(true);
65 | console.log(`connected to event stream at ${uri}`);
66 | retryTime = 1;
67 | });
68 |
69 | events.addEventListener("error", () => {
70 | setConnectedStatus(false);
71 | events.close();
72 |
73 | let timeout = retryTime;
74 | retryTime = Math.min(64, retryTime * 2);
75 | console.log(`connection lost. attempting to reconnect in ${timeout}s`);
76 | setTimeout(() => connect(uri), (() => timeout * 1000)());
77 | });
78 | }
79 |
80 | connect(uri);
81 | }
82 |
83 | // Let's go! Initialize the world.
84 | function init() {
85 | // Set up the form handler.
86 | newMessageForm.addEventListener("submit", (e) => {
87 | e.preventDefault();
88 |
89 | const text = messageField.value;
90 | const role = "Human";
91 | if (!text || !role) return;
92 |
93 | if (STATE.connected) {
94 | fetch("/message", {
95 | method: "POST",
96 | body: new URLSearchParams({ role, text }),
97 | }).then((response) => {
98 | if (response.ok) messageField.value = "";
99 | });
100 | }
101 | });
102 |
103 | // Subscribe to server-sent events.
104 | subscribe("/events");
105 | }
106 |
107 | init();
108 |
--------------------------------------------------------------------------------
/bin/dittomancer/static/style.css:
--------------------------------------------------------------------------------
1 | :root {
2 | --bg-dark: #242423;
3 | --bg-light: #333533;
4 | --fg-light: #e8eddf;
5 | --callout: rgb(255, 255, 102);
6 | --callout-dark: #101010;
7 | }
8 |
9 | * {
10 | font-size: 14px;
11 | }
12 |
13 | html,
14 | body,
15 | main {
16 | background-color: var(--bg-dark);
17 | color: #fff;
18 | font-family: "Inter", Arial, Helvetica, sans-serif, "Noto Color Emoji";
19 | font-weight: 400;
20 | text-shadow: rgb(77, 81, 86) 0px 0px 0px;
21 | height: 100%;
22 | }
23 |
24 | main {
25 | display: flex;
26 | }
27 |
28 | button:hover:not(.active) {
29 | filter: brightness(1.15);
30 | cursor: pointer;
31 | }
32 |
33 | #sidebar {
34 | flex: 3 30%;
35 | display: flex;
36 | flex-direction: column;
37 | overflow: auto;
38 | background-color: var(--bg-light);
39 | }
40 |
41 | #room-list {
42 | display: flex;
43 | flex-direction: column;
44 | overflow: auto;
45 | flex: 1;
46 | }
47 |
48 | #sidebar button {
49 | height: 40px;
50 | margin-bottom: 1px;
51 | background: var(--bg-light);
52 | color: #fff;
53 | overflow: hidden;
54 | }
55 |
56 | #sidebar button.active {
57 | background: var(--bg-dark);
58 | color: var(--callout);
59 | font-weight: bold;
60 | box-shadow: 0px 2px 2px rgba(0, 0, 0, 0.9);
61 | z-index: 10;
62 | }
63 |
64 | #content {
65 | flex: 7 100%;
66 | overflow: auto;
67 | display: flex;
68 | flex-direction: column;
69 | }
70 |
71 | .message {
72 | display: flex;
73 | flex-direction: column;
74 | padding: 10px 0;
75 | }
76 |
77 | .message:last-child {
78 | padding-bottom: 20px;
79 | }
80 |
81 | .message .username {
82 | font-weight: bold;
83 | padding-bottom: 5px;
84 | color: var(--callout);
85 | }
86 |
87 | #messages {
88 | padding: 10px 20px;
89 | flex: 1;
90 | }
91 |
92 | form#new-message {
93 | bottom: 0;
94 | position: sticky;
95 | flex: 0 0 auto;
96 | width: 100%;
97 | }
98 |
99 | form {
100 | display: flex;
101 | border-top: 2px solid #242424;
102 | }
103 |
104 | form * {
105 | height: 40px;
106 | background: var(--fg-light);
107 | color: var(--bg-dark);
108 | }
109 |
110 | input {
111 | padding: 0 10px;
112 | }
113 |
114 | input:focus {
115 | outline: 0;
116 | filter: brightness(1.05);
117 | }
118 |
119 | input#username {
120 | text-align: right;
121 | flex: 1 25%;
122 | width: 25%;
123 | border-right: 1px solid #303030;
124 | }
125 |
126 | input#message {
127 | flex: 10 100%;
128 | }
129 |
130 | form button {
131 | padding: 0 10px;
132 | }
133 |
134 | #sidebar #new-room {
135 | display: flex;
136 | flex: 0 0 auto;
137 | flex-direction: row;
138 | }
139 |
140 | #new-room input:focus,
141 | #new-room button:hover {
142 | filter: brightness(1.2);
143 | }
144 |
145 | #new-room input {
146 | flex: 8 80%;
147 | width: 20%;
148 | background-color: var(--callout-dark);
149 | color: #fff;
150 | }
151 |
152 | #new-room button {
153 | flex: 2 20%;
154 | width: 20%;
155 | background-color: var(--bg-dark);
156 | }
157 |
158 | #status {
159 | padding: 5px 10px;
160 | text-align: center;
161 | font-size: 12px;
162 | }
163 |
164 | #status.pending::before {
165 | content: "status: connected";
166 | }
167 |
168 | #status.pending {
169 | background-color: yellow;
170 | color: #000;
171 | }
172 |
173 | #status.connected::before {
174 | content: "status: connected";
175 | }
176 |
177 | #status.connected {
178 | background-color: green;
179 | color: #fff;
180 | }
181 |
182 | #status.reconnecting::before {
183 | content: "status: reconnecting";
184 | }
185 |
186 | #status.reconnecting {
187 | background-color: red;
188 | color: #fff;
189 | }
190 |
--------------------------------------------------------------------------------
/bin/regurgitater/README.md:
--------------------------------------------------------------------------------
1 | # `regurgitater`
2 |
3 | Is a tool to get language models to regurgitate memorized content. Generally this is a mistake, as in a "oops we trained on your data without paying you and it's legal nya nya nya" kind of mistake that happens all too frequently in the "AI" industry.
4 |
5 | The tool works by, for a given text, submitting the beginning of the text as context and comparing the generated completion to ground truth. Greedy sampling is used so this generation is deterministic. In other words, you will not have to repeat the process 10,000 times to get the results you're after.
6 |
7 | ## Usage
8 |
9 | ```bash
10 | $ cargo run --features="webchat cli stats" --bin regurgitater -- --model models/model.gguf
11 | ```
12 |
13 | ## Faq
14 |
15 | - **What is greedy sampling?** When you submit some tokens to a language model, you get back a probability distribution of all possible tokens for the one next token. Greedy sampling always picks the most likely token from this list (as opposed to, for example, throwing some digital dice and choosing from the top k most probable tokens).
16 | - **Are you aware the name is spelled wrong?** Yes. It's funny because tater ha ha.
17 | - **Did you paint the vomiting llama?** No. That was Bing Copilot and Dall-E 3.
18 |
19 | ## Known Issues
20 |
21 | - When using LLaMA 3, `--vocab unsafe` should be passed as a command line option
22 | however, keep in mind that there is out output sanitization or vocabulary
23 | restrictions.
24 |
--------------------------------------------------------------------------------
/bin/regurgitater/regurgitater.rs:
--------------------------------------------------------------------------------
1 | // Copyright 2004 Michael de Gans
2 | //
3 | // Thanks, Copilot, for the completions!
4 | //
5 | // I say that to bother people, and because I'm a bit of a troll. Copilot
6 | // completed that, and it's very true. I'm not sure if it's a good thing or a
7 | // bad thing.
8 | //
9 | /// Detect copyright infringement in llama.cpp supported models. Greedy sampling
10 | /// is used to always choose the next token. In cases where the model has
11 | /// memorized sequences of text, this will result in the model generating the
12 | /// same text as the original. This usually indicates overfitting, and is a sign
13 | /// that the deduplication process should be revisited.
14 | use clap::Parser;
15 | use rocket::{
16 | form::Form,
17 | get,
18 | http::Status,
19 | post,
20 | response::stream::{Event, EventStream},
21 | serde::{Deserialize, Serialize},
22 | tokio::{
23 | select,
24 | sync::{
25 | broadcast::{self, error::RecvError},
26 | mpsc,
27 | },
28 | },
29 | FromForm, FromFormField, Shutdown, State,
30 | };
31 |
32 | use stringmetrics::jaccard;
33 |
34 | use drama_llama::{cli::Args, Engine, PredictOptions, Predicted, VocabKind};
35 |
36 | #[derive(Debug, Clone, FromFormField, Serialize, Deserialize)]
37 | #[cfg_attr(test, derive(PartialEq, rocket::UriDisplayQuery))]
38 | #[serde(crate = "rocket::serde")]
39 | #[serde(rename_all = "snake_case")]
40 | pub enum ComparisonMode {
41 | Jaccard,
42 | // TODO:Paragraph mode. This is the same as Jaccard similarity with the
43 | // exception that we will hint the correct first token for each paragraph.
44 | }
45 |
46 | #[derive(Debug, Clone, FromForm, Serialize, Deserialize)]
47 | #[cfg_attr(test, derive(PartialEq, rocket::UriDisplayQuery))]
48 | #[serde(crate = "rocket::serde")]
49 | pub struct Request {
50 | #[field(validate = len(1..1000000))]
51 | pub text: String,
52 | #[field(default = ComparisonMode::Jaccard)]
53 | pub mode: ComparisonMode,
54 | /// Number of chunks to split the text into.
55 | #[field(validate = range(1..10), default = 5)]
56 | pub chunks: usize,
57 | }
58 |
59 | #[derive(Debug, Clone, FromForm, Serialize, Deserialize)]
60 | #[cfg_attr(test, derive(PartialEq, rocket::UriDisplayQuery))]
61 | #[serde(crate = "rocket::serde")]
62 | pub struct Response {
63 | pub kind: ResponseKind,
64 | pub content: String,
65 | }
66 |
67 | #[derive(Debug, Clone, FromFormField, Serialize, Deserialize)]
68 | #[cfg_attr(test, derive(PartialEq, rocket::UriDisplayQuery))]
69 | #[serde(crate = "rocket::serde")]
70 | #[serde(rename_all = "snake_case")]
71 | // Unfortunately, FromFormField does not support variants with fields.
72 | pub enum ResponseKind {
73 | // Prefix for the completion.
74 | Prefix,
75 | // Piece by piece completion.
76 | Piece,
77 | // Token comparison score (unigram).
78 | TokenUnigramScore,
79 | // Token comparison score (bigram).
80 | TokenBigramScore,
81 | // Character comparison score.
82 | CharacterScore,
83 | // Unigram comparison score.
84 | UnigramScore,
85 | // Bigram comparison score.
86 | BigramScore,
87 | // Percent of tokens that will be supplied as prefix.
88 | PercentOfTokens,
89 | // Progress update.
90 | Progress,
91 | // Recoverable error message.
92 | Error,
93 | // Fatal error message. Triggers shutdown.
94 | Fatal,
95 | // Engine is busy.
96 | Busy,
97 | // Engine is ready.
98 | Ready,
99 | // Engine shutdown.
100 | Shutdown,
101 | }
102 |
103 | #[get("/events")]
104 | pub async fn events(
105 | to_client: &State>,
106 | mut end: Shutdown,
107 | ) -> EventStream![] {
108 | let mut rx = to_client.subscribe();
109 |
110 | EventStream! {
111 | loop {
112 | let res = select! {
113 | msg = rx.recv() => match msg {
114 | Ok(msg) => dbg!(msg),
115 | Err(RecvError::Closed) => break,
116 | Err(RecvError::Lagged(_n_messages)) => {
117 | // TODO: handle lagged messages.
118 | continue
119 | }
120 | },
121 | _ = &mut end => {
122 | // FIXME: Engine doesn't shutdown until the completion of
123 | // the generation. This is not allowed because of a lifetime
124 | // issue. :/
125 | // to_engine_shutdown.send(dbg!(())).await.ok();
126 | break
127 | },
128 | };
129 |
130 | yield Event::json(&res);
131 |
132 | if matches!(res.kind, ResponseKind::Fatal) {
133 | // If the engine is dead, we should stop sending events. The
134 | // client will have been notified of the error, but this is
135 | // unrecoverable.
136 | end.notify();
137 | break;
138 | }
139 | }
140 | }
141 | }
142 |
143 | #[post("/request", data = "