├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── data └── documents │ └── nietzsche │ ├── train.documents │ └── valid.documents ├── docs └── source │ └── _images │ ├── swagger_example.png │ ├── tb_example.png │ └── telegram_example.png ├── examples └── language_generation.py ├── full_stack_transformer ├── __init__.py ├── core │ ├── __init__.py │ ├── constants.py │ ├── data │ │ ├── __init__.py │ │ ├── dataset.py │ │ ├── encodings_collate.py │ │ ├── encodings_producer.py │ │ ├── encodings_sampler.py │ │ ├── text_inputs_producer.py │ │ └── text_lines_parsers.py │ ├── encoding.py │ ├── model_input.py │ ├── model_output.py │ ├── modelling │ │ ├── __init__.py │ │ ├── lightning.py │ │ ├── loading.py │ │ └── model.py │ ├── nn │ │ ├── __init__.py │ │ └── unlikelihood_candidates_loss.py │ ├── task_runner.py │ ├── text_input.py │ └── tokenizer.py ├── tasks │ ├── __init__.py │ ├── document_lm │ │ ├── __init__.py │ │ ├── data │ │ │ ├── __init__.py │ │ │ ├── dataset.py │ │ │ ├── encodings_collate.py │ │ │ └── text_lines_parsers.py │ │ ├── examples │ │ │ ├── __init__.py │ │ │ └── language_generation.py │ │ ├── hf_gpt2_tokenizer.py │ │ ├── language_generator │ │ │ ├── __init__.py │ │ │ ├── callback.py │ │ │ ├── generator.py │ │ │ ├── logits_modifiers.py │ │ │ └── progress.py │ │ ├── modelling │ │ │ ├── __init__.py │ │ │ ├── lightning.py │ │ │ ├── loading.py │ │ │ └── model.py │ │ ├── ru_transformers_tokenizer.py │ │ ├── serving │ │ │ ├── __init__.py │ │ │ ├── app.py │ │ │ ├── run.sh │ │ │ ├── schemas.py │ │ │ └── views.py │ │ ├── task_runner.py │ │ ├── telegram │ │ │ ├── __init__.py │ │ │ ├── app.py │ │ │ └── handlers.py │ │ ├── text_input.py │ │ └── tokenizer.py │ └── static │ │ ├── gpt2_bpe │ │ ├── merges.txt │ │ └── vocab.json │ │ └── ru_transformers_sp │ │ ├── merges.txt │ │ └── vocab.json └── utilities │ ├── __init__.py │ ├── arguments.py │ ├── experiment.py │ ├── factory.py │ ├── files.py │ ├── log_config.py │ ├── queue_iterable_dataset.py │ ├── seeding.py │ ├── sequences.py │ └── strings.py ├── requirements.txt └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # idea 107 | /.idea/ 108 | 109 | # Custom 110 | 111 | /data/ -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | ## [0.2.0] - 2020-05-13 5 | ### Changed 6 | - Library structure changed to the tasks-oriented design 7 | 8 | ## [0.1.0] - 2020-05-11 9 | ### Added 10 | - Training with meta text ([CTRL](https://arxiv.org/pdf/1909.05858.pdf)). 11 | 12 | ### Changed 13 | - Global library structure refactored for further expansion. 14 | - Data format has changed. Now no need for intermediate data arrays preparation. 15 | Training is performed on raw text files. 16 | - Now data is processed during the training with multiprocessing workers. 17 | - Embeddings now resized with mean embeddings vector (not the random one). 18 | 19 | ## [0.0.2] 20 | ### Added 21 | - Added telegram client for text generator service. 22 | - Added unlikelihood candidates loss. -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT NO-AI LICENSE 2 | 3 | Copyright 2025 Alexey Karnachev 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so. 6 | 7 | Permission is not granted to use this software or any of the associated files as sample data for the purposes of building machine learning models. 8 | 9 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Full Stack Transformer 2 | Pytorch library for end-to-end transformer models training, inference and serving. 3 |
4 |
5 | [Powered by](#powered-by) a list of great libraries. 6 | 7 | ## Library Design 8 | The library is organized in such a way, that the [core](full_stack_transformer/core) 9 | sub-package contains all modelling, data structures and data streaming classes. 10 | 11 | In [tasks](full_stack_transformer/tasks) sub-package there are all tasks-specific code. 12 | 13 | Available tasks: 14 | - [Document Language Model](#document-language-model) - the classic document-base language 15 | model training. It also provides application serving for interactive text generation. 16 | 17 | ## Document Language Model 18 | 19 | *For now, there is only 1 task available in the library. So, I'll put an example of this 20 | task usage right here in README. When another tasks will be implemented, I'll move 21 | all examples in the documentation.* 22 | 23 | ### Features 24 | - Automatic LM dataset preparation 25 | - End-to-end transformer LM training 26 | - [Unlikelihood loss](https://arxiv.org/pdf/1908.04319.pdf) training 27 | - Training LM with meta data (control codes, [CTRL](https://arxiv.org/pdf/1909.05858.pdf)) 28 | - Text generation tricks (top-k, [nucleus](http://arxiv.org/abs/1904.09751), repetition penalty, etc) 29 | - Text generation as a service 30 | - Telegram bot client 31 | 32 | ### Prepare dataset 33 | First, you need two text files (train and validation) which contain raw documents. 34 | For this example, files are already placed here: 35 | ``` 36 | data/documents/nietzsche/ 37 | ├── train.documents 38 | └── valid.documents 39 | ``` 40 | (If you want to start with your own files, check 41 | [Input Document Files Format](#input-document-files-format) 42 | 43 | ### Train Model 44 | The library uses `pytorch-lightning` for training and arguments which are used by 45 | lightning `Trainer` class are allowed as a command-line arguments for the script below. 46 | 47 | To check them all (as well as task-specific args) execute: 48 | ``` 49 | python full_stack_transformer/tasks/document_lm/task_runner.py --help 50 | ``` 51 | 52 | Now, let's train the model: 53 | ``` 54 | python full_stack_transformer/tasks/document_lm/task_runner.py \ 55 | --experiments_root=./data/experiments/ \ 56 | --model_path=gpt2 \ 57 | --tokenizer_class_name=HFGPT2DocumentTokenizer \ 58 | --batch_size=4 \ 59 | --max_meta_len=0 \ 60 | --max_body_len=70 \ 61 | --ignore_meta_prob=1.0 \ 62 | --train_file=./data/documents/nietzsche/train.documents \ 63 | --valid_file=./data/documents/nietzsche/valid.documents \ 64 | --learning_rate=2.5e-05 \ 65 | --num_warmup_steps=10 \ 66 | --num_cycles=1 \ 67 | --gpus="0," \ 68 | --val_check_interval=1.0 \ 69 | --max_epochs=10 \ 70 | --unlikelihood_alpha=5.0 \ 71 | --accumulate_grad_batches=8 \ 72 | --experiment_name=nietzsche 73 | ``` 74 | 75 | If you don't have `gpt2` model downloaded, it'll be obtained from the huggingface server (548M). 76 | Also, if you want to use a pre-trained gpt weights, which are stored locally, pass the path 77 | to the model directory, like so: `--model_path path/to/local/gpt/model`. 78 | Make sure, that you use an appropriate `--tokenizer_class_name` with your model. Check the 79 | list of available tokenizers: [Available Tokenizers](#available-tokenizers). 80 | 81 | The training has been started. All experiment related files a placed in the experiment directory: 82 | ``` 83 | data/experiments/nietzsche_v0/ 84 | ├── description.json 85 | ├── generated.txt 86 | ├── logs 87 | │   ├── debug.log 88 | │   ├── errors.log 89 | │   ├── info.log 90 | │   └── nietzsche 91 | │   └── version_0 92 | │   ├── events.out.tfevents.1589375895 93 | │   └── meta_tags.csv 94 | └── models 95 | ├── epoch=5.ckpt 96 | └── epoch=6.ckpt 97 | ``` 98 | 99 | **WARNING!** 100 | 101 | *In 0.2.0 version dynamic data generation is used. There are multiprocessing workers 102 | that produce samples for training and there is no graceful shutdown for these workers now. 103 | It'll be fixed in next version, but for now please make sure, that all training processes are dead 104 | when the training is finished. Or you can kill them manually:* 105 | ``` 106 | pkill -f nietzsche 107 | ``` 108 | 109 | 110 | ### Monitor training 111 | Run `tensorboard`: 112 | ``` 113 | tensorboard --logdir=./data/tb_logs/ --port=6006 114 | ``` 115 | TensorBoard interface is available here: [http://localhost:6006/](http://localhost:6006/) 116 |
117 | ![tb_example](docs/source/_images/tb_example.png) 118 | 119 | 120 | Also, text samples are generated during the training on each validation step. 121 | They are logged here: 122 | ``` 123 | cat data/experiments/nietzsche_v0/generated.txt 124 | ``` 125 | ``` 126 | ... 127 | { 128 | { 129 | "Global step": 14, 130 | "Current epoch": 0, 131 | "Generator params": { 132 | "max_number_of_generated_tokens": 128, 133 | "num_return_sequences": 8, 134 | "repetition_penalty": 1.0, 135 | "temperature": 0.7, 136 | "top_k": 0, 137 | "top_p": 1.0 138 | }, 139 | "Generated samples": [ 140 | "The last, most important aspect of a good writer... 141 | ... 142 | ``` 143 | 144 | 145 | ### Serve application 146 | 147 | When the model is trained, it could be served for inference: 148 | ``` 149 | ./full_stack_transformer/tasks/document_lm/serving/run.sh 9228 1 ./ ./data/experiments/nietzsche_v0/models/epoch\=6.ckpt cuda:0 150 | ``` 151 | 152 | Swagger is available here: [http://localhost:9228/docs](http://localhost:9228/docs) 153 |
154 | ![swagger_example](docs/source/_images/swagger_example.png) 155 | 156 | 157 | ### Serve telegram bot 158 | If you want to play with the text generation via telegram bot, you need the service run 159 | (previous step). Also, you need to obtain telegram api token. It could be easily done 160 | via [@BotFather](https://t.me/botfather). 161 | 162 | After you run the application server and got the api token, execute the following: 163 | ``` 164 | python full_stack_transformer/tasks/document_lm/telegram/app.py \ 165 | --telegram_api_token="API-TOKEN-OBTAINED-FROM-BOTFATHER" \ 166 | --text_generator_service_url=http://127.0.0.1:9228 167 | ``` 168 | 169 | That's it. Go find your bot in telegram and chat: 170 | ![telegram_example](docs/source/_images/telegram_example.png) 171 | 172 | 173 | ## Inference 174 | After you train the model, you may want to perform inference in a code. Check an 175 | example of [Language Generation](full_stack_transformer/tasks/document_lm/examples/language_generation.py). 176 | 177 | 178 | ## Input Document Files Format 179 | Raw input document files (train and valid) contains one document per line. 180 | Each document is a dict (json-like) with `body` and `meta`(optional) fields. 181 | 182 | For example: 183 | ``` 184 | {"body": "Article about animals", "meta": "Cats, dogs"} 185 | {"body": "Article about movies"} 186 | {"body": "Another article or document or whatever", "meta": "Some tags"} 187 | ... 188 | ``` 189 | 190 | ## Available Tokenizers 191 | For now, there are two tokenizers available for the `dialog_lm` task. 192 | 193 | - `HFGPT2DocumentTokenizer` for huggingface models: 194 | - `gpt2` 195 | - `gpt2-medium` 196 | - `gpt2-large` 197 | - `gpt2-xl` 198 | - `distilgpt2` 199 | - maybe, there are some more models already. 200 | Check [official huggingface repo](https://github.com/huggingface/transformers/blob/master/src/transformers/configuration_gpt2.py) 201 | - `RuTransformersDocumentTokenizer`: 202 | - [ru_transformers](https://github.com/mgrankin/ru_transformers) medium size model 203 | 204 | ## Powered By 205 | - [tokenizers](https://github.com/huggingface/tokenizers) fast tokenization and dataset preparation 206 | - [transformers](https://github.com/huggingface/transformers) model backbones 207 | - [pytorch-lightning](https://github.com/PyTorchLightning/pytorch-lightning) training process 208 | - [fastapi](https://github.com/tiangolo/fastapi) application serving 209 | - [aiogram](https://github.com/aiogram/aiogram) telegram bot serving 210 | 211 | Also, I predominantly work with russian texts, so I actively used pre-trained gpt-model 212 | and tokenizer (which I wrapped in the fast sentence piece tokenizer from 213 | [tokenizers](https://github.com/huggingface/tokenizers) library) 214 | from the [ru_transformers](https://github.com/mgrankin/ru_transformers) repository. -------------------------------------------------------------------------------- /data/documents/nietzsche/valid.documents: -------------------------------------------------------------------------------- 1 | {"body": "=Degree of Moral Susceptibility Unknown.=--The fact that one has or has\nnot had certain profoundly moving impressions and insights into\nthings--for example, an unjustly executed, slain or martyred father, a\nfaithless wife, a shattering, serious accident,--is the factor upon\nwhich the excitation of our passions to white heat principally depends,\nas well as the course of our whole lives. No one knows to what lengths\ncircumstances (sympathy, emotion) may lead him. He does not know the\nfull extent of his own susceptibility. Wretched environment makes him\nwretched. It is as a rule not the quality of our experience but its\nquantity upon which depends the development of our superiority or\ninferiority, from the point of view of good and evil.\n"} 2 | {"body": "=The Martyr Against His Will.=--In a certain movement there was a man\nwho was too cowardly and vacillating ever to contradict his comrades. He\nwas made use of in each emergency, every sacrifice was demanded of him\nbecause he feared the disfavor of his comrades more than he feared\ndeath: he was a petty, abject spirit. They perceived this and upon the\nfoundation of the qualities just mentioned they elevated him to the\naltitude of a hero, and finally even of a martyr. Although the cowardly\ncreature always inwardly said No, he always said Yes with his lips, even\nupon the scaffold, where he died for the tenets of his party: for beside\nhim stood one of his old associates who so domineered him with look and\nword that he actually went to his death with the utmost fortitude and\nhas ever since been celebrated as a martyr and exalted character.\n"} 3 | {"body": "=General Standard.=--One will rarely err if extreme actions be ascribed\nto vanity, ordinary actions to habit and mean actions to fear.\n"} 4 | {"body": "=Misunderstanding of Virtue.=--Whoever has obtained his experience of\nvice in connection with pleasure as in the case of one with a youth of\nwild oats behind him, comes to the conclusion that virtue must be\nconnected with self denial. Whoever, on the other hand, has been very\nmuch plagued by his passions and vices, longs to find in virtue the rest\nand peace of the soul. That is why it is possible for two virtuous\npeople to misunderstand one another wholly.\n"} 5 | {"body": "=The Ascetic.=--The ascetic makes out of virtue a slavery.\n"} 6 | {"body": "=Honor Transferred from Persons to Things.=--Actions prompted by love or\nby the spirit of self sacrifice for others are universally honored\nwherever they are manifest. Hence is magnified the value set upon\nwhatever things may be loved or whatever things conduce to self\nsacrifice: although in themselves they may be worth nothing much. A\nvaliant army is evidence of the value of the thing it fights for.\n"} 7 | {"body": "=Ambition a Substitute for Moral Feeling.=--Moral feeling should never\nbecome extinct in natures that are destitute of ambition. The ambitious\ncan get along without moral feeling just as well as with it.--Hence the\nsons of retired, ambitionless families, generally become by a series of\nrapid gradations, when they lose moral feeling, the most absolute\nlunkheads.\n"} 8 | {"body": "=Vanity Enriches.=--How poor the human mind would be without vanity! As\nit is, it resembles a well stacked and ever renewed ware-emporium that\nattracts buyers of every class: they can find almost everything, have\nalmost everything, provided they bring with them the right kind of\nmoney--admiration.\n"} 9 | {"body": "=Senility and Death.=--Apart from the demands made by religion, it may\nwell be asked why it is more honorable in an aged man, who feels the\ndecline of his powers, to await slow extinction than to fix a term to\nhis existence himself? Suicide in such a case is a quite natural and due\nproceeding that ought to command respect as a triumph of reason: and did\nin fact command respect during the times of the masters of Greek\nphilosophy and the bravest Roman patriots, who usually died by their own\nhand. Eagerness, on the other hand, to keep alive from day to day with\nthe anxious counsel of physicians, without capacity to attain any nearer\nto one's ideal of life, is far less worthy of respect.--Religions are\nvery rich in refuges from the mandate of suicide: hence they ingratiate\nthemselves with those who cling to life.\n"} 10 | {"body": "=Delusions Regarding Victim and Regarding Evil Doer.=--When the rich man\ntakes a possession away from the poor man (for example, a prince who\ndeprives a plebeian of his beloved) there arises in the mind of the poor\nman a delusion: he thinks the rich man must be wholly perverted to take\nfrom him the little that he has. But the rich man appreciates the value\nof a single possession much less because he is accustomed to many\npossessions, so that he cannot put himself in the place of the poor man\nand does not act by any means as ill as the latter supposes. Both have a\ntotally false idea of each other. The iniquities of the mighty which\nbulk most largely in history are not nearly so monstrous as they seem.\nThe hereditary consciousness of being a superior being with superior\nenvironment renders one very callous and lulls the conscience to rest.\nWe all feel, when the difference between ourselves and some other being\nis exceedingly great, that no element of injustice can be involved, and\nwe kill a fly with no qualms of conscience whatever. So, too, it is no\nindication of wickedness in Xerxes (whom even the Greeks represent as\nexceptionally noble) that he deprived a father of his son and had him\ndrawn and quartered because the latter had manifested a troublesome,\nominous distrust of an entire expedition: the individual was in this\ncase brushed aside as a pestiferous insect. He was too low and mean to\njustify continued sentiments of compunction in the ruler of the world.\nIndeed no cruel man is ever as cruel, in the main, as his victim thinks.\nThe idea of pain is never the same as the sensation. The rule is\nprecisely analogous in the case of the unjust judge, and of the\njournalist who by means of devious rhetorical methods, leads public\nopinion astray. Cause and effect are in all these instances entwined\nwith totally different series of feeling and thoughts, whereas it is\nunconsciously assumed that principal and victim feel and think exactly\nalike, and because of this assumption the guilt of the one is based upon\nthe pain of the other.\n"} 11 | {"body": "=The Soul's Skin.=--As the bones, flesh, entrails and blood vessels are\nenclosed by a skin that renders the aspect of men endurable, so the\nimpulses and passions of the soul are enclosed by vanity: it is the skin\nof the soul.\n"} 12 | {"body": "=Sleep of Virtue.=--If virtue goes to sleep, it will be more vigorous\nwhen it awakes.\n"} 13 | {"body": "=Subtlety of Shame.=--Men are not ashamed of obscene thoughts, but they\nare ashamed when they suspect that obscene thoughts are attributed to\nthem.\n"} 14 | {"body": "=Naughtiness Is Rare.=--Most people are too much absorbed in themselves\nto be bad.\n"} 15 | {"body": "=The Mite in the Balance.=--We are praised or blamed, as the one or the\nother may be expedient, for displaying to advantage our power of\ndiscernment.\n"} 16 | {"body": "=Luke 18:14 Improved.=--He that humbleth himself wisheth to be exalted.\n"} 17 | {"body": "=Prevention of Suicide.=--There is a justice according to which we may\ndeprive a man of life, but none that permits us to deprive him of death:\nthis is merely cruelty.\n"} 18 | {"body": "=Vanity.=--We set store by the good opinion of men, first because it is\nof use to us and next because we wish to give them pleasure (children\ntheir parents, pupils their teacher, and well disposed persons all\nothers generally). Only when the good opinion of men is important to\nsomebody, apart from personal advantage or the desire to give pleasure,\ndo we speak of vanity. In this last case, a man wants to give himself\npleasure, but at the expense of his fellow creatures, inasmuch as he\ninspires them with a false opinion of himself or else inspires \"good\nopinion\" in such a way that it is a source of pain to others (by\narousing envy). The individual generally seeks, through the opinion of\nothers, to attest and fortify the opinion he has of himself; but the\npotent influence of authority--an influence as old as man himself--leads\nmany, also, to strengthen their own opinion of themselves by means of\nauthority, that is, to borrow from others the expedient of relying more\nupon the judgment of their fellow men than upon their own.--Interest in\noneself, the wish to please oneself attains, with the vain man, such\nproportions that he first misleads others into a false, unduly exalted\nestimate of himself and then relies upon the authority of others for his\nself estimate; he thus creates the delusion that he pins his faith\nto.--It must, however, be admitted that the vain man does not desire to\nplease others so much as himself and he will often go so far, on this\naccount, as to overlook his own interests: for he often inspires his\nfellow creatures with malicious envy and renders them ill disposed in\norder that he may thus increase his own delight in himself.\n"} 19 | {"body": "=Limits of the Love of Mankind.=--Every man who has declared that some\nother man is an ass or a scoundrel, gets angry when the other man\nconclusively shows that the assertion was erroneous.\n"} 20 | {"body": "=Weeping Morality.=--How much delight morality occasions! Think of the\nocean of pleasing tears that has flowed from the narration of noble,\ngreat-hearted deeds!--This charm of life would disappear if the belief\nin complete irresponsibility gained the upper hand.\n"} 21 | {"body": "=Origin of Justice.=--Justice (reasonableness) has its origin among\napproximate equals in power, as Thucydides (in the dreadful conferences\nof the Athenian and Melian envoys) has rightly conceived. Thus, where\nthere exists no demonstrable supremacy and a struggle leads but to\nmutual, useless damage, the reflection arises that an understanding\nwould best be arrived at and some compromise entered into. The\nreciprocal nature is hence the first nature of justice. Each party makes\nthe other content inasmuch as each receives what it prizes more highly\nthan the other. Each surrenders to the other what the other wants and\nreceives in return its own desire. Justice is therefore reprisal and\nexchange upon the basis of an approximate equality of power. Thus\nrevenge pertains originally to the domain of justice as it is a sort of\nreciprocity. Equally so, gratitude.--Justice reverts naturally to the\nstandpoint of self preservation, therefore to the egoism of this\nconsideration: \"why should I injure myself to no purpose and perhaps\nnever attain my end?\"--So much for the origin of justice. Only because\nmen, through mental habits, have forgotten the original motive of so\ncalled just and rational acts, and also because for thousands of years\nchildren have been brought to admire and imitate such acts, have they\ngradually assumed the appearance of being unegotistical. Upon this\nappearance is founded the high estimate of them, which, moreover, like\nall estimates, is continually developing, for whatever is highly\nesteemed is striven for, imitated, made the object of self sacrifice,\nwhile the merit of the pain and emulation thus expended is, by each\nindividual, ascribed to the thing esteemed.--How slightly moral would\nthe world appear without forgetfulness! A poet could say that God had\nposted forgetfulness as a sentinel at the portal of the temple of human\nmerit!\n"} 22 | {"body": "=Concerning the Law of the Weaker.=--Whenever any party, for instance, a\nbesieged city, yields to a stronger party, under stipulated conditions,\nthe counter stipulation is that there be a reduction to insignificance,\na burning and destruction of the city and thus a great damage inflicted\nupon the stronger party. Thus arises a sort of equalization principle\nupon the basis of which a law can be established. The enemy has an\nadvantage to gain by its maintenance.--To this extent there is also a\nlaw between slaves and masters, limited only by the extent to which the\nslave may be useful to his master. The law goes originally only so far\nas the one party may appear to the other potent, invincible, stable, and\nthe like. To such an extent, then, the weaker has rights, but very\nlimited ones. Hence the famous dictum that each has as much law on his\nside as his power extends (or more accurately, as his power is believed\nto extend).\n"} 23 | {"body": "=The Three Phases of Morality Hitherto.=--It is the first evidence that\nthe animal has become human when his conduct ceases to be based upon the\nimmediately expedient, but upon the permanently useful; when he has,\ntherefore, grown utilitarian, capable of purpose. Thus is manifested the\nfirst rule of reason. A still higher stage is attained when he regulates\nhis conduct upon the basis of honor, by means of which he gains mastery\nof himself and surrenders his desires to principles; this lifts him far\nabove the phase in which he was actuated only by considerations of\npersonal advantage as he understood it. He respects and wishes to be\nrespected. This means that he comprehends utility as a thing dependent\nupon what his opinion of others is and their opinion of him. Finally he\nregulates his conduct (the highest phase of morality hitherto attained)\nby his own standard of men and things. He himself decides, for himself\nand for others, what is honorable and what is useful. He has become a\nlaw giver to opinion, upon the basis of his ever higher developing\nconception of the utilitarian and the honorable. Knowledge makes him\ncapable of placing the highest utility, (that is, the universal,\nenduring utility) before merely personal utility,--of placing ennobling\nrecognition of the enduring and universal before the merely temporary:\nhe lives and acts as a collective individuality.\n"} 24 | {"body": "=Ethic of the Developed Individual.=--Hitherto the altruistic has been\nlooked upon as the distinctive characteristic of moral conduct, and it\nis manifest that it was the consideration of universal utility that\nprompted praise and recognition of altruistic conduct. Must not a\nradical departure from this point of view be imminent, now that it is\nbeing ever more clearly perceived that in the most personal\nconsiderations the most general welfare is attained: so that conduct\ninspired by the most personal considerations of advantage is just the\nsort which has its origin in the present conception of morality (as a\nuniversal utilitarianism)? To contemplate oneself as a complete\npersonality and bear the welfare of that personality in mind in all that\none does--this is productive of better results than any sympathetic\nsusceptibility and conduct in behalf of others. Indeed we all suffer\nfrom such disparagement of our own personalities, which are at present\nmade to deteriorate from neglect. Capacity is, in fact, divorced from\nour personality in most cases, and sacrificed to the state, to science,\nto the needy, as if it were the bad which deserved to be made a\nsacrifice. Now, we are willing to labor for our fellowmen but only to\nthe extent that we find our own highest advantage in so doing, no more,\nno less. The whole matter depends upon what may be understood as one's\nadvantage: the crude, undeveloped, rough individualities will be the\nvery ones to estimate it most inadequately.\n"} 25 | {"body": "=Usage and Ethic.=--To be moral, virtuous, praiseworthy means to yield\nobedience to ancient law and hereditary usage. Whether this obedience be\nrendered readily or with difficulty is long immaterial. Enough that it\nbe rendered. \"Good\" finally comes to mean him who acts in the\ntraditional manner, as a result of heredity or natural disposition, that\nis to say does what is customary with scarcely an effort, whatever that\nmay be (for example revenges injuries when revenge, as with the ancient\nGreeks, was part of good morals). He is called good because he is good\n\"to some purpose,\" and as benevolence, sympathy, considerateness,\nmoderation and the like come, in the general course of conduct, to be\nfinally recognized as \"good to some purpose\" (as utilitarian) the\nbenevolent man, the helpful man, is duly styled \"good\". (At first other\nand more important kinds of utilitarian qualities stand in the\nforeground.) Bad is \"not habitual\" (unusual), to do things not in\naccordance with usage, to oppose the traditional, however rational or\nthe reverse the traditional may be. To do injury to one's social group\nor community (and to one's neighbor as thus understood) is looked upon,\nthrough all the variations of moral laws, in different ages, as the\npeculiarly \"immoral\" act, so that to-day we associate the word \"bad\"\nwith deliberate injury to one's neighbor or community. \"Egoistic\" and\n\"non-egoistic\" do not constitute the fundamental opposites that have\nbrought mankind to make a distinction between moral and immoral, good\nand bad; but adherence to traditional custom, and emancipation from it.\nHow the traditional had its origin is quite immaterial; in any event it\nhad no reference to good and bad or any categorical imperative but to\nthe all important end of maintaining and sustaining the community, the\nrace, the confederation, the nation. Every superstitious custom that\noriginated in a misinterpreted event or casualty entailed some\ntradition, to adhere to which is moral. To break loose from it is\ndangerous, more prejudicial to the community than to the individual\n(because divinity visits the consequences of impiety and sacrilege upon\nthe community rather than upon the individual). Now every tradition\ngrows ever more venerable--the more remote is its origin, the more\nconfused that origin is. The reverence due to it increases from\ngeneration to generation. The tradition finally becomes holy and\ninspires awe. Thus it is that the precept of piety is a far loftier\nmorality than that inculcated by altruistic conduct.\n"} 26 | {"body": "=Delight in the Moral.=--A potent species of joy (and thereby the source\nof morality) is custom. The customary is done more easily, better,\ntherefore preferably. A pleasure is felt in it and experience thus shows\nthat since this practice has held its own it must be good. A manner or\nmoral that lives and lets live is thus demonstrated advantageous,\nnecessary, in contradistinction to all new and not yet adopted\npractices. The custom is therefore the blending of the agreeable and the\nuseful. Moreover it does not require deliberation. As soon as man can\nexercise compulsion, he exercises it to enforce and establish his\ncustoms, for they are to him attested lifewisdom. So, too, a community\nof individuals constrains each one of their number to adopt the same\nmoral or custom. The error herein is this: Because a certain custom has\nbeen agreeable to the feelings or at least because it proves a means of\nmaintenance, this custom must be imperative, for it is regarded as the\nonly thing that can possibly be consistent with well being. The well\nbeing of life seems to spring from it alone. This conception of the\ncustomary as a condition of existence is carried into the slightest\ndetail of morality. Inasmuch as insight into true causation is quite\nrestricted in all inferior peoples, a superstitious anxiety is felt that\neverything be done in due routine. Even when a custom is exceedingly\nburdensome it is preserved because of its supposed vital utility. It is\nnot known that the same degree of satisfaction can be experienced\nthrough some other custom and even higher degrees of satisfaction, too.\nBut it is fully appreciated that all customs do become more agreeable\nwith the lapse of time, no matter how difficult they may have been found\nin the beginning, and that even the severest way of life may be rendered\na matter of habit and therefore a pleasure.\n"} 27 | {"body": "=Pleasure and Social Instinct.=--Through his relations with other men,\nman derives a new species of delight in those pleasurable emotions which\nhis own personality affords him; whereby the domain of pleasurable\nemotions is made infinitely more comprehensive. No doubt he has\ninherited many of these feelings from the brutes, which palpably feel\ndelight when they sport with one another, as mothers with their young.\nSo, too, the sexual relations must be taken into account: they make\nevery young woman interesting to every young man from the standpoint of\npleasure, and conversely. The feeling of pleasure originating in human\nrelationships makes men in general better. The delight in common, the\npleasures enjoyed together heighten one another. The individual feels a\nsense of security. He becomes better natured. Distrust and malice\ndissolve. For the man feels the sense of benefit and observes the same\nfeeling in others. Mutual manifestations of pleasure inspire mutual\nsympathy, the sentiment of homogeneity. The same effect is felt also at\nmutual sufferings, in a common danger, in stormy weather. Upon such a\nfoundation are built the earliest alliances: the object of which is the\nmutual protection and safety from threatening misfortunes, and the\nwelfare of each individual. And thus the social instinct develops from\npleasure.\n"} 28 | {"body": "=The Guiltless Nature of So-Called Bad Acts.=--All \"bad\" acts are\ninspired by the impulse to self preservation or, more accurately, by\nthe desire for pleasure and for the avoidance of pain in the individual.\nThus are they occasioned, but they are not, therefore, bad. \"Pain self\nprepared\" does not exist, except in the brains of the philosophers, any\nmore than \"pleasure self prepared\" (sympathy in the Schopenhauer sense).\nIn the condition anterior to the state we kill the creature, be it man\nor ape, that attempts to pluck the fruit of a tree before we pluck it\nourselves should we happen to be hungry at the time and making for that\ntree: as we would do to-day, so far as the brute is concerned, if we\nwere wandering in savage regions.--The bad acts which most disturb us at\npresent do so because of the erroneous supposition that the one who is\nguilty of them towards us has a free will in the matter and that it was\nwithin his discretion not to have done these evil things. This belief in\ndiscretionary power inspires hate, thirst for revenge, malice, the\nentire perversion of the mental processes, whereas we would feel in no\nway incensed against the brute, as we hold it irresponsible. To inflict\npain not from the instinct of self preservation but in requital--this is\nthe consequence of false judgment and is equally a guiltless course of\nconduct. The individual can, in that condition which is anterior to the\nstate, act with fierceness and violence for the intimidation of another\ncreature, in order to render his own power more secure as a result of\nsuch acts of intimidation. Thus acts the powerful, the superior, the\noriginal state founder, who subjugates the weaker. He has the right to\ndo so, as the state nowadays assumes the same right, or, to be more\naccurate, there is no right that can conflict with this. A foundation\nfor all morality can first be laid only when a stronger individuality or\na collective individuality, for example society, the state, subjects the\nsingle personalities, hence builds upon their unification and\nestablishes a bond of union. Morality results from compulsion, it is\nindeed itself one long compulsion to which obedience is rendered in\norder that pain may be avoided. At first it is but custom, later free\nobedience and finally almost instinct. At last it is (like everything\nhabitual and natural) associated with pleasure--and is then called\nvirtue.\n"} 29 | {"body": "=Shame.=--Shame exists wherever a \"mystery\" exists: but this is a\nreligious notion which in the earlier period of human civilization had\ngreat vogue. Everywhere there were circumscribed spots to which access\nwas denied on account of some divine law, except in special\ncircumstances. At first these spots were quite extensive, inasmuch as\nstipulated areas could not be trod by the uninitiated, who, when near\nthem, felt tremors and anxieties. This sentiment was frequently\ntransferred to other relationships, for example to sexual relations,\nwhich, as the privilege and gateway of mature age, must be withdrawn\nfrom the contemplation of youth for its own advantage: relations which\nmany divinities were busy in preserving and sanctifying, images of which\ndivinities were duly placed in marital chambers as guardians. (In\nTurkish such an apartment is termed a harem or holy thing, the same word\nalso designating the vestibule of a mosque). So, too, Kingship is\nregarded as a centre from which power and brilliance stream forth, as a\nmystery to the subjects, impregnated with secrecy and shame, sentiments\nstill quite operative among peoples who in other respects are without\nany shame at all. So, too, is the whole world of inward states, the\nso-called \"soul,\" even now, for all non-philosophical persons, a\n\"mystery,\" and during countless ages it was looked upon as a something\nof divine origin, in direct communion with deity. It is, therefore, an\nadytum and occasions shame.\n"} 30 | {"body": "=Judge Not.=--Care must be taken, in the contemplation of earlier ages,\nthat there be no falling into unjust scornfulness. The injustice in\nslavery, the cruelty in the subjugation of persons and peoples must not\nbe estimated by our standard. For in that period the instinct of justice\nwas not so highly developed. Who dare reproach the Genoese Calvin for\nburning the physician Servetus at the stake? It was a proceeding growing\nout of his convictions. And the Inquisition, too, had its justification.\nThe only thing is that the prevailing views were false and led to those\nproceedings which seem so cruel to us, simply because such views have\nbecome foreign to us. Besides, what is the burning alive of one\nindividual compared with eternal hell pains for everybody else? And yet\nthis idea then had hold of all the world without in the least vitiating,\nwith its frightfulness, the other idea of a god. Even we nowadays are\nhard and merciless to political revolutionists, but that is because we\nare in the habit of believing the state a necessity, and hence the\ncruelty of the proceeding is not so much understood as in the other\ncases where the points of view are repudiated. The cruelty to animals\nshown by children and Italians is due to the same misunderstanding. The\nanimal, owing to the exigencies of the church catechism, is placed too\nfar below the level of mankind.--Much, too, that is frightful and\ninhuman in history, and which is almost incredible, is rendered less\natrocious by the reflection that the one who commands and the one who\nexecutes are different persons. The former does not witness the\nperformance and hence it makes no strong impression on him. The latter\nobeys a superior and hence feels no responsibility. Most princes and\nmilitary chieftains appear, through lack of true perception, cruel and\nhard without really being so.--Egoism is not bad because the idea of the\n\"neighbor\"--the word is of Christian origin and does not correspond to\ntruth--is very weak in us, and we feel ourselves, in regard to him, as\nfree from responsibility as if plants and stones were involved. That\nanother is in suffering must be learned and it can never be wholly\nlearned.\n"} 31 | {"body": "\"=Man Always Does Right.=\"--We do not blame nature when she sends a\nthunder storm and makes us wet: why then do we term the man who inflicts\ninjury immoral? Because in the latter case we assume a voluntary,\nruling, free will, and in the former necessity. But this distinction is\na delusion. Moreover, even the intentional infliction of injury is not,\nin all circumstances termed immoral. Thus, we kill a fly intentionally\nwithout thinking very much about it, simply because its buzzing about is\ndisagreeable; and we punish a criminal and inflict pain upon him in\norder to protect ourselves and society. In the first case it is the\nindividual who, for the sake of preserving himself or in order to spare\nhimself pain, does injury with design: in the second case, it is the\nstate. All ethic deems intentional infliction of injury justified by\nnecessity; that is when it is a matter of self preservation. But these\ntwo points of view are sufficient to explain all bad acts done by man to\nmen. It is desired to obtain pleasure or avoid pain. In any sense, it is\na question, always, of self preservation. Socrates and Plato are right:\nwhatever man does he always does right: that is, does what seems to him\ngood (advantageous) according to the degree of advancement his intellect\nhas attained, which is always the measure of his rational capacity.\n"} 32 | {"body": "=The Inoffensive in Badness.=--Badness has not for its object the\ninfliction of pain upon others but simply our own satisfaction as, for\ninstance, in the case of thirst for vengeance or of nerve excitation.\nEvery act of teasing shows what pleasure is caused by the display of\nour power over others and what feelings of delight are experienced in\nthe sense of domination. Is there, then, anything immoral in feeling\npleasure in the pain of others? Is malicious joy devilish, as\nSchopenhauer says? In the realm of nature we feel joy in breaking\nboughs, shattering rocks, fighting with wild beasts, simply to attest\nour strength thereby. Should not the knowledge that another suffers on\nour account here, in this case, make the same kind of act, (which, by\nthe way, arouses no qualms of conscience in us) immoral also? But if we\nhad not this knowledge there would be no pleasure in one's own\nsuperiority or power, for this pleasure is experienced only in the\nsuffering of another, as in the case of teasing. All pleasure is, in\nitself, neither good nor bad. Whence comes the conviction that one\nshould not cause pain in others in order to feel pleasure oneself?\nSimply from the standpoint of utility, that is, in consideration of the\nconsequences, of ultimate pain, since the injured party or state will\ndemand satisfaction and revenge. This consideration alone can have led\nto the determination to renounce such pleasure.--Sympathy has the\nsatisfaction of others in view no more than, as already stated, badness\nhas the pain of others in view. For there are at least two (perhaps many\nmore) elementary ingredients in personal gratification which enter\nlargely into our self satisfaction: one of them being the pleasure of\nthe emotion, of which species is sympathy with tragedy, and another,\nwhen the impulse is to action, being the pleasure of exercising one's\npower. Should a sufferer be very dear to us, we divest ourselves of pain\nby the performance of acts of sympathy.--With the exception of some few\nphilosophers, men have placed sympathy very low in the rank of moral\nfeelings: and rightly.\n"} 33 | {"body": "=Self Defence.=--If self defence is in general held a valid\njustification, then nearly every manifestation of so called immoral\negoism must be justified, too. Pain is inflicted, robbery or killing\ndone in order to maintain life or to protect oneself and ward off harm.\nA man lies when cunning and delusion are valid means of self\npreservation. To injure intentionally when our safety and our existence\nare involved, or the continuance of our well being, is conceded to be\nmoral. The state itself injures from this motive when it hangs\ncriminals. In unintentional injury the immoral, of course, can not be\npresent, as accident alone is involved. But is there any sort of\nintentional injury in which our existence and the maintenance of our\nwell being be not involved? Is there such a thing as injuring from\nabsolute badness, for example, in the case of cruelty? If a man does not\nknow what pain an act occasions, that act is not one of wickedness. Thus\nthe child is not bad to the animal, not evil. It disturbs and rends it\nas if it were one of its playthings. Does a man ever fully know how much\npain an act may cause another? As far as our nervous system extends, we\nshield ourselves from pain. If it extended further, that is, to our\nfellow men, we would never cause anyone else any pain (except in such\ncases as we cause it to ourselves, when we cut ourselves, surgically, to\nheal our ills, or strive and trouble ourselves to gain health). We\nconclude from analogy that something pains somebody and can in\nconsequence, through recollection and the power of imagination, feel\npain also. But what a difference there always is between the tooth ache\nand the pain (sympathy) that the spectacle of tooth ache occasions!\nTherefore when injury is inflicted from so called badness the degree of\npain thereby experienced is always unknown to us: in so far, however, as\npleasure is felt in the act (a sense of one's own power, of one's own\nexcitation) the act is committed to maintain the well being of the\nindividual and hence comes under the purview of self defence and lying\nfor self preservation. Without pleasure, there is no life; the struggle\nfor pleasure is the struggle for life. Whether the individual shall\ncarry on this struggle in such a way that he be called good or in such a\nway that he be called bad is something that the standard and the\ncapacity of his own intellect must determine for him.\n"} 34 | {"body": "=Justice that Rewards.=--Whoever has fully understood the doctrine of\nabsolute irresponsibility can no longer include the so called rewarding\nand punishing justice in the idea of justice, if the latter be taken to\nmean that to each be given his due. For he who is punished does not\ndeserve the punishment. He is used simply as a means to intimidate\nothers from certain acts. Equally, he who is rewarded does not merit the\nreward. He could not act any differently than he did act. Hence the\nreward has only the significance of an encouragement to him and others\nas a motive for subsequent acts. The praise is called out only to him\nwho is running in the race and not to him who has arrived at the goal.\nSomething that comes to someone as his own is neither a punishment nor a\nreward. It is given to him from utiliarian considerations, without his\nhaving any claim to it in justice. Hence one must say \"the wise man\npraises not because a good act has been done\" precisely as was once\nsaid: \"the wise man punishes not because a bad act has been done but in\norder that a bad act may not be done.\" If punishment and reward ceased,\nthere would cease with them the most powerful incentives to certain acts\nand away from other acts. The purposes of men demand their continuance\n[of punishment and reward] and inasmuch as punishment and reward, blame\nand praise operate most potently upon vanity, these same purposes of men\nimperatively require the continuance of vanity.\n"} 35 | {"body": "=The Water Fall.=--At the sight of a water fall we may opine that in the\ncountless curves, spirations and dashes of the waves we behold freedom\nof the will and of the impulses. But everything is compulsory,\neverything can be mathematically calculated. Thus it is, too, with human\nacts. We would be able to calculate in advance every single action if we\nwere all knowing, as well as every advance in knowledge, every delusion,\nevery bad deed. The acting individual himself is held fast in the\nillusion of volition. If, on a sudden, the entire movement of the world\nstopped short, and an all knowing and reasoning intelligence were there\nto take advantage of this pause, he could foretell the future of every\nbeing to the remotest ages and indicate the path that would be taken in\nthe world's further course. The deception of the acting individual as\nregards himself, the assumption of the freedom of the will, is a part of\nthis computable mechanism.\n"} 36 | {"body": "=Non-Responsibility and Non-Guilt.=--The absolute irresponsibility of\nman for his acts and his nature is the bitterest drop in the cup of him\nwho has knowledge, if he be accustomed to behold in responsibility and\nduty the patent of nobility of his human nature. All his estimates,\npreferences, dislikes are thus made worthless and false. His deepest\nsentiment, with which he honored the sufferer, the hero, sprang from an\nerror. He may no longer praise, no longer blame, for it is irrational to\nblame and praise nature and necessity. Just as he cherishes the\nbeautiful work of art, but does not praise it (as it is incapable of\ndoing anything for itself), just as he stands in the presence of plants,\nhe must stand in the presence of human conduct, his own included. He may\nadmire strength, beauty, capacity, therein, but he can discern no merit.\nThe chemical process and the conflict of the elements, the ordeal of\nthe invalid who strives for convalescence, are no more merits than the\nsoul-struggles and extremities in which one is torn this way and that by\ncontending motives until one finally decides in favor of the\nstrongest--as the phrase has it, although, in fact, it is the strongest\nmotive that decides for us. All these motives, however, whatever fine\nnames we may give them, have grown from the same roots in which we\nbelieve the baneful poisons lurk. Between good and bad actions there is\nno difference in kind but, at most, in degree. Good acts are sublimated\nevil. Bad acts are degraded, imbruted good. The very longing of the\nindividual for self gratification (together with the fear of being\ndeprived of it) obtains satisfaction in all circumstances, let the\nindividual act as he may, that is, as he must: be it in deeds of vanity,\nrevenge, pleasure, utility, badness, cunning, be it in deeds of self\nsacrifice, sympathy or knowledge. The degrees of rational capacity\ndetermine the direction in which this longing impels: every society,\nevery individual has constantly present a comparative classification of\nbenefits in accordance with which conduct is determined and others are\njudged. But this standard perpetually changes. Many acts are called bad\nthat are only stupid, because the degree of intelligence that decided\nfor them was low. Indeed, in a certain sense, all acts now are stupid,\nfor the highest degree of human intelligence that has yet been attained\nwill in time most certainly be surpassed and then, in retrospection, all\nour present conduct and opinion will appear as narrow and petty as we\nnow deem the conduct and opinion of savage peoples and ages.--To\nperceive all these things may occasion profound pain but there is,\nnevertheless, a consolation. Such pains are birth pains. The butterfly\ninsists upon breaking through the cocoon, he presses through it, tears\nit to pieces, only to be blinded and confused by the strange light, by\nthe realm of liberty. By such men as are capable of this sadness--how\nfew there are!--will the first attempt be made to see if humanity may\nconvert itself from a thing of morality to a thing of wisdom. The sun of\na new gospel sheds its first ray upon the loftiest height in the souls\nof those few: but the clouds are massed there, too, thicker than ever,\nand not far apart are the brightest sunlight and the deepest gloom.\nEverything is necessity--so says the new knowledge: and this knowledge\nis itself necessity. All is guiltlessness, and knowledge is the way to\ninsight into this guiltlessness. If pleasure, egoism, vanity be\nnecessary to attest the moral phenomena and their richest blooms, the\ninstinct for truth and accuracy of knowledge; if delusion and confusion\nof the imagination were the only means whereby mankind could gradually\nlift itself up to this degree of self enlightenment and self\nemancipation--who would venture to disparage the means? Who would have\nthe right to feel sad if made aware of the goal to which those paths\nlead? Everything in the domain of ethic is evolved, changeable,\ntottering; all things flow, it is true--but all things are also in the\nstream: to their goal. Though within us the hereditary habit of\nerroneous judgment, love, hate, may be ever dominant, yet under the\ninfluence of awaking knowledge it will ever become weaker: a new habit,\nthat of understanding, not-loving, not-hating, looking from above, grows\nup within us gradually and in the same soil, and may, perhaps, in\nthousands of years be powerful enough to endow mankind with capacity to\ndevelop the wise, guiltless man (conscious of guiltlessness) as\nunfailingly as it now developes the unwise, irrational, guilt-conscious\nman--that is to say, the necessary higher step, not the opposite of it.\n"} 37 | {"body": "THE RELIGIOUS LIFE.\n"} 38 | {"body": "=The Double Contest Against Evil.=--If an evil afflicts us we can either\nso deal with it as to remove its cause or else so deal with it that its\neffect upon our feeling is changed: hence look upon the evil as a\nbenefit of which the uses will perhaps first become evident in some\nsubsequent period. Religion and art (and also the metaphysical\nphilosophy) strive to effect an alteration of the feeling, partly by an\nalteration of our judgment respecting the experience (for example, with\nthe aid of the dictum \"whom God loves, he chastizes\") partly by the\nawakening of a joy in pain, in emotion especially (whence the art of\ntragedy had its origin). The more one is disposed to interpret away and\njustify, the less likely he is to look directly at the causes of evil\nand eliminate them. An instant alleviation and narcotizing of pain, as\nis usual in the case of tooth ache, is sufficient for him even in the\nseverest suffering. The more the domination of religions and of all\nnarcotic arts declines, the more searchingly do men look to the\nelimination of evil itself, which is a rather bad thing for the tragic\npoets--for there is ever less and less material for tragedy, since the\ndomain of unsparing, immutable destiny grows constantly more\ncircumscribed--and a still worse thing for the priests, for these last\nhave lived heretofore upon the narcoticizing of human ill.\n"} 39 | {"body": "=Sorrow is Knowledge.=--How willingly would not one exchange the false\nassertions of the homines religiosi that there is a god who commands us\nto be good, who is the sentinel and witness of every act, every moment,\nevery thought, who loves us, who plans our welfare in every\nmisfortune--how willingly would not one exchange these for truths as\nhealing, beneficial and grateful as those delusions! But there are no\nsuch truths. Philosophy can at most set up in opposition to them other\nmetaphysical plausibilities (fundamental untruths as well). The tragedy\nof it all is that, although one cannot believe these dogmas of religion\nand metaphysics if one adopts in heart and head the potent methods of\ntruth, one has yet become, through human evolution, so tender,\nsusceptible, sensitive, as to stand in need of the most effective means\nof rest and consolation. From this state of things arises the danger\nthat, through the perception of truth or, more accurately, seeing\nthrough delusion, one may bleed to death. Byron has put this into\ndeathless verse:\n"} 40 | {"body": " \"Sorrow is knowledge: they who know the most\n Must mourn the deepest o'er the fatal truth,\n The tree of knowledge is not that of life.\"\n"} 41 | {"body": "Against such cares there is no better protective than the light fancy of\nHorace, (at any rate during the darkest hours and sun eclipses of the\nsoul) expressed in the words\n"} 42 | {"body": " \"quid aeternis minorem\n consiliis animum fatigas?\n cur non sub alta vel platano vel hac\n pinu jacentes.\"[22]\n"} 43 | {"body": "[22] Then wherefore should you, who are mortal, outwear\n Your soul with a profitless burden of care\n Say, why should we not, flung at ease neath this pine,\n Or a plane-tree's broad umbrage, quaff gaily our wine?\n (Translation of Sir Theodore Martin.)\n"} 44 | {"body": "At any rate, light fancy or heavy heartedness of any degree must be\nbetter than a romantic retrogression and desertion of one's flag, an\napproach to Christianity in any form: for with it, in the present state\nof knowledge, one can have nothing to do without hopelessly defiling\none's intellectual integrity and surrendering it unconditionally. These\nwoes may be painful enough, but without pain one cannot become a leader\nand guide of humanity: and woe to him who would be such and lacks this\npure integrity of the intellect!\n"} 45 | {"body": "=The Truth in Religion.=--In the ages of enlightenment justice was not\ndone to the importance of religion, of this there can be no doubt. It is\nalso equally certain that in the ensuing reaction of enlightenment, the\ndemands of justice were far exceeded inasmuch as religion was treated\nwith love, even with infatuation and proclaimed as a profound, indeed\nthe most profound knowledge of the world, which science had but to\ndivest of its dogmatic garb in order to possess \"truth\" in its\nunmythical form. Religions must therefore--this was the contention of\nall foes of enlightenment--sensu allegorico, with regard for the\ncomprehension of the masses, give expression to that ancient truth which\nis wisdom in itself, inasmuch as all science of modern times has led up\nto it instead of away from it. So that between the most ancient wisdom\nof man and all later wisdom there prevails harmony, even similarity of\nviewpoint; and the advancement of knowledge--if one be disposed to\nconcede such a thing--has to do not with its nature but with its\npropagation. This whole conception of religion and science is through\nand through erroneous, and none would to-day be hardy enough to\ncountenance it had not Schopenhauer's rhetoric taken it under\nprotection, this high sounding rhetoric which now gains auditors after\nthe lapse of a generation. Much as may be gained from Schopenhauer's\nreligio-ethical human and cosmical oracle as regards the comprehension\nof Christianity and other religions, it is nevertheless certain that he\nerred regarding the value of religion to knowledge. He himself was in\nthis but a servile pupil of the scientific teachers of his time who had\nall taken romanticism under their protection and renounced the spirit of\nenlightenment. Had he been born in our own time it would have been\nimpossible for him to have spoken of the sensus allegoricus of religion.\nHe would instead have done truth the justice to say: never has a\nreligion, directly or indirectly, either as dogma or as allegory,\ncontained a truth. For all religions grew out of dread or necessity, and\ncame into existence through an error of the reason. They have, perhaps,\nin times of danger from science, incorporated some philosophical\ndoctrine or other into their systems in order to make it possible to\ncontinue one's existence within them. But this is but a theological work\nof art dating from the time in which a religion began to doubt of\nitself. These theological feats of art, which are most common in\nChristianity as the religion of a learned age, impregnated with\nphilosophy, have led to this superstition of the sensus allegoricus, as\nhas, even more, the habit of the philosophers (namely those\nhalf-natures, the poetical philosophers and the philosophising artists)\nof dealing with their own feelings as if they constituted the\nfundamental nature of humanity and hence of giving their own religious\nfeelings a predominant influence over the structure of their systems. As\nthe philosophers mostly philosophised under the influence of hereditary\nreligious habits, or at least under the traditional influence of this\n\"metaphysical necessity,\" they naturally arrived at conclusions\nclosely resembling the Judaic or Christian or Indian religious\ntenets--resembling, in the way that children are apt to look like their\nmothers: only in this case the fathers were not certain as to the\nmaternity, as easily happens--but in the innocence of their admiration,\nthey fabled regarding the family likeness of all religion and science.\nIn reality, there exists between religion and true science neither\nrelationship nor friendship, not even enmity: they dwell in different\nspheres. Every philosophy that lets the religious comet gleam through\nthe darkness of its last outposts renders everything within it that\npurports to be science, suspicious. It is all probably religion,\nalthough it may assume the guise of science.--Moreover, though all the\npeoples agree concerning certain religious things, for example, the\nexistence of a god (which, by the way, as regards this point, is not\nthe case) this fact would constitute an argument against the thing\nagreed upon, for example the very existence of a god. The consensus\ngentium and especially hominum can probably amount only to an absurdity.\nAgainst it there is no consensus omnium sapientium whatever, on any\npoint, with the exception of which Goethe's verse speaks:\n"} 46 | {"body": " \"All greatest sages to all latest ages\n Will smile, wink and slily agree\n 'Tis folly to wait till a fool's empty pate\n Has learned to be knowing and free.\n So children of wisdom must look upon fools\n As creatures who're never the better for schools.\"\n"} 47 | {"body": "Stated without rhyme or metre and adapted to our case: the consensus\nsapientium is to the effect that the consensus gentium amounts to an\nabsurdity.\n"} 48 | {"body": "=Origin of Religious Worship.=--Let us transport ourselves back to the\ntimes in which religious life flourished most vigorously and we will\nfind a fundamental conviction prevalent which we no longer share and\nwhich has resulted in the closing of the door to religious life once for\nall so far as we are concerned: this conviction has to do with nature\nand intercourse with her. In those times nothing is yet known of\nnature's laws. Neither for earth nor for heaven is there a must. A\nseason, sunshine, rain can come or stay away as it pleases. There is\nwanting, in particular, all idea of natural causation. If a man rows, it\nis not the oar that moves the boat, but rowing is a magical ceremony\nwhereby a demon is constrained to move the boat. All illness, death\nitself, is a consequence of magical influences. In sickness and death\nnothing natural is conceived. The whole idea of \"natural course\" is\nwanting. The idea dawns first upon the ancient Greeks, that is to say in\na very late period of humanity, in the conception of a Moira [fate]\nruling over the gods. If any person shoots off a bow, there is always an\nirrational strength and agency in the act. If the wells suddenly run\ndry, the first thought is of subterranean demons and their pranks. It\nmust have been the dart of a god beneath whose invisible influence a\nhuman being suddenly collapses. In India, the carpenter (according to\nLubbock) is in the habit of making devout offerings to his hammer and\nhatchet. A Brahmin treats the plume with which he writes, a soldier the\nweapon that he takes into the field, a mason his trowel, a laborer his\nplow, in the same way. All nature is, in the opinion of religious\npeople, a sum total of the doings of conscious and willing beings, an\nimmense mass of complex volitions. In regard to all that takes place\noutside of us no conclusion is permissible that anything will result\nthus and so, must result thus and so, that we are comparatively\ncalculable and certain in our experiences, that man is the rule, nature\nthe ruleless. This view forms the fundamental conviction that dominates\ncrude, religion-producing, early civilizations. We contemporary men feel\nexactly the opposite: the richer man now feels himself inwardly, the\nmore polyphone the music and the sounding of his soul, the more\npowerfully does the uniformity of nature impress him. We all, with\nGoethe, recognize in nature the great means of repose for the soul. We\nlisten to the pendulum stroke of this great clock with longing for rest,\nfor absolute calm and quiescence, as if we could drink in the uniformity\nof nature and thereby arrive first at an enjoyment of oneself. Formerly\nit was the reverse: if we carry ourselves back to the periods of crude\ncivilization, or if we contemplate contemporary savages, we will find\nthem most strongly influenced by rule, by tradition. The individual is\nalmost automatically bound to rule and tradition and moves with the\nuniformity of a pendulum. To him nature--the uncomprehended, fearful,\nmysterious nature--must seem the domain of freedom, of volition, of\nhigher power, indeed as an ultra-human degree of destiny, as god. Every\nindividual in such periods and circumstances feels that his existence,\nhis happiness, the existence and happiness of the family, the state,\nthe success or failure of every undertaking, must depend upon these\ndispositions of nature. Certain natural events must occur at the proper\ntime and certain others must not occur. How can influence be exercised\nover this fearful unknown, how can this domain of freedom be brought\nunder subjection? thus he asks himself, thus he worries: Is there no\nmeans to render these powers of nature as subject to rule and tradition\nas you are yourself?--The cogitation of the superstitious and\nmagic-deluded man is upon the theme of imposing a law upon nature: and\nto put it briefly, religious worship is the result of such cogitation.\nThe problem which is present to every man is closely connected with this\none: how can the weaker party dictate laws to the stronger, control its\nacts in reference to the weaker? At first the most harmless form of\ninfluence is recollected, that influence which is acquired when the\npartiality of anyone has been won. Through beseeching and prayer,\nthrough abject humiliation, through obligations to regular gifts and\npropitiations, through flattering homages, it is possible, therefore, to\nimpose some guidance upon the forces of nature, to the extent that their\npartiality be won: love binds and is bound. Then agreements can be\nentered into by means of which certain courses of conduct are mutually\nconcluded, vows are made and authorities prescribed. But far more potent\nis that species of power exercised by means of magic and incantation. As\na man is able to injure a powerful enemy by means of the magician and\nrender him helpless with fear, as the love potion operates at a\ndistance, so can the mighty forces of nature, in the opinion of weaker\nmankind, be controlled by similar means. The principal means of\neffecting incantations is to acquire control of something belonging to\nthe party to be influenced, hair, finger nails, food from his table,\neven his picture or his name. With such apparatus it is possible to act\nby means of magic, for the basic principle is that to everything\nspiritual corresponds something corporeal. With the aid of this\ncorporeal element the spirit may be bound, injured or destroyed. The\ncorporeal affords the handle by which the spiritual can be laid hold of.\nIn the same way that man influences mankind does he influences some\nspirit of nature, for this latter has also its corporeal element that\ncan be grasped. The tree, and on the same basis, the seed from which it\ngrew: this puzzling sequence seems to demonstrate that in both forms the\nsame spirit is embodied, now large, now small. A stone that suddenly\nrolls, is the body in which the spirit works. Does a huge boulder lie in\na lonely moor? It is impossible to think of mortal power having placed\nit there. The stone must have moved itself there. That is to say some\nspirit must dominate it. Everything that has a body is subject to magic,\nincluding, therefore, the spirits of nature. If a god is directly\nconnected with his portrait, a direct influence (by refraining from\ndevout offerings, by whippings, chainings and the like) can be brought\nto bear upon him. The lower classes in China tie cords around the\npicture of their god in order to defy his departing favor, when he has\nleft them in the lurch, and tear the picture to pieces, drag it through\nthe streets into dung heaps and gutters, crying: \"You dog of a spirit,\nwe housed you in a beautiful temple, we gilded you prettily, we fed you\nwell, we brought you offerings, and yet how ungrateful you are!\" Similar\ndisplays of resentment have been made against pictures of the mother of\ngod and pictures of saints in Catholic countries during the present\ncentury when such pictures would not do their duty during times of\npestilence and drought.\n"} 49 | {"body": "Through all these magical relationships to nature countless ceremonies\nare occasioned, and finally, when their complexity and confusion grow\ntoo great, pains are taken to systematize them, to arrange them so that\nthe favorable course of nature's progress, namely the great yearly\ncircle of the seasons, may be brought about by a corresponding course of\nthe ceremonial progress. The aim of religious worship is to influence\nnature to human advantage, and hence to instil a subjection to law into\nher that originally she has not, whereas at present man desires to find\nout the subjection to law of nature in order to guide himself thereby.\nIn brief, the system of religious worship rests upon the idea of magic\nbetween man and man, and the magician is older than the priest. But it\nrests equally upon other and higher ideas. It brings into prominence the\nsympathetic relation of man to man, the existence of benevolence,\ngratitude, prayer, of truces between enemies, of loans upon security, of\narrangements for the protection of property. Man, even in very inferior\ndegrees of civilization, does not stand in the presence of nature as a\nhelpless slave, he is not willy-nilly the absolute servant of nature. In\nthe Greek development of religion, especially in the relationship to the\nOlympian gods, it becomes possible to entertain the idea of an existence\nside by side of two castes, a higher, more powerful, and a lower, less\npowerful: but both are bound together in some way, on account of their\norigin and are one species. They need not be ashamed of one another.\nThis is the element of distinction in Greek religion.\n"} 50 | {"body": "=At the Contemplation of Certain Ancient Sacrificial Proceedings.=--How\nmany sentiments are lost to us is manifest in the union of the farcical,\neven of the obscene, with the religious feeling. The feeling that this\nmixture is possible is becoming extinct. We realize the mixture only\nhistorically, in the mysteries of Demeter and Dionysos and in the\nChristian Easter festivals and religious mysteries. But we still\nperceive the sublime in connection with the ridiculous, and the like,\nthe emotional with the absurd. Perhaps a later age will be unable to\nunderstand even these combinations.\n"} 51 | {"body": "=Christianity as Antiquity.=--When on a Sunday morning we hear the old\nbells ringing, we ask ourselves: Is it possible? All this for a Jew\ncrucified two thousand years ago who said he was God's son? The proof of\nsuch an assertion is lacking.--Certainly, the Christian religion\nconstitutes in our time a protruding bit of antiquity from very remote\nages and that its assertions are still generally believed--although men\nhave become so keen in the scrutiny of claims--constitutes the oldest\nrelic of this inheritance. A god who begets children by a mortal woman;\na sage who demands that no more work be done, that no more justice be\nadministered but that the signs of the approaching end of the world be\nheeded; a system of justice that accepts an innocent as a vicarious\nsacrifice in the place of the guilty; a person who bids his disciples\ndrink his blood; prayers for miracles; sins against a god expiated upon\na god; fear of a hereafter to which death is the portal; the figure of\nthe cross as a symbol in an age that no longer knows the purpose and the\nignominy of the cross--how ghostly all these things flit before us out\nof the grave of their primitive antiquity! Is one to believe that such\nthings can still be believed?\n"} 52 | {"body": "=The Un-Greek in Christianity.=--The Greeks did not look upon the\nHomeric gods above them as lords nor upon themselves beneath as\nservants, after the fashion of the Jews. They saw but the counterpart as\nin a mirror of the most perfect specimens of their own caste, hence an\nideal, but no contradiction of their own nature. There was a feeling of\nmutual relationship, resulting in a mutual interest, a sort of alliance.\nMan thinks well of himself when he gives himself such gods and places\nhimself in a relationship akin to that of the lower nobility with the\nhigher; whereas the Italian races have a decidedly vulgar religion,\ninvolving perpetual anxiety because of bad and mischievous powers and\nsoul disturbers. Wherever the Olympian gods receded into the background,\nthere even Greek life became gloomier and more perturbed.--Christianity,\non the other hand, oppressed and degraded humanity completely and sank\nit into deepest mire: into the feeling of utter abasement it suddenly\nflashed the gleam of divine compassion, so that the amazed and\ngrace-dazzled stupefied one gave a cry of delight and for a moment\nbelieved that the whole of heaven was within him. Upon this unhealthy\nexcess of feeling, upon the accompanying corruption of heart and head,\nChristianity attains all its psychological effects. It wants to\nannihilate, debase, stupefy, amaze, bedazzle. There is but one thing\nthat it does not want: measure, standard (das Maas) and therefore is it\nin the worst sense barbarous, asiatic, vulgar, un-Greek.\n"} 53 | {"body": "=Being Religious to Some Purpose.=--There are certain insipid,\ntraffic-virtuous people to whom religion is pinned like the hem of some\ngarb of a higher humanity. These people do well to remain religious: it\nadorns them. All who are not versed in some professional\nweapon--including tongue and pen as weapons--are servile: to all such\nthe Christian religion is very useful, for then their servility assumes\nthe aspect of Christian virtue and is amazingly adorned.--People whose\ndaily lives are empty and colorless are readily religious. This is\ncomprehensible and pardonable, but they have no right to demand that\nothers, whose daily lives are not empty and colorless, should be\nreligious also.\n"} 54 | {"body": "=The Everyday Christian.=--If Christianity, with its allegations of an\navenging God, universal sinfulness, choice of grace, and the danger of\neternal damnation, were true, it would be an indication of weakness of\nmind and character not to be a priest or an apostle or a hermit, and\ntoil for one's own salvation. It would be irrational to lose sight of\none's eternal well being in comparison with temporary advantage:\nAssuming these dogmas to be generally believed, the every day Christian\nis a pitiable figure, a man who really cannot count as far as three, and\nwho, for the rest, just because of his intellectual incapacity, does not\ndeserve to be as hard punished as Christianity promises he shall be.\n"} 55 | {"body": "=Concerning the Cleverness of Christianity.=--It is a master stroke of\nChristianity to so emphasize the unworthiness, sinfulness and\ndegradation of men in general that contempt of one's fellow creatures\nbecomes impossible. \"He may sin as much as he pleases, he is not by\nnature different from me. It is I who in every way am unworthy and\ncontemptible.\" So says the Christian to himself. But even this feeling\nhas lost its keenest sting for the Christian does not believe in his\nindividual degradation. He is bad in his general human capacity and he\nsoothes himself a little with the assertion that we are all alike.\n"} 56 | {"body": "=Personal Change.=--As soon as a religion rules, it has for its\nopponents those who were its first disciples.\n"} 57 | {"body": "=Fate of Christianity.=--Christianity arose to lighten the heart, but\nnow it must first make the heart heavy in order to be able to lighten it\nafterwards. Christianity will consequently go down.\n"} 58 | {"body": "=The Testimony of Pleasure.=--The agreeable opinion is accepted as true.\nThis is the testimony of pleasure (or as the church says, the evidence\nof strength) of which all religions are so proud, although they should\nall be ashamed of it. If a belief did not make blessed it would not be\nbelieved. How little it would be worth, then!\n"} 59 | {"body": "=Dangerous Play.=--Whoever gives religious feeling room, must then also\nlet it grow. He can do nothing else. Then his being gradually changes.\nThe religious element brings with it affinities and kinships. The whole\ncircle of his judgment and feeling is clouded and draped in religious\nshadows. Feeling cannot stand still. One should be on one's guard.\n"} 60 | {"body": "=The Blind Pupil.=--As long as one knows very well the strength and the\nweakness of one's dogma, one's art, one's religion, its strength is\nstill low. The pupil and apostle who has no eye for the weaknesses of a\ndogma, a religion and so on, dazzled by the aspect of the master and by\nhis own reverence for him, has, on that very account, generally more\npower than the master. Without blind pupils the influence of a man and\nhis work has never become great. To give victory to knowledge, often\namounts to no more than so allying it with stupidity that the brute\nforce of the latter forces triumph for the former.\n"} 61 | {"body": "=The Breaking off of Churches.=--There is not sufficient religion in the\nworld merely to put an end to the number of religions.\n"} 62 | {"body": "=Sinlessness of Men.=--If one have understood how \"Sin came into the\nworld,\" namely through errors of the reason, through which men in their\nintercourse with one another and even individual men looked upon\nthemselves as much blacker and wickeder than was really the case, one's\nwhole feeling is much lightened and man and the world appear together in\nsuch a halo of harmlessness that a sentiment of well being is instilled\ninto one's whole nature. Man in the midst of nature is as a child left\nto its own devices. This child indeed dreams a heavy, anxious dream. But\nwhen it opens its eyes it finds itself always in paradise.\n"} 63 | {"body": "=Irreligiousness of Artists.=--Homer is so much at home among his gods\nand is as a poet so good natured to them that he must have been\nprofoundly irreligious. That which was brought to him by the popular\nfaith--a mean, crude and partially repulsive superstition--he dealt with\nas freely as the Sculptor with his clay, therefore with the same freedom\nthat \u00c6schylus and Aristophanes evinced and with which in later times the\ngreat artists of the renaissance, and also Shakespeare and Goethe, drew\ntheir pictures.\n"} 64 | {"body": "=Art and Strength of False Interpretation.=--All the visions, fears,\nexhaustions and delights of the saint are well known symptoms of\nsickness, which in him, owing to deep rooted religious and psychological\ndelusions, are explained quite differently, that is not as symptoms of\nsickness.--So, too, perhaps, the demon of Socrates was nothing but a\nmalady of the ear that he explained, in view of his predominant moral\ntheory, in a manner different from what would be thought rational\nto-day. Nor is the case different with the frenzy and the frenzied\nspeeches of the prophets and of the priests of the oracles. It is always\nthe degree of wisdom, imagination, capacity and morality in the heart\nand mind of the interpreters that got so much out of them. It is among\nthe greatest feats of the men who are called geniuses and saints that\nthey made interpreters for themselves who, fortunately for mankind, did\nnot understand them.\n"} 65 | {"body": "=Reverence for Madness.=--Because it was perceived that an excitement of\nsome kind often made the head clearer and occasioned fortunate\ninspirations, it was concluded that the utmost excitement would occasion\nthe most fortunate inspirations. Hence the frenzied being was revered as\na sage and an oracle giver. A false conclusion lies at the bottom of all\nthis.\n"} 66 | {"body": "=Promises of Wisdom.=--Modern science has as its object as little pain\nas possible, as long a life as possible--hence a sort of eternal\nblessedness, but of a very limited kind in comparison with the promises\nof religion.\n"} 67 | {"body": "=Forbidden Generosity.=--There is not enough of love and goodness in the\nworld to throw any of it away on conceited people.\n"} 68 | {"body": "=Survival of Religious Training in the Disposition.=--The Catholic\nChurch, and before it all ancient education, controlled the whole domain\nof means through which man was put into certain unordinary moods and\nwithdrawn from the cold calculation of personal advantage and from calm,\nrational reflection. A church vibrating with deep tones; gloomy,\nregular, restraining exhortations from a priestly band, who\ninvoluntarily communicate their own tension to their congregation and\nlead them to listen almost with anxiety as if some miracle were in\ncourse of preparation; the awesome pile of architecture which, as the\nhouse of a god, rears itself vastly into the vague and in all its\nshadowy nooks inspires fear of its nerve-exciting power--who would care\nto reduce men to the level of these things if the ideas upon which they\nrest became extinct? But the results of all these things are\nnevertheless not thrown away: the inner world of exalted, emotional,\nprophetic, profoundly repentant, hope-blessed moods has become inborn in\nman largely through cultivation. What still exists in his soul was\nformerly, as he germinated, grew and bloomed, thoroughly disciplined.\n"} 69 | {"body": "=Religious After-Pains.=--Though one believe oneself absolutely weaned\naway from religion, the process has yet not been so thorough as to make\nimpossible a feeling of joy at the presence of religious feelings and\ndispositions without intelligible content, as, for example, in music;\nand if a philosophy alleges to us the validity of metaphysical hopes,\nthrough the peace of soul therein attainable, and also speaks of \"the\nwhole true gospel in the look of Raphael's Madonna,\" we greet such\ndeclarations and innuendoes with a welcome smile. The philosopher has\nhere a matter easy of demonstration. He responds with that which he is\nglad to give, namely a heart that is glad to accept. Hence it is\nobservable how the less reflective free spirits collide only with dogmas\nbut yield readily to the magic of religious feelings; it is a source of\npain to them to let the latter go simply on account of the\nformer.--Scientific philosophy must be very much on its guard lest on\naccount of this necessity--an evolved and hence, also, a transitory\nnecessity--delusions are smuggled in. Even logicians speak of\n\"presentiments\" of truth in ethics and in art (for example of the\npresentiment that the essence of things is unity) a thing which,\nnevertheless, ought to be prohibited. Between carefully deduced truths\nand such \"foreboded\" things there lies the abysmal distinction that the\nformer are products of the intellect and the latter of the necessity.\nHunger is no evidence that there is food at hand to appease it. Hunger\nmerely craves food. \"Presentiment\" does not denote that the existence of\na thing is known in any way whatever. It denotes merely that it is\ndeemed possible to the extent that it is desired or feared. The\n\"presentiment\" is not one step forward in the domain of certainty.--It\nis involuntarily believed that the religious tinted sections of a\nphilosophy are better attested than the others, but the case is at\nbottom just the opposite: there is simply the inner wish that it may be\nso, that the thing which beautifies may also be true. This wish leads us\nto accept bad grounds as good.\n"} 70 | {"body": "=Of the Christian Need of Salvation.=--Careful consideration must render\nit possible to propound some explanation of that process in the soul of\na Christian which is termed need of salvation, and to propound an\nexplanation, too, free from mythology: hence one purely psychological.\nHeretofore psychological explanations of religious conditions and\nprocesses have really been in disrepute, inasmuch as a theology calling\nitself free gave vent to its unprofitable nature in this domain; for its\nprincipal aim, so far as may be judged from the spirit of its creator,\nSchleier-macher, was the preservation of the Christian religion and the\nmaintenance of the Christian theology. It appeared that in the\npsychological analysis of religious \"facts\" a new anchorage and above\nall a new calling were to be gained. Undisturbed by such predecessors,\nwe venture the following exposition of the phenomena alluded to. Man is\nconscious of certain acts which are very firmly implanted in the general\ncourse of conduct: indeed he discovers in himself a predisposition to\nsuch acts that seems to him to be as unalterable as his very being. How\ngladly he would essay some other kind of acts which in the general\nestimate of conduct are rated the best and highest, how gladly he would\nwelcome the consciousness of well doing which ought to follow unselfish\nmotive! Unfortunately, however, it goes no further than this longing:\nthe discontent consequent upon being unable to satisfy it is added to\nall other kinds of discontent which result from his life destiny in\nparticular or which may be due to so called bad acts; so that a deep\ndepression ensues accompanied by a desire for some physician to remove\nit and all its causes.--This condition would not be found so bitter if\nthe individual but compared himself freely with other men: for then he\nwould have no reason to be discontented with himself in particular as he\nis merely bearing his share of the general burden of human discontent\nand incompleteness. But he compares himself with a being who alone must\nbe capable of the conduct that is called unegoistic and of an enduring\nconsciousness of unselfish motive, with God. It is because he gazes into\nthis clear mirror, that his own self seems so extraordinarily distracted\nand so troubled. Thereupon the thought of that being, in so far as it\nflits before his fancy as retributive justice, occasions him anxiety. In\nevery conceivable small and great experience he believes he sees the\nanger of the being, his threats, the very implements and manacles of his\njudge and prison. What succors him in this danger, which, in the\nprospect of an eternal duration of punishment, transcends in hideousness\nall the horrors that can be presented to the imagination?\n"} 71 | {"body": "Before we consider this condition in its further effects, we would admit\nto ourselves that man is betrayed into this condition not through his\n\"fault\" and \"sin\" but through a series of delusions of the reason; that\nit was the fault of the mirror if his own self appeared to him in the\nhighest degree dark and hateful, and that that mirror was his own work,\nthe very imperfect work of human imagination and judgment. In the first\nplace a being capable of absolutely unegoistic conduct is as fabulous as\nthe phoenix. Such a being is not even thinkable for the very reason that\nthe whole notion of \"unegoistic conduct,\" when closely examined,\nvanishes into air. Never yet has a man done anything solely for others\nand entirely without reference to a personal motive; indeed how could he\npossibly do anything that had no reference to himself, that is without\ninward compulsion (which must always have its basis in a personal need)?\nHow could the ego act without ego?--A god, who, on the other hand, is\nall love, as he is usually represented, would not be capable of a\nsolitary unegoistic act: whence one is reminded of a reflection of\nLichtenberg's which is, in truth, taken from a lower sphere: \"We cannot\npossibly feel for others, as the expression goes; we feel only for\nourselves. The assertion sounds hard, but it is not, if rightly\nunderstood. A man loves neither his father nor his mother nor his wife\nnor his child, but simply the feelings which they inspire.\" Or, as La\nRochefoucauld says: \"If you think you love your mistress for the mere\nlove of her, you are very much mistaken.\" Why acts of love are more\nhighly prized than others, namely not on account of their nature, but on\naccount of their utility, has already been explained in the section on\nthe origin of moral feelings. But if a man should wish to be all love\nlike the god aforesaid, and want to do all things for others and nothing\nfor himself, the procedure would be fundamentally impossible because he\n_must_ do a great deal for himself before there would be any possibility\nof doing anything for the love of others. It is also essential that\nothers be sufficiently egoistic to accept always and at all times this\nself sacrifice and living for others, so that the men of love and self\nsacrifice have an interest in the survival of unloving and selfish\negoists, while the highest morality, in order to maintain itself must\nformally enforce the existence of immorality (wherein it would be really\ndestroying itself.)--Further: the idea of a god perturbs and discourages\nas long as it is accepted but as to how it originated can no longer, in\nthe present state of comparative ethnological science, be a matter of\ndoubt, and with the insight into the origin of this belief all faith\ncollapses. What happens to the Christian who compares his nature with\nthat of God is exactly what happened to Don Quixote, who depreciated his\nown prowess because his head was filled with the wondrous deeds of the\nheroes of chivalrous romance. The standard of measurement which both\nemploy belongs to the domain of fable.--But if the idea of God\ncollapses, so too, does the feeling of \"sin\" as a violation of divine\nrescript, as a stain upon a god-like creation. There still apparently\nremains that discouragement which is closely allied with fear of the\npunishment of worldly justice or of the contempt of one's fellow men.\nThe keenest thorn in the sentiment of sin is dulled when it is perceived\nthat one's acts have contravened human tradition, human rules and human\nlaws without having thereby endangered the \"eternal salvation of the\nsoul\" and its relations with deity. If finally men attain to the\nconviction of the absolute necessity of all acts and of their utter\nirresponsibility and then absorb it into their flesh and blood, every\nrelic of conscience pangs will disappear.\n"} 72 | {"body": "If now, as stated, the Christian, through certain delusive feelings, is\nbetrayed into self contempt, that is by a false and unscientific view of\nhis acts and feelings, he must, nevertheless, perceive with the utmost\namazement that this state of self contempt, of conscience pangs, of\ndespair in particular, does not last, that there are hours during which\nall these things are wafted away from the soul and he feels himself once\nmore free and courageous. The truth is that joy in his own being, the\nfulness of his own powers in connection with the inevitable decline of\nhis profound excitation with the lapse of time, bore off the palm of\nvictory. The man loves himself once more, he feels it--but this very new\nlove, this new self esteem seems to him incredible. He can see in it\nonly the wholly unmerited stream of the light of grace shed down upon\nhim. If he formerly saw in every event merely warnings, threats,\npunishments and every kind of indication of divine anger, he now reads\ninto his experiences the grace of god. The latter circumstance seems to\nhim full of love, the former as a helpful pointing of the way, and his\nentirely joyful frame of mind now seems to him to be an absolute proof\nof the goodness of God. As formerly in his states of discouragement he\ninterpreted his conduct falsely so now he does the same with his\nexperiences. His state of consolation is now regarded as the effect\nproduced by some external power. The love with which, at bottom, he\nloves himself, seems to be the divine love. That which he calls grace\nand the preliminary of salvation is in reality self-grace,\nself-salvation.\n"} 73 | {"body": "Therefore a certain false psychology, a certain kind of imaginativeness\nin the interpretation of motives and experiences is the essential\npreliminary to being a Christian and to experiencing the need of\nsalvation. Upon gaining an insight into this wandering of the reason and\nthe imagination, one ceases to be a Christian.\n"} 74 | {"body": "=Of Christian Asceticism and Sanctity.=--Much as some thinkers have\nexerted themselves to impart an air of the miraculous to those singular\nphenomena known as asceticism and sanctity, to question which or to\naccount for which upon a rational basis would be wickedness and\nsacrilege, the temptation to this wickedness is none the less great. A\npowerful impulse of nature has in every age led to protest against such\nphenomena. At any rate science, inasmuch as it is the imitation of\nnature, permits the casting of doubts upon the inexplicable character\nand the supernal degree of such phenomena. It is true that heretofore\nscience has not succeeded in its attempts at explanation. The phenomena\nremain unexplained still, to the great satisfaction of those who revere\nmoral miracles. For, speaking generally, the unexplained must rank as\nthe inexplicable, the inexplicable as the non-natural, supernatural,\nmiraculous--so runs the demand in the souls of all the religious and all\nthe metaphysicians (even the artists if they happen to be thinkers),\nwhereas the scientific man sees in this demand the \"evil\nprinciple.\"--The universal, first, apparent truth that is encountered in\nthe contemplation of sanctity and asceticism is that their nature is\ncomplicated; for nearly always, within the physical world as well as in\nthe moral, the apparently miraculous may be traced successfully to the\ncomplex, the obscure, the multi-conditioned. Let us venture then to\nisolate a few impulses in the soul of the saint and the ascetic, to\nconsider them separately and then view them as a synthetic development.\n"} 75 | {"body": "There is an obstinacy against oneself, certain sublimated forms of which\nare included in asceticism. Certain kinds of men are under such a strong\nnecessity of exercising their power and dominating impulses that, if\nother objects are lacking or if they have not succeeded with other\nobjects they will actually tyrannize over some portions of their own\nnature or over sections and stages of their own personality. Thus do\nmany thinkers bring themselves to views which are far from likely to\nincrease or improve their fame. Many deliberately bring down the\ncontempt of others upon themselves although they could easily have\nretained consideration by silence. Others contradict earlier opinions\nand do not shrink from the ordeal of being deemed inconsistent. On the\ncontrary they strive for this and act like eager riders who enjoy\nhorseback exercise most when the horse is skittish. Thus will men in\ndangerous paths ascend to the highest steeps in order to laugh to scorn\ntheir own fear and their own trembling limbs. Thus will the philosopher\nembrace the dogmas of asceticism, humility, sanctity, in the light of\nwhich his own image appears in its most hideous aspect. This crushing of\nself, this mockery of one's own nature, this spernere se sperni out of\nwhich religions have made so much is in reality but a very high\ndevelopment of vanity. The whole ethic of the sermon on the mount\nbelongs in this category: man has a true delight in mastering himself\nthrough exaggerated pretensions or excessive expedients and later\ndeifying this tyrannically exacting something within him. In every\nscheme of ascetic ethics, man prays to one part of himself as if it were\ngod and hence it is necessary for him to treat the rest of himself as\ndevil.\n"} 76 | {"body": "=Man is Not at All Hours Equally Moral=; this is established. If one's\nmorality be judged according to one's capacity for great, self\nsacrificing resolutions and abnegations (which when continual, and made\na habit are known as sanctity) one is, in affection, or disposition, the\nmost moral: while higher excitement supplies wholly new impulses which,\nwere one calm and cool as ordinarily, one would not deem oneself even\ncapable of. How comes this? Apparently from the propinquity of all great\nand lofty emotional states. If a man is brought to an extraordinary\npitch of feeling he can resolve upon a fearful revenge or upon a fearful\nrenunciation of his thirst for vengeance indifferently. He craves, under\nthe influences of powerful emotion, the great, the powerful, the\nimmense, and if he chances to perceive that the sacrifice of himself\nwill afford him as much satisfaction as the sacrifice of another, or\nwill afford him more, he will choose self sacrifice. What concerns him\nparticularly is simply the unloading of his emotion. Hence he readily,\nto relieve his tension, grasps the darts of the enemy and buries them in\nhis own breast. That in self abnegation and not in revenge the element\nof greatness consisted must have been brought home to mankind only after\nlong habituation. A god who sacrifices himself would be the most\npowerful and most effective symbol of this sort of greatness. As the\nconquest of the most hardly conquered enemy, the sudden mastering of a\npassion--thus does such abnegation _appear_: hence it passes for the\nsummit of morality. In reality all that is involved is the exchange of\none idea for another whilst the temperament remained at a like altitude,\na like tidal state. Men when coming out of the spell, or resting from\nsuch passionate excitation, no longer understand the morality of such\ninstants, but the admiration of all who participated in the occasion\nsustains them. Pride is their support if the passion and the\ncomprehension of their act weaken. Therefore, at bottom even such acts\nof self-abnegation are not moral inasmuch as they are not done with a\nstrict regard for others. Rather do others afford the high strung\ntemperament an opportunity to lighten itself through such abnegation.\n"} 77 | {"body": "=Even the Ascetic Seeks to Make Life Easier=, and generally by means of\nabsolute subjection to another will or to an all inclusive rule and\nritual, pretty much as the Brahmin leaves absolutely nothing to his own\nvolition but is guided in every moment of his life by some holy\ninjunction or other. This subjection is a potent means of acquiring\ndominion over oneself. One is occupied, hence time does not bang heavy\nand there is no incitement of the personal will and of the individual\npassion. The deed once done there is no feeling of responsibility nor\nthe sting of regret. One has given up one's own will once for all and\nthis is easier than to give it up occasionally, as it is also easier\nwholly to renounce a desire than to yield to it in measured degree. When\nwe consider the present relation of man to the state we perceive\nunconditional obedience is easier than conditional. The holy person also\nmakes his lot easier through the complete surrender of his life\npersonality and it is all delusion to admire such a phenomenon as the\nloftiest heroism of morality. It is always more difficult to assert\none's personality without shrinking and without hesitation than to give\nit up altogether in the manner indicated, and it requires moreover more\nintellect and thought.\n"} 78 | {"body": "After having discovered in many of the less comprehensible actions mere\nmanifestations of pleasure in emotion for its own sake, I fancy I can\ndetect in the self contempt which characterises holy persons, and also\nin their acts of self torture (through hunger and scourgings,\ndistortions and chaining of the limbs, acts of madness) simply a means\nwhereby such natures may resist the general exhaustion of their will to\nlive (their nerves). They employ the most painful expedients to escape\nif only for a time from the heaviness and weariness in which they are\nsteeped by their great mental indolence and their subjection to a will\nother than their own.\n"} 79 | {"body": "=The Most Usual Means= by which the ascetic and the sanctified\nindividual seeks to make life more endurable comprises certain combats\nof an inner nature involving alternations of victory and prostration.\nFor this purpose an enemy is necessary and he is found in the so called\n\"inner enemy.\" That is, the holy individual makes use of his tendency to\nvanity, domineering and pride, and of his mental longings in order to\ncontemplate his life as a sort of continuous battle and himself as a\nbattlefield, in which good and evil spirits wage war with varying\nfortune. It is an established fact that the imagination is restrained\nthrough the regularity and adequacy of sexual intercourse while on the\nother hand abstention from or great irregularity in sexual intercourse\nwill cause the imagination to run riot. The imaginations of many of the\nChristian saints were obscene to a degree; and because of the theory\nthat sexual desires were in reality demons that raged within them, the\nsaints did not feel wholly responsible for them. It is to this\nconviction that we are indebted for the highly instructive sincerity of\ntheir evidence against themselves. It was to their interest that this\ncontest should always be kept up in some fashion because by means of\nthis contest, as already stated, their empty lives gained distraction.\nIn order that the contest might seem sufficiently great to inspire\nsympathy and admiration in the unsanctified, it was essential that\nsexual capacity be ever more and more damned and denounced. Indeed the\ndanger of eternal damnation was so closely allied to this capacity that\nfor whole generations Christians showed their children with actual\nconscience pangs. What evil may not have been done to humanity through\nthis! And yet here the truth is just upside down: an exceedingly\nunseemly attitude for the truth. Christianity, it is true, had said that\nevery man is conceived and born in sin, and in the intolerable and\nexcessive Christianity of Calderon this thought is again perverted and\nentangled into the most distorted paradox extant in the well known lines\n"} 80 | {"body": " The greatest sin of man\n Is the sin of being born.\n"} 81 | {"body": "In all pessimistic religions the act of procreation is looked upon as\nevil in itself. This is far from being the general human opinion. It is\nnot even the opinion of all pessimists. Empedocles, for example, knows\nnothing of anything shameful, devilish and sinful in it. He sees rather\nin the great field of bliss of unholiness simply a healthful and hopeful\nphenomenon, Aphrodite. She is to him an evidence that strife does not\nalways rage but that some time a gentle demon is to wield the sceptre.\nThe Christian pessimists of practice, had, as stated, a direct interest\nin the prevalence of an opposite belief. They needed in the loneliness\nand the spiritual wilderness of their lives an ever living enemy, and a\nuniversally known enemy through whose conquest they might appear to the\nunsanctified as utterly incomprehensible and half unnatural beings. When\nthis enemy at last, as a result of their mode of life and their\nshattered health, took flight forever, they were able immediately to\npeople their inner selves with new demons. The rise and fall of the\nbalance of cheerfulness and despair maintained their addled brains in a\ntotally new fluctuation of longing and peace of soul. And in that period\npsychology served not only to cast suspicion on everything human but to\nwound and scourge it, to crucify it. Man wanted to find himself as base\nand evil as possible. Man sought to become anxious about the state of\nhis soul, he wished to be doubtful of his own capacity. Everything\nnatural with which man connects the idea of badness and sinfulness (as,\nfor instance, is still customary in regard to the erotic) injures and\ndegrades the imagination, occasions a shamed aspect, leads man to war\nupon himself and makes him uncertain, distrustful of himself. Even his\ndreams acquire a tincture of the unclean conscience. And yet this\nsuffering because of the natural element in certain things is wholly\nsuperfluous. It is simply the result of opinions regarding the things.\nIt is easy to understand why men become worse than they are if they are\nbrought to look upon the unavoidably natural as bad and later to feel it\nas of evil origin. It is the master stroke of religions and metaphysics\nthat wish to make man out bad and sinful by nature, to render nature\nsuspicious in his eyes and to so make himself evil, for he learns to\nfeel himself evil when he cannot divest himself of nature. He gradually\ncomes to look upon himself, after a long life lived naturally, so\noppressed by a weight of sin that supernatural powers become necessary\nto relieve him of the burden; and with this notion comes the so called\nneed of salvation, which is the result not of a real but of an imaginary\nsinfulness. Go through the separate moral expositions in the vouchers of\nchristianity and it will always be found that the demands are excessive\nin order that it may be impossible for man to satisfy them. The object\nis not that he may become moral but that he may feel as sinful as\npossible. If this feeling had not been rendered agreeable to man--why\nshould he have improvised such an ideal and clung to it so long? As in\nthe ancient world an incalculable strength of intellect and capacity for\nfeeling was squandered in order to increase the joy of living through\nfeastful systems of worship, so in the era of christianity an equally\nincalculable quantity of intellectual capacity has been sacrificed in\nanother endeavor: that man should in every way feel himself sinful and\nthereby be moved, inspired, inspirited. To move, to inspire, to inspirit\nat any cost--is not this the freedom cry of an exhausted, over-ripe,\nover cultivated age? The circle of all the natural sensations had been\ngone through a hundred times: the soul had grown weary. Then the saints\nand the ascetics found a new order of ecstacies. They set themselves\nbefore the eyes of all not alone as models for imitation to many, but as\nfearful and yet delightful spectacles on the boundary line between this\nworld and the next world, where in that period everyone thought he saw\nat one time rays of heavenly light, at another fearful, threatening\ntongues of flame. The eye of the saint, directed upon the fearful\nsignificance of the shortness of earthly life, upon the imminence of the\nlast judgment, upon eternal life hereafter; this glowering eye in an\nemaciated body caused men, in the old time world, to tremble to the\ndepths of their being. To look, to look away and shudder, to feel anew\nthe fascination of the spectacle, to yield to it, sate oneself upon it\nuntil the soul trembled with ardor and fever--that was the last pleasure\nleft to classical antiquity when its sensibilities had been blunted by\nthe arena and the gladiatorial show.\n"} 82 | {"body": "=To Sum Up All That Has Been Said=: that condition of soul at which the\nsaint or expectant saint is rejoiced is a combination of elements which\nwe are all familiar with, except that under other influences than those\nof mere religious ideation they customarily arouse the censure of men in\nthe same way that when combined with religion itself and regarded as the\nsupreme attainment of sanctity, they are object of admiration and even\nof prayer--at least in more simple times. Very soon the saint turns upon\nhimself that severity that is so closely allied to the instinct of\ndomination at any price and which inspire even in the most solitary\nindividual the sense of power. Soon his swollen sensitiveness of feeling\nbreaks forth from the longing to restrain his passions within it and is\ntransformed into a longing to master them as if they were wild steeds,\nthe master impulse being ever that of a proud spirit; next he craves a\ncomplete cessation of all perturbing, fascinating feelings, a waking\nsleep, an enduring repose in the lap of a dull, animal, plant-like\nindolence. Next he seeks the battle and extinguishes it within himself\nbecause weariness and boredom confront him. He binds his\nself-deification with self-contempt. He delights in the wild tumult of\nhis desires and the sharp pain of sin, in the very idea of being lost.\nHe is able to play his very passions, for instance the desire to\ndomineer, a trick so that he goes to the other extreme of abject\nhumiliation and subjection, so that his overwrought soul is without any\nrestraint through this antithesis. And, finally, when indulgence in\nvisions, in talks with the dead or with divine beings overcomes him,\nthis is really but a form of gratification that he craves, perhaps a\nform of gratification in which all other gratifications are blended.\nNovalis, one of the authorities in matters of sanctity, because of his\nexperience and instinct, betrays the whole secret with the utmost\nsimplicity when he says: \"It is remarkable that the close connection of\ngratification, religion and cruelty has not long ago made men aware of\ntheir inner relationship and common tendency.\"\n"} 83 | {"body": "=Not What the Saint is but what he was in= the eyes of the\nnon-sanctified gives him his historical importance. Because there\nexisted a delusion respecting the saint, his soul states being falsely\nviewed and his personality being sundered as much as possible from\nhumanity as a something incomparable and supernatural, because of these\nthings he attained the extraordinary with which he swayed the\nimaginations of whole nations and whole ages. Even he knew himself not\nfor even he regarded his dispositions, passions and actions in\naccordance with a system of interpretation as artificial and exaggerated\nas the pneumatic interpretation of the bible. The distorted and diseased\nin his own nature with its blending of spiritual poverty, defective\nknowledge, ruined health, overwrought nerves, remained as hidden from\nhis view as from the view of his beholders. He was neither a\nparticularly good man nor a particularly bad man but he stood for\nsomething that was far above the human standard in wisdom and goodness.\nFaith in him sustained faith in the divine and miraculous, in a\nreligious significance of all existence, in an impending day of\njudgment. In the last rays of the setting sun of the ancient world,\nwhich fell upon the christian peoples, the shadowy form of the saint\nattained enormous proportions--to such enormous proportions, indeed,\nthat down even to our own age, which no longer believes in god, there\nare thinkers who believe in the saints.\n"} 84 | -------------------------------------------------------------------------------- /docs/source/_images/swagger_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexeykarnachev/full_stack_transformer/2ac4a583c1ea610a4786a65258f33a941251bdea/docs/source/_images/swagger_example.png -------------------------------------------------------------------------------- /docs/source/_images/tb_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexeykarnachev/full_stack_transformer/2ac4a583c1ea610a4786a65258f33a941251bdea/docs/source/_images/tb_example.png -------------------------------------------------------------------------------- /docs/source/_images/telegram_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexeykarnachev/full_stack_transformer/2ac4a583c1ea610a4786a65258f33a941251bdea/docs/source/_images/telegram_example.png -------------------------------------------------------------------------------- /examples/language_generation.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | 3 | import torch 4 | 5 | from full_stack_transformer import ( 6 | LanguageGenerator, 7 | LanguageGeneratorParams, 8 | Document, 9 | load_language_model_from_checkpoint, 10 | load_tokenizer_from_checkpoint 11 | ) 12 | 13 | if __name__ == '__main__': 14 | device = 'cuda:0' 15 | experiment_dir = pathlib.Path('../static/experiments/nietzsche_v0/') 16 | ckpt_path = experiment_dir / 'models' / 'epoch=9.ckpt' 17 | 18 | generator_params = LanguageGeneratorParams( 19 | max_number_of_generated_tokens=64, 20 | num_return_sequences=8, 21 | repetition_penalty=3.0, 22 | temperature=0.5, 23 | top_k=50, 24 | top_p=1.0 25 | ) 26 | 27 | ckpt = torch.load(f=str(ckpt_path), map_location='cpu') 28 | 29 | model = load_language_model_from_checkpoint( 30 | ckpt=ckpt, device=device, unlikelihood_alpha=None 31 | ) 32 | 33 | tokenizer = load_tokenizer_from_checkpoint( 34 | ckpt=ckpt, max_body_len=64, max_meta_len=0 35 | ) 36 | 37 | generator = LanguageGenerator( 38 | model=model, eos_token_id=tokenizer.eos_token_id 39 | ) 40 | 41 | document = Document(body='The best filosopher of the 19th century is') 42 | 43 | inp_encoding = tokenizer.encode_document( 44 | document=document, with_eos=False 45 | ) 46 | 47 | out_encodings = generator(inp_encoding, params=generator_params) 48 | 49 | for enc in out_encodings: 50 | text = tokenizer.decode_encoding(enc) 51 | print(text + '\n\n') 52 | -------------------------------------------------------------------------------- /full_stack_transformer/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from full_stack_transformer.utilities import log_config 4 | 5 | sys.excepthook = log_config.handle_unhandled_exception 6 | 7 | __version__ = '0.2.0' 8 | -------------------------------------------------------------------------------- /full_stack_transformer/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexeykarnachev/full_stack_transformer/2ac4a583c1ea610a4786a65258f33a941251bdea/full_stack_transformer/core/__init__.py -------------------------------------------------------------------------------- /full_stack_transformer/core/constants.py: -------------------------------------------------------------------------------- 1 | LOSS_IGNORE = -100 -------------------------------------------------------------------------------- /full_stack_transformer/core/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexeykarnachev/full_stack_transformer/2ac4a583c1ea610a4786a65258f33a941251bdea/full_stack_transformer/core/data/__init__.py -------------------------------------------------------------------------------- /full_stack_transformer/core/data/dataset.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import pathlib 3 | from multiprocessing import Queue 4 | from typing import Optional, Callable 5 | 6 | import torch.utils.data 7 | 8 | from full_stack_transformer.core.data.encodings_collate import EncodingsCollate 9 | from full_stack_transformer.core.data.encodings_producer import EncodingsProducer 10 | from full_stack_transformer.core.data.encodings_sampler import EncodingsSampler 11 | from full_stack_transformer.core.data.text_inputs_producer import TextInputsProducer 12 | from full_stack_transformer.core.data.text_lines_parsers import TextLinesParser 13 | from full_stack_transformer.core.tokenizer import Tokenizer 14 | from full_stack_transformer.utilities.queue_iterable_dataset import \ 15 | QueueIterableDataset 16 | 17 | _QUEUE_MAX_SIZE = 10 18 | 19 | 20 | class Dataset(QueueIterableDataset): 21 | def __init__( 22 | self, 23 | file_path: pathlib.Path, 24 | tokenizer: Tokenizer, 25 | text_lines_parser: TextLinesParser, 26 | encodings_collate: Callable, 27 | n_producer_workers: int = 4, 28 | chunk_size: Optional[int] = 10000 29 | ): 30 | self._file_path = file_path 31 | self._tokenizer = tokenizer 32 | self._n_producer_workers = n_producer_workers 33 | self._chunk_size = chunk_size 34 | self._text_lines_parser = text_lines_parser 35 | self._collate = encodings_collate 36 | 37 | sorted_samples_queue = self._initialize() 38 | 39 | super().__init__(inp_queue=sorted_samples_queue, 40 | ) 41 | 42 | def _initialize(self): 43 | text_inputs_queue = Queue(maxsize=_QUEUE_MAX_SIZE) 44 | encodings_queue = Queue(maxsize=_QUEUE_MAX_SIZE) 45 | sorted_encodings_queue = Queue(maxsize=_QUEUE_MAX_SIZE) 46 | 47 | text_inputs_producer = TextInputsProducer( 48 | file_path=self._file_path, 49 | out_text_inputs_queue=text_inputs_queue, 50 | out_chunk_size=self._chunk_size, 51 | text_lines_parser=self._text_lines_parser 52 | ) 53 | 54 | model_inputs_producers = [] 55 | 56 | for _ in range(self._n_producer_workers): 57 | samples_producer = EncodingsProducer( 58 | inp_text_inputs_queue=text_inputs_queue, 59 | out_encodings_queue=encodings_queue, 60 | tokenizer=self._tokenizer 61 | ) 62 | model_inputs_producers.append(samples_producer) 63 | 64 | encodings_sampler = EncodingsSampler( 65 | inp_encodings_queue=encodings_queue, 66 | out_encodings_queue=sorted_encodings_queue 67 | ) 68 | 69 | text_inputs_producer.start() 70 | [p.start() for p in model_inputs_producers] 71 | encodings_sampler.start() 72 | 73 | return sorted_encodings_queue 74 | 75 | @abc.abstractmethod 76 | def __len__(self) -> int: 77 | pass 78 | 79 | def get_data_loader( 80 | self, 81 | batch_size: int, 82 | num_workers: int 83 | ) -> 'DataLoader': 84 | dl = DataLoader( 85 | dataset=self, 86 | batch_size=batch_size, 87 | num_workers=num_workers, 88 | encodings_collate=self._collate 89 | ) 90 | return dl 91 | 92 | 93 | class DataLoader(torch.utils.data.DataLoader): 94 | def __init__( 95 | self, 96 | dataset: Dataset, 97 | batch_size: int, 98 | num_workers: int, 99 | encodings_collate: EncodingsCollate 100 | ): 101 | self._dataset = dataset 102 | self._batch_size = batch_size 103 | self._collate = encodings_collate 104 | 105 | super().__init__( 106 | dataset=self._dataset, 107 | batch_size=self._batch_size, 108 | num_workers=num_workers, 109 | collate_fn=self._collate 110 | ) 111 | 112 | def __len__(self): 113 | return (len(self._dataset) // self._batch_size) - 1 114 | -------------------------------------------------------------------------------- /full_stack_transformer/core/data/encodings_collate.py: -------------------------------------------------------------------------------- 1 | import abc 2 | from typing import Sequence 3 | 4 | from full_stack_transformer.core.encoding import Encoding 5 | from full_stack_transformer.core.model_input import ModelInput 6 | 7 | 8 | class EncodingsCollate: 9 | @abc.abstractmethod 10 | def __call__(self, encodings: Sequence[Encoding]) -> ModelInput: 11 | pass 12 | -------------------------------------------------------------------------------- /full_stack_transformer/core/data/encodings_producer.py: -------------------------------------------------------------------------------- 1 | from multiprocessing import Process, Queue 2 | 3 | from full_stack_transformer.core.tokenizer import Tokenizer 4 | 5 | 6 | class EncodingsProducer(Process): 7 | def __init__( 8 | self, 9 | tokenizer: Tokenizer, 10 | inp_text_inputs_queue: Queue, 11 | out_encodings_queue: Queue 12 | ): 13 | super().__init__() 14 | 15 | self._tokenizer = tokenizer 16 | self._inp_queue = inp_text_inputs_queue 17 | self._out_queue = out_encodings_queue 18 | 19 | def run(self) -> None: 20 | while True: 21 | input_chunk = self._inp_queue.get() 22 | encodings = [] 23 | for text_input in input_chunk: 24 | enc = self._tokenizer.encode_for_train(text_input) 25 | encodings.extend(enc) 26 | 27 | self._out_queue.put(encodings) 28 | -------------------------------------------------------------------------------- /full_stack_transformer/core/data/encodings_sampler.py: -------------------------------------------------------------------------------- 1 | from multiprocessing import Process, Queue 2 | 3 | 4 | class EncodingsSampler(Process): 5 | def __init__(self, inp_encodings_queue: Queue, out_encodings_queue: Queue): 6 | super().__init__() 7 | 8 | self._inp_queue = inp_encodings_queue 9 | self._out_queue = out_encodings_queue 10 | 11 | def run(self) -> None: 12 | while True: 13 | encodings = self._inp_queue.get() 14 | encodings = sorted(encodings, key=lambda e: -len(e.token_ids)) 15 | for enc in encodings: 16 | self._out_queue.put(enc) 17 | -------------------------------------------------------------------------------- /full_stack_transformer/core/data/text_inputs_producer.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | from multiprocessing import Process, Queue 3 | 4 | from full_stack_transformer.core.data.text_lines_parsers import TextLinesParser 5 | 6 | 7 | class TextInputsProducer(Process): 8 | def __init__( 9 | self, 10 | file_path: pathlib.Path, 11 | out_text_inputs_queue: Queue, 12 | out_chunk_size: int, 13 | text_lines_parser: TextLinesParser 14 | ): 15 | super().__init__() 16 | 17 | self._file_path = file_path 18 | self._chunk_size = out_chunk_size 19 | self._out_queue = out_text_inputs_queue 20 | self._parser = text_lines_parser 21 | 22 | def run(self) -> None: 23 | while True: 24 | chunk = [] 25 | with self._file_path.open() as file: 26 | for line in file: 27 | 28 | text_input = self._parser.parse(line) 29 | if text_input is not None: 30 | chunk.append(text_input) 31 | 32 | if len(chunk) >= self._chunk_size: 33 | self._out_queue.put(chunk) 34 | chunk = [] 35 | 36 | if len(chunk) > 0: 37 | self._out_queue.put(chunk) 38 | -------------------------------------------------------------------------------- /full_stack_transformer/core/data/text_lines_parsers.py: -------------------------------------------------------------------------------- 1 | import abc 2 | from typing import Optional 3 | 4 | from full_stack_transformer.core.text_input import TextInput 5 | 6 | 7 | class TextLinesParser: 8 | @abc.abstractmethod 9 | def parse(self, text: str) -> Optional[TextInput]: 10 | pass 11 | -------------------------------------------------------------------------------- /full_stack_transformer/core/encoding.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List 3 | 4 | 5 | @dataclass 6 | class Encoding: 7 | """Base data container which represents encoded text.""" 8 | token_ids: List[int] 9 | lm_labels: List[int] -------------------------------------------------------------------------------- /full_stack_transformer/core/model_input.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Optional 3 | 4 | import torch 5 | 6 | 7 | class ModelInputError(Exception): 8 | pass 9 | 10 | 11 | @dataclass 12 | class ModelInput(dict): 13 | """Base data container which represents model input.""" 14 | input_ids: torch.Tensor 15 | token_type_ids: Optional[torch.Tensor] = None 16 | lm_labels: Optional[torch.Tensor] = None 17 | past: Optional[torch.Tensor] = None 18 | 19 | def __post_init__(self): 20 | super().__init__(**self.__dict__) 21 | 22 | self._check_inputs_validity() 23 | 24 | def _check_inputs_validity(self): 25 | if self.token_type_ids is not None and \ 26 | self.input_ids.size() != self.token_type_ids.size(): 27 | raise ModelInputError( 28 | '`input_ids` and `token_type_ids` shapes must be equal.' 29 | ) 30 | elif self.lm_labels is not None and \ 31 | self.lm_labels.size() != self.input_ids.size(): 32 | raise ModelInputError( 33 | '`lm_labels` were passed, bu there shape does not match ' 34 | '`input_ids` shape.' 35 | ) 36 | elif self.past is not None and self.input_ids.size()[1] != 1: 37 | raise ModelInputError( 38 | 'If `past` passed, `input_ids` must contain only one token, ' 39 | 'i.e sequence length must be equal to 1.' 40 | ) 41 | 42 | def cuda(self, gpu_id): 43 | """Sends object field to cuda.""" 44 | fields = self.__dict__ 45 | 46 | for name, field in fields.items(): 47 | if isinstance(field, torch.Tensor): 48 | self.__dict__[name] = field.cuda(gpu_id) 49 | 50 | return self 51 | -------------------------------------------------------------------------------- /full_stack_transformer/core/model_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Optional 3 | 4 | import torch 5 | 6 | 7 | @dataclass 8 | class ModelOutput: 9 | """Base data container for model output tensors.""" 10 | logits: torch.Tensor 11 | loss: Optional[torch.Tensor] 12 | 13 | 14 | @dataclass 15 | class LanguageModelOutput(ModelOutput): 16 | """Data container for language model output tensors.""" 17 | hidden: torch.Tensor 18 | past: torch.Tensor 19 | lm_loss: Optional[torch.Tensor] = None 20 | ul_loss: Optional[torch.Tensor] = None 21 | -------------------------------------------------------------------------------- /full_stack_transformer/core/modelling/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexeykarnachev/full_stack_transformer/2ac4a583c1ea610a4786a65258f33a941251bdea/full_stack_transformer/core/modelling/__init__.py -------------------------------------------------------------------------------- /full_stack_transformer/core/modelling/lightning.py: -------------------------------------------------------------------------------- 1 | import abc 2 | from typing import Mapping, Tuple, Dict, Sequence 3 | 4 | import torch 5 | from pytorch_lightning import LightningModule 6 | from pytorch_lightning.loggers.base import merge_dicts 7 | 8 | from full_stack_transformer.core.model_input import ModelInput 9 | from full_stack_transformer.core.model_output import ModelOutput 10 | from full_stack_transformer.core.modelling.model import Model 11 | from full_stack_transformer.utilities.arguments import ArgparserExtender 12 | 13 | 14 | class PLModule(LightningModule, ArgparserExtender): 15 | def __init__(self, model: Model): 16 | super().__init__() 17 | 18 | self.model = model 19 | 20 | def forward(self, model_inp: ModelInput) -> ModelOutput: 21 | output = self.model(model_inp) 22 | return output 23 | 24 | def training_step(self, model_inp: ModelInput, batch_idx: int) -> Dict: 25 | loss, log = self._step(model_inp=model_inp) 26 | return {'loss': loss, 'log': log} 27 | 28 | def validation_step(self, model_inp: ModelInput, batch_idx: int) -> Dict: 29 | loss, log = self._step(model_inp=model_inp) 30 | return {'val_loss': loss, 'log': log} 31 | 32 | def validation_epoch_end(self, val_step_results: Sequence): 33 | validation_epoch_result = merge_dicts( 34 | dicts=val_step_results, 35 | default_func=lambda x: torch.stack(x).mean().item() 36 | ) 37 | 38 | return validation_epoch_result 39 | 40 | def configure_optimizers(self): 41 | optimizer = self._get_optimizer() 42 | scheduler = self._get_lr_scheduler(optimizer=optimizer) 43 | 44 | return [optimizer], [scheduler] 45 | 46 | def _step(self, model_inp: ModelInput) -> Tuple[torch.Tensor, Mapping]: 47 | output = self.forward(model_inp=model_inp) 48 | log = self._get_step_log(model_output=output) 49 | return output.loss, log 50 | 51 | @abc.abstractmethod 52 | def _get_optimizer(self): 53 | pass 54 | 55 | @abc.abstractmethod 56 | def _get_lr_scheduler(self, optimizer): 57 | pass 58 | 59 | @abc.abstractmethod 60 | def _get_step_log(self, model_output: ModelOutput) -> Dict: 61 | pass 62 | 63 | @abc.abstractmethod 64 | def get_description(self) -> Dict: 65 | pass 66 | -------------------------------------------------------------------------------- /full_stack_transformer/core/modelling/loading.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import pathlib 3 | from typing import Optional, Union 4 | 5 | import transformers 6 | 7 | 8 | class ModelLoadingError(Exception): 9 | pass 10 | 11 | 12 | def load_transformer_model_from_path( 13 | model_path: Union[str, pathlib.Path], 14 | vocab_size: Optional[int] 15 | ) -> transformers.PreTrainedModel: 16 | config = transformers.AutoConfig.from_pretrained(model_path) 17 | modified_config = _modify_transformers_config(config) 18 | 19 | model = transformers.AutoModelForPreTraining.from_pretrained( 20 | pretrained_model_name_or_path=model_path, 21 | config=modified_config 22 | ) 23 | 24 | _resize_embeddings_if_needed(model, vocab_size) 25 | 26 | return model 27 | 28 | 29 | def initialize_transformer_model_from_config( 30 | config: transformers.PretrainedConfig, 31 | vocab_size: Optional[int] 32 | ) -> transformers.PreTrainedModel: 33 | modified_config = _modify_transformers_config(config) 34 | model = transformers.AutoModelForPreTraining.from_config(modified_config) 35 | 36 | _resize_embeddings_if_needed(model, vocab_size) 37 | 38 | return model 39 | 40 | 41 | def _modify_transformers_config( 42 | config: transformers.PretrainedConfig 43 | ) -> transformers.PretrainedConfig: 44 | config_copy = copy.deepcopy(config) 45 | config_copy.output_past = True 46 | config_copy.output_hidden_states = True 47 | return config_copy 48 | 49 | 50 | def _resize_embeddings_if_needed( 51 | model: transformers.PreTrainedModel, 52 | vocab_size: int 53 | ) -> None: 54 | if vocab_size is not None: 55 | mean_emb = model.base_model.wte.weight.data.mean(0) 56 | old_size = model.base_model.wte.weight.data.size()[0] 57 | n_new = vocab_size - old_size 58 | 59 | if n_new < 0: 60 | raise ModelLoadingError( 61 | "Can't resize embeddings: new vocab size can not be less than " 62 | "the old embeddings number (old vocab size)." 63 | ) 64 | 65 | model.resize_token_embeddings(vocab_size) 66 | idx = vocab_size - n_new 67 | model.base_model.wte.weight.data[idx:] = mean_emb.unsqueeze(0) 68 | -------------------------------------------------------------------------------- /full_stack_transformer/core/modelling/model.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | import torch 4 | import torch.nn as nn 5 | 6 | from full_stack_transformer.core.model_input import ModelInput 7 | from full_stack_transformer.core.model_output import ModelOutput 8 | 9 | 10 | class Model(nn.Module): 11 | 12 | @property 13 | def device(self): 14 | return self.parameters().__next__().device 15 | 16 | @abc.abstractmethod 17 | def forward(self, inp: ModelInput) -> ModelOutput: 18 | pass 19 | 20 | @torch.no_grad() 21 | @abc.abstractmethod 22 | def infer(self, inp: ModelInput) -> ModelOutput: 23 | pass 24 | -------------------------------------------------------------------------------- /full_stack_transformer/core/nn/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexeykarnachev/full_stack_transformer/2ac4a583c1ea610a4786a65258f33a941251bdea/full_stack_transformer/core/nn/__init__.py -------------------------------------------------------------------------------- /full_stack_transformer/core/nn/unlikelihood_candidates_loss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional 3 | 4 | 5 | def unlikelihood_candidates_loss(logits, target): 6 | """Loss which helps model not to predict already appeared tokens. 7 | Args: 8 | logits (tensor): 9 | Torch tensor of shape (bs, seq_len, vocab_size), output language 10 | model scores. 11 | target (tensor): 12 | Torch tensor of shape (bs, seq_len), language model target (model 13 | input tokens itself). 14 | Returns: 15 | Not-scaled unlikelihood candidates loss-value. 16 | Notes: 17 | This loss is based on penalizing of the previous context tokens. 18 | Original paper - Welleck et al. https://arxiv.org/pdf/1908.04319.pdf. 19 | """ 20 | 21 | candidates_logp = _get_candidates_logp(logits=logits, target=target) 22 | loss = _get_loss(candidates_logp) 23 | return loss 24 | 25 | 26 | def _get_candidates_logp(logits, target): 27 | logp = torch.nn.functional.log_softmax(logits, 2) 28 | bs = logits.size()[0] 29 | seq_len = logits.size()[1] 30 | 31 | logp_flat = logp.view(bs * seq_len, -1) 32 | tril = torch.tril(torch.ones((seq_len, seq_len), device=logits.device), diagonal=-1) 33 | 34 | cols = target.repeat(1, seq_len).view(seq_len * bs, -1) 35 | rows = torch.arange(0, cols.size()[0]).unsqueeze(-1).repeat(1, seq_len) 36 | 37 | not_ignore_mask = ~(cols == target.flatten().unsqueeze(-1).repeat(1, seq_len)) 38 | candidates_mask = tril.repeat(bs, 1).bool() 39 | candidates_mask *= not_ignore_mask 40 | 41 | cols_flat = cols[candidates_mask] 42 | rows_flat = rows[candidates_mask] 43 | 44 | logp = [] if len(cols_flat) == 0 else logp_flat[rows_flat, cols_flat] 45 | 46 | return logp 47 | 48 | 49 | def _get_loss(candidates_logp): 50 | if len(candidates_logp) == 0: 51 | return 0 52 | 53 | probs = torch.clamp((1.0 - candidates_logp.exp()), min=1e-5) 54 | 55 | loss = -torch.log(probs) 56 | loss = loss.mean() 57 | 58 | return loss 59 | -------------------------------------------------------------------------------- /full_stack_transformer/core/task_runner.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import argparse 3 | import copy 4 | from typing import Dict, Type, List 5 | 6 | from pytorch_lightning import Trainer, Callback 7 | from pytorch_lightning.callbacks import ModelCheckpoint 8 | from pytorch_lightning.loggers import TensorBoardLogger 9 | 10 | from full_stack_transformer.core.modelling.lightning import PLModule 11 | from full_stack_transformer.utilities.experiment import Workspace 12 | 13 | 14 | class TaskRunner(abc.ABC): 15 | def __init__(self, pl_module_cls: Type[PLModule]): 16 | self.pl_module_cls = pl_module_cls 17 | args = self._parse_args() 18 | 19 | self.workspace = Workspace( 20 | experiments_root=args['experiments_root'], 21 | experiment_name=args['experiment_name'] 22 | ) 23 | 24 | def _parse_args(self) -> Dict: 25 | parser = argparse.ArgumentParser() 26 | parser = Trainer.add_argparse_args(parser) 27 | parser = Workspace.add_argparse_args(parser) 28 | parser = self.pl_module_cls.add_argparse_args(parser) 29 | 30 | args = parser.parse_args() 31 | 32 | return args.__dict__ 33 | 34 | def run(self): 35 | args = self._parse_args() 36 | 37 | module = self.pl_module_cls(**args) 38 | 39 | description = { 40 | 'Arguments': args, 41 | 'Module': module.get_description() 42 | } 43 | self.workspace.save_description(description) 44 | 45 | trainer = self._prepare_trainer(args=args) 46 | 47 | trainer.fit(model=module) 48 | 49 | def _prepare_trainer(self, args): 50 | trainer_args = copy.deepcopy(args) 51 | 52 | _fix_trainer_args(args=trainer_args) 53 | 54 | trainer_args.update( 55 | { 56 | 'logger': TensorBoardLogger( 57 | save_dir=self.workspace.logs_dir, 58 | name=self.workspace.name, 59 | version=self.workspace.version 60 | ), 61 | 'checkpoint_callback': ModelCheckpoint( 62 | filepath=self.workspace.models_dir, 63 | verbose=True, 64 | save_top_k=2, 65 | period=0 66 | ) 67 | } 68 | ) 69 | 70 | trainer_args['callbacks'] = self._get_trainer_callbacks() 71 | 72 | trainer = Trainer(**trainer_args) 73 | 74 | return trainer 75 | 76 | @abc.abstractmethod 77 | def _get_trainer_callbacks(self) -> List[Callback]: 78 | pass 79 | 80 | 81 | def _fix_trainer_args(args: Dict) -> None: 82 | val_check_interval = args['val_check_interval'] 83 | if val_check_interval <= 1: 84 | val_check_interval = float(val_check_interval) 85 | else: 86 | val_check_interval = int(val_check_interval) 87 | 88 | args['val_check_interval'] = val_check_interval 89 | -------------------------------------------------------------------------------- /full_stack_transformer/core/text_input.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | 4 | @dataclass 5 | class TextInput: 6 | pass 7 | -------------------------------------------------------------------------------- /full_stack_transformer/core/tokenizer.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import re 3 | from typing import List 4 | 5 | from tokenizers.implementations import BaseTokenizer 6 | from transformers.tokenization_utils import PreTrainedTokenizerFast 7 | 8 | from full_stack_transformer.core.encoding import Encoding 9 | from full_stack_transformer.core.text_input import TextInput 10 | from full_stack_transformer.utilities.factory import get_object 11 | 12 | _END = '[END]' 13 | 14 | _SPECIAL_TOKENS = [_END] 15 | 16 | 17 | class Tokenizer(PreTrainedTokenizerFast): 18 | @property 19 | def eos_token_id(self): 20 | return self.convert_tokens_to_ids(self.eos_token) 21 | 22 | @property 23 | def eos_token(self): 24 | return _END 25 | 26 | @property 27 | def vocab_size(self): 28 | return max(self.all_special_ids) + 1 29 | 30 | def __init__(self, tokenizer: BaseTokenizer, **kwargs): 31 | super().__init__(tokenizer, **kwargs) 32 | 33 | self.add_special_tokens({'additional_special_tokens': _SPECIAL_TOKENS}) 34 | 35 | def encode_for_train(self, text_input: TextInput) -> List[Encoding]: 36 | text_input = self._preprocess_input(text_input=text_input) 37 | encodings = self._encode_for_train(text_input=text_input) 38 | 39 | return encodings 40 | 41 | def encode_for_inference(self, text_input: TextInput) -> List[Encoding]: 42 | text_input = self._preprocess_input(text_input=text_input) 43 | encodings = self._encode_for_inference(text_input=text_input) 44 | 45 | return encodings 46 | 47 | @abc.abstractmethod 48 | def _encode_for_train(self, text_input: TextInput) -> List[Encoding]: 49 | pass 50 | 51 | @abc.abstractmethod 52 | def _encode_for_inference(self, text_input: TextInput) -> List[Encoding]: 53 | pass 54 | 55 | @abc.abstractmethod 56 | def _preprocess_input(self, text_input: TextInput) -> TextInput: 57 | pass 58 | 59 | @abc.abstractmethod 60 | def _postprocess_text(self, text: str) -> str: 61 | pass 62 | 63 | def decode_encoding(self, encoding: Encoding) -> str: 64 | token_ids = encoding.token_ids 65 | text = self.decode(token_ids=token_ids, skip_special_tokens=True) 66 | 67 | for token in _SPECIAL_TOKENS: 68 | text = re.sub(re.escape(token), '', text) 69 | 70 | text = self._postprocess_text(text) 71 | 72 | text = text.strip() 73 | 74 | return text 75 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexeykarnachev/full_stack_transformer/2ac4a583c1ea610a4786a65258f33a941251bdea/full_stack_transformer/tasks/__init__.py -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/__init__.py: -------------------------------------------------------------------------------- 1 | from full_stack_transformer.tasks.document_lm.hf_gpt2_tokenizer import \ 2 | HFGPT2DocumentTokenizer 3 | from full_stack_transformer.tasks.document_lm.language_generator.generator import ( 4 | LanguageGeneratorParams, 5 | LanguageGenerator 6 | ) 7 | from full_stack_transformer.tasks.document_lm.modelling.loading import ( 8 | load_model_from_checkpoint, 9 | load_tokenizer_from_checkpoint 10 | ) 11 | from full_stack_transformer.tasks.document_lm.ru_transformers_tokenizer import \ 12 | RuTransformersDocumentTokenizer 13 | from full_stack_transformer.tasks.document_lm.text_input import DocumentInput 14 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexeykarnachev/full_stack_transformer/2ac4a583c1ea610a4786a65258f33a941251bdea/full_stack_transformer/tasks/document_lm/data/__init__.py -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/data/dataset.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | 3 | from full_stack_transformer.core.data.dataset import Dataset 4 | from full_stack_transformer.tasks.document_lm.data.encodings_collate import \ 5 | DocumentEncodingsCollate 6 | from full_stack_transformer.tasks.document_lm.data.text_lines_parsers import \ 7 | DocumentLinesParser 8 | from full_stack_transformer.tasks.document_lm.tokenizer import DocumentTokenizer 9 | from full_stack_transformer.utilities.files import count_lines_in_file 10 | 11 | 12 | class DocumentDataset(Dataset): 13 | def __init__( 14 | self, 15 | file_path: pathlib.Path, 16 | tokenizer: DocumentTokenizer, 17 | ): 18 | collate = DocumentEncodingsCollate(pad_value=tokenizer.pad_token_id) 19 | text_lines_parser = DocumentLinesParser() 20 | 21 | super().__init__( 22 | file_path=file_path, 23 | tokenizer=tokenizer, 24 | text_lines_parser=text_lines_parser, 25 | encodings_collate=collate 26 | ) 27 | 28 | def __len__(self) -> int: 29 | return count_lines_in_file(self._file_path) 30 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/data/encodings_collate.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence, Union, Optional 2 | 3 | import torch 4 | 5 | from full_stack_transformer.core.constants import LOSS_IGNORE 6 | from full_stack_transformer.core.encoding import Encoding 7 | from full_stack_transformer.core.model_input import ModelInput 8 | from full_stack_transformer.utilities.sequences import pad_sequences_from_right 9 | 10 | 11 | class DocumentEncodingsCollate: 12 | def __init__(self, pad_value: int): 13 | self._pad_value = pad_value 14 | 15 | def __call__( 16 | self, 17 | encodings: Sequence[Encoding], 18 | device: Optional[Union[torch.device, str]] = None 19 | ) -> ModelInput: 20 | model_input = _collate_encodings( 21 | encodings=encodings, 22 | pad_value=self._pad_value, 23 | device=device 24 | ) 25 | 26 | return model_input 27 | 28 | 29 | def _collate_encodings( 30 | encodings: Sequence[Encoding], 31 | pad_value: int, 32 | device: Optional[Union[torch.device, str]] = None 33 | ) -> ModelInput: 34 | token_ids = pad_sequences_from_right( 35 | sequences=[e.token_ids for e in encodings], 36 | pad_value=pad_value, 37 | max_len=None, 38 | ) 39 | lm_labels = pad_sequences_from_right( 40 | sequences=[e.lm_labels for e in encodings], 41 | pad_value=LOSS_IGNORE, 42 | max_len=None 43 | ) 44 | 45 | input_ids = torch.tensor(token_ids, dtype=torch.long, device=device) 46 | lm_labels = torch.tensor(lm_labels, dtype=torch.long, device=device) 47 | model_input = ModelInput(input_ids=input_ids, lm_labels=lm_labels) 48 | 49 | return model_input 50 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/data/text_lines_parsers.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import Optional 3 | 4 | from full_stack_transformer.core.data.text_lines_parsers import TextLinesParser 5 | from full_stack_transformer.tasks.document_lm.text_input import DocumentInput 6 | 7 | 8 | class DocumentLinesParser(TextLinesParser): 9 | def __init__(self): 10 | pass 11 | 12 | def parse(self, text: str) -> Optional[DocumentInput]: 13 | inp = DocumentInput(**json.loads(text)) 14 | return inp 15 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/examples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexeykarnachev/full_stack_transformer/2ac4a583c1ea610a4786a65258f33a941251bdea/full_stack_transformer/tasks/document_lm/examples/__init__.py -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/examples/language_generation.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | 3 | import torch 4 | 5 | from full_stack_transformer.tasks.document_lm import ( 6 | LanguageGeneratorParams, 7 | load_model_from_checkpoint, 8 | load_tokenizer_from_checkpoint, 9 | LanguageGenerator, 10 | DocumentInput 11 | ) 12 | 13 | if __name__ == '__main__': 14 | device = 'cuda:0' 15 | experiment_dir = pathlib.Path('../../../../data/experiments/nietzsche_v0/') 16 | ckpt_path = experiment_dir / 'models' / 'epoch=6.ckpt' 17 | 18 | generator_params = LanguageGeneratorParams( 19 | max_number_of_generated_tokens=64, 20 | num_return_sequences=8, 21 | repetition_penalty=3.0, 22 | temperature=1.0, 23 | top_k=50, 24 | top_p=1.0 25 | ) 26 | 27 | ckpt = torch.load(f=str(ckpt_path), map_location='cpu') 28 | model = load_model_from_checkpoint(ckpt=ckpt, device=device) 29 | tokenizer = load_tokenizer_from_checkpoint(ckpt=ckpt) 30 | generator = LanguageGenerator( 31 | model=model, eos_token_id=tokenizer.eos_token_id 32 | ) 33 | 34 | document = DocumentInput(body='The best filosopher of the 19th century is') 35 | inp_encoding = tokenizer.encode_for_inference(text_input=document)[0] 36 | 37 | out_encodings = generator(inp_encoding, params=generator_params) 38 | for enc in out_encodings: 39 | text = tokenizer.decode_encoding(enc) 40 | print(text + '\n\n') 41 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/hf_gpt2_tokenizer.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | 3 | from tokenizers import ByteLevelBPETokenizer 4 | 5 | from full_stack_transformer.tasks.document_lm.text_input import DocumentInput 6 | from full_stack_transformer.tasks.document_lm.tokenizer import \ 7 | DocumentTokenizer 8 | 9 | _THIS_DIR = pathlib.Path(__file__).parent 10 | _VOCAB = _THIS_DIR / '..' / 'static' / 'gpt2_bpe' / 'vocab.json' 11 | _MERGES = _THIS_DIR / '..' / 'static' / 'gpt2_bpe' / 'merges.txt' 12 | 13 | 14 | class HFGPT2DocumentTokenizer(DocumentTokenizer): 15 | def __init__( 16 | self, 17 | max_meta_len: int, 18 | max_body_len: int, 19 | ignore_meta_prob: float 20 | ): 21 | tokenizer = ByteLevelBPETokenizer( 22 | vocab_file=str(_VOCAB), 23 | merges_file=str(_MERGES) 24 | ) 25 | 26 | super().__init__( 27 | tokenizer=tokenizer, 28 | pad_token='', 29 | max_meta_len=max_meta_len, 30 | max_body_len=max_body_len, 31 | ignore_meta_prob=ignore_meta_prob 32 | ) 33 | 34 | def _preprocess_input(self, text_input: DocumentInput) -> DocumentInput: 35 | return text_input 36 | 37 | def _postprocess_text(self, text: str) -> str: 38 | return text 39 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/language_generator/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexeykarnachev/full_stack_transformer/2ac4a583c1ea610a4786a65258f33a941251bdea/full_stack_transformer/tasks/document_lm/language_generator/__init__.py -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/language_generator/callback.py: -------------------------------------------------------------------------------- 1 | import json 2 | import pathlib 3 | from collections import Mapping 4 | 5 | from pytorch_lightning import Callback, Trainer 6 | 7 | from full_stack_transformer.tasks.document_lm.language_generator.generator import LanguageGeneratorParams, \ 8 | LanguageGenerator 9 | from full_stack_transformer.tasks.document_lm.modelling.lightning import DocumentPLModule 10 | from full_stack_transformer.tasks.document_lm.text_input import DocumentInput 11 | from full_stack_transformer.utilities.experiment import Workspace 12 | 13 | 14 | class LanguageGeneratorCallback(Callback): 15 | _OUTPUT_FILE_NAME = 'generated.txt' 16 | 17 | @property 18 | def _default_params(self): 19 | return LanguageGeneratorParams( 20 | max_number_of_generated_tokens=128, 21 | num_return_sequences=8, 22 | repetition_penalty=1.0, 23 | temperature=0.7, 24 | top_k=0, 25 | top_p=1.0 26 | ) 27 | 28 | @property 29 | def _default_document(self): 30 | return DocumentInput(body='', meta=None) 31 | 32 | @property 33 | def _out_file(self) -> pathlib.Path: 34 | return self._workspace.experiment_dir / self._OUTPUT_FILE_NAME 35 | 36 | def __init__(self, experiment_workspace: Workspace): 37 | self._workspace = experiment_workspace 38 | 39 | def on_validation_end( 40 | self, 41 | trainer: Trainer, 42 | pl_module: DocumentPLModule 43 | ): 44 | params = self._default_params 45 | tokenizer = pl_module.tokenizer 46 | generator = LanguageGenerator( 47 | model=pl_module.model, 48 | eos_token_id=tokenizer.eos_token_id 49 | ) 50 | 51 | inp_encoding = tokenizer.encode_for_inference( 52 | text_input=self._default_document, 53 | )[0] 54 | 55 | encodings = generator(encoding=inp_encoding, params=params) 56 | 57 | text_samples = [tokenizer.decode_encoding(e) for e in encodings] 58 | 59 | result = { 60 | 'Global step': trainer.global_step, 61 | 'Current epoch': trainer.current_epoch, 62 | 'Generator params': params.__dict__, 63 | 'Generated samples': text_samples 64 | } 65 | 66 | self._dump_result(result=result) 67 | 68 | def _dump_result(self, result: Mapping): 69 | with self._out_file.open('a') as file: 70 | out_str = json.dumps(obj=result, ensure_ascii=False, indent=4) 71 | out_str += '\n' 72 | file.write(out_str) 73 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/language_generator/generator.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Sequence 3 | 4 | import torch 5 | import torch.nn.functional 6 | 7 | from full_stack_transformer.core.encoding import Encoding 8 | from full_stack_transformer.core.model_input import ModelInput 9 | from full_stack_transformer.tasks.document_lm.data.encodings_collate import DocumentEncodingsCollate 10 | from full_stack_transformer.tasks.document_lm.language_generator.logits_modifiers import ( 11 | IgnoredTokensModifier, 12 | RepetitiveTokensModifier, 13 | TemperatureModifier, 14 | TopKNucleusModifier 15 | ) 16 | from full_stack_transformer.tasks.document_lm.language_generator.progress import GenerationProgressTracker 17 | from full_stack_transformer.tasks.document_lm.modelling.model import DocumentModel 18 | 19 | 20 | @dataclass 21 | class LanguageGeneratorParams: 22 | max_number_of_generated_tokens: int 23 | num_return_sequences: int 24 | 25 | repetition_penalty: float 26 | temperature: float 27 | top_k: float 28 | top_p: float 29 | 30 | 31 | class LanguageGenerator: 32 | def __init__(self, model: DocumentModel, eos_token_id: int): 33 | self._model = model 34 | self._eos_token_ids = eos_token_id 35 | 36 | # LanguageGenerator never needs to pad sequences, so the `pad_value` 37 | # could be any here (e.g. 0). 38 | self._collator = DocumentEncodingsCollate(pad_value=0) 39 | 40 | def __call__( 41 | self, 42 | encoding: Encoding, 43 | params: LanguageGeneratorParams 44 | ) -> Sequence[Encoding]: 45 | 46 | self._model.eval() 47 | 48 | encodings = [encoding] * params.num_return_sequences 49 | model_inp = self._collator( 50 | encodings=encodings, 51 | device=self._model.device 52 | ) 53 | 54 | progress = GenerationProgressTracker( 55 | eos_token_id=self._eos_token_ids, 56 | max_length=params.max_number_of_generated_tokens 57 | ) 58 | 59 | generated_token_ids = torch.zeros( 60 | params.num_return_sequences, 61 | params.max_number_of_generated_tokens, 62 | dtype=torch.long 63 | ) 64 | 65 | generated_token_ids = generated_token_ids.to(self._model.device) 66 | 67 | past_token_ids = model_inp.input_ids.detach().clone() 68 | not_eos_mask = ~(past_token_ids == self._eos_token_ids).all(0) 69 | past_token_ids = past_token_ids[:, not_eos_mask] 70 | 71 | while not progress.finished: 72 | model_out = self._model.infer(inp=model_inp) 73 | next_token_logits = model_out.logits[:, -1, :] 74 | past_token_ids = torch.cat( 75 | tensors=[past_token_ids, generated_token_ids], 76 | dim=1 77 | ) 78 | _modify_next_token_logits( 79 | next_token_logits=next_token_logits, 80 | ignored_token_ids=[], 81 | token_ids_to_penalize=past_token_ids, 82 | repetition_penalty=params.repetition_penalty, 83 | temperature=params.temperature, 84 | top_k=params.top_k, 85 | top_p=params.top_p 86 | ) 87 | next_token_ids = _sample_next_token_ids(next_token_logits) 88 | progress.update(next_token_ids) 89 | 90 | generated_token_ids[:, progress.current_length - 1] = next_token_ids 91 | 92 | input_ids = next_token_ids.unsqueeze(1) 93 | 94 | if model_inp.token_type_ids is not None: 95 | token_type_ids = model_inp.token_type_ids[:, -1:] 96 | else: 97 | token_type_ids = None 98 | 99 | model_inp = ModelInput( 100 | input_ids=input_ids, 101 | token_type_ids=token_type_ids, 102 | past=model_out.past 103 | ) 104 | 105 | candidates = _get_candidates( 106 | generated_tokens=generated_token_ids, 107 | generated_sample_lengths=progress.generated_sample_lengths 108 | ) 109 | 110 | return candidates 111 | 112 | 113 | def _modify_next_token_logits( 114 | next_token_logits, 115 | ignored_token_ids, 116 | token_ids_to_penalize, 117 | repetition_penalty, 118 | temperature, 119 | top_k, 120 | top_p 121 | ): 122 | modifiers = [ 123 | IgnoredTokensModifier( 124 | ignored_token_ids=ignored_token_ids 125 | ), 126 | RepetitiveTokensModifier( 127 | penalty=repetition_penalty, 128 | token_ids_to_penalize=token_ids_to_penalize 129 | ), 130 | TemperatureModifier( 131 | temperature=temperature 132 | ), 133 | TopKNucleusModifier( 134 | top_k=top_k, 135 | top_p=top_p 136 | ) 137 | ] 138 | 139 | _ = [modifier(next_token_logits) for modifier in modifiers] 140 | 141 | 142 | def _sample_next_token_ids(next_token_logits: torch.tensor) -> torch.tensor: 143 | probabilities = torch.nn.functional.softmax( 144 | input=next_token_logits, 145 | dim=-1 146 | ) 147 | 148 | next_tokens = torch.multinomial( 149 | probabilities, num_samples=1 150 | ) 151 | return next_tokens.squeeze(1) 152 | 153 | 154 | def _get_candidates(generated_tokens, generated_sample_lengths): 155 | candidates = [] 156 | for i in range(generated_tokens.size()[0]): 157 | token_ids = generated_tokens[i, :generated_sample_lengths[i]] 158 | token_ids = token_ids.detach().cpu().numpy().tolist() 159 | candidate = Encoding(token_ids=token_ids, lm_labels=token_ids) 160 | candidates.append(candidate) 161 | 162 | return candidates 163 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/language_generator/logits_modifiers.py: -------------------------------------------------------------------------------- 1 | import abc 2 | from typing import Sequence, Optional 3 | 4 | import torch 5 | from transformers import modeling_utils 6 | 7 | _MINUS_INF = -float("Inf") 8 | 9 | 10 | class NextTokenLogitsModifierError(Exception): 11 | pass 12 | 13 | 14 | class NextTokenLogitsModifier(abc.ABC): 15 | @abc.abstractmethod 16 | def modify(self, logits: torch.tensor): 17 | """Modifies next token logits. Should modify them inplace.""" 18 | 19 | def __call__(self, logits: torch.tensor): 20 | return self.modify(logits) 21 | 22 | 23 | class TemperatureModifier(NextTokenLogitsModifier): 24 | """Classic temperature logits distribution modifier.""" 25 | 26 | def __init__(self, temperature: float): 27 | """ 28 | Args: 29 | temperature: 30 | The value used to module the next token probabilities. 31 | Must be >= 0. 0 Means, that the token with maximum logits value 32 | will be taken (no sampling). 33 | Raises: 34 | NextTokenLogitsModifierError: in case of incorrect input arguments. 35 | """ 36 | self._temperature = temperature 37 | self._check_arguments_validity() 38 | 39 | def _check_arguments_validity(self) -> None: 40 | if self._temperature < 0: 41 | raise NextTokenLogitsModifierError('`temperature` must be >= 0.') 42 | 43 | def modify(self, logits: torch.tensor): 44 | logits.mul_(1 / self._temperature) 45 | 46 | 47 | class TopKNucleusModifier(NextTokenLogitsModifier): 48 | """Filters a distribution of logits using top-k and/or top-p filtering""" 49 | 50 | def __init__(self, top_k: int, top_p: float): 51 | """ 52 | Args: 53 | top_k: 54 | The number of highest probability vocabulary tokens to keep for 55 | top-k-filtering. Between 0 and infinity. If 0, top-k filtering 56 | will not be using. 57 | top_p: 58 | The cumulative probability of parameter highest probability 59 | vocabulary tokens to keep for nucleus sampling. Must be between 60 | 0 and 1. 61 | Notes: 62 | `top_k` is performed before `top_p`. It means, that cumulative 63 | probability will be counted only for top k tokens. 64 | Raises: 65 | NextTokenLogitsModifierError: in case of incorrect input arguments. 66 | """ 67 | self._top_k = top_k 68 | self._top_p = top_p 69 | 70 | self._check_arguments_validity() 71 | 72 | def _check_arguments_validity(self) -> None: 73 | if self._top_k < 0: 74 | raise NextTokenLogitsModifierError( 75 | '`top_k` must be >= 0. Use `top_k` = 0 if you want to switch ' 76 | 'off the top-k filtering.') 77 | elif not (0 <= self._top_p <= 1): 78 | raise NextTokenLogitsModifierError( 79 | '`top_p` must be between 0 and 1.') 80 | 81 | def modify(self, logits: torch.tensor): 82 | """Delegates call to the transformers `top_k_top_p_filtering` func.""" 83 | modeling_utils.top_k_top_p_filtering( 84 | logits=logits, 85 | top_k=self._top_k, 86 | top_p=self._top_p, 87 | filter_value=_MINUS_INF) 88 | 89 | 90 | class IgnoredTokensModifier(NextTokenLogitsModifier): 91 | """Assigns zero probabilities logits to the ignored tokens.""" 92 | 93 | def __init__(self, ignored_token_ids: Optional[Sequence[int]]): 94 | """ 95 | Args: 96 | ignored_token_ids: 97 | Ignored token indexes sequence. 98 | """ 99 | self._ignored_token_ids = list(set(ignored_token_ids)) 100 | 101 | def modify(self, logits: torch.tensor): 102 | if self._ignored_token_ids: 103 | logits[:, self._ignored_token_ids] = _MINUS_INF 104 | 105 | 106 | class RepetitiveTokensModifier(NextTokenLogitsModifier): 107 | """Decreases probability (and logits) for tokens which have already been.""" 108 | 109 | def __init__( 110 | self, 111 | penalty: float, 112 | token_ids_to_penalize: torch.tensor 113 | ): 114 | """ 115 | Args: 116 | penalty: 117 | Repetitive tokens penalization strength (must be >= 1.0). 118 | 1.0 means no penalty. 119 | token_ids_to_penalize: 120 | Tensor with shape (batch_size, seq_len). 121 | Raises: 122 | NextTokenLogitsModifierError: in case of incorrect input arguments. 123 | """ 124 | self._token_ids_to_penalize = token_ids_to_penalize 125 | self._penalty = penalty 126 | self._check_arguments_validity() 127 | 128 | def _check_arguments_validity(self) -> None: 129 | if self._penalty < 1.0: 130 | raise NextTokenLogitsModifierError( 131 | "`penalty` must be >= 1.0. Use 1.0 if you don't want to apply " 132 | "repetition penalty.") 133 | 134 | def modify(self, logits: torch.tensor): 135 | for i in range(logits.size()[0]): 136 | ids_to_penalize = self._token_ids_to_penalize[i] 137 | _penalize_logits_tensor(logits[i], ids_to_penalize, self._penalty) 138 | 139 | 140 | def _penalize_logits_tensor(logits, penalty_idx, penalty): 141 | if penalty == 1.0: 142 | return 143 | 144 | idx = torch.unique(penalty_idx) 145 | logits -= logits.max() 146 | 147 | full_exp = torch.exp(logits) 148 | 149 | e = full_exp[idx] 150 | sum_e = torch.sum(e) 151 | s = torch.sum(full_exp) - sum_e 152 | 153 | n = torch.log((e * s) / (penalty * s + penalty * sum_e - sum_e)) 154 | logits[idx] = n 155 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/language_generator/progress.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class GenerationProgressTrackerError(Exception): 5 | pass 6 | 7 | 8 | def _return_value_if_not_initialized(value): 9 | def _return_value_if_not_initialized_inner(function_or_prop): 10 | def decorated(self, *args, **kwargs): 11 | if self._n_samples is None: 12 | return value 13 | else: 14 | return function_or_prop(self, *args, **kwargs) 15 | 16 | return decorated 17 | 18 | return _return_value_if_not_initialized_inner 19 | 20 | 21 | class GenerationProgressTracker: 22 | """Tracks generation progress.""" 23 | 24 | @property 25 | @_return_value_if_not_initialized(value=False) 26 | def max_length_reached(self): 27 | """Will be True, when all sequence lengths will reach max_length.""" 28 | return self.current_length >= self._max_length 29 | 30 | @property 31 | @_return_value_if_not_initialized(value=False) 32 | def all_samples_finished(self): 33 | """Will be True, when eos_token_id will appear in every sequence.""" 34 | return self._unfinished_mask.max() == 0 35 | 36 | @property 37 | @_return_value_if_not_initialized(value=False) 38 | def finished(self): 39 | return self.max_length_reached or self.all_samples_finished 40 | 41 | @property 42 | def generated_sample_lengths(self): 43 | return self._gen_lengths 44 | 45 | def __init__(self, eos_token_id: int, max_length: int): 46 | """ 47 | Args: 48 | eos_token_id: 49 | End of string token id. It's needed for GeneratorProgress to 50 | understand which sample is finished. 51 | max_length: 52 | Maximum length of the generated sequences. 53 | """ 54 | self._eos_token_id = eos_token_id 55 | self._max_length = max_length 56 | self.current_length = 0 57 | 58 | self._n_samples = None 59 | self._unfinished_mask = None 60 | self._gen_lengths = None 61 | 62 | self._check_arguments_validity() 63 | 64 | def _check_arguments_validity(self) -> None: 65 | if self._max_length < 1: 66 | raise GenerationProgressTrackerError("`max_length` must be >= 1.") 67 | elif self._eos_token_id < 0: 68 | raise GenerationProgressTrackerError("`eos_token_id` must be >= 0.") 69 | 70 | def update(self, next_token_ids) -> None: 71 | """Updates generation progress status.""" 72 | self._assert_update_is_possible() 73 | self._initialize_if_needed(next_token_ids) 74 | 75 | not_eos_tokens_mask = next_token_ids.ne(self._eos_token_id).bool() 76 | self._gen_lengths[self._unfinished_mask] += 1 77 | 78 | self._unfinished_mask *= not_eos_tokens_mask 79 | self.current_length += 1 80 | 81 | def _assert_update_is_possible(self): 82 | if self.finished: 83 | raise GenerationProgressTrackerError( 84 | "Can't update generation progress, because it's already " 85 | "finished.") 86 | 87 | def _initialize_if_needed(self, next_tokens): 88 | if self._n_samples is None: 89 | device = next_tokens.device 90 | self._n_samples = len(next_tokens) 91 | self._unfinished_mask = torch.ones( 92 | self._n_samples, 93 | dtype=torch.bool, 94 | device=device 95 | ) 96 | self._gen_lengths = torch.zeros( 97 | self._n_samples, 98 | dtype=torch.long, 99 | device=device 100 | ) 101 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/modelling/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexeykarnachev/full_stack_transformer/2ac4a583c1ea610a4786a65258f33a941251bdea/full_stack_transformer/tasks/document_lm/modelling/__init__.py -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/modelling/lightning.py: -------------------------------------------------------------------------------- 1 | import json 2 | import pathlib 3 | from typing import Dict, Optional 4 | 5 | import transformers 6 | from transformers import get_cosine_with_hard_restarts_schedule_with_warmup 7 | 8 | from full_stack_transformer.core.data.dataset import DataLoader 9 | from full_stack_transformer.core.model_output import LanguageModelOutput 10 | from full_stack_transformer.core.modelling.lightning import PLModule 11 | from full_stack_transformer.core.modelling.loading import load_transformer_model_from_path 12 | from full_stack_transformer.tasks.document_lm.data.dataset import DocumentDataset 13 | from full_stack_transformer.tasks.document_lm.modelling.model import DocumentModel 14 | from full_stack_transformer.tasks.document_lm.tokenizer import get_tokenizer 15 | from full_stack_transformer.utilities.arguments import get_func_arg_values_as_namespace 16 | 17 | 18 | class DocumentPLModule(PLModule): 19 | def __init__( 20 | self, 21 | model_path: str, 22 | tokenizer_class_name: str, 23 | batch_size: int, 24 | max_meta_len: int, 25 | max_body_len: int, 26 | ignore_meta_prob: float, 27 | train_file: str, 28 | valid_file: str, 29 | learning_rate: float, 30 | num_warmup_steps: int, 31 | num_cycles: int, 32 | unlikelihood_alpha: float, 33 | **kwargs 34 | ): 35 | """ 36 | Args: 37 | model_path (str): 38 | Path to the pre-trained transformer model. 39 | tokenizer_class_name (str): 40 | Name of the tokenizer class which is importable from 41 | `dialog_models.dialog_data`. 42 | batch_size (int): 43 | Batch size (the same for training and validation). 44 | max_meta_len (int): 45 | Max number of tokens for `meta` field encoding. Longer meta 46 | field will be cut from the left side (right side will be 47 | remained). 48 | max_body_len (int): 49 | Max number of tokens for `body` field encoding. Longer bodies 50 | will be chunked. Encoding of the `meta` will be appended to 51 | each part of the encoded body. 52 | ignore_meta_prob (float): 53 | The probability to ignore `meta` field in the document and don't 54 | add it to the final encoding. 55 | train_file (str): 56 | Path to the training raw dialog samples file. 57 | valid_file (str): 58 | Path to the validation raw dialog samples file. 59 | learning_rate (float): 60 | Base learning rate for AdamW optimizer. 61 | num_warmup_steps (int): 62 | Number of warmup steps for the cosine with hard restarts 63 | scheduler. 64 | num_cycles (int): 65 | Number of cycles for the cosine with hard restarts scheduler. 66 | If 0, the scheduler will perform as a constant scheduler with 67 | warmup. 68 | unlikelihood_alpha (float): 69 | Unlikelihood loss multiplier. If None, no unlikelihood loss will 70 | be used. 71 | """ 72 | self.train_file = pathlib.Path(train_file) 73 | self.valid_file = pathlib.Path(valid_file) 74 | self.learning_rate = learning_rate 75 | self.batch_size = batch_size 76 | self.num_warmup_steps = num_warmup_steps 77 | self.num_cycles = num_cycles 78 | 79 | tokenizer_config = dict( 80 | name=tokenizer_class_name, 81 | max_meta_len=max_meta_len, 82 | max_body_len=max_body_len, 83 | ignore_meta_prob=ignore_meta_prob 84 | ) 85 | 86 | self.tokenizer = get_tokenizer(**tokenizer_config) 87 | 88 | lm_head_model = load_transformer_model_from_path( 89 | model_path=model_path, 90 | vocab_size=self.tokenizer.vocab_size 91 | ) 92 | model = DocumentModel( 93 | lm_head_model=lm_head_model, 94 | unlikelihood_alpha=unlikelihood_alpha 95 | ) 96 | 97 | self.train_dataset: Optional[DocumentDataset] = None 98 | self.valid_dataset: Optional[DocumentDataset] = None 99 | 100 | locals_ = locals() 101 | self.transformer_config = json.dumps( 102 | lm_head_model.config.__dict__, 103 | ensure_ascii=False 104 | ) 105 | 106 | super().__init__(model=model) 107 | 108 | self.hparams = get_func_arg_values_as_namespace( 109 | locals_=locals_, 110 | func=self.__init__, 111 | transformer_config=self.transformer_config, 112 | tokenizer_config=tokenizer_config 113 | ) 114 | 115 | def prepare_data(self) -> None: 116 | self.train_dataset = DocumentDataset( 117 | file_path=self.train_file, 118 | tokenizer=self.tokenizer, 119 | ) 120 | 121 | self.valid_dataset = DocumentDataset( 122 | file_path=self.valid_file, 123 | tokenizer=self.tokenizer 124 | ) 125 | 126 | def train_dataloader(self) -> DataLoader: 127 | return self.train_dataset.get_data_loader( 128 | batch_size=self.batch_size, 129 | num_workers=4 130 | ) 131 | 132 | def val_dataloader(self) -> DataLoader: 133 | return self.valid_dataset.get_data_loader( 134 | batch_size=self.batch_size, 135 | num_workers=1 136 | ) 137 | 138 | def _get_optimizer(self): 139 | parameters = self.model.parameters() 140 | optimizer = transformers.AdamW( 141 | params=parameters, 142 | lr=self.learning_rate 143 | ) 144 | 145 | return optimizer 146 | 147 | def _get_lr_scheduler(self, optimizer): 148 | total_steps = len(self.train_dataloader()) * self.trainer.max_epochs 149 | training_steps = total_steps // self.trainer.accumulate_grad_batches 150 | 151 | lr_scheduler = get_cosine_with_hard_restarts_schedule_with_warmup( 152 | optimizer=optimizer, 153 | num_warmup_steps=self.num_warmup_steps, 154 | num_training_steps=training_steps, 155 | num_cycles=self.num_cycles 156 | ) 157 | scheduler = { 158 | 'scheduler': lr_scheduler, 159 | 'interval': 'step', 160 | 'frequency': self.trainer.accumulate_grad_batches, 161 | 'monitor': 'Loss/valid' 162 | } 163 | 164 | return scheduler 165 | 166 | def _get_step_log(self, model_output: LanguageModelOutput) -> Dict: 167 | postfix = 'train' if self.training else 'valid' 168 | log = { 169 | f'LM-Loss/{postfix}': model_output.lm_loss, 170 | f'UL-Loss/{postfix}': model_output.ul_loss, 171 | f'Loss/{postfix}': model_output.loss 172 | } 173 | 174 | if self.training: 175 | current_lr = self.trainer.optimizers[0].param_groups[0]['lr'] 176 | log['Learning-Rate'] = current_lr 177 | 178 | return log 179 | 180 | def get_description(self) -> Dict: 181 | return {'Transformer': self.transformer_config} 182 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/modelling/loading.py: -------------------------------------------------------------------------------- 1 | import json 2 | import re 3 | 4 | from transformers import GPT2Config 5 | 6 | from full_stack_transformer.core.modelling.loading import initialize_transformer_model_from_config 7 | from full_stack_transformer.tasks.document_lm.modelling.model import DocumentModel 8 | from full_stack_transformer.tasks.document_lm.tokenizer import get_tokenizer, DocumentTokenizer 9 | 10 | 11 | def load_model_from_checkpoint(ckpt, device): 12 | state_dict = dict() 13 | 14 | for k, v in ckpt['state_dict'].items(): 15 | new_key = re.search(r'^model\.(.+)$', k).group(1) 16 | state_dict[new_key] = v 17 | 18 | vocab_size = state_dict['lm_head_model.transformer.wte.weight'].size()[0] 19 | 20 | transformer_config = json.loads(ckpt['hparams']['transformer_config']) 21 | transformer_config = GPT2Config(**transformer_config) 22 | 23 | lm_head_model = initialize_transformer_model_from_config( 24 | config=transformer_config, 25 | vocab_size=vocab_size 26 | ) 27 | 28 | model = DocumentModel( 29 | lm_head_model=lm_head_model, 30 | unlikelihood_alpha=None 31 | ) 32 | 33 | model.load_state_dict(state_dict=state_dict) 34 | model = model.to(device) 35 | 36 | return model 37 | 38 | 39 | def load_tokenizer_from_checkpoint(ckpt) -> DocumentTokenizer: 40 | config = ckpt['hparams']['tokenizer_config'] 41 | config['ignore_meta_prob'] = 0 42 | tokenizer = get_tokenizer(**config) 43 | return tokenizer 44 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/modelling/model.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import torch 4 | import transformers 5 | 6 | from full_stack_transformer.core.model_input import ModelInput 7 | from full_stack_transformer.core.model_output import LanguageModelOutput 8 | from full_stack_transformer.core.modelling.model import Model 9 | from full_stack_transformer.core.nn.unlikelihood_candidates_loss import \ 10 | unlikelihood_candidates_loss 11 | 12 | 13 | class DocumentModel(Model): 14 | 15 | @property 16 | def device(self): 17 | return self.parameters().__next__().device 18 | 19 | def __init__( 20 | self, 21 | lm_head_model: transformers.GPT2LMHeadModel, 22 | unlikelihood_alpha: Optional[float] 23 | ): 24 | super().__init__() 25 | 26 | self.lm_head_model = lm_head_model 27 | self._ul_alpha = unlikelihood_alpha 28 | 29 | def forward(self, inp: ModelInput) -> LanguageModelOutput: 30 | lm_loss, logits, past, hidden = self.lm_head_model( 31 | input_ids=inp.input_ids, 32 | token_type_ids=inp.token_type_ids, 33 | labels=inp.lm_labels, 34 | past=inp.past 35 | ) 36 | 37 | if self._ul_alpha is not None: 38 | ul_loss = unlikelihood_candidates_loss( 39 | logits=logits, 40 | target=inp.input_ids 41 | ) 42 | 43 | loss = lm_loss + self._ul_alpha * ul_loss 44 | else: 45 | loss = lm_loss 46 | ul_loss = None 47 | 48 | output = LanguageModelOutput( 49 | lm_loss=lm_loss, 50 | ul_loss=ul_loss, 51 | loss=loss, 52 | logits=logits, 53 | past=past, 54 | hidden=hidden 55 | ) 56 | 57 | return output 58 | 59 | @torch.no_grad() 60 | def infer(self, inp: ModelInput) -> LanguageModelOutput: 61 | """Performs forward pass without loss calculation.""" 62 | 63 | logits, past, hidden = self.lm_head_model( 64 | input_ids=inp.input_ids, 65 | token_type_ids=inp.token_type_ids, 66 | past=inp.past 67 | ) 68 | 69 | output = LanguageModelOutput( 70 | logits=logits, 71 | past=past, 72 | hidden=hidden, 73 | loss=None 74 | ) 75 | 76 | return output 77 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/ru_transformers_tokenizer.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import re 3 | from typing import Optional 4 | 5 | from tokenizers import SentencePieceBPETokenizer 6 | 7 | from full_stack_transformer.tasks.document_lm.text_input import DocumentInput 8 | from full_stack_transformer.tasks.document_lm.tokenizer import \ 9 | DocumentTokenizer 10 | 11 | _THIS_DIR = pathlib.Path(__file__).parent 12 | _VOCAB = _THIS_DIR / '..' / 'static' / 'ru_transformers_sp' / 'vocab.json' 13 | _MERGES = _THIS_DIR / '..' / 'static' / 'ru_transformers_sp' / 'merges.txt' 14 | 15 | _NEW_LINE_REP = '<|n|>' 16 | 17 | 18 | class RuTransformersDocumentTokenizer(DocumentTokenizer): 19 | def __init__( 20 | self, 21 | max_meta_len: int, 22 | max_body_len: int, 23 | ignore_meta_prob: float 24 | ): 25 | tokenizer = SentencePieceBPETokenizer( 26 | vocab_file=str(_VOCAB), 27 | merges_file=str(_MERGES) 28 | ) 29 | 30 | super().__init__( 31 | tokenizer=tokenizer, 32 | max_meta_len=max_meta_len, 33 | max_body_len=max_body_len, 34 | ignore_meta_prob=ignore_meta_prob, 35 | pad_token='' 36 | ) 37 | 38 | def _preprocess_input(self, text_input: DocumentInput) -> DocumentInput: 39 | body = _preprocess(text_input.body) 40 | meta = _preprocess(text_input.meta) 41 | 42 | new_input = DocumentInput(body=body, meta=meta) 43 | 44 | return new_input 45 | 46 | def _postprocess_text(self, text: str) -> str: 47 | return _postrpocess(text=text) 48 | 49 | 50 | def _preprocess(text: Optional[str]) -> Optional[str]: 51 | if text is None: 52 | return None 53 | 54 | if text and text[0] != ' ': 55 | text = ' ' + text 56 | 57 | text = re.sub(r'(?=[^ ])([\W])([\w])', r'\g<1> \g<2>', text) 58 | text = text.replace('\n', f' {_NEW_LINE_REP}') 59 | return text 60 | 61 | 62 | def _postrpocess(text: str) -> str: 63 | text = re.sub(re.escape(_NEW_LINE_REP), '\n', text) 64 | text = re.sub(r'([\n(]) (\w)', r'\g<1>\g<2>', text) 65 | text = re.sub(r'(\W|^)([«"''\n(]|^) (\w)', r'\g<1>\g<2>\g<3>', text) 66 | text = re.sub(r'(\w)- (\w)', r'\g<1>-\g<2>', text) 67 | return text 68 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/serving/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexeykarnachev/full_stack_transformer/2ac4a583c1ea610a4786a65258f33a941251bdea/full_stack_transformer/tasks/document_lm/serving/__init__.py -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/serving/app.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import pathlib 3 | 4 | import torch 5 | from fastapi import FastAPI 6 | 7 | import full_stack_transformer 8 | from full_stack_transformer.tasks.document_lm.language_generator.generator import LanguageGenerator 9 | from full_stack_transformer.tasks.document_lm.modelling.loading import ( 10 | load_tokenizer_from_checkpoint, 11 | load_model_from_checkpoint 12 | ) 13 | from full_stack_transformer.tasks.document_lm.serving.views import ViewsRegister 14 | from full_stack_transformer.utilities.log_config import prepare_logging 15 | 16 | _LOGGER = logging.getLogger(__name__) 17 | 18 | 19 | def prepare(checkpoint_path, device, logs_dir) -> FastAPI: 20 | prepare_logging(pathlib.Path(logs_dir)) 21 | ckpt = torch.load(f=checkpoint_path, map_location='cpu') 22 | 23 | tokenizer = load_tokenizer_from_checkpoint(ckpt=ckpt) 24 | 25 | model = load_model_from_checkpoint(ckpt=ckpt, device=device) 26 | generator = LanguageGenerator( 27 | model=model, 28 | eos_token_id=tokenizer.eos_token_id 29 | ) 30 | version = _get_version_from_ckpt( 31 | ckpt=ckpt, 32 | checkpoint_path=checkpoint_path 33 | ) 34 | app = _prepare_app( 35 | generator=generator, 36 | tokenizer=tokenizer, 37 | version=version 38 | ) 39 | 40 | _LOGGER.info( 41 | 'All text_generator_service components were successfully initialized.' 42 | ) 43 | 44 | return app 45 | 46 | 47 | def _get_version_from_ckpt(ckpt, **kwargs): 48 | version = dict() 49 | 50 | version['package_version'] = full_stack_transformer.__version__ 51 | version['epoch'] = ckpt['epoch'] 52 | version['global_step'] = ckpt['global_step'] 53 | version['hparams'] = ckpt['hparams'] 54 | version.update(kwargs) 55 | 56 | return version 57 | 58 | 59 | def _prepare_app(generator, tokenizer, version): 60 | app = FastAPI() 61 | views_register = ViewsRegister( 62 | app=app, 63 | generator=generator, 64 | version=version, 65 | tokenizer=tokenizer 66 | ) 67 | views_register.register_all_views() 68 | 69 | return app 70 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/serving/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | PORT=$1 4 | WORKERS=$2 5 | LOGS_DIR=$3 6 | CHECKPOINT_PATH=$4 7 | DEVICE=$5 8 | 9 | TIMEOUT=600 10 | WORKER_CLASS="uvicorn.workers.UvicornWorker" 11 | MODULE_PATH="full_stack_transformer.tasks.document_lm.serving.app" 12 | ACCESS_LOGFILE=$LOGS_DIR"/gunicorn_access.log" 13 | ERROR_LOGFILE=$LOGS_DIR"/gunicorn_error.log" 14 | LOG_LEVEL="DEBUG" 15 | 16 | mkdir -p "${LOGS_DIR}" 17 | 18 | exec gunicorn ${MODULE_PATH}":prepare(checkpoint_path='${CHECKPOINT_PATH}', device='${DEVICE}', logs_dir='${LOGS_DIR}')" \ 19 | -b :"${PORT}" \ 20 | --timeout "${TIMEOUT}" \ 21 | -k "${WORKER_CLASS}" \ 22 | --workers "${WORKERS}" \ 23 | --access-logfile "${ACCESS_LOGFILE}" \ 24 | --error-logfile "${ERROR_LOGFILE}" \ 25 | --log-level "${LOG_LEVEL}" \ 26 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/serving/schemas.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | from pydantic import BaseModel, Field 4 | 5 | 6 | class GeneratedTexts(BaseModel): 7 | texts: List[str] = Field() 8 | 9 | 10 | class LanguageGeneratorAppParams(BaseModel): 11 | body: str = Field(default='', max_length=1024) 12 | meta: Optional[str] = Field(default=None, max_length=1024) 13 | 14 | max_number_of_generated_tokens: int = Field(default=128, ge=1, le=512) 15 | temperature: float = Field(default=0.7, gt=0, le=100) 16 | top_k: int = Field(default=50, ge=0) 17 | top_p: float = Field(default=1.0, gt=0.0, le=1.0) 18 | repetition_penalty: float = Field(default=5.0, ge=1.0, le=100) 19 | num_return_sequences: int = Field(default=1, ge=1, le=64) -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/serving/views.py: -------------------------------------------------------------------------------- 1 | import re 2 | from typing import Mapping 3 | 4 | from fastapi import FastAPI 5 | 6 | from full_stack_transformer.tasks.document_lm.language_generator.generator import LanguageGenerator, \ 7 | LanguageGeneratorParams 8 | from full_stack_transformer.tasks.document_lm.serving.schemas import ( 9 | GeneratedTexts, 10 | LanguageGeneratorAppParams 11 | ) 12 | from full_stack_transformer.tasks.document_lm.text_input import DocumentInput 13 | from full_stack_transformer.tasks.document_lm.tokenizer import DocumentTokenizer 14 | 15 | 16 | class ViewsRegister: 17 | def __init__( 18 | self, 19 | app: FastAPI, 20 | generator: LanguageGenerator, 21 | tokenizer: DocumentTokenizer, 22 | version: Mapping 23 | ): 24 | self._app = app 25 | self._generator = generator 26 | self._version = version 27 | self._tokenizer = tokenizer 28 | 29 | def register_generated_texts_view(self): 30 | @self._app.post("/generated_texts/", response_model=GeneratedTexts) 31 | def generated_texts( 32 | app_params: LanguageGeneratorAppParams 33 | ) -> GeneratedTexts: 34 | params = LanguageGeneratorParams( 35 | app_params.max_number_of_generated_tokens, 36 | num_return_sequences=app_params.num_return_sequences, 37 | repetition_penalty=app_params.repetition_penalty, 38 | temperature=app_params.temperature, 39 | top_p=app_params.top_p, 40 | top_k=app_params.top_k 41 | ) 42 | document = DocumentInput( 43 | body=app_params.body, 44 | meta=app_params.meta 45 | ) 46 | inp_encoding = self._tokenizer.encode_for_inference( 47 | text_input=document 48 | )[0] 49 | encodings = self._generator( 50 | encoding=inp_encoding, 51 | params=params 52 | ) 53 | texts = [self._tokenizer.decode_encoding(e) for e in encodings] 54 | return GeneratedTexts(texts=texts) 55 | 56 | return generated_texts 57 | 58 | def register_health_check_view(self): 59 | @self._app.get("/health_check/") 60 | def health_check(): 61 | return "ok" 62 | 63 | return health_check 64 | 65 | def register_version_view(self): 66 | @self._app.get("/version/") 67 | def version(): 68 | return self._version 69 | 70 | return version 71 | 72 | def register_all_views(self): 73 | for field in dir(self): 74 | if re.match('^register.+view$', field): 75 | getattr(self, field)() -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/task_runner.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from pytorch_lightning import Callback 4 | 5 | from full_stack_transformer.core.task_runner import TaskRunner 6 | from full_stack_transformer.tasks.document_lm.language_generator.callback import LanguageGeneratorCallback 7 | from full_stack_transformer.tasks.document_lm.modelling.lightning import DocumentPLModule 8 | 9 | 10 | class DocumentLMTaskRunner(TaskRunner): 11 | def __init__(self): 12 | super().__init__(pl_module_cls=DocumentPLModule) 13 | 14 | def _get_trainer_callbacks(self) -> List[Callback]: 15 | return [LanguageGeneratorCallback(experiment_workspace=self.workspace)] 16 | 17 | 18 | if __name__ == '__main__': 19 | runner = DocumentLMTaskRunner() 20 | runner.run() -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/telegram/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexeykarnachev/full_stack_transformer/2ac4a583c1ea610a4786a65258f33a941251bdea/full_stack_transformer/tasks/document_lm/telegram/__init__.py -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/telegram/app.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import pathlib 3 | from typing import Optional 4 | 5 | import aiohttp 6 | from aiogram import Dispatcher, Bot 7 | from aiogram.utils import executor 8 | 9 | from full_stack_transformer.tasks.document_lm.telegram.handlers import HandlersRegister 10 | 11 | THIS_DIR = pathlib.Path(__file__).parent 12 | 13 | 14 | def _parse_args(): 15 | logs_dir = THIS_DIR / '../../data/telegram_logs/' 16 | 17 | parser = argparse.ArgumentParser( 18 | description='This script runs telegram client for the text generator ' 19 | 'service' 20 | ) 21 | 22 | parser.add_argument( 23 | '--telegram_api_token', type=str, required=True, 24 | help='Telegram client API token. Could be obtained via `@BotFather` ' 25 | 'bot in telegram.' 26 | ) 27 | parser.add_argument( 28 | '--text_generator_service_url', type=str, required=True 29 | ) 30 | parser.add_argument( 31 | '--text_generator_service_login', type=str, required=False 32 | ) 33 | parser.add_argument( 34 | '--text_generator_service_password', type=str, required=False 35 | ) 36 | parser.add_argument( 37 | '--logs_dir', type=str, required=False, default=logs_dir 38 | ) 39 | 40 | args = parser.parse_args() 41 | return args 42 | 43 | 44 | def main(): 45 | args = _parse_args() 46 | 47 | dispatcher = prepare( 48 | logs_dir=args.logs_dir, 49 | telegram_api_token=args.telegram_api_token, 50 | text_generator_service_url=args.text_generator_service_url, 51 | text_generator_service_login=args.text_generator_service_login, 52 | text_generator_service_password=args.text_generator_service_password 53 | ) 54 | 55 | executor.start_polling(dispatcher, skip_updates=True) 56 | 57 | 58 | def prepare( 59 | telegram_api_token: str, 60 | text_generator_service_url: str, 61 | logs_dir: pathlib.Path, 62 | text_generator_service_login: Optional[str] = None, 63 | text_generator_service_password: Optional[str] = None 64 | ) -> Dispatcher: 65 | """Prepares dispatcher object.""" 66 | 67 | bot = Bot(token=telegram_api_token) 68 | dispatcher = Dispatcher(bot) 69 | 70 | if text_generator_service_login and text_generator_service_password: 71 | text_generator_service_auth = aiohttp.BasicAuth( 72 | login=text_generator_service_login, 73 | password=text_generator_service_password 74 | ) 75 | else: 76 | text_generator_service_auth = None 77 | 78 | handlers_register = HandlersRegister( 79 | dispatcher=dispatcher, 80 | text_generator_service_url=text_generator_service_url, 81 | text_generator_service_auth=text_generator_service_auth, 82 | logs_dir=logs_dir 83 | ) 84 | 85 | handlers_register.register_all_handlers() 86 | 87 | return dispatcher 88 | 89 | 90 | if __name__ == '__main__': 91 | main() -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/telegram/handlers.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import pathlib 5 | from http import HTTPStatus 6 | from typing import Optional, Tuple 7 | 8 | import aiohttp 9 | from aiogram import Dispatcher, types 10 | from aiogram.types import InlineKeyboardMarkup, InlineKeyboardButton, ParseMode 11 | from aiohttp import ServerDisconnectedError 12 | 13 | from full_stack_transformer.utilities.strings import get_string_md5 14 | 15 | 16 | class HandlersRegister: 17 | """Class which registers all text generator telegram client handlers.""" 18 | 19 | WELCOME_MESSAGE = "Hello, send me a message and I'll continue it." 20 | TEXT_GENERATOR_SERVICE_ERROR_MESSAGE = "I'm broken, or tired..." 21 | CANT_REPEAT_MESSAGE = "Can't repeat, please send me a new message." 22 | 23 | REPEAT_CALLBACK_DATA_PREFIX = '__repeat__' 24 | 25 | DEFAULT_PARAMS = { 26 | 'max_number_of_generated_tokens': 128, 27 | 'temperature': 1.0, 28 | 'top_k': 0, 29 | 'top_p': 1.0, 30 | 'repetition_penalty': 1.0 31 | } 32 | 33 | def __init__( 34 | self, 35 | dispatcher: Dispatcher, 36 | text_generator_service_url: str, 37 | text_generator_service_auth: Optional, 38 | logs_dir: pathlib.Path 39 | ): 40 | self._dispatcher = dispatcher 41 | self._msg_cache = MessagesCache() 42 | self._url = text_generator_service_url 43 | self._auth = text_generator_service_auth 44 | self._log_handler = LoggingHandler(logs_dir=logs_dir) 45 | 46 | def register_start_message_handler(self): 47 | """Handles `/start` command and sends welcome message.""" 48 | 49 | @self._dispatcher.message_handler(commands=['start']) 50 | async def start(message: types.Message): 51 | await message.answer(self.WELCOME_MESSAGE) 52 | 53 | def register_send_reply_message_handler(self): 54 | """Replies on user input message.""" 55 | 56 | @self._dispatcher.message_handler() 57 | async def send_reply(message: types.Message): 58 | message_hash = self._msg_cache.add_message(message.text) 59 | 60 | await self._get_response_and_send_reply( 61 | message=message, 62 | seed_string=message.text, 63 | callback_data=message_hash 64 | ) 65 | 66 | def register_send_reply_callback_query_handler(self): 67 | """Replies with user's previous seed text.""" 68 | 69 | @self._dispatcher.callback_query_handler( 70 | lambda q: q.data.startswith(self.REPEAT_CALLBACK_DATA_PREFIX) 71 | ) 72 | async def send_reply(callback_query: types.CallbackQuery): 73 | message_hash = callback_query.data.split(':', 1)[1] 74 | message_text = self._msg_cache.get_message(message_hash) 75 | 76 | if message_text is None: 77 | await callback_query.message.answer(self.CANT_REPEAT_MESSAGE) 78 | else: 79 | await self._get_response_and_send_reply( 80 | message=callback_query.message, 81 | seed_string=message_text, 82 | callback_data=message_hash 83 | ) 84 | 85 | async def _get_response_and_send_reply( 86 | self, 87 | message, 88 | seed_string, 89 | callback_data 90 | ): 91 | response = await self._get_text_generator_service_response( 92 | seed_string=seed_string 93 | ) 94 | 95 | reply_text = _prepare_reply_text( 96 | text_generator_service_response=response, 97 | prefix_string=seed_string 98 | ) 99 | 100 | keyboard = _get_inline_keyboard(callback_data=callback_data) 101 | 102 | self._log_handler.log( 103 | user_id=_get_user_id_from_message(message=message), 104 | log_msg=f'\n{reply_text}\n' 105 | ) 106 | 107 | await message.answer( 108 | text=reply_text, 109 | reply_markup=keyboard, 110 | parse_mode=ParseMode.HTML 111 | ) 112 | 113 | async def _get_text_generator_service_response( 114 | self, 115 | seed_string: str 116 | ) -> Tuple[Optional[str], int]: 117 | url = os.path.join(self._url, 'generated_texts') 118 | 119 | payload = {'body': seed_string} 120 | payload.update(self.DEFAULT_PARAMS) 121 | 122 | headers = {'Content-Type': 'application/json'} 123 | async with aiohttp.ClientSession(auth=self._auth) as session: 124 | try: 125 | async with session.post( 126 | url=url, 127 | data=json.dumps(payload), 128 | headers=headers 129 | ) as response: 130 | status = response.status 131 | reply = await response.text() 132 | return reply, status 133 | except ServerDisconnectedError: 134 | return None, HTTPStatus.INTERNAL_SERVER_ERROR 135 | 136 | def register_all_handlers(self): 137 | self.register_start_message_handler() 138 | self.register_send_reply_message_handler() 139 | self.register_send_reply_callback_query_handler() 140 | 141 | 142 | def _prepare_reply_text( 143 | text_generator_service_response: Tuple[Optional[str], int], 144 | prefix_string: str 145 | ) -> str: 146 | response_text, status = text_generator_service_response 147 | if status == 200: 148 | response_dict = json.loads(response_text) 149 | generated_text = response_dict['texts'][0] 150 | generated_text = f'{prefix_string} {generated_text}' 151 | else: 152 | generated_text = HandlersRegister.TEXT_GENERATOR_SERVICE_ERROR_MESSAGE 153 | 154 | return generated_text 155 | 156 | 157 | def _get_user_id_from_message(message): 158 | username = str(message.chat['username']) 159 | chat_id = str(message.chat['id']) 160 | user_name = "".join(x for x in username if x.isalnum()) 161 | user_id = user_name + '_' + chat_id 162 | return user_id 163 | 164 | 165 | def _get_inline_keyboard(callback_data: str) -> InlineKeyboardMarkup: 166 | buttons = [] 167 | 168 | data = f'{HandlersRegister.REPEAT_CALLBACK_DATA_PREFIX}:{callback_data}' 169 | repeat_button = InlineKeyboardButton(text='Repeat', callback_data=data) 170 | buttons.append(repeat_button) 171 | keyboard = InlineKeyboardMarkup(inline_keyboard=[buttons]) 172 | 173 | return keyboard 174 | 175 | 176 | class MessagesCache: 177 | def __init__(self): 178 | self._cache = dict() 179 | 180 | def add_message(self, message: str) -> str: 181 | message_hash = get_string_md5(message)[:16] 182 | self._cache[message_hash] = message 183 | return message_hash 184 | 185 | def get_message(self, message_hash: str) -> str: 186 | message = self._cache.get(message_hash) 187 | return message 188 | 189 | 190 | class LoggingHandler: 191 | """Performs logging for users in their individual files.""" 192 | 193 | FORMATTER = '%(asctime)s %(message)s' 194 | 195 | def __init__(self, logs_dir: pathlib.Path): 196 | self._logs_dir = logs_dir 197 | self._cache = dict() 198 | 199 | self._logs_dir.mkdir(parents=True, exist_ok=True) 200 | 201 | def _get_logger(self, user_id: str): 202 | if user_id not in self._cache: 203 | log_file = self._logs_dir / f'{user_id}.log' 204 | formatter = logging.Formatter(self.FORMATTER) 205 | handler = logging.FileHandler(str(log_file)) 206 | handler.setFormatter(formatter) 207 | 208 | logger = logging.getLogger(f'{user_id}') 209 | logger.setLevel(logging.INFO) 210 | logger.addHandler(handler) 211 | self._cache[user_id] = logger 212 | else: 213 | logger = self._cache[user_id] 214 | 215 | return logger 216 | 217 | def log(self, user_id: str, log_msg: str): 218 | logger = self._get_logger(user_id=user_id) 219 | logger.info(log_msg) 220 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/text_input.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Optional 3 | 4 | from full_stack_transformer.core.text_input import TextInput 5 | 6 | 7 | @dataclass 8 | class DocumentInput(TextInput): 9 | body: str 10 | meta: Optional[str] = None 11 | -------------------------------------------------------------------------------- /full_stack_transformer/tasks/document_lm/tokenizer.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import numpy as np 4 | from tokenizers.implementations import BaseTokenizer 5 | 6 | from full_stack_transformer.core.constants import LOSS_IGNORE 7 | from full_stack_transformer.core.encoding import Encoding 8 | from full_stack_transformer.core.tokenizer import Tokenizer 9 | from full_stack_transformer.tasks.document_lm.text_input import DocumentInput 10 | from full_stack_transformer.utilities.factory import get_object 11 | 12 | _START_OF_DOCUMENT = '[START_OF_DOCUMENT]' 13 | _END_OF_META = '[END_OF_META]' 14 | 15 | _SPECIAL_TOKENS = [_START_OF_DOCUMENT, _END_OF_META] 16 | 17 | 18 | class DocumentTokenizer(Tokenizer): 19 | def __init__( 20 | self, 21 | tokenizer: BaseTokenizer, 22 | max_meta_len: int, 23 | max_body_len: int, 24 | ignore_meta_prob: float = 0, 25 | **kwargs 26 | ): 27 | super().__init__(tokenizer, **kwargs) 28 | 29 | self._max_meta_len = max_meta_len 30 | self._max_body_len = max_body_len 31 | self._ignore_meta_prob = ignore_meta_prob 32 | self.add_special_tokens({'additional_special_tokens': _SPECIAL_TOKENS}) 33 | 34 | def _encode_for_train( 35 | self, 36 | text_input: DocumentInput 37 | ) -> List[Encoding]: 38 | return self._encode(document=text_input, with_eos=True) 39 | 40 | def _encode_for_inference( 41 | self, 42 | text_input: DocumentInput 43 | ) -> List[Encoding]: 44 | return self._encode(document=text_input, with_eos=False) 45 | 46 | def _encode( 47 | self, 48 | document: DocumentInput, 49 | with_eos: bool = True 50 | ) -> List[Encoding]: 51 | 52 | body = document.body 53 | if np.random.rand() > self._ignore_meta_prob: 54 | meta = document.meta 55 | else: 56 | meta = None 57 | 58 | body_ids, body_lm_labels = self._get_body_ids_and_labels( 59 | body=body, 60 | with_eos=with_eos 61 | ) 62 | meta_ids, meta_lm_labels = self._get_meta_ids_and_labels( 63 | meta=meta 64 | ) 65 | 66 | encoding = self._get_encoding_from_ids_and_labels( 67 | body_ids=body_ids, 68 | body_lm_labels=body_lm_labels, 69 | meta_ids=meta_ids, 70 | meta_lm_labels=meta_lm_labels 71 | ) 72 | 73 | return [encoding] 74 | 75 | def _get_body_ids_and_labels(self, body: str, with_eos: bool): 76 | body = self.prepare_for_tokenization(text=body) 77 | body = f'{_START_OF_DOCUMENT}{body}' 78 | 79 | if with_eos: 80 | body += self.eos_token 81 | 82 | body_ids = self.encode(text=body) 83 | body_lm_labels = list(body_ids) 84 | body_lm_labels[0] = LOSS_IGNORE 85 | 86 | return body_ids, body_lm_labels 87 | 88 | def _get_meta_ids_and_labels(self, meta: str): 89 | if meta is not None: 90 | meta = self.prepare_for_tokenization(text=meta) 91 | meta = f'{meta}{_END_OF_META}' 92 | meta_ids = self.encode(text=meta) 93 | meta_ids = meta_ids[-self._max_meta_len:] 94 | meta_lm_labels = [LOSS_IGNORE] * len(meta_ids) 95 | else: 96 | meta_ids = [] 97 | meta_lm_labels = [] 98 | 99 | return meta_ids, meta_lm_labels 100 | 101 | def _get_encoding_from_ids_and_labels( 102 | self, 103 | body_ids: List[int], 104 | body_lm_labels: List[int], 105 | meta_ids: List[int], 106 | meta_lm_labels: List[int] 107 | ) -> Encoding: 108 | token_ids = meta_ids + body_ids[:self._max_body_len] 109 | lm_labels = meta_lm_labels + body_lm_labels[:self._max_body_len] 110 | encoding = Encoding( 111 | token_ids=token_ids, 112 | lm_labels=lm_labels 113 | ) 114 | 115 | return encoding 116 | 117 | 118 | def get_tokenizer( 119 | name: str, 120 | max_meta_len: int, 121 | max_body_len: int, 122 | ignore_meta_prob: float 123 | ) -> DocumentTokenizer: 124 | path = f'full_stack_transformer.tasks.document_lm.{name}' 125 | tokenizer = get_object( 126 | class_path=path, 127 | max_meta_len=max_meta_len, 128 | max_body_len=max_body_len, 129 | ignore_meta_prob=ignore_meta_prob 130 | ) 131 | 132 | if not isinstance(tokenizer, DocumentTokenizer): 133 | raise ValueError(f'{name} is not a `DocumentTokenizer` subclass.') 134 | 135 | return tokenizer -------------------------------------------------------------------------------- /full_stack_transformer/utilities/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexeykarnachev/full_stack_transformer/2ac4a583c1ea610a4786a65258f33a941251bdea/full_stack_transformer/utilities/__init__.py -------------------------------------------------------------------------------- /full_stack_transformer/utilities/arguments.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | from argparse import ArgumentParser, Namespace 3 | from dataclasses import dataclass 4 | from typing import List, Any 5 | 6 | from docstring_parser.google import parse 7 | 8 | 9 | @dataclass 10 | class DocstringArgument: 11 | name: str 12 | annotation: Any 13 | description: str 14 | required: bool 15 | default: Any 16 | 17 | 18 | def parse_docstring_arguments(obj) -> List[DocstringArgument]: 19 | parsed_doc = parse(obj.__doc__) 20 | signature = inspect.signature(obj) 21 | parsed_arguments = [] 22 | for doc_param in parsed_doc.params: 23 | name = doc_param.arg_name 24 | param_sign = signature.parameters[name] 25 | annotation = param_sign.annotation 26 | description = doc_param.description 27 | required = not doc_param.is_optional 28 | default = param_sign.default 29 | 30 | argument = [name, annotation, description, required, default] 31 | argument = DocstringArgument(*argument) 32 | 33 | parsed_arguments.append(argument) 34 | 35 | return parsed_arguments 36 | 37 | 38 | class ArgparserExtender: 39 | @classmethod 40 | def add_argparse_args(cls, parser: ArgumentParser) -> ArgumentParser: 41 | arguments = parse_docstring_arguments(cls.__init__) 42 | 43 | for arg in arguments: 44 | parser.add_argument( 45 | f'--{arg.name}', 46 | required=arg.required, 47 | type=arg.annotation, 48 | help=arg.description, 49 | default=arg.default 50 | ) 51 | 52 | return parser 53 | 54 | 55 | def get_func_arg_values_as_namespace(locals_, func, **kwargs): 56 | signature = inspect.signature(func) 57 | 58 | params = kwargs 59 | 60 | for name, param in signature.parameters.items(): 61 | value = locals_[name] 62 | if name == 'self': 63 | continue 64 | elif name == 'kwargs': 65 | params.update(value) 66 | else: 67 | params[name] = value 68 | 69 | return Namespace(**params) 70 | -------------------------------------------------------------------------------- /full_stack_transformer/utilities/experiment.py: -------------------------------------------------------------------------------- 1 | import json 2 | import pathlib 3 | from typing import Mapping 4 | 5 | from full_stack_transformer.utilities.arguments import ArgparserExtender 6 | from full_stack_transformer.utilities.log_config import prepare_logging 7 | 8 | 9 | class Workspace(ArgparserExtender): 10 | """Experiment workspace, which handles files and folders creation.""" 11 | 12 | _LOGS_DIR_NAME = 'logs' 13 | _MODELS_DIR_NAME = 'models' 14 | _MAX_VERSION = 63 15 | 16 | @property 17 | def name(self): 18 | return self._name 19 | 20 | @property 21 | def version(self): 22 | return self._version 23 | 24 | @property 25 | def logs_dir(self): 26 | return self._logs_dir 27 | 28 | @property 29 | def models_dir(self): 30 | return self._models_dir 31 | 32 | @property 33 | def experiment_dir(self): 34 | return self._main_dir 35 | 36 | def __init__( 37 | self, 38 | experiments_root: str, 39 | experiment_name: str = 'default' 40 | ): 41 | """ 42 | Args: 43 | experiments_root (str): 44 | Root dir where all experiments folders are stored. 45 | 46 | experiment_name (str, optional): 47 | Base name of the experiment. Experiment folder name also will 48 | contain a version number postfix. Defaults to default. 49 | """ 50 | 51 | self._root = pathlib.Path(experiments_root) 52 | self._name = experiment_name 53 | 54 | self._version = self._get_version() 55 | self._main_dir = self._get_main_dir(self._version) 56 | self._logs_dir = self._main_dir / self._LOGS_DIR_NAME 57 | self._models_dir = self._main_dir / self._MODELS_DIR_NAME 58 | 59 | self._main_dir.mkdir(exist_ok=True, parents=True) 60 | self._logs_dir.mkdir(exist_ok=True, parents=True) 61 | self._models_dir.mkdir(exist_ok=True, parents=True) 62 | 63 | prepare_logging(logs_dir=self._logs_dir) 64 | 65 | def _get_main_dir(self, version: int): 66 | return self._root / f'{self._name}_v{version}' 67 | 68 | def _get_version(self) -> int: 69 | version = 0 70 | 71 | while self._get_main_dir(version).is_dir(): 72 | version += 1 73 | 74 | if version > self._MAX_VERSION: 75 | raise ValueError( 76 | f'Too many experiments with name: {self._name}. Please, ' 77 | f'remove old ones from {self._root}' 78 | ) 79 | 80 | return version 81 | 82 | def save_json(self, name: str, content: Mapping): 83 | with (self._main_dir / name).open('w') as file: 84 | json.dump(content, file, ensure_ascii=False, indent=2) 85 | 86 | def save_description(self, content: Mapping): 87 | self.save_json(name='description.json', content=content) 88 | -------------------------------------------------------------------------------- /full_stack_transformer/utilities/factory.py: -------------------------------------------------------------------------------- 1 | """Utilities for dynamic objects and classes imports.""" 2 | 3 | import pydoc 4 | 5 | 6 | class FactoryError(Exception): 7 | pass 8 | 9 | 10 | def get_object(class_path: str, *args, **kwargs): 11 | """Imports class and instantiate an object. 12 | 13 | Args: 14 | class_path (str): 15 | Class path of the required object to instantiate. 16 | 17 | *args: 18 | Class constructor arguments. 19 | 20 | **kwargs: 21 | Class constructor key value arguments. 22 | 23 | Returns: 24 | Object of a given class. 25 | """ 26 | cls = get_class(class_path) 27 | 28 | obj = cls(*args, **kwargs) 29 | return obj 30 | 31 | 32 | def get_class(class_path): 33 | """Imports and returns a class. 34 | 35 | Args: 36 | class_path (str): 37 | Class path of the required class object. 38 | 39 | Returns: 40 | Desired class object. 41 | """ 42 | cls = pydoc.locate(class_path) 43 | if cls is None: 44 | raise FactoryError("Can't locate class: {}".format(class_path)) 45 | return cls 46 | -------------------------------------------------------------------------------- /full_stack_transformer/utilities/files.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | 3 | 4 | def count_lines_in_file(file_path: pathlib.Path) -> int: 5 | number_of_lines = 0 6 | with file_path.open() as file: 7 | for line in file: 8 | if line != '\n': 9 | number_of_lines += 1 10 | 11 | return number_of_lines 12 | -------------------------------------------------------------------------------- /full_stack_transformer/utilities/log_config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import logging.config 3 | import pathlib 4 | import sys 5 | from typing import Dict 6 | 7 | _LOGGER = logging.getLogger(__name__) 8 | _FORMATTER = '[%(asctime)s %(module)s %(funcName)s %(levelname)s] %(message)s' 9 | 10 | 11 | def prepare_logging(logs_dir: pathlib.Path): 12 | """Configures logging.""" 13 | log_config = _get_log_config(logs_dir) 14 | logging.config.dictConfig(log_config) 15 | 16 | 17 | def handle_unhandled_exception(exc_type, exc_value, exc_traceback): 18 | """Handler for unhandled exceptions that will write to the logs""" 19 | if issubclass(exc_type, KeyboardInterrupt): 20 | # call the default excepthook saved at __excepthook__ 21 | sys.__excepthook__(exc_type, exc_value, exc_traceback) 22 | return 23 | _LOGGER.critical( 24 | "Unhandled exception", 25 | exc_info=(exc_type, exc_value, exc_traceback) 26 | ) 27 | 28 | 29 | def _get_rotating_file_handler( 30 | log_file: str, 31 | level: str, 32 | max_bytes: int = 10485760, 33 | backup_count: int = 5 34 | ) -> Dict: 35 | handler_dict = { 36 | 'class': 'logging.handlers.RotatingFileHandler', 37 | 'level': level, 38 | 'formatter': 'default', 39 | 'filename': log_file, 40 | 'mode': 'a', 41 | 'maxBytes': max_bytes, 42 | 'backupCount': backup_count, 43 | } 44 | 45 | return handler_dict 46 | 47 | 48 | def _get_console_output_handler(level) -> Dict: 49 | handler_dict = { 50 | 'class': 'logging.StreamHandler', 51 | 'level': level, 52 | 'formatter': 'default', 53 | } 54 | 55 | return handler_dict 56 | 57 | 58 | def _get_log_config(log_dir: pathlib.Path) -> dict: 59 | log_dir = pathlib.Path(log_dir) 60 | 61 | log_dir.mkdir(exist_ok=True, parents=True) 62 | info_file = str(log_dir / 'info.log') 63 | errors_file = str(log_dir / 'errors.log') 64 | debug_file = str(log_dir / 'debug.log') 65 | 66 | handlers = { 67 | 'info_file': _get_rotating_file_handler(info_file, 'INFO'), 68 | 'debug_file': _get_rotating_file_handler(debug_file, 'DEBUG'), 69 | 'errors_file': _get_rotating_file_handler(errors_file, 'ERROR'), 70 | 'console': _get_console_output_handler('INFO') 71 | } 72 | 73 | log_config = { 74 | 'disable_existing_loggers': False, 75 | 'version': 1, 76 | 'formatters': { 77 | 'default': { 78 | 'format': _FORMATTER 79 | } 80 | }, 81 | 'handlers': handlers, 82 | 83 | 'loggers': { 84 | '': { 85 | 'handlers': list(handlers.keys()), 86 | 'level': 'DEBUG' 87 | } 88 | } 89 | } 90 | 91 | return log_config 92 | -------------------------------------------------------------------------------- /full_stack_transformer/utilities/queue_iterable_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | from multiprocessing import Queue 3 | from typing import Optional 4 | 5 | from torch.utils.data.dataset import IterableDataset 6 | 7 | 8 | class QueueIterableDataset(IterableDataset): 9 | def __init__( 10 | self, 11 | inp_queue: Queue, 12 | timeout: Optional[float] = None 13 | ): 14 | self._inp_queue = inp_queue 15 | self._timeout = timeout 16 | 17 | def __iter__(self): 18 | while True: 19 | sample = self._inp_queue.get(timeout=self._timeout) 20 | yield sample 21 | -------------------------------------------------------------------------------- /full_stack_transformer/utilities/seeding.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | 4 | import numpy as np 5 | import torch 6 | 7 | 8 | def seed_everything(seed: int): 9 | """Seeds and fixes every possible random state.""" 10 | random.seed(seed) 11 | os.environ['PYTHONHASHSEED'] = str(seed) 12 | np.random.seed(seed) 13 | torch.manual_seed(seed) 14 | torch.cuda.manual_seed(seed) 15 | torch.backends.cudnn.deterministic = True 16 | -------------------------------------------------------------------------------- /full_stack_transformer/utilities/sequences.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence, TypeVar, Optional, List 2 | 3 | TNum = TypeVar('TNum', int, float) 4 | 5 | 6 | def pad_sequences_from_right( 7 | sequences: Sequence[Sequence[TNum]], 8 | max_len: Optional[int], 9 | pad_value: TNum 10 | ) -> List[List[TNum]]: 11 | max_seq_len = max([len(x) for x in sequences]) 12 | max_len = min(max_len, max_seq_len) if max_len else max_seq_len 13 | 14 | new_seqs = [] 15 | 16 | for seq in sequences: 17 | new_seq = list(seq[:max_len]) 18 | new_seq.extend([pad_value] * (max_len - len(new_seq))) 19 | new_seqs.append(new_seq) 20 | 21 | return new_seqs 22 | -------------------------------------------------------------------------------- /full_stack_transformer/utilities/strings.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | 3 | 4 | def get_string_md5(string: str) -> str: 5 | """Gets input string md5 hexdigest hash.""" 6 | return hashlib.md5(string.encode()).hexdigest() -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | torch==1.4.0 2 | numpy==1.18.3 3 | transformers==2.9.0 4 | tokenizers==0.7.0 5 | pytorch-lightning==0.7.5 6 | more-itertools==8.2.0 7 | cached-property==1.5.1 8 | docstring-parser==0.7.1 9 | tqdm==4.45.0 10 | pytest==5.4.1 11 | pydantic==1.5.1 12 | fastapi==0.54.1 13 | gunicorn==20.0.4 14 | uvicorn==0.11.5 15 | aiohttp==3.6.2 16 | aiogram==2.8 17 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | from setuptools import find_packages, setup 5 | 6 | THIS_DIR = Path(__file__).parent 7 | 8 | 9 | def _get_version(filename): 10 | from re import findall 11 | with open(filename) as f: 12 | metadata = dict(findall("__([a-z]+)__ = '([^']+)'", f.read())) 13 | return metadata['version'] 14 | 15 | 16 | def _load_requirements(path_dir=THIS_DIR, comment_char='#'): 17 | with open(os.path.join(path_dir, 'requirements.txt'), 'r') as file: 18 | lines = [ln.strip() for ln in file.readlines()] 19 | reqs = [] 20 | for ln in lines: 21 | # filer all comments 22 | if comment_char in ln: 23 | ln = ln[:ln.index(comment_char)] 24 | if ln: # if requirement is not empty 25 | reqs.append(ln) 26 | return reqs 27 | 28 | 29 | setup( 30 | name='full_stack_transformer', 31 | version=_get_version('full_stack_transformer/__init__.py'), 32 | description='Language Models trainer.', 33 | packages=find_packages(exclude=['tests', 'tests.*']), 34 | install_requires=_load_requirements(THIS_DIR), 35 | package_dir={'full_stack_transformer': 'full_stack_transformer'}, 36 | package_data={'full_stack_transformer': ['tasks/static/*']} 37 | ) 38 | --------------------------------------------------------------------------------