├── .dockerignore ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── feature_request.md │ └── user_story.md ├── dependabot.yml └── workflows │ ├── build-image.yml │ ├── build-library.yml │ ├── lint-code.yml │ └── publish-library.yml ├── .gitignore ├── .isort.cfg ├── .pre-commit-config.yaml ├── .prettierignore ├── .pylintrc ├── .whitesource ├── CODEOWNERS ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── SECURITY.md ├── benchmarks ├── README.md └── logs │ └── llama2-7b │ ├── 20230905_183655.output │ ├── 20230905_184809.output │ ├── 20230905_191650.output │ ├── 20230905_194133.output │ └── 20230906_135211.output ├── caikit_nlp ├── __init__.py ├── config │ ├── __init__.py │ └── config.yml ├── data_model │ ├── __init__.py │ └── generation.py ├── model_management │ ├── __init__.py │ └── tgis_auto_finder.py ├── modules │ ├── __init__.py │ ├── text_classification │ │ ├── __init__.py │ │ └── sequence_classification.py │ ├── text_embedding │ │ ├── __init__.py │ │ ├── crossencoder.py │ │ ├── embedding.py │ │ └── utils.py │ ├── text_generation │ │ ├── __init__.py │ │ ├── peft_config.py │ │ ├── peft_prompt_tuning.py │ │ ├── peft_tgis_remote.py │ │ ├── text_generation_local.py │ │ └── text_generation_tgis.py │ ├── token_classification │ │ ├── __init__.py │ │ └── filtered_span_classification.py │ └── tokenization │ │ ├── __init__.py │ │ └── regex_sentence_splitter.py ├── resources │ ├── __init__.py │ └── pretrained_model │ │ ├── __init__.py │ │ ├── base.py │ │ ├── hf_auto_causal_lm.py │ │ ├── hf_auto_seq2seq_lm.py │ │ └── hf_auto_seq_classifier.py ├── toolkit │ ├── __init__.py │ ├── data_stream_wrapper.py │ ├── data_type_utils.py │ ├── task_specific_utils.py │ ├── text_generation │ │ ├── __init__.py │ │ ├── model_run_utils.py │ │ └── tgis_utils.py │ ├── torch_run.py │ ├── trainer_utils.py │ └── verbalizer_utils.py └── version.py ├── code-of-conduct.md ├── examples ├── Caikit_Getting_Started.ipynb ├── compare_local_vs_tgis_models.py ├── evaluate_model.py ├── kill-text-generation-launcher.sh ├── load_and_run_distributed_peft.py ├── run_fine_tuning.py ├── run_peft_tuning.py ├── text-generation-launcher └── utils.py ├── prompt_tuning_parameter_selection.md ├── pyproject.toml ├── runtime_config.yaml ├── runtime_template └── run_with_gateway.sh ├── scripts ├── dump_apis.sh ├── fmt.sh └── run_local.sh ├── setup_requirements.txt ├── tests ├── __init__.py ├── conftest.py ├── data_model │ └── test_generation.py ├── fixtures │ ├── __init__.py │ ├── data_model │ │ └── sample_objects.py │ └── tiny_models │ │ ├── BertForSequenceClassification │ │ ├── config.json │ │ ├── pytorch_model.bin │ │ ├── special_tokens_map.json │ │ ├── tf_model.h5 │ │ ├── tokenizer.json │ │ ├── tokenizer_config.json │ │ └── vocab.txt │ │ ├── BloomForCausalLM │ │ ├── config.json │ │ ├── generation_config.json │ │ ├── pytorch_model.bin │ │ ├── special_tokens_map.json │ │ ├── tokenizer.json │ │ └── tokenizer_config.json │ │ ├── README.md │ │ └── T5ForConditionalGeneration │ │ ├── config.json │ │ ├── generation_config.json │ │ ├── pytorch_model.bin │ │ ├── special_tokens_map.json │ │ ├── tokenizer.json │ │ └── tokenizer_config.json ├── model_management │ ├── __init__.py │ └── test_tgis_auto_finder.py ├── modules │ ├── __init__.py │ ├── text_classification │ │ └── test_sequence_classification.py │ ├── text_embedding │ │ ├── test_crossencoder.py │ │ └── test_embedding.py │ ├── text_generation │ │ ├── __init__.py │ │ ├── test_peft_config.py │ │ ├── test_peft_prompt_tuning.py │ │ ├── test_peft_tgis_remote.py │ │ ├── test_text_generation_local.py │ │ └── test_text_generation_tgis.py │ ├── token_classification │ │ └── test_filtered_span_classification.py │ └── tokenization │ │ └── test_regex_sentence_splitter.py ├── resources │ ├── __init__.py │ └── test_pretrained_model.py └── toolkit │ ├── __init__.py │ ├── test_data_stream_wrapper.py │ ├── test_data_type_utils.py │ ├── test_task_specific_utils.py │ ├── test_verbalizers.py │ └── text_generation │ ├── test_model_run_utils.py │ └── test_tgis_utils.py └── tox.ini /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !README.md 3 | !LICENSE 4 | !caikit_nlp 5 | !pyproject.toml 6 | !tox.ini 7 | !.git 8 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/.github/ISSUE_TEMPLATE/bug_report.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/.github/ISSUE_TEMPLATE/feature_request.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/user_story.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/.github/ISSUE_TEMPLATE/user_story.md -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/.github/dependabot.yml -------------------------------------------------------------------------------- /.github/workflows/build-image.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/.github/workflows/build-image.yml -------------------------------------------------------------------------------- /.github/workflows/build-library.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/.github/workflows/build-library.yml -------------------------------------------------------------------------------- /.github/workflows/lint-code.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/.github/workflows/lint-code.yml -------------------------------------------------------------------------------- /.github/workflows/publish-library.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/.github/workflows/publish-library.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/.gitignore -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/.isort.cfg -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/.prettierignore -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/.pylintrc -------------------------------------------------------------------------------- /.whitesource: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/.whitesource -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/CODEOWNERS -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/Dockerfile -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/README.md -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/SECURITY.md -------------------------------------------------------------------------------- /benchmarks/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/benchmarks/README.md -------------------------------------------------------------------------------- /benchmarks/logs/llama2-7b/20230905_183655.output: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/benchmarks/logs/llama2-7b/20230905_183655.output -------------------------------------------------------------------------------- /benchmarks/logs/llama2-7b/20230905_184809.output: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/benchmarks/logs/llama2-7b/20230905_184809.output -------------------------------------------------------------------------------- /benchmarks/logs/llama2-7b/20230905_191650.output: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/benchmarks/logs/llama2-7b/20230905_191650.output -------------------------------------------------------------------------------- /benchmarks/logs/llama2-7b/20230905_194133.output: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/benchmarks/logs/llama2-7b/20230905_194133.output -------------------------------------------------------------------------------- /benchmarks/logs/llama2-7b/20230906_135211.output: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/benchmarks/logs/llama2-7b/20230906_135211.output -------------------------------------------------------------------------------- /caikit_nlp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/__init__.py -------------------------------------------------------------------------------- /caikit_nlp/config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/config/__init__.py -------------------------------------------------------------------------------- /caikit_nlp/config/config.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/config/config.yml -------------------------------------------------------------------------------- /caikit_nlp/data_model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/data_model/__init__.py -------------------------------------------------------------------------------- /caikit_nlp/data_model/generation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/data_model/generation.py -------------------------------------------------------------------------------- /caikit_nlp/model_management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/model_management/__init__.py -------------------------------------------------------------------------------- /caikit_nlp/model_management/tgis_auto_finder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/model_management/tgis_auto_finder.py -------------------------------------------------------------------------------- /caikit_nlp/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/modules/__init__.py -------------------------------------------------------------------------------- /caikit_nlp/modules/text_classification/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/modules/text_classification/__init__.py -------------------------------------------------------------------------------- /caikit_nlp/modules/text_classification/sequence_classification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/modules/text_classification/sequence_classification.py -------------------------------------------------------------------------------- /caikit_nlp/modules/text_embedding/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/modules/text_embedding/__init__.py -------------------------------------------------------------------------------- /caikit_nlp/modules/text_embedding/crossencoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/modules/text_embedding/crossencoder.py -------------------------------------------------------------------------------- /caikit_nlp/modules/text_embedding/embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/modules/text_embedding/embedding.py -------------------------------------------------------------------------------- /caikit_nlp/modules/text_embedding/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/modules/text_embedding/utils.py -------------------------------------------------------------------------------- /caikit_nlp/modules/text_generation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/modules/text_generation/__init__.py -------------------------------------------------------------------------------- /caikit_nlp/modules/text_generation/peft_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/modules/text_generation/peft_config.py -------------------------------------------------------------------------------- /caikit_nlp/modules/text_generation/peft_prompt_tuning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/modules/text_generation/peft_prompt_tuning.py -------------------------------------------------------------------------------- /caikit_nlp/modules/text_generation/peft_tgis_remote.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/modules/text_generation/peft_tgis_remote.py -------------------------------------------------------------------------------- /caikit_nlp/modules/text_generation/text_generation_local.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/modules/text_generation/text_generation_local.py -------------------------------------------------------------------------------- /caikit_nlp/modules/text_generation/text_generation_tgis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/modules/text_generation/text_generation_tgis.py -------------------------------------------------------------------------------- /caikit_nlp/modules/token_classification/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/modules/token_classification/__init__.py -------------------------------------------------------------------------------- /caikit_nlp/modules/token_classification/filtered_span_classification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/modules/token_classification/filtered_span_classification.py -------------------------------------------------------------------------------- /caikit_nlp/modules/tokenization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/modules/tokenization/__init__.py -------------------------------------------------------------------------------- /caikit_nlp/modules/tokenization/regex_sentence_splitter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/modules/tokenization/regex_sentence_splitter.py -------------------------------------------------------------------------------- /caikit_nlp/resources/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/resources/__init__.py -------------------------------------------------------------------------------- /caikit_nlp/resources/pretrained_model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/resources/pretrained_model/__init__.py -------------------------------------------------------------------------------- /caikit_nlp/resources/pretrained_model/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/resources/pretrained_model/base.py -------------------------------------------------------------------------------- /caikit_nlp/resources/pretrained_model/hf_auto_causal_lm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/resources/pretrained_model/hf_auto_causal_lm.py -------------------------------------------------------------------------------- /caikit_nlp/resources/pretrained_model/hf_auto_seq2seq_lm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/resources/pretrained_model/hf_auto_seq2seq_lm.py -------------------------------------------------------------------------------- /caikit_nlp/resources/pretrained_model/hf_auto_seq_classifier.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/resources/pretrained_model/hf_auto_seq_classifier.py -------------------------------------------------------------------------------- /caikit_nlp/toolkit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /caikit_nlp/toolkit/data_stream_wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/toolkit/data_stream_wrapper.py -------------------------------------------------------------------------------- /caikit_nlp/toolkit/data_type_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/toolkit/data_type_utils.py -------------------------------------------------------------------------------- /caikit_nlp/toolkit/task_specific_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/toolkit/task_specific_utils.py -------------------------------------------------------------------------------- /caikit_nlp/toolkit/text_generation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /caikit_nlp/toolkit/text_generation/model_run_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/toolkit/text_generation/model_run_utils.py -------------------------------------------------------------------------------- /caikit_nlp/toolkit/text_generation/tgis_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/toolkit/text_generation/tgis_utils.py -------------------------------------------------------------------------------- /caikit_nlp/toolkit/torch_run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/toolkit/torch_run.py -------------------------------------------------------------------------------- /caikit_nlp/toolkit/trainer_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/toolkit/trainer_utils.py -------------------------------------------------------------------------------- /caikit_nlp/toolkit/verbalizer_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/toolkit/verbalizer_utils.py -------------------------------------------------------------------------------- /caikit_nlp/version.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/caikit_nlp/version.py -------------------------------------------------------------------------------- /code-of-conduct.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/code-of-conduct.md -------------------------------------------------------------------------------- /examples/Caikit_Getting_Started.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/examples/Caikit_Getting_Started.ipynb -------------------------------------------------------------------------------- /examples/compare_local_vs_tgis_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/examples/compare_local_vs_tgis_models.py -------------------------------------------------------------------------------- /examples/evaluate_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/examples/evaluate_model.py -------------------------------------------------------------------------------- /examples/kill-text-generation-launcher.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/examples/kill-text-generation-launcher.sh -------------------------------------------------------------------------------- /examples/load_and_run_distributed_peft.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/examples/load_and_run_distributed_peft.py -------------------------------------------------------------------------------- /examples/run_fine_tuning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/examples/run_fine_tuning.py -------------------------------------------------------------------------------- /examples/run_peft_tuning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/examples/run_peft_tuning.py -------------------------------------------------------------------------------- /examples/text-generation-launcher: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/examples/text-generation-launcher -------------------------------------------------------------------------------- /examples/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/examples/utils.py -------------------------------------------------------------------------------- /prompt_tuning_parameter_selection.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/prompt_tuning_parameter_selection.md -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/pyproject.toml -------------------------------------------------------------------------------- /runtime_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/runtime_config.yaml -------------------------------------------------------------------------------- /runtime_template/run_with_gateway.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/runtime_template/run_with_gateway.sh -------------------------------------------------------------------------------- /scripts/dump_apis.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/scripts/dump_apis.sh -------------------------------------------------------------------------------- /scripts/fmt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/scripts/fmt.sh -------------------------------------------------------------------------------- /scripts/run_local.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/scripts/run_local.sh -------------------------------------------------------------------------------- /setup_requirements.txt: -------------------------------------------------------------------------------- 1 | tox>=4.4.2,<5 2 | build>=0.10.0,<2.0 -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/conftest.py -------------------------------------------------------------------------------- /tests/data_model/test_generation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/data_model/test_generation.py -------------------------------------------------------------------------------- /tests/fixtures/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/__init__.py -------------------------------------------------------------------------------- /tests/fixtures/data_model/sample_objects.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/data_model/sample_objects.py -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/BertForSequenceClassification/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/BertForSequenceClassification/config.json -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/BertForSequenceClassification/pytorch_model.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/BertForSequenceClassification/pytorch_model.bin -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/BertForSequenceClassification/special_tokens_map.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/BertForSequenceClassification/special_tokens_map.json -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/BertForSequenceClassification/tf_model.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/BertForSequenceClassification/tf_model.h5 -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/BertForSequenceClassification/tokenizer.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/BertForSequenceClassification/tokenizer.json -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/BertForSequenceClassification/tokenizer_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/BertForSequenceClassification/tokenizer_config.json -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/BertForSequenceClassification/vocab.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/BertForSequenceClassification/vocab.txt -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/BloomForCausalLM/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/BloomForCausalLM/config.json -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/BloomForCausalLM/generation_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/BloomForCausalLM/generation_config.json -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/BloomForCausalLM/pytorch_model.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/BloomForCausalLM/pytorch_model.bin -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/BloomForCausalLM/special_tokens_map.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/BloomForCausalLM/special_tokens_map.json -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/BloomForCausalLM/tokenizer.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/BloomForCausalLM/tokenizer.json -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/BloomForCausalLM/tokenizer_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/BloomForCausalLM/tokenizer_config.json -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/README.md -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/T5ForConditionalGeneration/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/T5ForConditionalGeneration/config.json -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/T5ForConditionalGeneration/generation_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/T5ForConditionalGeneration/generation_config.json -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/T5ForConditionalGeneration/pytorch_model.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/T5ForConditionalGeneration/pytorch_model.bin -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/T5ForConditionalGeneration/special_tokens_map.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/T5ForConditionalGeneration/special_tokens_map.json -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/T5ForConditionalGeneration/tokenizer.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/T5ForConditionalGeneration/tokenizer.json -------------------------------------------------------------------------------- /tests/fixtures/tiny_models/T5ForConditionalGeneration/tokenizer_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/fixtures/tiny_models/T5ForConditionalGeneration/tokenizer_config.json -------------------------------------------------------------------------------- /tests/model_management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/model_management/test_tgis_auto_finder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/model_management/test_tgis_auto_finder.py -------------------------------------------------------------------------------- /tests/modules/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/modules/text_classification/test_sequence_classification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/modules/text_classification/test_sequence_classification.py -------------------------------------------------------------------------------- /tests/modules/text_embedding/test_crossencoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/modules/text_embedding/test_crossencoder.py -------------------------------------------------------------------------------- /tests/modules/text_embedding/test_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/modules/text_embedding/test_embedding.py -------------------------------------------------------------------------------- /tests/modules/text_generation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/modules/text_generation/test_peft_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/modules/text_generation/test_peft_config.py -------------------------------------------------------------------------------- /tests/modules/text_generation/test_peft_prompt_tuning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/modules/text_generation/test_peft_prompt_tuning.py -------------------------------------------------------------------------------- /tests/modules/text_generation/test_peft_tgis_remote.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/modules/text_generation/test_peft_tgis_remote.py -------------------------------------------------------------------------------- /tests/modules/text_generation/test_text_generation_local.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/modules/text_generation/test_text_generation_local.py -------------------------------------------------------------------------------- /tests/modules/text_generation/test_text_generation_tgis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/modules/text_generation/test_text_generation_tgis.py -------------------------------------------------------------------------------- /tests/modules/token_classification/test_filtered_span_classification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/modules/token_classification/test_filtered_span_classification.py -------------------------------------------------------------------------------- /tests/modules/tokenization/test_regex_sentence_splitter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/modules/tokenization/test_regex_sentence_splitter.py -------------------------------------------------------------------------------- /tests/resources/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/resources/test_pretrained_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/resources/test_pretrained_model.py -------------------------------------------------------------------------------- /tests/toolkit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/toolkit/test_data_stream_wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/toolkit/test_data_stream_wrapper.py -------------------------------------------------------------------------------- /tests/toolkit/test_data_type_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/toolkit/test_data_type_utils.py -------------------------------------------------------------------------------- /tests/toolkit/test_task_specific_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/toolkit/test_task_specific_utils.py -------------------------------------------------------------------------------- /tests/toolkit/test_verbalizers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/toolkit/test_verbalizers.py -------------------------------------------------------------------------------- /tests/toolkit/text_generation/test_model_run_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/toolkit/text_generation/test_model_run_utils.py -------------------------------------------------------------------------------- /tests/toolkit/text_generation/test_tgis_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tests/toolkit/text_generation/test_tgis_utils.py -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caikit/caikit-nlp/HEAD/tox.ini --------------------------------------------------------------------------------