├── .gitignore ├── .pre-commit-config.yaml ├── LICENSE ├── README.md ├── assets ├── RedPajama-INCITE-Base-3B-v1-dialogue-summary-topic.jpeg ├── RedPajama-INCITE-Base-3B-v1-paraphrase-tone.jpeg └── falcon-7b-paraphrase-tone-dialogue-summary-topic.jpeg ├── data ├── dialogue_summary_topic.json ├── dialogue_summary_topic_test.json └── paraphrase_tone.json ├── llm_toys ├── __init__.py ├── config.py ├── eval.py ├── hf │ ├── __init__.py │ ├── peft │ │ ├── __init__.py │ │ ├── auto.py │ │ ├── import_utils.py │ │ ├── mapping.py │ │ ├── peft_model.py │ │ ├── py.typed │ │ ├── tuners │ │ │ ├── __init__.py │ │ │ ├── adalora.py │ │ │ ├── adaption_prompt.py │ │ │ ├── ia3.py │ │ │ ├── lora.py │ │ │ ├── p_tuning.py │ │ │ ├── prefix_tuning.py │ │ │ └── prompt_tuning.py │ │ └── utils │ │ │ ├── __init__.py │ │ │ ├── config.py │ │ │ ├── hub_utils.py │ │ │ ├── other.py │ │ │ └── save_and_load.py │ └── transformers │ │ ├── __init__.py │ │ ├── activations.py │ │ ├── activations_tf.py │ │ ├── audio_utils.py │ │ ├── benchmark │ │ ├── __init__.py │ │ ├── benchmark.py │ │ ├── benchmark_args.py │ │ ├── benchmark_args_tf.py │ │ ├── benchmark_args_utils.py │ │ ├── benchmark_tf.py │ │ └── benchmark_utils.py │ │ ├── commands │ │ ├── __init__.py │ │ ├── add_new_model.py │ │ ├── add_new_model_like.py │ │ ├── convert.py │ │ ├── download.py │ │ ├── env.py │ │ ├── lfs.py │ │ ├── pt_to_tf.py │ │ ├── run.py │ │ ├── serving.py │ │ ├── train.py │ │ ├── transformers_cli.py │ │ └── user.py │ │ ├── configuration_utils.py │ │ ├── convert_graph_to_onnx.py │ │ ├── convert_pytorch_checkpoint_to_tf2.py │ │ ├── convert_slow_tokenizer.py │ │ ├── convert_slow_tokenizers_checkpoints_to_fast.py │ │ ├── convert_tf_hub_seq_to_seq_bert_to_pytorch.py │ │ ├── data │ │ ├── __init__.py │ │ ├── data_collator.py │ │ ├── datasets │ │ │ ├── __init__.py │ │ │ ├── glue.py │ │ │ ├── language_modeling.py │ │ │ └── squad.py │ │ ├── metrics │ │ │ ├── __init__.py │ │ │ └── squad_metrics.py │ │ └── processors │ │ │ ├── __init__.py │ │ │ ├── glue.py │ │ │ ├── squad.py │ │ │ ├── utils.py │ │ │ └── xnli.py │ │ ├── debug_utils.py │ │ ├── deepspeed.py │ │ ├── dependency_versions_check.py │ │ ├── dependency_versions_table.py │ │ ├── dynamic_module_utils.py │ │ ├── feature_extraction_sequence_utils.py │ │ ├── feature_extraction_utils.py │ │ ├── file_utils.py │ │ ├── generation │ │ ├── __init__.py │ │ ├── beam_constraints.py │ │ ├── beam_search.py │ │ ├── configuration_utils.py │ │ ├── flax_logits_process.py │ │ ├── flax_utils.py │ │ ├── logits_process.py │ │ ├── stopping_criteria.py │ │ ├── streamers.py │ │ ├── tf_logits_process.py │ │ ├── tf_utils.py │ │ └── utils.py │ │ ├── generation_flax_utils.py │ │ ├── generation_tf_utils.py │ │ ├── generation_utils.py │ │ ├── hf_argparser.py │ │ ├── hyperparameter_search.py │ │ ├── image_processing_utils.py │ │ ├── image_transforms.py │ │ ├── image_utils.py │ │ ├── integrations.py │ │ ├── keras_callbacks.py │ │ ├── kernels │ │ ├── deformable_detr │ │ │ ├── cpu │ │ │ │ ├── ms_deform_attn_cpu.cpp │ │ │ │ └── ms_deform_attn_cpu.h │ │ │ ├── cuda │ │ │ │ ├── ms_deform_attn_cuda.cu │ │ │ │ ├── ms_deform_attn_cuda.cuh │ │ │ │ ├── ms_deform_attn_cuda.h │ │ │ │ └── ms_deform_im2col_cuda.cuh │ │ │ ├── ms_deform_attn.h │ │ │ └── vision.cpp │ │ ├── mra │ │ │ ├── cuda_kernel.cu │ │ │ ├── cuda_kernel.h │ │ │ ├── cuda_launch.cu │ │ │ ├── cuda_launch.h │ │ │ └── torch_extension.cpp │ │ ├── rwkv │ │ │ ├── wkv_cuda.cu │ │ │ ├── wkv_cuda_bf16.cu │ │ │ └── wkv_op.cpp │ │ └── yoso │ │ │ ├── common.h │ │ │ ├── common_cuda.h │ │ │ ├── common_cuda_device.h │ │ │ ├── fast_lsh_cumulation.cu │ │ │ ├── fast_lsh_cumulation.h │ │ │ ├── fast_lsh_cumulation_cuda.cu │ │ │ ├── fast_lsh_cumulation_cuda.h │ │ │ └── fast_lsh_cumulation_torch.cpp │ │ ├── modelcard.py │ │ ├── modeling_flax_outputs.py │ │ ├── modeling_flax_pytorch_utils.py │ │ ├── modeling_flax_utils.py │ │ ├── modeling_outputs.py │ │ ├── modeling_tf_outputs.py │ │ ├── modeling_tf_pytorch_utils.py │ │ ├── modeling_tf_utils.py │ │ ├── modeling_utils.py │ │ ├── onnx │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── config.py │ │ ├── convert.py │ │ ├── features.py │ │ └── utils.py │ │ ├── optimization.py │ │ ├── optimization_tf.py │ │ ├── pipelines │ │ ├── __init__.py │ │ ├── audio_classification.py │ │ ├── audio_utils.py │ │ ├── automatic_speech_recognition.py │ │ ├── base.py │ │ ├── conversational.py │ │ ├── depth_estimation.py │ │ ├── document_question_answering.py │ │ ├── feature_extraction.py │ │ ├── fill_mask.py │ │ ├── image_classification.py │ │ ├── image_segmentation.py │ │ ├── image_to_text.py │ │ ├── mask_generation.py │ │ ├── object_detection.py │ │ ├── pt_utils.py │ │ ├── question_answering.py │ │ ├── table_question_answering.py │ │ ├── text2text_generation.py │ │ ├── text_classification.py │ │ ├── text_generation.py │ │ ├── token_classification.py │ │ ├── video_classification.py │ │ ├── visual_question_answering.py │ │ ├── zero_shot_audio_classification.py │ │ ├── zero_shot_classification.py │ │ ├── zero_shot_image_classification.py │ │ └── zero_shot_object_detection.py │ │ ├── processing_utils.py │ │ ├── pytorch_utils.py │ │ ├── sagemaker │ │ ├── __init__.py │ │ ├── trainer_sm.py │ │ └── training_args_sm.py │ │ ├── testing_utils.py │ │ ├── tf_utils.py │ │ ├── time_series_utils.py │ │ ├── tokenization_utils.py │ │ ├── tokenization_utils_base.py │ │ ├── tokenization_utils_fast.py │ │ ├── tools │ │ ├── __init__.py │ │ ├── agent_types.py │ │ ├── agents.py │ │ ├── base.py │ │ ├── document_question_answering.py │ │ ├── evaluate_agent.py │ │ ├── image_captioning.py │ │ ├── image_question_answering.py │ │ ├── image_segmentation.py │ │ ├── prompts.py │ │ ├── python_interpreter.py │ │ ├── speech_to_text.py │ │ ├── text_classification.py │ │ ├── text_question_answering.py │ │ ├── text_summarization.py │ │ ├── text_to_speech.py │ │ └── translation.py │ │ ├── trainer.py │ │ ├── trainer_callback.py │ │ ├── trainer_pt_utils.py │ │ ├── trainer_seq2seq.py │ │ ├── trainer_tf.py │ │ ├── trainer_utils.py │ │ ├── training_args.py │ │ ├── training_args_seq2seq.py │ │ ├── training_args_tf.py │ │ └── utils │ │ ├── __init__.py │ │ ├── backbone_utils.py │ │ ├── bitsandbytes.py │ │ ├── constants.py │ │ ├── doc.py │ │ ├── dummy_detectron2_objects.py │ │ ├── dummy_flax_objects.py │ │ ├── dummy_keras_nlp_objects.py │ │ ├── dummy_pt_objects.py │ │ ├── dummy_sentencepiece_and_tokenizers_objects.py │ │ ├── dummy_sentencepiece_objects.py │ │ ├── dummy_speech_objects.py │ │ ├── dummy_tensorflow_text_objects.py │ │ ├── dummy_tf_objects.py │ │ ├── dummy_tokenizers_objects.py │ │ ├── dummy_vision_objects.py │ │ ├── fx.py │ │ ├── generic.py │ │ ├── hp_naming.py │ │ ├── hub.py │ │ ├── import_utils.py │ │ ├── logging.py │ │ ├── model_parallel_utils.py │ │ ├── notebook.py │ │ ├── quantization_config.py │ │ ├── sentencepiece_model_pb2.py │ │ ├── sentencepiece_model_pb2_new.py │ │ └── versions.py ├── prompts.py ├── tasks │ ├── __init__.py │ ├── all_tasks.py │ ├── paraphrase.py │ ├── predict.py │ └── summary_topic.py ├── train.py └── utils.py ├── pyproject.toml ├── requirements.txt └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | 162 | .vscode 163 | .aim/ 164 | bkup/ 165 | models/ -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/psf/black 3 | rev: 23.7.0 4 | hooks: 5 | - id: black 6 | language_version: python3.10 7 | args: ["--line-length", "120"] 8 | # - repo: https://github.com/pre-commit/mirrors-mypy 9 | # rev: v1.4.1 10 | # hooks: 11 | # - id: mypy 12 | # args: [--ignore-missing-imports] 13 | -------------------------------------------------------------------------------- /assets/RedPajama-INCITE-Base-3B-v1-dialogue-summary-topic.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuutsav/llm-toys/76ed7ded502d5a38e1a4055fcaf15a76476d2bff/assets/RedPajama-INCITE-Base-3B-v1-dialogue-summary-topic.jpeg -------------------------------------------------------------------------------- /assets/RedPajama-INCITE-Base-3B-v1-paraphrase-tone.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuutsav/llm-toys/76ed7ded502d5a38e1a4055fcaf15a76476d2bff/assets/RedPajama-INCITE-Base-3B-v1-paraphrase-tone.jpeg -------------------------------------------------------------------------------- /assets/falcon-7b-paraphrase-tone-dialogue-summary-topic.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuutsav/llm-toys/76ed7ded502d5a38e1a4055fcaf15a76476d2bff/assets/falcon-7b-paraphrase-tone-dialogue-summary-topic.jpeg -------------------------------------------------------------------------------- /llm_toys/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.1.1" 2 | 3 | 4 | from pathlib import Path 5 | import sys 6 | 7 | from llm_toys.utils import print_warning 8 | 9 | 10 | print_warning( 11 | "Note that we are using the transformers and peft packages from the source directory, " 12 | "not the installed package. 4bit bitsandbytes quantization was only working with the " 13 | "main brach of transformers and peft. Once transformers version 4.31.0 and peft version 0.4.0 is " 14 | "published to pypi we will use the published version." 15 | ) 16 | 17 | sys.path.append(str(Path(__file__).parent / "hf")) 18 | -------------------------------------------------------------------------------- /llm_toys/config.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | from enum import Enum 3 | from pathlib import Path 4 | from typing import TYPE_CHECKING 5 | 6 | import torch 7 | 8 | 9 | if TYPE_CHECKING: 10 | from transformers import BitsAndBytesConfig 11 | 12 | 13 | # General 14 | 15 | DEVICE = "mps" if torch.backends.mps.is_available() else torch.device("cuda" if torch.cuda.is_available() else "cpu") 16 | DATA_DIR = Path(__file__).parent.parent / "data" 17 | 18 | 19 | # Model and Prompt related 20 | 21 | class TaskType(str, Enum): 22 | PARAPHRASE_TONE = "paraphrase_tone" 23 | DIALOGUE_SUMMARY_TOPIC = "dialogue_summary_topic" 24 | 25 | 26 | RESPONSE_FORMAT = "\n\n### Response:" 27 | EOC_FORMAT = "\n\n### END" 28 | SUPPORTED_MODEL_SIZES = ["3B", "7B"] 29 | SUPPORTED_MODEL_SIZES_ALL_TASKS = ["7B"] 30 | DEFAULT_3B_MODEL = "togethercomputer/RedPajama-INCITE-Base-3B-v1" 31 | DEFAULT_7B_MODEL = "tiiuae/falcon-7b" 32 | MODEL_CONFIG = { 33 | "3B": { 34 | TaskType.PARAPHRASE_TONE: { 35 | "model_name": DEFAULT_3B_MODEL, 36 | "peft_model_id": "llm-toys/RedPajama-INCITE-Base-3B-v1-paraphrase-tone", 37 | }, 38 | TaskType.DIALOGUE_SUMMARY_TOPIC: { 39 | "model_name": DEFAULT_3B_MODEL, 40 | "peft_model_id": "llm-toys/RedPajama-INCITE-Base-3B-v1-dialogue-summary-topic", 41 | }, 42 | }, 43 | "7B": { 44 | "all": { 45 | "model_name": DEFAULT_7B_MODEL, 46 | "peft_model_id": "llm-toys/falcon-7b-paraphrase-tone-dialogue-summary-topic", 47 | }, 48 | }, 49 | } 50 | SUPPORTED_END_TONES = ["casual", "professional", "witty"] 51 | TASK_TYPES = [tt.value for tt in TaskType] 52 | 53 | 54 | def get_bnb_config() -> BitsAndBytesConfig: 55 | import torch 56 | from transformers import BitsAndBytesConfig 57 | 58 | return BitsAndBytesConfig( 59 | load_in_4bit=True, 60 | bnb_4bit_use_double_quant=True, 61 | bnb_4bit_quant_type="nf4", 62 | bnb_4bit_compute_dtype=torch.bfloat16, 63 | ) 64 | -------------------------------------------------------------------------------- /llm_toys/eval.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | from typing import TYPE_CHECKING 3 | 4 | import torch.nn.functional as F 5 | 6 | from llm_toys.config import DATA_DIR, DEVICE 7 | from llm_toys.utils import load_json, save_json 8 | 9 | 10 | if TYPE_CHECKING: 11 | from torch import Tensor 12 | from transformers import AutoTokenizer, AutoModel 13 | 14 | dialogue_summary_topic_test_data_f = DATA_DIR / "dialogue_summary_topic_test.json" 15 | 16 | 17 | def summary_topic_test_pred(model_size: str = "3B") -> None: 18 | """Generates summary and topic for each dialogue in the 19 | dialogue_summary_topic_test data and saves it to the same file.""" 20 | from llm_toys.tasks import SummaryAndTopicGenerator 21 | 22 | model = SummaryAndTopicGenerator(model_size=model_size) 23 | test_data = load_json(dialogue_summary_topic_test_data_f) 24 | 25 | for i, d in enumerate(test_data, start=1): 26 | pred = model.generate_summary_and_topic(d["dialogue"]) 27 | d["pred_summary"] = pred["summary"] 28 | d["pred_topic"] = pred["topic"] 29 | if i % 10 == 0: 30 | print(f"Processed {i:<3}/{len(test_data)} dialogues") 31 | 32 | save_json(dialogue_summary_topic_test_data_f, test_data) 33 | 34 | 35 | def _max_rouge_score(scores: list) -> dict: 36 | return sorted(scores, key=lambda x: x["rouge1"].fmeasure, reverse=True)[0] 37 | 38 | 39 | def _average_pool(last_hidden_states: Tensor, attention_mask: Tensor) -> Tensor: 40 | last_hidden = last_hidden_states.masked_fill(~attention_mask[..., None].bool(), 0.0) 41 | return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None] 42 | 43 | 44 | def _encode(encoder: AutoModel, tokenizer: AutoTokenizer, input_texts: list[str]) -> Tensor: 45 | batch_dict = tokenizer(input_texts, max_length=512, padding=True, truncation=True, return_tensors="pt") 46 | batch_dict["input_ids"] = batch_dict["input_ids"].to(DEVICE) 47 | batch_dict["attention_mask"] = batch_dict["attention_mask"].to(DEVICE) 48 | outputs = encoder(input_ids=batch_dict["input_ids"], attention_mask=batch_dict["attention_mask"]) 49 | embeddings = _average_pool(outputs.last_hidden_state, batch_dict["attention_mask"]) 50 | embeddings = F.normalize(embeddings, p=2, dim=1) 51 | return embeddings 52 | 53 | 54 | def summary_topic_test_eval() -> dict[str, float]: 55 | """Generates evaluation metrics on dialogue_summary_topic_test after the predictions have been generated 56 | using summary_topic_test_pred(). 57 | 58 | The eval metrics are: 59 | - rouge1, rouge2, rougeL scores for the predicted summary; max from the 3 ground truth summaries 60 | - topic similarity score between the predicted topic and the ground truth topics; max from the 3 ground truth topics 61 | """ 62 | from rouge_score import rouge_scorer 63 | from transformers import AutoTokenizer, AutoModel 64 | 65 | tokenizer = AutoTokenizer.from_pretrained("intfloat/e5-base-v2") 66 | encoder = AutoModel.from_pretrained("intfloat/e5-base-v2") 67 | encoder = encoder.to(DEVICE) 68 | scorer = rouge_scorer.RougeScorer(["rouge1", "rouge2", "rougeL"], use_stemmer=True) 69 | test_data = load_json(dialogue_summary_topic_test_data_f) 70 | 71 | all_scores = [] 72 | for d in test_data: 73 | if "pred_summary" in d and "pred_topic" in d: 74 | summary_scores = [scorer.score(d["pred_summary"], d[f"summary{i+1}"]) for i in range(2)] 75 | topic_scores = [_encode(encoder, tokenizer, [d["pred_topic"], d[f"topic{i+1}"]]) for i in range(2)] 76 | max_topic_sim_score = max([(a @ b.T).item() for a, b in topic_scores]) 77 | scores = {**_max_rouge_score(summary_scores), "topic_similarity": max_topic_sim_score} 78 | all_scores.append(scores) 79 | 80 | avg_scores = {"rouge1": 0.0, "rouge2": 0.0, "rougeL": 0.0, "topic_similarity": 0.0} 81 | for s in all_scores: 82 | for k in avg_scores: 83 | if k == "topic_similarity": 84 | avg_scores[k] += s[k] 85 | else: 86 | avg_scores[k] += s[k].fmeasure 87 | for k in avg_scores: 88 | avg_scores[k] = round(avg_scores[k] / len(all_scores), 3) 89 | 90 | return avg_scores 91 | -------------------------------------------------------------------------------- /llm_toys/hf/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuutsav/llm-toys/76ed7ded502d5a38e1a4055fcaf15a76476d2bff/llm_toys/hf/__init__.py -------------------------------------------------------------------------------- /llm_toys/hf/peft/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | # There's no way to ignore "F401 '...' imported but unused" warnings in this 3 | # module, but to preserve other warnings. So, don't check this module at all. 4 | 5 | # coding=utf-8 6 | # Copyright 2023-present the HuggingFace Inc. team. 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | 20 | __version__ = "0.4.0.dev0" 21 | 22 | from .auto import ( 23 | AutoPeftModel, 24 | AutoPeftModelForCausalLM, 25 | AutoPeftModelForSequenceClassification, 26 | AutoPeftModelForSeq2SeqLM, 27 | AutoPeftModelForTokenClassification, 28 | AutoPeftModelForQuestionAnswering, 29 | AutoPeftModelForFeatureExtraction, 30 | ) 31 | from .mapping import MODEL_TYPE_TO_PEFT_MODEL_MAPPING, PEFT_TYPE_TO_CONFIG_MAPPING, get_peft_config, get_peft_model 32 | from .peft_model import ( 33 | PeftModel, 34 | PeftModelForCausalLM, 35 | PeftModelForSeq2SeqLM, 36 | PeftModelForSequenceClassification, 37 | PeftModelForTokenClassification, 38 | PeftModelForQuestionAnswering, 39 | PeftModelForFeatureExtraction, 40 | ) 41 | from .tuners import ( 42 | AdaptionPromptConfig, 43 | AdaptionPromptModel, 44 | LoraConfig, 45 | LoraModel, 46 | IA3Config, 47 | IA3Model, 48 | AdaLoraConfig, 49 | AdaLoraModel, 50 | PrefixEncoder, 51 | PrefixTuningConfig, 52 | PromptEmbedding, 53 | PromptEncoder, 54 | PromptEncoderConfig, 55 | PromptEncoderReparameterizationType, 56 | PromptTuningConfig, 57 | PromptTuningInit, 58 | ) 59 | from .utils import ( 60 | TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING, 61 | PeftConfig, 62 | PeftType, 63 | PromptLearningConfig, 64 | TaskType, 65 | bloom_model_postprocess_past_key_value, 66 | get_peft_model_state_dict, 67 | prepare_model_for_int8_training, 68 | prepare_model_for_kbit_training, 69 | set_peft_model_state_dict, 70 | shift_tokens_right, 71 | ) 72 | -------------------------------------------------------------------------------- /llm_toys/hf/peft/import_utils.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2023-present the HuggingFace Inc. team. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import importlib 16 | 17 | 18 | def is_bnb_available(): 19 | return importlib.util.find_spec("bitsandbytes") is not None 20 | 21 | 22 | def is_bnb_4bit_available(): 23 | if not is_bnb_available(): 24 | return False 25 | 26 | import bitsandbytes as bnb 27 | 28 | return hasattr(bnb.nn, "Linear4bit") 29 | -------------------------------------------------------------------------------- /llm_toys/hf/peft/mapping.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2023-present the HuggingFace Inc. team. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from __future__ import annotations 17 | 18 | from typing import TYPE_CHECKING, Any, Dict 19 | 20 | from .peft_model import ( 21 | PeftModel, 22 | PeftModelForCausalLM, 23 | PeftModelForFeatureExtraction, 24 | PeftModelForQuestionAnswering, 25 | PeftModelForSeq2SeqLM, 26 | PeftModelForSequenceClassification, 27 | PeftModelForTokenClassification, 28 | ) 29 | from .tuners import ( 30 | AdaLoraConfig, 31 | AdaptionPromptConfig, 32 | IA3Config, 33 | LoraConfig, 34 | PrefixTuningConfig, 35 | PromptEncoderConfig, 36 | PromptTuningConfig, 37 | ) 38 | from .utils import PromptLearningConfig, _prepare_prompt_learning_config 39 | 40 | 41 | if TYPE_CHECKING: 42 | from transformers import PreTrainedModel 43 | 44 | from .utils.config import PeftConfig 45 | 46 | 47 | MODEL_TYPE_TO_PEFT_MODEL_MAPPING = { 48 | "SEQ_CLS": PeftModelForSequenceClassification, 49 | "SEQ_2_SEQ_LM": PeftModelForSeq2SeqLM, 50 | "CAUSAL_LM": PeftModelForCausalLM, 51 | "TOKEN_CLS": PeftModelForTokenClassification, 52 | "QUESTION_ANS": PeftModelForQuestionAnswering, 53 | "FEATURE_EXTRACTION": PeftModelForFeatureExtraction, 54 | } 55 | 56 | PEFT_TYPE_TO_CONFIG_MAPPING = { 57 | "ADAPTION_PROMPT": AdaptionPromptConfig, 58 | "PROMPT_TUNING": PromptTuningConfig, 59 | "PREFIX_TUNING": PrefixTuningConfig, 60 | "P_TUNING": PromptEncoderConfig, 61 | "LORA": LoraConfig, 62 | "ADALORA": AdaLoraConfig, 63 | "IA3": IA3Config, 64 | } 65 | 66 | 67 | def get_peft_config(config_dict: Dict[str, Any]): 68 | """ 69 | Returns a Peft config object from a dictionary. 70 | 71 | Args: 72 | config_dict (`Dict[str, Any]`): Dictionary containing the configuration parameters. 73 | """ 74 | 75 | return PEFT_TYPE_TO_CONFIG_MAPPING[config_dict["peft_type"]](**config_dict) 76 | 77 | 78 | def get_peft_model(model: PreTrainedModel, peft_config: PeftConfig, adapter_name: str = "default") -> PeftModel: 79 | """ 80 | Returns a Peft model object from a model and a config. 81 | 82 | Args: 83 | model ([`transformers.PreTrainedModel`]): Model to be wrapped. 84 | peft_config ([`PeftConfig`]): Configuration object containing the parameters of the Peft model. 85 | """ 86 | model_config = model.config.to_dict() if hasattr(model.config, "to_dict") else model.config 87 | peft_config.base_model_name_or_path = model.__dict__.get("name_or_path", None) 88 | 89 | if peft_config.task_type not in MODEL_TYPE_TO_PEFT_MODEL_MAPPING.keys() and not isinstance( 90 | peft_config, PromptLearningConfig 91 | ): 92 | return PeftModel(model, peft_config, adapter_name=adapter_name) 93 | if isinstance(peft_config, PromptLearningConfig): 94 | peft_config = _prepare_prompt_learning_config(peft_config, model_config) 95 | return MODEL_TYPE_TO_PEFT_MODEL_MAPPING[peft_config.task_type](model, peft_config, adapter_name=adapter_name) 96 | -------------------------------------------------------------------------------- /llm_toys/hf/peft/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuutsav/llm-toys/76ed7ded502d5a38e1a4055fcaf15a76476d2bff/llm_toys/hf/peft/py.typed -------------------------------------------------------------------------------- /llm_toys/hf/peft/tuners/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | # There's no way to ignore "F401 '...' imported but unused" warnings in this 3 | # module, but to preserve other warnings. So, don't check this module at all 4 | 5 | # coding=utf-8 6 | # Copyright 2023-present the HuggingFace Inc. team. 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | 20 | from .adaption_prompt import AdaptionPromptConfig, AdaptionPromptModel 21 | from .lora import LoraConfig, LoraModel 22 | from .ia3 import IA3Config, IA3Model 23 | from .adalora import AdaLoraConfig, AdaLoraModel 24 | from .p_tuning import PromptEncoder, PromptEncoderConfig, PromptEncoderReparameterizationType 25 | from .prefix_tuning import PrefixEncoder, PrefixTuningConfig 26 | from .prompt_tuning import PromptEmbedding, PromptTuningConfig, PromptTuningInit 27 | -------------------------------------------------------------------------------- /llm_toys/hf/peft/tuners/prefix_tuning.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2023-present the HuggingFace Inc. team. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from dataclasses import dataclass, field 18 | 19 | import torch 20 | 21 | from ..utils import PeftType, PromptLearningConfig 22 | 23 | 24 | @dataclass 25 | class PrefixTuningConfig(PromptLearningConfig): 26 | """ 27 | This is the configuration class to store the configuration of a [`PrefixEncoder`]. 28 | 29 | Args: 30 | encoder_hidden_size (`int`): The hidden size of the prompt encoder. 31 | prefix_projection (`bool`): Whether to project the prefix embeddings. 32 | """ 33 | 34 | encoder_hidden_size: int = field( 35 | default=None, 36 | metadata={"help": "The hidden size of the encoder"}, 37 | ) 38 | prefix_projection: bool = field( 39 | default=False, 40 | metadata={"help": "Whether to project the prefix tokens"}, 41 | ) 42 | 43 | def __post_init__(self): 44 | self.peft_type = PeftType.PREFIX_TUNING 45 | 46 | 47 | # Based on https://github.com/THUDM/P-tuning-v2/blob/main/model/prefix_encoder.py 48 | # with some refactor 49 | class PrefixEncoder(torch.nn.Module): 50 | r""" 51 | The `torch.nn` model to encode the prefix. 52 | 53 | Args: 54 | config ([`PrefixTuningConfig`]): The configuration of the prefix encoder. 55 | 56 | Example: 57 | 58 | ```py 59 | >>> from peft import PrefixEncoder, PrefixTuningConfig 60 | 61 | >>> config = PrefixTuningConfig( 62 | ... peft_type="PREFIX_TUNING", 63 | ... task_type="SEQ_2_SEQ_LM", 64 | ... num_virtual_tokens=20, 65 | ... token_dim=768, 66 | ... num_transformer_submodules=1, 67 | ... num_attention_heads=12, 68 | ... num_layers=12, 69 | ... encoder_hidden_size=768, 70 | ... ) 71 | >>> prefix_encoder = PrefixEncoder(config) 72 | ``` 73 | 74 | **Attributes**: 75 | - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prefix encoder. 76 | - **transform** (`torch.nn.Sequential`) -- The two-layer MLP to transform the prefix embeddings if 77 | `prefix_projection` is `True`. 78 | - **prefix_projection** (`bool`) -- Whether to project the prefix embeddings. 79 | 80 | Input shape: (`batch_size`, `num_virtual_tokens`) 81 | 82 | Output shape: (`batch_size`, `num_virtual_tokens`, `2*layers*hidden`) 83 | """ 84 | 85 | def __init__(self, config): 86 | super().__init__() 87 | self.prefix_projection = config.prefix_projection 88 | token_dim = config.token_dim 89 | num_layers = config.num_layers 90 | encoder_hidden_size = config.encoder_hidden_size 91 | num_virtual_tokens = config.num_virtual_tokens 92 | if self.prefix_projection and not config.inference_mode: 93 | # Use a two-layer MLP to encode the prefix 94 | self.embedding = torch.nn.Embedding(num_virtual_tokens, token_dim) 95 | self.transform = torch.nn.Sequential( 96 | torch.nn.Linear(token_dim, encoder_hidden_size), 97 | torch.nn.Tanh(), 98 | torch.nn.Linear(encoder_hidden_size, num_layers * 2 * token_dim), 99 | ) 100 | else: 101 | self.embedding = torch.nn.Embedding(num_virtual_tokens, num_layers * 2 * token_dim) 102 | 103 | def forward(self, prefix: torch.Tensor): 104 | if self.prefix_projection: 105 | prefix_tokens = self.embedding(prefix) 106 | past_key_values = self.transform(prefix_tokens) 107 | else: 108 | past_key_values = self.embedding(prefix) 109 | return past_key_values 110 | -------------------------------------------------------------------------------- /llm_toys/hf/peft/tuners/prompt_tuning.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2023-present the HuggingFace Inc. team. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import enum 17 | import math 18 | from dataclasses import dataclass, field 19 | from typing import Optional, Union 20 | 21 | import torch 22 | 23 | from ..utils import PeftType, PromptLearningConfig 24 | 25 | 26 | class PromptTuningInit(str, enum.Enum): 27 | TEXT = "TEXT" 28 | RANDOM = "RANDOM" 29 | 30 | 31 | @dataclass 32 | class PromptTuningConfig(PromptLearningConfig): 33 | """ 34 | This is the configuration class to store the configuration of a [`PromptEmbedding`]. 35 | 36 | Args: 37 | prompt_tuning_init (Union[[`PromptTuningInit`], `str`]): The initialization of the prompt embedding. 38 | prompt_tuning_init_text (`str`, *optional*): 39 | The text to initialize the prompt embedding. Only used if `prompt_tuning_init` is `TEXT`. 40 | tokenizer_name_or_path (`str`, *optional*): 41 | The name or path of the tokenizer. Only used if `prompt_tuning_init` is `TEXT`. 42 | """ 43 | 44 | prompt_tuning_init: Union[PromptTuningInit, str] = field( 45 | default=PromptTuningInit.RANDOM, 46 | metadata={"help": "How to initialize the prompt tuning parameters"}, 47 | ) 48 | prompt_tuning_init_text: Optional[str] = field( 49 | default=None, 50 | metadata={ 51 | "help": "The text to use for prompt tuning initialization. Only used if prompt_tuning_init is `TEXT`" 52 | }, 53 | ) 54 | tokenizer_name_or_path: Optional[str] = field( 55 | default=None, 56 | metadata={ 57 | "help": "The tokenizer to use for prompt tuning initialization. Only used if prompt_tuning_init is `TEXT`" 58 | }, 59 | ) 60 | 61 | def __post_init__(self): 62 | self.peft_type = PeftType.PROMPT_TUNING 63 | 64 | 65 | class PromptEmbedding(torch.nn.Module): 66 | """ 67 | The model to encode virtual tokens into prompt embeddings. 68 | 69 | Args: 70 | config ([`PromptTuningConfig`]): The configuration of the prompt embedding. 71 | word_embeddings (`torch.nn.Module`): The word embeddings of the base transformer model. 72 | 73 | **Attributes**: 74 | - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prompt embedding. 75 | 76 | Example: 77 | 78 | ```py 79 | >>> from peft import PromptEmbedding, PromptTuningConfig 80 | 81 | >>> config = PromptTuningConfig( 82 | ... peft_type="PROMPT_TUNING", 83 | ... task_type="SEQ_2_SEQ_LM", 84 | ... num_virtual_tokens=20, 85 | ... token_dim=768, 86 | ... num_transformer_submodules=1, 87 | ... num_attention_heads=12, 88 | ... num_layers=12, 89 | ... prompt_tuning_init="TEXT", 90 | ... prompt_tuning_init_text="Predict if sentiment of this review is positive, negative or neutral", 91 | ... tokenizer_name_or_path="t5-base", 92 | ... ) 93 | 94 | >>> # t5_model.shared is the word embeddings of the base model 95 | >>> prompt_embedding = PromptEmbedding(config, t5_model.shared) 96 | ``` 97 | 98 | Input Shape: (`batch_size`, `total_virtual_tokens`) 99 | 100 | Output Shape: (`batch_size`, `total_virtual_tokens`, `token_dim`) 101 | """ 102 | 103 | def __init__(self, config, word_embeddings): 104 | super().__init__() 105 | 106 | total_virtual_tokens = config.num_virtual_tokens * config.num_transformer_submodules 107 | self.embedding = torch.nn.Embedding(total_virtual_tokens, config.token_dim) 108 | if config.prompt_tuning_init == PromptTuningInit.TEXT: 109 | from transformers import AutoTokenizer 110 | 111 | tokenizer = AutoTokenizer.from_pretrained(config.tokenizer_name_or_path) 112 | init_text = config.prompt_tuning_init_text 113 | init_token_ids = tokenizer(init_text)["input_ids"] 114 | # Trim or iterate until num_text_tokens matches total_virtual_tokens 115 | num_text_tokens = len(init_token_ids) 116 | if num_text_tokens > total_virtual_tokens: 117 | init_token_ids = init_token_ids[:total_virtual_tokens] 118 | elif num_text_tokens < total_virtual_tokens: 119 | num_reps = math.ceil(total_virtual_tokens / num_text_tokens) 120 | init_token_ids = init_token_ids * num_reps 121 | init_token_ids = init_token_ids[:total_virtual_tokens] 122 | 123 | word_embedding_weights = word_embeddings(torch.LongTensor(init_token_ids)).detach().clone() 124 | word_embedding_weights = word_embedding_weights.to(torch.float32) 125 | self.embedding.weight = torch.nn.Parameter(word_embedding_weights) 126 | 127 | def forward(self, indices): 128 | # Just get embeddings 129 | prompt_embeddings = self.embedding(indices) 130 | return prompt_embeddings 131 | -------------------------------------------------------------------------------- /llm_toys/hf/peft/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | # There's no way to ignore "F401 '...' imported but unused" warnings in this 3 | # module, but to preserve other warnings. So, don't check this module at all 4 | 5 | # coding=utf-8 6 | # Copyright 2023-present the HuggingFace Inc. team. 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | 20 | from .config import PeftConfig, PeftType, PromptLearningConfig, TaskType 21 | from .other import ( 22 | TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING, 23 | TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING, 24 | TRANSFORMERS_MODELS_TO_ADALORA_TARGET_MODULES_MAPPING, 25 | TRANSFORMERS_MODELS_TO_IA3_TARGET_MODULES_MAPPING, 26 | TRANSFORMERS_MODELS_TO_IA3_FEEDFORWARD_MODULES_MAPPING, 27 | COMMON_LAYERS_PATTERN, 28 | CONFIG_NAME, 29 | WEIGHTS_NAME, 30 | SAFETENSORS_WEIGHTS_NAME, 31 | CLAMP_QUANTILE, 32 | _set_trainable, 33 | add_library_to_model_card, 34 | bloom_model_postprocess_past_key_value, 35 | prepare_model_for_int8_training, 36 | prepare_model_for_kbit_training, 37 | shift_tokens_right, 38 | transpose, 39 | _get_submodules, 40 | _set_adapter, 41 | _freeze_adapter, 42 | ModulesToSaveWrapper, 43 | _prepare_prompt_learning_config, 44 | ) 45 | from .hub_utils import hub_file_exists 46 | from .save_and_load import get_peft_model_state_dict, set_peft_model_state_dict 47 | -------------------------------------------------------------------------------- /llm_toys/hf/peft/utils/hub_utils.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2023-present the HuggingFace Inc. team. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from huggingface_hub import get_hf_file_metadata, hf_hub_url 17 | from huggingface_hub.utils import EntryNotFoundError 18 | 19 | 20 | def hub_file_exists(repo_id: str, filename: str, revision: str = None, repo_type: str = None) -> bool: 21 | r""" 22 | Checks if a file exists in a remote Hub repository. 23 | """ 24 | url = hf_hub_url(repo_id=repo_id, filename=filename, repo_type=repo_type, revision=revision) 25 | try: 26 | get_hf_file_metadata(url) 27 | return True 28 | except EntryNotFoundError: 29 | return False 30 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/activations_tf.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import math 16 | 17 | import tensorflow as tf 18 | from packaging import version 19 | 20 | 21 | def _gelu(x): 22 | """ 23 | Gaussian Error Linear Unit. Original Implementation of the gelu activation function in Google Bert repo when 24 | initially created. For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): 25 | 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) Also see 26 | https://arxiv.org/abs/1606.08415 27 | """ 28 | x = tf.convert_to_tensor(x) 29 | cdf = 0.5 * (1.0 + tf.math.erf(x / tf.cast(tf.sqrt(2.0), x.dtype))) 30 | 31 | return x * cdf 32 | 33 | 34 | def _gelu_new(x): 35 | """ 36 | Gaussian Error Linear Unit. This is a smoother version of the GELU. Original paper: https://arxiv.org/abs/1606.0841 37 | 38 | Args: 39 | x: float Tensor to perform activation 40 | 41 | Returns: 42 | `x` with the GELU activation applied. 43 | """ 44 | x = tf.convert_to_tensor(x) 45 | pi = tf.cast(math.pi, x.dtype) 46 | coeff = tf.cast(0.044715, x.dtype) 47 | cdf = 0.5 * (1.0 + tf.tanh(tf.sqrt(2.0 / pi) * (x + coeff * tf.pow(x, 3)))) 48 | 49 | return x * cdf 50 | 51 | 52 | def mish(x): 53 | x = tf.convert_to_tensor(x) 54 | 55 | return x * tf.tanh(tf.math.softplus(x)) 56 | 57 | 58 | def gelu_fast(x): 59 | x = tf.convert_to_tensor(x) 60 | coeff1 = tf.cast(0.044715, x.dtype) 61 | coeff2 = tf.cast(0.7978845608, x.dtype) 62 | 63 | return 0.5 * x * (1.0 + tf.tanh(x * coeff2 * (1.0 + coeff1 * x * x))) 64 | 65 | 66 | def quick_gelu(x): 67 | x = tf.convert_to_tensor(x) 68 | coeff = tf.cast(1.702, x.dtype) 69 | return x * tf.math.sigmoid(coeff * x) 70 | 71 | 72 | def gelu_10(x): 73 | """ 74 | Clip the range of possible GeLU outputs between [-10, 10]. This is especially useful for quantization purpose, as 75 | it allows mapping 2 negatives values in the GeLU spectrum. For more information on this trick, please refer to 76 | https://arxiv.org/abs/2004.09602 77 | 78 | Gaussian Error Linear Unit. Original Implementation of the gelu activation function in Google Bert repo when 79 | initially created. For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): 80 | 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) Also see 81 | https://arxiv.org/abs/1606.08415 :param x: :return: 82 | """ 83 | return tf.clip_by_value(_gelu(x), -10, 10) 84 | 85 | 86 | def glu(x, axis=-1): 87 | """ 88 | Gated Linear Unit. Implementation as defined in the original paper (see https://arxiv.org/abs/1612.08083), where 89 | the input `x` is split in two halves across a dimension (`axis`), A and B, returning A * sigmoid(B). 90 | 91 | Args: 92 | `x`: float Tensor to perform activation 93 | `axis`: dimension across which `x` be split in half 94 | 95 | Returns: 96 | `x` with the GLU activation applied (with its size halved across the dimension `axis`). 97 | """ 98 | a, b = tf.split(x, 2, axis=axis) 99 | return a * tf.math.sigmoid(b) 100 | 101 | 102 | if version.parse(tf.version.VERSION) >= version.parse("2.4"): 103 | 104 | def approximate_gelu_wrap(x): 105 | return tf.keras.activations.gelu(x, approximate=True) 106 | 107 | gelu = tf.keras.activations.gelu 108 | gelu_new = approximate_gelu_wrap 109 | else: 110 | gelu = _gelu 111 | gelu_new = _gelu_new 112 | 113 | 114 | ACT2FN = { 115 | "gelu": gelu, 116 | "gelu_10": gelu_10, 117 | "gelu_fast": gelu_fast, 118 | "gelu_new": gelu_new, 119 | "glu": glu, 120 | "mish": mish, 121 | "quick_gelu": quick_gelu, 122 | "relu": tf.keras.activations.relu, 123 | "sigmoid": tf.keras.activations.sigmoid, 124 | "silu": tf.keras.activations.swish, 125 | "swish": tf.keras.activations.swish, 126 | "tanh": tf.keras.activations.tanh, 127 | } 128 | 129 | 130 | def get_tf_activation(activation_string): 131 | if activation_string in ACT2FN: 132 | return ACT2FN[activation_string] 133 | else: 134 | raise KeyError(f"function {activation_string} not found in ACT2FN mapping {list(ACT2FN.keys())}") 135 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/benchmark/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuutsav/llm-toys/76ed7ded502d5a38e1a4055fcaf15a76476d2bff/llm_toys/hf/transformers/benchmark/__init__.py -------------------------------------------------------------------------------- /llm_toys/hf/transformers/benchmark/benchmark_args.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The HuggingFace Inc. team. 3 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from dataclasses import dataclass, field 18 | from typing import Tuple 19 | 20 | from ..utils import cached_property, is_torch_available, is_torch_tpu_available, logging, requires_backends 21 | from .benchmark_args_utils import BenchmarkArguments 22 | 23 | 24 | if is_torch_available(): 25 | import torch 26 | 27 | if is_torch_tpu_available(check_device=False): 28 | import torch_xla.core.xla_model as xm 29 | 30 | 31 | logger = logging.get_logger(__name__) 32 | 33 | 34 | @dataclass 35 | class PyTorchBenchmarkArguments(BenchmarkArguments): 36 | deprecated_args = [ 37 | "no_inference", 38 | "no_cuda", 39 | "no_tpu", 40 | "no_speed", 41 | "no_memory", 42 | "no_env_print", 43 | "no_multi_process", 44 | ] 45 | 46 | def __init__(self, **kwargs): 47 | """ 48 | This __init__ is there for legacy code. When removing deprecated args completely, the class can simply be 49 | deleted 50 | """ 51 | for deprecated_arg in self.deprecated_args: 52 | if deprecated_arg in kwargs: 53 | positive_arg = deprecated_arg[3:] 54 | setattr(self, positive_arg, not kwargs.pop(deprecated_arg)) 55 | logger.warning( 56 | f"{deprecated_arg} is depreciated. Please use --no_{positive_arg} or" 57 | f" {positive_arg}={kwargs[positive_arg]}" 58 | ) 59 | 60 | self.torchscript = kwargs.pop("torchscript", self.torchscript) 61 | self.torch_xla_tpu_print_metrics = kwargs.pop("torch_xla_tpu_print_metrics", self.torch_xla_tpu_print_metrics) 62 | self.fp16_opt_level = kwargs.pop("fp16_opt_level", self.fp16_opt_level) 63 | super().__init__(**kwargs) 64 | 65 | torchscript: bool = field(default=False, metadata={"help": "Trace the models using torchscript"}) 66 | torch_xla_tpu_print_metrics: bool = field(default=False, metadata={"help": "Print Xla/PyTorch tpu metrics"}) 67 | fp16_opt_level: str = field( 68 | default="O1", 69 | metadata={ 70 | "help": ( 71 | "For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']. " 72 | "See details at https://nvidia.github.io/apex/amp.html" 73 | ) 74 | }, 75 | ) 76 | 77 | @cached_property 78 | def _setup_devices(self) -> Tuple["torch.device", int]: 79 | requires_backends(self, ["torch"]) 80 | logger.info("PyTorch: setting up devices") 81 | if not self.cuda: 82 | device = torch.device("cpu") 83 | n_gpu = 0 84 | elif is_torch_tpu_available(): 85 | device = xm.xla_device() 86 | n_gpu = 0 87 | else: 88 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 89 | n_gpu = torch.cuda.device_count() 90 | return device, n_gpu 91 | 92 | @property 93 | def is_tpu(self): 94 | return is_torch_tpu_available() and self.tpu 95 | 96 | @property 97 | def device_idx(self) -> int: 98 | requires_backends(self, ["torch"]) 99 | # TODO(PVP): currently only single GPU is supported 100 | return torch.cuda.current_device() 101 | 102 | @property 103 | def device(self) -> "torch.device": 104 | requires_backends(self, ["torch"]) 105 | return self._setup_devices[0] 106 | 107 | @property 108 | def n_gpu(self): 109 | requires_backends(self, ["torch"]) 110 | return self._setup_devices[1] 111 | 112 | @property 113 | def is_gpu(self): 114 | return self.n_gpu > 0 115 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/benchmark/benchmark_args_tf.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The HuggingFace Inc. team. 3 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from dataclasses import dataclass, field 18 | from typing import Tuple 19 | 20 | from ..utils import cached_property, is_tf_available, logging, requires_backends 21 | from .benchmark_args_utils import BenchmarkArguments 22 | 23 | 24 | if is_tf_available(): 25 | import tensorflow as tf 26 | 27 | 28 | logger = logging.get_logger(__name__) 29 | 30 | 31 | @dataclass 32 | class TensorFlowBenchmarkArguments(BenchmarkArguments): 33 | deprecated_args = [ 34 | "no_inference", 35 | "no_cuda", 36 | "no_tpu", 37 | "no_speed", 38 | "no_memory", 39 | "no_env_print", 40 | "no_multi_process", 41 | ] 42 | 43 | def __init__(self, **kwargs): 44 | """ 45 | This __init__ is there for legacy code. When removing deprecated args completely, the class can simply be 46 | deleted 47 | """ 48 | for deprecated_arg in self.deprecated_args: 49 | if deprecated_arg in kwargs: 50 | positive_arg = deprecated_arg[3:] 51 | kwargs[positive_arg] = not kwargs.pop(deprecated_arg) 52 | logger.warning( 53 | f"{deprecated_arg} is depreciated. Please use --no-{positive_arg} or" 54 | f" {positive_arg}={kwargs[positive_arg]}" 55 | ) 56 | self.tpu_name = kwargs.pop("tpu_name", self.tpu_name) 57 | self.device_idx = kwargs.pop("device_idx", self.device_idx) 58 | self.eager_mode = kwargs.pop("eager_mode", self.eager_mode) 59 | self.use_xla = kwargs.pop("use_xla", self.use_xla) 60 | super().__init__(**kwargs) 61 | 62 | tpu_name: str = field( 63 | default=None, 64 | metadata={"help": "Name of TPU"}, 65 | ) 66 | device_idx: int = field( 67 | default=0, 68 | metadata={"help": "CPU / GPU device index. Defaults to 0."}, 69 | ) 70 | eager_mode: bool = field(default=False, metadata={"help": "Benchmark models in eager model."}) 71 | use_xla: bool = field( 72 | default=False, 73 | metadata={ 74 | "help": "Benchmark models using XLA JIT compilation. Note that `eager_model` has to be set to `False`." 75 | }, 76 | ) 77 | 78 | @cached_property 79 | def _setup_tpu(self) -> Tuple["tf.distribute.cluster_resolver.TPUClusterResolver"]: 80 | requires_backends(self, ["tf"]) 81 | tpu = None 82 | if self.tpu: 83 | try: 84 | if self.tpu_name: 85 | tpu = tf.distribute.cluster_resolver.TPUClusterResolver(self.tpu_name) 86 | else: 87 | tpu = tf.distribute.cluster_resolver.TPUClusterResolver() 88 | except ValueError: 89 | tpu = None 90 | return tpu 91 | 92 | @cached_property 93 | def _setup_strategy(self) -> Tuple["tf.distribute.Strategy", "tf.distribute.cluster_resolver.TPUClusterResolver"]: 94 | requires_backends(self, ["tf"]) 95 | if self.is_tpu: 96 | tf.config.experimental_connect_to_cluster(self._setup_tpu) 97 | tf.tpu.experimental.initialize_tpu_system(self._setup_tpu) 98 | 99 | strategy = tf.distribute.TPUStrategy(self._setup_tpu) 100 | else: 101 | # currently no multi gpu is allowed 102 | if self.is_gpu: 103 | # TODO: Currently only single GPU is supported 104 | tf.config.set_visible_devices(self.gpu_list[self.device_idx], "GPU") 105 | strategy = tf.distribute.OneDeviceStrategy(device=f"/gpu:{self.device_idx}") 106 | else: 107 | tf.config.set_visible_devices([], "GPU") # disable GPU 108 | strategy = tf.distribute.OneDeviceStrategy(device=f"/cpu:{self.device_idx}") 109 | 110 | return strategy 111 | 112 | @property 113 | def is_tpu(self) -> bool: 114 | requires_backends(self, ["tf"]) 115 | return self._setup_tpu is not None 116 | 117 | @property 118 | def strategy(self) -> "tf.distribute.Strategy": 119 | requires_backends(self, ["tf"]) 120 | return self._setup_strategy 121 | 122 | @property 123 | def gpu_list(self): 124 | requires_backends(self, ["tf"]) 125 | return tf.config.list_physical_devices("GPU") 126 | 127 | @property 128 | def n_gpu(self) -> int: 129 | requires_backends(self, ["tf"]) 130 | if self.cuda: 131 | return len(self.gpu_list) 132 | return 0 133 | 134 | @property 135 | def is_gpu(self) -> bool: 136 | return self.n_gpu > 0 137 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from abc import ABC, abstractmethod 16 | from argparse import ArgumentParser 17 | 18 | 19 | class BaseTransformersCLICommand(ABC): 20 | @staticmethod 21 | @abstractmethod 22 | def register_subcommand(parser: ArgumentParser): 23 | raise NotImplementedError() 24 | 25 | @abstractmethod 26 | def run(self): 27 | raise NotImplementedError() 28 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/commands/download.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from argparse import ArgumentParser 16 | 17 | from . import BaseTransformersCLICommand 18 | 19 | 20 | def download_command_factory(args): 21 | return DownloadCommand(args.model, args.cache_dir, args.force, args.trust_remote_code) 22 | 23 | 24 | class DownloadCommand(BaseTransformersCLICommand): 25 | @staticmethod 26 | def register_subcommand(parser: ArgumentParser): 27 | download_parser = parser.add_parser("download") 28 | download_parser.add_argument("--cache-dir", type=str, default=None, help="Path to location to store the models") 29 | download_parser.add_argument( 30 | "--force", action="store_true", help="Force the model to be download even if already in cache-dir" 31 | ) 32 | download_parser.add_argument( 33 | "--trust-remote-code", 34 | action="store_true", 35 | help="Whether or not to allow for custom models defined on the Hub in their own modeling files. Use only if you've reviewed the code as it will execute on your local machine", 36 | ) 37 | download_parser.add_argument("model", type=str, help="Name of the model to download") 38 | download_parser.set_defaults(func=download_command_factory) 39 | 40 | def __init__(self, model: str, cache: str, force: bool, trust_remote_code: bool): 41 | self._model = model 42 | self._cache = cache 43 | self._force = force 44 | self._trust_remote_code = trust_remote_code 45 | 46 | def run(self): 47 | from ..models.auto import AutoModel, AutoTokenizer 48 | 49 | AutoModel.from_pretrained( 50 | self._model, cache_dir=self._cache, force_download=self._force, trust_remote_code=self._trust_remote_code 51 | ) 52 | AutoTokenizer.from_pretrained( 53 | self._model, cache_dir=self._cache, force_download=self._force, trust_remote_code=self._trust_remote_code 54 | ) 55 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/commands/run.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from argparse import ArgumentParser 16 | 17 | from ..pipelines import Pipeline, PipelineDataFormat, get_supported_tasks, pipeline 18 | from ..utils import logging 19 | from . import BaseTransformersCLICommand 20 | 21 | 22 | logger = logging.get_logger(__name__) # pylint: disable=invalid-name 23 | 24 | 25 | def try_infer_format_from_ext(path: str): 26 | if not path: 27 | return "pipe" 28 | 29 | for ext in PipelineDataFormat.SUPPORTED_FORMATS: 30 | if path.endswith(ext): 31 | return ext 32 | 33 | raise Exception( 34 | f"Unable to determine file format from file extension {path}. " 35 | f"Please provide the format through --format {PipelineDataFormat.SUPPORTED_FORMATS}" 36 | ) 37 | 38 | 39 | def run_command_factory(args): 40 | nlp = pipeline( 41 | task=args.task, 42 | model=args.model if args.model else None, 43 | config=args.config, 44 | tokenizer=args.tokenizer, 45 | device=args.device, 46 | ) 47 | format = try_infer_format_from_ext(args.input) if args.format == "infer" else args.format 48 | reader = PipelineDataFormat.from_str( 49 | format=format, 50 | output_path=args.output, 51 | input_path=args.input, 52 | column=args.column if args.column else nlp.default_input_names, 53 | overwrite=args.overwrite, 54 | ) 55 | return RunCommand(nlp, reader) 56 | 57 | 58 | class RunCommand(BaseTransformersCLICommand): 59 | def __init__(self, nlp: Pipeline, reader: PipelineDataFormat): 60 | self._nlp = nlp 61 | self._reader = reader 62 | 63 | @staticmethod 64 | def register_subcommand(parser: ArgumentParser): 65 | run_parser = parser.add_parser("run", help="Run a pipeline through the CLI") 66 | run_parser.add_argument("--task", choices=get_supported_tasks(), help="Task to run") 67 | run_parser.add_argument("--input", type=str, help="Path to the file to use for inference") 68 | run_parser.add_argument("--output", type=str, help="Path to the file that will be used post to write results.") 69 | run_parser.add_argument("--model", type=str, help="Name or path to the model to instantiate.") 70 | run_parser.add_argument("--config", type=str, help="Name or path to the model's config to instantiate.") 71 | run_parser.add_argument( 72 | "--tokenizer", type=str, help="Name of the tokenizer to use. (default: same as the model name)" 73 | ) 74 | run_parser.add_argument( 75 | "--column", 76 | type=str, 77 | help="Name of the column to use as input. (For multi columns input as QA use column1,columns2)", 78 | ) 79 | run_parser.add_argument( 80 | "--format", 81 | type=str, 82 | default="infer", 83 | choices=PipelineDataFormat.SUPPORTED_FORMATS, 84 | help="Input format to read from", 85 | ) 86 | run_parser.add_argument( 87 | "--device", 88 | type=int, 89 | default=-1, 90 | help="Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU (default: -1)", 91 | ) 92 | run_parser.add_argument("--overwrite", action="store_true", help="Allow overwriting the output file.") 93 | run_parser.set_defaults(func=run_command_factory) 94 | 95 | def run(self): 96 | nlp, outputs = self._nlp, [] 97 | 98 | for entry in self._reader: 99 | output = nlp(**entry) if self._reader.is_multi_columns else nlp(entry) 100 | if isinstance(output, dict): 101 | outputs.append(output) 102 | else: 103 | outputs += output 104 | 105 | # Saving data 106 | if self._nlp.binary_output: 107 | binary_path = self._reader.save_binary(outputs) 108 | logger.warning(f"Current pipeline requires output to be in binary format, saving at {binary_path}") 109 | else: 110 | self._reader.save(outputs) 111 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/commands/transformers_cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2020 The HuggingFace Team. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from argparse import ArgumentParser 17 | 18 | from .add_new_model import AddNewModelCommand 19 | from .add_new_model_like import AddNewModelLikeCommand 20 | from .convert import ConvertCommand 21 | from .download import DownloadCommand 22 | from .env import EnvironmentCommand 23 | from .lfs import LfsCommands 24 | from .pt_to_tf import PTtoTFCommand 25 | from .run import RunCommand 26 | from .serving import ServeCommand 27 | from .user import UserCommands 28 | 29 | 30 | def main(): 31 | parser = ArgumentParser("Transformers CLI tool", usage="transformers-cli []") 32 | commands_parser = parser.add_subparsers(help="transformers-cli command helpers") 33 | 34 | # Register commands 35 | ConvertCommand.register_subcommand(commands_parser) 36 | DownloadCommand.register_subcommand(commands_parser) 37 | EnvironmentCommand.register_subcommand(commands_parser) 38 | RunCommand.register_subcommand(commands_parser) 39 | ServeCommand.register_subcommand(commands_parser) 40 | UserCommands.register_subcommand(commands_parser) 41 | AddNewModelCommand.register_subcommand(commands_parser) 42 | AddNewModelLikeCommand.register_subcommand(commands_parser) 43 | LfsCommands.register_subcommand(commands_parser) 44 | PTtoTFCommand.register_subcommand(commands_parser) 45 | 46 | # Let's go 47 | args = parser.parse_args() 48 | 49 | if not hasattr(args, "func"): 50 | parser.print_help() 51 | exit(1) 52 | 53 | # Run 54 | service = args.func(args) 55 | service.run() 56 | 57 | 58 | if __name__ == "__main__": 59 | main() 60 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/convert_slow_tokenizers_checkpoints_to_fast.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The HuggingFace Inc. team. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """ Convert slow tokenizers checkpoints in fast (serialization format of the `tokenizers` library)""" 16 | 17 | import argparse 18 | import os 19 | 20 | import transformers 21 | 22 | from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS 23 | from .utils import logging 24 | 25 | 26 | logging.set_verbosity_info() 27 | 28 | logger = logging.get_logger(__name__) 29 | 30 | 31 | TOKENIZER_CLASSES = {name: getattr(transformers, name + "Fast") for name in SLOW_TO_FAST_CONVERTERS} 32 | 33 | 34 | def convert_slow_checkpoint_to_fast(tokenizer_name, checkpoint_name, dump_path, force_download): 35 | if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES: 36 | raise ValueError(f"Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys())}.") 37 | 38 | if tokenizer_name is None: 39 | tokenizer_names = TOKENIZER_CLASSES 40 | else: 41 | tokenizer_names = {tokenizer_name: getattr(transformers, tokenizer_name + "Fast")} 42 | 43 | logger.info(f"Loading tokenizer classes: {tokenizer_names}") 44 | 45 | for tokenizer_name in tokenizer_names: 46 | tokenizer_class = TOKENIZER_CLASSES[tokenizer_name] 47 | 48 | add_prefix = True 49 | if checkpoint_name is None: 50 | checkpoint_names = list(tokenizer_class.max_model_input_sizes.keys()) 51 | else: 52 | checkpoint_names = [checkpoint_name] 53 | 54 | logger.info(f"For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}") 55 | 56 | for checkpoint in checkpoint_names: 57 | logger.info(f"Loading {tokenizer_class.__class__.__name__} {checkpoint}") 58 | 59 | # Load tokenizer 60 | tokenizer = tokenizer_class.from_pretrained(checkpoint, force_download=force_download) 61 | 62 | # Save fast tokenizer 63 | logger.info(f"Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}") 64 | 65 | # For organization names we create sub-directories 66 | if "/" in checkpoint: 67 | checkpoint_directory, checkpoint_prefix_name = checkpoint.split("/") 68 | dump_path_full = os.path.join(dump_path, checkpoint_directory) 69 | elif add_prefix: 70 | checkpoint_prefix_name = checkpoint 71 | dump_path_full = dump_path 72 | else: 73 | checkpoint_prefix_name = None 74 | dump_path_full = dump_path 75 | 76 | logger.info(f"=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}") 77 | 78 | if checkpoint in list(tokenizer.pretrained_vocab_files_map.values())[0]: 79 | file_path = list(tokenizer.pretrained_vocab_files_map.values())[0][checkpoint] 80 | next_char = file_path.split(checkpoint)[-1][0] 81 | if next_char == "/": 82 | dump_path_full = os.path.join(dump_path_full, checkpoint_prefix_name) 83 | checkpoint_prefix_name = None 84 | 85 | logger.info(f"=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}") 86 | 87 | file_names = tokenizer.save_pretrained( 88 | dump_path_full, legacy_format=False, filename_prefix=checkpoint_prefix_name 89 | ) 90 | logger.info(f"=> File names {file_names}") 91 | 92 | for file_name in file_names: 93 | if not file_name.endswith("tokenizer.json"): 94 | os.remove(file_name) 95 | logger.info(f"=> removing {file_name}") 96 | 97 | 98 | if __name__ == "__main__": 99 | parser = argparse.ArgumentParser() 100 | # Required parameters 101 | parser.add_argument( 102 | "--dump_path", default=None, type=str, required=True, help="Path to output generated fast tokenizer files." 103 | ) 104 | parser.add_argument( 105 | "--tokenizer_name", 106 | default=None, 107 | type=str, 108 | help=( 109 | f"Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will " 110 | "download and convert all the checkpoints from AWS." 111 | ), 112 | ) 113 | parser.add_argument( 114 | "--checkpoint_name", 115 | default=None, 116 | type=str, 117 | help="Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.", 118 | ) 119 | parser.add_argument( 120 | "--force_download", 121 | action="store_true", 122 | help="Re-download checkpoints.", 123 | ) 124 | args = parser.parse_args() 125 | 126 | convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download) 127 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/convert_tf_hub_seq_to_seq_bert_to_pytorch.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2020 The HuggingFace Inc. team. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Convert Seq2Seq TF Hub checkpoint.""" 16 | 17 | 18 | import argparse 19 | 20 | from . import ( 21 | BertConfig, 22 | BertGenerationConfig, 23 | BertGenerationDecoder, 24 | BertGenerationEncoder, 25 | load_tf_weights_in_bert_generation, 26 | logging, 27 | ) 28 | 29 | 30 | logging.set_verbosity_info() 31 | 32 | 33 | def convert_tf_checkpoint_to_pytorch(tf_hub_path, pytorch_dump_path, is_encoder_named_decoder, vocab_size, is_encoder): 34 | # Initialise PyTorch model 35 | bert_config = BertConfig.from_pretrained( 36 | "bert-large-cased", 37 | vocab_size=vocab_size, 38 | max_position_embeddings=512, 39 | is_decoder=True, 40 | add_cross_attention=True, 41 | ) 42 | bert_config_dict = bert_config.to_dict() 43 | del bert_config_dict["type_vocab_size"] 44 | config = BertGenerationConfig(**bert_config_dict) 45 | if is_encoder: 46 | model = BertGenerationEncoder(config) 47 | else: 48 | model = BertGenerationDecoder(config) 49 | print(f"Building PyTorch model from configuration: {config}") 50 | 51 | # Load weights from tf checkpoint 52 | load_tf_weights_in_bert_generation( 53 | model, 54 | tf_hub_path, 55 | model_class="bert", 56 | is_encoder_named_decoder=is_encoder_named_decoder, 57 | is_encoder=is_encoder, 58 | ) 59 | 60 | # Save pytorch-model 61 | print(f"Save PyTorch model and config to {pytorch_dump_path}") 62 | model.save_pretrained(pytorch_dump_path) 63 | 64 | 65 | if __name__ == "__main__": 66 | parser = argparse.ArgumentParser() 67 | # Required parameters 68 | parser.add_argument( 69 | "--tf_hub_path", default=None, type=str, required=True, help="Path to the TensorFlow checkpoint path." 70 | ) 71 | parser.add_argument( 72 | "--pytorch_dump_path", default=None, type=str, required=True, help="Path to the output PyTorch model." 73 | ) 74 | parser.add_argument( 75 | "--is_encoder_named_decoder", 76 | action="store_true", 77 | help="If decoder has to be renamed to encoder in PyTorch model.", 78 | ) 79 | parser.add_argument("--is_encoder", action="store_true", help="If model is an encoder.") 80 | parser.add_argument("--vocab_size", default=50358, type=int, help="Vocab size of model") 81 | args = parser.parse_args() 82 | convert_tf_checkpoint_to_pytorch( 83 | args.tf_hub_path, 84 | args.pytorch_dump_path, 85 | args.is_encoder_named_decoder, 86 | args.vocab_size, 87 | is_encoder=args.is_encoder, 88 | ) 89 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/data/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .data_collator import ( 16 | DataCollatorForLanguageModeling, 17 | DataCollatorForPermutationLanguageModeling, 18 | DataCollatorForSeq2Seq, 19 | DataCollatorForSOP, 20 | DataCollatorForTokenClassification, 21 | DataCollatorForWholeWordMask, 22 | DataCollatorWithPadding, 23 | DefaultDataCollator, 24 | default_data_collator, 25 | ) 26 | from .metrics import glue_compute_metrics, xnli_compute_metrics 27 | from .processors import ( 28 | DataProcessor, 29 | InputExample, 30 | InputFeatures, 31 | SingleSentenceClassificationProcessor, 32 | SquadExample, 33 | SquadFeatures, 34 | SquadV1Processor, 35 | SquadV2Processor, 36 | glue_convert_examples_to_features, 37 | glue_output_modes, 38 | glue_processors, 39 | glue_tasks_num_labels, 40 | squad_convert_examples_to_features, 41 | xnli_output_modes, 42 | xnli_processors, 43 | xnli_tasks_num_labels, 44 | ) 45 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/data/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .glue import GlueDataset, GlueDataTrainingArguments 16 | from .language_modeling import ( 17 | LineByLineTextDataset, 18 | LineByLineWithRefDataset, 19 | LineByLineWithSOPTextDataset, 20 | TextDataset, 21 | TextDatasetForNextSentencePrediction, 22 | ) 23 | from .squad import SquadDataset, SquadDataTrainingArguments 24 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/data/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | import warnings 14 | 15 | from ...utils import is_sklearn_available, requires_backends 16 | 17 | 18 | if is_sklearn_available(): 19 | from scipy.stats import pearsonr, spearmanr 20 | from sklearn.metrics import f1_score, matthews_corrcoef 21 | 22 | 23 | DEPRECATION_WARNING = ( 24 | "This metric will be removed from the library soon, metrics should be handled with the 🤗 Evaluate " 25 | "library. You can have a look at this example script for pointers: " 26 | "https://github.com/huggingface/transformers/blob/main/examples/pytorch/text-classification/run_glue.py" 27 | ) 28 | 29 | 30 | def simple_accuracy(preds, labels): 31 | warnings.warn(DEPRECATION_WARNING, FutureWarning) 32 | requires_backends(simple_accuracy, "sklearn") 33 | return (preds == labels).mean() 34 | 35 | 36 | def acc_and_f1(preds, labels): 37 | warnings.warn(DEPRECATION_WARNING, FutureWarning) 38 | requires_backends(acc_and_f1, "sklearn") 39 | acc = simple_accuracy(preds, labels) 40 | f1 = f1_score(y_true=labels, y_pred=preds) 41 | return { 42 | "acc": acc, 43 | "f1": f1, 44 | "acc_and_f1": (acc + f1) / 2, 45 | } 46 | 47 | 48 | def pearson_and_spearman(preds, labels): 49 | warnings.warn(DEPRECATION_WARNING, FutureWarning) 50 | requires_backends(pearson_and_spearman, "sklearn") 51 | pearson_corr = pearsonr(preds, labels)[0] 52 | spearman_corr = spearmanr(preds, labels)[0] 53 | return { 54 | "pearson": pearson_corr, 55 | "spearmanr": spearman_corr, 56 | "corr": (pearson_corr + spearman_corr) / 2, 57 | } 58 | 59 | 60 | def glue_compute_metrics(task_name, preds, labels): 61 | warnings.warn(DEPRECATION_WARNING, FutureWarning) 62 | requires_backends(glue_compute_metrics, "sklearn") 63 | assert len(preds) == len(labels), f"Predictions and labels have mismatched lengths {len(preds)} and {len(labels)}" 64 | if task_name == "cola": 65 | return {"mcc": matthews_corrcoef(labels, preds)} 66 | elif task_name == "sst-2": 67 | return {"acc": simple_accuracy(preds, labels)} 68 | elif task_name == "mrpc": 69 | return acc_and_f1(preds, labels) 70 | elif task_name == "sts-b": 71 | return pearson_and_spearman(preds, labels) 72 | elif task_name == "qqp": 73 | return acc_and_f1(preds, labels) 74 | elif task_name == "mnli": 75 | return {"mnli/acc": simple_accuracy(preds, labels)} 76 | elif task_name == "mnli-mm": 77 | return {"mnli-mm/acc": simple_accuracy(preds, labels)} 78 | elif task_name == "qnli": 79 | return {"acc": simple_accuracy(preds, labels)} 80 | elif task_name == "rte": 81 | return {"acc": simple_accuracy(preds, labels)} 82 | elif task_name == "wnli": 83 | return {"acc": simple_accuracy(preds, labels)} 84 | elif task_name == "hans": 85 | return {"acc": simple_accuracy(preds, labels)} 86 | else: 87 | raise KeyError(task_name) 88 | 89 | 90 | def xnli_compute_metrics(task_name, preds, labels): 91 | warnings.warn(DEPRECATION_WARNING, FutureWarning) 92 | requires_backends(xnli_compute_metrics, "sklearn") 93 | if len(preds) != len(labels): 94 | raise ValueError(f"Predictions and labels have mismatched lengths {len(preds)} and {len(labels)}") 95 | if task_name == "xnli": 96 | return {"acc": simple_accuracy(preds, labels)} 97 | else: 98 | raise KeyError(task_name) 99 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/data/processors/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .glue import glue_convert_examples_to_features, glue_output_modes, glue_processors, glue_tasks_num_labels 16 | from .squad import SquadExample, SquadFeatures, SquadV1Processor, SquadV2Processor, squad_convert_examples_to_features 17 | from .utils import DataProcessor, InputExample, InputFeatures, SingleSentenceClassificationProcessor 18 | from .xnli import xnli_output_modes, xnli_processors, xnli_tasks_num_labels 19 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/data/processors/xnli.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. 3 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | """ XNLI utils (dataset loading and evaluation)""" 17 | 18 | 19 | import os 20 | 21 | from ...utils import logging 22 | from .utils import DataProcessor, InputExample 23 | 24 | 25 | logger = logging.get_logger(__name__) 26 | 27 | 28 | class XnliProcessor(DataProcessor): 29 | """ 30 | Processor for the XNLI dataset. Adapted from 31 | https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/run_classifier.py#L207 32 | """ 33 | 34 | def __init__(self, language, train_language=None): 35 | self.language = language 36 | self.train_language = train_language 37 | 38 | def get_train_examples(self, data_dir): 39 | """See base class.""" 40 | lg = self.language if self.train_language is None else self.train_language 41 | lines = self._read_tsv(os.path.join(data_dir, f"XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv")) 42 | examples = [] 43 | for i, line in enumerate(lines): 44 | if i == 0: 45 | continue 46 | guid = f"train-{i}" 47 | text_a = line[0] 48 | text_b = line[1] 49 | label = "contradiction" if line[2] == "contradictory" else line[2] 50 | if not isinstance(text_a, str): 51 | raise ValueError(f"Training input {text_a} is not a string") 52 | if not isinstance(text_b, str): 53 | raise ValueError(f"Training input {text_b} is not a string") 54 | if not isinstance(label, str): 55 | raise ValueError(f"Training label {label} is not a string") 56 | examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) 57 | return examples 58 | 59 | def get_test_examples(self, data_dir): 60 | """See base class.""" 61 | lines = self._read_tsv(os.path.join(data_dir, "XNLI-1.0/xnli.test.tsv")) 62 | examples = [] 63 | for i, line in enumerate(lines): 64 | if i == 0: 65 | continue 66 | language = line[0] 67 | if language != self.language: 68 | continue 69 | guid = f"test-{i}" 70 | text_a = line[6] 71 | text_b = line[7] 72 | label = line[1] 73 | if not isinstance(text_a, str): 74 | raise ValueError(f"Training input {text_a} is not a string") 75 | if not isinstance(text_b, str): 76 | raise ValueError(f"Training input {text_b} is not a string") 77 | if not isinstance(label, str): 78 | raise ValueError(f"Training label {label} is not a string") 79 | examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) 80 | return examples 81 | 82 | def get_labels(self): 83 | """See base class.""" 84 | return ["contradiction", "entailment", "neutral"] 85 | 86 | 87 | xnli_processors = { 88 | "xnli": XnliProcessor, 89 | } 90 | 91 | xnli_output_modes = { 92 | "xnli": "classification", 93 | } 94 | 95 | xnli_tasks_num_labels = { 96 | "xnli": 3, 97 | } 98 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/dependency_versions_check.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .dependency_versions_table import deps 16 | from .utils.versions import require_version, require_version_core 17 | 18 | 19 | # define which module versions we always want to check at run time 20 | # (usually the ones defined in `install_requires` in setup.py) 21 | # 22 | # order specific notes: 23 | # - tqdm must be checked before tokenizers 24 | 25 | pkgs_to_check_at_runtime = [ 26 | "python", 27 | "tqdm", 28 | "regex", 29 | "requests", 30 | "packaging", 31 | "filelock", 32 | "numpy", 33 | "tokenizers", 34 | "huggingface-hub", 35 | "safetensors", 36 | "accelerate", 37 | "pyyaml", 38 | ] 39 | 40 | for pkg in pkgs_to_check_at_runtime: 41 | if pkg in deps: 42 | if pkg == "tokenizers": 43 | # must be loaded here, or else tqdm check may fail 44 | from .utils import is_tokenizers_available 45 | 46 | if not is_tokenizers_available(): 47 | continue # not required, check version only if installed 48 | elif pkg == "accelerate": 49 | # must be loaded here, or else tqdm check may fail 50 | from .utils import is_accelerate_available 51 | 52 | # Maybe switch to is_torch_available in the future here so that Accelerate is hard dep of 53 | # Transformers with PyTorch 54 | if not is_accelerate_available(): 55 | continue # not required, check version only if installed 56 | 57 | require_version_core(deps[pkg]) 58 | else: 59 | raise ValueError(f"can't find {pkg} in {deps.keys()}, check dependency_versions_table.py") 60 | 61 | 62 | def dep_version_check(pkg, hint=None): 63 | require_version(deps[pkg], hint) 64 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/dependency_versions_table.py: -------------------------------------------------------------------------------- 1 | # THIS FILE HAS BEEN AUTOGENERATED. To update: 2 | # 1. modify the `_deps` dict in setup.py 3 | # 2. run `make deps_table_update`` 4 | deps = { 5 | "Pillow": "Pillow<10.0.0", 6 | "accelerate": "accelerate>=0.20.3", 7 | "av": "av==9.2.0", 8 | "beautifulsoup4": "beautifulsoup4", 9 | "black": "black~=23.1", 10 | "codecarbon": "codecarbon==1.2.0", 11 | "cookiecutter": "cookiecutter==1.7.3", 12 | "dataclasses": "dataclasses", 13 | "datasets": "datasets!=2.5.0", 14 | "decord": "decord==0.6.0", 15 | "deepspeed": "deepspeed>=0.9.3", 16 | "diffusers": "diffusers", 17 | "dill": "dill<0.3.5", 18 | "evaluate": "evaluate>=0.2.0", 19 | "fairscale": "fairscale>0.3", 20 | "faiss-cpu": "faiss-cpu", 21 | "fastapi": "fastapi", 22 | "filelock": "filelock", 23 | "flax": "flax>=0.4.1,<=0.7.0", 24 | "ftfy": "ftfy", 25 | "fugashi": "fugashi>=1.0", 26 | "GitPython": "GitPython<3.1.19", 27 | "hf-doc-builder": "hf-doc-builder>=0.3.0", 28 | "huggingface-hub": "huggingface-hub>=0.14.1,<1.0", 29 | "importlib_metadata": "importlib_metadata", 30 | "ipadic": "ipadic>=1.0.0,<2.0", 31 | "isort": "isort>=5.5.4", 32 | "jax": "jax>=0.2.8,!=0.3.2,<=0.4.13", 33 | "jaxlib": "jaxlib>=0.1.65,<=0.4.13", 34 | "jieba": "jieba", 35 | "kenlm": "kenlm", 36 | "keras-nlp": "keras-nlp>=0.3.1", 37 | "librosa": "librosa", 38 | "nltk": "nltk", 39 | "natten": "natten>=0.14.6", 40 | "numpy": "numpy>=1.17", 41 | "onnxconverter-common": "onnxconverter-common", 42 | "onnxruntime-tools": "onnxruntime-tools>=1.4.2", 43 | "onnxruntime": "onnxruntime>=1.4.0", 44 | "opencv-python": "opencv-python", 45 | "optuna": "optuna", 46 | "optax": "optax>=0.0.8,<=0.1.4", 47 | "packaging": "packaging>=20.0", 48 | "parameterized": "parameterized", 49 | "phonemizer": "phonemizer", 50 | "protobuf": "protobuf", 51 | "psutil": "psutil", 52 | "pyyaml": "pyyaml>=5.1", 53 | "pydantic": "pydantic<2", 54 | "pytest": "pytest>=7.2.0", 55 | "pytest-timeout": "pytest-timeout", 56 | "pytest-xdist": "pytest-xdist", 57 | "python": "python>=3.8.0", 58 | "ray[tune]": "ray[tune]", 59 | "regex": "regex!=2019.12.17", 60 | "requests": "requests", 61 | "rhoknp": "rhoknp>=1.1.0,<1.3.1", 62 | "rjieba": "rjieba", 63 | "rouge-score": "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1", 64 | "ruff": "ruff>=0.0.241,<=0.0.259", 65 | "sacrebleu": "sacrebleu>=1.4.12,<2.0.0", 66 | "sacremoses": "sacremoses", 67 | "safetensors": "safetensors>=0.3.1", 68 | "sagemaker": "sagemaker>=2.31.0", 69 | "scikit-learn": "scikit-learn", 70 | "sentencepiece": "sentencepiece>=0.1.91,!=0.1.92", 71 | "sigopt": "sigopt", 72 | "starlette": "starlette", 73 | "sudachipy": "sudachipy>=0.6.6", 74 | "sudachidict_core": "sudachidict_core>=20220729", 75 | "tensorflow-cpu": "tensorflow-cpu>=2.6,<2.14", 76 | "tensorflow": "tensorflow>=2.6,<2.14", 77 | "tensorflow-text": "tensorflow-text<2.14", 78 | "tf2onnx": "tf2onnx", 79 | "timeout-decorator": "timeout-decorator", 80 | "timm": "timm", 81 | "tokenizers": "tokenizers>=0.11.1,!=0.11.3,<0.14", 82 | "torch": "torch>=1.9,!=1.12.0", 83 | "torchaudio": "torchaudio", 84 | "torchvision": "torchvision", 85 | "pyctcdecode": "pyctcdecode>=0.4.0", 86 | "tqdm": "tqdm>=4.27", 87 | "unidic": "unidic>=1.0.2", 88 | "unidic_lite": "unidic_lite>=1.0.7", 89 | "urllib3": "urllib3<2.0.0", 90 | "uvicorn": "uvicorn", 91 | } 92 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/file_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | File utilities: utilities related to download and cache models 16 | 17 | This module should not be update anymore and is only left for backward compatibility. 18 | """ 19 | 20 | from . import __version__ 21 | 22 | # Backward compatibility imports, to make sure all those objects can be found in file_utils 23 | from .utils import ( 24 | CLOUDFRONT_DISTRIB_PREFIX, 25 | CONFIG_NAME, 26 | DISABLE_TELEMETRY, 27 | DUMMY_INPUTS, 28 | DUMMY_MASK, 29 | ENV_VARS_TRUE_AND_AUTO_VALUES, 30 | ENV_VARS_TRUE_VALUES, 31 | FEATURE_EXTRACTOR_NAME, 32 | FLAX_WEIGHTS_NAME, 33 | HF_MODULES_CACHE, 34 | HUGGINGFACE_CO_PREFIX, 35 | HUGGINGFACE_CO_RESOLVE_ENDPOINT, 36 | MODEL_CARD_NAME, 37 | MULTIPLE_CHOICE_DUMMY_INPUTS, 38 | PYTORCH_PRETRAINED_BERT_CACHE, 39 | PYTORCH_TRANSFORMERS_CACHE, 40 | S3_BUCKET_PREFIX, 41 | SENTENCEPIECE_UNDERLINE, 42 | SPIECE_UNDERLINE, 43 | TF2_WEIGHTS_NAME, 44 | TF_WEIGHTS_NAME, 45 | TORCH_FX_REQUIRED_VERSION, 46 | TRANSFORMERS_CACHE, 47 | TRANSFORMERS_DYNAMIC_MODULE_NAME, 48 | USE_JAX, 49 | USE_TF, 50 | USE_TORCH, 51 | WEIGHTS_INDEX_NAME, 52 | WEIGHTS_NAME, 53 | ContextManagers, 54 | DummyObject, 55 | EntryNotFoundError, 56 | ExplicitEnum, 57 | ModelOutput, 58 | PaddingStrategy, 59 | PushToHubMixin, 60 | RepositoryNotFoundError, 61 | RevisionNotFoundError, 62 | TensorType, 63 | _LazyModule, 64 | add_code_sample_docstrings, 65 | add_end_docstrings, 66 | add_start_docstrings, 67 | add_start_docstrings_to_model_forward, 68 | cached_property, 69 | copy_func, 70 | default_cache_path, 71 | define_sagemaker_information, 72 | get_cached_models, 73 | get_file_from_repo, 74 | get_full_repo_name, 75 | get_torch_version, 76 | has_file, 77 | http_user_agent, 78 | is_apex_available, 79 | is_bs4_available, 80 | is_coloredlogs_available, 81 | is_datasets_available, 82 | is_detectron2_available, 83 | is_faiss_available, 84 | is_flax_available, 85 | is_ftfy_available, 86 | is_in_notebook, 87 | is_ipex_available, 88 | is_librosa_available, 89 | is_offline_mode, 90 | is_onnx_available, 91 | is_pandas_available, 92 | is_phonemizer_available, 93 | is_protobuf_available, 94 | is_psutil_available, 95 | is_py3nvml_available, 96 | is_pyctcdecode_available, 97 | is_pytesseract_available, 98 | is_pytorch_quantization_available, 99 | is_rjieba_available, 100 | is_sagemaker_dp_enabled, 101 | is_sagemaker_mp_enabled, 102 | is_scipy_available, 103 | is_sentencepiece_available, 104 | is_seqio_available, 105 | is_sklearn_available, 106 | is_soundfile_availble, 107 | is_spacy_available, 108 | is_speech_available, 109 | is_tensor, 110 | is_tensorflow_probability_available, 111 | is_tf2onnx_available, 112 | is_tf_available, 113 | is_timm_available, 114 | is_tokenizers_available, 115 | is_torch_available, 116 | is_torch_bf16_available, 117 | is_torch_cuda_available, 118 | is_torch_fx_available, 119 | is_torch_fx_proxy, 120 | is_torch_mps_available, 121 | is_torch_tf32_available, 122 | is_torch_tpu_available, 123 | is_torchaudio_available, 124 | is_training_run_on_sagemaker, 125 | is_vision_available, 126 | replace_return_docstrings, 127 | requires_backends, 128 | to_numpy, 129 | to_py_obj, 130 | torch_only_method, 131 | ) 132 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/generation_flax_utils.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2021 The Google AI Flax Team Authors, and The HuggingFace Inc. team. 3 | # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import warnings 18 | 19 | from .generation import FlaxGenerationMixin 20 | 21 | 22 | class FlaxGenerationMixin(FlaxGenerationMixin): 23 | # warning at import time 24 | warnings.warn( 25 | "Importing `FlaxGenerationMixin` from `src/transformers/generation_flax_utils.py` is deprecated and will " 26 | "be removed in Transformers v5. Import as `from transformers import FlaxGenerationMixin` instead.", 27 | FutureWarning, 28 | ) 29 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/generation_tf_utils.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. 3 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import warnings 18 | 19 | from .generation import TFGenerationMixin 20 | 21 | 22 | class TFGenerationMixin(TFGenerationMixin): 23 | # warning at import time 24 | warnings.warn( 25 | "Importing `TFGenerationMixin` from `src/transformers/generation_tf_utils.py` is deprecated and will " 26 | "be removed in Transformers v5. Import as `from transformers import TFGenerationMixin` instead.", 27 | FutureWarning, 28 | ) 29 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/generation_utils.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2020 The Google AI Language Team Authors, Facebook AI Research authors and The HuggingFace Inc. team. 3 | # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import warnings 18 | 19 | from .generation import GenerationMixin 20 | 21 | 22 | class GenerationMixin(GenerationMixin): 23 | # warning at import time 24 | warnings.warn( 25 | "Importing `GenerationMixin` from `src/transformers/generation_utils.py` is deprecated and will " 26 | "be removed in Transformers v5. Import as `from transformers import GenerationMixin` instead.", 27 | FutureWarning, 28 | ) 29 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/hyperparameter_search.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2023-present the HuggingFace Inc. team. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from .integrations import ( 17 | is_optuna_available, 18 | is_ray_available, 19 | is_sigopt_available, 20 | is_wandb_available, 21 | run_hp_search_optuna, 22 | run_hp_search_ray, 23 | run_hp_search_sigopt, 24 | run_hp_search_wandb, 25 | ) 26 | from .trainer_utils import ( 27 | HPSearchBackend, 28 | default_hp_space_optuna, 29 | default_hp_space_ray, 30 | default_hp_space_sigopt, 31 | default_hp_space_wandb, 32 | ) 33 | from .utils import logging 34 | 35 | 36 | logger = logging.get_logger(__name__) 37 | 38 | 39 | class HyperParamSearchBackendBase: 40 | name: str 41 | pip_package: str = None 42 | 43 | @staticmethod 44 | def is_available(): 45 | raise NotImplementedError 46 | 47 | def run(self, trainer, n_trials: int, direction: str, **kwargs): 48 | raise NotImplementedError 49 | 50 | def default_hp_space(self, trial): 51 | raise NotImplementedError 52 | 53 | def ensure_available(self): 54 | if not self.is_available(): 55 | raise RuntimeError( 56 | f"You picked the {self.name} backend, but it is not installed. Run {self.pip_install()}." 57 | ) 58 | 59 | @classmethod 60 | def pip_install(cls): 61 | return f"`pip install {cls.pip_package or cls.name}`" 62 | 63 | 64 | class OptunaBackend(HyperParamSearchBackendBase): 65 | name = "optuna" 66 | 67 | @staticmethod 68 | def is_available(): 69 | return is_optuna_available() 70 | 71 | def run(self, trainer, n_trials: int, direction: str, **kwargs): 72 | return run_hp_search_optuna(trainer, n_trials, direction, **kwargs) 73 | 74 | def default_hp_space(self, trial): 75 | return default_hp_space_optuna(trial) 76 | 77 | 78 | class RayTuneBackend(HyperParamSearchBackendBase): 79 | name = "ray" 80 | pip_package = "'ray[tune]'" 81 | 82 | @staticmethod 83 | def is_available(): 84 | return is_ray_available() 85 | 86 | def run(self, trainer, n_trials: int, direction: str, **kwargs): 87 | return run_hp_search_ray(trainer, n_trials, direction, **kwargs) 88 | 89 | def default_hp_space(self, trial): 90 | return default_hp_space_ray(trial) 91 | 92 | 93 | class SigOptBackend(HyperParamSearchBackendBase): 94 | name = "sigopt" 95 | 96 | @staticmethod 97 | def is_available(): 98 | return is_sigopt_available() 99 | 100 | def run(self, trainer, n_trials: int, direction: str, **kwargs): 101 | return run_hp_search_sigopt(trainer, n_trials, direction, **kwargs) 102 | 103 | def default_hp_space(self, trial): 104 | return default_hp_space_sigopt(trial) 105 | 106 | 107 | class WandbBackend(HyperParamSearchBackendBase): 108 | name = "wandb" 109 | 110 | @staticmethod 111 | def is_available(): 112 | return is_wandb_available() 113 | 114 | def run(self, trainer, n_trials: int, direction: str, **kwargs): 115 | return run_hp_search_wandb(trainer, n_trials, direction, **kwargs) 116 | 117 | def default_hp_space(self, trial): 118 | return default_hp_space_wandb(trial) 119 | 120 | 121 | ALL_HYPERPARAMETER_SEARCH_BACKENDS = { 122 | HPSearchBackend(backend.name): backend for backend in [OptunaBackend, RayTuneBackend, SigOptBackend, WandbBackend] 123 | } 124 | 125 | 126 | def default_hp_search_backend() -> str: 127 | available_backends = [backend for backend in ALL_HYPERPARAMETER_SEARCH_BACKENDS.values() if backend.is_available()] 128 | if len(available_backends) > 0: 129 | name = available_backends[0].name 130 | if len(available_backends) > 1: 131 | logger.info( 132 | f"{len(available_backends)} hyperparameter search backends available. Using {name} as the default." 133 | ) 134 | return name 135 | raise RuntimeError( 136 | "No hyperparameter search backend available.\n" 137 | + "\n".join( 138 | f" - To install {backend.name} run {backend.pip_install()}" 139 | for backend in ALL_HYPERPARAMETER_SEARCH_BACKENDS.values() 140 | ) 141 | ) 142 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/kernels/deformable_detr/cpu/ms_deform_attn_cpu.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | ************************************************************************************************** 3 | * Deformable DETR 4 | * Copyright (c) 2020 SenseTime. All Rights Reserved. 5 | * Licensed under the Apache License, Version 2.0 [see LICENSE for details] 6 | ************************************************************************************************** 7 | * Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 8 | ************************************************************************************************** 9 | */ 10 | 11 | #include 12 | 13 | #include 14 | #include 15 | 16 | 17 | at::Tensor 18 | ms_deform_attn_cpu_forward( 19 | const at::Tensor &value, 20 | const at::Tensor &spatial_shapes, 21 | const at::Tensor &level_start_index, 22 | const at::Tensor &sampling_loc, 23 | const at::Tensor &attn_weight, 24 | const int im2col_step) 25 | { 26 | AT_ERROR("Not implement on cpu"); 27 | } 28 | 29 | std::vector 30 | ms_deform_attn_cpu_backward( 31 | const at::Tensor &value, 32 | const at::Tensor &spatial_shapes, 33 | const at::Tensor &level_start_index, 34 | const at::Tensor &sampling_loc, 35 | const at::Tensor &attn_weight, 36 | const at::Tensor &grad_output, 37 | const int im2col_step) 38 | { 39 | AT_ERROR("Not implement on cpu"); 40 | } 41 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/kernels/deformable_detr/cpu/ms_deform_attn_cpu.h: -------------------------------------------------------------------------------- 1 | /*! 2 | ************************************************************************************************** 3 | * Deformable DETR 4 | * Copyright (c) 2020 SenseTime. All Rights Reserved. 5 | * Licensed under the Apache License, Version 2.0 [see LICENSE for details] 6 | ************************************************************************************************** 7 | * Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 8 | ************************************************************************************************** 9 | */ 10 | 11 | #pragma once 12 | #include 13 | 14 | at::Tensor 15 | ms_deform_attn_cpu_forward( 16 | const at::Tensor &value, 17 | const at::Tensor &spatial_shapes, 18 | const at::Tensor &level_start_index, 19 | const at::Tensor &sampling_loc, 20 | const at::Tensor &attn_weight, 21 | const int im2col_step); 22 | 23 | std::vector 24 | ms_deform_attn_cpu_backward( 25 | const at::Tensor &value, 26 | const at::Tensor &spatial_shapes, 27 | const at::Tensor &level_start_index, 28 | const at::Tensor &sampling_loc, 29 | const at::Tensor &attn_weight, 30 | const at::Tensor &grad_output, 31 | const int im2col_step); 32 | 33 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/kernels/deformable_detr/cuda/ms_deform_attn_cuda.h: -------------------------------------------------------------------------------- 1 | /*! 2 | ************************************************************************************************** 3 | * Deformable DETR 4 | * Copyright (c) 2020 SenseTime. All Rights Reserved. 5 | * Licensed under the Apache License, Version 2.0 [see LICENSE for details] 6 | ************************************************************************************************** 7 | * Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 8 | ************************************************************************************************** 9 | */ 10 | 11 | #pragma once 12 | #include 13 | 14 | at::Tensor ms_deform_attn_cuda_forward( 15 | const at::Tensor &value, 16 | const at::Tensor &spatial_shapes, 17 | const at::Tensor &level_start_index, 18 | const at::Tensor &sampling_loc, 19 | const at::Tensor &attn_weight, 20 | const int im2col_step); 21 | 22 | std::vector ms_deform_attn_cuda_backward( 23 | const at::Tensor &value, 24 | const at::Tensor &spatial_shapes, 25 | const at::Tensor &level_start_index, 26 | const at::Tensor &sampling_loc, 27 | const at::Tensor &attn_weight, 28 | const at::Tensor &grad_output, 29 | const int im2col_step); 30 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/kernels/deformable_detr/ms_deform_attn.h: -------------------------------------------------------------------------------- 1 | /*! 2 | ************************************************************************************************** 3 | * Deformable DETR 4 | * Copyright (c) 2020 SenseTime. All Rights Reserved. 5 | * Licensed under the Apache License, Version 2.0 [see LICENSE for details] 6 | ************************************************************************************************** 7 | * Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 8 | ************************************************************************************************** 9 | */ 10 | 11 | #pragma once 12 | 13 | #include "cpu/ms_deform_attn_cpu.h" 14 | 15 | #ifdef WITH_CUDA 16 | #include "cuda/ms_deform_attn_cuda.h" 17 | #endif 18 | 19 | 20 | at::Tensor 21 | ms_deform_attn_forward( 22 | const at::Tensor &value, 23 | const at::Tensor &spatial_shapes, 24 | const at::Tensor &level_start_index, 25 | const at::Tensor &sampling_loc, 26 | const at::Tensor &attn_weight, 27 | const int im2col_step) 28 | { 29 | if (value.type().is_cuda()) 30 | { 31 | #ifdef WITH_CUDA 32 | return ms_deform_attn_cuda_forward( 33 | value, spatial_shapes, level_start_index, sampling_loc, attn_weight, im2col_step); 34 | #else 35 | AT_ERROR("Not compiled with GPU support"); 36 | #endif 37 | } 38 | AT_ERROR("Not implemented on the CPU"); 39 | } 40 | 41 | std::vector 42 | ms_deform_attn_backward( 43 | const at::Tensor &value, 44 | const at::Tensor &spatial_shapes, 45 | const at::Tensor &level_start_index, 46 | const at::Tensor &sampling_loc, 47 | const at::Tensor &attn_weight, 48 | const at::Tensor &grad_output, 49 | const int im2col_step) 50 | { 51 | if (value.type().is_cuda()) 52 | { 53 | #ifdef WITH_CUDA 54 | return ms_deform_attn_cuda_backward( 55 | value, spatial_shapes, level_start_index, sampling_loc, attn_weight, grad_output, im2col_step); 56 | #else 57 | AT_ERROR("Not compiled with GPU support"); 58 | #endif 59 | } 60 | AT_ERROR("Not implemented on the CPU"); 61 | } 62 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/kernels/deformable_detr/vision.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | ************************************************************************************************** 3 | * Deformable DETR 4 | * Copyright (c) 2020 SenseTime. All Rights Reserved. 5 | * Licensed under the Apache License, Version 2.0 [see LICENSE for details] 6 | ************************************************************************************************** 7 | * Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 8 | ************************************************************************************************** 9 | */ 10 | 11 | #include "ms_deform_attn.h" 12 | 13 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 14 | m.def("ms_deform_attn_forward", &ms_deform_attn_forward, "ms_deform_attn_forward"); 15 | m.def("ms_deform_attn_backward", &ms_deform_attn_backward, "ms_deform_attn_backward"); 16 | } -------------------------------------------------------------------------------- /llm_toys/hf/transformers/kernels/mra/cuda_kernel.h: -------------------------------------------------------------------------------- 1 | 2 | #define WARP_SIZE 32 3 | #define FULL_MASK 0xffffffff 4 | #define OPTIMAL_THREADS 256 5 | 6 | __global__ void index_max_cuda_kernel( 7 | float *index_vals, // [batch_size, 32, num_block] 8 | int *indices, // [batch_size, num_block] 9 | float *max_vals, // [batch_size, A_num_block * 32] 10 | float *max_vals_scatter, // [batch_size, 32, num_block] 11 | long batch_size, 12 | long A_num_block, 13 | long B_num_block, 14 | long num_block 15 | ); 16 | 17 | __global__ void mm_to_sparse_cuda_kernel( 18 | float *dense_A, // [batch_size, A_num_block, dim, 32] 19 | float *dense_B, // [batch_size, B_num_block, dim, 32] 20 | int *indices, // [batch_size, num_block] 21 | float *sparse_C, // [batch_size, num_block, 32, 32] 22 | long batch_size, 23 | long A_num_block, 24 | long B_num_block, 25 | long dim, 26 | long num_block 27 | ); 28 | 29 | __global__ void sparse_dense_mm_cuda_kernel( 30 | float *sparse_A, // [batch_size, num_block, 32, 32] 31 | int *indices, // [batch_size, num_block] 32 | float *dense_B, // [batch_size, B_num_block, dim, 32] 33 | float *dense_C, // [batch_size, A_num_block, dim, 32] 34 | long batch_size, 35 | long A_num_block, 36 | long B_num_block, 37 | long dim, 38 | long num_block 39 | ); 40 | 41 | __global__ void reduce_sum_cuda_kernel( 42 | float *sparse_A, // [batch_size, num_block, 32, 32] 43 | int *indices, // [batch_size, num_block] 44 | float *dense_C, // [batch_size, A_num_block, 32] 45 | long batch_size, 46 | long A_num_block, 47 | long B_num_block, 48 | long num_block 49 | ); 50 | 51 | __global__ void scatter_cuda_kernel( 52 | float *dense_A, // [batch_size, A_num_block, 32] 53 | int *indices, // [batch_size, num_block] 54 | float *sparse_C, // [batch_size, num_block, 32, 32] 55 | long batch_size, 56 | long A_num_block, 57 | long B_num_block, 58 | long num_block 59 | ); 60 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/kernels/mra/cuda_launch.cu: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "cuda_launch.h" 4 | #include "cuda_kernel.h" 5 | #include 6 | 7 | ////////////////////////////////////////////////////////////////////////////////////////////////// 8 | ////////////////////////////////////////////////////////////////////////////////////////////////// 9 | 10 | std::vector index_max_kernel( 11 | at::Tensor index_vals, // [batch_size, 32, num_block] 12 | at::Tensor indices, // [batch_size, num_block], 13 | int A_num_block, 14 | int B_num_block 15 | ) { 16 | int batch_size = indices.size(0); 17 | int num_block = indices.size(1); 18 | 19 | at::Tensor max_vals = at::zeros({batch_size, A_num_block * 32}, index_vals.options()); 20 | at::Tensor max_vals_scatter = at::zeros({batch_size, 32, num_block}, index_vals.options()); 21 | 22 | dim3 threads(256); 23 | dim3 blocks(batch_size); 24 | int shared_mem = A_num_block * 32 * sizeof(float); 25 | 26 | index_max_cuda_kernel<<>>( 27 | index_vals.data_ptr(), 28 | indices.data_ptr(), 29 | max_vals.data_ptr(), 30 | max_vals_scatter.data_ptr(), 31 | batch_size, 32 | A_num_block, 33 | B_num_block, 34 | num_block 35 | ); 36 | 37 | return {max_vals, max_vals_scatter}; 38 | } 39 | 40 | at::Tensor mm_to_sparse_kernel( 41 | at::Tensor dense_A, // [batch_size, A_num_block, dim, 32] 42 | at::Tensor dense_B, // [batch_size, B_num_block, dim, 32] 43 | at::Tensor indices // [batch_size, num_block] 44 | ) { 45 | int batch_size = dense_A.size(0); 46 | int A_num_block = dense_A.size(1); 47 | int B_num_block = dense_B.size(1); 48 | int dim = dense_A.size(2); 49 | int num_block = indices.size(1); 50 | 51 | at::Tensor sparse_C = at::zeros({batch_size, num_block, 32, 32}, dense_A.options()); 52 | 53 | dim3 threads(64, 4); 54 | dim3 blocks(num_block / 4, batch_size); 55 | 56 | mm_to_sparse_cuda_kernel<<>>( 57 | dense_A.data_ptr(), 58 | dense_B.data_ptr(), 59 | indices.data_ptr(), 60 | sparse_C.data_ptr(), 61 | batch_size, 62 | A_num_block, 63 | B_num_block, 64 | dim, 65 | num_block 66 | ); 67 | 68 | return sparse_C; 69 | } 70 | 71 | at::Tensor sparse_dense_mm_kernel( 72 | at::Tensor sparse_A, // [batch_size, num_block, 32, 32] 73 | at::Tensor indices, // [batch_size, num_block] 74 | at::Tensor dense_B, // [batch_size, B_num_block, dim, 32] 75 | int A_num_block 76 | ) { 77 | int batch_size = sparse_A.size(0); 78 | int num_block = sparse_A.size(1); 79 | int B_num_block = dense_B.size(1); 80 | int dim = dense_B.size(2); 81 | 82 | at::Tensor dense_C = at::zeros({batch_size, A_num_block, dim, 32}, dense_B.options()); 83 | 84 | dim3 threads(128, 2); 85 | dim3 blocks(num_block / 2, batch_size); 86 | 87 | sparse_dense_mm_cuda_kernel<<>>( 88 | sparse_A.data_ptr(), 89 | indices.data_ptr(), 90 | dense_B.data_ptr(), 91 | dense_C.data_ptr(), 92 | batch_size, 93 | A_num_block, 94 | B_num_block, 95 | dim, 96 | num_block 97 | ); 98 | 99 | return dense_C; 100 | } 101 | 102 | at::Tensor reduce_sum_kernel( 103 | at::Tensor sparse_A, // [batch_size, num_block, 32, 32] 104 | at::Tensor indices, // [batch_size, num_block] 105 | int A_num_block, 106 | int B_num_block 107 | ) { 108 | int batch_size = sparse_A.size(0); 109 | int num_block = sparse_A.size(1); 110 | 111 | at::Tensor dense_C = at::zeros({batch_size, A_num_block, 32}, sparse_A.options()); 112 | 113 | dim3 threads(32, 4); 114 | dim3 blocks(num_block / 4, batch_size); 115 | 116 | reduce_sum_cuda_kernel<<>>( 117 | sparse_A.data_ptr(), 118 | indices.data_ptr(), 119 | dense_C.data_ptr(), 120 | batch_size, 121 | A_num_block, 122 | B_num_block, 123 | num_block 124 | ); 125 | 126 | return dense_C; 127 | } 128 | 129 | at::Tensor scatter_kernel( 130 | at::Tensor dense_A, // [batch_size, A_num_block, 32] 131 | at::Tensor indices, // [batch_size, num_block] 132 | int B_num_block 133 | ) { 134 | int batch_size = dense_A.size(0); 135 | int A_num_block = dense_A.size(1); 136 | int num_block = indices.size(1); 137 | 138 | at::Tensor sparse_C = at::zeros({batch_size, num_block, 32, 32}, dense_A.options()); 139 | 140 | dim3 threads(32, 4); 141 | dim3 blocks(num_block / 4, batch_size); 142 | 143 | scatter_cuda_kernel<<>>( 144 | dense_A.data_ptr(), 145 | indices.data_ptr(), 146 | sparse_C.data_ptr(), 147 | batch_size, 148 | A_num_block, 149 | B_num_block, 150 | num_block 151 | ); 152 | 153 | return sparse_C; 154 | } 155 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/kernels/mra/cuda_launch.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #define min(a, b) ((a)<(b)?(a):(b)) 6 | #define max(a, b) ((a)>(b)?(a):(b)) 7 | 8 | std::vector index_max_kernel( 9 | at::Tensor index_vals, 10 | at::Tensor indices, 11 | int A_num_block, 12 | int B_num_block 13 | ); 14 | 15 | at::Tensor mm_to_sparse_kernel( 16 | at::Tensor dense_A, 17 | at::Tensor dense_B, 18 | at::Tensor indices 19 | ); 20 | 21 | at::Tensor sparse_dense_mm_kernel( 22 | at::Tensor sparse_A, 23 | at::Tensor indices, 24 | at::Tensor dense_B, 25 | int A_num_block 26 | ); 27 | 28 | at::Tensor reduce_sum_kernel( 29 | at::Tensor sparse_A, 30 | at::Tensor indices, 31 | int A_num_block, 32 | int B_num_block 33 | ); 34 | 35 | at::Tensor scatter_kernel( 36 | at::Tensor dense_A, 37 | at::Tensor indices, 38 | int B_num_block 39 | ); 40 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/kernels/mra/torch_extension.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "cuda_launch.h" 4 | #include 5 | 6 | std::vector index_max( 7 | at::Tensor index_vals, 8 | at::Tensor indices, 9 | int A_num_block, 10 | int B_num_block 11 | ) { 12 | return index_max_kernel( 13 | index_vals, 14 | indices, 15 | A_num_block, 16 | B_num_block 17 | ); 18 | } 19 | 20 | at::Tensor mm_to_sparse( 21 | at::Tensor dense_A, 22 | at::Tensor dense_B, 23 | at::Tensor indices 24 | ) { 25 | return mm_to_sparse_kernel( 26 | dense_A, 27 | dense_B, 28 | indices 29 | ); 30 | } 31 | 32 | at::Tensor sparse_dense_mm( 33 | at::Tensor sparse_A, 34 | at::Tensor indices, 35 | at::Tensor dense_B, 36 | int A_num_block 37 | ) { 38 | return sparse_dense_mm_kernel( 39 | sparse_A, 40 | indices, 41 | dense_B, 42 | A_num_block 43 | ); 44 | } 45 | 46 | at::Tensor reduce_sum( 47 | at::Tensor sparse_A, 48 | at::Tensor indices, 49 | int A_num_block, 50 | int B_num_block 51 | ) { 52 | return reduce_sum_kernel( 53 | sparse_A, 54 | indices, 55 | A_num_block, 56 | B_num_block 57 | ); 58 | } 59 | 60 | at::Tensor scatter( 61 | at::Tensor dense_A, 62 | at::Tensor indices, 63 | int B_num_block 64 | ) { 65 | return scatter_kernel( 66 | dense_A, 67 | indices, 68 | B_num_block 69 | ); 70 | } 71 | 72 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 73 | m.def("index_max", &index_max, "index_max (CUDA)"); 74 | m.def("mm_to_sparse", &mm_to_sparse, "mm_to_sparse (CUDA)"); 75 | m.def("sparse_dense_mm", &sparse_dense_mm, "sparse_dense_mm (CUDA)"); 76 | m.def("reduce_sum", &reduce_sum, "reduce_sum (CUDA)"); 77 | m.def("scatter", &scatter, "scatter (CUDA)"); 78 | } 79 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/kernels/rwkv/wkv_op.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "ATen/ATen.h" 3 | typedef at::BFloat16 bf16; 4 | 5 | void cuda_forward(int B, int T, int C, float *w, float *u, float *k, float *v, float *y); 6 | void cuda_forward_bf16(int B, int T, int C, float *w, bf16 *u, bf16 *k, bf16 *v, bf16 *y); 7 | void cuda_forward_with_state(int B, int T, int C, float *w, float *u, float *k, float *v, float *y, float *s); 8 | void cuda_forward_with_state_bf16(int B, int T, int C, float *w, bf16 *u, bf16 *k, bf16 *v, bf16 *y, float *s); 9 | void cuda_backward(int B, int T, int C, float *w, float *u, float *k, float *v, float *y, float *gy, float *gw, float *gu, float *gk, float *gv); 10 | void cuda_backward_bf16(int B, int T, int C, float *w, bf16 *u, bf16 *k, bf16 *v, bf16 *y, bf16 *gy, bf16 *gw, bf16 *gu, bf16 *gk, bf16 *gv); 11 | 12 | void forward(torch::Tensor &w, torch::Tensor &u, torch::Tensor &k, torch::Tensor &v, torch::Tensor &y) { 13 | const int B = k.size(0); 14 | const int T = k.size(1); 15 | const int C = k.size(2); 16 | cuda_forward(B, T, C, w.data_ptr(), u.data_ptr(), k.data_ptr(), v.data_ptr(), y.data_ptr()); 17 | } 18 | void forward_bf16(torch::Tensor &w, torch::Tensor &u, torch::Tensor &k, torch::Tensor &v, torch::Tensor &y) { 19 | const int B = k.size(0); 20 | const int T = k.size(1); 21 | const int C = k.size(2); 22 | cuda_forward_bf16(B, T, C, w.data_ptr(), u.data_ptr(), k.data_ptr(), v.data_ptr(), y.data_ptr()); 23 | } 24 | void forward_with_state(torch::Tensor &w, torch::Tensor &u, torch::Tensor &k, torch::Tensor &v, torch::Tensor &y, torch::Tensor &s) { 25 | const int B = k.size(0); 26 | const int T = k.size(1); 27 | const int C = k.size(2); 28 | cuda_forward_with_state(B, T, C, w.data_ptr(), u.data_ptr(), k.data_ptr(), v.data_ptr(), y.data_ptr(), s.data_ptr()); 29 | } 30 | void forward_with_state_bf16(torch::Tensor &w, torch::Tensor &u, torch::Tensor &k, torch::Tensor &v, torch::Tensor &y, torch::Tensor &s) { 31 | const int B = k.size(0); 32 | const int T = k.size(1); 33 | const int C = k.size(2); 34 | cuda_forward_with_state_bf16(B, T, C, w.data_ptr(), u.data_ptr(), k.data_ptr(), v.data_ptr(), y.data_ptr(), s.data_ptr()); 35 | } 36 | void backward(torch::Tensor &w, torch::Tensor &u, torch::Tensor &k, torch::Tensor &v, torch::Tensor &y, torch::Tensor &gy, torch::Tensor &gw, torch::Tensor &gu, torch::Tensor &gk, torch::Tensor &gv) { 37 | const int B = k.size(0); 38 | const int T = k.size(1); 39 | const int C = k.size(2); 40 | cuda_backward(B, T, C, w.data_ptr(), u.data_ptr(), k.data_ptr(), v.data_ptr(), y.data_ptr(), gy.data_ptr(), gw.data_ptr(), gu.data_ptr(), gk.data_ptr(), gv.data_ptr()); 41 | } 42 | void backward_bf16(torch::Tensor &w, torch::Tensor &u, torch::Tensor &k, torch::Tensor &v, torch::Tensor &y, torch::Tensor &gy, torch::Tensor &gw, torch::Tensor &gu, torch::Tensor &gk, torch::Tensor &gv) { 43 | const int B = k.size(0); 44 | const int T = k.size(1); 45 | const int C = k.size(2); 46 | cuda_backward_bf16(B, T, C, w.data_ptr(), u.data_ptr(), k.data_ptr(), v.data_ptr(), y.data_ptr(), 47 | gy.data_ptr(), gw.data_ptr(), gu.data_ptr(), gk.data_ptr(), gv.data_ptr()); 48 | } 49 | 50 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 51 | m.def("forward", &forward, "wkv forward"); 52 | m.def("forward_bf16", &forward_bf16, "wkv forward bf16"); 53 | m.def("forward_with_state", &forward_with_state, "wkv forward with state"); 54 | m.def("forward_with_state_bf16", &forward_with_state_bf16, "wkv forward with state bf16"); 55 | m.def("backward", &backward, "wkv backward"); 56 | m.def("backward_bf16", &backward_bf16, "wkv backward bf16"); 57 | } 58 | 59 | TORCH_LIBRARY(wkv, m) { 60 | m.def("forward", forward); 61 | m.def("forward_bf16", forward_bf16); 62 | m.def("forward_with_state", forward_with_state); 63 | m.def("forward_with_state_bf16", forward_with_state_bf16); 64 | m.def("backward", backward); 65 | m.def("backward_bf16", backward_bf16); 66 | } 67 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/kernels/yoso/common.h: -------------------------------------------------------------------------------- 1 | 2 | #define min(a, b) ((a)<(b)?(a):(b)) 3 | #define max(a, b) ((a)>(b)?(a):(b)) 4 | #define ceil_divide(a, b) ((a)/(b)+((a)%(b)!=0)) 5 | #define select(cond, a, b) ((cond)?(a):(b)) 6 | #define PI 3.141592 7 | #define EPSILON 1e-8 8 | #define MAX_VAL 1e12 9 | #define MIN_VAL -1e12 10 | #define EMPTY_VALUE -1 11 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/kernels/yoso/common_cuda.h: -------------------------------------------------------------------------------- 1 | 2 | #define MAX_THREADS_PER_BLOCK 1024 3 | #define OPTIMAL_THREADS_PER_BLOCK 256 4 | #define WARP_SIZE 32 5 | #define MAX_NUM_BLOCK_X 2147483647 6 | #define MAX_NUM_BLOCK_Y 65535 7 | #define MAX_NUM_BLOCK_Z 65535 8 | #define MAX_SHARED_MEM_PER_BLOCK 48000 9 | #define FULL_MASK 0xffffffff 10 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/kernels/yoso/common_cuda_device.h: -------------------------------------------------------------------------------- 1 | 2 | #include "common.h" 3 | 4 | template 5 | __device__ int set_insert(T *set, int set_size, T value) { 6 | int slot = value % set_size; 7 | int start_slot = slot; 8 | while (true) { 9 | T prev = atomicCAS(&set[slot], EMPTY_VALUE, value); 10 | if (prev == EMPTY_VALUE || prev == value) { 11 | return slot; 12 | } 13 | slot = (slot + 1) % set_size; 14 | if (slot == start_slot) { 15 | return -1; 16 | } 17 | } 18 | return -1; 19 | } 20 | 21 | template 22 | __device__ int set_lookup(T *set, int set_size, T value) { 23 | int slot = value % set_size; 24 | int start_slot = slot; 25 | while (true) { 26 | if (set[slot] == value) { 27 | return slot; 28 | } 29 | slot = (slot + 1) % set_size; 30 | if (slot == start_slot) { 31 | return -1; 32 | } 33 | } 34 | return -1; 35 | } 36 | 37 | template 38 | __device__ void init_buffer(T init_value, T *buffer, int buffer_size, int num_threads, int thread_id) { 39 | __syncthreads(); 40 | for (int i = 0; i < buffer_size; i = i + num_threads) { 41 | int offset_idx = i + thread_id; 42 | if (offset_idx < buffer_size) { 43 | buffer[offset_idx] = init_value; 44 | } 45 | } 46 | __syncthreads(); 47 | } 48 | 49 | template 50 | __device__ void copy_data(T *src_pt, T *dist_pt, int data_length, int num_threads, int thread_id) { 51 | __syncthreads(); 52 | for (int i = 0; i < data_length; i = i + num_threads) { 53 | int offset_idx = i + thread_id; 54 | if (offset_idx < data_length) { 55 | dist_pt[offset_idx] = src_pt[offset_idx]; 56 | } 57 | } 58 | __syncthreads(); 59 | } 60 | 61 | template 62 | __device__ void init_buffer_nonblocking(T init_value, T *buffer, int buffer_size, int num_threads, int thread_id) { 63 | for (int i = 0; i < buffer_size; i = i + num_threads) { 64 | int offset_idx = i + thread_id; 65 | if (offset_idx < buffer_size) { 66 | buffer[offset_idx] = init_value; 67 | } 68 | } 69 | } 70 | 71 | template 72 | __device__ void copy_data_nonblocking(T *src_pt, T *dist_pt, int data_length, int num_threads, int thread_id) { 73 | for (int i = 0; i < data_length; i = i + num_threads) { 74 | int offset_idx = i + thread_id; 75 | if (offset_idx < data_length) { 76 | dist_pt[offset_idx] = src_pt[offset_idx]; 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/kernels/yoso/fast_lsh_cumulation.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | std::vector fast_hash_ver1_kernel( 6 | at::Tensor query_mask, 7 | at::Tensor query_vector, 8 | at::Tensor key_mask, 9 | at::Tensor key_vector, 10 | int num_hash_f, 11 | int hash_code_len, 12 | bool use_cuda 13 | ); 14 | 15 | at::Tensor lsh_cumulation_ver1_kernel( 16 | at::Tensor query_mask, 17 | at::Tensor query_hash_code, 18 | at::Tensor key_mask, 19 | at::Tensor key_hash_code, 20 | at::Tensor value, 21 | int hashtable_capacity, 22 | bool use_cuda 23 | ); 24 | 25 | at::Tensor lsh_weighted_cumulation_ver1_kernel( 26 | at::Tensor query_mask, 27 | at::Tensor query_hash_code, 28 | at::Tensor query_weight, 29 | at::Tensor key_mask, 30 | at::Tensor key_hash_code, 31 | at::Tensor key_weight, 32 | at::Tensor value, 33 | int hashtable_capacity, 34 | bool use_cuda 35 | ); 36 | 37 | at::Tensor lsh_weighted_cumulation_ver2_kernel( 38 | at::Tensor query_mask, 39 | at::Tensor query_hash_code, 40 | at::Tensor query_weight, 41 | at::Tensor key_mask, 42 | at::Tensor key_hash_code, 43 | at::Tensor key_weight, 44 | at::Tensor value, 45 | int hashtable_capacity, 46 | bool use_cuda 47 | ); 48 | 49 | at::Tensor lsh_weighted_cumulation_ver3_kernel( 50 | at::Tensor query_mask, 51 | at::Tensor query_hash_code, 52 | at::Tensor query_weight, 53 | at::Tensor key_mask, 54 | at::Tensor key_hash_code, 55 | at::Tensor key_weight, 56 | at::Tensor value, 57 | int hashtable_capacity, 58 | bool use_cuda 59 | ); 60 | 61 | at::Tensor lsh_weighted_cumulation_ver4_kernel( 62 | at::Tensor query_mask, 63 | at::Tensor query_hash_code, 64 | at::Tensor query_weight, 65 | at::Tensor key_mask, 66 | at::Tensor key_hash_code, 67 | at::Tensor key_weight, 68 | at::Tensor value, 69 | int hashtable_capacity, 70 | bool use_cuda 71 | ); 72 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/kernels/yoso/fast_lsh_cumulation_torch.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "fast_lsh_cumulation.h" 4 | #include "common_cuda.h" 5 | #include 6 | 7 | std::vector fast_hash( 8 | at::Tensor query_mask, 9 | at::Tensor query_vector, 10 | at::Tensor key_mask, 11 | at::Tensor key_vector, 12 | int num_hash_f, 13 | int hash_code_len, 14 | bool use_cuda, 15 | int version 16 | ) { 17 | return fast_hash_ver1_kernel( 18 | query_mask, 19 | query_vector, 20 | key_mask, 21 | key_vector, 22 | num_hash_f, 23 | hash_code_len, 24 | use_cuda 25 | ); 26 | } 27 | 28 | at::Tensor lsh_cumulation( 29 | at::Tensor query_mask, // [batch_size, num_query] 30 | at::Tensor query_hash_code, // [batch_size, num_query, num_hash_f] 31 | at::Tensor key_mask, // [batch_size, num_key] 32 | at::Tensor key_hash_code, // [batch_size, num_key, num_hash_f] 33 | at::Tensor value, // [batch_size, num_key, value_dim] 34 | int hashtable_capacity, 35 | bool use_cuda, 36 | int version 37 | ) { 38 | return lsh_cumulation_ver1_kernel( 39 | query_mask, 40 | query_hash_code, 41 | key_mask, 42 | key_hash_code, 43 | value, 44 | hashtable_capacity, 45 | use_cuda 46 | ); 47 | } 48 | 49 | at::Tensor lsh_weighted_cumulation( 50 | at::Tensor query_mask, // [batch_size, num_query] 51 | at::Tensor query_hash_code, // [batch_size, num_query, num_hash_f] 52 | at::Tensor query_weight, // [batch_size, num_query, weight_dim] 53 | at::Tensor key_mask, // [batch_size, num_key] 54 | at::Tensor key_hash_code, // [batch_size, num_key, num_hash_f] 55 | at::Tensor key_weight, // [batch_size, num_key, weight_dim] 56 | at::Tensor value, // [batch_size, num_key, value_dim] 57 | int hashtable_capacity, 58 | bool use_cuda, 59 | int version 60 | ) { 61 | if (version == 1) { 62 | return lsh_weighted_cumulation_ver1_kernel( 63 | query_mask, 64 | query_hash_code, 65 | query_weight, 66 | key_mask, 67 | key_hash_code, 68 | key_weight, 69 | value, 70 | hashtable_capacity, 71 | use_cuda 72 | ); 73 | } else if (version == 2) { 74 | return lsh_weighted_cumulation_ver2_kernel( 75 | query_mask, 76 | query_hash_code, 77 | query_weight, 78 | key_mask, 79 | key_hash_code, 80 | key_weight, 81 | value, 82 | hashtable_capacity, 83 | use_cuda 84 | ); 85 | } else if (version == 3) { 86 | return lsh_weighted_cumulation_ver3_kernel( 87 | query_mask, 88 | query_hash_code, 89 | query_weight, 90 | key_mask, 91 | key_hash_code, 92 | key_weight, 93 | value, 94 | hashtable_capacity, 95 | use_cuda 96 | ); 97 | } else if (version == 4) { 98 | return lsh_weighted_cumulation_ver4_kernel( 99 | query_mask, 100 | query_hash_code, 101 | query_weight, 102 | key_mask, 103 | key_hash_code, 104 | key_weight, 105 | value, 106 | hashtable_capacity, 107 | use_cuda 108 | ); 109 | } else { 110 | return lsh_weighted_cumulation_ver3_kernel( 111 | query_mask, 112 | query_hash_code, 113 | query_weight, 114 | key_mask, 115 | key_hash_code, 116 | key_weight, 117 | value, 118 | hashtable_capacity, 119 | use_cuda 120 | ); 121 | } 122 | } 123 | 124 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 125 | m.def("fast_hash", &fast_hash, "Fast Hash (CUDA)"); 126 | m.def("lsh_cumulation", &lsh_cumulation, "LSH Cumulation (CUDA)"); 127 | m.def("lsh_weighted_cumulation", &lsh_weighted_cumulation, "LSH Weighted Cumulation (CUDA)"); 128 | } 129 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/onnx/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import TYPE_CHECKING 16 | 17 | from ..utils import _LazyModule 18 | 19 | 20 | _import_structure = { 21 | "config": [ 22 | "EXTERNAL_DATA_FORMAT_SIZE_LIMIT", 23 | "OnnxConfig", 24 | "OnnxConfigWithPast", 25 | "OnnxSeq2SeqConfigWithPast", 26 | "PatchingSpec", 27 | ], 28 | "convert": ["export", "validate_model_outputs"], 29 | "features": ["FeaturesManager"], 30 | "utils": ["ParameterFormat", "compute_serialized_parameters_size"], 31 | } 32 | 33 | 34 | if TYPE_CHECKING: 35 | from .config import ( 36 | EXTERNAL_DATA_FORMAT_SIZE_LIMIT, 37 | OnnxConfig, 38 | OnnxConfigWithPast, 39 | OnnxSeq2SeqConfigWithPast, 40 | PatchingSpec, 41 | ) 42 | from .convert import export, validate_model_outputs 43 | from .features import FeaturesManager 44 | from .utils import ParameterFormat, compute_serialized_parameters_size 45 | 46 | else: 47 | import sys 48 | 49 | sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__) 50 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/onnx/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from ctypes import c_float, sizeof 16 | from enum import Enum 17 | from typing import TYPE_CHECKING, Optional, Union 18 | 19 | 20 | if TYPE_CHECKING: 21 | from .. import AutoFeatureExtractor, AutoProcessor, AutoTokenizer # tests_ignore 22 | 23 | 24 | class ParameterFormat(Enum): 25 | Float = c_float 26 | 27 | @property 28 | def size(self) -> int: 29 | """ 30 | Number of byte required for this data type 31 | 32 | Returns: 33 | Integer > 0 34 | """ 35 | return sizeof(self.value) 36 | 37 | 38 | def compute_effective_axis_dimension(dimension: int, fixed_dimension: int, num_token_to_add: int = 0) -> int: 39 | """ 40 | 41 | Args: 42 | dimension: 43 | fixed_dimension: 44 | num_token_to_add: 45 | 46 | Returns: 47 | 48 | """ 49 | # < 0 is possible if using a dynamic axis 50 | if dimension <= 0: 51 | dimension = fixed_dimension 52 | 53 | dimension -= num_token_to_add 54 | return dimension 55 | 56 | 57 | def compute_serialized_parameters_size(num_parameters: int, dtype: ParameterFormat) -> int: 58 | """ 59 | Compute the size taken by all the parameters in the given the storage format when serializing the model 60 | 61 | Args: 62 | num_parameters: Number of parameters to be saved 63 | dtype: The data format each parameter will be saved 64 | 65 | Returns: 66 | Size (in byte) taken to save all the parameters 67 | """ 68 | return num_parameters * dtype.size 69 | 70 | 71 | def get_preprocessor(model_name: str) -> Optional[Union["AutoTokenizer", "AutoFeatureExtractor", "AutoProcessor"]]: 72 | """ 73 | Gets a preprocessor (tokenizer, feature extractor or processor) that is available for `model_name`. 74 | 75 | Args: 76 | model_name (`str`): Name of the model for which a preprocessor are loaded. 77 | 78 | Returns: 79 | `Optional[Union[AutoTokenizer, AutoFeatureExtractor, AutoProcessor]]`: 80 | If a processor is found, it is returned. Otherwise, if a tokenizer or a feature extractor exists, it is 81 | returned. If both a tokenizer and a feature extractor exist, an error is raised. The function returns 82 | `None` if no preprocessor is found. 83 | """ 84 | # Avoid circular imports by only importing this here. 85 | from .. import AutoFeatureExtractor, AutoProcessor, AutoTokenizer # tests_ignore 86 | 87 | try: 88 | return AutoProcessor.from_pretrained(model_name) 89 | except (ValueError, OSError, KeyError): 90 | tokenizer, feature_extractor = None, None 91 | try: 92 | tokenizer = AutoTokenizer.from_pretrained(model_name) 93 | except (OSError, KeyError): 94 | pass 95 | try: 96 | feature_extractor = AutoFeatureExtractor.from_pretrained(model_name) 97 | except (OSError, KeyError): 98 | pass 99 | 100 | if tokenizer is not None and feature_extractor is not None: 101 | raise ValueError( 102 | f"Couldn't auto-detect preprocessor for {model_name}. Found both a tokenizer and a feature extractor." 103 | ) 104 | elif tokenizer is None and feature_extractor is None: 105 | return None 106 | elif tokenizer is not None: 107 | return tokenizer 108 | else: 109 | return feature_extractor 110 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/pipelines/depth_estimation.py: -------------------------------------------------------------------------------- 1 | from typing import List, Union 2 | 3 | import numpy as np 4 | 5 | from ..utils import add_end_docstrings, is_torch_available, is_vision_available, logging, requires_backends 6 | from .base import PIPELINE_INIT_ARGS, Pipeline 7 | 8 | 9 | if is_vision_available(): 10 | from PIL import Image 11 | 12 | from ..image_utils import load_image 13 | 14 | if is_torch_available(): 15 | import torch 16 | 17 | from ..models.auto.modeling_auto import MODEL_FOR_DEPTH_ESTIMATION_MAPPING 18 | 19 | logger = logging.get_logger(__name__) 20 | 21 | 22 | @add_end_docstrings(PIPELINE_INIT_ARGS) 23 | class DepthEstimationPipeline(Pipeline): 24 | """ 25 | Depth estimation pipeline using any `AutoModelForDepthEstimation`. This pipeline predicts the depth of an image. 26 | 27 | Example: 28 | 29 | ```python 30 | >>> from transformers import pipeline 31 | 32 | >>> depth_estimator = pipeline(task="depth-estimation", model="Intel/dpt-large") 33 | >>> output = depth_estimator("http://images.cocodataset.org/val2017/000000039769.jpg") 34 | >>> # This is a tensor with the values being the depth expressed in meters for each pixel 35 | >>> output["predicted_depth"].shape 36 | torch.Size([1, 384, 384]) 37 | ``` 38 | 39 | Learn more about the basics of using a pipeline in the [pipeline tutorial](../pipeline_tutorial) 40 | 41 | 42 | This depth estimation pipeline can currently be loaded from [`pipeline`] using the following task identifier: 43 | `"depth-estimation"`. 44 | 45 | See the list of available models on [huggingface.co/models](https://huggingface.co/models?filter=depth-estimation). 46 | """ 47 | 48 | def __init__(self, *args, **kwargs): 49 | super().__init__(*args, **kwargs) 50 | requires_backends(self, "vision") 51 | self.check_model_type(MODEL_FOR_DEPTH_ESTIMATION_MAPPING) 52 | 53 | def __call__(self, images: Union[str, List[str], "Image.Image", List["Image.Image"]], **kwargs): 54 | """ 55 | Assign labels to the image(s) passed as inputs. 56 | 57 | Args: 58 | images (`str`, `List[str]`, `PIL.Image` or `List[PIL.Image]`): 59 | The pipeline handles three types of images: 60 | 61 | - A string containing a http link pointing to an image 62 | - A string containing a local path to an image 63 | - An image loaded in PIL directly 64 | 65 | The pipeline accepts either a single image or a batch of images, which must then be passed as a string. 66 | Images in a batch must all be in the same format: all as http links, all as local paths, or all as PIL 67 | images. 68 | top_k (`int`, *optional*, defaults to 5): 69 | The number of top labels that will be returned by the pipeline. If the provided number is higher than 70 | the number of labels available in the model configuration, it will default to the number of labels. 71 | 72 | Return: 73 | A dictionary or a list of dictionaries containing result. If the input is a single image, will return a 74 | dictionary, if the input is a list of several images, will return a list of dictionaries corresponding to 75 | the images. 76 | 77 | The dictionaries contain the following keys: 78 | 79 | - **label** (`str`) -- The label identified by the model. 80 | - **score** (`int`) -- The score attributed by the model for that label. 81 | """ 82 | return super().__call__(images, **kwargs) 83 | 84 | def _sanitize_parameters(self, **kwargs): 85 | return {}, {}, {} 86 | 87 | def preprocess(self, image): 88 | image = load_image(image) 89 | self.image_size = image.size 90 | model_inputs = self.image_processor(images=image, return_tensors=self.framework) 91 | return model_inputs 92 | 93 | def _forward(self, model_inputs): 94 | model_outputs = self.model(**model_inputs) 95 | return model_outputs 96 | 97 | def postprocess(self, model_outputs): 98 | predicted_depth = model_outputs.predicted_depth 99 | prediction = torch.nn.functional.interpolate( 100 | predicted_depth.unsqueeze(1), size=self.image_size[::-1], mode="bicubic", align_corners=False 101 | ) 102 | output = prediction.squeeze().cpu().numpy() 103 | formatted = (output * 255 / np.max(output)).astype("uint8") 104 | depth = Image.fromarray(formatted) 105 | output_dict = {} 106 | output_dict["predicted_depth"] = predicted_depth 107 | output_dict["depth"] = depth 108 | return output_dict 109 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/pipelines/feature_extraction.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | from .base import GenericTensor, Pipeline 4 | 5 | 6 | # Can't use @add_end_docstrings(PIPELINE_INIT_ARGS) here because this one does not accept `binary_output` 7 | class FeatureExtractionPipeline(Pipeline): 8 | """ 9 | Feature extraction pipeline using no model head. This pipeline extracts the hidden states from the base 10 | transformer, which can be used as features in downstream tasks. 11 | 12 | Example: 13 | 14 | ```python 15 | >>> from transformers import pipeline 16 | 17 | >>> extractor = pipeline(model="bert-base-uncased", task="feature-extraction") 18 | >>> result = extractor("This is a simple test.", return_tensors=True) 19 | >>> result.shape # This is a tensor of shape [1, sequence_lenth, hidden_dimension] representing the input string. 20 | torch.Size([1, 8, 768]) 21 | ``` 22 | 23 | Learn more about the basics of using a pipeline in the [pipeline tutorial](../pipeline_tutorial) 24 | 25 | This feature extraction pipeline can currently be loaded from [`pipeline`] using the task identifier: 26 | `"feature-extraction"`. 27 | 28 | All models may be used for this pipeline. See a list of all models, including community-contributed models on 29 | [huggingface.co/models](https://huggingface.co/models). 30 | 31 | Arguments: 32 | model ([`PreTrainedModel`] or [`TFPreTrainedModel`]): 33 | The model that will be used by the pipeline to make predictions. This needs to be a model inheriting from 34 | [`PreTrainedModel`] for PyTorch and [`TFPreTrainedModel`] for TensorFlow. 35 | tokenizer ([`PreTrainedTokenizer`]): 36 | The tokenizer that will be used by the pipeline to encode data for the model. This object inherits from 37 | [`PreTrainedTokenizer`]. 38 | modelcard (`str` or [`ModelCard`], *optional*): 39 | Model card attributed to the model for this pipeline. 40 | framework (`str`, *optional*): 41 | The framework to use, either `"pt"` for PyTorch or `"tf"` for TensorFlow. The specified framework must be 42 | installed. 43 | 44 | If no framework is specified, will default to the one currently installed. If no framework is specified and 45 | both frameworks are installed, will default to the framework of the `model`, or to PyTorch if no model is 46 | provided. 47 | return_tensors (`bool`, *optional*): 48 | If `True`, returns a tensor according to the specified framework, otherwise returns a list. 49 | task (`str`, defaults to `""`): 50 | A task-identifier for the pipeline. 51 | args_parser ([`~pipelines.ArgumentHandler`], *optional*): 52 | Reference to the object in charge of parsing supplied pipeline parameters. 53 | device (`int`, *optional*, defaults to -1): 54 | Device ordinal for CPU/GPU supports. Setting this to -1 will leverage CPU, a positive will run the model on 55 | the associated CUDA device id. 56 | tokenize_kwargs (`dict`, *optional*): 57 | Additional dictionary of keyword arguments passed along to the tokenizer. 58 | """ 59 | 60 | def _sanitize_parameters(self, truncation=None, tokenize_kwargs=None, return_tensors=None, **kwargs): 61 | if tokenize_kwargs is None: 62 | tokenize_kwargs = {} 63 | 64 | if truncation is not None: 65 | if "truncation" in tokenize_kwargs: 66 | raise ValueError( 67 | "truncation parameter defined twice (given as keyword argument as well as in tokenize_kwargs)" 68 | ) 69 | tokenize_kwargs["truncation"] = truncation 70 | 71 | preprocess_params = tokenize_kwargs 72 | 73 | postprocess_params = {} 74 | if return_tensors is not None: 75 | postprocess_params["return_tensors"] = return_tensors 76 | 77 | return preprocess_params, {}, postprocess_params 78 | 79 | def preprocess(self, inputs, **tokenize_kwargs) -> Dict[str, GenericTensor]: 80 | return_tensors = self.framework 81 | model_inputs = self.tokenizer(inputs, return_tensors=return_tensors, **tokenize_kwargs) 82 | return model_inputs 83 | 84 | def _forward(self, model_inputs): 85 | model_outputs = self.model(**model_inputs) 86 | return model_outputs 87 | 88 | def postprocess(self, model_outputs, return_tensors=False): 89 | # [0] is the first available tensor, logits or last_hidden_state. 90 | if return_tensors: 91 | return model_outputs[0] 92 | if self.framework == "pt": 93 | return model_outputs[0].tolist() 94 | elif self.framework == "tf": 95 | return model_outputs[0].numpy().tolist() 96 | 97 | def __call__(self, *args, **kwargs): 98 | """ 99 | Extract the features of the input(s). 100 | 101 | Args: 102 | args (`str` or `List[str]`): One or several texts (or one list of texts) to get the features of. 103 | 104 | Return: 105 | A nested list of `float`: The features computed by the model. 106 | """ 107 | return super().__call__(*args, **kwargs) 108 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/pipelines/image_classification.py: -------------------------------------------------------------------------------- 1 | from typing import List, Union 2 | 3 | from ..utils import ( 4 | add_end_docstrings, 5 | is_tf_available, 6 | is_torch_available, 7 | is_vision_available, 8 | logging, 9 | requires_backends, 10 | ) 11 | from .base import PIPELINE_INIT_ARGS, Pipeline 12 | 13 | 14 | if is_vision_available(): 15 | from PIL import Image 16 | 17 | from ..image_utils import load_image 18 | 19 | if is_tf_available(): 20 | import tensorflow as tf 21 | 22 | from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING 23 | from ..tf_utils import stable_softmax 24 | 25 | if is_torch_available(): 26 | from ..models.auto.modeling_auto import MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING 27 | 28 | logger = logging.get_logger(__name__) 29 | 30 | 31 | @add_end_docstrings(PIPELINE_INIT_ARGS) 32 | class ImageClassificationPipeline(Pipeline): 33 | """ 34 | Image classification pipeline using any `AutoModelForImageClassification`. This pipeline predicts the class of an 35 | image. 36 | 37 | Example: 38 | 39 | ```python 40 | >>> from transformers import pipeline 41 | 42 | >>> classifier = pipeline(model="microsoft/beit-base-patch16-224-pt22k-ft22k") 43 | >>> classifier("https://huggingface.co/datasets/Narsil/image_dummy/raw/main/parrots.png") 44 | [{'score': 0.442, 'label': 'macaw'}, {'score': 0.088, 'label': 'popinjay'}, {'score': 0.075, 'label': 'parrot'}, {'score': 0.073, 'label': 'parodist, lampooner'}, {'score': 0.046, 'label': 'poll, poll_parrot'}] 45 | ``` 46 | 47 | Learn more about the basics of using a pipeline in the [pipeline tutorial](../pipeline_tutorial) 48 | 49 | This image classification pipeline can currently be loaded from [`pipeline`] using the following task identifier: 50 | `"image-classification"`. 51 | 52 | See the list of available models on 53 | [huggingface.co/models](https://huggingface.co/models?filter=image-classification). 54 | """ 55 | 56 | def __init__(self, *args, **kwargs): 57 | super().__init__(*args, **kwargs) 58 | requires_backends(self, "vision") 59 | self.check_model_type( 60 | TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING 61 | if self.framework == "tf" 62 | else MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING 63 | ) 64 | 65 | def _sanitize_parameters(self, top_k=None): 66 | postprocess_params = {} 67 | if top_k is not None: 68 | postprocess_params["top_k"] = top_k 69 | return {}, {}, postprocess_params 70 | 71 | def __call__(self, images: Union[str, List[str], "Image.Image", List["Image.Image"]], **kwargs): 72 | """ 73 | Assign labels to the image(s) passed as inputs. 74 | 75 | Args: 76 | images (`str`, `List[str]`, `PIL.Image` or `List[PIL.Image]`): 77 | The pipeline handles three types of images: 78 | 79 | - A string containing a http link pointing to an image 80 | - A string containing a local path to an image 81 | - An image loaded in PIL directly 82 | 83 | The pipeline accepts either a single image or a batch of images, which must then be passed as a string. 84 | Images in a batch must all be in the same format: all as http links, all as local paths, or all as PIL 85 | images. 86 | top_k (`int`, *optional*, defaults to 5): 87 | The number of top labels that will be returned by the pipeline. If the provided number is higher than 88 | the number of labels available in the model configuration, it will default to the number of labels. 89 | 90 | Return: 91 | A dictionary or a list of dictionaries containing result. If the input is a single image, will return a 92 | dictionary, if the input is a list of several images, will return a list of dictionaries corresponding to 93 | the images. 94 | 95 | The dictionaries contain the following keys: 96 | 97 | - **label** (`str`) -- The label identified by the model. 98 | - **score** (`int`) -- The score attributed by the model for that label. 99 | """ 100 | return super().__call__(images, **kwargs) 101 | 102 | def preprocess(self, image): 103 | image = load_image(image) 104 | model_inputs = self.image_processor(images=image, return_tensors=self.framework) 105 | return model_inputs 106 | 107 | def _forward(self, model_inputs): 108 | model_outputs = self.model(**model_inputs) 109 | return model_outputs 110 | 111 | def postprocess(self, model_outputs, top_k=5): 112 | if top_k > self.model.config.num_labels: 113 | top_k = self.model.config.num_labels 114 | 115 | if self.framework == "pt": 116 | probs = model_outputs.logits.softmax(-1)[0] 117 | scores, ids = probs.topk(top_k) 118 | elif self.framework == "tf": 119 | probs = stable_softmax(model_outputs.logits, axis=-1)[0] 120 | topk = tf.math.top_k(probs, k=top_k) 121 | scores, ids = topk.values.numpy(), topk.indices.numpy() 122 | else: 123 | raise ValueError(f"Unsupported framework: {self.framework}") 124 | 125 | scores = scores.tolist() 126 | ids = ids.tolist() 127 | return [{"score": score, "label": self.model.config.id2label[_id]} for score, _id in zip(scores, ids)] 128 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/pipelines/video_classification.py: -------------------------------------------------------------------------------- 1 | from io import BytesIO 2 | from typing import List, Union 3 | 4 | import requests 5 | 6 | from ..utils import add_end_docstrings, is_decord_available, is_torch_available, logging, requires_backends 7 | from .base import PIPELINE_INIT_ARGS, Pipeline 8 | 9 | 10 | if is_decord_available(): 11 | import numpy as np 12 | from decord import VideoReader 13 | 14 | 15 | if is_torch_available(): 16 | from ..models.auto.modeling_auto import MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING 17 | 18 | logger = logging.get_logger(__name__) 19 | 20 | 21 | @add_end_docstrings(PIPELINE_INIT_ARGS) 22 | class VideoClassificationPipeline(Pipeline): 23 | """ 24 | Video classification pipeline using any `AutoModelForVideoClassification`. This pipeline predicts the class of a 25 | video. 26 | 27 | This video classification pipeline can currently be loaded from [`pipeline`] using the following task identifier: 28 | `"video-classification"`. 29 | 30 | See the list of available models on 31 | [huggingface.co/models](https://huggingface.co/models?filter=video-classification). 32 | """ 33 | 34 | def __init__(self, *args, **kwargs): 35 | super().__init__(*args, **kwargs) 36 | requires_backends(self, "decord") 37 | self.check_model_type(MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING) 38 | 39 | def _sanitize_parameters(self, top_k=None, num_frames=None, frame_sampling_rate=None): 40 | preprocess_params = {} 41 | if frame_sampling_rate is not None: 42 | preprocess_params["frame_sampling_rate"] = frame_sampling_rate 43 | if num_frames is not None: 44 | preprocess_params["num_frames"] = num_frames 45 | 46 | postprocess_params = {} 47 | if top_k is not None: 48 | postprocess_params["top_k"] = top_k 49 | return preprocess_params, {}, postprocess_params 50 | 51 | def __call__(self, videos: Union[str, List[str]], **kwargs): 52 | """ 53 | Assign labels to the video(s) passed as inputs. 54 | 55 | Args: 56 | videos (`str`, `List[str]`): 57 | The pipeline handles three types of videos: 58 | 59 | - A string containing a http link pointing to a video 60 | - A string containing a local path to a video 61 | 62 | The pipeline accepts either a single video or a batch of videos, which must then be passed as a string. 63 | Videos in a batch must all be in the same format: all as http links or all as local paths. 64 | top_k (`int`, *optional*, defaults to 5): 65 | The number of top labels that will be returned by the pipeline. If the provided number is higher than 66 | the number of labels available in the model configuration, it will default to the number of labels. 67 | num_frames (`int`, *optional*, defaults to `self.model.config.num_frames`): 68 | The number of frames sampled from the video to run the classification on. If not provided, will default 69 | to the number of frames specified in the model configuration. 70 | frame_sampling_rate (`int`, *optional*, defaults to 1): 71 | The sampling rate used to select frames from the video. If not provided, will default to 1, i.e. every 72 | frame will be used. 73 | 74 | Return: 75 | A dictionary or a list of dictionaries containing result. If the input is a single video, will return a 76 | dictionary, if the input is a list of several videos, will return a list of dictionaries corresponding to 77 | the videos. 78 | 79 | The dictionaries contain the following keys: 80 | 81 | - **label** (`str`) -- The label identified by the model. 82 | - **score** (`int`) -- The score attributed by the model for that label. 83 | """ 84 | return super().__call__(videos, **kwargs) 85 | 86 | def preprocess(self, video, num_frames=None, frame_sampling_rate=1): 87 | if num_frames is None: 88 | num_frames = self.model.config.num_frames 89 | 90 | if video.startswith("http://") or video.startswith("https://"): 91 | video = BytesIO(requests.get(video).content) 92 | 93 | videoreader = VideoReader(video) 94 | videoreader.seek(0) 95 | 96 | start_idx = 0 97 | end_idx = num_frames * frame_sampling_rate - 1 98 | indices = np.linspace(start_idx, end_idx, num=num_frames, dtype=np.int64) 99 | 100 | video = videoreader.get_batch(indices).asnumpy() 101 | video = list(video) 102 | 103 | model_inputs = self.image_processor(video, return_tensors=self.framework) 104 | return model_inputs 105 | 106 | def _forward(self, model_inputs): 107 | model_outputs = self.model(**model_inputs) 108 | return model_outputs 109 | 110 | def postprocess(self, model_outputs, top_k=5): 111 | if top_k > self.model.config.num_labels: 112 | top_k = self.model.config.num_labels 113 | 114 | if self.framework == "pt": 115 | probs = model_outputs.logits.softmax(-1)[0] 116 | scores, ids = probs.topk(top_k) 117 | else: 118 | raise ValueError(f"Unsupported framework: {self.framework}") 119 | 120 | scores = scores.tolist() 121 | ids = ids.tolist() 122 | return [{"score": score, "label": self.model.config.id2label[_id]} for score, _id in zip(scores, ids)] 123 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/sagemaker/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .trainer_sm import SageMakerTrainer 16 | from .training_args_sm import SageMakerTrainingArguments, is_sagemaker_dp_enabled 17 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/sagemaker/trainer_sm.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import warnings 15 | 16 | from ..trainer import Trainer 17 | from ..utils import logging 18 | 19 | 20 | logger = logging.get_logger(__name__) 21 | 22 | 23 | class SageMakerTrainer(Trainer): 24 | def __init__(self, args=None, **kwargs): 25 | warnings.warn( 26 | "`SageMakerTrainer` is deprecated and will be removed in v5 of Transformers. You can use `Trainer` " 27 | "instead.", 28 | FutureWarning, 29 | ) 30 | super().__init__(args=args, **kwargs) 31 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/tools/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | # Copyright 2023 The HuggingFace Inc. team. All rights reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | from typing import TYPE_CHECKING 18 | 19 | from ..utils import ( 20 | OptionalDependencyNotAvailable, 21 | _LazyModule, 22 | is_torch_available, 23 | ) 24 | 25 | 26 | _import_structure = { 27 | "agents": ["Agent", "AzureOpenAiAgent", "HfAgent", "LocalAgent", "OpenAiAgent"], 28 | "base": ["PipelineTool", "RemoteTool", "Tool", "launch_gradio_demo", "load_tool"], 29 | } 30 | 31 | try: 32 | if not is_torch_available(): 33 | raise OptionalDependencyNotAvailable() 34 | except OptionalDependencyNotAvailable: 35 | pass 36 | else: 37 | _import_structure["document_question_answering"] = ["DocumentQuestionAnsweringTool"] 38 | _import_structure["image_captioning"] = ["ImageCaptioningTool"] 39 | _import_structure["image_question_answering"] = ["ImageQuestionAnsweringTool"] 40 | _import_structure["image_segmentation"] = ["ImageSegmentationTool"] 41 | _import_structure["speech_to_text"] = ["SpeechToTextTool"] 42 | _import_structure["text_classification"] = ["TextClassificationTool"] 43 | _import_structure["text_question_answering"] = ["TextQuestionAnsweringTool"] 44 | _import_structure["text_summarization"] = ["TextSummarizationTool"] 45 | _import_structure["text_to_speech"] = ["TextToSpeechTool"] 46 | _import_structure["translation"] = ["TranslationTool"] 47 | 48 | if TYPE_CHECKING: 49 | from .agents import Agent, AzureOpenAiAgent, HfAgent, LocalAgent, OpenAiAgent 50 | from .base import PipelineTool, RemoteTool, Tool, launch_gradio_demo, load_tool 51 | 52 | try: 53 | if not is_torch_available(): 54 | raise OptionalDependencyNotAvailable() 55 | except OptionalDependencyNotAvailable: 56 | pass 57 | else: 58 | from .document_question_answering import DocumentQuestionAnsweringTool 59 | from .image_captioning import ImageCaptioningTool 60 | from .image_question_answering import ImageQuestionAnsweringTool 61 | from .image_segmentation import ImageSegmentationTool 62 | from .speech_to_text import SpeechToTextTool 63 | from .text_classification import TextClassificationTool 64 | from .text_question_answering import TextQuestionAnsweringTool 65 | from .text_summarization import TextSummarizationTool 66 | from .text_to_speech import TextToSpeechTool 67 | from .translation import TranslationTool 68 | else: 69 | import sys 70 | 71 | sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__) 72 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/tools/document_question_answering.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | # Copyright 2023 The HuggingFace Inc. team. All rights reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | import re 18 | 19 | from ..models.auto import AutoProcessor 20 | from ..models.vision_encoder_decoder import VisionEncoderDecoderModel 21 | from ..utils import is_vision_available 22 | from .base import PipelineTool 23 | 24 | 25 | if is_vision_available(): 26 | from PIL import Image 27 | 28 | 29 | class DocumentQuestionAnsweringTool(PipelineTool): 30 | default_checkpoint = "naver-clova-ix/donut-base-finetuned-docvqa" 31 | description = ( 32 | "This is a tool that answers a question about an document (pdf). It takes an input named `document` which " 33 | "should be the document containing the information, as well as a `question` that is the question about the " 34 | "document. It returns a text that contains the answer to the question." 35 | ) 36 | name = "document_qa" 37 | pre_processor_class = AutoProcessor 38 | model_class = VisionEncoderDecoderModel 39 | 40 | inputs = ["image", "text"] 41 | outputs = ["text"] 42 | 43 | def __init__(self, *args, **kwargs): 44 | if not is_vision_available(): 45 | raise ValueError("Pillow must be installed to use the DocumentQuestionAnsweringTool.") 46 | 47 | super().__init__(*args, **kwargs) 48 | 49 | def encode(self, document: "Image", question: str): 50 | task_prompt = "{user_input}" 51 | prompt = task_prompt.replace("{user_input}", question) 52 | decoder_input_ids = self.pre_processor.tokenizer( 53 | prompt, add_special_tokens=False, return_tensors="pt" 54 | ).input_ids 55 | pixel_values = self.pre_processor(document, return_tensors="pt").pixel_values 56 | 57 | return {"decoder_input_ids": decoder_input_ids, "pixel_values": pixel_values} 58 | 59 | def forward(self, inputs): 60 | return self.model.generate( 61 | inputs["pixel_values"].to(self.device), 62 | decoder_input_ids=inputs["decoder_input_ids"].to(self.device), 63 | max_length=self.model.decoder.config.max_position_embeddings, 64 | early_stopping=True, 65 | pad_token_id=self.pre_processor.tokenizer.pad_token_id, 66 | eos_token_id=self.pre_processor.tokenizer.eos_token_id, 67 | use_cache=True, 68 | num_beams=1, 69 | bad_words_ids=[[self.pre_processor.tokenizer.unk_token_id]], 70 | return_dict_in_generate=True, 71 | ).sequences 72 | 73 | def decode(self, outputs): 74 | sequence = self.pre_processor.batch_decode(outputs)[0] 75 | sequence = sequence.replace(self.pre_processor.tokenizer.eos_token, "") 76 | sequence = sequence.replace(self.pre_processor.tokenizer.pad_token, "") 77 | sequence = re.sub(r"<.*?>", "", sequence, count=1).strip() # remove first task start token 78 | sequence = self.pre_processor.token2json(sequence) 79 | 80 | return sequence["answer"] 81 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/tools/image_captioning.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | # Copyright 2023 The HuggingFace Inc. team. All rights reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | from typing import TYPE_CHECKING 18 | 19 | from ..models.auto import AutoModelForVision2Seq 20 | from ..utils import requires_backends 21 | from .base import PipelineTool 22 | 23 | 24 | if TYPE_CHECKING: 25 | from PIL import Image 26 | 27 | 28 | class ImageCaptioningTool(PipelineTool): 29 | default_checkpoint = "Salesforce/blip-image-captioning-base" 30 | description = ( 31 | "This is a tool that generates a description of an image. It takes an input named `image` which should be the " 32 | "image to caption, and returns a text that contains the description in English." 33 | ) 34 | name = "image_captioner" 35 | model_class = AutoModelForVision2Seq 36 | 37 | inputs = ["image"] 38 | outputs = ["text"] 39 | 40 | def __init__(self, *args, **kwargs): 41 | requires_backends(self, ["vision"]) 42 | super().__init__(*args, **kwargs) 43 | 44 | def encode(self, image: "Image"): 45 | return self.pre_processor(images=image, return_tensors="pt") 46 | 47 | def forward(self, inputs): 48 | return self.model.generate(**inputs) 49 | 50 | def decode(self, outputs): 51 | return self.pre_processor.batch_decode(outputs, skip_special_tokens=True)[0].strip() 52 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/tools/image_question_answering.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | # Copyright 2023 The HuggingFace Inc. team. All rights reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | from typing import TYPE_CHECKING 18 | 19 | import torch 20 | 21 | from ..models.auto import AutoModelForVisualQuestionAnswering, AutoProcessor 22 | from ..utils import requires_backends 23 | from .base import PipelineTool 24 | 25 | 26 | if TYPE_CHECKING: 27 | from PIL import Image 28 | 29 | 30 | class ImageQuestionAnsweringTool(PipelineTool): 31 | default_checkpoint = "dandelin/vilt-b32-finetuned-vqa" 32 | description = ( 33 | "This is a tool that answers a question about an image. It takes an input named `image` which should be the " 34 | "image containing the information, as well as a `question` which should be the question in English. It " 35 | "returns a text that is the answer to the question." 36 | ) 37 | name = "image_qa" 38 | pre_processor_class = AutoProcessor 39 | model_class = AutoModelForVisualQuestionAnswering 40 | 41 | inputs = ["image", "text"] 42 | outputs = ["text"] 43 | 44 | def __init__(self, *args, **kwargs): 45 | requires_backends(self, ["vision"]) 46 | super().__init__(*args, **kwargs) 47 | 48 | def encode(self, image: "Image", question: str): 49 | return self.pre_processor(image, question, return_tensors="pt") 50 | 51 | def forward(self, inputs): 52 | with torch.no_grad(): 53 | return self.model(**inputs).logits 54 | 55 | def decode(self, outputs): 56 | idx = outputs.argmax(-1).item() 57 | return self.model.config.id2label[idx] 58 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/tools/image_segmentation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | # Copyright 2023 The HuggingFace Inc. team. All rights reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | import numpy as np 18 | import torch 19 | 20 | from ..models.clipseg import CLIPSegForImageSegmentation 21 | from ..utils import is_vision_available, requires_backends 22 | from .base import PipelineTool 23 | 24 | 25 | if is_vision_available(): 26 | from PIL import Image 27 | 28 | 29 | class ImageSegmentationTool(PipelineTool): 30 | description = ( 31 | "This is a tool that creates a segmentation mask of an image according to a label. It cannot create an image." 32 | "It takes two arguments named `image` which should be the original image, and `label` which should be a text " 33 | "describing the elements what should be identified in the segmentation mask. The tool returns the mask." 34 | ) 35 | default_checkpoint = "CIDAS/clipseg-rd64-refined" 36 | name = "image_segmenter" 37 | model_class = CLIPSegForImageSegmentation 38 | 39 | inputs = ["image", "text"] 40 | outputs = ["image"] 41 | 42 | def __init__(self, *args, **kwargs): 43 | requires_backends(self, ["vision"]) 44 | super().__init__(*args, **kwargs) 45 | 46 | def encode(self, image: "Image", label: str): 47 | return self.pre_processor(text=[label], images=[image], padding=True, return_tensors="pt") 48 | 49 | def forward(self, inputs): 50 | with torch.no_grad(): 51 | logits = self.model(**inputs).logits 52 | return logits 53 | 54 | def decode(self, outputs): 55 | array = outputs.cpu().detach().numpy() 56 | array[array <= 0] = 0 57 | array[array > 0] = 1 58 | return Image.fromarray((array * 255).astype(np.uint8)) 59 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/tools/prompts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | # Copyright 2023 The HuggingFace Inc. team. All rights reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | import re 18 | 19 | from ..utils import cached_file 20 | 21 | 22 | # docstyle-ignore 23 | CHAT_MESSAGE_PROMPT = """ 24 | Human: <> 25 | 26 | Assistant: """ 27 | 28 | 29 | DEFAULT_PROMPTS_REPO = "huggingface-tools/default-prompts" 30 | PROMPT_FILES = {"chat": "chat_prompt_template.txt", "run": "run_prompt_template.txt"} 31 | 32 | 33 | def download_prompt(prompt_or_repo_id, agent_name, mode="run"): 34 | """ 35 | Downloads and caches the prompt from a repo and returns it contents (if necessary) 36 | """ 37 | if prompt_or_repo_id is None: 38 | prompt_or_repo_id = DEFAULT_PROMPTS_REPO 39 | 40 | # prompt is considered a repo ID when it does not contain any kind of space 41 | if re.search("\\s", prompt_or_repo_id) is not None: 42 | return prompt_or_repo_id 43 | 44 | prompt_file = cached_file( 45 | prompt_or_repo_id, PROMPT_FILES[mode], repo_type="dataset", user_agent={"agent": agent_name} 46 | ) 47 | with open(prompt_file, "r", encoding="utf-8") as f: 48 | return f.read() 49 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/tools/speech_to_text.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | # Copyright 2023 The HuggingFace Inc. team. All rights reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | from ..models.whisper import WhisperForConditionalGeneration, WhisperProcessor 18 | from .base import PipelineTool 19 | 20 | 21 | class SpeechToTextTool(PipelineTool): 22 | default_checkpoint = "openai/whisper-base" 23 | description = ( 24 | "This is a tool that transcribes an audio into text. It takes an input named `audio` and returns the " 25 | "transcribed text." 26 | ) 27 | name = "transcriber" 28 | pre_processor_class = WhisperProcessor 29 | model_class = WhisperForConditionalGeneration 30 | 31 | inputs = ["audio"] 32 | outputs = ["text"] 33 | 34 | def encode(self, audio): 35 | return self.pre_processor(audio, return_tensors="pt").input_features 36 | 37 | def forward(self, inputs): 38 | return self.model.generate(inputs=inputs) 39 | 40 | def decode(self, outputs): 41 | return self.pre_processor.batch_decode(outputs, skip_special_tokens=True)[0] 42 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/tools/text_classification.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | # Copyright 2023 The HuggingFace Inc. team. All rights reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | import torch 18 | 19 | from ..models.auto import AutoModelForSequenceClassification, AutoTokenizer 20 | from .base import PipelineTool 21 | 22 | 23 | class TextClassificationTool(PipelineTool): 24 | """ 25 | Example: 26 | 27 | ```py 28 | from transformers.tools import TextClassificationTool 29 | 30 | classifier = TextClassificationTool() 31 | classifier("This is a super nice API!", labels=["positive", "negative"]) 32 | ``` 33 | """ 34 | 35 | default_checkpoint = "facebook/bart-large-mnli" 36 | description = ( 37 | "This is a tool that classifies an English text using provided labels. It takes two inputs: `text`, which " 38 | "should be the text to classify, and `labels`, which should be the list of labels to use for classification. " 39 | "It returns the most likely label in the list of provided `labels` for the input text." 40 | ) 41 | name = "text_classifier" 42 | pre_processor_class = AutoTokenizer 43 | model_class = AutoModelForSequenceClassification 44 | 45 | inputs = ["text", ["text"]] 46 | outputs = ["text"] 47 | 48 | def setup(self): 49 | super().setup() 50 | config = self.model.config 51 | self.entailment_id = -1 52 | for idx, label in config.id2label.items(): 53 | if label.lower().startswith("entail"): 54 | self.entailment_id = int(idx) 55 | if self.entailment_id == -1: 56 | raise ValueError("Could not determine the entailment ID from the model config, please pass it at init.") 57 | 58 | def encode(self, text, labels): 59 | self._labels = labels 60 | return self.pre_processor( 61 | [text] * len(labels), 62 | [f"This example is {label}" for label in labels], 63 | return_tensors="pt", 64 | padding="max_length", 65 | ) 66 | 67 | def decode(self, outputs): 68 | logits = outputs.logits 69 | label_id = torch.argmax(logits[:, 2]).item() 70 | return self._labels[label_id] 71 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/tools/text_question_answering.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | # Copyright 2023 The HuggingFace Inc. team. All rights reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | from ..models.auto import AutoModelForSeq2SeqLM, AutoTokenizer 18 | from .base import PipelineTool 19 | 20 | 21 | QA_PROMPT = """Here is a text containing a lot of information: '''{text}'''. 22 | 23 | Can you answer this question about the text: '{question}'""" 24 | 25 | 26 | class TextQuestionAnsweringTool(PipelineTool): 27 | default_checkpoint = "google/flan-t5-base" 28 | description = ( 29 | "This is a tool that answers questions related to a text. It takes two arguments named `text`, which is the " 30 | "text where to find the answer, and `question`, which is the question, and returns the answer to the question." 31 | ) 32 | name = "text_qa" 33 | pre_processor_class = AutoTokenizer 34 | model_class = AutoModelForSeq2SeqLM 35 | 36 | inputs = ["text", "text"] 37 | outputs = ["text"] 38 | 39 | def encode(self, text: str, question: str): 40 | prompt = QA_PROMPT.format(text=text, question=question) 41 | return self.pre_processor(prompt, return_tensors="pt") 42 | 43 | def forward(self, inputs): 44 | output_ids = self.model.generate(**inputs) 45 | 46 | in_b, _ = inputs["input_ids"].shape 47 | out_b = output_ids.shape[0] 48 | 49 | return output_ids.reshape(in_b, out_b // in_b, *output_ids.shape[1:])[0][0] 50 | 51 | def decode(self, outputs): 52 | return self.pre_processor.decode(outputs, skip_special_tokens=True, clean_up_tokenization_spaces=True) 53 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/tools/text_summarization.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | # Copyright 2023 The HuggingFace Inc. team. All rights reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | from ..models.auto import AutoModelForSeq2SeqLM, AutoTokenizer 18 | from .base import PipelineTool 19 | 20 | 21 | class TextSummarizationTool(PipelineTool): 22 | """ 23 | Example: 24 | 25 | ```py 26 | from transformers.tools import TextSummarizationTool 27 | 28 | summarizer = TextSummarizationTool() 29 | summarizer(long_text) 30 | ``` 31 | """ 32 | 33 | default_checkpoint = "philschmid/bart-large-cnn-samsum" 34 | description = ( 35 | "This is a tool that summarizes an English text. It takes an input `text` containing the text to summarize, " 36 | "and returns a summary of the text." 37 | ) 38 | name = "summarizer" 39 | pre_processor_class = AutoTokenizer 40 | model_class = AutoModelForSeq2SeqLM 41 | 42 | inputs = ["text"] 43 | outputs = ["text"] 44 | 45 | def encode(self, text): 46 | return self.pre_processor(text, return_tensors="pt", truncation=True) 47 | 48 | def forward(self, inputs): 49 | return self.model.generate(**inputs)[0] 50 | 51 | def decode(self, outputs): 52 | return self.pre_processor.decode(outputs, skip_special_tokens=True, clean_up_tokenization_spaces=True) 53 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/tools/text_to_speech.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | # Copyright 2023 The HuggingFace Inc. team. All rights reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | import torch 18 | 19 | from ..models.speecht5 import SpeechT5ForTextToSpeech, SpeechT5HifiGan, SpeechT5Processor 20 | from ..utils import is_datasets_available 21 | from .base import PipelineTool 22 | 23 | 24 | if is_datasets_available(): 25 | from datasets import load_dataset 26 | 27 | 28 | class TextToSpeechTool(PipelineTool): 29 | default_checkpoint = "microsoft/speecht5_tts" 30 | description = ( 31 | "This is a tool that reads an English text out loud. It takes an input named `text` which should contain the " 32 | "text to read (in English) and returns a waveform object containing the sound." 33 | ) 34 | name = "text_reader" 35 | pre_processor_class = SpeechT5Processor 36 | model_class = SpeechT5ForTextToSpeech 37 | post_processor_class = SpeechT5HifiGan 38 | 39 | inputs = ["text"] 40 | outputs = ["audio"] 41 | 42 | def setup(self): 43 | if self.post_processor is None: 44 | self.post_processor = "microsoft/speecht5_hifigan" 45 | super().setup() 46 | 47 | def encode(self, text, speaker_embeddings=None): 48 | inputs = self.pre_processor(text=text, return_tensors="pt", truncation=True) 49 | 50 | if speaker_embeddings is None: 51 | if not is_datasets_available(): 52 | raise ImportError("Datasets needs to be installed if not passing speaker embeddings.") 53 | 54 | embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation") 55 | speaker_embeddings = torch.tensor(embeddings_dataset[7305]["xvector"]).unsqueeze(0) 56 | 57 | return {"input_ids": inputs["input_ids"], "speaker_embeddings": speaker_embeddings} 58 | 59 | def forward(self, inputs): 60 | with torch.no_grad(): 61 | return self.model.generate_speech(**inputs) 62 | 63 | def decode(self, outputs): 64 | with torch.no_grad(): 65 | return self.post_processor(outputs).cpu().detach() 66 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/training_args_seq2seq.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | from dataclasses import dataclass, field 17 | from pathlib import Path 18 | from typing import Optional, Union 19 | 20 | from .generation.configuration_utils import GenerationConfig 21 | from .training_args import TrainingArguments 22 | from .utils import add_start_docstrings 23 | 24 | 25 | logger = logging.getLogger(__name__) 26 | 27 | 28 | @dataclass 29 | @add_start_docstrings(TrainingArguments.__doc__) 30 | class Seq2SeqTrainingArguments(TrainingArguments): 31 | """ 32 | Args: 33 | sortish_sampler (`bool`, *optional*, defaults to `False`): 34 | Whether to use a *sortish sampler* or not. Only possible if the underlying datasets are *Seq2SeqDataset* 35 | for now but will become generally available in the near future. 36 | 37 | It sorts the inputs according to lengths in order to minimize the padding size, with a bit of randomness 38 | for the training set. 39 | predict_with_generate (`bool`, *optional*, defaults to `False`): 40 | Whether to use generate to calculate generative metrics (ROUGE, BLEU). 41 | generation_max_length (`int`, *optional*): 42 | The `max_length` to use on each evaluation loop when `predict_with_generate=True`. Will default to the 43 | `max_length` value of the model configuration. 44 | generation_num_beams (`int`, *optional*): 45 | The `num_beams` to use on each evaluation loop when `predict_with_generate=True`. Will default to the 46 | `num_beams` value of the model configuration. 47 | generation_config (`str` or `Path` or [`~generation.GenerationConfig`], *optional*): 48 | Allows to load a [`~generation.GenerationConfig`] from the `from_pretrained` method. This can be either: 49 | 50 | - a string, the *model id* of a pretrained model configuration hosted inside a model repo on 51 | huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced 52 | under a user or organization name, like `dbmdz/bert-base-german-cased`. 53 | - a path to a *directory* containing a configuration file saved using the 54 | [`~GenerationConfig.save_pretrained`] method, e.g., `./my_model_directory/`. 55 | - a [`~generation.GenerationConfig`] object. 56 | """ 57 | 58 | sortish_sampler: bool = field(default=False, metadata={"help": "Whether to use SortishSampler or not."}) 59 | predict_with_generate: bool = field( 60 | default=False, metadata={"help": "Whether to use generate to calculate generative metrics (ROUGE, BLEU)."} 61 | ) 62 | generation_max_length: Optional[int] = field( 63 | default=None, 64 | metadata={ 65 | "help": ( 66 | "The `max_length` to use on each evaluation loop when `predict_with_generate=True`. Will default " 67 | "to the `max_length` value of the model configuration." 68 | ) 69 | }, 70 | ) 71 | generation_num_beams: Optional[int] = field( 72 | default=None, 73 | metadata={ 74 | "help": ( 75 | "The `num_beams` to use on each evaluation loop when `predict_with_generate=True`. Will default " 76 | "to the `num_beams` value of the model configuration." 77 | ) 78 | }, 79 | ) 80 | generation_config: Optional[Union[str, Path, GenerationConfig]] = field( 81 | default=None, 82 | metadata={ 83 | "help": "Model id, file path or url pointing to a GenerationConfig json file, to use during prediction." 84 | }, 85 | ) 86 | 87 | def to_dict(self): 88 | """ 89 | Serializes this instance while replace `Enum` by their values and `GenerationConfig` by dictionaries (for JSON 90 | serialization support). It obfuscates the token values by removing their value. 91 | """ 92 | # filter out fields that are defined as field(init=False) 93 | d = super().to_dict() 94 | for k, v in d.items(): 95 | if isinstance(v, GenerationConfig): 96 | d[k] = v.to_dict() 97 | return d 98 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/utils/constants.py: -------------------------------------------------------------------------------- 1 | IMAGENET_DEFAULT_MEAN = [0.485, 0.456, 0.406] 2 | IMAGENET_DEFAULT_STD = [0.229, 0.224, 0.225] 3 | IMAGENET_STANDARD_MEAN = [0.5, 0.5, 0.5] 4 | IMAGENET_STANDARD_STD = [0.5, 0.5, 0.5] 5 | OPENAI_CLIP_MEAN = [0.48145466, 0.4578275, 0.40821073] 6 | OPENAI_CLIP_STD = [0.26862954, 0.26130258, 0.27577711] 7 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/utils/dummy_detectron2_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import requires_backends 3 | 4 | 5 | LAYOUTLM_V2_PRETRAINED_MODEL_ARCHIVE_LIST = None 6 | 7 | 8 | class LayoutLMv2Model: 9 | def __init__(self, *args, **kwargs): 10 | requires_backends(self, ["detectron2"]) 11 | 12 | @classmethod 13 | def from_pretrained(cls, *args, **kwargs): 14 | requires_backends(cls, ["detectron2"]) 15 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/utils/dummy_keras_nlp_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class TFGPT2Tokenizer(metaclass=DummyObject): 6 | _backends = ["keras_nlp"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["keras_nlp"]) 10 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/utils/dummy_sentencepiece_and_tokenizers_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | SLOW_TO_FAST_CONVERTERS = None 6 | 7 | 8 | def convert_slow_tokenizer(*args, **kwargs): 9 | requires_backends(convert_slow_tokenizer, ["sentencepiece", "tokenizers"]) 10 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/utils/dummy_speech_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class ASTFeatureExtractor(metaclass=DummyObject): 6 | _backends = ["speech"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["speech"]) 10 | 11 | 12 | class Speech2TextFeatureExtractor(metaclass=DummyObject): 13 | _backends = ["speech"] 14 | 15 | def __init__(self, *args, **kwargs): 16 | requires_backends(self, ["speech"]) 17 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/utils/dummy_tensorflow_text_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class TFBertTokenizer(metaclass=DummyObject): 6 | _backends = ["tensorflow_text"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["tensorflow_text"]) 10 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/utils/model_parallel_utils.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2020 The HuggingFace Team. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from math import ceil 17 | 18 | 19 | def assert_device_map(device_map, num_blocks): 20 | blocks = list(range(0, num_blocks)) 21 | 22 | device_map_blocks = [item for sublist in list(device_map.values()) for item in sublist] 23 | 24 | # Duplicate check 25 | duplicate_blocks = [] 26 | for i in device_map_blocks: 27 | if device_map_blocks.count(i) > 1 and i not in duplicate_blocks: 28 | duplicate_blocks.append(i) 29 | # Missing blocks 30 | missing_blocks = [i for i in blocks if i not in device_map_blocks] 31 | extra_blocks = [i for i in device_map_blocks if i not in blocks] 32 | 33 | if len(duplicate_blocks) != 0: 34 | raise ValueError( 35 | "Duplicate attention blocks specified in device_map. Attention blocks must be specified to one device." 36 | " These attention blocks were specified more than once: " + str(duplicate_blocks) 37 | ) 38 | if len(missing_blocks) != 0: 39 | raise ValueError( 40 | "There are attention blocks for this model that are not specified in the device_map. Add these attention " 41 | "blocks to a device on the device_map: " + str(missing_blocks) 42 | ) 43 | if len(extra_blocks) != 0: 44 | raise ValueError( 45 | "The device_map contains more attention blocks than this model has. Remove these from the device_map:" 46 | + str(extra_blocks) 47 | ) 48 | 49 | 50 | def get_device_map(n_layers, devices): 51 | """Returns a dictionary of layers distributed evenly across all devices.""" 52 | layers = list(range(n_layers)) 53 | n_blocks = int(ceil(n_layers / len(devices))) 54 | layers_list = [layers[i : i + n_blocks] for i in range(0, n_layers, n_blocks)] 55 | 56 | return dict(zip(devices, layers_list)) 57 | -------------------------------------------------------------------------------- /llm_toys/hf/transformers/utils/versions.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | Utilities for working with package versions 16 | """ 17 | 18 | import importlib.metadata 19 | import operator 20 | import re 21 | import sys 22 | from typing import Optional 23 | 24 | from packaging import version 25 | 26 | 27 | ops = { 28 | "<": operator.lt, 29 | "<=": operator.le, 30 | "==": operator.eq, 31 | "!=": operator.ne, 32 | ">=": operator.ge, 33 | ">": operator.gt, 34 | } 35 | 36 | 37 | def _compare_versions(op, got_ver, want_ver, requirement, pkg, hint): 38 | if got_ver is None or want_ver is None: 39 | raise ValueError( 40 | f"Unable to compare versions for {requirement}: need={want_ver} found={got_ver}. This is unusual. Consider" 41 | f" reinstalling {pkg}." 42 | ) 43 | if not ops[op](version.parse(got_ver), version.parse(want_ver)): 44 | raise ImportError( 45 | f"{requirement} is required for a normal functioning of this module, but found {pkg}=={got_ver}.{hint}" 46 | ) 47 | 48 | 49 | def require_version(requirement: str, hint: Optional[str] = None) -> None: 50 | """ 51 | Perform a runtime check of the dependency versions, using the exact same syntax used by pip. 52 | 53 | The installed module version comes from the *site-packages* dir via *importlib.metadata*. 54 | 55 | Args: 56 | requirement (`str`): pip style definition, e.g., "tokenizers==0.9.4", "tqdm>=4.27", "numpy" 57 | hint (`str`, *optional*): what suggestion to print in case of requirements not being met 58 | 59 | Example: 60 | 61 | ```python 62 | require_version("pandas>1.1.2") 63 | require_version("numpy>1.18.5", "this is important to have for whatever reason") 64 | ```""" 65 | 66 | hint = f"\n{hint}" if hint is not None else "" 67 | 68 | # non-versioned check 69 | if re.match(r"^[\w_\-\d]+$", requirement): 70 | pkg, op, want_ver = requirement, None, None 71 | else: 72 | match = re.findall(r"^([^!=<>\s]+)([\s!=<>]{1,2}.+)", requirement) 73 | if not match: 74 | raise ValueError( 75 | "requirement needs to be in the pip package format, .e.g., package_a==1.23, or package_b>=1.23, but" 76 | f" got {requirement}" 77 | ) 78 | pkg, want_full = match[0] 79 | want_range = want_full.split(",") # there could be multiple requirements 80 | wanted = {} 81 | for w in want_range: 82 | match = re.findall(r"^([\s!=<>]{1,2})(.+)", w) 83 | if not match: 84 | raise ValueError( 85 | "requirement needs to be in the pip package format, .e.g., package_a==1.23, or package_b>=1.23," 86 | f" but got {requirement}" 87 | ) 88 | op, want_ver = match[0] 89 | wanted[op] = want_ver 90 | if op not in ops: 91 | raise ValueError(f"{requirement}: need one of {list(ops.keys())}, but got {op}") 92 | 93 | # special case 94 | if pkg == "python": 95 | got_ver = ".".join([str(x) for x in sys.version_info[:3]]) 96 | for op, want_ver in wanted.items(): 97 | _compare_versions(op, got_ver, want_ver, requirement, pkg, hint) 98 | return 99 | 100 | # check if any version is installed 101 | try: 102 | got_ver = importlib.metadata.version(pkg) 103 | except importlib.metadata.PackageNotFoundError: 104 | raise importlib.metadata.PackageNotFoundError( 105 | f"The '{requirement}' distribution was not found and is required by this application. {hint}" 106 | ) 107 | 108 | # check that the right version is installed if version number or a range was provided 109 | if want_ver is not None: 110 | for op, want_ver in wanted.items(): 111 | _compare_versions(op, got_ver, want_ver, requirement, pkg, hint) 112 | 113 | 114 | def require_version_core(requirement): 115 | """require_version wrapper which emits a core-specific hint on failure""" 116 | hint = "Try: pip install transformers -U or pip install -e '.[dev]' if you're working with git main" 117 | return require_version(requirement, hint) 118 | -------------------------------------------------------------------------------- /llm_toys/prompts.py: -------------------------------------------------------------------------------- 1 | from llm_toys.config import EOC_FORMAT 2 | 3 | 4 | # Paraphrasing + Tone change 5 | 6 | PARAPHRASE_PREDICT_FORMAT = ( 7 | "### Instruction:\nGenerate a paraphrase for the following Input sentence.\n\n" 8 | "### Input:\n{input_text}\n\n### Response:\n" 9 | ) 10 | PARAPHRASE_TRAIN_FORMAT = PARAPHRASE_PREDICT_FORMAT + "{response}" + EOC_FORMAT 11 | 12 | TONE_CHANGE_PREDICT_FORMAT = ( 13 | "### Instruction:\nChange the tone of the following Input sentence to {tone}.\n\n" 14 | "### Input:\n{input_text}\n\n### Response:\n" 15 | ) 16 | TONE_CHANGE_TRAIN_FORMAT = TONE_CHANGE_PREDICT_FORMAT + "{response}" + EOC_FORMAT 17 | 18 | 19 | # Dialogue Summary + Topic generation 20 | 21 | DIALOGUE_SUMMARY_TOPIC_PREDICT_FORMAT = ( 22 | "### Instruction:\nGenerate a Summary and Topic for the following dialogue.\n\n" 23 | "### Input:\n{input_text}\n\n### Response:\n" 24 | ) 25 | DIALOGUE_SUMMARY_TOPIC_TRAIN_FORMAT = ( 26 | DIALOGUE_SUMMARY_TOPIC_PREDICT_FORMAT + "# Summary:\n{summary}\n\n# Topic:\n{topic}" + EOC_FORMAT 27 | ) 28 | -------------------------------------------------------------------------------- /llm_toys/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | from .all_tasks import GeneralTaskAssitant 2 | from .paraphrase import Paraphraser 3 | from .summary_topic import SummaryAndTopicGenerator 4 | 5 | 6 | __all__ = ["GeneralTaskAssitant", "Paraphraser", "SummaryAndTopicGenerator"] 7 | -------------------------------------------------------------------------------- /llm_toys/tasks/paraphrase.py: -------------------------------------------------------------------------------- 1 | from time import time 2 | 3 | from llm_toys.config import MODEL_CONFIG, SUPPORTED_END_TONES, SUPPORTED_MODEL_SIZES, TaskType 4 | from llm_toys.prompts import PARAPHRASE_PREDICT_FORMAT, TONE_CHANGE_PREDICT_FORMAT 5 | from llm_toys.tasks.predict import PeftQLoraPredictor 6 | from llm_toys.utils import print_warning 7 | 8 | 9 | TXTS_TO_EXPLORE = """ 10 | I undrstand you're facing an isue. Lts find a solution together.--- 11 | I'm srry, but I'm unble to resolv this at the moment. Plase try again later.--- 12 | It appeas that you're facing a technial difficulty. Let's wk through it.--- 13 | I appoligize for the inconveience cause by our system. We're workng on a fix.--- 14 | Plase provice me with more dtails so I can bettr assist you. 15 | """.strip() 16 | 17 | 18 | class Paraphraser(PeftQLoraPredictor): 19 | paraphrase_format = PARAPHRASE_PREDICT_FORMAT 20 | tone_change_format = TONE_CHANGE_PREDICT_FORMAT 21 | 22 | def __init__(self, model_size: str = "3B", max_length: int = 512): 23 | assert ( 24 | model_size.upper() in SUPPORTED_MODEL_SIZES 25 | ), f"Invalid model size; Supported sizes: {SUPPORTED_MODEL_SIZES}" 26 | super().__init__(**MODEL_CONFIG[model_size.upper()][TaskType.PARAPHRASE_TONE], max_length=max_length) 27 | 28 | def paraphrase( 29 | self, 30 | input_text: str, 31 | tone: str = None, 32 | max_new_tokens: int = 512, 33 | temperature: float = 0.3, 34 | top_p: float = 0.95, 35 | adjust_token_lengths: bool = False, 36 | ) -> str: 37 | if tone: 38 | if tone.lower() not in SUPPORTED_END_TONES: 39 | print_warning( 40 | f"Model has been fine-tuned to change the tone to one of following: {SUPPORTED_END_TONES}; " 41 | "Using a different tone may result in unexpected output." 42 | ) 43 | input_text = self.tone_change_format.format(input_text=input_text, tone=tone) 44 | else: 45 | input_text = self.paraphrase_format.format(input_text=input_text) 46 | return super().predict( 47 | input_text, max_new_tokens, temperature, top_p, adjust_token_lengths=adjust_token_lengths 48 | )[0] 49 | 50 | def explore(self, input_texts: list[str] = None, temperatures: list[float] = [0.1, 0.3, 0.5, 0.8, 1.0]) -> None: 51 | """This method can be used to explore outputs on a range of temperatures. 52 | This can help in finding the best temperature that suits the user.""" 53 | preds, total_time = [], 0 54 | for input_text in input_texts or TXTS_TO_EXPLORE.split("---"): 55 | input_text = input_text.strip() 56 | print(f"\n{input_text}\n{'-'*len(input_text)}") 57 | for i, temperature in enumerate(temperatures): 58 | if i != 0: 59 | print() 60 | print(f"Temperature: {temperature}") 61 | for tone in [None] + SUPPORTED_END_TONES: 62 | st = time() 63 | pred = self.paraphrase(input_text, tone=tone, temperature=temperature) 64 | total_time += time() - st 65 | print(f" ↪ {tone or 'paraphrase'}: {pred}") 66 | preds.append(pred) 67 | 68 | n_tokens = sum((len(self.tokenizer.encode(txt)) for txt in preds)) 69 | n_tokens_per_sec = n_tokens / total_time 70 | print(f"\n---\nGenerated {len(preds)} predictions in {total_time:.2f}s: {n_tokens_per_sec:.2f} tokens/s") 71 | -------------------------------------------------------------------------------- /llm_toys/tasks/predict.py: -------------------------------------------------------------------------------- 1 | from peft import PeftModel 2 | import torch 3 | from transformers import ( 4 | AutoModelForCausalLM, 5 | AutoTokenizer, 6 | StoppingCriteria, 7 | StoppingCriteriaList, 8 | ) 9 | 10 | from llm_toys.config import DEVICE, EOC_FORMAT, get_bnb_config, RESPONSE_FORMAT 11 | 12 | 13 | class StoppingCriteriaSub(StoppingCriteria): 14 | """Helps in stopping the generation when a certain sequence of tokens is generated.""" 15 | 16 | def __init__(self, stops: list = []): 17 | super().__init__() 18 | self.stops = stops 19 | 20 | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> bool: 21 | return input_ids[0][-len(self.stops) :].tolist() == self.stops 22 | 23 | 24 | class PeftQLoraPredictor: 25 | """Peft quantized lora model predictor for a model finetuned on generative task.""" 26 | 27 | def __init__(self, model_name: str, peft_model_id: str, max_length: int = 512): 28 | self.model_name = model_name 29 | self.tokenizer = AutoTokenizer.from_pretrained(self.model_name) 30 | self.tokenizer.pad_token = self.tokenizer.eos_token 31 | self.max_length = max_length 32 | self._response_ids = self.tokenizer(RESPONSE_FORMAT, return_tensors="pt")["input_ids"][0] 33 | 34 | self.model = AutoModelForCausalLM.from_pretrained( 35 | self.model_name, 36 | device_map="auto", 37 | trust_remote_code=True, 38 | quantization_config=get_bnb_config(), 39 | ) 40 | self.model = PeftModel.from_pretrained(self.model, peft_model_id) 41 | self.model = self.model.to(DEVICE) 42 | self.model.eval() 43 | 44 | self.stopping_criteria = StoppingCriteriaList( 45 | [StoppingCriteriaSub(stops=self.tokenizer(EOC_FORMAT)["input_ids"])] 46 | ) 47 | 48 | def _adjust_token_lengths(self, tokenized: dict) -> None: 49 | """If tokenized length is more than max length, we truncate to 50 | include the RESPONSE ids to get a prediction from the model.""" 51 | for i in range(len(tokenized["input_ids"])): 52 | if len(tokenized["input_ids"][i]) == self.max_length: 53 | tokenized["input_ids"][i] = torch.concat( 54 | (tokenized["input_ids"][i][: -len(self._response_ids)], self._response_ids) 55 | ) 56 | 57 | def predict( 58 | self, 59 | input_text: str, 60 | max_new_tokens: int = 512, 61 | temperature: float = 0.3, 62 | top_p: float = 0.95, 63 | num_return_sequences: int = 1, 64 | adjust_token_lengths: bool = False, 65 | ) -> list[str]: 66 | """If you know that the input token length is going to exceed the max length then set adjust_token_lengths 67 | to True so that the EOC tokens can be added back to the truncated tokens.""" 68 | tokenized = self.tokenizer( 69 | input_text, 70 | max_length=self.max_length, 71 | padding=True, 72 | truncation=True, 73 | return_tensors="pt", 74 | ) 75 | if adjust_token_lengths: 76 | self._adjust_token_lengths(tokenized) 77 | 78 | with torch.no_grad(): 79 | out = self.model.generate( 80 | input_ids=tokenized["input_ids"].to(DEVICE), 81 | attention_mask=tokenized["attention_mask"].to(DEVICE), 82 | pad_token_id=self.tokenizer.eos_token_id, 83 | max_new_tokens=max_new_tokens, 84 | num_return_sequences=num_return_sequences, 85 | do_sample=True, 86 | temperature=temperature, 87 | top_p=top_p, 88 | stopping_criteria=self.stopping_criteria, 89 | ) 90 | 91 | out_texts = [self.tokenizer.decode(o, skip_special_tokens=True) for o in out] 92 | preds = [] 93 | for txt in out_texts: 94 | response_ix = txt.find(RESPONSE_FORMAT) 95 | if response_ix != -1: 96 | response_txt = txt[response_ix + len(RESPONSE_FORMAT) :] 97 | end_ix = response_txt.find(EOC_FORMAT) 98 | if end_ix != -1: 99 | preds.append(response_txt[:end_ix].strip()) 100 | return preds 101 | -------------------------------------------------------------------------------- /llm_toys/tasks/summary_topic.py: -------------------------------------------------------------------------------- 1 | from time import time 2 | 3 | from llm_toys.config import MODEL_CONFIG, SUPPORTED_MODEL_SIZES, TaskType 4 | from llm_toys.prompts import DIALOGUE_SUMMARY_TOPIC_PREDICT_FORMAT 5 | from llm_toys.tasks.predict import PeftQLoraPredictor 6 | from llm_toys.utils import print_warning 7 | 8 | 9 | TXTS_TO_EXPLORE = """ 10 | #Person1#: I'm so excited for the premiere of the latest Studio Ghibli movie! 11 | #Person2#: What's got you so hyped? 12 | #Person1#: Studio Ghibli movies are pure magic! The animation, storytelling, everything is incredible. 13 | #Person2#: Which movie is it? 14 | #Person1#: It's called "Whisper of the Wind." It's about a girl on a magical journey to save her village. 15 | #Person2#: Sounds amazing! I'm in for the premiere. 16 | #Person1#: Great! We're in for a visual masterpiece and a heartfelt story. 17 | #Person2#: Can't wait to be transported to their world. 18 | #Person1#: It'll be an unforgettable experience, for sure! 19 | --- 20 | #Person1#: Have you seen the floods happening worldwide? Climate change is causing havoc. 21 | #Person2#: Yes, it's distressing. We must act fast. 22 | #Person1#: Rising temperatures intensify rainfall, leading to severe floods. 23 | #Person2#: Vulnerable communities suffer the most. We need global cooperation. 24 | #Person1#: Mitigation and adaptation strategies are crucial. 25 | #Person2#: Governments and international organizations must act now. 26 | #Person1#: Let's push for a sustainable future. Time is running out. 27 | """.strip() 28 | 29 | 30 | class SummaryAndTopicGenerator(PeftQLoraPredictor): 31 | dialogue_summary_topic_format = DIALOGUE_SUMMARY_TOPIC_PREDICT_FORMAT 32 | 33 | def __init__(self, model_size: str = "3B", max_length: int = 512): 34 | assert ( 35 | model_size.upper() in SUPPORTED_MODEL_SIZES 36 | ), f"Invalid model size; Supported sizes: {SUPPORTED_MODEL_SIZES}" 37 | super().__init__(**MODEL_CONFIG[model_size.upper()][TaskType.DIALOGUE_SUMMARY_TOPIC], max_length=max_length) 38 | 39 | @staticmethod 40 | def parse_summary_and_topic(pred: str) -> dict[str, str]: 41 | topic_ix = pred.index("# Topic:") 42 | summary = pred[len("# Summary:") : topic_ix].strip() 43 | topic = pred[topic_ix + len("# Topic:") :].strip() 44 | return {"summary": summary, "topic": topic} 45 | 46 | def generate_summary_and_topic( 47 | self, 48 | input_text: str, 49 | max_new_tokens: int = 512, 50 | temperature: float = 0.3, 51 | top_p: float = 0.95, 52 | adjust_token_lengths: bool = False, 53 | ) -> dict[str, str]: 54 | input_text = self.dialogue_summary_topic_format.format(input_text=input_text) 55 | try: 56 | pred = super().predict( 57 | input_text, max_new_tokens, temperature, top_p, adjust_token_lengths=adjust_token_lengths 58 | )[0] 59 | except IndexError: 60 | print_warning( 61 | "Failed prediction; This could be due to a long input or a small value for `max_new_tokens`. " 62 | "If your input is long, try with `adjust_token_lengths=True`. Try increasing `max_new_tokens` otherwise." 63 | ) 64 | return {"summary": "", "topic": ""} 65 | return self.parse_summary_and_topic(pred) 66 | 67 | def explore(self, input_texts: list[str] = None, temperatures: list[float] = [0.1, 0.3, 0.5, 0.8, 1.0]) -> None: 68 | """This method can be used to explore outputs on a range of temperatures. 69 | This can help in finding the best temperature that suits the user.""" 70 | preds, total_time = [], 0 71 | for input_text in input_texts or TXTS_TO_EXPLORE.split("---"): 72 | input_text = input_text.strip() 73 | print(f"\n{input_text}\n{'-'*110}") 74 | for i, temperature in enumerate(temperatures): 75 | if i != 0: 76 | print() 77 | print(f"Temperature: {temperature}") 78 | st = time() 79 | pred = self.generate_summary_and_topic(input_text, temperature=temperature) 80 | total_time += time() - st 81 | print(f" ↪ summary: {pred['summary']}") 82 | print(f" ↪ topic: {pred['topic']}") 83 | preds.append(pred) 84 | 85 | # This is rough count of the tokens as the model also generates new lines and EOC tokens. 86 | n_tokens = sum( 87 | (len(self.tokenizer.encode(p["summary"])) + len(self.tokenizer.encode(p["topic"])) + 20 for p in preds) 88 | ) 89 | n_tokens_per_sec = n_tokens / total_time 90 | print(f"\n---\nGenerated {len(preds)} predictions in {total_time:.2f}s: {n_tokens_per_sec:.2f} tokens/s") 91 | -------------------------------------------------------------------------------- /llm_toys/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | import json 3 | from typing import TYPE_CHECKING 4 | 5 | from llm_toys.config import DATA_DIR, SUPPORTED_END_TONES 6 | from llm_toys.prompts import DIALOGUE_SUMMARY_TOPIC_TRAIN_FORMAT, PARAPHRASE_TRAIN_FORMAT, TONE_CHANGE_TRAIN_FORMAT 7 | 8 | 9 | if TYPE_CHECKING: 10 | from pathlib import Path 11 | 12 | paraphrase_tone_data_f = DATA_DIR / "paraphrase_tone.json" 13 | dialogue_summary_topic_data_f = DATA_DIR / "dialogue_summary_topic.json" 14 | 15 | 16 | def print_warning(message: str) -> None: 17 | print(f"\033[93mWARNING: {message}\033[0m") 18 | 19 | 20 | def load_json(fname: str | Path) -> dict: 21 | with open(fname, "r") as f: 22 | return json.load(f) 23 | 24 | 25 | def save_json(fname: str | Path, data: dict) -> None: 26 | with open(fname, "w") as f: 27 | json.dump(data, f) 28 | 29 | 30 | def save_text(fname: str | Path, data: str) -> None: 31 | with open(fname, "w") as f: 32 | f.write(data) 33 | 34 | 35 | def token_stats(texts: list[str]) -> None: 36 | from llm_toys.config import DEFAULT_3B_MODEL 37 | from transformers import AutoTokenizer 38 | 39 | tokenier = AutoTokenizer.from_pretrained(DEFAULT_3B_MODEL) 40 | 41 | token_lengths = [len(tokenier.encode(txt)) for txt in texts] 42 | print(f"# Data points : {len(texts)}") 43 | print(f"Min token len : {min(token_lengths)}") 44 | print(f"Max token len : {max(token_lengths)}") 45 | print(f"Mean token len : {int(sum(token_lengths) / len(token_lengths))}") 46 | print(f"Median token len : {sorted(token_lengths)[len(token_lengths) // 2]}") 47 | print(f"Q3 token len : {sorted(token_lengths)[int(len(token_lengths) * 0.75)]}") 48 | 49 | 50 | def paraphrase_tone_training_data(print_token_stats: bool = False) -> list[str]: 51 | """Generates training data for both `Paraphrasing` and `Tone Change`. Also prints out the 52 | token stats for the training data to help with picking the `max_length` during training.""" 53 | data = load_json(paraphrase_tone_data_f) 54 | 55 | training_data = [] 56 | for d in data: 57 | # Tone: Original -> Casual, Professional, Witty 58 | for start_tone in ["original"]: 59 | for end_tone in SUPPORTED_END_TONES: 60 | if start_tone == end_tone: 61 | continue 62 | training_data.append( 63 | TONE_CHANGE_TRAIN_FORMAT.format(tone=end_tone, input_text=d[start_tone], response=d[end_tone]) 64 | ) 65 | # Paraphrase: Original -> Paraphrase 66 | training_data.append(PARAPHRASE_TRAIN_FORMAT.format(input_text=d["original"], response=d["paraphrase"])) 67 | 68 | if print_token_stats: 69 | token_stats(training_data) 70 | 71 | return training_data 72 | 73 | 74 | def dialogue_summary_topic_training_data(print_token_stats: bool = False) -> list[str]: 75 | """Generates training data for both `Summary` and `Topic` generation from dialogues. Also prints out the 76 | token stats for the training data to help with picking the `max_length` during training.""" 77 | data = load_json(dialogue_summary_topic_data_f) 78 | 79 | training_data = [ 80 | DIALOGUE_SUMMARY_TOPIC_TRAIN_FORMAT.format(input_text=d["dialogue"], summary=d["summary"], topic=d["topic"]) 81 | for d in data 82 | ] 83 | 84 | if print_token_stats: 85 | token_stats(training_data) 86 | 87 | return training_data 88 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel"] 3 | build-backend = "setuptools.build_meta" -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==1.4.0 2 | accelerate==0.20.3 3 | aim==3.17.5 4 | aim-ui==3.17.5 5 | aimrecords==0.0.7 6 | aimrocks==0.4.0 7 | aiofiles==23.1.0 8 | alembic==1.11.1 9 | annotated-types==0.5.0 10 | anyio==3.7.1 11 | backoff==2.2.1 12 | base58==2.0.1 13 | bitsandbytes==0.39.1 14 | cachetools==5.3.1 15 | certifi==2023.5.7 16 | cffi==1.15.1 17 | charset-normalizer==3.2.0 18 | click==8.1.5 19 | cmake==3.26.4 20 | cryptography==41.0.2 21 | einops==0.6.1 22 | exceptiongroup==1.1.2 23 | fastapi==0.100.0 24 | filelock==3.12.2 25 | fsspec==2023.6.0 26 | greenlet==2.0.2 27 | grpcio==1.56.0 28 | h11==0.14.0 29 | huggingface-hub==0.16.4 30 | idna==3.4 31 | Jinja2==3.1.2 32 | joblib==1.3.1 33 | lit==16.0.6 34 | Mako==1.2.4 35 | MarkupSafe==2.1.3 36 | monotonic==1.6 37 | mpmath==1.3.0 38 | networkx==3.1 39 | nltk==3.8.1 40 | numpy==1.25.1 41 | nvidia-cublas-cu11==11.10.3.66 42 | nvidia-cuda-cupti-cu11==11.7.101 43 | nvidia-cuda-nvrtc-cu11==11.7.99 44 | nvidia-cuda-runtime-cu11==11.7.99 45 | nvidia-cudnn-cu11==8.5.0.96 46 | nvidia-cufft-cu11==10.9.0.58 47 | nvidia-curand-cu11==10.2.10.91 48 | nvidia-cusolver-cu11==11.4.0.1 49 | nvidia-cusparse-cu11==11.7.4.91 50 | nvidia-nccl-cu11==2.14.3 51 | nvidia-nvtx-cu11==11.7.91 52 | packaging==23.1 53 | Pillow==10.0.0 54 | protobuf==4.23.4 55 | psutil==5.9.5 56 | py3nvml==0.2.7 57 | pycparser==2.21 58 | pydantic==2.0.3 59 | pydantic_core==2.3.0 60 | python-dateutil==2.8.2 61 | pytz==2023.3 62 | PyYAML==6.0 63 | regex==2023.6.3 64 | requests==2.31.0 65 | RestrictedPython==6.1 66 | rouge-score==0.1.2 67 | safetensors==0.3.1 68 | scipy==1.11.1 69 | segment-analytics-python==2.2.3 70 | six==1.16.0 71 | sniffio==1.3.0 72 | SQLAlchemy==1.4.49 73 | starlette==0.27.0 74 | sympy==1.12 75 | tokenizers==0.13.3 76 | torch==2.0.1 77 | tqdm==4.65.0 78 | triton==2.0.0 79 | typing_extensions==4.7.1 80 | urllib3==2.0.3 81 | uvicorn==0.23.0 82 | xmltodict==0.13.0 83 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | import os 3 | import re 4 | from typing import List 5 | 6 | import setuptools 7 | from setuptools import find_packages 8 | 9 | 10 | here = os.path.abspath(os.path.dirname(__file__)) 11 | 12 | 13 | with open("README.md", "r") as fh: 14 | long_description = fh.read() 15 | 16 | 17 | def parse_requirements(file_name: str) -> List[str]: 18 | with open(os.path.join(here, file_name), "r") as f: 19 | return [require.strip() for require in f if require.strip() and not require.startswith("#")] 20 | 21 | 22 | def read(*parts): 23 | with codecs.open(os.path.join(here, *parts), "r") as fp: 24 | return fp.read() 25 | 26 | 27 | def find_version(*file_paths): 28 | version_file = read(*file_paths) 29 | version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M) 30 | if version_match: 31 | return version_match.group(1) 32 | raise RuntimeError("Unable to find version string.") 33 | 34 | 35 | setuptools.setup( 36 | name="llm-toys", 37 | packages=find_packages(), 38 | version=find_version("llm_toys", "__init__.py"), 39 | author="Kumar Utsav", 40 | author_email="krum.utsav@gmail.com", 41 | description="Small(7B and below), production-ready finetuned LLMs for a diverse set of useful tasks.", 42 | long_description=long_description, 43 | long_description_content_type="text/markdown", 44 | install_requires=parse_requirements("requirements.txt"), 45 | url="https://github.com/kuutsav/llm-toys", 46 | python_requires=">=3.10.0", 47 | ) 48 | --------------------------------------------------------------------------------