├── .gitignore ├── .pre-commit-config.yaml ├── ACKNOWLEDGEMENTS.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── LICENSE_WEIGHTS ├── README.md ├── assets ├── flextok_pull_darkmode.png └── flextok_pull_lightmode.png ├── flextok ├── __init__.py ├── flextok_wrapper.py ├── flow_matching │ ├── __init__.py │ ├── cfg_utils.py │ ├── noise_modules.py │ └── pipelines.py ├── model │ ├── __init__.py │ ├── layers │ │ ├── __init__.py │ │ ├── attention.py │ │ ├── drop_path.py │ │ ├── mlp.py │ │ ├── mup_readout.py │ │ ├── norm.py │ │ └── transformer_blocks.py │ ├── postprocessors │ │ ├── __init__.py │ │ ├── heads.py │ │ └── seq_unpacking.py │ ├── preprocessors │ │ ├── __init__.py │ │ ├── flex_seq_packing.py │ │ ├── linear.py │ │ ├── mask_tokens.py │ │ ├── nullcond.py │ │ ├── patching.py │ │ ├── registers.py │ │ ├── time_embedding.py │ │ └── token_dropout.py │ ├── trunks │ │ ├── __init__.py │ │ └── transformers.py │ └── utils │ │ ├── __init__.py │ │ ├── dict_ops.py │ │ ├── packed_ops.py │ │ ├── posembs.py │ │ └── wrappers.py ├── regularizers │ ├── __init__.py │ └── quantize_fsq.py ├── utils │ ├── __init__.py │ ├── checkpoint.py │ ├── demo.py │ └── misc.py └── vae_wrapper.py ├── notebooks └── flextok_inference.ipynb └── pyproject.toml /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /ACKNOWLEDGEMENTS.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/ACKNOWLEDGEMENTS.md -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/LICENSE -------------------------------------------------------------------------------- /LICENSE_WEIGHTS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/LICENSE_WEIGHTS -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/README.md -------------------------------------------------------------------------------- /assets/flextok_pull_darkmode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/assets/flextok_pull_darkmode.png -------------------------------------------------------------------------------- /assets/flextok_pull_lightmode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/assets/flextok_pull_lightmode.png -------------------------------------------------------------------------------- /flextok/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/__init__.py -------------------------------------------------------------------------------- /flextok/flextok_wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/flextok_wrapper.py -------------------------------------------------------------------------------- /flextok/flow_matching/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/flow_matching/__init__.py -------------------------------------------------------------------------------- /flextok/flow_matching/cfg_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/flow_matching/cfg_utils.py -------------------------------------------------------------------------------- /flextok/flow_matching/noise_modules.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/flow_matching/noise_modules.py -------------------------------------------------------------------------------- /flextok/flow_matching/pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/flow_matching/pipelines.py -------------------------------------------------------------------------------- /flextok/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/__init__.py -------------------------------------------------------------------------------- /flextok/model/layers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/layers/__init__.py -------------------------------------------------------------------------------- /flextok/model/layers/attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/layers/attention.py -------------------------------------------------------------------------------- /flextok/model/layers/drop_path.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/layers/drop_path.py -------------------------------------------------------------------------------- /flextok/model/layers/mlp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/layers/mlp.py -------------------------------------------------------------------------------- /flextok/model/layers/mup_readout.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/layers/mup_readout.py -------------------------------------------------------------------------------- /flextok/model/layers/norm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/layers/norm.py -------------------------------------------------------------------------------- /flextok/model/layers/transformer_blocks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/layers/transformer_blocks.py -------------------------------------------------------------------------------- /flextok/model/postprocessors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/postprocessors/__init__.py -------------------------------------------------------------------------------- /flextok/model/postprocessors/heads.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/postprocessors/heads.py -------------------------------------------------------------------------------- /flextok/model/postprocessors/seq_unpacking.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/postprocessors/seq_unpacking.py -------------------------------------------------------------------------------- /flextok/model/preprocessors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/preprocessors/__init__.py -------------------------------------------------------------------------------- /flextok/model/preprocessors/flex_seq_packing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/preprocessors/flex_seq_packing.py -------------------------------------------------------------------------------- /flextok/model/preprocessors/linear.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/preprocessors/linear.py -------------------------------------------------------------------------------- /flextok/model/preprocessors/mask_tokens.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/preprocessors/mask_tokens.py -------------------------------------------------------------------------------- /flextok/model/preprocessors/nullcond.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/preprocessors/nullcond.py -------------------------------------------------------------------------------- /flextok/model/preprocessors/patching.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/preprocessors/patching.py -------------------------------------------------------------------------------- /flextok/model/preprocessors/registers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/preprocessors/registers.py -------------------------------------------------------------------------------- /flextok/model/preprocessors/time_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/preprocessors/time_embedding.py -------------------------------------------------------------------------------- /flextok/model/preprocessors/token_dropout.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/preprocessors/token_dropout.py -------------------------------------------------------------------------------- /flextok/model/trunks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/trunks/__init__.py -------------------------------------------------------------------------------- /flextok/model/trunks/transformers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/trunks/transformers.py -------------------------------------------------------------------------------- /flextok/model/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/utils/__init__.py -------------------------------------------------------------------------------- /flextok/model/utils/dict_ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/utils/dict_ops.py -------------------------------------------------------------------------------- /flextok/model/utils/packed_ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/utils/packed_ops.py -------------------------------------------------------------------------------- /flextok/model/utils/posembs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/utils/posembs.py -------------------------------------------------------------------------------- /flextok/model/utils/wrappers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/model/utils/wrappers.py -------------------------------------------------------------------------------- /flextok/regularizers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/regularizers/__init__.py -------------------------------------------------------------------------------- /flextok/regularizers/quantize_fsq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/regularizers/quantize_fsq.py -------------------------------------------------------------------------------- /flextok/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/utils/__init__.py -------------------------------------------------------------------------------- /flextok/utils/checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/utils/checkpoint.py -------------------------------------------------------------------------------- /flextok/utils/demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/utils/demo.py -------------------------------------------------------------------------------- /flextok/utils/misc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/utils/misc.py -------------------------------------------------------------------------------- /flextok/vae_wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/flextok/vae_wrapper.py -------------------------------------------------------------------------------- /notebooks/flextok_inference.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/notebooks/flextok_inference.ipynb -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple/ml-flextok/HEAD/pyproject.toml --------------------------------------------------------------------------------