├── .gitignore ├── .gitmodules ├── README.md ├── README_zh.md ├── cuda ├── wkv5_cuda.cu ├── wkv5_op.cpp ├── wkv6_cuda.cu ├── wkv6_op.cpp ├── wkv6infctx_cuda.cu ├── wkv6infctx_op.cpp ├── wkv6state_cuda.cu ├── wkv6state_op.cpp ├── wkv7_cuda.cu └── wkv7_op.cpp ├── docs ├── 03-Confusing-Pictures.jpg └── Streaming input TTS.md ├── eval ├── api.sh ├── hf_api.sh ├── hf_server.py ├── hf_test.py ├── test.py └── vlm_eval.sh ├── infer ├── __init__.py ├── rwkv │ ├── __init__.py │ ├── cuda │ │ ├── gemm_fp16_cublas.cpp │ │ ├── operators.cu │ │ ├── rwkv5.cu │ │ ├── rwkv5_op.cpp │ │ ├── rwkv6.cu │ │ ├── rwkv6_op.cpp │ │ ├── rwkv7.cu │ │ ├── rwkv7_op.cpp │ │ └── wrapper.cpp │ ├── hip │ │ ├── gemm_fp16_cublas.cpp │ │ ├── operators.hip │ │ ├── rwkv7.hip │ │ ├── rwkv7_op.cpp │ │ └── wrapper.cpp │ ├── model.py │ ├── rwkv_tokenizer.py │ ├── utils.py │ └── wr_vocab_v20230424.txt ├── server.py └── worldmodel.py ├── requirements.txt ├── scripts ├── asr.sh ├── stt.sh └── vtt.sh ├── src ├── __init__.py ├── binidx.py ├── operator │ └── rwkvop.py ├── rwkv7 │ ├── __init__.py │ ├── att.py │ ├── block.py │ ├── ffn.py │ └── model.py ├── trainer.py ├── trick │ ├── __init__.py │ └── lrs.py └── utils.py ├── train.py ├── web ├── audio_gen.py ├── audio_multiturns_web.py ├── audio_web.py ├── check_img.py ├── multimodal_web.py ├── visual_gen.py └── visual_web.py ├── wlm ├── __init__.py ├── configuration_wlm.py ├── encoder │ ├── clip_encoder.py │ ├── siglip_encoder.py │ ├── speech_encoder.py │ ├── visual_encoder.py │ └── whisper_encoder.py ├── modeling_rwkv7_wlm.py ├── processing_wlm.py ├── projector │ ├── __init__.py │ └── test.py ├── pyproject.toml ├── registry.py └── wlm.egg-info │ ├── PKG-INFO │ ├── SOURCES.txt │ ├── dependency_links.txt │ └── top_level.txt └── world ├── __init__.py ├── dataset.py ├── encoder ├── clip_encoder.py ├── siglip_encoder.py ├── speech_encoder.py ├── visual_encoder.py └── whisper_encoder.py ├── loss.py ├── model.py ├── prepare └── make_vtensor.py ├── projector ├── __init__.py └── test.py ├── registry.py ├── utils.py └── world_load.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/.gitignore -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/.gitmodules -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/README.md -------------------------------------------------------------------------------- /README_zh.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/README_zh.md -------------------------------------------------------------------------------- /cuda/wkv5_cuda.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/cuda/wkv5_cuda.cu -------------------------------------------------------------------------------- /cuda/wkv5_op.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/cuda/wkv5_op.cpp -------------------------------------------------------------------------------- /cuda/wkv6_cuda.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/cuda/wkv6_cuda.cu -------------------------------------------------------------------------------- /cuda/wkv6_op.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/cuda/wkv6_op.cpp -------------------------------------------------------------------------------- /cuda/wkv6infctx_cuda.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/cuda/wkv6infctx_cuda.cu -------------------------------------------------------------------------------- /cuda/wkv6infctx_op.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/cuda/wkv6infctx_op.cpp -------------------------------------------------------------------------------- /cuda/wkv6state_cuda.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/cuda/wkv6state_cuda.cu -------------------------------------------------------------------------------- /cuda/wkv6state_op.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/cuda/wkv6state_op.cpp -------------------------------------------------------------------------------- /cuda/wkv7_cuda.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/cuda/wkv7_cuda.cu -------------------------------------------------------------------------------- /cuda/wkv7_op.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/cuda/wkv7_op.cpp -------------------------------------------------------------------------------- /docs/03-Confusing-Pictures.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/docs/03-Confusing-Pictures.jpg -------------------------------------------------------------------------------- /docs/Streaming input TTS.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/docs/Streaming input TTS.md -------------------------------------------------------------------------------- /eval/api.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/eval/api.sh -------------------------------------------------------------------------------- /eval/hf_api.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/eval/hf_api.sh -------------------------------------------------------------------------------- /eval/hf_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/eval/hf_server.py -------------------------------------------------------------------------------- /eval/hf_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/eval/hf_test.py -------------------------------------------------------------------------------- /eval/test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/eval/test.py -------------------------------------------------------------------------------- /eval/vlm_eval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/eval/vlm_eval.sh -------------------------------------------------------------------------------- /infer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /infer/rwkv/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /infer/rwkv/cuda/gemm_fp16_cublas.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/rwkv/cuda/gemm_fp16_cublas.cpp -------------------------------------------------------------------------------- /infer/rwkv/cuda/operators.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/rwkv/cuda/operators.cu -------------------------------------------------------------------------------- /infer/rwkv/cuda/rwkv5.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/rwkv/cuda/rwkv5.cu -------------------------------------------------------------------------------- /infer/rwkv/cuda/rwkv5_op.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/rwkv/cuda/rwkv5_op.cpp -------------------------------------------------------------------------------- /infer/rwkv/cuda/rwkv6.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/rwkv/cuda/rwkv6.cu -------------------------------------------------------------------------------- /infer/rwkv/cuda/rwkv6_op.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/rwkv/cuda/rwkv6_op.cpp -------------------------------------------------------------------------------- /infer/rwkv/cuda/rwkv7.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/rwkv/cuda/rwkv7.cu -------------------------------------------------------------------------------- /infer/rwkv/cuda/rwkv7_op.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/rwkv/cuda/rwkv7_op.cpp -------------------------------------------------------------------------------- /infer/rwkv/cuda/wrapper.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/rwkv/cuda/wrapper.cpp -------------------------------------------------------------------------------- /infer/rwkv/hip/gemm_fp16_cublas.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/rwkv/hip/gemm_fp16_cublas.cpp -------------------------------------------------------------------------------- /infer/rwkv/hip/operators.hip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/rwkv/hip/operators.hip -------------------------------------------------------------------------------- /infer/rwkv/hip/rwkv7.hip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/rwkv/hip/rwkv7.hip -------------------------------------------------------------------------------- /infer/rwkv/hip/rwkv7_op.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/rwkv/hip/rwkv7_op.cpp -------------------------------------------------------------------------------- /infer/rwkv/hip/wrapper.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/rwkv/hip/wrapper.cpp -------------------------------------------------------------------------------- /infer/rwkv/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/rwkv/model.py -------------------------------------------------------------------------------- /infer/rwkv/rwkv_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/rwkv/rwkv_tokenizer.py -------------------------------------------------------------------------------- /infer/rwkv/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/rwkv/utils.py -------------------------------------------------------------------------------- /infer/rwkv/wr_vocab_v20230424.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/rwkv/wr_vocab_v20230424.txt -------------------------------------------------------------------------------- /infer/server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/server.py -------------------------------------------------------------------------------- /infer/worldmodel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/infer/worldmodel.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/asr.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/scripts/asr.sh -------------------------------------------------------------------------------- /scripts/stt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/scripts/stt.sh -------------------------------------------------------------------------------- /scripts/vtt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/scripts/vtt.sh -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/binidx.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/src/binidx.py -------------------------------------------------------------------------------- /src/operator/rwkvop.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/src/operator/rwkvop.py -------------------------------------------------------------------------------- /src/rwkv7/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/rwkv7/att.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/src/rwkv7/att.py -------------------------------------------------------------------------------- /src/rwkv7/block.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/src/rwkv7/block.py -------------------------------------------------------------------------------- /src/rwkv7/ffn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/src/rwkv7/ffn.py -------------------------------------------------------------------------------- /src/rwkv7/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/src/rwkv7/model.py -------------------------------------------------------------------------------- /src/trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/src/trainer.py -------------------------------------------------------------------------------- /src/trick/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/trick/lrs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/src/trick/lrs.py -------------------------------------------------------------------------------- /src/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/src/utils.py -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/train.py -------------------------------------------------------------------------------- /web/audio_gen.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/web/audio_gen.py -------------------------------------------------------------------------------- /web/audio_multiturns_web.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/web/audio_multiturns_web.py -------------------------------------------------------------------------------- /web/audio_web.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/web/audio_web.py -------------------------------------------------------------------------------- /web/check_img.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/web/check_img.py -------------------------------------------------------------------------------- /web/multimodal_web.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/web/multimodal_web.py -------------------------------------------------------------------------------- /web/visual_gen.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/web/visual_gen.py -------------------------------------------------------------------------------- /web/visual_web.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/web/visual_web.py -------------------------------------------------------------------------------- /wlm/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /wlm/configuration_wlm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/wlm/configuration_wlm.py -------------------------------------------------------------------------------- /wlm/encoder/clip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/wlm/encoder/clip_encoder.py -------------------------------------------------------------------------------- /wlm/encoder/siglip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/wlm/encoder/siglip_encoder.py -------------------------------------------------------------------------------- /wlm/encoder/speech_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/wlm/encoder/speech_encoder.py -------------------------------------------------------------------------------- /wlm/encoder/visual_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/wlm/encoder/visual_encoder.py -------------------------------------------------------------------------------- /wlm/encoder/whisper_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/wlm/encoder/whisper_encoder.py -------------------------------------------------------------------------------- /wlm/modeling_rwkv7_wlm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/wlm/modeling_rwkv7_wlm.py -------------------------------------------------------------------------------- /wlm/processing_wlm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/wlm/processing_wlm.py -------------------------------------------------------------------------------- /wlm/projector/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /wlm/projector/test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/wlm/projector/test.py -------------------------------------------------------------------------------- /wlm/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/wlm/pyproject.toml -------------------------------------------------------------------------------- /wlm/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/wlm/registry.py -------------------------------------------------------------------------------- /wlm/wlm.egg-info/PKG-INFO: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/wlm/wlm.egg-info/PKG-INFO -------------------------------------------------------------------------------- /wlm/wlm.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/wlm/wlm.egg-info/SOURCES.txt -------------------------------------------------------------------------------- /wlm/wlm.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /wlm/wlm.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | encoder 2 | projector 3 | -------------------------------------------------------------------------------- /world/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /world/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/world/dataset.py -------------------------------------------------------------------------------- /world/encoder/clip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/world/encoder/clip_encoder.py -------------------------------------------------------------------------------- /world/encoder/siglip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/world/encoder/siglip_encoder.py -------------------------------------------------------------------------------- /world/encoder/speech_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/world/encoder/speech_encoder.py -------------------------------------------------------------------------------- /world/encoder/visual_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/world/encoder/visual_encoder.py -------------------------------------------------------------------------------- /world/encoder/whisper_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/world/encoder/whisper_encoder.py -------------------------------------------------------------------------------- /world/loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/world/loss.py -------------------------------------------------------------------------------- /world/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/world/model.py -------------------------------------------------------------------------------- /world/prepare/make_vtensor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/world/prepare/make_vtensor.py -------------------------------------------------------------------------------- /world/projector/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /world/projector/test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/world/projector/test.py -------------------------------------------------------------------------------- /world/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/world/registry.py -------------------------------------------------------------------------------- /world/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/world/utils.py -------------------------------------------------------------------------------- /world/world_load.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Joluck/WorldRWKV/HEAD/world/world_load.py --------------------------------------------------------------------------------