├── .github └── workflows │ ├── bladellm_bench_test.yml │ ├── bladellm_correctness_test.yml │ ├── bladellm_migration_test.yml │ ├── bladellm_register_service_test.yml │ ├── bladellm_server_test.yml │ ├── bladellm_trace_request_test.yml │ ├── bladellm_unit_test.yml │ ├── offline_test.yml │ ├── pylint.yml │ ├── vllm_bench_test.yml │ ├── vllm_correctness_test.yml │ ├── vllm_migration_test.yml │ ├── vllm_register_service_test.yml │ ├── vllm_trace_request_test.yml │ ├── vllm_unit_test.yml │ └── whl_build.yml ├── .gitignore ├── .gitmodules ├── .pylintrc ├── LICENSE ├── Makefile ├── README.md ├── benchmark └── benchmark_serving.py ├── configs ├── bladellm.yml └── vllm.yml ├── docs ├── Arguments.md ├── Fault_Tolerance.md ├── Prefill-decode_Disaggregation.md ├── Quickstart.md ├── Simulator.md ├── Supported_Models.md ├── pdd_design.png ├── pdd_rationale.png ├── performance.png └── v0.1.0_benchmark.png ├── examples └── offline_inference.py ├── llumnix ├── __init__.py ├── arg_utils.py ├── backends │ ├── __init__.py │ ├── backend_interface.py │ ├── bladellm │ │ ├── __init__.py │ │ ├── llm_engine.py │ │ ├── metrics.py │ │ ├── migration_backend.py │ │ ├── migration_worker.py │ │ ├── proto │ │ │ ├── __init__.py │ │ │ └── migration_worker.proto │ │ ├── protocol.py │ │ ├── scheduler.py │ │ ├── sequence.py │ │ ├── utils.py │ │ ├── worker.py │ │ └── worker_client.py │ ├── envs.py │ ├── migration_backend_interface.py │ ├── output_forwarder.py │ ├── profiling.py │ ├── utils.py │ ├── vllm │ │ ├── __init__.py │ │ ├── executor.py │ │ ├── llm_engine.py │ │ ├── migration_backend.py │ │ ├── scheduler.py │ │ ├── sequence.py │ │ ├── sim_executor.py │ │ ├── sim_llm_engine.py │ │ ├── utils.py │ │ └── worker.py │ ├── vllm_utils.py │ └── vllm_v1 │ │ ├── __init__.py │ │ ├── core.py │ │ ├── executor.py │ │ ├── migration_frontend.py │ │ └── scheduler.py ├── config │ ├── __init__.py │ ├── config.py │ ├── default.py │ ├── mock_query_client_config.json │ └── utils.py ├── constants.py ├── dp_manager.py ├── entrypoints │ ├── __init__.py │ ├── api_server_actor.py │ ├── bladellm │ │ ├── __init__.py │ │ ├── api_server.py │ │ ├── api_server_actor.py │ │ ├── arg_utils.py │ │ ├── client.py │ │ ├── register_service.py │ │ ├── serve.py │ │ └── utils.py │ ├── client.py │ ├── setup.py │ ├── utils.py │ ├── vllm │ │ ├── __init__.py │ │ ├── api_server.py │ │ ├── api_server_actor.py │ │ ├── arg_utils.py │ │ ├── client.py │ │ ├── register_service.py │ │ └── serve.py │ └── vllm_v1 │ │ ├── __init__.py │ │ ├── api_server.py │ │ ├── api_server_actor.py │ │ ├── arg_utils.py │ │ ├── client.py │ │ ├── register_service.py │ │ └── serve.py ├── envs.py ├── global_scheduler │ ├── __init__.py │ ├── dispatch_filter.py │ ├── dispatch_policy.py │ ├── dispatch_scheduler.py │ ├── global_scheduler.py │ ├── migration_filter.py │ ├── migration_policy.py │ ├── migration_scheduler.py │ ├── query_client.py │ ├── scaling_policy.py │ └── scaling_scheduler.py ├── instance_info.py ├── internal_config.py ├── llumlet │ ├── __init__.py │ ├── llumlet.py │ ├── local_migration_scheduler.py │ ├── migration_coordinator.py │ └── request.py ├── load_computation.py ├── logging │ ├── __init__.py │ ├── formatter.py │ ├── handler.py │ └── logger.py ├── manager.py ├── metrics │ ├── __init__.py │ ├── base_metrics.py │ ├── engine_metrics.py │ ├── exporters.py │ ├── global_scheduler_metrics.py │ ├── llumlet_metrics.py │ ├── llumnix_client_metrics.py │ ├── manager_metrics.py │ ├── metrics_types.py │ ├── queue_server_metrics.py │ ├── timestamps.py │ └── utils.py ├── queue │ ├── __init__.py │ ├── queue_client_base.py │ ├── queue_server_base.py │ ├── queue_type.py │ ├── ray_queue_client.py │ ├── ray_queue_server.py │ ├── utils.py │ ├── zmq_client.py │ ├── zmq_server.py │ └── zmq_utils.py ├── ray_utils.py ├── request_output.py ├── request_processing_context.py ├── scaler.py ├── server_info.py ├── utils.py └── version.py ├── pytest.ini ├── requirements ├── requirements_bladellm.txt ├── requirements_vllm.txt └── requirements_vllm_v1.txt ├── setup.py ├── tests ├── __init__.py ├── conftest.py ├── e2e_test │ ├── __init__.py │ ├── bladellm_utils.py │ ├── test_bench.py │ ├── test_correctness.py │ ├── test_migration.py │ ├── test_register_service.py │ ├── test_server.py │ ├── test_trace_request.py │ └── utils.py ├── unit_test │ ├── __init__.py │ ├── args │ │ ├── bladellm │ │ │ └── test_arg_utils.py │ │ └── vllm │ │ │ └── test_arg_utils.py │ ├── backends │ │ ├── __init__.py │ │ ├── vllm │ │ │ ├── __init__.py │ │ │ ├── test_llm_engine.py │ │ │ ├── test_migration.py │ │ │ ├── test_migration_backend.py │ │ │ ├── test_scheduler.py │ │ │ ├── test_simulator.py │ │ │ ├── test_worker.py │ │ │ └── utils.py │ │ └── vllm_v1 │ │ │ └── test_migration.py │ ├── entrypoints │ │ ├── __init__.py │ │ ├── bladellm │ │ │ └── test_client.py │ │ ├── test_utils.py │ │ └── vllm │ │ │ ├── __init__.py │ │ │ ├── api.py │ │ │ ├── api_server.py │ │ │ ├── api_server_actor.py │ │ │ ├── test_api_server.py │ │ │ ├── test_client.py │ │ │ └── test_config_file.py │ ├── global_scheduler │ │ ├── __init__.py │ │ ├── test_dispatch_scheduler.py │ │ ├── test_global_scheduler.py │ │ ├── test_manager.py │ │ ├── test_migration_scheduler.py │ │ └── test_scaler.py │ ├── llumlet │ │ ├── __init__.py │ │ ├── test_llumlet.py │ │ ├── test_local_migration_scheduler.py │ │ └── test_migration_coordinator.py │ ├── logging │ │ └── test_logger.py │ ├── metrics │ │ ├── __init__.py │ │ └── test_time_recorder.py │ ├── queue │ │ ├── __init__.py │ │ ├── test_zmq.py │ │ └── utils.py │ ├── test_dp_manager.py │ └── test_utils.py └── utils.py └── tools ├── cupy_cuda_install.sh ├── dev_image_build.sh ├── docker ├── Dockerfile ├── Dockerfile.bladellm └── Dockerfile.vllm ├── pygloo_install.sh ├── release_image_build.sh └── run_test.sh /.github/workflows/bladellm_bench_test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.github/workflows/bladellm_bench_test.yml -------------------------------------------------------------------------------- /.github/workflows/bladellm_correctness_test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.github/workflows/bladellm_correctness_test.yml -------------------------------------------------------------------------------- /.github/workflows/bladellm_migration_test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.github/workflows/bladellm_migration_test.yml -------------------------------------------------------------------------------- /.github/workflows/bladellm_register_service_test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.github/workflows/bladellm_register_service_test.yml -------------------------------------------------------------------------------- /.github/workflows/bladellm_server_test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.github/workflows/bladellm_server_test.yml -------------------------------------------------------------------------------- /.github/workflows/bladellm_trace_request_test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.github/workflows/bladellm_trace_request_test.yml -------------------------------------------------------------------------------- /.github/workflows/bladellm_unit_test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.github/workflows/bladellm_unit_test.yml -------------------------------------------------------------------------------- /.github/workflows/offline_test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.github/workflows/offline_test.yml -------------------------------------------------------------------------------- /.github/workflows/pylint.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.github/workflows/pylint.yml -------------------------------------------------------------------------------- /.github/workflows/vllm_bench_test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.github/workflows/vllm_bench_test.yml -------------------------------------------------------------------------------- /.github/workflows/vllm_correctness_test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.github/workflows/vllm_correctness_test.yml -------------------------------------------------------------------------------- /.github/workflows/vllm_migration_test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.github/workflows/vllm_migration_test.yml -------------------------------------------------------------------------------- /.github/workflows/vllm_register_service_test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.github/workflows/vllm_register_service_test.yml -------------------------------------------------------------------------------- /.github/workflows/vllm_trace_request_test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.github/workflows/vllm_trace_request_test.yml -------------------------------------------------------------------------------- /.github/workflows/vllm_unit_test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.github/workflows/vllm_unit_test.yml -------------------------------------------------------------------------------- /.github/workflows/whl_build.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.github/workflows/whl_build.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.gitignore -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.gitmodules -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/.pylintrc -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/LICENSE -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/README.md -------------------------------------------------------------------------------- /benchmark/benchmark_serving.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/benchmark/benchmark_serving.py -------------------------------------------------------------------------------- /configs/bladellm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/configs/bladellm.yml -------------------------------------------------------------------------------- /configs/vllm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/configs/vllm.yml -------------------------------------------------------------------------------- /docs/Arguments.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/docs/Arguments.md -------------------------------------------------------------------------------- /docs/Fault_Tolerance.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/docs/Fault_Tolerance.md -------------------------------------------------------------------------------- /docs/Prefill-decode_Disaggregation.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/docs/Prefill-decode_Disaggregation.md -------------------------------------------------------------------------------- /docs/Quickstart.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/docs/Quickstart.md -------------------------------------------------------------------------------- /docs/Simulator.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/docs/Simulator.md -------------------------------------------------------------------------------- /docs/Supported_Models.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/docs/Supported_Models.md -------------------------------------------------------------------------------- /docs/pdd_design.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/docs/pdd_design.png -------------------------------------------------------------------------------- /docs/pdd_rationale.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/docs/pdd_rationale.png -------------------------------------------------------------------------------- /docs/performance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/docs/performance.png -------------------------------------------------------------------------------- /docs/v0.1.0_benchmark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/docs/v0.1.0_benchmark.png -------------------------------------------------------------------------------- /examples/offline_inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/examples/offline_inference.py -------------------------------------------------------------------------------- /llumnix/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/__init__.py -------------------------------------------------------------------------------- /llumnix/arg_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/arg_utils.py -------------------------------------------------------------------------------- /llumnix/backends/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/__init__.py -------------------------------------------------------------------------------- /llumnix/backends/backend_interface.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/backend_interface.py -------------------------------------------------------------------------------- /llumnix/backends/bladellm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/bladellm/__init__.py -------------------------------------------------------------------------------- /llumnix/backends/bladellm/llm_engine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/bladellm/llm_engine.py -------------------------------------------------------------------------------- /llumnix/backends/bladellm/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/bladellm/metrics.py -------------------------------------------------------------------------------- /llumnix/backends/bladellm/migration_backend.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/bladellm/migration_backend.py -------------------------------------------------------------------------------- /llumnix/backends/bladellm/migration_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/bladellm/migration_worker.py -------------------------------------------------------------------------------- /llumnix/backends/bladellm/proto/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/bladellm/proto/__init__.py -------------------------------------------------------------------------------- /llumnix/backends/bladellm/proto/migration_worker.proto: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/bladellm/proto/migration_worker.proto -------------------------------------------------------------------------------- /llumnix/backends/bladellm/protocol.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/bladellm/protocol.py -------------------------------------------------------------------------------- /llumnix/backends/bladellm/scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/bladellm/scheduler.py -------------------------------------------------------------------------------- /llumnix/backends/bladellm/sequence.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/bladellm/sequence.py -------------------------------------------------------------------------------- /llumnix/backends/bladellm/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/bladellm/utils.py -------------------------------------------------------------------------------- /llumnix/backends/bladellm/worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/bladellm/worker.py -------------------------------------------------------------------------------- /llumnix/backends/bladellm/worker_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/bladellm/worker_client.py -------------------------------------------------------------------------------- /llumnix/backends/envs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/envs.py -------------------------------------------------------------------------------- /llumnix/backends/migration_backend_interface.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/migration_backend_interface.py -------------------------------------------------------------------------------- /llumnix/backends/output_forwarder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/output_forwarder.py -------------------------------------------------------------------------------- /llumnix/backends/profiling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/profiling.py -------------------------------------------------------------------------------- /llumnix/backends/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/utils.py -------------------------------------------------------------------------------- /llumnix/backends/vllm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/vllm/__init__.py -------------------------------------------------------------------------------- /llumnix/backends/vllm/executor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/vllm/executor.py -------------------------------------------------------------------------------- /llumnix/backends/vllm/llm_engine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/vllm/llm_engine.py -------------------------------------------------------------------------------- /llumnix/backends/vllm/migration_backend.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/vllm/migration_backend.py -------------------------------------------------------------------------------- /llumnix/backends/vllm/scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/vllm/scheduler.py -------------------------------------------------------------------------------- /llumnix/backends/vllm/sequence.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/vllm/sequence.py -------------------------------------------------------------------------------- /llumnix/backends/vllm/sim_executor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/vllm/sim_executor.py -------------------------------------------------------------------------------- /llumnix/backends/vllm/sim_llm_engine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/vllm/sim_llm_engine.py -------------------------------------------------------------------------------- /llumnix/backends/vllm/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/vllm/utils.py -------------------------------------------------------------------------------- /llumnix/backends/vllm/worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/vllm/worker.py -------------------------------------------------------------------------------- /llumnix/backends/vllm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/vllm_utils.py -------------------------------------------------------------------------------- /llumnix/backends/vllm_v1/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/vllm_v1/__init__.py -------------------------------------------------------------------------------- /llumnix/backends/vllm_v1/core.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/vllm_v1/core.py -------------------------------------------------------------------------------- /llumnix/backends/vllm_v1/executor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/vllm_v1/executor.py -------------------------------------------------------------------------------- /llumnix/backends/vllm_v1/migration_frontend.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/vllm_v1/migration_frontend.py -------------------------------------------------------------------------------- /llumnix/backends/vllm_v1/scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/backends/vllm_v1/scheduler.py -------------------------------------------------------------------------------- /llumnix/config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/config/__init__.py -------------------------------------------------------------------------------- /llumnix/config/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/config/config.py -------------------------------------------------------------------------------- /llumnix/config/default.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/config/default.py -------------------------------------------------------------------------------- /llumnix/config/mock_query_client_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/config/mock_query_client_config.json -------------------------------------------------------------------------------- /llumnix/config/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/config/utils.py -------------------------------------------------------------------------------- /llumnix/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/constants.py -------------------------------------------------------------------------------- /llumnix/dp_manager.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/dp_manager.py -------------------------------------------------------------------------------- /llumnix/entrypoints/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/__init__.py -------------------------------------------------------------------------------- /llumnix/entrypoints/api_server_actor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/api_server_actor.py -------------------------------------------------------------------------------- /llumnix/entrypoints/bladellm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/bladellm/__init__.py -------------------------------------------------------------------------------- /llumnix/entrypoints/bladellm/api_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/bladellm/api_server.py -------------------------------------------------------------------------------- /llumnix/entrypoints/bladellm/api_server_actor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/bladellm/api_server_actor.py -------------------------------------------------------------------------------- /llumnix/entrypoints/bladellm/arg_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/bladellm/arg_utils.py -------------------------------------------------------------------------------- /llumnix/entrypoints/bladellm/client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/bladellm/client.py -------------------------------------------------------------------------------- /llumnix/entrypoints/bladellm/register_service.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/bladellm/register_service.py -------------------------------------------------------------------------------- /llumnix/entrypoints/bladellm/serve.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/bladellm/serve.py -------------------------------------------------------------------------------- /llumnix/entrypoints/bladellm/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/bladellm/utils.py -------------------------------------------------------------------------------- /llumnix/entrypoints/client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/client.py -------------------------------------------------------------------------------- /llumnix/entrypoints/setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/setup.py -------------------------------------------------------------------------------- /llumnix/entrypoints/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/utils.py -------------------------------------------------------------------------------- /llumnix/entrypoints/vllm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/vllm/__init__.py -------------------------------------------------------------------------------- /llumnix/entrypoints/vllm/api_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/vllm/api_server.py -------------------------------------------------------------------------------- /llumnix/entrypoints/vllm/api_server_actor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/vllm/api_server_actor.py -------------------------------------------------------------------------------- /llumnix/entrypoints/vllm/arg_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/vllm/arg_utils.py -------------------------------------------------------------------------------- /llumnix/entrypoints/vllm/client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/vllm/client.py -------------------------------------------------------------------------------- /llumnix/entrypoints/vllm/register_service.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/vllm/register_service.py -------------------------------------------------------------------------------- /llumnix/entrypoints/vllm/serve.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/vllm/serve.py -------------------------------------------------------------------------------- /llumnix/entrypoints/vllm_v1/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/vllm_v1/__init__.py -------------------------------------------------------------------------------- /llumnix/entrypoints/vllm_v1/api_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/vllm_v1/api_server.py -------------------------------------------------------------------------------- /llumnix/entrypoints/vllm_v1/api_server_actor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/vllm_v1/api_server_actor.py -------------------------------------------------------------------------------- /llumnix/entrypoints/vllm_v1/arg_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/vllm_v1/arg_utils.py -------------------------------------------------------------------------------- /llumnix/entrypoints/vllm_v1/client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/vllm_v1/client.py -------------------------------------------------------------------------------- /llumnix/entrypoints/vllm_v1/register_service.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/vllm_v1/register_service.py -------------------------------------------------------------------------------- /llumnix/entrypoints/vllm_v1/serve.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/entrypoints/vllm_v1/serve.py -------------------------------------------------------------------------------- /llumnix/envs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/envs.py -------------------------------------------------------------------------------- /llumnix/global_scheduler/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/global_scheduler/__init__.py -------------------------------------------------------------------------------- /llumnix/global_scheduler/dispatch_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/global_scheduler/dispatch_filter.py -------------------------------------------------------------------------------- /llumnix/global_scheduler/dispatch_policy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/global_scheduler/dispatch_policy.py -------------------------------------------------------------------------------- /llumnix/global_scheduler/dispatch_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/global_scheduler/dispatch_scheduler.py -------------------------------------------------------------------------------- /llumnix/global_scheduler/global_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/global_scheduler/global_scheduler.py -------------------------------------------------------------------------------- /llumnix/global_scheduler/migration_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/global_scheduler/migration_filter.py -------------------------------------------------------------------------------- /llumnix/global_scheduler/migration_policy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/global_scheduler/migration_policy.py -------------------------------------------------------------------------------- /llumnix/global_scheduler/migration_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/global_scheduler/migration_scheduler.py -------------------------------------------------------------------------------- /llumnix/global_scheduler/query_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/global_scheduler/query_client.py -------------------------------------------------------------------------------- /llumnix/global_scheduler/scaling_policy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/global_scheduler/scaling_policy.py -------------------------------------------------------------------------------- /llumnix/global_scheduler/scaling_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/global_scheduler/scaling_scheduler.py -------------------------------------------------------------------------------- /llumnix/instance_info.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/instance_info.py -------------------------------------------------------------------------------- /llumnix/internal_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/internal_config.py -------------------------------------------------------------------------------- /llumnix/llumlet/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/llumlet/__init__.py -------------------------------------------------------------------------------- /llumnix/llumlet/llumlet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/llumlet/llumlet.py -------------------------------------------------------------------------------- /llumnix/llumlet/local_migration_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/llumlet/local_migration_scheduler.py -------------------------------------------------------------------------------- /llumnix/llumlet/migration_coordinator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/llumlet/migration_coordinator.py -------------------------------------------------------------------------------- /llumnix/llumlet/request.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/llumlet/request.py -------------------------------------------------------------------------------- /llumnix/load_computation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/load_computation.py -------------------------------------------------------------------------------- /llumnix/logging/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/logging/__init__.py -------------------------------------------------------------------------------- /llumnix/logging/formatter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/logging/formatter.py -------------------------------------------------------------------------------- /llumnix/logging/handler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/logging/handler.py -------------------------------------------------------------------------------- /llumnix/logging/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/logging/logger.py -------------------------------------------------------------------------------- /llumnix/manager.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/manager.py -------------------------------------------------------------------------------- /llumnix/metrics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/metrics/__init__.py -------------------------------------------------------------------------------- /llumnix/metrics/base_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/metrics/base_metrics.py -------------------------------------------------------------------------------- /llumnix/metrics/engine_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/metrics/engine_metrics.py -------------------------------------------------------------------------------- /llumnix/metrics/exporters.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/metrics/exporters.py -------------------------------------------------------------------------------- /llumnix/metrics/global_scheduler_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/metrics/global_scheduler_metrics.py -------------------------------------------------------------------------------- /llumnix/metrics/llumlet_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/metrics/llumlet_metrics.py -------------------------------------------------------------------------------- /llumnix/metrics/llumnix_client_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/metrics/llumnix_client_metrics.py -------------------------------------------------------------------------------- /llumnix/metrics/manager_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/metrics/manager_metrics.py -------------------------------------------------------------------------------- /llumnix/metrics/metrics_types.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/metrics/metrics_types.py -------------------------------------------------------------------------------- /llumnix/metrics/queue_server_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/metrics/queue_server_metrics.py -------------------------------------------------------------------------------- /llumnix/metrics/timestamps.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/metrics/timestamps.py -------------------------------------------------------------------------------- /llumnix/metrics/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/metrics/utils.py -------------------------------------------------------------------------------- /llumnix/queue/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/queue/__init__.py -------------------------------------------------------------------------------- /llumnix/queue/queue_client_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/queue/queue_client_base.py -------------------------------------------------------------------------------- /llumnix/queue/queue_server_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/queue/queue_server_base.py -------------------------------------------------------------------------------- /llumnix/queue/queue_type.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/queue/queue_type.py -------------------------------------------------------------------------------- /llumnix/queue/ray_queue_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/queue/ray_queue_client.py -------------------------------------------------------------------------------- /llumnix/queue/ray_queue_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/queue/ray_queue_server.py -------------------------------------------------------------------------------- /llumnix/queue/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/queue/utils.py -------------------------------------------------------------------------------- /llumnix/queue/zmq_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/queue/zmq_client.py -------------------------------------------------------------------------------- /llumnix/queue/zmq_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/queue/zmq_server.py -------------------------------------------------------------------------------- /llumnix/queue/zmq_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/queue/zmq_utils.py -------------------------------------------------------------------------------- /llumnix/ray_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/ray_utils.py -------------------------------------------------------------------------------- /llumnix/request_output.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/request_output.py -------------------------------------------------------------------------------- /llumnix/request_processing_context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/request_processing_context.py -------------------------------------------------------------------------------- /llumnix/scaler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/scaler.py -------------------------------------------------------------------------------- /llumnix/server_info.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/server_info.py -------------------------------------------------------------------------------- /llumnix/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/utils.py -------------------------------------------------------------------------------- /llumnix/version.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/llumnix/version.py -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/pytest.ini -------------------------------------------------------------------------------- /requirements/requirements_bladellm.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/requirements/requirements_bladellm.txt -------------------------------------------------------------------------------- /requirements/requirements_vllm.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/requirements/requirements_vllm.txt -------------------------------------------------------------------------------- /requirements/requirements_vllm_v1.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/requirements/requirements_vllm_v1.txt -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/setup.py -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/conftest.py -------------------------------------------------------------------------------- /tests/e2e_test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/e2e_test/__init__.py -------------------------------------------------------------------------------- /tests/e2e_test/bladellm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/e2e_test/bladellm_utils.py -------------------------------------------------------------------------------- /tests/e2e_test/test_bench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/e2e_test/test_bench.py -------------------------------------------------------------------------------- /tests/e2e_test/test_correctness.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/e2e_test/test_correctness.py -------------------------------------------------------------------------------- /tests/e2e_test/test_migration.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/e2e_test/test_migration.py -------------------------------------------------------------------------------- /tests/e2e_test/test_register_service.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/e2e_test/test_register_service.py -------------------------------------------------------------------------------- /tests/e2e_test/test_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/e2e_test/test_server.py -------------------------------------------------------------------------------- /tests/e2e_test/test_trace_request.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/e2e_test/test_trace_request.py -------------------------------------------------------------------------------- /tests/e2e_test/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/e2e_test/utils.py -------------------------------------------------------------------------------- /tests/unit_test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/__init__.py -------------------------------------------------------------------------------- /tests/unit_test/args/bladellm/test_arg_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/args/bladellm/test_arg_utils.py -------------------------------------------------------------------------------- /tests/unit_test/args/vllm/test_arg_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/args/vllm/test_arg_utils.py -------------------------------------------------------------------------------- /tests/unit_test/backends/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/backends/__init__.py -------------------------------------------------------------------------------- /tests/unit_test/backends/vllm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/backends/vllm/__init__.py -------------------------------------------------------------------------------- /tests/unit_test/backends/vllm/test_llm_engine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/backends/vllm/test_llm_engine.py -------------------------------------------------------------------------------- /tests/unit_test/backends/vllm/test_migration.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/backends/vllm/test_migration.py -------------------------------------------------------------------------------- /tests/unit_test/backends/vllm/test_migration_backend.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/backends/vllm/test_migration_backend.py -------------------------------------------------------------------------------- /tests/unit_test/backends/vllm/test_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/backends/vllm/test_scheduler.py -------------------------------------------------------------------------------- /tests/unit_test/backends/vllm/test_simulator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/backends/vllm/test_simulator.py -------------------------------------------------------------------------------- /tests/unit_test/backends/vllm/test_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/backends/vllm/test_worker.py -------------------------------------------------------------------------------- /tests/unit_test/backends/vllm/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/backends/vllm/utils.py -------------------------------------------------------------------------------- /tests/unit_test/backends/vllm_v1/test_migration.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/backends/vllm_v1/test_migration.py -------------------------------------------------------------------------------- /tests/unit_test/entrypoints/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/entrypoints/__init__.py -------------------------------------------------------------------------------- /tests/unit_test/entrypoints/bladellm/test_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/entrypoints/bladellm/test_client.py -------------------------------------------------------------------------------- /tests/unit_test/entrypoints/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/entrypoints/test_utils.py -------------------------------------------------------------------------------- /tests/unit_test/entrypoints/vllm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/entrypoints/vllm/__init__.py -------------------------------------------------------------------------------- /tests/unit_test/entrypoints/vllm/api.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/entrypoints/vllm/api.py -------------------------------------------------------------------------------- /tests/unit_test/entrypoints/vllm/api_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/entrypoints/vllm/api_server.py -------------------------------------------------------------------------------- /tests/unit_test/entrypoints/vllm/api_server_actor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/entrypoints/vllm/api_server_actor.py -------------------------------------------------------------------------------- /tests/unit_test/entrypoints/vllm/test_api_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/entrypoints/vllm/test_api_server.py -------------------------------------------------------------------------------- /tests/unit_test/entrypoints/vllm/test_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/entrypoints/vllm/test_client.py -------------------------------------------------------------------------------- /tests/unit_test/entrypoints/vllm/test_config_file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/entrypoints/vllm/test_config_file.py -------------------------------------------------------------------------------- /tests/unit_test/global_scheduler/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/global_scheduler/__init__.py -------------------------------------------------------------------------------- /tests/unit_test/global_scheduler/test_dispatch_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/global_scheduler/test_dispatch_scheduler.py -------------------------------------------------------------------------------- /tests/unit_test/global_scheduler/test_global_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/global_scheduler/test_global_scheduler.py -------------------------------------------------------------------------------- /tests/unit_test/global_scheduler/test_manager.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/global_scheduler/test_manager.py -------------------------------------------------------------------------------- /tests/unit_test/global_scheduler/test_migration_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/global_scheduler/test_migration_scheduler.py -------------------------------------------------------------------------------- /tests/unit_test/global_scheduler/test_scaler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/global_scheduler/test_scaler.py -------------------------------------------------------------------------------- /tests/unit_test/llumlet/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/llumlet/__init__.py -------------------------------------------------------------------------------- /tests/unit_test/llumlet/test_llumlet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/llumlet/test_llumlet.py -------------------------------------------------------------------------------- /tests/unit_test/llumlet/test_local_migration_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/llumlet/test_local_migration_scheduler.py -------------------------------------------------------------------------------- /tests/unit_test/llumlet/test_migration_coordinator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/llumlet/test_migration_coordinator.py -------------------------------------------------------------------------------- /tests/unit_test/logging/test_logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/logging/test_logger.py -------------------------------------------------------------------------------- /tests/unit_test/metrics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/metrics/__init__.py -------------------------------------------------------------------------------- /tests/unit_test/metrics/test_time_recorder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/metrics/test_time_recorder.py -------------------------------------------------------------------------------- /tests/unit_test/queue/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/queue/__init__.py -------------------------------------------------------------------------------- /tests/unit_test/queue/test_zmq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/queue/test_zmq.py -------------------------------------------------------------------------------- /tests/unit_test/queue/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/queue/utils.py -------------------------------------------------------------------------------- /tests/unit_test/test_dp_manager.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/test_dp_manager.py -------------------------------------------------------------------------------- /tests/unit_test/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/unit_test/test_utils.py -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tests/utils.py -------------------------------------------------------------------------------- /tools/cupy_cuda_install.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tools/cupy_cuda_install.sh -------------------------------------------------------------------------------- /tools/dev_image_build.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tools/dev_image_build.sh -------------------------------------------------------------------------------- /tools/docker/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tools/docker/Dockerfile -------------------------------------------------------------------------------- /tools/docker/Dockerfile.bladellm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tools/docker/Dockerfile.bladellm -------------------------------------------------------------------------------- /tools/docker/Dockerfile.vllm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tools/docker/Dockerfile.vllm -------------------------------------------------------------------------------- /tools/pygloo_install.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tools/pygloo_install.sh -------------------------------------------------------------------------------- /tools/release_image_build.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tools/release_image_build.sh -------------------------------------------------------------------------------- /tools/run_test.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlibabaPAI/llumnix/HEAD/tools/run_test.sh --------------------------------------------------------------------------------