├── .gitignore ├── LICENSE ├── README.md ├── azure_openai ├── REAMDE.md ├── function_calling │ ├── data │ │ ├── flight.db │ │ ├── flight_db.db │ │ ├── flight_policy.json │ │ ├── flights.db │ │ ├── function_call.jsonl │ │ ├── function_call1.jsonl │ │ ├── hotel.db │ │ ├── hotel_policy.json │ │ ├── policy_vector.json │ │ ├── test.jsonl │ │ ├── test_data.jsonl │ │ ├── train.jsonl │ │ └── user_profile.json │ ├── generate_training_data.py │ ├── prompt.txt │ └── test.py ├── python_analytic │ ├── data │ │ ├── processed_test_data_v3.json │ │ ├── processed_test_data_v5.json │ │ ├── processed_test_data_v6.json │ │ ├── processed_train_data_v3.json │ │ ├── processed_train_data_v5.json │ │ ├── processed_train_data_v6.json │ │ ├── python_test.jsonl │ │ ├── python_test_0908.jsonl │ │ ├── python_test_0909.jsonl │ │ ├── python_train.jsonl │ │ ├── python_train_0908.jsonl │ │ ├── python_train_0909.jsonl │ │ ├── test_data_v2.json │ │ ├── test_data_v3.json │ │ ├── train_data.json │ │ ├── train_data_v2.json │ │ └── train_data_v3.json │ ├── evaluation │ │ ├── evaluation.ipynb │ │ ├── evaluation_results_python.csv │ │ ├── evaluation_results_python21.csv │ │ ├── evaluation_results_pythonft_0908.csv │ │ ├── evaluation_results_pythonft_0908_1.csv │ │ ├── evaluation_results_pythonft_0908_2.csv │ │ ├── evaluation_results_pythonft_0908_v5.csv │ │ └── evaluation_results_pythonft_0909_v1.csv │ └── gen_data.ipynb └── sql_gen │ ├── .env.sample │ ├── README.md │ ├── data │ ├── analytic_graph.json │ ├── data_gen_program.py │ ├── northwind.db │ ├── open_ai_data_v2.jsonl │ └── open_ai_data_with_reasoning_v2.jsonl │ ├── evaluate_fine_tuned_model.py │ ├── evaluation │ └── compare_deployments_concurrent.py │ └── ft_job_and_deploy.py ├── azureopenai └── README.md └── opensource_llm ├── README.md ├── archived ├── azureml_unsloth │ ├── pipeline │ │ ├── conda.yml │ │ ├── fine_tune.yml │ │ ├── fine_tune_src │ │ │ └── fine_tune.py │ │ ├── pipeline.yml │ │ ├── pipeline2.yml │ │ ├── pretrain.yml │ │ └── pretrain_src │ │ │ └── pretrain.py │ └── pretrain │ │ ├── conda.yml │ │ ├── hyperparam_tune.yml │ │ ├── main.py │ │ └── pipeline.yml ├── conda.yml ├── data_prep.ipynb ├── data_processing │ └── generate_osllm_ft_data_short.py ├── evaluate_component.yml ├── evaluate_component_bk.yml ├── evaluate_vllm.py ├── fine_tune_ac.py ├── merge_component.yml ├── pipeline.yml ├── pipeline_fsdp.yml ├── pipeline_fsdp_bk.yml ├── pipelinellama.yml ├── pipelinellama_fsdp.yml ├── pipelinellama_vllm.yml ├── quick_fine_tune.ipynb ├── test.ipynb ├── train_component.yml └── train_component_fsdp.yml ├── data ├── all_scenarios_questions_v1.json ├── all_scenarios_questions_v1_unique.json ├── analytic_graph.json ├── analytic_graph_v2.json ├── compare_deployments_concurrent.py ├── compare_deployments_concurrent_v2.py ├── compare_openai_deployments.py ├── compare_queries.py ├── dedupe_questions.py ├── dedupe_questions_v2.py ├── detailed_query_results_v2.0.xlsx ├── detailed_query_results_v2.1.xlsx ├── gen_openai.py ├── gen_openai_v2.py ├── gen_openai_v3.py ├── hs6_descriptions.txt ├── hs_chapters.json ├── hscodes.csv ├── northwind.db ├── open_ai_data.jsonl ├── open_ai_data_train.jsonl ├── open_ai_data_v2.jsonl ├── open_ai_data_val.jsonl ├── open_ai_data_with_reasoning.jsonl ├── open_ai_data_with_reasoning_v2.jsonl ├── question_gen.py ├── question_gen_v2.py ├── question_gen_v4.py ├── questions_by_difficulty_v1.json ├── questions_v2.json ├── questions_v3.json ├── questions_v4.json ├── questions_v4_deduped.json ├── refine_sql_queries.py ├── results.jsonl ├── split_data.py ├── sql_examples.jsonl ├── sql_gen_deepseek.py ├── sql_gen_deepseek_v2.py ├── sql_gen_deepseek_v3.py ├── sql_gen_deepseek_v4.py ├── sql_gen_deepseek_v5.py ├── sql_gen_o3.py ├── sql_result__test_v2.jsonl ├── sql_result_test_v5.jsonl ├── sql_result_train_v5.jsonl ├── sql_result_v2.jsonl ├── sql_result_v4.jsonl ├── sql_result_v5.jsonl ├── sql_results_refined.jsonl ├── test_deepseek.py └── test_sql.ipynb ├── deployment ├── aml │ ├── conda.yml │ ├── deployment.yml │ ├── endpoint.yml │ ├── instance_type.yaml │ ├── k8s_deployment.yml │ ├── k8s_endpoint.yml │ ├── nvidia_device.yaml │ ├── score.py │ ├── score_chat.py │ ├── score_chat70b.py │ ├── script.txt │ └── test_deployent.ipynb ├── aml_ray │ ├── command.txt │ ├── conda.yml │ ├── deployment.yml │ ├── endpoint.yml │ ├── request.json │ └── score.py └── aml_vllm │ ├── Dockerfile │ ├── command.sh │ ├── create_or_update.ps1 │ ├── data │ ├── 106.png │ └── 152.png │ ├── endpoint.yml │ ├── environment.yml │ ├── test_gpt4omini.py │ ├── test_inference.py │ ├── test_inference_single.py │ ├── test_scoring.py │ ├── vllm__custom_model_deployment.yml │ └── vllm_deployment.yml ├── pipelines ├── analytic_graph copy.json ├── analytic_graph.json ├── azureml_hf_sft │ ├── configs │ │ ├── ds_config.json │ │ ├── ds_config_stage1.json │ │ ├── ds_config_stage2.json │ │ └── fsdp_config.yaml │ ├── docker-context │ │ ├── Dockerfile │ │ └── requirements.txt │ ├── evaluate │ │ ├── evaluate.py │ │ ├── evaluate_bk.py │ │ └── merge.py │ └── train │ │ └── train_bk.py ├── configs │ ├── deepspeed_config_zero3.yaml │ ├── ds_config.json │ ├── ds_config_stage1.json │ ├── ds_config_stage2.json │ └── fsdp_config.yaml ├── data │ ├── northwind.db │ └── test_sql.ipynb ├── docker-context │ ├── Dockerfile │ └── requirements.txt ├── evaluate.py ├── northwind.db ├── pipeline.yml ├── quick_fine_tune.ipynb ├── run_deepspeed_phi4_14b_nc80h100_lora.yml ├── run_deepspeed_phi4_14b_nc96a100_fullweight.yml ├── run_fsdp_phi4_14b_nc80h100_lora.yml ├── split_data.py ├── sql_result_test.jsonl ├── sql_result_train.jsonl ├── sql_results.jsonl ├── train.py └── utils.py └── single_step ├── README.md ├── configs ├── deepspeed_config_zero3.yaml ├── ds_config.json ├── ds_config_stage1.json ├── ds_config_stage2.json └── fsdp_config.yaml ├── docker-context ├── Dockerfile └── requirements.txt ├── run_ddp_mistral7b_lora_nc12v3_2nodes.yml ├── run_deepspeed_llama3_70b_nc96a100_fullweight.yml ├── run_deepspeed_phi4_14b_nc96a100_fullweight.yml ├── run_fsdp_llama3_8b_instruct_nc48a100_fullweight.yml ├── run_fsdp_mistral7b_lora_nc12v3_2nodes.yml ├── run_fsdp_nc80h100_llama3.3_70b_lora.yml ├── run_fsdp_nd40v2_llama3.3_70b_instruct_lora.yml ├── run_fsdp_phi4_14b_nc96a100_fullweight.yml ├── train.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/README.md -------------------------------------------------------------------------------- /azure_openai/REAMDE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/REAMDE.md -------------------------------------------------------------------------------- /azure_openai/function_calling/data/flight.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/function_calling/data/flight.db -------------------------------------------------------------------------------- /azure_openai/function_calling/data/flight_db.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/function_calling/data/flight_db.db -------------------------------------------------------------------------------- /azure_openai/function_calling/data/flight_policy.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/function_calling/data/flight_policy.json -------------------------------------------------------------------------------- /azure_openai/function_calling/data/flights.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/function_calling/data/flights.db -------------------------------------------------------------------------------- /azure_openai/function_calling/data/function_call.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/function_calling/data/function_call.jsonl -------------------------------------------------------------------------------- /azure_openai/function_calling/data/function_call1.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/function_calling/data/function_call1.jsonl -------------------------------------------------------------------------------- /azure_openai/function_calling/data/hotel.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/function_calling/data/hotel.db -------------------------------------------------------------------------------- /azure_openai/function_calling/data/hotel_policy.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/function_calling/data/hotel_policy.json -------------------------------------------------------------------------------- /azure_openai/function_calling/data/policy_vector.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/function_calling/data/policy_vector.json -------------------------------------------------------------------------------- /azure_openai/function_calling/data/test.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/function_calling/data/test.jsonl -------------------------------------------------------------------------------- /azure_openai/function_calling/data/test_data.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/function_calling/data/test_data.jsonl -------------------------------------------------------------------------------- /azure_openai/function_calling/data/train.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/function_calling/data/train.jsonl -------------------------------------------------------------------------------- /azure_openai/function_calling/data/user_profile.json: -------------------------------------------------------------------------------- 1 | {"customer_id":"12345", "name":"John Doe"} -------------------------------------------------------------------------------- /azure_openai/function_calling/generate_training_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/function_calling/generate_training_data.py -------------------------------------------------------------------------------- /azure_openai/function_calling/prompt.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/function_calling/prompt.txt -------------------------------------------------------------------------------- /azure_openai/function_calling/test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/function_calling/test.py -------------------------------------------------------------------------------- /azure_openai/python_analytic/data/processed_test_data_v3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/data/processed_test_data_v3.json -------------------------------------------------------------------------------- /azure_openai/python_analytic/data/processed_test_data_v5.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/data/processed_test_data_v5.json -------------------------------------------------------------------------------- /azure_openai/python_analytic/data/processed_test_data_v6.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/data/processed_test_data_v6.json -------------------------------------------------------------------------------- /azure_openai/python_analytic/data/processed_train_data_v3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/data/processed_train_data_v3.json -------------------------------------------------------------------------------- /azure_openai/python_analytic/data/processed_train_data_v5.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/data/processed_train_data_v5.json -------------------------------------------------------------------------------- /azure_openai/python_analytic/data/processed_train_data_v6.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/data/processed_train_data_v6.json -------------------------------------------------------------------------------- /azure_openai/python_analytic/data/python_test.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/data/python_test.jsonl -------------------------------------------------------------------------------- /azure_openai/python_analytic/data/python_test_0908.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/data/python_test_0908.jsonl -------------------------------------------------------------------------------- /azure_openai/python_analytic/data/python_test_0909.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/data/python_test_0909.jsonl -------------------------------------------------------------------------------- /azure_openai/python_analytic/data/python_train.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/data/python_train.jsonl -------------------------------------------------------------------------------- /azure_openai/python_analytic/data/python_train_0908.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/data/python_train_0908.jsonl -------------------------------------------------------------------------------- /azure_openai/python_analytic/data/python_train_0909.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/data/python_train_0909.jsonl -------------------------------------------------------------------------------- /azure_openai/python_analytic/data/test_data_v2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/data/test_data_v2.json -------------------------------------------------------------------------------- /azure_openai/python_analytic/data/test_data_v3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/data/test_data_v3.json -------------------------------------------------------------------------------- /azure_openai/python_analytic/data/train_data.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/data/train_data.json -------------------------------------------------------------------------------- /azure_openai/python_analytic/data/train_data_v2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/data/train_data_v2.json -------------------------------------------------------------------------------- /azure_openai/python_analytic/data/train_data_v3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/data/train_data_v3.json -------------------------------------------------------------------------------- /azure_openai/python_analytic/evaluation/evaluation.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/evaluation/evaluation.ipynb -------------------------------------------------------------------------------- /azure_openai/python_analytic/evaluation/evaluation_results_python.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/evaluation/evaluation_results_python.csv -------------------------------------------------------------------------------- /azure_openai/python_analytic/evaluation/evaluation_results_python21.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/evaluation/evaluation_results_python21.csv -------------------------------------------------------------------------------- /azure_openai/python_analytic/evaluation/evaluation_results_pythonft_0908.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/evaluation/evaluation_results_pythonft_0908.csv -------------------------------------------------------------------------------- /azure_openai/python_analytic/evaluation/evaluation_results_pythonft_0908_1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/evaluation/evaluation_results_pythonft_0908_1.csv -------------------------------------------------------------------------------- /azure_openai/python_analytic/evaluation/evaluation_results_pythonft_0908_2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/evaluation/evaluation_results_pythonft_0908_2.csv -------------------------------------------------------------------------------- /azure_openai/python_analytic/evaluation/evaluation_results_pythonft_0908_v5.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/evaluation/evaluation_results_pythonft_0908_v5.csv -------------------------------------------------------------------------------- /azure_openai/python_analytic/evaluation/evaluation_results_pythonft_0909_v1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/evaluation/evaluation_results_pythonft_0909_v1.csv -------------------------------------------------------------------------------- /azure_openai/python_analytic/gen_data.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/python_analytic/gen_data.ipynb -------------------------------------------------------------------------------- /azure_openai/sql_gen/.env.sample: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/sql_gen/.env.sample -------------------------------------------------------------------------------- /azure_openai/sql_gen/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/sql_gen/README.md -------------------------------------------------------------------------------- /azure_openai/sql_gen/data/analytic_graph.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/sql_gen/data/analytic_graph.json -------------------------------------------------------------------------------- /azure_openai/sql_gen/data/data_gen_program.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/sql_gen/data/data_gen_program.py -------------------------------------------------------------------------------- /azure_openai/sql_gen/data/northwind.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/sql_gen/data/northwind.db -------------------------------------------------------------------------------- /azure_openai/sql_gen/data/open_ai_data_v2.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/sql_gen/data/open_ai_data_v2.jsonl -------------------------------------------------------------------------------- /azure_openai/sql_gen/data/open_ai_data_with_reasoning_v2.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/sql_gen/data/open_ai_data_with_reasoning_v2.jsonl -------------------------------------------------------------------------------- /azure_openai/sql_gen/evaluate_fine_tuned_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/sql_gen/evaluate_fine_tuned_model.py -------------------------------------------------------------------------------- /azure_openai/sql_gen/evaluation/compare_deployments_concurrent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/sql_gen/evaluation/compare_deployments_concurrent.py -------------------------------------------------------------------------------- /azure_openai/sql_gen/ft_job_and_deploy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azure_openai/sql_gen/ft_job_and_deploy.py -------------------------------------------------------------------------------- /azureopenai/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/azureopenai/README.md -------------------------------------------------------------------------------- /opensource_llm/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/README.md -------------------------------------------------------------------------------- /opensource_llm/archived/azureml_unsloth/pipeline/conda.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/azureml_unsloth/pipeline/conda.yml -------------------------------------------------------------------------------- /opensource_llm/archived/azureml_unsloth/pipeline/fine_tune.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/azureml_unsloth/pipeline/fine_tune.yml -------------------------------------------------------------------------------- /opensource_llm/archived/azureml_unsloth/pipeline/fine_tune_src/fine_tune.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/azureml_unsloth/pipeline/fine_tune_src/fine_tune.py -------------------------------------------------------------------------------- /opensource_llm/archived/azureml_unsloth/pipeline/pipeline.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/azureml_unsloth/pipeline/pipeline.yml -------------------------------------------------------------------------------- /opensource_llm/archived/azureml_unsloth/pipeline/pipeline2.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/azureml_unsloth/pipeline/pipeline2.yml -------------------------------------------------------------------------------- /opensource_llm/archived/azureml_unsloth/pipeline/pretrain.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/azureml_unsloth/pipeline/pretrain.yml -------------------------------------------------------------------------------- /opensource_llm/archived/azureml_unsloth/pipeline/pretrain_src/pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/azureml_unsloth/pipeline/pretrain_src/pretrain.py -------------------------------------------------------------------------------- /opensource_llm/archived/azureml_unsloth/pretrain/conda.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/azureml_unsloth/pretrain/conda.yml -------------------------------------------------------------------------------- /opensource_llm/archived/azureml_unsloth/pretrain/hyperparam_tune.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/azureml_unsloth/pretrain/hyperparam_tune.yml -------------------------------------------------------------------------------- /opensource_llm/archived/azureml_unsloth/pretrain/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/azureml_unsloth/pretrain/main.py -------------------------------------------------------------------------------- /opensource_llm/archived/azureml_unsloth/pretrain/pipeline.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/azureml_unsloth/pretrain/pipeline.yml -------------------------------------------------------------------------------- /opensource_llm/archived/conda.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/conda.yml -------------------------------------------------------------------------------- /opensource_llm/archived/data_prep.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/data_prep.ipynb -------------------------------------------------------------------------------- /opensource_llm/archived/data_processing/generate_osllm_ft_data_short.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/data_processing/generate_osllm_ft_data_short.py -------------------------------------------------------------------------------- /opensource_llm/archived/evaluate_component.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/evaluate_component.yml -------------------------------------------------------------------------------- /opensource_llm/archived/evaluate_component_bk.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/evaluate_component_bk.yml -------------------------------------------------------------------------------- /opensource_llm/archived/evaluate_vllm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/evaluate_vllm.py -------------------------------------------------------------------------------- /opensource_llm/archived/fine_tune_ac.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/fine_tune_ac.py -------------------------------------------------------------------------------- /opensource_llm/archived/merge_component.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/merge_component.yml -------------------------------------------------------------------------------- /opensource_llm/archived/pipeline.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/pipeline.yml -------------------------------------------------------------------------------- /opensource_llm/archived/pipeline_fsdp.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/pipeline_fsdp.yml -------------------------------------------------------------------------------- /opensource_llm/archived/pipeline_fsdp_bk.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/pipeline_fsdp_bk.yml -------------------------------------------------------------------------------- /opensource_llm/archived/pipelinellama.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/pipelinellama.yml -------------------------------------------------------------------------------- /opensource_llm/archived/pipelinellama_fsdp.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/pipelinellama_fsdp.yml -------------------------------------------------------------------------------- /opensource_llm/archived/pipelinellama_vllm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/pipelinellama_vllm.yml -------------------------------------------------------------------------------- /opensource_llm/archived/quick_fine_tune.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/quick_fine_tune.ipynb -------------------------------------------------------------------------------- /opensource_llm/archived/test.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/test.ipynb -------------------------------------------------------------------------------- /opensource_llm/archived/train_component.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/train_component.yml -------------------------------------------------------------------------------- /opensource_llm/archived/train_component_fsdp.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/archived/train_component_fsdp.yml -------------------------------------------------------------------------------- /opensource_llm/data/all_scenarios_questions_v1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/all_scenarios_questions_v1.json -------------------------------------------------------------------------------- /opensource_llm/data/all_scenarios_questions_v1_unique.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/all_scenarios_questions_v1_unique.json -------------------------------------------------------------------------------- /opensource_llm/data/analytic_graph.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/analytic_graph.json -------------------------------------------------------------------------------- /opensource_llm/data/analytic_graph_v2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/analytic_graph_v2.json -------------------------------------------------------------------------------- /opensource_llm/data/compare_deployments_concurrent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/compare_deployments_concurrent.py -------------------------------------------------------------------------------- /opensource_llm/data/compare_deployments_concurrent_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/compare_deployments_concurrent_v2.py -------------------------------------------------------------------------------- /opensource_llm/data/compare_openai_deployments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/compare_openai_deployments.py -------------------------------------------------------------------------------- /opensource_llm/data/compare_queries.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/compare_queries.py -------------------------------------------------------------------------------- /opensource_llm/data/dedupe_questions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/dedupe_questions.py -------------------------------------------------------------------------------- /opensource_llm/data/dedupe_questions_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/dedupe_questions_v2.py -------------------------------------------------------------------------------- /opensource_llm/data/detailed_query_results_v2.0.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/detailed_query_results_v2.0.xlsx -------------------------------------------------------------------------------- /opensource_llm/data/detailed_query_results_v2.1.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/detailed_query_results_v2.1.xlsx -------------------------------------------------------------------------------- /opensource_llm/data/gen_openai.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/gen_openai.py -------------------------------------------------------------------------------- /opensource_llm/data/gen_openai_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/gen_openai_v2.py -------------------------------------------------------------------------------- /opensource_llm/data/gen_openai_v3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/gen_openai_v3.py -------------------------------------------------------------------------------- /opensource_llm/data/hs6_descriptions.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/hs6_descriptions.txt -------------------------------------------------------------------------------- /opensource_llm/data/hs_chapters.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/hs_chapters.json -------------------------------------------------------------------------------- /opensource_llm/data/hscodes.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/hscodes.csv -------------------------------------------------------------------------------- /opensource_llm/data/northwind.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/northwind.db -------------------------------------------------------------------------------- /opensource_llm/data/open_ai_data.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/open_ai_data.jsonl -------------------------------------------------------------------------------- /opensource_llm/data/open_ai_data_train.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/open_ai_data_train.jsonl -------------------------------------------------------------------------------- /opensource_llm/data/open_ai_data_v2.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/open_ai_data_v2.jsonl -------------------------------------------------------------------------------- /opensource_llm/data/open_ai_data_val.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/open_ai_data_val.jsonl -------------------------------------------------------------------------------- /opensource_llm/data/open_ai_data_with_reasoning.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/open_ai_data_with_reasoning.jsonl -------------------------------------------------------------------------------- /opensource_llm/data/open_ai_data_with_reasoning_v2.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/open_ai_data_with_reasoning_v2.jsonl -------------------------------------------------------------------------------- /opensource_llm/data/question_gen.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/question_gen.py -------------------------------------------------------------------------------- /opensource_llm/data/question_gen_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/question_gen_v2.py -------------------------------------------------------------------------------- /opensource_llm/data/question_gen_v4.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/question_gen_v4.py -------------------------------------------------------------------------------- /opensource_llm/data/questions_by_difficulty_v1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/questions_by_difficulty_v1.json -------------------------------------------------------------------------------- /opensource_llm/data/questions_v2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/questions_v2.json -------------------------------------------------------------------------------- /opensource_llm/data/questions_v3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/questions_v3.json -------------------------------------------------------------------------------- /opensource_llm/data/questions_v4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/questions_v4.json -------------------------------------------------------------------------------- /opensource_llm/data/questions_v4_deduped.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/questions_v4_deduped.json -------------------------------------------------------------------------------- /opensource_llm/data/refine_sql_queries.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/refine_sql_queries.py -------------------------------------------------------------------------------- /opensource_llm/data/results.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/results.jsonl -------------------------------------------------------------------------------- /opensource_llm/data/split_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/split_data.py -------------------------------------------------------------------------------- /opensource_llm/data/sql_examples.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/sql_examples.jsonl -------------------------------------------------------------------------------- /opensource_llm/data/sql_gen_deepseek.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/sql_gen_deepseek.py -------------------------------------------------------------------------------- /opensource_llm/data/sql_gen_deepseek_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/sql_gen_deepseek_v2.py -------------------------------------------------------------------------------- /opensource_llm/data/sql_gen_deepseek_v3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/sql_gen_deepseek_v3.py -------------------------------------------------------------------------------- /opensource_llm/data/sql_gen_deepseek_v4.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/sql_gen_deepseek_v4.py -------------------------------------------------------------------------------- /opensource_llm/data/sql_gen_deepseek_v5.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/sql_gen_deepseek_v5.py -------------------------------------------------------------------------------- /opensource_llm/data/sql_gen_o3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/sql_gen_o3.py -------------------------------------------------------------------------------- /opensource_llm/data/sql_result__test_v2.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/sql_result__test_v2.jsonl -------------------------------------------------------------------------------- /opensource_llm/data/sql_result_test_v5.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/sql_result_test_v5.jsonl -------------------------------------------------------------------------------- /opensource_llm/data/sql_result_train_v5.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/sql_result_train_v5.jsonl -------------------------------------------------------------------------------- /opensource_llm/data/sql_result_v2.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/sql_result_v2.jsonl -------------------------------------------------------------------------------- /opensource_llm/data/sql_result_v4.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/sql_result_v4.jsonl -------------------------------------------------------------------------------- /opensource_llm/data/sql_result_v5.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/sql_result_v5.jsonl -------------------------------------------------------------------------------- /opensource_llm/data/sql_results_refined.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/sql_results_refined.jsonl -------------------------------------------------------------------------------- /opensource_llm/data/test_deepseek.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/data/test_deepseek.py -------------------------------------------------------------------------------- /opensource_llm/data/test_sql.ipynb: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /opensource_llm/deployment/aml/conda.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml/conda.yml -------------------------------------------------------------------------------- /opensource_llm/deployment/aml/deployment.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml/deployment.yml -------------------------------------------------------------------------------- /opensource_llm/deployment/aml/endpoint.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml/endpoint.yml -------------------------------------------------------------------------------- /opensource_llm/deployment/aml/instance_type.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml/instance_type.yaml -------------------------------------------------------------------------------- /opensource_llm/deployment/aml/k8s_deployment.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml/k8s_deployment.yml -------------------------------------------------------------------------------- /opensource_llm/deployment/aml/k8s_endpoint.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml/k8s_endpoint.yml -------------------------------------------------------------------------------- /opensource_llm/deployment/aml/nvidia_device.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml/nvidia_device.yaml -------------------------------------------------------------------------------- /opensource_llm/deployment/aml/score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml/score.py -------------------------------------------------------------------------------- /opensource_llm/deployment/aml/score_chat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml/score_chat.py -------------------------------------------------------------------------------- /opensource_llm/deployment/aml/score_chat70b.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml/score_chat70b.py -------------------------------------------------------------------------------- /opensource_llm/deployment/aml/script.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml/script.txt -------------------------------------------------------------------------------- /opensource_llm/deployment/aml/test_deployent.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml/test_deployent.ipynb -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_ray/command.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_ray/command.txt -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_ray/conda.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_ray/conda.yml -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_ray/deployment.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_ray/deployment.yml -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_ray/endpoint.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_ray/endpoint.yml -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_ray/request.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_ray/request.json -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_ray/score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_ray/score.py -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_vllm/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_vllm/Dockerfile -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_vllm/command.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_vllm/command.sh -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_vllm/create_or_update.ps1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_vllm/create_or_update.ps1 -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_vllm/data/106.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_vllm/data/106.png -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_vllm/data/152.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_vllm/data/152.png -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_vllm/endpoint.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_vllm/endpoint.yml -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_vllm/environment.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_vllm/environment.yml -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_vllm/test_gpt4omini.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_vllm/test_gpt4omini.py -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_vllm/test_inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_vllm/test_inference.py -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_vllm/test_inference_single.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_vllm/test_inference_single.py -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_vllm/test_scoring.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_vllm/test_scoring.py -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_vllm/vllm__custom_model_deployment.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_vllm/vllm__custom_model_deployment.yml -------------------------------------------------------------------------------- /opensource_llm/deployment/aml_vllm/vllm_deployment.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/deployment/aml_vllm/vllm_deployment.yml -------------------------------------------------------------------------------- /opensource_llm/pipelines/analytic_graph copy.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/analytic_graph copy.json -------------------------------------------------------------------------------- /opensource_llm/pipelines/analytic_graph.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/analytic_graph.json -------------------------------------------------------------------------------- /opensource_llm/pipelines/azureml_hf_sft/configs/ds_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/azureml_hf_sft/configs/ds_config.json -------------------------------------------------------------------------------- /opensource_llm/pipelines/azureml_hf_sft/configs/ds_config_stage1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/azureml_hf_sft/configs/ds_config_stage1.json -------------------------------------------------------------------------------- /opensource_llm/pipelines/azureml_hf_sft/configs/ds_config_stage2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/azureml_hf_sft/configs/ds_config_stage2.json -------------------------------------------------------------------------------- /opensource_llm/pipelines/azureml_hf_sft/configs/fsdp_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/azureml_hf_sft/configs/fsdp_config.yaml -------------------------------------------------------------------------------- /opensource_llm/pipelines/azureml_hf_sft/docker-context/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/azureml_hf_sft/docker-context/Dockerfile -------------------------------------------------------------------------------- /opensource_llm/pipelines/azureml_hf_sft/docker-context/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/azureml_hf_sft/docker-context/requirements.txt -------------------------------------------------------------------------------- /opensource_llm/pipelines/azureml_hf_sft/evaluate/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/azureml_hf_sft/evaluate/evaluate.py -------------------------------------------------------------------------------- /opensource_llm/pipelines/azureml_hf_sft/evaluate/evaluate_bk.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/azureml_hf_sft/evaluate/evaluate_bk.py -------------------------------------------------------------------------------- /opensource_llm/pipelines/azureml_hf_sft/evaluate/merge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/azureml_hf_sft/evaluate/merge.py -------------------------------------------------------------------------------- /opensource_llm/pipelines/azureml_hf_sft/train/train_bk.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/azureml_hf_sft/train/train_bk.py -------------------------------------------------------------------------------- /opensource_llm/pipelines/configs/deepspeed_config_zero3.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/configs/deepspeed_config_zero3.yaml -------------------------------------------------------------------------------- /opensource_llm/pipelines/configs/ds_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/configs/ds_config.json -------------------------------------------------------------------------------- /opensource_llm/pipelines/configs/ds_config_stage1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/configs/ds_config_stage1.json -------------------------------------------------------------------------------- /opensource_llm/pipelines/configs/ds_config_stage2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/configs/ds_config_stage2.json -------------------------------------------------------------------------------- /opensource_llm/pipelines/configs/fsdp_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/configs/fsdp_config.yaml -------------------------------------------------------------------------------- /opensource_llm/pipelines/data/northwind.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/data/northwind.db -------------------------------------------------------------------------------- /opensource_llm/pipelines/data/test_sql.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/data/test_sql.ipynb -------------------------------------------------------------------------------- /opensource_llm/pipelines/docker-context/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/docker-context/Dockerfile -------------------------------------------------------------------------------- /opensource_llm/pipelines/docker-context/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/docker-context/requirements.txt -------------------------------------------------------------------------------- /opensource_llm/pipelines/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/evaluate.py -------------------------------------------------------------------------------- /opensource_llm/pipelines/northwind.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/northwind.db -------------------------------------------------------------------------------- /opensource_llm/pipelines/pipeline.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/pipeline.yml -------------------------------------------------------------------------------- /opensource_llm/pipelines/quick_fine_tune.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/quick_fine_tune.ipynb -------------------------------------------------------------------------------- /opensource_llm/pipelines/run_deepspeed_phi4_14b_nc80h100_lora.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/run_deepspeed_phi4_14b_nc80h100_lora.yml -------------------------------------------------------------------------------- /opensource_llm/pipelines/run_deepspeed_phi4_14b_nc96a100_fullweight.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/run_deepspeed_phi4_14b_nc96a100_fullweight.yml -------------------------------------------------------------------------------- /opensource_llm/pipelines/run_fsdp_phi4_14b_nc80h100_lora.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/run_fsdp_phi4_14b_nc80h100_lora.yml -------------------------------------------------------------------------------- /opensource_llm/pipelines/split_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/split_data.py -------------------------------------------------------------------------------- /opensource_llm/pipelines/sql_result_test.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/sql_result_test.jsonl -------------------------------------------------------------------------------- /opensource_llm/pipelines/sql_result_train.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/sql_result_train.jsonl -------------------------------------------------------------------------------- /opensource_llm/pipelines/sql_results.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/sql_results.jsonl -------------------------------------------------------------------------------- /opensource_llm/pipelines/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/train.py -------------------------------------------------------------------------------- /opensource_llm/pipelines/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/pipelines/utils.py -------------------------------------------------------------------------------- /opensource_llm/single_step/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/single_step/README.md -------------------------------------------------------------------------------- /opensource_llm/single_step/configs/deepspeed_config_zero3.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/single_step/configs/deepspeed_config_zero3.yaml -------------------------------------------------------------------------------- /opensource_llm/single_step/configs/ds_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/single_step/configs/ds_config.json -------------------------------------------------------------------------------- /opensource_llm/single_step/configs/ds_config_stage1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/single_step/configs/ds_config_stage1.json -------------------------------------------------------------------------------- /opensource_llm/single_step/configs/ds_config_stage2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/single_step/configs/ds_config_stage2.json -------------------------------------------------------------------------------- /opensource_llm/single_step/configs/fsdp_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/single_step/configs/fsdp_config.yaml -------------------------------------------------------------------------------- /opensource_llm/single_step/docker-context/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/single_step/docker-context/Dockerfile -------------------------------------------------------------------------------- /opensource_llm/single_step/docker-context/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/single_step/docker-context/requirements.txt -------------------------------------------------------------------------------- /opensource_llm/single_step/run_ddp_mistral7b_lora_nc12v3_2nodes.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/single_step/run_ddp_mistral7b_lora_nc12v3_2nodes.yml -------------------------------------------------------------------------------- /opensource_llm/single_step/run_deepspeed_llama3_70b_nc96a100_fullweight.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/single_step/run_deepspeed_llama3_70b_nc96a100_fullweight.yml -------------------------------------------------------------------------------- /opensource_llm/single_step/run_deepspeed_phi4_14b_nc96a100_fullweight.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/single_step/run_deepspeed_phi4_14b_nc96a100_fullweight.yml -------------------------------------------------------------------------------- /opensource_llm/single_step/run_fsdp_llama3_8b_instruct_nc48a100_fullweight.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/single_step/run_fsdp_llama3_8b_instruct_nc48a100_fullweight.yml -------------------------------------------------------------------------------- /opensource_llm/single_step/run_fsdp_mistral7b_lora_nc12v3_2nodes.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/single_step/run_fsdp_mistral7b_lora_nc12v3_2nodes.yml -------------------------------------------------------------------------------- /opensource_llm/single_step/run_fsdp_nc80h100_llama3.3_70b_lora.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/single_step/run_fsdp_nc80h100_llama3.3_70b_lora.yml -------------------------------------------------------------------------------- /opensource_llm/single_step/run_fsdp_nd40v2_llama3.3_70b_instruct_lora.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/single_step/run_fsdp_nd40v2_llama3.3_70b_instruct_lora.yml -------------------------------------------------------------------------------- /opensource_llm/single_step/run_fsdp_phi4_14b_nc96a100_fullweight.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/single_step/run_fsdp_phi4_14b_nc96a100_fullweight.yml -------------------------------------------------------------------------------- /opensource_llm/single_step/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/single_step/train.py -------------------------------------------------------------------------------- /opensource_llm/single_step/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/james-tn/llm-fine-tuning/HEAD/opensource_llm/single_step/utils.py --------------------------------------------------------------------------------