├── .gitignore ├── LICENSE ├── README.md ├── imgs ├── 990_pro.png ├── claude-sae.png ├── maxpool_grad.png ├── nv-chat-rtx-version.png ├── nvme_partition.png ├── rag-agent.png ├── rag-chain.png ├── self-corrective-rag.png └── tot_dfs_bfs.png ├── results ├── image-0421-78fd7400.png └── image-0421-f5ab6e15.png └── tutorials ├── 01_image_caption.ipynb ├── 02_text2img.ipynb ├── agents ├── GraphRAG.ipynb ├── agents_research.ipynb ├── applications │ ├── translation-agent-demo.ipynb │ └── utils.py ├── langchain │ ├── AgentExecutor.ipynb │ ├── LangGraph-MultiAgents.ipynb │ ├── LangGraph-rag.ipynb │ ├── LangGraph_basics.ipynb │ ├── LangSmith.ipynb │ ├── agent_scratchpad_intermediate_steps.ipynb │ ├── conversational_agents.ipynb │ ├── examples │ │ ├── agent-simulation.ipynb │ │ ├── llm_compiler.ipynb │ │ ├── plan_and_execute.ipynb │ │ └── reflexion_agent.ipynb │ ├── imgs │ │ ├── dag.png │ │ ├── react-output.png │ │ ├── structure_output_option1.png │ │ └── structure_output_option2.png │ ├── langchain_agents.ipynb │ ├── langchain_graph_details.ipynb │ ├── lc_lg_core_concepts.ipynb │ ├── output_parser.ipynb │ └── structured_output.ipynb ├── oai_function_call_rag.ipynb ├── prompt_engineering.ipynb ├── reasoners │ └── reasoner_enhenced.ipynb └── tot_tree_of_thought.ipynb ├── ai_app ├── ai-copilot │ ├── NotebookLM.ipynb │ ├── copilot.ipynb │ ├── cursor-advanced.ipynb │ ├── cursor-composer.ipynb │ └── cursor.ipynb ├── audio_video │ ├── audio.wav │ ├── audio_notes.ipynb │ ├── imgs │ │ └── audio-notes-framework.png │ ├── speech_generation.ipynb │ ├── video.mp4 │ ├── videolingo.ipynb │ └── whisper_whisperx.ipynb ├── db │ └── postgresql.ipynb ├── imgs │ ├── 0_Parade_Parade_0_873.jpg │ ├── 0_Parade_marchingband_1_765.jpg │ ├── 11_Meeting_Meeting_11_Meeting_Meeting_11_176.jpg │ ├── 11_Meeting_Meeting_11_Meeting_Meeting_11_26.jpg │ ├── gh_copilot_suggest.png │ └── tab.jpeg ├── openrouter.ipynb ├── pipeline_工具.ipynb ├── streamlit(web).ipynb ├── tools │ ├── Monica.ipynb │ └── ai_tools.ipynb ├── vision │ ├── face_detect.ipynb │ ├── rembg-U2Net.ipynb │ └── 图像视频.ipynb └── 项目参考.ipynb ├── cv ├── PIL.ipynb ├── ViT.ipynb ├── cnn_3dcnn_se.ipynb ├── pytorch_cifar10.ipynb ├── timm.ipynb └── torchvision.ipynb ├── diffusion └── vae.ipynb ├── dnn ├── ae │ ├── ksparse-ae.ipynb │ └── ksparse-ae.py └── 对比学习 │ ├── SimCLR.ipynb │ ├── constrative_loss_infoNCE.ipynb │ └── 对比学习_contrastive_learning.ipynb ├── gnn ├── GTN_Transformer.ipynb ├── PyG.ipynb ├── gcn_gat.ipynb ├── gnn_basics.ipynb └── imgs │ ├── gnn_layers.png │ └── two-layers-gcn.png ├── mcp ├── about_mcp.ipynb └── 实用 MCP.ipynb ├── multi_modals.ipynb ├── nn_basics ├── 01_pytorch_wandb.ipynb ├── 02_gelu_grad.ipynb ├── 04_embedding_details.ipynb ├── activate_function │ └── swish.ipynb ├── bce_loss_cross_entropy.ipynb ├── data_prepare │ └── dataset_dataloader.ipynb ├── einsum_einops │ ├── einops.ipynb │ ├── einsum_bilinear.ipynb │ ├── einsum_numpy_torch.ipynb │ ├── einsum_supp_einops_初步.ipynb │ └── shape.ipynb ├── grad │ ├── gradient_autograd.ipynb │ ├── mask_grad.ipynb │ ├── max_pooling_grad.ipynb │ ├── readme.md │ ├── retain_grad_retatin_graph.ipynb │ ├── sae_topK.ipynb │ ├── sigmoid_grad.ipynb │ ├── tensorboard_graph.ipynb │ ├── torch_jacobian_hessian.ipynb │ ├── torch_jvp.ipynb │ ├── torch_常见错误.ipynb │ └── 可微才可学.ipynb ├── imgs │ ├── axies-flat.png │ ├── axis-swap-flat.png │ ├── axis-swap.png │ ├── axis0-1-2.png │ ├── boo.jpg │ ├── fan_in_fan_out.png │ ├── graph-legend.png │ ├── linear.png │ ├── multi_loss.PNG │ ├── numpy_einsum.png │ ├── shape_1x8.png │ ├── shape_1x8_2.png │ ├── shape_1x8_3.png │ ├── shape_1x8_4.png │ ├── shape_1x8_5.png │ ├── shape_2x4.png │ ├── shape_4x2.png │ ├── shape_4x2_2.png │ └── shape_4x2_3.png ├── loss │ ├── cross_entroy_loss_ignore_index.ipynb │ └── mse_crossentropy.ipynb ├── module_load_save.ipynb ├── multinomail_categorical.ipynb ├── optimizer │ ├── adadelta_adam.ipynb │ ├── adagrad_rmsprop.ipynb │ ├── airfoil_self_noise.dat │ ├── l2_reg_weight_decay_adaw.ipynb │ ├── lr_scheduler.ipynb │ ├── momentum_exp_avg.ipynb │ ├── pytorch_as_a_general_optimizer.ipynb │ └── pytorch_lbfgs.ipynb ├── performance_optimization │ └── pytorch_optimization.ipynb ├── quantization.ipynb ├── shape_axis_dim │ ├── axis_dim_2.ipynb │ ├── axis_dim_processing.ipynb │ ├── fan_in_fan_out_parameters.ipynb │ └── shape_modified.ipynb ├── torch_dtype.ipynb ├── tricks_norms │ ├── layer_norm.ipynb │ ├── nn_and_training_tricks.ipynb │ ├── normalization.ipynb │ ├── spectral_norm.ipynb │ ├── torch_dropout.ipynb │ └── why_norm.ipynb └── utils │ ├── functional.ipynb │ ├── softmax.ipynb │ ├── torch_triangle_mask.ipynb │ └── torch_utils_scripts.ipynb ├── nvidia_chat_with_rtx.ipynb ├── oai ├── account_login.ipynb ├── async_demo.py ├── async_oai.ipynb ├── async_scripts │ ├── Semaphore.py │ ├── async_call.py │ ├── async_call_semaphore.py │ ├── async_llm_call.py │ └── io_cpu.py ├── concurrent-api-threads.ipynb ├── gpt_openai_生态.ipynb ├── logP.ipynb ├── logP_supp.ipynb └── openai_api.ipynb ├── ocr └── llm_ocr.ipynb ├── rag_in_action ├── chap1-simple-rag │ └── simple_rag.py ├── requirements.txt └── 图文检索.ipynb └── 炼丹 ├── batch_size_learning_rate.ipynb └── 炼丹&调参_tricks.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | wandb/ 132 | tutorials/nn_basics/data/ 133 | 134 | runs/ 135 | 136 | .idea/ 137 | data/ 138 | logs/ 139 | tutorials/rag/ 140 | 141 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # llm_aigc 2 | everything about llm & aigc 3 | -------------------------------------------------------------------------------- /imgs/990_pro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/imgs/990_pro.png -------------------------------------------------------------------------------- /imgs/claude-sae.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/imgs/claude-sae.png -------------------------------------------------------------------------------- /imgs/maxpool_grad.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/imgs/maxpool_grad.png -------------------------------------------------------------------------------- /imgs/nv-chat-rtx-version.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/imgs/nv-chat-rtx-version.png -------------------------------------------------------------------------------- /imgs/nvme_partition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/imgs/nvme_partition.png -------------------------------------------------------------------------------- /imgs/rag-agent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/imgs/rag-agent.png -------------------------------------------------------------------------------- /imgs/rag-chain.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/imgs/rag-chain.png -------------------------------------------------------------------------------- /imgs/self-corrective-rag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/imgs/self-corrective-rag.png -------------------------------------------------------------------------------- /imgs/tot_dfs_bfs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/imgs/tot_dfs_bfs.png -------------------------------------------------------------------------------- /results/image-0421-78fd7400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/results/image-0421-78fd7400.png -------------------------------------------------------------------------------- /results/image-0421-f5ab6e15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/results/image-0421-f5ab6e15.png -------------------------------------------------------------------------------- /tutorials/agents/GraphRAG.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "4cb46b19-0ab3-49e2-a700-663a8b47ca1d", 6 | "metadata": {}, 7 | "source": [ 8 | "## GraphRAG" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "d0857959-b2fb-4c54-a9bb-3cac3750a54f", 14 | "metadata": {}, 15 | "source": [ 16 | "- 传统 RAG\n", 17 | " - 文本 chunks 间的复杂关系与层次结构；\n", 18 | " - 固定数量的文本 chunks\n", 19 | " - “这本小说的主旨”\n", 20 | "- GraphRAG 的核心：“全局性”\n", 21 | " - 提取实体（Entity）\n", 22 | " - 提取关系（Relation）" 23 | ] 24 | } 25 | ], 26 | "metadata": { 27 | "kernelspec": { 28 | "display_name": "Python 3 (ipykernel)", 29 | "language": "python", 30 | "name": "python3" 31 | }, 32 | "language_info": { 33 | "codemirror_mode": { 34 | "name": "ipython", 35 | "version": 3 36 | }, 37 | "file_extension": ".py", 38 | "mimetype": "text/x-python", 39 | "name": "python", 40 | "nbconvert_exporter": "python", 41 | "pygments_lexer": "ipython3", 42 | "version": "3.10.13" 43 | } 44 | }, 45 | "nbformat": 4, 46 | "nbformat_minor": 5 47 | } 48 | -------------------------------------------------------------------------------- /tutorials/agents/agents_research.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "59ba6798-ab66-433f-903f-fc4cfabe7068", 6 | "metadata": {}, 7 | "source": [ 8 | "- https://sites.google.com/view/cs-159-2024/lectures" 9 | ] 10 | } 11 | ], 12 | "metadata": { 13 | "kernelspec": { 14 | "display_name": "Python 3 (ipykernel)", 15 | "language": "python", 16 | "name": "python3" 17 | }, 18 | "language_info": { 19 | "codemirror_mode": { 20 | "name": "ipython", 21 | "version": 3 22 | }, 23 | "file_extension": ".py", 24 | "mimetype": "text/x-python", 25 | "name": "python", 26 | "nbconvert_exporter": "python", 27 | "pygments_lexer": "ipython3", 28 | "version": "3.10.13" 29 | } 30 | }, 31 | "nbformat": 4, 32 | "nbformat_minor": 5 33 | } 34 | -------------------------------------------------------------------------------- /tutorials/agents/langchain/LangSmith.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "46376c75-a9fa-43f7-b064-a7bcb4b3fbed", 6 | "metadata": { 7 | "execution": { 8 | "iopub.execute_input": "2024-08-03T05:22:44.453765Z", 9 | "iopub.status.busy": "2024-08-03T05:22:44.453383Z", 10 | "iopub.status.idle": "2024-08-03T05:22:44.457785Z", 11 | "shell.execute_reply": "2024-08-03T05:22:44.457227Z", 12 | "shell.execute_reply.started": "2024-08-03T05:22:44.453746Z" 13 | } 14 | }, 15 | "source": [ 16 | "- https://github.com/langchain-ai/langsmith-cookbook" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 1, 22 | "id": "91ca12cf-d580-4cac-9700-755b8b786907", 23 | "metadata": { 24 | "execution": { 25 | "iopub.execute_input": "2024-08-03T04:35:55.989174Z", 26 | "iopub.status.busy": "2024-08-03T04:35:55.988599Z", 27 | "iopub.status.idle": "2024-08-03T04:35:55.994826Z", 28 | "shell.execute_reply": "2024-08-03T04:35:55.994062Z", 29 | "shell.execute_reply.started": "2024-08-03T04:35:55.989119Z" 30 | } 31 | }, 32 | "outputs": [], 33 | "source": [ 34 | "import os" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 2, 40 | "id": "6bf770cd-58e8-45f0-bb78-4fb5cfdf6b03", 41 | "metadata": { 42 | "execution": { 43 | "iopub.execute_input": "2024-08-03T04:35:57.269696Z", 44 | "iopub.status.busy": "2024-08-03T04:35:57.268966Z", 45 | "iopub.status.idle": "2024-08-03T04:35:57.294027Z", 46 | "shell.execute_reply": "2024-08-03T04:35:57.292956Z", 47 | "shell.execute_reply.started": "2024-08-03T04:35:57.269614Z" 48 | } 49 | }, 50 | "outputs": [ 51 | { 52 | "data": { 53 | "text/plain": [ 54 | "True" 55 | ] 56 | }, 57 | "execution_count": 2, 58 | "metadata": {}, 59 | "output_type": "execute_result" 60 | } 61 | ], 62 | "source": [ 63 | "from dotenv import load_dotenv\n", 64 | "load_dotenv()" 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "id": "74d43b0c-d681-4726-a9b5-442b7bb3c802", 70 | "metadata": {}, 71 | "source": [ 72 | "### 配置环境变量即可关联 langsmith 后台" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": 3, 78 | "id": "93d1504d-d26f-4ff6-acd8-ce0e53d10cb4", 79 | "metadata": { 80 | "execution": { 81 | "iopub.execute_input": "2024-08-03T04:40:35.684973Z", 82 | "iopub.status.busy": "2024-08-03T04:40:35.684590Z", 83 | "iopub.status.idle": "2024-08-03T04:40:35.689162Z", 84 | "shell.execute_reply": "2024-08-03T04:40:35.688436Z", 85 | "shell.execute_reply.started": "2024-08-03T04:40:35.684949Z" 86 | } 87 | }, 88 | "outputs": [], 89 | "source": [ 90 | "import os\n", 91 | "os.environ['LANGCHAIN_TRACING_V2'] = 'true'\n", 92 | "os.environ['LANGCHAIN_API_KEY'] = 'xx'" 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "id": "defa3ce8-eb39-4791-a500-a099457d2a89", 98 | "metadata": {}, 99 | "source": [ 100 | "### basics" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "id": "ebebc66e-905a-4f21-85f4-6e73b4a7b961", 106 | "metadata": {}, 107 | "source": [ 108 | "- workflow 中间每一步过程的展示\n", 109 | " - input 是什么，output 是什么\n", 110 | "- evaluation" 111 | ] 112 | } 113 | ], 114 | "metadata": { 115 | "kernelspec": { 116 | "display_name": "agent", 117 | "language": "python", 118 | "name": "agent" 119 | }, 120 | "language_info": { 121 | "codemirror_mode": { 122 | "name": "ipython", 123 | "version": 3 124 | }, 125 | "file_extension": ".py", 126 | "mimetype": "text/x-python", 127 | "name": "python", 128 | "nbconvert_exporter": "python", 129 | "pygments_lexer": "ipython3", 130 | "version": "3.12.2" 131 | } 132 | }, 133 | "nbformat": 4, 134 | "nbformat_minor": 5 135 | } 136 | -------------------------------------------------------------------------------- /tutorials/agents/langchain/examples/llm_compiler.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "8754839f-e9a0-4a25-9a67-303fd1447ed9", 6 | "metadata": {}, 7 | "source": [ 8 | "- https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/llm-compiler/LLMCompiler.ipynb" 9 | ] 10 | } 11 | ], 12 | "metadata": { 13 | "kernelspec": { 14 | "display_name": "agent", 15 | "language": "python", 16 | "name": "agent" 17 | }, 18 | "language_info": { 19 | "codemirror_mode": { 20 | "name": "ipython", 21 | "version": 3 22 | }, 23 | "file_extension": ".py", 24 | "mimetype": "text/x-python", 25 | "name": "python", 26 | "nbconvert_exporter": "python", 27 | "pygments_lexer": "ipython3", 28 | "version": "3.12.2" 29 | } 30 | }, 31 | "nbformat": 4, 32 | "nbformat_minor": 5 33 | } 34 | -------------------------------------------------------------------------------- /tutorials/agents/langchain/imgs/dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/agents/langchain/imgs/dag.png -------------------------------------------------------------------------------- /tutorials/agents/langchain/imgs/react-output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/agents/langchain/imgs/react-output.png -------------------------------------------------------------------------------- /tutorials/agents/langchain/imgs/structure_output_option1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/agents/langchain/imgs/structure_output_option1.png -------------------------------------------------------------------------------- /tutorials/agents/langchain/imgs/structure_output_option2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/agents/langchain/imgs/structure_output_option2.png -------------------------------------------------------------------------------- /tutorials/agents/prompt_engineering.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "5d4f7651", 6 | "metadata": {}, 7 | "source": [ 8 | "- `but PLEASE follows the rule STRICTLY.`\n", 9 | " - 形容词，副词，大写，等修饰来强调；" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "id": "cedc6977", 15 | "metadata": {}, 16 | "source": [ 17 | "## system prompt" 18 | ] 19 | }, 20 | { 21 | "cell_type": "markdown", 22 | "id": "ffe89921", 23 | "metadata": {}, 24 | "source": [ 25 | "- ios ChatGPT app\n", 26 | " - https://chat.openai.com/c/16968942-7507-4db6-be84-5217d2e933d6" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "id": "a6558736", 32 | "metadata": {}, 33 | "source": [ 34 | "## ReAct" 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "id": "67dae220", 40 | "metadata": {}, 41 | "source": [ 42 | "- ReAct：Reasoning & Acting\n", 43 | " - https://github.com/ysymyth/ReAct/blob/master/hotpotqa.ipynb" 44 | ] 45 | } 46 | ], 47 | "metadata": { 48 | "kernelspec": { 49 | "display_name": "Python 3 (ipykernel)", 50 | "language": "python", 51 | "name": "python3" 52 | }, 53 | "language_info": { 54 | "codemirror_mode": { 55 | "name": "ipython", 56 | "version": 3 57 | }, 58 | "file_extension": ".py", 59 | "mimetype": "text/x-python", 60 | "name": "python", 61 | "nbconvert_exporter": "python", 62 | "pygments_lexer": "ipython3", 63 | "version": "3.10.13" 64 | }, 65 | "toc": { 66 | "base_numbering": 1, 67 | "nav_menu": {}, 68 | "number_sections": true, 69 | "sideBar": true, 70 | "skip_h1_title": false, 71 | "title_cell": "Table of Contents", 72 | "title_sidebar": "Contents", 73 | "toc_cell": false, 74 | "toc_position": {}, 75 | "toc_section_display": true, 76 | "toc_window_display": false 77 | } 78 | }, 79 | "nbformat": 4, 80 | "nbformat_minor": 5 81 | } 82 | -------------------------------------------------------------------------------- /tutorials/agents/reasoners/reasoner_enhenced.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "49471438-c2f1-4699-b139-07944e3349ef", 6 | "metadata": {}, 7 | "source": [ 8 | "> reasoner 在 agent 中的必要性；\n", 9 | "\n", 10 | "- https://medium.com/@deansaco/reasoning-recursive-retrieval-with-deepseek-r1-tavily-and-langgraph-79b3336731e2\n", 11 | " - https://github.com/deansaco/r1-reasoning-rag\n", 12 | " - https://github.com/astordu/r1-reasoning-rag" 13 | ] 14 | } 15 | ], 16 | "metadata": { 17 | "kernelspec": { 18 | "display_name": "verl", 19 | "language": "python", 20 | "name": "verl" 21 | }, 22 | "language_info": { 23 | "codemirror_mode": { 24 | "name": "ipython", 25 | "version": 3 26 | }, 27 | "file_extension": ".py", 28 | "mimetype": "text/x-python", 29 | "name": "python", 30 | "nbconvert_exporter": "python", 31 | "pygments_lexer": "ipython3", 32 | "version": "3.9.0" 33 | } 34 | }, 35 | "nbformat": 4, 36 | "nbformat_minor": 5 37 | } 38 | -------------------------------------------------------------------------------- /tutorials/ai_app/ai-copilot/NotebookLM.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "b04f24e0-4e4f-40e6-9f60-c92e393126a6", 6 | "metadata": {}, 7 | "source": [ 8 | "- conversional learning：对话式文档学习\n", 9 | "- 基于 Gemini 2.0\n", 10 | "- 14 天免费试用\n", 11 | "- 局限与不足\n", 12 | " - pdf 转换为text时，还是丢了很多格式化信息\n", 13 | " - 比如 latex formula；" 14 | ] 15 | } 16 | ], 17 | "metadata": { 18 | "kernelspec": { 19 | "display_name": "Python 3 (ipykernel)", 20 | "language": "python", 21 | "name": "python3" 22 | }, 23 | "language_info": { 24 | "codemirror_mode": { 25 | "name": "ipython", 26 | "version": 3 27 | }, 28 | "file_extension": ".py", 29 | "mimetype": "text/x-python", 30 | "name": "python", 31 | "nbconvert_exporter": "python", 32 | "pygments_lexer": "ipython3", 33 | "version": "3.10.13" 34 | } 35 | }, 36 | "nbformat": 4, 37 | "nbformat_minor": 5 38 | } 39 | -------------------------------------------------------------------------------- /tutorials/ai_app/ai-copilot/copilot.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "dfa1cb0a-b4d1-4110-b9fa-aaeaba916e52", 7 | "metadata": { 8 | "execution": { 9 | "iopub.execute_input": "2024-07-14T03:38:48.710895Z", 10 | "iopub.status.busy": "2024-07-14T03:38:48.710305Z", 11 | "iopub.status.idle": "2024-07-14T03:38:48.720499Z", 12 | "shell.execute_reply": "2024-07-14T03:38:48.718881Z", 13 | "shell.execute_reply.started": "2024-07-14T03:38:48.710849Z" 14 | } 15 | }, 16 | "outputs": [], 17 | "source": [ 18 | "from IPython.display import Image" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "id": "bbd22bd5", 24 | "metadata": {}, 25 | "source": [ 26 | "- 支持的 ide\n", 27 | " - vscode\n", 28 | " - jetbrains：暂不支持 chat\n", 29 | " - pycharm\n", 30 | " - idea\n", 31 | "\n", 32 | "- 主要作用\n", 33 | " - 协助阅读源码；\n", 34 | " - 自动代码补全；（非常智能）\n", 35 | "\n", 36 | "- 用法\n", 37 | " - `@workspace`：整个工作区的提问；" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "id": "61437bbc-c448-4776-8e6f-66cdf5031ccd", 43 | "metadata": {}, 44 | "source": [ 45 | "## gh cli" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "id": "2ae46090-c019-4187-aedf-6660faf99324", 51 | "metadata": {}, 52 | "source": [ 53 | "> 工作在 cli，也专注于命令行命令的解释、建议与输出；" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "id": "1d80e92e-a0ec-4580-8eea-a70940c279ef", 59 | "metadata": {}, 60 | "source": [ 61 | "### install" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "id": "5df9fc01-6ad7-4e7d-95a3-ccbe4fff6371", 67 | "metadata": {}, 68 | "source": [ 69 | "- install\n", 70 | " - https://github.com/cli/cli/blob/trunk/docs/install_linux.md\n", 71 | "\n", 72 | "```\n", 73 | "$ gh auth login\n", 74 | "\n", 75 | "$ gh pr status\n", 76 | "\n", 77 | "$ gh extension list\n", 78 | "$ gh extension install github/gh-copilot\n", 79 | "$ gh extension install yusukebe/gh-markdown-preview\n", 80 | "$ gh extension list\n", 81 | "```" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "id": "79a8556f-7e3b-45c8-bace-c437e4c49890", 87 | "metadata": {}, 88 | "source": [ 89 | "### usage" 90 | ] 91 | }, 92 | { 93 | "cell_type": "markdown", 94 | "id": "dedb55ff-e688-4482-88eb-3d9974c1c605", 95 | "metadata": {}, 96 | "source": [ 97 | "> 这里其实我们可以通过这样一个成熟的现代AI产品，学习如何对软件开发进行抽象的能力；\n", 98 | "\n", 99 | "- `explain`：解释\n", 100 | " - `$ gh copilot explain \"traceroute github.com\"`\n", 101 | "- `suggest`：交互模式（支持中文输入）\n", 102 | " - `$ gh copilot suggest \"Install git\"`\n", 103 | " - 其会生成一个 suggestion 的命令\n", 104 | " - 进一步可以选择 copy/explain/execute/revise(修改)/rate(评分，feedback)" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 3, 110 | "id": "bb63ceaf-2088-456e-87bc-9391db17572a", 111 | "metadata": { 112 | "execution": { 113 | "iopub.execute_input": "2024-07-14T03:39:55.856536Z", 114 | "iopub.status.busy": "2024-07-14T03:39:55.855834Z", 115 | "iopub.status.idle": "2024-07-14T03:39:55.862842Z", 116 | "shell.execute_reply": "2024-07-14T03:39:55.861601Z", 117 | "shell.execute_reply.started": "2024-07-14T03:39:55.856499Z" 118 | } 119 | }, 120 | "outputs": [ 121 | { 122 | "data": { 123 | "text/html": [ 124 | "" 125 | ], 126 | "text/plain": [ 127 | "" 128 | ] 129 | }, 130 | "execution_count": 3, 131 | "metadata": {}, 132 | "output_type": "execute_result" 133 | } 134 | ], 135 | "source": [ 136 | "Image(url='./imgs/gh_copilot_suggest.png', width=400)" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": null, 142 | "id": "4a45f2a6-64da-4b26-9a9c-69ac24fe4ade", 143 | "metadata": {}, 144 | "outputs": [], 145 | "source": [] 146 | } 147 | ], 148 | "metadata": { 149 | "kernelspec": { 150 | "display_name": "Python 3 (ipykernel)", 151 | "language": "python", 152 | "name": "python3" 153 | }, 154 | "language_info": { 155 | "codemirror_mode": { 156 | "name": "ipython", 157 | "version": 3 158 | }, 159 | "file_extension": ".py", 160 | "mimetype": "text/x-python", 161 | "name": "python", 162 | "nbconvert_exporter": "python", 163 | "pygments_lexer": "ipython3", 164 | "version": "3.10.13" 165 | }, 166 | "toc": { 167 | "base_numbering": 1, 168 | "nav_menu": {}, 169 | "number_sections": true, 170 | "sideBar": true, 171 | "skip_h1_title": false, 172 | "title_cell": "Table of Contents", 173 | "title_sidebar": "Contents", 174 | "toc_cell": false, 175 | "toc_position": {}, 176 | "toc_section_display": true, 177 | "toc_window_display": false 178 | } 179 | }, 180 | "nbformat": 4, 181 | "nbformat_minor": 5 182 | } 183 | -------------------------------------------------------------------------------- /tutorials/ai_app/ai-copilot/cursor-advanced.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "261b6f95-935b-4ee8-9201-1d7f4c9fe233", 6 | "metadata": {}, 7 | "source": [ 8 | "- ignore files\n", 9 | " - https://docs.cursor.com/context/ignore-files" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "id": "2202d527-d369-4186-aee8-cb56799e3be4", 15 | "metadata": {}, 16 | "source": [ 17 | "### `.cursorrules`" 18 | ] 19 | }, 20 | { 21 | "cell_type": "markdown", 22 | "id": "aced2015-7dab-494c-93da-b21f0b1eff2e", 23 | "metadata": {}, 24 | "source": [ 25 | "- https://cursor.directory/\n", 26 | " - https://cursor.directory/fastapi-python-microservices-serverless-cursor-rules\n", 27 | " - https://cursor.directory/learn\n", 28 | "\n", 29 | "```\n", 30 | "Copy and add a .cursorrules file in the root of your project.\n", 31 | "\n", 32 | "The instructions in the .cursorrules file will be included for features such as Cursor Chat and Ctrl/⌘ K.\n", 33 | "\n", 34 | "The more specific your rules for your project, the better.\n", 35 | "\n", 36 | "Feel free to create your own directory using our template on GitHub.\n", 37 | "```" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "id": "8c46fa26-c6ba-4298-b2c2-ae25d32ec85a", 43 | "metadata": {}, 44 | "source": [ 45 | "### MCP (model context protocol)" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "id": "78439003-97fa-4fef-98ba-f6be5e00feaf", 51 | "metadata": {}, 52 | "source": [ 53 | "> 一种特别的，又 general 的 function calling\n", 54 | "- 实现 agent（computer use）的基础；\n", 55 | " - 学习 mcp，除了更加的自动化外，也是一个再次接触前端的机会；\n", 56 | "- 经典基础 mcp: 配置好之后各自都会提供一组 available tools\n", 57 | " - fetch：网页爬取；\n", 58 | " - https://github.com/modelcontextprotocol/servers/tree/main/src/fetch\n", 59 | " - filesystem：文件读写；\n", 60 | " - https://github.com/modelcontextprotocol/servers/tree/main/src/filesystem" 61 | ] 62 | } 63 | ], 64 | "metadata": { 65 | "kernelspec": { 66 | "display_name": "Python 3 (ipykernel)", 67 | "language": "python", 68 | "name": "python3" 69 | }, 70 | "language_info": { 71 | "codemirror_mode": { 72 | "name": "ipython", 73 | "version": 3 74 | }, 75 | "file_extension": ".py", 76 | "mimetype": "text/x-python", 77 | "name": "python", 78 | "nbconvert_exporter": "python", 79 | "pygments_lexer": "ipython3", 80 | "version": "3.10.0" 81 | } 82 | }, 83 | "nbformat": 4, 84 | "nbformat_minor": 5 85 | } 86 | -------------------------------------------------------------------------------- /tutorials/ai_app/ai-copilot/cursor-composer.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "1b226f8c-f083-43d3-bf3b-6868940d84a8", 6 | "metadata": {}, 7 | "source": [ 8 | "- cmd + k：行级别\n", 9 | "- cmd + l：文件级别\n", 10 | "- cmd + i：跨文件；" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "id": "0d7c91e2-29df-4851-87f3-aab195a3dddd", 16 | "metadata": {}, 17 | "source": [ 18 | "### composer 与 checkpoint" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "id": "39eb9d8e-2faa-41a2-bb8e-4d432068042c", 24 | "metadata": {}, 25 | "source": [ 26 | "- 适合一个项目的 startup，以搭建起相对完善的模块及框架；\n", 27 | " - composer 直接跨文件编写似乎上下文是整个 codebase；\n", 28 | "- composer\n", 29 | " - normal vs. agent\n", 30 | " - agent: agent composers can use tools\n", 31 | " - 可以新建新的 composer\n", 32 | " - composer history" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "id": "f2f072a8-3d5e-4e16-b06d-85e2d10d2062", 38 | "metadata": {}, 39 | "source": [ 40 | "### new features" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "id": "1fb73684-d3f8-470f-a552-143af574abcb", 46 | "metadata": {}, 47 | "source": [ 48 | "- `Generate Commit Message`\n", 49 | " - 在 Source Control tab" 50 | ] 51 | } 52 | ], 53 | "metadata": { 54 | "kernelspec": { 55 | "display_name": "Python 3 (ipykernel)", 56 | "language": "python", 57 | "name": "python3" 58 | }, 59 | "language_info": { 60 | "codemirror_mode": { 61 | "name": "ipython", 62 | "version": 3 63 | }, 64 | "file_extension": ".py", 65 | "mimetype": "text/x-python", 66 | "name": "python", 67 | "nbconvert_exporter": "python", 68 | "pygments_lexer": "ipython3", 69 | "version": "3.10.13" 70 | } 71 | }, 72 | "nbformat": 4, 73 | "nbformat_minor": 5 74 | } 75 | -------------------------------------------------------------------------------- /tutorials/ai_app/ai-copilot/cursor.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "f65e2fbc-0bf6-4284-9141-6bb18ab5add7", 7 | "metadata": { 8 | "execution": { 9 | "iopub.execute_input": "2024-12-09T15:01:37.901339Z", 10 | "iopub.status.busy": "2024-12-09T15:01:37.900698Z", 11 | "iopub.status.idle": "2024-12-09T15:01:37.919635Z", 12 | "shell.execute_reply": "2024-12-09T15:01:37.917865Z", 13 | "shell.execute_reply.started": "2024-12-09T15:01:37.901291Z" 14 | } 15 | }, 16 | "outputs": [], 17 | "source": [ 18 | "from IPython.display import Image" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 2, 24 | "id": "dc33b224-8a35-46cf-9f11-bac50ed92def", 25 | "metadata": { 26 | "execution": { 27 | "iopub.execute_input": "2024-12-09T15:01:39.151916Z", 28 | "iopub.status.busy": "2024-12-09T15:01:39.151360Z", 29 | "iopub.status.idle": "2024-12-09T15:01:39.169286Z", 30 | "shell.execute_reply": "2024-12-09T15:01:39.167225Z", 31 | "shell.execute_reply.started": "2024-12-09T15:01:39.151872Z" 32 | } 33 | }, 34 | "outputs": [ 35 | { 36 | "data": { 37 | "text/html": [ 38 | "" 39 | ], 40 | "text/plain": [ 41 | "" 42 | ] 43 | }, 44 | "execution_count": 2, 45 | "metadata": {}, 46 | "output_type": "execute_result" 47 | } 48 | ], 49 | "source": [ 50 | "Image(url='../imgs/tab.jpeg', width=500)" 51 | ] 52 | }, 53 | { 54 | "cell_type": "markdown", 55 | "id": "7b0f672f-c703-4ee1-81bf-a8887876946c", 56 | "metadata": {}, 57 | "source": [ 58 | "### 如何基于 cursor 从 0-1 实现项目开发（1013）" 59 | ] 60 | }, 61 | { 62 | "cell_type": "markdown", 63 | "id": "7f0d8f55-6e68-490a-993d-9b88e48c2e5c", 64 | "metadata": {}, 65 | "source": [ 66 | "- 创建文件夹先写一个 pipeline.md 的文件；\n", 67 | " - 在这个文件中用自然语言写整个项目的目标，及开发上的模块和流程；\n", 68 | " - 思路必须清晰，考验架构、抽象及模块化的能力；" 69 | ] 70 | }, 71 | { 72 | "cell_type": "markdown", 73 | "id": "85d71a79-0874-45c9-96e1-8267b6a3337d", 74 | "metadata": {}, 75 | "source": [ 76 | "## cursor vs. (github) copilot" 77 | ] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "id": "12a814a7-6bd9-4734-b606-6703758745ae", 82 | "metadata": {}, 83 | "source": [ 84 | "> Claude 3.5 sonnect 卓越的代码能力的加持下，使得 Cursor 的代码能力得到显著地飞升；\n", 85 | "\n", 86 | "- 传统的补全\n", 87 | " - words 匹配\n", 88 | "- copilot 只是一个高级的代码补全\n", 89 | " - vscode等其他的 plugin\n", 90 | " - 局部的自动高级补全，基于代码语义\n", 91 | " - 只做补全，接下文；\n", 92 | "- cursor：AI-first code editor\n", 93 | " - features\n", 94 | " - https://www.cursor.com/features\n", 95 | " - 本身就是一个 app （AI-powered IDE），real 结对编程\n", 96 | " - 继承了 vscode（无缝迁移 vscode 的插件）\n", 97 | " - 继承自 vscode 的快捷键\n", 98 | " - ctrl + shift + o：当前文件中定位 symbol\n", 99 | " - ctrl + t：跨文件定位 symbol\n", 100 | " - ctrl + p：基于文件名搜索文件\n", 101 | " - 更全局（global）的 context window\n", 102 | " - context of the entire project\n", 103 | " - 不只是尾部做提示补全，还可在内部做提示，还可做修改；\n", 104 | " - chat tab 支持图片输入；\n", 105 | " - 不需要图片文件，截图的拷贝，右键的拷贝，都可以直接粘贴进对话框中；\n" 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "id": "8189c517-5167-4f45-9712-0462c7732a3b", 111 | "metadata": {}, 112 | "source": [ 113 | "### ctrl + k, ctrl + l" 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "id": "d0f2d5ab-ad3a-4b9a-94fb-4c7e65ea948c", 119 | "metadata": {}, 120 | "source": [ 121 | "- ctrl + l：chat 模式下生成的 code snippets 可以直接 apply\n", 122 | " - tab tab ..." 123 | ] 124 | }, 125 | { 126 | "cell_type": "markdown", 127 | "id": "a920786e-b432-4fe2-b93e-a9bc1b1b3417", 128 | "metadata": {}, 129 | "source": [ 130 | "### prompts" 131 | ] 132 | }, 133 | { 134 | "cell_type": "markdown", 135 | "id": "f201c290-4379-4e94-8c9b-f5c6c8f90b42", 136 | "metadata": {}, 137 | "source": [ 138 | "- 对话框中\n", 139 | " - 复制当前文件中的一段代码，\n", 140 | " - 解释这段代码；\n", 141 | " - 重构这段代码；\n", 142 | "- code editor 基于注释的自动补全：自然语言编程；\n", 143 | " - 先写注释\n", 144 | " - 换行，让其自动补全；\n", 145 | "- RAG\n", 146 | " - 读 api 文档（官方 doc），基于 api 或者官方 doc 写代码" 147 | ] 148 | }, 149 | { 150 | "cell_type": "markdown", 151 | "id": "65a84e5e-c9b1-42dd-aff5-b6cc3c52d5f8", 152 | "metadata": {}, 153 | "source": [ 154 | "### terminal" 155 | ] 156 | }, 157 | { 158 | "cell_type": "markdown", 159 | "id": "afc96c7c-d335-4081-935d-fae896eaf99f", 160 | "metadata": {}, 161 | "source": [ 162 | "- 输入时\n", 163 | " - ⌘ + k：generate a command\n", 164 | " - ⌘ + /：autocomplete\n", 165 | "- 执行命令时\n", 166 | " - 自动提取错误到右侧的 chat 对话框；\n", 167 | " - Please help me debug this code." 168 | ] 169 | }, 170 | { 171 | "cell_type": "markdown", 172 | "id": "cf0955da-f01c-4ead-94c2-b52d87b54786", 173 | "metadata": {}, 174 | "source": [ 175 | "### 3.5 sonnect artifact" 176 | ] 177 | }, 178 | { 179 | "cell_type": "markdown", 180 | "id": "af9e612e-b4cc-4da8-bf00-3920a0bd905e", 181 | "metadata": {}, 182 | "source": [ 183 | "\n", 184 | "- 基于 Mermaid 画流程图（而且是 svg），太秒了；\n", 185 | " - 后续可以基于此，也即是将一些复杂逻辑的代码丢进去，让其先画流程图出来，很好/很快地去理解和消化复杂的代码逻辑；" 186 | ] 187 | } 188 | ], 189 | "metadata": { 190 | "kernelspec": { 191 | "display_name": "Python 3 (ipykernel)", 192 | "language": "python", 193 | "name": "python3" 194 | }, 195 | "language_info": { 196 | "codemirror_mode": { 197 | "name": "ipython", 198 | "version": 3 199 | }, 200 | "file_extension": ".py", 201 | "mimetype": "text/x-python", 202 | "name": "python", 203 | "nbconvert_exporter": "python", 204 | "pygments_lexer": "ipython3", 205 | "version": "3.10.13" 206 | } 207 | }, 208 | "nbformat": 4, 209 | "nbformat_minor": 5 210 | } 211 | -------------------------------------------------------------------------------- /tutorials/ai_app/audio_video/audio.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/ai_app/audio_video/audio.wav -------------------------------------------------------------------------------- /tutorials/ai_app/audio_video/audio_notes.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "36f6b62e-5505-42e0-bc33-6119bb1e3b8e", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "from IPython.display import Image" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "id": "88ded059-af1a-4433-8356-1a806f8a8e2c", 16 | "metadata": {}, 17 | "source": [ 18 | "- https://github.com/harry0703/AudioNotes.git" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 2, 24 | "id": "32980ce2-581b-47dd-aa5e-884a2c80c232", 25 | "metadata": {}, 26 | "outputs": [ 27 | { 28 | "data": { 29 | "text/html": [ 30 | "" 31 | ], 32 | "text/plain": [ 33 | "" 34 | ] 35 | }, 36 | "execution_count": 2, 37 | "metadata": {}, 38 | "output_type": "execute_result" 39 | } 40 | ], 41 | "source": [ 42 | "Image(url='./imgs/audio-notes-framework.png', width=400)" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "id": "21f3d25d-d355-4dc9-bfd1-abd8a70572b8", 48 | "metadata": {}, 49 | "source": [ 50 | "```python\n", 51 | "@cl.on_chat_start\n", 52 | "async def on_chat_start():\n", 53 | " ...\n", 54 | " asr_result = await transcribe_file(file)\n", 55 | " await summarize_notes(asr_result)\n", 56 | " await msg.send()\n", 57 | " ...\n", 58 | "```" 59 | ] 60 | }, 61 | { 62 | "cell_type": "markdown", 63 | "id": "afdc3984-10fe-475e-9e3c-2d715b6d26b9", 64 | "metadata": {}, 65 | "source": [ 66 | "### transcribe_file" 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "id": "952f14be-b122-42ce-b6cf-924cbf80644e", 72 | "metadata": {}, 73 | "source": [ 74 | "```python\n", 75 | "from funasr import AutoModel\n", 76 | "...\n", 77 | " self.__model = AutoModel(model=\"paraformer-zh\",\n", 78 | " vad_model=\"fsmn-vad\",\n", 79 | " punc_model=\"ct-punc\",\n", 80 | " spk_model=\"cam++\",\n", 81 | " log_level=\"error\",\n", 82 | " hub=\"ms\" # hub：表示模型仓库，ms为选择modelscope下载，hf为选择huggingface下载。\n", 83 | " )\n", 84 | "```\n", 85 | "\n", 86 | "- `~/.cache/modelscope/hub/models`" 87 | ] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "id": "5aa79b0e-a282-4471-898a-f2f15b8bf703", 92 | "metadata": {}, 93 | "source": [ 94 | "### chat " 95 | ] 96 | }, 97 | { 98 | "cell_type": "markdown", 99 | "id": "2a024a88-ff8f-4bfb-80b2-3f7bcfdca6e4", 100 | "metadata": {}, 101 | "source": [ 102 | "```python\n", 103 | "async def summarize_notes(text):\n", 104 | " messages = [\n", 105 | " {\"role\": \"system\", \"content\": \"你是一名笔记整理专家，根据用户提供的内容，整理出一份内容详尽的结构化的笔记\"},\n", 106 | " {\"role\": \"user\", \"content\": text},\n", 107 | " ]\n", 108 | "```" 109 | ] 110 | } 111 | ], 112 | "metadata": { 113 | "kernelspec": { 114 | "display_name": "Python 3 (ipykernel)", 115 | "language": "python", 116 | "name": "python3" 117 | }, 118 | "language_info": { 119 | "codemirror_mode": { 120 | "name": "ipython", 121 | "version": 3 122 | }, 123 | "file_extension": ".py", 124 | "mimetype": "text/x-python", 125 | "name": "python", 126 | "nbconvert_exporter": "python", 127 | "pygments_lexer": "ipython3", 128 | "version": "3.10.0" 129 | } 130 | }, 131 | "nbformat": 4, 132 | "nbformat_minor": 5 133 | } 134 | -------------------------------------------------------------------------------- /tutorials/ai_app/audio_video/imgs/audio-notes-framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/ai_app/audio_video/imgs/audio-notes-framework.png -------------------------------------------------------------------------------- /tutorials/ai_app/audio_video/speech_generation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "7847bdbc-4bb8-4b5d-84be-60c37b11f3fb", 6 | "metadata": {}, 7 | "source": [ 8 | "- 声音克隆\n", 9 | " - https://github.com/FunAudioLLM/CosyVoice\n", 10 | " - https://funaudiollm.github.io/" 11 | ] 12 | } 13 | ], 14 | "metadata": { 15 | "kernelspec": { 16 | "display_name": "Python 3 (ipykernel)", 17 | "language": "python", 18 | "name": "python3" 19 | }, 20 | "language_info": { 21 | "codemirror_mode": { 22 | "name": "ipython", 23 | "version": 3 24 | }, 25 | "file_extension": ".py", 26 | "mimetype": "text/x-python", 27 | "name": "python", 28 | "nbconvert_exporter": "python", 29 | "pygments_lexer": "ipython3", 30 | "version": "3.10.13" 31 | } 32 | }, 33 | "nbformat": 4, 34 | "nbformat_minor": 5 35 | } 36 | -------------------------------------------------------------------------------- /tutorials/ai_app/audio_video/video.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/ai_app/audio_video/video.mp4 -------------------------------------------------------------------------------- /tutorials/ai_app/audio_video/whisper_whisperx.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "baa28a74-f874-42e4-a57d-b48a0af5b229", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "# pip install pytube moviepy openai-whisper" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 2, 16 | "id": "1f12058d-29cd-4d37-90ad-f0440436a971", 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "# !pip install you-get\n", 21 | "# !pip install yt-dlp" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "id": "5101bc6a-f847-4f53-bbc9-e8b5b6a3155a", 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "import pytube\n", 32 | "import moviepy.editor as mp\n", 33 | "import whisper\n", 34 | "import os\n", 35 | "import subprocess\n", 36 | "\n", 37 | "\n", 38 | "# 使用 yt-dlp 下载视频\n", 39 | "def download_video(url, output_path='video.mp4'):\n", 40 | " subprocess.run(['yt-dlp', '-o', output_path, url])\n", 41 | " return output_path\n", 42 | "\n", 43 | "# 从视频中提取音频\n", 44 | "def extract_audio(video_path, output_path='audio.wav'):\n", 45 | " video = mp.VideoFileClip(video_path)\n", 46 | " video.audio.write_audiofile(output_path)\n", 47 | " return output_path\n", 48 | "\n", 49 | "# 转录音频\n", 50 | "def transcribe_audio_with_whisper(audio_path):\n", 51 | " model = whisper.load_model(\"large\")\n", 52 | " # result = model.transcribe(audio_path, language='en')\n", 53 | " result = model.transcribe(audio_path)\n", 54 | " return result['text']" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 2, 60 | "id": "6912b1ae-ed9d-4947-8e7b-f16ccdbddfeb", 61 | "metadata": {}, 62 | "outputs": [ 63 | { 64 | "name": "stdout", 65 | "output_type": "stream", 66 | "text": [ 67 | "Downloading video...\n", 68 | "[BiliBili] Extracting URL: https://www.bilibili.com/video/BV12J4m1N7wU/\n", 69 | "[BiliBili] 12J4m1N7wU: Downloading webpage\n", 70 | "[BiliBili] BV12J4m1N7wU: Extracting videos in anthology\n", 71 | "[BiliBili] 1254605812: Extracting chapters\n", 72 | "[BiliBili] Format(s) 1080P 高码率, 1080P 高清, 720P 高清 are missing; you have to login or become premium member to download them. Use --cookies-from-browser or --cookies for the authentication. See https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for how to manually pass cookies\n", 73 | "[info] BV12J4m1N7wU: Downloading 1 format(s): 100110+30280\n", 74 | "[download] Destination: ./video.f100110.mp4\n", 75 | "[download] 100% of 425.01KiB in 00:00:00 at 7.64MiB/s \n", 76 | "[download] Destination: ./video.f30280.m4a\n", 77 | "[download] 100% of 274.45KiB in 00:00:00 at 10.31MiB/s \n", 78 | "[Merger] Merging formats into \"./video.mp4\"\n", 79 | "Deleting original file ./video.f30280.m4a (pass -k to keep)\n", 80 | "Deleting original file ./video.f100110.mp4 (pass -k to keep)\n", 81 | "Extracting audio...\n", 82 | "MoviePy - Writing audio in ./audio.wav\n" 83 | ] 84 | }, 85 | { 86 | "name": "stderr", 87 | "output_type": "stream", 88 | "text": [ 89 | " \r" 90 | ] 91 | }, 92 | { 93 | "name": "stdout", 94 | "output_type": "stream", 95 | "text": [ 96 | "MoviePy - Done.\n", 97 | "Transcribing audio with Whisper...\n", 98 | "Transcription:\n", 99 | " Saying goodbye, I don't think is ever nice, but saying goodbye without Feeling sad or feeling hurt or whatever that would that would just mean that the time we spent together was Right great and had a great time. So it was always clear that it will be tough and I know it will be tough\n" 100 | ] 101 | } 102 | ], 103 | "source": [ 104 | "\n", 105 | "video_url = 'https://www.bilibili.com/video/BV12J4m1N7wU/'\n", 106 | "video_path = './video.mp4'\n", 107 | "audio_path = './audio.wav'\n", 108 | "\n", 109 | "print(\"Downloading video...\")\n", 110 | "download_video(video_url, video_path)\n", 111 | "\n", 112 | "print(\"Extracting audio...\")\n", 113 | "extract_audio(video_path, audio_path)\n", 114 | "\n", 115 | "print(\"Transcribing audio with Whisper...\")\n", 116 | "transcription = transcribe_audio_with_whisper(audio_path)\n", 117 | "\n", 118 | "print(\"Transcription:\")\n", 119 | "print(transcription)\n", 120 | "\n", 121 | "# 清理临时文件\n", 122 | "# os.remove(video_path)\n", 123 | "# os.remove(audio_path)\n" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "id": "21bbd3ef-11f5-4238-b128-e626a60d9fed", 129 | "metadata": {}, 130 | "source": [ 131 | "### whisper" 132 | ] 133 | }, 134 | { 135 | "cell_type": "markdown", 136 | "id": "a38dddf6-8349-415a-863f-e46a4e8d3b22", 137 | "metadata": {}, 138 | "source": [ 139 | "- `--task`\n", 140 | " - transcribe：转录\n", 141 | " - translate：翻译；" 142 | ] 143 | }, 144 | { 145 | "cell_type": "markdown", 146 | "id": "311847fb-838b-4cc9-8446-42e0e1979f53", 147 | "metadata": {}, 148 | "source": [ 149 | "### whisperx" 150 | ] 151 | }, 152 | { 153 | "cell_type": "markdown", 154 | "id": "74aa0570-2c77-49d6-94ef-660b19e67db2", 155 | "metadata": {}, 156 | "source": [ 157 | "- https://github.com/m-bain/whisperX" 158 | ] 159 | }, 160 | { 161 | "cell_type": "markdown", 162 | "id": "f37ccfab-b771-476a-8eb2-03d01d0fd831", 163 | "metadata": {}, 164 | "source": [ 165 | "- 说话人分离（--diarize）\n", 166 | " - 注意一些 hf 的 models，也许需要在 hf 页面做申请，比如 `pyannote/segmentation-3.0`\n", 167 | " - 也许需要在环境变量中配置 `hf_token`\n", 168 | "- srt file：（--highlight_words）" 169 | ] 170 | } 171 | ], 172 | "metadata": { 173 | "kernelspec": { 174 | "display_name": "Python 3 (ipykernel)", 175 | "language": "python", 176 | "name": "python3" 177 | }, 178 | "language_info": { 179 | "codemirror_mode": { 180 | "name": "ipython", 181 | "version": 3 182 | }, 183 | "file_extension": ".py", 184 | "mimetype": "text/x-python", 185 | "name": "python", 186 | "nbconvert_exporter": "python", 187 | "pygments_lexer": "ipython3", 188 | "version": "3.10.13" 189 | } 190 | }, 191 | "nbformat": 4, 192 | "nbformat_minor": 5 193 | } 194 | -------------------------------------------------------------------------------- /tutorials/ai_app/db/postgresql.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "82bfff79-6e0b-4617-b05a-5645d55c5a81", 6 | "metadata": {}, 7 | "source": [ 8 | "```sh\n", 9 | "psql -U user_name -d db_name\n", 10 | "\\list # list db\n", 11 | "\\c db_name # change db\n", 12 | "\\dt # 列出当前数据库中的所有表 \n", 13 | "\\d table_name # 查看特定表的详细结构\n", 14 | "\n", 15 | "\\q # quit\n", 16 | "```" 17 | ] 18 | } 19 | ], 20 | "metadata": { 21 | "kernelspec": { 22 | "display_name": "Python 3 (ipykernel)", 23 | "language": "python", 24 | "name": "python3" 25 | }, 26 | "language_info": { 27 | "codemirror_mode": { 28 | "name": "ipython", 29 | "version": 3 30 | }, 31 | "file_extension": ".py", 32 | "mimetype": "text/x-python", 33 | "name": "python", 34 | "nbconvert_exporter": "python", 35 | "pygments_lexer": "ipython3", 36 | "version": "3.10.0" 37 | } 38 | }, 39 | "nbformat": 4, 40 | "nbformat_minor": 5 41 | } 42 | -------------------------------------------------------------------------------- /tutorials/ai_app/imgs/0_Parade_Parade_0_873.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/ai_app/imgs/0_Parade_Parade_0_873.jpg -------------------------------------------------------------------------------- /tutorials/ai_app/imgs/0_Parade_marchingband_1_765.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/ai_app/imgs/0_Parade_marchingband_1_765.jpg -------------------------------------------------------------------------------- /tutorials/ai_app/imgs/11_Meeting_Meeting_11_Meeting_Meeting_11_176.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/ai_app/imgs/11_Meeting_Meeting_11_Meeting_Meeting_11_176.jpg -------------------------------------------------------------------------------- /tutorials/ai_app/imgs/11_Meeting_Meeting_11_Meeting_Meeting_11_26.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/ai_app/imgs/11_Meeting_Meeting_11_Meeting_Meeting_11_26.jpg -------------------------------------------------------------------------------- /tutorials/ai_app/imgs/gh_copilot_suggest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/ai_app/imgs/gh_copilot_suggest.png -------------------------------------------------------------------------------- /tutorials/ai_app/imgs/tab.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/ai_app/imgs/tab.jpeg -------------------------------------------------------------------------------- /tutorials/ai_app/openrouter.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "c95eb1c2-1a8a-4ad6-95bc-23b9abe507ba", 6 | "metadata": {}, 7 | "source": [ 8 | "> A unified interface for LLMs。大模型的中间商，也可能是中间商的中间商；\n", 9 | "\n", 10 | "- 不是所有的模型都可以方便地本地部署 (*2 gpu memory)\n", 11 | " - qwen/qwen-2.5-72b-instruct\n", 12 | " - deepseek-v3；\n", 13 | " - llama3.1-405b；\n", 14 | "- 远端api快速验证，evaluate，科研或者工程；\n", 15 | "- 虚拟信用卡\n", 16 | " - https://bewildcard.com/i/CHUNHUI3\n", 17 | "- 关于一个模型的不同 provider 的路由策略（provider routing）\n", 18 | " - https://openrouter.ai/docs/provider-routing" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 3, 24 | "id": "8839a1bb-019b-42be-bbf3-5ed0ee5942d5", 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "from openai import OpenAI" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 4, 34 | "id": "449661f7-3031-4ccd-bb00-9d625f7ff440", 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "from dotenv import load_dotenv, find_dotenv\n", 39 | "import os" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 5, 45 | "id": "24dba6a5-9fc6-4e00-8fad-0b5c5c284c41", 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "assert load_dotenv(find_dotenv())" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 7, 55 | "id": "3cc1ad49-dfd8-44a8-8a8b-accad01b7b8d", 56 | "metadata": {}, 57 | "outputs": [ 58 | { 59 | "name": "stdout", 60 | "output_type": "stream", 61 | "text": [ 62 | "I am an instance of OpenAI's language model, specifically based on the GPT-4 architecture. My design allows me to understand and generate human-like text based on the input I receive. I can assist with a wide range of tasks, from answering questions and providing explanations to generating creative content and offering advice. Let me know how I can help you today!\n" 63 | ] 64 | } 65 | ], 66 | "source": [ 67 | "client = OpenAI(\n", 68 | " base_url=\"https://openrouter.ai/api/v1\",\n", 69 | " api_key=os.getenv('OPENROUTER_API_KEY'),\n", 70 | ")\n", 71 | "\n", 72 | "completion = client.chat.completions.create(\n", 73 | " model=\"deepseek/deepseek-chat\",\n", 74 | " messages=[\n", 75 | " {\n", 76 | " \"role\": \"user\",\n", 77 | " \"content\": \"what model are you?\"\n", 78 | " }\n", 79 | " ]\n", 80 | ")\n", 81 | "print(completion.choices[0].message.content)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "id": "08e42d1d-2faf-41ee-b867-f4786bfb6c5b", 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [] 91 | } 92 | ], 93 | "metadata": { 94 | "kernelspec": { 95 | "display_name": "Python 3 (ipykernel)", 96 | "language": "python", 97 | "name": "python3" 98 | }, 99 | "language_info": { 100 | "codemirror_mode": { 101 | "name": "ipython", 102 | "version": 3 103 | }, 104 | "file_extension": ".py", 105 | "mimetype": "text/x-python", 106 | "name": "python", 107 | "nbconvert_exporter": "python", 108 | "pygments_lexer": "ipython3", 109 | "version": "3.10.13" 110 | } 111 | }, 112 | "nbformat": 4, 113 | "nbformat_minor": 5 114 | } 115 | -------------------------------------------------------------------------------- /tutorials/ai_app/pipeline_工具.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "8f0e85be-7fb0-4027-b1cc-b80aa7c8912c", 6 | "metadata": {}, 7 | "source": [ 8 | "### 微信聊天记录导出" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "adfa7307-2681-4b57-9baf-94784dd47f46", 14 | "metadata": {}, 15 | "source": [ 16 | "- https://github.com/LC044/WeChatMsg\n", 17 | " - 甚至可以导出为支持 GLM 训练的 json 格式；" 18 | ] 19 | }, 20 | { 21 | "cell_type": "markdown", 22 | "id": "4af4610d-dfbf-4cad-a553-aea251628a20", 23 | "metadata": {}, 24 | "source": [ 25 | "### misc" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "id": "4621f7d2-179f-417b-aecb-f1c1de1b0697", 31 | "metadata": {}, 32 | "source": [ 33 | "- markitdown\n", 34 | " - https://github.com/microsoft/markitdown\n", 35 | " - 论文 pdf => markdown\n", 36 | " - latex/formula 转换效果不佳；\n", 37 | " - 轻量级，简单但也能力有限；\n", 38 | "- **MinerU**：挺惊艳的；\n", 39 | " - https://github.com/opendatalab/MinerU\n", 40 | " - 环境配置\n", 41 | " - model weights download\n", 42 | " - https://github.com/opendatalab/MinerU/blob/master/docs/how_to_download_models_en.md\n", 43 | " - 配置文件：`~/magic-pdf.json`\n", 44 | " - 命令行使用\n", 45 | " - https://mineru.readthedocs.io/en/latest/user_guide/quick_start/command_line.html" 46 | ] 47 | } 48 | ], 49 | "metadata": { 50 | "kernelspec": { 51 | "display_name": "Python 3 (ipykernel)", 52 | "language": "python", 53 | "name": "python3" 54 | }, 55 | "language_info": { 56 | "codemirror_mode": { 57 | "name": "ipython", 58 | "version": 3 59 | }, 60 | "file_extension": ".py", 61 | "mimetype": "text/x-python", 62 | "name": "python", 63 | "nbconvert_exporter": "python", 64 | "pygments_lexer": "ipython3", 65 | "version": "3.10.13" 66 | } 67 | }, 68 | "nbformat": 4, 69 | "nbformat_minor": 5 70 | } 71 | -------------------------------------------------------------------------------- /tutorials/ai_app/streamlit(web).ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "9c559a42-c664-44da-b22d-7f69a67d3f4f", 7 | "metadata": { 8 | "execution": { 9 | "iopub.execute_input": "2024-12-14T03:31:31.255262Z", 10 | "iopub.status.busy": "2024-12-14T03:31:31.254614Z", 11 | "iopub.status.idle": "2024-12-14T03:31:31.473634Z", 12 | "shell.execute_reply": "2024-12-14T03:31:31.472825Z", 13 | "shell.execute_reply.started": "2024-12-14T03:31:31.255226Z" 14 | } 15 | }, 16 | "outputs": [], 17 | "source": [ 18 | "import streamlit as st" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "id": "0f7df014-bb4b-48c8-97c8-5b155f4d907a", 24 | "metadata": {}, 25 | "source": [ 26 | "```\n", 27 | "st.set_page_config(\n", 28 | "st.title(\n", 29 | "st.markdown(\n", 30 | "user_query = st.text_input(\"Enter your query:\", placeholder=\"e.g., How many 'R's are in the word strawberry?\")\n", 31 | "```" 32 | ] 33 | }, 34 | { 35 | "cell_type": "markdown", 36 | "id": "66bac8e5-9c8f-419e-9508-43e3c2e96460", 37 | "metadata": {}, 38 | "source": [ 39 | "### 运行" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "id": "5388d98b-d544-4076-a88c-451569263d84", 45 | "metadata": {}, 46 | "source": [ 47 | "```\n", 48 | "streamlit run app.py\n", 49 | "```" 50 | ] 51 | } 52 | ], 53 | "metadata": { 54 | "kernelspec": { 55 | "display_name": "Python 3 (ipykernel)", 56 | "language": "python", 57 | "name": "python3" 58 | }, 59 | "language_info": { 60 | "codemirror_mode": { 61 | "name": "ipython", 62 | "version": 3 63 | }, 64 | "file_extension": ".py", 65 | "mimetype": "text/x-python", 66 | "name": "python", 67 | "nbconvert_exporter": "python", 68 | "pygments_lexer": "ipython3", 69 | "version": "3.10.13" 70 | } 71 | }, 72 | "nbformat": 4, 73 | "nbformat_minor": 5 74 | } 75 | -------------------------------------------------------------------------------- /tutorials/ai_app/tools/Monica.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "2bb10555-109d-4c43-8540-b0395415f758", 6 | "metadata": {}, 7 | "source": [ 8 | "- gpt 套壳\n", 9 | "- 浏览器插件：Modern AI tools\n", 10 | " - （任意）网页内容：几乎作为了浏览器任意网页的侧边栏及右键选项；\n", 11 | " - summary：带 index；\n", 12 | " - 选中之后的翻译\n", 13 | " - editor：扩写，润色等\n", 14 | " - （youtube）summary\n", 15 | " - chatbot" 16 | ] 17 | } 18 | ], 19 | "metadata": { 20 | "kernelspec": { 21 | "display_name": "Python 3 (ipykernel)", 22 | "language": "python", 23 | "name": "python3" 24 | }, 25 | "language_info": { 26 | "codemirror_mode": { 27 | "name": "ipython", 28 | "version": 3 29 | }, 30 | "file_extension": ".py", 31 | "mimetype": "text/x-python", 32 | "name": "python", 33 | "nbconvert_exporter": "python", 34 | "pygments_lexer": "ipython3", 35 | "version": "3.10.13" 36 | } 37 | }, 38 | "nbformat": 4, 39 | "nbformat_minor": 5 40 | } 41 | -------------------------------------------------------------------------------- /tutorials/ai_app/tools/ai_tools.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "cae47a17", 6 | "metadata": {}, 7 | "source": [ 8 | "- ocr\n", 9 | " - paddleocr\n", 10 | "- asr\n", 11 | " - whisper" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "id": "b3e0d1d9-18e7-4f1a-9c75-2cdd95638aa0", 17 | "metadata": {}, 18 | "source": [ 19 | "### whisper" 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "id": "daa28c8f-26e7-47ae-8da4-94bf7d30fa7a", 25 | "metadata": {}, 26 | "source": [ 27 | "- https://github.com/chidiwilliams/buzz\n", 28 | " - release：https://github.com/chidiwilliams/buzz/releases\n", 29 | " - 使用：https://www.youtube.com/watch?v=qZ3T5hunOuQ" 30 | ] 31 | } 32 | ], 33 | "metadata": { 34 | "kernelspec": { 35 | "display_name": "Python 3 (ipykernel)", 36 | "language": "python", 37 | "name": "python3" 38 | }, 39 | "language_info": { 40 | "codemirror_mode": { 41 | "name": "ipython", 42 | "version": 3 43 | }, 44 | "file_extension": ".py", 45 | "mimetype": "text/x-python", 46 | "name": "python", 47 | "nbconvert_exporter": "python", 48 | "pygments_lexer": "ipython3", 49 | "version": "3.10.13" 50 | }, 51 | "toc": { 52 | "base_numbering": 1, 53 | "nav_menu": {}, 54 | "number_sections": true, 55 | "sideBar": true, 56 | "skip_h1_title": false, 57 | "title_cell": "Table of Contents", 58 | "title_sidebar": "Contents", 59 | "toc_cell": false, 60 | "toc_position": {}, 61 | "toc_section_display": true, 62 | "toc_window_display": false 63 | } 64 | }, 65 | "nbformat": 4, 66 | "nbformat_minor": 5 67 | } 68 | -------------------------------------------------------------------------------- /tutorials/ai_app/vision/rembg-U2Net.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "ac82e26e-c981-40c5-992c-4e698a04af42", 6 | "metadata": {}, 7 | "source": [ 8 | "## u2net" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "bdeec8d8-6e0e-40ce-9173-ac87499058c6", 14 | "metadata": {}, 15 | "source": [ 16 | "- https://github.com/xuebinqin/U-2-Net" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "id": "327a78a1-96e8-48b0-bfce-8f8135718bd6", 22 | "metadata": {}, 23 | "source": [ 24 | "### human portrait" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "id": "dcda27c5-d2b7-4795-8871-e9a8597571c6", 30 | "metadata": {}, 31 | "source": [ 32 | "- `python u2net_portrait_composite.py -s 20 -a 0.5`\n", 33 | " - -s indicates the sigma of gaussian function for blurring the orignal image\n", 34 | " - -a denotes the alpha weights of the orignal image when fusing them.\n", 35 | " - 代码层面要修改\n", 36 | " - `im_comp = im_comp.astype(np.uint8)`（增加这一步处理，适配新版本的 pillow？）" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "id": "298c3c49-4566-4295-90b5-e2c7c3ecf55d", 42 | "metadata": {}, 43 | "source": [ 44 | "### human segmentation" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "id": "0450941b-d491-445e-8b49-df32925166ac", 50 | "metadata": {}, 51 | "source": [ 52 | "- `python u2net_human_seg_test.py`" 53 | ] 54 | }, 55 | { 56 | "cell_type": "markdown", 57 | "id": "14fa93d2-1c9d-4323-b870-512bfe9c400b", 58 | "metadata": {}, 59 | "source": [ 60 | "## rembg" 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "id": "3e69eda4-2d17-4995-94e6-dcc8622e039d", 66 | "metadata": {}, 67 | "source": [ 68 | "- `pip install \"rembg[gpu,cli]\"`\n", 69 | " - 注意 onnxruntime-gpu 的版本兼容性问题\n", 70 | " - `pip install onnxruntime-gpu==1.11`\n", 71 | " - `pip install onnxruntime-gpu==1.12`\n", 72 | " - https://onnxruntime.ai/docs/execution-providers/TensorRT-ExecutionProvider.html#requirements\n", 73 | "- https://github.com/danielgatis/rembg\n", 74 | " - Rembg is a tool to remove images background " 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "id": "a8e8309b-4be6-4167-a54d-009144203cff", 80 | "metadata": {}, 81 | "source": [ 82 | "- rembg i path/to/input.png path/to/output.png\n", 83 | "- rembg i -m u2netp path/to/input.png path/to/output.png\n", 84 | " - 内部使用的也是 u2net" 85 | ] 86 | } 87 | ], 88 | "metadata": { 89 | "kernelspec": { 90 | "display_name": "Python 3 (ipykernel)", 91 | "language": "python", 92 | "name": "python3" 93 | }, 94 | "language_info": { 95 | "codemirror_mode": { 96 | "name": "ipython", 97 | "version": 3 98 | }, 99 | "file_extension": ".py", 100 | "mimetype": "text/x-python", 101 | "name": "python", 102 | "nbconvert_exporter": "python", 103 | "pygments_lexer": "ipython3", 104 | "version": "3.10.13" 105 | } 106 | }, 107 | "nbformat": 4, 108 | "nbformat_minor": 5 109 | } 110 | -------------------------------------------------------------------------------- /tutorials/ai_app/vision/图像视频.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "9a42d307-b1ef-4ef9-9f61-1e7092caba21", 6 | "metadata": {}, 7 | "source": [ 8 | "### 图像" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "cf14d1d8-985a-47b7-b164-f1efb0c76bb3", 14 | "metadata": {}, 15 | "source": [ 16 | "- textbehindimage" 17 | ] 18 | } 19 | ], 20 | "metadata": { 21 | "kernelspec": { 22 | "display_name": "Python 3 (ipykernel)", 23 | "language": "python", 24 | "name": "python3" 25 | }, 26 | "language_info": { 27 | "codemirror_mode": { 28 | "name": "ipython", 29 | "version": 3 30 | }, 31 | "file_extension": ".py", 32 | "mimetype": "text/x-python", 33 | "name": "python", 34 | "nbconvert_exporter": "python", 35 | "pygments_lexer": "ipython3", 36 | "version": "3.10.13" 37 | } 38 | }, 39 | "nbformat": 4, 40 | "nbformat_minor": 5 41 | } 42 | -------------------------------------------------------------------------------- /tutorials/ai_app/项目参考.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "178d3861-5fe5-4ea3-b9e1-d8cbc6e783f4", 6 | "metadata": {}, 7 | "source": [ 8 | "- https://github.com/LYiHub/mad-professor-public\n", 9 | " - https://www.bilibili.com/video/BV18j5jz2Eco/" 10 | ] 11 | } 12 | ], 13 | "metadata": { 14 | "kernelspec": { 15 | "display_name": "casual", 16 | "language": "python", 17 | "name": "casual" 18 | }, 19 | "language_info": { 20 | "codemirror_mode": { 21 | "name": "ipython", 22 | "version": 3 23 | }, 24 | "file_extension": ".py", 25 | "mimetype": "text/x-python", 26 | "name": "python", 27 | "nbconvert_exporter": "python", 28 | "pygments_lexer": "ipython3", 29 | "version": "3.10.0" 30 | } 31 | }, 32 | "nbformat": 4, 33 | "nbformat_minor": 5 34 | } 35 | -------------------------------------------------------------------------------- /tutorials/cv/PIL.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "8116ee9e", 7 | "metadata": { 8 | "ExecuteTime": { 9 | "end_time": "2024-02-07T13:55:21.637323Z", 10 | "start_time": "2024-02-07T13:55:20.304107Z" 11 | } 12 | }, 13 | "outputs": [], 14 | "source": [ 15 | "from einops import rearrange\n", 16 | "from torchvision import transforms\n", 17 | "from PIL import Image\n", 18 | "\n", 19 | "transform = transforms.Compose(\n", 20 | " [\n", 21 | " transforms.Resize((224, 224)),\n", 22 | "# transforms.RandomResizedCrop(224),\n", 23 | "# transforms.RandomHorizontalFlip(),\n", 24 | " transforms.ToTensor(),\n", 25 | " ]\n", 26 | ")" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": 2, 32 | "id": "deff7797", 33 | "metadata": { 34 | "ExecuteTime": { 35 | "end_time": "2024-02-08T12:24:38.743729Z", 36 | "start_time": "2024-02-08T12:24:38.591058Z" 37 | } 38 | }, 39 | "outputs": [], 40 | "source": [ 41 | "import numpy as np" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "id": "58989dc8", 47 | "metadata": {}, 48 | "source": [ 49 | "## Image" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 3, 55 | "id": "cbbac838", 56 | "metadata": { 57 | "ExecuteTime": { 58 | "end_time": "2024-02-07T14:02:25.305528Z", 59 | "start_time": "2024-02-07T14:02:25.298925Z" 60 | } 61 | }, 62 | "outputs": [], 63 | "source": [ 64 | "from PIL import Image" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 4, 70 | "id": "6894b6b4", 71 | "metadata": { 72 | "ExecuteTime": { 73 | "end_time": "2024-02-07T14:02:39.646406Z", 74 | "start_time": "2024-02-07T14:02:39.626204Z" 75 | } 76 | }, 77 | "outputs": [], 78 | "source": [ 79 | "image = Image.open('../nn_basics/imgs/boo.jpg')" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": 5, 85 | "id": "b0185f3f", 86 | "metadata": { 87 | "ExecuteTime": { 88 | "end_time": "2024-02-07T14:02:45.354535Z", 89 | "start_time": "2024-02-07T14:02:45.338394Z" 90 | } 91 | }, 92 | "outputs": [ 93 | { 94 | "data": { 95 | "text/plain": [ 96 | "(1500, 1500)" 97 | ] 98 | }, 99 | "execution_count": 5, 100 | "metadata": {}, 101 | "output_type": "execute_result" 102 | } 103 | ], 104 | "source": [ 105 | "image.size" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": 11, 111 | "id": "5038a39a", 112 | "metadata": { 113 | "ExecuteTime": { 114 | "end_time": "2024-02-07T14:23:11.461043Z", 115 | "start_time": "2024-02-07T14:23:11.450438Z" 116 | } 117 | }, 118 | "outputs": [ 119 | { 120 | "data": { 121 | "text/plain": [ 122 | "'RGB'" 123 | ] 124 | }, 125 | "execution_count": 11, 126 | "metadata": {}, 127 | "output_type": "execute_result" 128 | } 129 | ], 130 | "source": [ 131 | "# channel\n", 132 | "\n", 133 | "# \"L\": 8位灰度图像。\n", 134 | "# \"RGB\": 24位真彩色图像（8位×3）。\n", 135 | "# \"RGBA\": 32位真彩色图像+透明通道（8位×4）。\n", 136 | "image.mode" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": 1, 142 | "id": "d79e2cf3", 143 | "metadata": { 144 | "ExecuteTime": { 145 | "end_time": "2024-02-08T12:24:10.167103Z", 146 | "start_time": "2024-02-08T12:24:10.128469Z" 147 | } 148 | }, 149 | "outputs": [], 150 | "source": [ 151 | "from PIL.Image import fromarray" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": null, 157 | "id": "0691c1d9", 158 | "metadata": {}, 159 | "outputs": [], 160 | "source": [] 161 | } 162 | ], 163 | "metadata": { 164 | "kernelspec": { 165 | "display_name": "Python 3 (ipykernel)", 166 | "language": "python", 167 | "name": "python3" 168 | }, 169 | "language_info": { 170 | "codemirror_mode": { 171 | "name": "ipython", 172 | "version": 3 173 | }, 174 | "file_extension": ".py", 175 | "mimetype": "text/x-python", 176 | "name": "python", 177 | "nbconvert_exporter": "python", 178 | "pygments_lexer": "ipython3", 179 | "version": "3.10.13" 180 | }, 181 | "toc": { 182 | "base_numbering": 1, 183 | "nav_menu": {}, 184 | "number_sections": true, 185 | "sideBar": true, 186 | "skip_h1_title": false, 187 | "title_cell": "Table of Contents", 188 | "title_sidebar": "Contents", 189 | "toc_cell": false, 190 | "toc_position": {}, 191 | "toc_section_display": true, 192 | "toc_window_display": false 193 | } 194 | }, 195 | "nbformat": 4, 196 | "nbformat_minor": 5 197 | } 198 | -------------------------------------------------------------------------------- /tutorials/cv/ViT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "0b5abd4b", 6 | "metadata": {}, 7 | "source": [ 8 | "- 不再需要词表维度的 embedding lookup table；\n", 9 | " - 直接通过 learnable 的线性映射；\n", 10 | " - 或者 pretrained ResNet 提取；" 11 | ] 12 | } 13 | ], 14 | "metadata": { 15 | "kernelspec": { 16 | "display_name": "Python 3 (ipykernel)", 17 | "language": "python", 18 | "name": "python3" 19 | }, 20 | "language_info": { 21 | "codemirror_mode": { 22 | "name": "ipython", 23 | "version": 3 24 | }, 25 | "file_extension": ".py", 26 | "mimetype": "text/x-python", 27 | "name": "python", 28 | "nbconvert_exporter": "python", 29 | "pygments_lexer": "ipython3", 30 | "version": "3.10.13" 31 | }, 32 | "toc": { 33 | "base_numbering": 1, 34 | "nav_menu": {}, 35 | "number_sections": true, 36 | "sideBar": true, 37 | "skip_h1_title": false, 38 | "title_cell": "Table of Contents", 39 | "title_sidebar": "Contents", 40 | "toc_cell": false, 41 | "toc_position": {}, 42 | "toc_section_display": true, 43 | "toc_window_display": false 44 | } 45 | }, 46 | "nbformat": 4, 47 | "nbformat_minor": 5 48 | } 49 | -------------------------------------------------------------------------------- /tutorials/cv/cnn_3dcnn_se.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "79cbc3d8", 6 | "metadata": {}, 7 | "source": [ 8 | "- SE module:\n", 9 | " - Squeeze-and-Excitation 模块\n", 10 | " - Squeeze：压缩；\n", 11 | " - Excitation：激励；" 12 | ] 13 | } 14 | ], 15 | "metadata": { 16 | "kernelspec": { 17 | "display_name": "Python 3 (ipykernel)", 18 | "language": "python", 19 | "name": "python3" 20 | }, 21 | "language_info": { 22 | "codemirror_mode": { 23 | "name": "ipython", 24 | "version": 3 25 | }, 26 | "file_extension": ".py", 27 | "mimetype": "text/x-python", 28 | "name": "python", 29 | "nbconvert_exporter": "python", 30 | "pygments_lexer": "ipython3", 31 | "version": "3.10.13" 32 | }, 33 | "toc": { 34 | "base_numbering": 1, 35 | "nav_menu": {}, 36 | "number_sections": true, 37 | "sideBar": true, 38 | "skip_h1_title": false, 39 | "title_cell": "Table of Contents", 40 | "title_sidebar": "Contents", 41 | "toc_cell": false, 42 | "toc_position": {}, 43 | "toc_section_display": true, 44 | "toc_window_display": false 45 | } 46 | }, 47 | "nbformat": 4, 48 | "nbformat_minor": 5 49 | } 50 | -------------------------------------------------------------------------------- /tutorials/cv/timm.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "8d612bef", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [] 10 | } 11 | ], 12 | "metadata": { 13 | "kernelspec": { 14 | "display_name": "Python 3 (ipykernel)", 15 | "language": "python", 16 | "name": "python3" 17 | }, 18 | "language_info": { 19 | "codemirror_mode": { 20 | "name": "ipython", 21 | "version": 3 22 | }, 23 | "file_extension": ".py", 24 | "mimetype": "text/x-python", 25 | "name": "python", 26 | "nbconvert_exporter": "python", 27 | "pygments_lexer": "ipython3", 28 | "version": "3.10.9" 29 | }, 30 | "toc": { 31 | "base_numbering": 1, 32 | "nav_menu": {}, 33 | "number_sections": true, 34 | "sideBar": true, 35 | "skip_h1_title": false, 36 | "title_cell": "Table of Contents", 37 | "title_sidebar": "Contents", 38 | "toc_cell": false, 39 | "toc_position": {}, 40 | "toc_section_display": true, 41 | "toc_window_display": false 42 | } 43 | }, 44 | "nbformat": 4, 45 | "nbformat_minor": 5 46 | } 47 | -------------------------------------------------------------------------------- /tutorials/diffusion/vae.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "c0871741", 6 | "metadata": {}, 7 | "source": [ 8 | "- references\n", 9 | " - https://towardsdatascience.com/intuitively-understanding-variational-autoencoders-1bfe67eb5daf\n", 10 | " - https://mohitjain.me/2018/10/26/variational-autoencoder/" 11 | ] 12 | } 13 | ], 14 | "metadata": { 15 | "kernelspec": { 16 | "display_name": "Python 3 (ipykernel)", 17 | "language": "python", 18 | "name": "python3" 19 | }, 20 | "language_info": { 21 | "codemirror_mode": { 22 | "name": "ipython", 23 | "version": 3 24 | }, 25 | "file_extension": ".py", 26 | "mimetype": "text/x-python", 27 | "name": "python", 28 | "nbconvert_exporter": "python", 29 | "pygments_lexer": "ipython3", 30 | "version": "3.10.9" 31 | }, 32 | "toc": { 33 | "base_numbering": 1, 34 | "nav_menu": {}, 35 | "number_sections": true, 36 | "sideBar": true, 37 | "skip_h1_title": false, 38 | "title_cell": "Table of Contents", 39 | "title_sidebar": "Contents", 40 | "toc_cell": false, 41 | "toc_position": {}, 42 | "toc_section_display": true, 43 | "toc_window_display": false 44 | } 45 | }, 46 | "nbformat": 4, 47 | "nbformat_minor": 5 48 | } 49 | -------------------------------------------------------------------------------- /tutorials/dnn/ae/ksparse-ae.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.optim as optim 4 | from torch.utils.data import DataLoader 5 | from torchvision import datasets, transforms 6 | import matplotlib.pyplot as plt 7 | import numpy as np 8 | from tqdm import tqdm 9 | from torch.nn import functional as F 10 | 11 | # 设置随机种子和设备 12 | torch.manual_seed(42) 13 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 14 | 15 | # 数据加载和预处理 16 | transform = transforms.Compose([ 17 | # 0-1 之间 18 | transforms.ToTensor(), 19 | # 这种运行不会出错的，数据/逻辑bug，很难排查 20 | # transforms.Normalize((0.1307,), (0.3081,)) 21 | ]) 22 | 23 | train_dataset = datasets.MNIST('./data', train=True, download=True, transform=transform) 24 | test_dataset = datasets.MNIST('./data', train=False, transform=transform) 25 | 26 | train_loader = DataLoader(train_dataset, batch_size=1024, shuffle=True) 27 | test_loader = DataLoader(test_dataset, batch_size=128, shuffle=False) 28 | 29 | class KSparseAutoencoder(nn.Module): 30 | def __init__(self, input_dim=784, hidden_dim=256, k=60): 31 | super(KSparseAutoencoder, self).__init__() 32 | self.k = k 33 | 34 | # Encoder 35 | self.encoder = nn.Sequential( 36 | nn.Linear(input_dim, hidden_dim), 37 | # nn.ReLU() 38 | ) 39 | 40 | # Decoder 41 | self.decoder = nn.Sequential( 42 | nn.Linear(hidden_dim, input_dim), 43 | nn.Sigmoid() 44 | ) 45 | 46 | def k_sparse(self, x): 47 | # 实现k-sparse约束 48 | topk, indices = torch.topk(x, self.k, dim=1) 49 | mask = torch.zeros_like(x).scatter_(1, indices, 1) 50 | return x * mask 51 | 52 | def forward(self, x): 53 | # 编码 54 | x = x.view(x.size(0), -1) 55 | encoded = self.encoder(x) 56 | # 应用k-sparse约束 57 | sparse_encoded = self.k_sparse(encoded) 58 | # 解码 59 | decoded = self.decoder(sparse_encoded) 60 | return decoded, sparse_encoded 61 | 62 | def train_model(model, train_loader, num_epochs=20): 63 | criterion = nn.MSELoss() 64 | optimizer = optim.Adam(model.parameters(), lr=1e-3) 65 | 66 | train_losses = [] 67 | 68 | for epoch in range(num_epochs): 69 | model.train() 70 | total_loss = 0 71 | 72 | for data, _ in tqdm(train_loader, desc=f'Epoch {epoch+1}/{num_epochs}'): 73 | data = data.to(device) 74 | optimizer.zero_grad() 75 | 76 | decoded, _ = model(data) 77 | loss = criterion(decoded, data.view(data.size(0), -1)) 78 | 79 | loss.backward() 80 | optimizer.step() 81 | 82 | total_loss += loss.item() 83 | 84 | avg_loss = total_loss / len(train_loader) 85 | train_losses.append(avg_loss) 86 | print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {avg_loss:.6f}') 87 | 88 | return train_losses 89 | 90 | def visualize_results(model, test_loader, num_images=10, k=20): 91 | model.eval() 92 | with torch.no_grad(): 93 | data = next(iter(test_loader))[0][:num_images].to(device) 94 | labels = next(iter(test_loader))[1][:num_images] 95 | decoded, encoded = model(data) 96 | 97 | # 可视化原始图像和重构图像 98 | fig, axes = plt.subplots(2, num_images, figsize=(20, 4)) 99 | 100 | for i in range(num_images): 101 | # 原始图像 102 | err = F.mse_loss(data[i].cpu().squeeze(), decoded[i].cpu().view(28, 28)) 103 | print(labels[i].item(), err.item()) 104 | axes[0, i].imshow(data[i].cpu().squeeze(), cmap='gray') 105 | axes[0, i].axis('off') 106 | axes[0, i].set_title('Original') 107 | 108 | # 重构图像 109 | axes[1, i].imshow(decoded[i].cpu().view(28, 28), cmap='gray') 110 | axes[1, i].axis('off') 111 | 112 | axes[1, i].set_title(f'Reconstructed, {err.item():.2f}') 113 | 114 | plt.tight_layout() 115 | # plt.show() 116 | plt.savefig(f'./figs/ksae-reconstructed-{k}.png') 117 | 118 | # 可视化隐层激活 119 | plt.figure(figsize=(10, 4)) 120 | plt.imshow(encoded.cpu().T, aspect='auto', cmap='viridis') 121 | plt.colorbar() 122 | plt.title('Hidden Layer Activations') 123 | plt.xlabel('Sample') 124 | plt.ylabel('Hidden Unit') 125 | # plt.show() 126 | plt.savefig(f'./figs/ksae-activations-{k}.png') 127 | 128 | def plot_training_loss(losses, k): 129 | plt.figure(figsize=(10, 4)) 130 | plt.plot(losses) 131 | plt.title(f'Training Loss Over Time (k={k})') 132 | plt.xlabel('Epoch') 133 | plt.ylabel('Loss') 134 | plt.grid(True) 135 | # plt.show() 136 | plt.savefig(f'./figs/ksae-loss-{k}.png') 137 | 138 | # 主函数 139 | def main(): 140 | k = 20 141 | model = KSparseAutoencoder(k=k).to(device) 142 | losses = train_model(model, train_loader) 143 | plot_training_loss(losses, k) 144 | visualize_results(model, test_loader, k=k) 145 | 146 | if __name__ == "__main__": 147 | main() 148 | -------------------------------------------------------------------------------- /tutorials/dnn/对比学习/对比学习_contrastive_learning.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "65cb11f2", 6 | "metadata": {}, 7 | "source": [ 8 | "- https://lilianweng.github.io/posts/2021-05-31-contrastive/\n", 9 | "- siamese-triplet\n", 10 | " - https://github.com/adambielski/siamese-triplet/tree/master\n", 11 | "- contrastive-image-text\n", 12 | " - CLIP：https://github.com/huggingface/transformers/tree/main/examples/pytorch/contrastive-image-text" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "id": "e0083f42-32c4-4466-887c-7a4f51729825", 18 | "metadata": {}, 19 | "source": [ 20 | "### constrastive learning & reward modeling" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "id": "c955783d-6ac2-4f13-b967-8e609897ad95", 26 | "metadata": {}, 27 | "source": [ 28 | "> contrastive learning 学习表示（representation），基于表示计算距离，远近；\n", 29 | "> Reward modelling：学习reward function，学习 好坏/高低；\n", 30 | "\n", 31 | "- constrastive learning\n", 32 | " - 一种通过对样本对进行比较来学习有用表征的方法。目标是使相似样本的表示更接近，不相似样本的表示更远离。\n", 33 | " - representation learning\n", 34 | " - datapoint_i => representation_i\n", 35 | " - a measure of distance between data points.\n", 36 | " - embedding learning：学习的是表示；\n", 37 | " - 传统对比学习主要关注学习数据的表示（embedding），使得相似样本的表示更接近，以便在下游任务中使用。\n", 38 | "- learning to rank（learning from preference）\n", 39 | " - 从类别到偏序\n", 40 | "- contrastive learning vs. reward modeling\n", 41 | " - contrastive learning 学到的是一个表示（representation），或者说基于表示还可进一步算距离 $\\|f(x)-f(x_+)\\|$；\n", 42 | " - Reward modelling 你最终学到的是一个 reward/value function：$r_\\theta(x,y_w)$；\n", 43 | "- 数据集（样本对，正例、负例，或者偏序对）的构造，以及损失函数的设计；" 44 | ] 45 | }, 46 | { 47 | "cell_type": "markdown", 48 | "id": "775bcad4-d0c6-4d81-9062-9678f8a46243", 49 | "metadata": {}, 50 | "source": [ 51 | "### supervised vs. un-supervised" 52 | ] 53 | }, 54 | { 55 | "cell_type": "markdown", 56 | "id": "93bcb46e-7db3-4d68-b4e6-cf1161eb077f", 57 | "metadata": {}, 58 | "source": [ 59 | "\n", 60 | "> 监督跟无监督的一个直观的区别就是有没有/用不用 label 信息；\n", 61 | "\n", 62 | "- supervised\n", 63 | " - 比如用真实的 label 做监督\n", 64 | " - dog vs. \n", 65 | "- unsupervised\n", 66 | " - 对 input 做增广（augmentation）" 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "id": "703c9121-a94f-426e-99a4-46bdc7cfae57", 72 | "metadata": {}, 73 | "source": [ 74 | "### CLIP (Contrastive Language–Image Pretraining)" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "id": "9a95e624-1fe6-48a9-a516-53af5986feb6", 80 | "metadata": {}, 81 | "source": [ 82 | "> 相应的图像和文本在特征空间中接近，而不相关的图文对则远离\n", 83 | "\n", 84 | "- 在训练过程中，CLIP 处理一批（batch）图文对。例如，在一个批次中有 N 个图像和 N 个对应的文本描述。模型将每个图像和文本分别编码为向量表示。然后，它计算每个图像向量和每个文本向量之间的相似度，形成一个 N x N 的相似度矩阵。\n", 85 | " - 对于这个矩阵中的每一行，模型希望对应的正确匹配（即第 i 个图像和第 i 个文本）的相似度最大，而其他不匹配的相似度较小。为此，CLIP 使用了一种基于交叉熵的对比损失函数，它考虑了图像到文本和文本到图像的匹配，这意味着模型同时优化图像检索文本和文本检索图像的任务。\n", 86 | " - 这种损失函数的设计使得模型学会在高维特征空间中，将相关的图像和文本映射到相近的位置，从而实现跨模态的检索和匹配。\n", 87 | "- 设在一个批次中，有 N 个图像和 N 个对应的文本描述，形成 N 个匹配的图文对。\n", 88 | " - 令 $I_i$ 表示第 $i$ 个图像，$T_i$ 表示第 $i$ 个文本\n", 89 | " - 定义图像编码器 $f$ 和文本编码器 $g$，将图像和文本映射到一个共同的特征空间。（表征学习 representation learning）\n", 90 | " - 获取归一化的图像和文本特征向量：\n", 91 | "\n", 92 | " $$\n", 93 | " \\mathbf v_i=\\frac{f(I_i)}{\\|f(I_t)\\|}, \\mathbf u_i=\\frac{g(T_i)}{\\|g(T_i)\\|}\n", 94 | " $$\n", 95 | " - 相似度矩阵 $S$ 的元素定义为（其中，$\\tau$ 是可学习的温度（temperature）参数，用于控制分布的锐化程度。）\n", 96 | " - 注意 $v_i$ 和 $u_i$ 维度未必一致，也许还需要一层映射，映射到一个共同的表征空间；\n", 97 | " $$\n", 98 | " s_{ij}=\\frac{v_i^Tu_i}{\\tau}\n", 99 | " $$\n", 100 | " - 对于图像到文本的方向，定义预测的概率分布：\n", 101 | " $$\n", 102 | " p_{ij}=\\frac{\\exp(s_{ij})}{\\sum_{k=1}^N\\exp(s_{ik})}\n", 103 | " $$\n", 104 | " - 相应的损失函数为：$\\mathcal L_{\\text{img2txt}}=-\\frac1N\\sum_{i=1}^N\\log p_{i,i}$\n", 105 | " - 对于文本到图像的方向，定义预测的概率分布为：\n", 106 | " $$\n", 107 | " q_{ij}=\\frac{\\exp(s_{ij})}{\\sum_{k=1}^N\\exp(s_{k,j})}\n", 108 | " $$\n", 109 | " - 相应的损失函数为：$\\mathcal L_{\\text{txt2img}}=-\\frac1N\\sum_{i=1}^N\\log q_{i,i}$\n", 110 | " - 总的损失\n", 111 | " $$\n", 112 | " \\mathcal L=\\frac12({L_{\\text{img2txt}}+L_{\\text{txt2img}}})\n", 113 | " $$ " 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "id": "908edeb4-8070-4851-a935-affc9f6682ec", 119 | "metadata": {}, 120 | "source": [ 121 | "### reward modeling" 122 | ] 123 | }, 124 | { 125 | "cell_type": "markdown", 126 | "id": "962b4a5d-9530-439f-9c0d-8ed5cc2c5aea", 127 | "metadata": {}, 128 | "source": [ 129 | "- 而 instruct GPT 开创的 reward modeling 追求的是学习 reward function，即给定一个 input, response pair，输出一个好坏的分数（scalar output）\n", 130 | " $$\n", 131 | " \\text{loss}(\\theta) = -\\frac{1}{\\left(\\frac{K}{2}\\right)} \\mathbb{E}_{(x, y_w, y_l) \\sim \\mathcal{D}} \\left[ \\log \\left( \\sigma \\left( r_\\theta(x, y_w) - r_\\theta(x, y_l) \\right) \\right) \\right]\n", 132 | " $$ \n", 133 | "- 对比学习，强调的是 $f(I_i),g(T_i)$ 将原始的输入，映射到 hidden space，得到的 feature，即为表征学习（representation learning）；" 134 | ] 135 | } 136 | ], 137 | "metadata": { 138 | "kernelspec": { 139 | "display_name": "Python 3 (ipykernel)", 140 | "language": "python", 141 | "name": "python3" 142 | }, 143 | "language_info": { 144 | "codemirror_mode": { 145 | "name": "ipython", 146 | "version": 3 147 | }, 148 | "file_extension": ".py", 149 | "mimetype": "text/x-python", 150 | "name": "python", 151 | "nbconvert_exporter": "python", 152 | "pygments_lexer": "ipython3", 153 | "version": "3.10.13" 154 | }, 155 | "toc": { 156 | "base_numbering": 1, 157 | "nav_menu": {}, 158 | "number_sections": true, 159 | "sideBar": true, 160 | "skip_h1_title": false, 161 | "title_cell": "Table of Contents", 162 | "title_sidebar": "Contents", 163 | "toc_cell": false, 164 | "toc_position": {}, 165 | "toc_section_display": true, 166 | "toc_window_display": false 167 | } 168 | }, 169 | "nbformat": 4, 170 | "nbformat_minor": 5 171 | } 172 | -------------------------------------------------------------------------------- /tutorials/gnn/GTN_Transformer.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "b8f5b88b-70fe-4c9a-87c9-c89932db593e", 6 | "metadata": {}, 7 | "source": [ 8 | "- cs224w\n", 9 | " - https://web.stanford.edu/class/cs224w/slides/14-graph-transformer.pdf\n", 10 | "\n", 11 | "| **Graph** | **Transformers** |\n", 12 | "|-----------------------------|---------------------------|\n", 13 | "| (1) Node features? | (1) Tokenizing |\n", 14 | "| (2) Adjacency information? | (2) Positional encoding |\n", 15 | "| (3) Edge features? | (3) Self-attention |" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "id": "632aad59-dfa3-49b7-8809-26ff3b61948c", 21 | "metadata": {}, 22 | "source": [ 23 | "### 拉普拉斯矩阵" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "id": "27dc50f6-33f1-4e70-9fcf-44a556a45b94", 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [] 33 | } 34 | ], 35 | "metadata": { 36 | "kernelspec": { 37 | "display_name": "Python 3 (ipykernel)", 38 | "language": "python", 39 | "name": "python3" 40 | }, 41 | "language_info": { 42 | "codemirror_mode": { 43 | "name": "ipython", 44 | "version": 3 45 | }, 46 | "file_extension": ".py", 47 | "mimetype": "text/x-python", 48 | "name": "python", 49 | "nbconvert_exporter": "python", 50 | "pygments_lexer": "ipython3", 51 | "version": "3.10.13" 52 | } 53 | }, 54 | "nbformat": 4, 55 | "nbformat_minor": 5 56 | } 57 | -------------------------------------------------------------------------------- /tutorials/gnn/PyG.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 5, 6 | "id": "9cfcbc26-2f73-4893-b4ad-70153466678c", 7 | "metadata": { 8 | "execution": { 9 | "iopub.execute_input": "2024-10-14T15:43:53.563630Z", 10 | "iopub.status.busy": "2024-10-14T15:43:53.562997Z", 11 | "iopub.status.idle": "2024-10-14T15:43:53.572555Z", 12 | "shell.execute_reply": "2024-10-14T15:43:53.570259Z", 13 | "shell.execute_reply.started": "2024-10-14T15:43:53.563584Z" 14 | } 15 | }, 16 | "outputs": [], 17 | "source": [ 18 | "from IPython.display import Image" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 6, 24 | "id": "2fb2db2b-effd-46fe-8c53-22f7cca1795c", 25 | "metadata": { 26 | "execution": { 27 | "iopub.execute_input": "2024-10-14T15:43:54.968278Z", 28 | "iopub.status.busy": "2024-10-14T15:43:54.967636Z", 29 | "iopub.status.idle": "2024-10-14T15:43:54.980845Z", 30 | "shell.execute_reply": "2024-10-14T15:43:54.978750Z", 31 | "shell.execute_reply.started": "2024-10-14T15:43:54.968222Z" 32 | } 33 | }, 34 | "outputs": [ 35 | { 36 | "data": { 37 | "text/html": [ 38 | "" 39 | ], 40 | "text/plain": [ 41 | "" 42 | ] 43 | }, 44 | "execution_count": 6, 45 | "metadata": {}, 46 | "output_type": "execute_result" 47 | } 48 | ], 49 | "source": [ 50 | "Image(url='./imgs/gnn_layers.png', width=500)" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 7, 56 | "id": "8a01bc67-10d2-42c7-ab15-5566f383cd1b", 57 | "metadata": { 58 | "execution": { 59 | "iopub.execute_input": "2024-10-14T15:45:04.557978Z", 60 | "iopub.status.busy": "2024-10-14T15:45:04.557351Z", 61 | "iopub.status.idle": "2024-10-14T15:45:08.682441Z", 62 | "shell.execute_reply": "2024-10-14T15:45:08.680521Z", 63 | "shell.execute_reply.started": "2024-10-14T15:45:04.557932Z" 64 | } 65 | }, 66 | "outputs": [], 67 | "source": [ 68 | "import torch_geometric as pyg" 69 | ] 70 | }, 71 | { 72 | "cell_type": "markdown", 73 | "id": "b7bea46c-52cc-4f6a-87f8-39a6de5de259", 74 | "metadata": {}, 75 | "source": [ 76 | "- 2708 nodes, 10556 edges\n", 77 | "- Each node describes a document.\n", 78 | "- Each node is described by a FEATURE TENSOR.\n", 79 | " - x_1, ..., x_1433\n", 80 | "- Two documents are connected (with an edge) if there exists a citation link in CORA between them." 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "id": "6bdca450-8147-4843-b17c-6634da63be66", 86 | "metadata": {}, 87 | "source": [ 88 | "### InMemoryDataset" 89 | ] 90 | }, 91 | { 92 | "cell_type": "markdown", 93 | "id": "fb731a18-4ddb-4149-87dd-ddd02ba3d028", 94 | "metadata": {}, 95 | "source": [ 96 | "- Dataset base class for creating graph datasets which easily fit into CPU memory." 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 3, 102 | "id": "dae55b20-1c01-4143-a4bb-28f7075d1427", 103 | "metadata": { 104 | "execution": { 105 | "iopub.execute_input": "2024-10-19T03:03:50.870103Z", 106 | "iopub.status.busy": "2024-10-19T03:03:50.869429Z", 107 | "iopub.status.idle": "2024-10-19T03:03:50.878801Z", 108 | "shell.execute_reply": "2024-10-19T03:03:50.876627Z", 109 | "shell.execute_reply.started": "2024-10-19T03:03:50.870053Z" 110 | } 111 | }, 112 | "outputs": [], 113 | "source": [ 114 | "# !pip install rdkit" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 7, 120 | "id": "621ffa4e-9029-4131-b37f-2e09d164b9d1", 121 | "metadata": { 122 | "execution": { 123 | "iopub.execute_input": "2024-10-19T03:05:12.239026Z", 124 | "iopub.status.busy": "2024-10-19T03:05:12.238693Z", 125 | "iopub.status.idle": "2024-10-19T03:05:12.246822Z", 126 | "shell.execute_reply": "2024-10-19T03:05:12.244862Z", 127 | "shell.execute_reply.started": "2024-10-19T03:05:12.239004Z" 128 | } 129 | }, 130 | "outputs": [], 131 | "source": [ 132 | "from torch_geometric.datasets import QM9" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": 10, 138 | "id": "d6bf3868-d4ca-45b8-9679-347de10160f9", 139 | "metadata": { 140 | "execution": { 141 | "iopub.execute_input": "2024-10-19T03:08:34.780446Z", 142 | "iopub.status.busy": "2024-10-19T03:08:34.779792Z", 143 | "iopub.status.idle": "2024-10-19T03:10:15.343084Z", 144 | "shell.execute_reply": "2024-10-19T03:10:15.342250Z", 145 | "shell.execute_reply.started": "2024-10-19T03:08:34.780397Z" 146 | } 147 | }, 148 | "outputs": [ 149 | { 150 | "name": "stderr", 151 | "output_type": "stream", 152 | "text": [ 153 | "Processing...\n", 154 | "100%|██████████| 133885/133885 [01:34<00:00, 1413.15it/s]\n", 155 | "Done!\n" 156 | ] 157 | } 158 | ], 159 | "source": [ 160 | "dp = QM9(root='./data/QM9/').to_datapipe()" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 21, 166 | "id": "3bc0d371-c824-40c3-9621-dabc6996fb0b", 167 | "metadata": { 168 | "execution": { 169 | "iopub.execute_input": "2024-10-19T03:12:37.817601Z", 170 | "iopub.status.busy": "2024-10-19T03:12:37.817102Z", 171 | "iopub.status.idle": "2024-10-19T03:12:37.828669Z", 172 | "shell.execute_reply": "2024-10-19T03:12:37.826502Z", 173 | "shell.execute_reply.started": "2024-10-19T03:12:37.817566Z" 174 | } 175 | }, 176 | "outputs": [ 177 | { 178 | "data": { 179 | "text/plain": [ 180 | "Batcher" 181 | ] 182 | }, 183 | "execution_count": 21, 184 | "metadata": {}, 185 | "output_type": "execute_result" 186 | } 187 | ], 188 | "source": [ 189 | "dp = dp.batch_graphs(batch_size=16, drop_last=True)\n", 190 | "dp" 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": 22, 196 | "id": "6825ac76-675f-4a77-8ead-3c03243b5f15", 197 | "metadata": { 198 | "execution": { 199 | "iopub.execute_input": "2024-10-19T03:12:39.340635Z", 200 | "iopub.status.busy": "2024-10-19T03:12:39.340037Z", 201 | "iopub.status.idle": "2024-10-19T03:12:40.342465Z", 202 | "shell.execute_reply": "2024-10-19T03:12:40.340739Z", 203 | "shell.execute_reply.started": "2024-10-19T03:12:39.340590Z" 204 | } 205 | }, 206 | "outputs": [ 207 | { 208 | "data": { 209 | "text/plain": [ 210 | "DataBatch(x=[12622, 11], edge_index=[2, 24802], edge_attr=[24802, 4], y=[1024, 19], pos=[12622, 3], z=[12622], smiles=[16], name=[16], idx=[1024], batch=[12622])" 211 | ] 212 | }, 213 | "execution_count": 22, 214 | "metadata": {}, 215 | "output_type": "execute_result" 216 | } 217 | ], 218 | "source": [ 219 | "batch = next(iter(dp))\n", 220 | "batch" 221 | ] 222 | } 223 | ], 224 | "metadata": { 225 | "kernelspec": { 226 | "display_name": "Python 3 (ipykernel)", 227 | "language": "python", 228 | "name": "python3" 229 | }, 230 | "language_info": { 231 | "codemirror_mode": { 232 | "name": "ipython", 233 | "version": 3 234 | }, 235 | "file_extension": ".py", 236 | "mimetype": "text/x-python", 237 | "name": "python", 238 | "nbconvert_exporter": "python", 239 | "pygments_lexer": "ipython3", 240 | "version": "3.10.13" 241 | } 242 | }, 243 | "nbformat": 4, 244 | "nbformat_minor": 5 245 | } 246 | -------------------------------------------------------------------------------- /tutorials/gnn/imgs/gnn_layers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/gnn/imgs/gnn_layers.png -------------------------------------------------------------------------------- /tutorials/gnn/imgs/two-layers-gcn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/gnn/imgs/two-layers-gcn.png -------------------------------------------------------------------------------- /tutorials/mcp/about_mcp.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "56b26f85-92db-44be-8d73-d397f41950c5", 6 | "metadata": {}, 7 | "source": [ 8 | "### uv 与 npx" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "e2c3f25a-8b92-4f4c-9a33-f5504c9ab36e", 14 | "metadata": {}, 15 | "source": [ 16 | "- uv 是 python 的运行环境\n", 17 | " - `uvx` == `uv run tool`\n", 18 | "- npx: node js 的运行环境" 19 | ] 20 | } 21 | ], 22 | "metadata": { 23 | "kernelspec": { 24 | "display_name": "Python 3 (ipykernel)", 25 | "language": "python", 26 | "name": "python3" 27 | }, 28 | "language_info": { 29 | "codemirror_mode": { 30 | "name": "ipython", 31 | "version": 3 32 | }, 33 | "file_extension": ".py", 34 | "mimetype": "text/x-python", 35 | "name": "python", 36 | "nbconvert_exporter": "python", 37 | "pygments_lexer": "ipython3", 38 | "version": "3.10.13" 39 | } 40 | }, 41 | "nbformat": 4, 42 | "nbformat_minor": 5 43 | } 44 | -------------------------------------------------------------------------------- /tutorials/mcp/实用 MCP.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "2c94417a-d16b-42ec-b08c-8ca587d6ee73", 6 | "metadata": {}, 7 | "source": [ 8 | "## 爬虫类" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "754e82fe-ade2-458c-be72-521ef9bf8643", 14 | "metadata": {}, 15 | "source": [ 16 | "- fetch\n", 17 | " - 抓网页\n", 18 | "- firecrawl" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "id": "f7b98afa-241c-4400-9f40-ef83693517b2", 24 | "metadata": {}, 25 | "source": [ 26 | "### firecrawl" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "id": "37ca105b-fd80-4442-9de9-8219a511d15e", 32 | "metadata": {}, 33 | "source": [ 34 | "## Location" 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "id": "74e619be-92ae-4779-aa19-67fda12ed2f3", 40 | "metadata": {}, 41 | "source": [ 42 | "- 高德 amap server\n", 43 | " - 旅行规划" 44 | ] 45 | } 46 | ], 47 | "metadata": { 48 | "kernelspec": { 49 | "display_name": "Python 3 (ipykernel)", 50 | "language": "python", 51 | "name": "python3" 52 | }, 53 | "language_info": { 54 | "codemirror_mode": { 55 | "name": "ipython", 56 | "version": 3 57 | }, 58 | "file_extension": ".py", 59 | "mimetype": "text/x-python", 60 | "name": "python", 61 | "nbconvert_exporter": "python", 62 | "pygments_lexer": "ipython3", 63 | "version": "3.10.13" 64 | } 65 | }, 66 | "nbformat": 4, 67 | "nbformat_minor": 5 68 | } 69 | -------------------------------------------------------------------------------- /tutorials/multi_modals.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "da1d765a", 6 | "metadata": {}, 7 | "source": [ 8 | "## LIP" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "0aa004b1", 14 | "metadata": {}, 15 | "source": [ 16 | "LIP：Language Image pretraining\n", 17 | "- Clip：constrative，learning from image-text pairs\n", 18 | "- Glip：grounded\n", 19 | "- Blip：bootstrapping" 20 | ] 21 | } 22 | ], 23 | "metadata": { 24 | "kernelspec": { 25 | "display_name": "Python 3 (ipykernel)", 26 | "language": "python", 27 | "name": "python3" 28 | }, 29 | "language_info": { 30 | "codemirror_mode": { 31 | "name": "ipython", 32 | "version": 3 33 | }, 34 | "file_extension": ".py", 35 | "mimetype": "text/x-python", 36 | "name": "python", 37 | "nbconvert_exporter": "python", 38 | "pygments_lexer": "ipython3", 39 | "version": "3.10.9" 40 | }, 41 | "toc": { 42 | "base_numbering": 1, 43 | "nav_menu": {}, 44 | "number_sections": true, 45 | "sideBar": true, 46 | "skip_h1_title": false, 47 | "title_cell": "Table of Contents", 48 | "title_sidebar": "Contents", 49 | "toc_cell": false, 50 | "toc_position": {}, 51 | "toc_section_display": true, 52 | "toc_window_display": false 53 | } 54 | }, 55 | "nbformat": 4, 56 | "nbformat_minor": 5 57 | } 58 | -------------------------------------------------------------------------------- /tutorials/nn_basics/bce_loss_cross_entropy.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "bac3a6cd", 6 | "metadata": {}, 7 | "source": [ 8 | "- 原理上都适用于分类问题\n", 9 | " - bce：二分类问题 （0-1）\n", 10 | " - 概率的过程通过 sigmoid，\n", 11 | " - 1d(scalar) 的 output `logits` => 2d `[1-sigmoid(z), sigmoid(z)]`\n", 12 | " - cross_entropy：多分类问题（`0, 1, 2, ... , C-1`）\n", 13 | " - 概率化的过程通过 softmax，\n", 14 | " - log softmax" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 12, 20 | "id": "39602fc9", 21 | "metadata": { 22 | "ExecuteTime": { 23 | "end_time": "2023-11-26T10:25:23.813874Z", 24 | "start_time": "2023-11-26T10:25:23.806626Z" 25 | } 26 | }, 27 | "outputs": [], 28 | "source": [ 29 | "import torch\n", 30 | "import torch.nn.functional as F\n", 31 | "from torch import nn" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 40, 37 | "id": "8a0d30b5", 38 | "metadata": { 39 | "ExecuteTime": { 40 | "end_time": "2023-11-26T10:46:14.026352Z", 41 | "start_time": "2023-11-26T10:46:14.014195Z" 42 | } 43 | }, 44 | "outputs": [ 45 | { 46 | "data": { 47 | "text/plain": [ 48 | "tensor([ 0.5031, -0.1608, 1.1623], requires_grad=True)" 49 | ] 50 | }, 51 | "execution_count": 40, 52 | "metadata": {}, 53 | "output_type": "execute_result" 54 | } 55 | ], 56 | "source": [ 57 | "# prediction\n", 58 | "# number samples: 3\n", 59 | "input = torch.randn(3, requires_grad=True)\n", 60 | "input" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": 63, 66 | "id": "2d5f5258", 67 | "metadata": { 68 | "ExecuteTime": { 69 | "end_time": "2023-11-26T10:54:17.246800Z", 70 | "start_time": "2023-11-26T10:54:17.239182Z" 71 | } 72 | }, 73 | "outputs": [], 74 | "source": [ 75 | "# true\n", 76 | "target = torch.empty(3, dtype=torch.long).random_(2)" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 64, 82 | "id": "edad24b1", 83 | "metadata": { 84 | "ExecuteTime": { 85 | "end_time": "2023-11-26T10:54:18.515522Z", 86 | "start_time": "2023-11-26T10:54:18.504170Z" 87 | } 88 | }, 89 | "outputs": [ 90 | { 91 | "data": { 92 | "text/plain": [ 93 | "tensor([0, 1, 1])" 94 | ] 95 | }, 96 | "execution_count": 64, 97 | "metadata": {}, 98 | "output_type": "execute_result" 99 | } 100 | ], 101 | "source": [ 102 | "target" 103 | ] 104 | }, 105 | { 106 | "cell_type": "markdown", 107 | "id": "96b9924e", 108 | "metadata": {}, 109 | "source": [ 110 | "## F.binary_cross_entropy" 111 | ] 112 | }, 113 | { 114 | "cell_type": "markdown", 115 | "id": "05a5c9a1", 116 | "metadata": {}, 117 | "source": [ 118 | "- F.binary_cross_entropy 的 target 可以是浮点数，未必非得是 0-1 (integer)\n", 119 | " - VAE 在图像重构时的 loss，通过 bce loss 定义和计算的；" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": 65, 125 | "id": "26b93e19", 126 | "metadata": { 127 | "ExecuteTime": { 128 | "end_time": "2023-11-26T10:54:20.291711Z", 129 | "start_time": "2023-11-26T10:54:20.283599Z" 130 | } 131 | }, 132 | "outputs": [], 133 | "source": [ 134 | "loss = F.binary_cross_entropy(torch.sigmoid(input), target.to(torch.float))" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": 66, 140 | "id": "eb777c7e", 141 | "metadata": { 142 | "ExecuteTime": { 143 | "end_time": "2023-11-26T10:54:21.719561Z", 144 | "start_time": "2023-11-26T10:54:21.708177Z" 145 | } 146 | }, 147 | "outputs": [ 148 | { 149 | "data": { 150 | "text/plain": [ 151 | "tensor(0.6750, grad_fn=)" 152 | ] 153 | }, 154 | "execution_count": 66, 155 | "metadata": {}, 156 | "output_type": "execute_result" 157 | } 158 | ], 159 | "source": [ 160 | "loss" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 67, 166 | "id": "ad3b2515", 167 | "metadata": { 168 | "ExecuteTime": { 169 | "end_time": "2023-11-26T10:54:22.647777Z", 170 | "start_time": "2023-11-26T10:54:22.633838Z" 171 | }, 172 | "scrolled": true 173 | }, 174 | "outputs": [ 175 | { 176 | "data": { 177 | "text/plain": [ 178 | "tensor(0.6750, grad_fn=)" 179 | ] 180 | }, 181 | "execution_count": 67, 182 | "metadata": {}, 183 | "output_type": "execute_result" 184 | } 185 | ], 186 | "source": [ 187 | "-torch.mean(target * torch.log(F.sigmoid(input)) + (1-target) * torch.log(1-F.sigmoid(input)))" 188 | ] 189 | }, 190 | { 191 | "cell_type": "markdown", 192 | "id": "8bc4a899", 193 | "metadata": {}, 194 | "source": [ 195 | "## nn.BCELoss" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": 68, 201 | "id": "1b90daa6", 202 | "metadata": { 203 | "ExecuteTime": { 204 | "end_time": "2023-11-26T10:54:24.222542Z", 205 | "start_time": "2023-11-26T10:54:24.210501Z" 206 | } 207 | }, 208 | "outputs": [ 209 | { 210 | "data": { 211 | "text/plain": [ 212 | "tensor(0.6750, grad_fn=)" 213 | ] 214 | }, 215 | "execution_count": 68, 216 | "metadata": {}, 217 | "output_type": "execute_result" 218 | } 219 | ], 220 | "source": [ 221 | "m = nn.BCELoss()\n", 222 | "m(torch.sigmoid(input), target.to(torch.float))" 223 | ] 224 | }, 225 | { 226 | "cell_type": "markdown", 227 | "id": "93fb5059", 228 | "metadata": {}, 229 | "source": [ 230 | "## F.cross_entropy" 231 | ] 232 | }, 233 | { 234 | "cell_type": "markdown", 235 | "id": "f7a410ac", 236 | "metadata": {}, 237 | "source": [ 238 | "- 1d => 2d\n", 239 | "- ignore_index=-100," 240 | ] 241 | }, 242 | { 243 | "cell_type": "code", 244 | "execution_count": 69, 245 | "id": "cfdf9940", 246 | "metadata": { 247 | "ExecuteTime": { 248 | "end_time": "2023-11-26T10:54:25.816943Z", 249 | "start_time": "2023-11-26T10:54:25.804104Z" 250 | } 251 | }, 252 | "outputs": [ 253 | { 254 | "data": { 255 | "text/plain": [ 256 | "tensor([[-0.9760, -0.4729],\n", 257 | " [-0.6160, -0.7768],\n", 258 | " [-1.4345, -0.2721]], grad_fn=)" 259 | ] 260 | }, 261 | "execution_count": 69, 262 | "metadata": {}, 263 | "output_type": "execute_result" 264 | } 265 | ], 266 | "source": [ 267 | "input_2 = torch.stack([1-torch.sigmoid(input), torch.sigmoid(input)], dim=1)\n", 268 | "# log softmax\n", 269 | "input_2 = torch.log(input_2)\n", 270 | "input_2" 271 | ] 272 | }, 273 | { 274 | "cell_type": "code", 275 | "execution_count": 70, 276 | "id": "2be7d192", 277 | "metadata": { 278 | "ExecuteTime": { 279 | "end_time": "2023-11-26T10:54:27.122765Z", 280 | "start_time": "2023-11-26T10:54:27.112910Z" 281 | } 282 | }, 283 | "outputs": [ 284 | { 285 | "data": { 286 | "text/plain": [ 287 | "tensor([0, 1, 1])" 288 | ] 289 | }, 290 | "execution_count": 70, 291 | "metadata": {}, 292 | "output_type": "execute_result" 293 | } 294 | ], 295 | "source": [ 296 | "target" 297 | ] 298 | }, 299 | { 300 | "cell_type": "code", 301 | "execution_count": 71, 302 | "id": "6206b200", 303 | "metadata": { 304 | "ExecuteTime": { 305 | "end_time": "2023-11-26T10:54:28.236956Z", 306 | "start_time": "2023-11-26T10:54:28.225731Z" 307 | } 308 | }, 309 | "outputs": [ 310 | { 311 | "data": { 312 | "text/plain": [ 313 | "tensor(0.6750, grad_fn=)" 314 | ] 315 | }, 316 | "execution_count": 71, 317 | "metadata": {}, 318 | "output_type": "execute_result" 319 | } 320 | ], 321 | "source": [ 322 | "F.nll_loss(input_2, target)" 323 | ] 324 | }, 325 | { 326 | "cell_type": "code", 327 | "execution_count": 75, 328 | "id": "c127e0f1", 329 | "metadata": { 330 | "ExecuteTime": { 331 | "end_time": "2023-11-26T10:58:39.352335Z", 332 | "start_time": "2023-11-26T10:58:39.342689Z" 333 | } 334 | }, 335 | "outputs": [ 336 | { 337 | "data": { 338 | "text/plain": [ 339 | "0.6749666666666667" 340 | ] 341 | }, 342 | "execution_count": 75, 343 | "metadata": {}, 344 | "output_type": "execute_result" 345 | } 346 | ], 347 | "source": [ 348 | "-((-0.9760 + (-0.7768)+(-0.2721))/3)" 349 | ] 350 | } 351 | ], 352 | "metadata": { 353 | "kernelspec": { 354 | "display_name": "Python 3 (ipykernel)", 355 | "language": "python", 356 | "name": "python3" 357 | }, 358 | "language_info": { 359 | "codemirror_mode": { 360 | "name": "ipython", 361 | "version": 3 362 | }, 363 | "file_extension": ".py", 364 | "mimetype": "text/x-python", 365 | "name": "python", 366 | "nbconvert_exporter": "python", 367 | "pygments_lexer": "ipython3", 368 | "version": "3.10.13" 369 | }, 370 | "toc": { 371 | "base_numbering": 1, 372 | "nav_menu": {}, 373 | "number_sections": true, 374 | "sideBar": true, 375 | "skip_h1_title": false, 376 | "title_cell": "Table of Contents", 377 | "title_sidebar": "Contents", 378 | "toc_cell": false, 379 | "toc_position": {}, 380 | "toc_section_display": true, 381 | "toc_window_display": false 382 | } 383 | }, 384 | "nbformat": 4, 385 | "nbformat_minor": 5 386 | } 387 | -------------------------------------------------------------------------------- /tutorials/nn_basics/einsum_einops/einops.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "id": "9cc62bc7-5408-4dfe-a9e5-cc773a1595ed", 7 | "metadata": { 8 | "execution": { 9 | "iopub.execute_input": "2024-11-23T10:06:06.583433Z", 10 | "iopub.status.busy": "2024-11-23T10:06:06.583022Z", 11 | "iopub.status.idle": "2024-11-23T10:06:06.590218Z", 12 | "shell.execute_reply": "2024-11-23T10:06:06.588478Z", 13 | "shell.execute_reply.started": "2024-11-23T10:06:06.583410Z" 14 | } 15 | }, 16 | "outputs": [], 17 | "source": [ 18 | "import torch\n", 19 | "import einops\n", 20 | "from einops import repeat" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "id": "1b9275d2-1c60-4914-8701-df5497814309", 26 | "metadata": {}, 27 | "source": [ 28 | "#### repeat" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "id": "22bfd1db-1cb7-4a41-be90-15e7986d49da", 34 | "metadata": {}, 35 | "source": [ 36 | "- 模式 \"i -> (i j)\" 的含义：\n", 37 | " - 输入维度：i，即原始的一维张量。\n", 38 | " - 输出维度：(i j)，表示将每个元素 i 重复 j 次后展平成一个新的维度。\n", 39 | "- 参数 j=p 指定每个元素重复的次数为 p。" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 3, 45 | "id": "59b0e5ad-dffa-4831-bbfa-8a14c971c899", 46 | "metadata": { 47 | "execution": { 48 | "iopub.execute_input": "2024-11-23T10:06:39.266087Z", 49 | "iopub.status.busy": "2024-11-23T10:06:39.265444Z", 50 | "iopub.status.idle": "2024-11-23T10:06:40.299609Z", 51 | "shell.execute_reply": "2024-11-23T10:06:40.297826Z", 52 | "shell.execute_reply.started": "2024-11-23T10:06:39.266042Z" 53 | } 54 | }, 55 | "outputs": [ 56 | { 57 | "data": { 58 | "text/plain": [ 59 | "tensor([1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5,\n", 60 | " 5])" 61 | ] 62 | }, 63 | "execution_count": 3, 64 | "metadata": {}, 65 | "output_type": "execute_result" 66 | } 67 | ], 68 | "source": [ 69 | "repeat(torch.tensor([1, 2, 3, 4, 5]), 'i -> (i j)', j=5)" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 5, 75 | "id": "c16cdedd-eefd-47ba-a9dd-9be4fc2e09be", 76 | "metadata": { 77 | "execution": { 78 | "iopub.execute_input": "2024-11-23T10:09:08.088125Z", 79 | "iopub.status.busy": "2024-11-23T10:09:08.087482Z", 80 | "iopub.status.idle": "2024-11-23T10:09:08.102022Z", 81 | "shell.execute_reply": "2024-11-23T10:09:08.099919Z", 82 | "shell.execute_reply.started": "2024-11-23T10:09:08.088080Z" 83 | } 84 | }, 85 | "outputs": [ 86 | { 87 | "data": { 88 | "text/plain": [ 89 | "(tensor([10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10]), torch.Size([12]))" 90 | ] 91 | }, 92 | "execution_count": 5, 93 | "metadata": {}, 94 | "output_type": "execute_result" 95 | } 96 | ], 97 | "source": [ 98 | "p = repeat(torch.tensor(10), '-> (i j)', i=3, j=4)\n", 99 | "p, p.shape" 100 | ] 101 | } 102 | ], 103 | "metadata": { 104 | "kernelspec": { 105 | "display_name": "Python 3 (ipykernel)", 106 | "language": "python", 107 | "name": "python3" 108 | }, 109 | "language_info": { 110 | "codemirror_mode": { 111 | "name": "ipython", 112 | "version": 3 113 | }, 114 | "file_extension": ".py", 115 | "mimetype": "text/x-python", 116 | "name": "python", 117 | "nbconvert_exporter": "python", 118 | "pygments_lexer": "ipython3", 119 | "version": "3.10.13" 120 | } 121 | }, 122 | "nbformat": 4, 123 | "nbformat_minor": 5 124 | } 125 | -------------------------------------------------------------------------------- /tutorials/nn_basics/einsum_einops/shape.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "id": "c361ce2b-f769-4b6b-839a-6dd9888d5b16", 7 | "metadata": { 8 | "execution": { 9 | "iopub.execute_input": "2024-12-02T15:38:49.174204Z", 10 | "iopub.status.busy": "2024-12-02T15:38:49.173796Z", 11 | "iopub.status.idle": "2024-12-02T15:38:49.182531Z", 12 | "shell.execute_reply": "2024-12-02T15:38:49.180432Z", 13 | "shell.execute_reply.started": "2024-12-02T15:38:49.174180Z" 14 | } 15 | }, 16 | "outputs": [], 17 | "source": [ 18 | "import torch\n", 19 | "import torch.nn.functional as F" 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "id": "2788bc19-8899-43ea-8ca0-c0aeb4587983", 25 | "metadata": {}, 26 | "source": [ 27 | "### broadcast" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "id": "10baa1fb-7ce2-43a3-bbe6-8bd55e92010e", 33 | "metadata": {}, 34 | "source": [ 35 | "```\n", 36 | "cos_sim = F.cosine_similarity(feats[:,None,:], feats[None,:,:], dim=-1)\n", 37 | "```\n", 38 | "\n", 39 | "假设 feats 的原始形状是 (N, D)，其中：\n", 40 | "- N 是批次大小（在这里是 2 batch_size）\n", 41 | "- D 是特征维度\n", 42 | "\n", 43 | "通过添加 None（等同于 unsqueeze）操作：\n", 44 | "- feats[:,None,:] 将形状从 (N, D) 变为 (N, 1, D)\n", 45 | "- feats[None,:,:] 将形状从 (N, D) 变为 (1, N, D)\n", 46 | "\n", 47 | "广播\n", 48 | "\n", 49 | "当这两个张量进行运算时，PyTorch 的广播机制会将它们扩展为相同的形状：\n", 50 | "- (N, 1, D) → (N, N, D)\n", 51 | "- (1, N, D) → (N, N, D)" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 8, 57 | "id": "98fbf721-ad5c-4f5d-b0b4-3b97ca9065de", 58 | "metadata": { 59 | "execution": { 60 | "iopub.execute_input": "2024-12-02T15:42:08.403174Z", 61 | "iopub.status.busy": "2024-12-02T15:42:08.402753Z", 62 | "iopub.status.idle": "2024-12-02T15:42:08.411563Z", 63 | "shell.execute_reply": "2024-12-02T15:42:08.409423Z", 64 | "shell.execute_reply.started": "2024-12-02T15:42:08.403145Z" 65 | } 66 | }, 67 | "outputs": [], 68 | "source": [ 69 | "# 假设 feats 的形状是 (3, 2)，即 3 个样本，每个特征维度为 2\n", 70 | "X = torch.tensor([\n", 71 | " [1, 2], # 样本1\n", 72 | " [3, 4], # 样本2\n", 73 | " [5, 6] # 样本3\n", 74 | "], dtype=torch.float32)\n" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 9, 80 | "id": "09ab1af8-55a5-43ed-9eb9-7d9385246821", 81 | "metadata": { 82 | "execution": { 83 | "iopub.execute_input": "2024-12-02T15:42:10.142626Z", 84 | "iopub.status.busy": "2024-12-02T15:42:10.141960Z", 85 | "iopub.status.idle": "2024-12-02T15:42:10.154511Z", 86 | "shell.execute_reply": "2024-12-02T15:42:10.152208Z", 87 | "shell.execute_reply.started": "2024-12-02T15:42:10.142577Z" 88 | } 89 | }, 90 | "outputs": [ 91 | { 92 | "data": { 93 | "text/plain": [ 94 | "torch.Size([3, 2])" 95 | ] 96 | }, 97 | "execution_count": 9, 98 | "metadata": {}, 99 | "output_type": "execute_result" 100 | } 101 | ], 102 | "source": [ 103 | "X.shape" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": 10, 109 | "id": "d1521eba-3cfa-42bf-a62b-b254b5538fbc", 110 | "metadata": { 111 | "execution": { 112 | "iopub.execute_input": "2024-12-02T15:42:11.164067Z", 113 | "iopub.status.busy": "2024-12-02T15:42:11.163472Z", 114 | "iopub.status.idle": "2024-12-02T15:42:11.182647Z", 115 | "shell.execute_reply": "2024-12-02T15:42:11.180286Z", 116 | "shell.execute_reply.started": "2024-12-02T15:42:11.164023Z" 117 | } 118 | }, 119 | "outputs": [ 120 | { 121 | "data": { 122 | "text/plain": [ 123 | "tensor([[1.0000, 0.9839, 0.9734],\n", 124 | " [0.9839, 1.0000, 0.9987],\n", 125 | " [0.9734, 0.9987, 1.0000]])" 126 | ] 127 | }, 128 | "execution_count": 10, 129 | "metadata": {}, 130 | "output_type": "execute_result" 131 | } 132 | ], 133 | "source": [ 134 | "cos_sim = F.cosine_similarity(X[:, None, :], X[None,:,:], dim=-1)\n", 135 | "cos_sim" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": 13, 141 | "id": "4a3860bf-badd-4486-87e5-3624faaf546a", 142 | "metadata": { 143 | "execution": { 144 | "iopub.execute_input": "2024-12-02T15:43:27.145713Z", 145 | "iopub.status.busy": "2024-12-02T15:43:27.145058Z", 146 | "iopub.status.idle": "2024-12-02T15:43:27.161644Z", 147 | "shell.execute_reply": "2024-12-02T15:43:27.159393Z", 148 | "shell.execute_reply.started": "2024-12-02T15:43:27.145665Z" 149 | } 150 | }, 151 | "outputs": [ 152 | { 153 | "data": { 154 | "text/plain": [ 155 | "tensor(0.9839)" 156 | ] 157 | }, 158 | "execution_count": 13, 159 | "metadata": {}, 160 | "output_type": "execute_result" 161 | } 162 | ], 163 | "source": [ 164 | "(1*3+2*4) / torch.norm(torch.tensor([1., 2.])) / torch.norm(torch.tensor([3., 4.]))" 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": 6, 170 | "id": "18c4d844-7cc8-4ecd-b8d7-04f34ec89a8a", 171 | "metadata": { 172 | "execution": { 173 | "iopub.execute_input": "2024-12-02T15:41:31.404536Z", 174 | "iopub.status.busy": "2024-12-02T15:41:31.403859Z", 175 | "iopub.status.idle": "2024-12-02T15:41:31.424728Z", 176 | "shell.execute_reply": "2024-12-02T15:41:31.422568Z", 177 | "shell.execute_reply.started": "2024-12-02T15:41:31.404487Z" 178 | } 179 | }, 180 | "outputs": [ 181 | { 182 | "data": { 183 | "text/plain": [ 184 | "[[[1, 2], [3, 4], [5, 6]], [[1, 2], [3, 4], [5, 6]], [[1, 2], [3, 4], [5, 6]]]" 185 | ] 186 | }, 187 | "execution_count": 6, 188 | "metadata": {}, 189 | "output_type": "execute_result" 190 | } 191 | ], 192 | "source": [ 193 | "# feats[:,None,:] 的形状是 (3, 1, 2)：\n", 194 | "[\n", 195 | " [[1, 2]],\n", 196 | " [[3, 4]],\n", 197 | " [[5, 6]]\n", 198 | "]\n", 199 | "\n", 200 | "# feats[None,:,:] 的形状是 (1, 3, 2)：\n", 201 | "[\n", 202 | " [[1, 2],\n", 203 | " [3, 4],\n", 204 | " [5, 6]]\n", 205 | "]\n", 206 | "\n", 207 | "# 广播后，两个张量都变成 (3, 3, 2)：\n", 208 | "# 第一个张量（垂直方向重复）：\n", 209 | "[\n", 210 | " [[1, 2], \n", 211 | " [1, 2], \n", 212 | " [1, 2]],\n", 213 | " [[3, 4], \n", 214 | " [3, 4], \n", 215 | " [3, 4]],\n", 216 | " [[5, 6], \n", 217 | " [5, 6], \n", 218 | " [5, 6]]\n", 219 | "]\n", 220 | "\n", 221 | "# 第二个张量（水平方向重复）：\n", 222 | "[\n", 223 | " [[1, 2], \n", 224 | " [3, 4], \n", 225 | " [5, 6]],\n", 226 | " [[1, 2], \n", 227 | " [3, 4], \n", 228 | " [5, 6]],\n", 229 | " [[1, 2], \n", 230 | " [3, 4], \n", 231 | " [5, 6]]\n", 232 | "]" 233 | ] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "execution_count": 14, 238 | "id": "a913f2e6-4bb1-4a0b-afb1-d83344094a0a", 239 | "metadata": { 240 | "execution": { 241 | "iopub.execute_input": "2024-12-02T15:51:21.822340Z", 242 | "iopub.status.busy": "2024-12-02T15:51:21.821069Z", 243 | "iopub.status.idle": "2024-12-02T15:51:21.831886Z", 244 | "shell.execute_reply": "2024-12-02T15:51:21.830626Z", 245 | "shell.execute_reply.started": "2024-12-02T15:51:21.822291Z" 246 | } 247 | }, 248 | "outputs": [ 249 | { 250 | "data": { 251 | "text/plain": [ 252 | "torch.Size([100])" 253 | ] 254 | }, 255 | "execution_count": 14, 256 | "metadata": {}, 257 | "output_type": "execute_result" 258 | } 259 | ], 260 | "source": [ 261 | "input1 = torch.randn(100, 128)\n", 262 | "input2 = torch.randn(100, 128)\n", 263 | "output = F.cosine_similarity(input1, input2)\n", 264 | "output.shape" 265 | ] 266 | } 267 | ], 268 | "metadata": { 269 | "kernelspec": { 270 | "display_name": "Python 3 (ipykernel)", 271 | "language": "python", 272 | "name": "python3" 273 | }, 274 | "language_info": { 275 | "codemirror_mode": { 276 | "name": "ipython", 277 | "version": 3 278 | }, 279 | "file_extension": ".py", 280 | "mimetype": "text/x-python", 281 | "name": "python", 282 | "nbconvert_exporter": "python", 283 | "pygments_lexer": "ipython3", 284 | "version": "3.10.13" 285 | } 286 | }, 287 | "nbformat": 4, 288 | "nbformat_minor": 5 289 | } 290 | -------------------------------------------------------------------------------- /tutorials/nn_basics/grad/gradient_autograd.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "25aba748", 6 | "metadata": {}, 7 | "source": [ 8 | "## grad fn" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "f16be65e", 14 | "metadata": {}, 15 | "source": [ 16 | "- MulBackward0\n", 17 | "- MeanBackward0\n", 18 | "- ..." 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "id": "2a8799d5", 24 | "metadata": {}, 25 | "source": [ 26 | "## `requires_grad = True`" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": 9, 32 | "id": "3aec21fe", 33 | "metadata": { 34 | "ExecuteTime": { 35 | "end_time": "2023-11-14T13:12:52.080492Z", 36 | "start_time": "2023-11-14T13:12:52.067867Z" 37 | } 38 | }, 39 | "outputs": [ 40 | { 41 | "name": "stdout", 42 | "output_type": "stream", 43 | "text": [ 44 | "tensor([8.], grad_fn=)\n", 45 | "tensor(8., grad_fn=)\n", 46 | "tensor([2.], requires_grad=True) True\n" 47 | ] 48 | } 49 | ], 50 | "source": [ 51 | "import torch\n", 52 | "\n", 53 | "# 创建一个可跟踪梯度的张量\n", 54 | "x = torch.tensor([2.0], requires_grad=True)\n", 55 | "\n", 56 | "# 对张量进行一些操作\n", 57 | "# 2x^2\n", 58 | "y = x * x * 2\n", 59 | "print(y)\n", 60 | "z = y.mean()\n", 61 | "print(z)\n", 62 | "print(x, x.is_leaf)" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 7, 68 | "id": "86b1b9dd", 69 | "metadata": { 70 | "ExecuteTime": { 71 | "end_time": "2023-11-14T13:12:14.120433Z", 72 | "start_time": "2023-11-14T13:12:14.111021Z" 73 | } 74 | }, 75 | "outputs": [ 76 | { 77 | "name": "stdout", 78 | "output_type": "stream", 79 | "text": [ 80 | "tensor([8.])\n" 81 | ] 82 | } 83 | ], 84 | "source": [ 85 | "# 执行反向传播\n", 86 | "z.backward()\n", 87 | "\n", 88 | "# 输出梯度, 4x\n", 89 | "print(x.grad)" 90 | ] 91 | } 92 | ], 93 | "metadata": { 94 | "kernelspec": { 95 | "display_name": "Python 3 (ipykernel)", 96 | "language": "python", 97 | "name": "python3" 98 | }, 99 | "language_info": { 100 | "codemirror_mode": { 101 | "name": "ipython", 102 | "version": 3 103 | }, 104 | "file_extension": ".py", 105 | "mimetype": "text/x-python", 106 | "name": "python", 107 | "nbconvert_exporter": "python", 108 | "pygments_lexer": "ipython3", 109 | "version": "3.10.13" 110 | }, 111 | "toc": { 112 | "base_numbering": 1, 113 | "nav_menu": {}, 114 | "number_sections": true, 115 | "sideBar": true, 116 | "skip_h1_title": false, 117 | "title_cell": "Table of Contents", 118 | "title_sidebar": "Contents", 119 | "toc_cell": false, 120 | "toc_position": {}, 121 | "toc_section_display": true, 122 | "toc_window_display": false 123 | } 124 | }, 125 | "nbformat": 4, 126 | "nbformat_minor": 5 127 | } 128 | -------------------------------------------------------------------------------- /tutorials/nn_basics/grad/readme.md: -------------------------------------------------------------------------------- 1 | 2 | - 标量关于谁求导，返回的 shape，就跟谁 shape 保持一致； 3 | - $y=x^TAx$ 4 | - $\nabla_xy=(A+A^T)x$ 5 | - $\nabla_Ay=xx^T$ -------------------------------------------------------------------------------- /tutorials/nn_basics/grad/torch_常见错误.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "078e2471", 7 | "metadata": { 8 | "ExecuteTime": { 9 | "end_time": "2024-03-18T13:18:37.760538Z", 10 | "start_time": "2024-03-18T13:18:35.712909Z" 11 | } 12 | }, 13 | "outputs": [], 14 | "source": [ 15 | "import torch" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "id": "a81d8a5c", 21 | "metadata": { 22 | "ExecuteTime": { 23 | "end_time": "2024-03-18T13:25:54.748902Z", 24 | "start_time": "2024-03-18T13:25:54.742465Z" 25 | } 26 | }, 27 | "source": [ 28 | "## no_grad & enable_grad" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "id": "61774502", 34 | "metadata": {}, 35 | "source": [ 36 | "- torch module 前向构建计算图的过程中，不能使用 `torch.enable_grad()`\n", 37 | " - 因为外部在推理的时候，大概率还会有 `torch.no_grad` 的 context" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": 3, 43 | "id": "7cdf2f49", 44 | "metadata": { 45 | "ExecuteTime": { 46 | "end_time": "2024-03-18T13:18:42.334361Z", 47 | "start_time": "2024-03-18T13:18:42.326367Z" 48 | } 49 | }, 50 | "outputs": [ 51 | { 52 | "data": { 53 | "text/plain": [ 54 | "True" 55 | ] 56 | }, 57 | "execution_count": 3, 58 | "metadata": {}, 59 | "output_type": "execute_result" 60 | } 61 | ], 62 | "source": [ 63 | "x = torch.tensor([1., 2.], requires_grad=True)\n", 64 | "y = x ** 2\n", 65 | "y.requires_grad" 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": 4, 71 | "id": "2467ef40", 72 | "metadata": { 73 | "ExecuteTime": { 74 | "end_time": "2024-03-18T13:18:55.687029Z", 75 | "start_time": "2024-03-18T13:18:55.679115Z" 76 | } 77 | }, 78 | "outputs": [ 79 | { 80 | "data": { 81 | "text/plain": [ 82 | "True" 83 | ] 84 | }, 85 | "execution_count": 4, 86 | "metadata": {}, 87 | "output_type": "execute_result" 88 | } 89 | ], 90 | "source": [ 91 | "with torch.no_grad():\n", 92 | " with torch.enable_grad():\n", 93 | " y = x**2\n", 94 | "y.requires_grad" 95 | ] 96 | } 97 | ], 98 | "metadata": { 99 | "kernelspec": { 100 | "display_name": "Python 3 (ipykernel)", 101 | "language": "python", 102 | "name": "python3" 103 | }, 104 | "language_info": { 105 | "codemirror_mode": { 106 | "name": "ipython", 107 | "version": 3 108 | }, 109 | "file_extension": ".py", 110 | "mimetype": "text/x-python", 111 | "name": "python", 112 | "nbconvert_exporter": "python", 113 | "pygments_lexer": "ipython3", 114 | "version": "3.10.13" 115 | }, 116 | "toc": { 117 | "base_numbering": 1, 118 | "nav_menu": {}, 119 | "number_sections": true, 120 | "sideBar": true, 121 | "skip_h1_title": false, 122 | "title_cell": "Table of Contents", 123 | "title_sidebar": "Contents", 124 | "toc_cell": false, 125 | "toc_position": {}, 126 | "toc_section_display": true, 127 | "toc_window_display": false 128 | } 129 | }, 130 | "nbformat": 4, 131 | "nbformat_minor": 5 132 | } 133 | -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/axies-flat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/axies-flat.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/axis-swap-flat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/axis-swap-flat.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/axis-swap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/axis-swap.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/axis0-1-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/axis0-1-2.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/boo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/boo.jpg -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/fan_in_fan_out.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/fan_in_fan_out.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/graph-legend.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/graph-legend.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/linear.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/linear.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/multi_loss.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/multi_loss.PNG -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/numpy_einsum.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/numpy_einsum.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/shape_1x8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/shape_1x8.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/shape_1x8_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/shape_1x8_2.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/shape_1x8_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/shape_1x8_3.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/shape_1x8_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/shape_1x8_4.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/shape_1x8_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/shape_1x8_5.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/shape_2x4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/shape_2x4.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/shape_4x2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/shape_4x2.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/shape_4x2_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/shape_4x2_2.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/shape_4x2_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/35aa29da73cd1b7155ff305312914d95bbab4328/tutorials/nn_basics/imgs/shape_4x2_3.png -------------------------------------------------------------------------------- /tutorials/nn_basics/module_load_save.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "33a38d29", 7 | "metadata": { 8 | "ExecuteTime": { 9 | "end_time": "2023-11-14T14:52:10.373949Z", 10 | "start_time": "2023-11-14T14:52:09.182936Z" 11 | } 12 | }, 13 | "outputs": [], 14 | "source": [ 15 | "import torch\n", 16 | "from torch import nn" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "id": "cd46809a-ae0b-4a49-89ee-7b346b3faabe", 22 | "metadata": {}, 23 | "source": [ 24 | "## basics" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "id": "32da4ec6-bb98-4f5b-8390-3b6311853aa1", 30 | "metadata": {}, 31 | "source": [ 32 | "- `for name, child in module.named_children():`\n", 33 | " - module 本质上维护了一个树结构；\n", 34 | " - 参考 SyncBatchNorm（https://pytorch.org/docs/stable/_modules/torch/nn/modules/batchnorm.html#SyncBatchNorm）\n", 35 | " - `def convert_sync_batchnorm`：这个函数递归地处理所有的 module\n", 36 | "- PyTorch Module — quick reference：https://medium.com/howsofcoding/pytorch-module-quick-reference-b35d5a0f9a00" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "id": "eb2f75ea-2d67-49b9-b3ef-1bd1093af56b", 42 | "metadata": {}, 43 | "source": [ 44 | "## `named_children` vs. `named_modules`" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 6, 50 | "id": "206bf950-cc6e-4cf0-acb7-d30237390fb3", 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "class MyModule(nn.Module):\n", 55 | " def __init__(self):\n", 56 | " super(MyModule, self).__init__()\n", 57 | " self.layer1 = nn.Sequential(\n", 58 | " nn.Conv2d(16, 32, 3, 1),\n", 59 | " nn.ReLU(inplace=True)\n", 60 | " )\n", 61 | " self.layer2 = nn.Sequential(\n", 62 | " nn.Linear(32,10)\n", 63 | " )\n", 64 | " self.misc = nn.Linear(1, 1)\n", 65 | " \n", 66 | " def forward(self,x):\n", 67 | " x = self.layer1(x)\n", 68 | " x = self.layer2(x)\n", 69 | " \n", 70 | "model = MyModule()" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 7, 76 | "id": "4c782cac-2a5b-4e87-968b-7556c6fe4557", 77 | "metadata": {}, 78 | "outputs": [ 79 | { 80 | "name": "stdout", 81 | "output_type": "stream", 82 | "text": [ 83 | "layer1 -> Sequential(\n", 84 | " (0): Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1))\n", 85 | " (1): ReLU(inplace=True)\n", 86 | ")\n", 87 | "layer2 -> Sequential(\n", 88 | " (0): Linear(in_features=32, out_features=10, bias=True)\n", 89 | ")\n", 90 | "misc -> Linear(in_features=1, out_features=1, bias=True)\n" 91 | ] 92 | } 93 | ], 94 | "source": [ 95 | "for name, module in model.named_children():\n", 96 | " print(name, '->', module)" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 8, 102 | "id": "cf93557e-23fd-4cc1-9d20-735711922a1d", 103 | "metadata": {}, 104 | "outputs": [ 105 | { 106 | "name": "stdout", 107 | "output_type": "stream", 108 | "text": [ 109 | " -> MyModule(\n", 110 | " (layer1): Sequential(\n", 111 | " (0): Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1))\n", 112 | " (1): ReLU(inplace=True)\n", 113 | " )\n", 114 | " (layer2): Sequential(\n", 115 | " (0): Linear(in_features=32, out_features=10, bias=True)\n", 116 | " )\n", 117 | " (misc): Linear(in_features=1, out_features=1, bias=True)\n", 118 | ")\n", 119 | "layer1 -> Sequential(\n", 120 | " (0): Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1))\n", 121 | " (1): ReLU(inplace=True)\n", 122 | ")\n", 123 | "layer1.0 -> Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1))\n", 124 | "layer1.1 -> ReLU(inplace=True)\n", 125 | "layer2 -> Sequential(\n", 126 | " (0): Linear(in_features=32, out_features=10, bias=True)\n", 127 | ")\n", 128 | "layer2.0 -> Linear(in_features=32, out_features=10, bias=True)\n", 129 | "misc -> Linear(in_features=1, out_features=1, bias=True)\n" 130 | ] 131 | } 132 | ], 133 | "source": [ 134 | "for name, module in model.named_modules():\n", 135 | " print(name, '->', module)" 136 | ] 137 | }, 138 | { 139 | "cell_type": "markdown", 140 | "id": "0ee438f1", 141 | "metadata": {}, 142 | "source": [ 143 | "## `model.parameters()`" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": 2, 149 | "id": "19c5fba6-2f0d-4f0d-9185-c5dcd3143d7b", 150 | "metadata": {}, 151 | "outputs": [], 152 | "source": [ 153 | "model = nn.Linear(5, 3)" 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": 9, 159 | "id": "29d04b31", 160 | "metadata": { 161 | "ExecuteTime": { 162 | "end_time": "2023-11-14T14:52:57.383206Z", 163 | "start_time": "2023-11-14T14:52:57.369346Z" 164 | } 165 | }, 166 | "outputs": [ 167 | { 168 | "data": { 169 | "text/plain": [ 170 | "[Parameter containing:\n", 171 | " tensor([[-0.2888, -0.1333, 0.2464, -0.2843, -0.2926],\n", 172 | " [-0.1179, -0.3022, -0.4258, 0.4373, 0.0226],\n", 173 | " [ 0.1486, -0.1183, -0.3256, -0.3780, -0.4289]], requires_grad=True),\n", 174 | " Parameter containing:\n", 175 | " tensor([-0.2695, -0.1645, -0.4121], requires_grad=True)]" 176 | ] 177 | }, 178 | "execution_count": 9, 179 | "metadata": {}, 180 | "output_type": "execute_result" 181 | } 182 | ], 183 | "source": [ 184 | "list(model.parameters())" 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": 10, 190 | "id": "a1168a17", 191 | "metadata": { 192 | "ExecuteTime": { 193 | "end_time": "2023-11-14T14:53:01.334448Z", 194 | "start_time": "2023-11-14T14:53:01.321081Z" 195 | } 196 | }, 197 | "outputs": [ 198 | { 199 | "data": { 200 | "text/plain": [ 201 | "[('weight',\n", 202 | " Parameter containing:\n", 203 | " tensor([[-0.2888, -0.1333, 0.2464, -0.2843, -0.2926],\n", 204 | " [-0.1179, -0.3022, -0.4258, 0.4373, 0.0226],\n", 205 | " [ 0.1486, -0.1183, -0.3256, -0.3780, -0.4289]], requires_grad=True)),\n", 206 | " ('bias',\n", 207 | " Parameter containing:\n", 208 | " tensor([-0.2695, -0.1645, -0.4121], requires_grad=True))]" 209 | ] 210 | }, 211 | "execution_count": 10, 212 | "metadata": {}, 213 | "output_type": "execute_result" 214 | } 215 | ], 216 | "source": [ 217 | "list(model.named_parameters())" 218 | ] 219 | } 220 | ], 221 | "metadata": { 222 | "kernelspec": { 223 | "display_name": "Python 3 (ipykernel)", 224 | "language": "python", 225 | "name": "python3" 226 | }, 227 | "language_info": { 228 | "codemirror_mode": { 229 | "name": "ipython", 230 | "version": 3 231 | }, 232 | "file_extension": ".py", 233 | "mimetype": "text/x-python", 234 | "name": "python", 235 | "nbconvert_exporter": "python", 236 | "pygments_lexer": "ipython3", 237 | "version": "3.10.13" 238 | }, 239 | "toc": { 240 | "base_numbering": 1, 241 | "nav_menu": {}, 242 | "number_sections": true, 243 | "sideBar": true, 244 | "skip_h1_title": false, 245 | "title_cell": "Table of Contents", 246 | "title_sidebar": "Contents", 247 | "toc_cell": false, 248 | "toc_position": {}, 249 | "toc_section_display": true, 250 | "toc_window_display": false 251 | } 252 | }, 253 | "nbformat": 4, 254 | "nbformat_minor": 5 255 | } 256 | -------------------------------------------------------------------------------- /tutorials/nn_basics/optimizer/momentum_exp_avg.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "dbed347c-181f-4178-ab67-915064a44985", 6 | "metadata": {}, 7 | "source": [ 8 | "\n", 9 | "- 基于 Momentum 的 SGD\n", 10 | "\n", 11 | " $$\n", 12 | " \\begin{split}\n", 13 | " v_t&=\\beta v_{t-1}+g_t\\\\\n", 14 | " \\theta_{t+1}&=\\theta_t-\\alpha v_t\n", 15 | " \\end{split}\n", 16 | " $$\n", 17 | " \n", 18 | "\n", 19 | " - 对于 $v_t$ 部分递归的结果：$v_t=\\sum_{i=0}^{t-i}\\beta^ig_{t-i}$ （$v_0$ 初始化为 0）" 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "id": "ee28e34f-3c8d-4815-a457-b65f203db2d2", 25 | "metadata": {}, 26 | "source": [ 27 | "- EMA (Exponential Moving Average)\n", 28 | "\n", 29 | "$$\n", 30 | "\\begin{split}\n", 31 | "\\text{EMA}_t&=\\alpha x_t+(1-\\alpha) \\text{EMA}_{t-1}\\\\\n", 32 | "&=\\alpha x_t+(1-\\alpha)\\alpha x_{t-1}+(1-\\alpha)^2\\alpha x_{t-2}+\\cdots+(1-\\alpha)^{t-1}\\alpha x_1\n", 33 | "\\end{split}\n", 34 | "$$" 35 | ] 36 | } 37 | ], 38 | "metadata": { 39 | "kernelspec": { 40 | "display_name": "Python 3 (ipykernel)", 41 | "language": "python", 42 | "name": "python3" 43 | }, 44 | "language_info": { 45 | "codemirror_mode": { 46 | "name": "ipython", 47 | "version": 3 48 | }, 49 | "file_extension": ".py", 50 | "mimetype": "text/x-python", 51 | "name": "python", 52 | "nbconvert_exporter": "python", 53 | "pygments_lexer": "ipython3", 54 | "version": "3.10.13" 55 | } 56 | }, 57 | "nbformat": 4, 58 | "nbformat_minor": 5 59 | } 60 | -------------------------------------------------------------------------------- /tutorials/nn_basics/quantization.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "e8311baa", 6 | "metadata": {}, 7 | "source": [ 8 | "$$\n", 9 | "f = \\frac{f_{\\max}-f_{\\min}}{q_{\\max}-q_{\\min}} (q-Z)=S(q-Z)\\\\\n", 10 | "q = \\frac fS+Z\n", 11 | "$$" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "id": "777ff93d", 17 | "metadata": {}, 18 | "source": [ 19 | "- unsigned 8 bit\n", 20 | " - [0, 255]（q_max, q_min）\n", 21 | "- signed 8 bit\n", 22 | " - [-128, 127]（q_max, q_min)" 23 | ] 24 | }, 25 | { 26 | "cell_type": "markdown", 27 | "id": "e6173f22", 28 | "metadata": {}, 29 | "source": [ 30 | "## quantize" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": null, 36 | "id": "aaa1dfcd", 37 | "metadata": { 38 | "ExecuteTime": { 39 | "end_time": "2023-07-05T14:14:16.541441Z", 40 | "start_time": "2023-07-05T14:14:13.318860Z" 41 | } 42 | }, 43 | "outputs": [], 44 | "source": [ 45 | "from transformers import AutoModel\n", 46 | "model = AutoModel.from_pretrained('bert-base-uncased')" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": null, 52 | "id": "99551ab4", 53 | "metadata": { 54 | "ExecuteTime": { 55 | "end_time": "2023-07-05T14:49:57.427112Z", 56 | "start_time": "2023-07-05T14:49:57.417737Z" 57 | } 58 | }, 59 | "outputs": [], 60 | "source": [ 61 | "model" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "id": "d27e9e11", 68 | "metadata": { 69 | "ExecuteTime": { 70 | "end_time": "2023-07-05T14:50:20.658732Z", 71 | "start_time": "2023-07-05T14:50:20.643717Z" 72 | } 73 | }, 74 | "outputs": [], 75 | "source": [ 76 | "weights = model.state_dict()['encoder.layer.0.attention.self.query.weight']\n", 77 | "weights.dtype" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": null, 83 | "id": "5790feca", 84 | "metadata": { 85 | "ExecuteTime": { 86 | "end_time": "2023-07-05T14:50:23.825669Z", 87 | "start_time": "2023-07-05T14:50:23.816251Z" 88 | } 89 | }, 90 | "outputs": [], 91 | "source": [ 92 | "# (4h, h)\n", 93 | "weights.shape" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "id": "b542aab4", 100 | "metadata": { 101 | "ExecuteTime": { 102 | "end_time": "2023-07-05T14:16:25.055670Z", 103 | "start_time": "2023-07-05T14:16:24.749026Z" 104 | } 105 | }, 106 | "outputs": [], 107 | "source": [ 108 | "import matplotlib.pyplot as plt" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": null, 114 | "id": "86c52d95", 115 | "metadata": { 116 | "ExecuteTime": { 117 | "end_time": "2023-07-05T14:50:26.835664Z", 118 | "start_time": "2023-07-05T14:50:26.353195Z" 119 | } 120 | }, 121 | "outputs": [], 122 | "source": [ 123 | "_ = plt.hist(weights.flatten().numpy(), bins=250, )" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": null, 129 | "id": "459fa0a7", 130 | "metadata": { 131 | "ExecuteTime": { 132 | "end_time": "2023-07-05T14:50:29.783138Z", 133 | "start_time": "2023-07-05T14:50:29.776069Z" 134 | } 135 | }, 136 | "outputs": [], 137 | "source": [ 138 | "# signed 8-bit integer\n", 139 | "zero_point = 0\n", 140 | "scale = (weights.max() - weights.min())/(127-(-128))\n", 141 | "scale" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": null, 147 | "id": "a3d70b6c", 148 | "metadata": { 149 | "ExecuteTime": { 150 | "end_time": "2023-07-05T14:50:32.030900Z", 151 | "start_time": "2023-07-05T14:50:32.023089Z" 152 | } 153 | }, 154 | "outputs": [], 155 | "source": [ 156 | "(weights/scale + zero_point).round().char()" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": null, 162 | "id": "634bd6dc", 163 | "metadata": { 164 | "ExecuteTime": { 165 | "end_time": "2023-07-05T14:50:34.313978Z", 166 | "start_time": "2023-07-05T14:50:34.308514Z" 167 | } 168 | }, 169 | "outputs": [], 170 | "source": [ 171 | "import torch\n", 172 | "from torch import quantize_per_tensor" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": null, 178 | "id": "710238a7", 179 | "metadata": { 180 | "ExecuteTime": { 181 | "end_time": "2023-07-05T14:50:35.250675Z", 182 | "start_time": "2023-07-05T14:50:35.243881Z" 183 | } 184 | }, 185 | "outputs": [], 186 | "source": [ 187 | "q_weights = quantize_per_tensor(weights, scale, zero_point, torch.qint8)\n", 188 | "q_weights.int_repr()" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": null, 194 | "id": "369c7f54", 195 | "metadata": { 196 | "ExecuteTime": { 197 | "end_time": "2023-07-05T14:26:06.329258Z", 198 | "start_time": "2023-07-05T14:26:06.320153Z" 199 | } 200 | }, 201 | "outputs": [], 202 | "source": [ 203 | "q_weights.shape" 204 | ] 205 | }, 206 | { 207 | "cell_type": "markdown", 208 | "id": "f88b02d0", 209 | "metadata": {}, 210 | "source": [ 211 | "## analysis" 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": null, 217 | "id": "6d5290d3", 218 | "metadata": { 219 | "ExecuteTime": { 220 | "end_time": "2023-07-05T14:50:37.753592Z", 221 | "start_time": "2023-07-05T14:50:37.405342Z" 222 | } 223 | }, 224 | "outputs": [], 225 | "source": [ 226 | "_ = plt.hist(q_weights.int_repr().flatten().numpy(), bins=250, range=(-60, 60))" 227 | ] 228 | }, 229 | { 230 | "cell_type": "code", 231 | "execution_count": null, 232 | "id": "36711b39", 233 | "metadata": { 234 | "ExecuteTime": { 235 | "end_time": "2023-07-05T14:50:42.578299Z", 236 | "start_time": "2023-07-05T14:50:40.284362Z" 237 | } 238 | }, 239 | "outputs": [], 240 | "source": [ 241 | "%%timeit\n", 242 | "weights @ weights.T" 243 | ] 244 | }, 245 | { 246 | "cell_type": "code", 247 | "execution_count": 1, 248 | "id": "9a5e532d", 249 | "metadata": { 250 | "ExecuteTime": { 251 | "end_time": "2023-07-05T14:56:53.187851Z", 252 | "start_time": "2023-07-05T14:56:51.936817Z" 253 | } 254 | }, 255 | "outputs": [], 256 | "source": [ 257 | "import torch\n", 258 | "from torch.nn.quantized import QFunctional\n", 259 | "q_fn = QFunctional()" 260 | ] 261 | }, 262 | { 263 | "cell_type": "code", 264 | "execution_count": 2, 265 | "id": "99eef38a", 266 | "metadata": { 267 | "ExecuteTime": { 268 | "end_time": "2023-07-05T14:56:54.275759Z", 269 | "start_time": "2023-07-05T14:56:54.268386Z" 270 | } 271 | }, 272 | "outputs": [ 273 | { 274 | "data": { 275 | "text/plain": [ 276 | "'x86'" 277 | ] 278 | }, 279 | "execution_count": 2, 280 | "metadata": {}, 281 | "output_type": "execute_result" 282 | } 283 | ], 284 | "source": [ 285 | "# qconfig = torch.quantization.get_default_qconfig('qnnpack')\n", 286 | "torch.backends.quantized.engine" 287 | ] 288 | }, 289 | { 290 | "cell_type": "code", 291 | "execution_count": null, 292 | "id": "bfacb5b5", 293 | "metadata": { 294 | "ExecuteTime": { 295 | "end_time": "2023-07-05T14:56:22.410054Z", 296 | "start_time": "2023-07-05T14:56:22.268136Z" 297 | } 298 | }, 299 | "outputs": [], 300 | "source": [ 301 | "# %%timeit \n", 302 | "q_fn.mul(q_weights, q_weights)" 303 | ] 304 | }, 305 | { 306 | "cell_type": "code", 307 | "execution_count": null, 308 | "id": "c70332bd", 309 | "metadata": {}, 310 | "outputs": [], 311 | "source": [] 312 | } 313 | ], 314 | "metadata": { 315 | "kernelspec": { 316 | "display_name": "Python 3 (ipykernel)", 317 | "language": "python", 318 | "name": "python3" 319 | }, 320 | "language_info": { 321 | "codemirror_mode": { 322 | "name": "ipython", 323 | "version": 3 324 | }, 325 | "file_extension": ".py", 326 | "mimetype": "text/x-python", 327 | "name": "python", 328 | "nbconvert_exporter": "python", 329 | "pygments_lexer": "ipython3", 330 | "version": "3.10.13" 331 | }, 332 | "toc": { 333 | "base_numbering": 1, 334 | "nav_menu": {}, 335 | "number_sections": true, 336 | "sideBar": true, 337 | "skip_h1_title": false, 338 | "title_cell": "Table of Contents", 339 | "title_sidebar": "Contents", 340 | "toc_cell": false, 341 | "toc_position": {}, 342 | "toc_section_display": true, 343 | "toc_window_display": false 344 | } 345 | }, 346 | "nbformat": 4, 347 | "nbformat_minor": 5 348 | } 349 | -------------------------------------------------------------------------------- /tutorials/nn_basics/tricks_norms/layer_norm.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "207a7f06-e0dd-4d0b-8761-99ad25c790c2", 6 | "metadata": {}, 7 | "source": [ 8 | "$$\n", 9 | "\\begin{split}\n", 10 | "x_1 &= x_0 - \\text{mean}(x_0)\\\\\n", 11 | "x_2 &= \\frac{x_1}{\\sqrt{\\text{mean}(x_1^2)}}\\\\\n", 12 | "x_3 &= x_2 \\cdot w\\\\\n", 13 | "x_4 &= x_3 + b\\\\\n", 14 | "\\end{split}\n", 15 | "$$\n", 16 | "\n", 17 | "- Unlike BatchNorm, it cannot be turned off at inference time, as it significantly alters the mathematical function implemented by the transformer.\n", 18 | "- 定义在 feature 维度，样本甚至是 token 级别，而非 batchnorm 的跨样本；\n", 19 | "- element-wise 操作；" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 41, 25 | "id": "76a755e2-9348-4135-9e1f-5ab8851c1a7c", 26 | "metadata": { 27 | "execution": { 28 | "iopub.execute_input": "2024-12-19T14:20:31.028856Z", 29 | "iopub.status.busy": "2024-12-19T14:20:31.028199Z", 30 | "iopub.status.idle": "2024-12-19T14:20:31.042105Z", 31 | "shell.execute_reply": "2024-12-19T14:20:31.039936Z", 32 | "shell.execute_reply.started": "2024-12-19T14:20:31.028809Z" 33 | } 34 | }, 35 | "outputs": [ 36 | { 37 | "data": { 38 | "text/plain": [ 39 | "" 40 | ] 41 | }, 42 | "execution_count": 41, 43 | "metadata": {}, 44 | "output_type": "execute_result" 45 | } 46 | ], 47 | "source": [ 48 | "import torch\n", 49 | "from torch import nn\n", 50 | "torch.manual_seed(42)" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 42, 56 | "id": "e451a3e7-f801-41e4-aa13-f8f8cc48b85b", 57 | "metadata": { 58 | "execution": { 59 | "iopub.execute_input": "2024-12-19T14:20:32.493381Z", 60 | "iopub.status.busy": "2024-12-19T14:20:32.492778Z", 61 | "iopub.status.idle": "2024-12-19T14:20:32.501645Z", 62 | "shell.execute_reply": "2024-12-19T14:20:32.499494Z", 63 | "shell.execute_reply.started": "2024-12-19T14:20:32.493335Z" 64 | } 65 | }, 66 | "outputs": [], 67 | "source": [ 68 | "batch, sentence_length, embedding_dim = 2, 3, 4" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 43, 74 | "id": "25a3b546-7b63-445a-aa16-49ff6615009e", 75 | "metadata": { 76 | "execution": { 77 | "iopub.execute_input": "2024-12-19T14:20:33.562922Z", 78 | "iopub.status.busy": "2024-12-19T14:20:33.562326Z", 79 | "iopub.status.idle": "2024-12-19T14:20:33.578811Z", 80 | "shell.execute_reply": "2024-12-19T14:20:33.576355Z", 81 | "shell.execute_reply.started": "2024-12-19T14:20:33.562877Z" 82 | } 83 | }, 84 | "outputs": [ 85 | { 86 | "data": { 87 | "text/plain": [ 88 | "tensor([[[2., 2., 1., 4.],\n", 89 | " [1., 0., 0., 4.],\n", 90 | " [0., 3., 3., 4.]],\n", 91 | "\n", 92 | " [[0., 4., 1., 2.],\n", 93 | " [0., 0., 2., 1.],\n", 94 | " [4., 1., 3., 1.]]])" 95 | ] 96 | }, 97 | "execution_count": 43, 98 | "metadata": {}, 99 | "output_type": "execute_result" 100 | } 101 | ], 102 | "source": [ 103 | "embedding = torch.randint(0, 5, (batch, sentence_length, embedding_dim)).float()\n", 104 | "embedding" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 44, 110 | "id": "e2e4e0fc-5050-4a2c-b156-3a5824e35632", 111 | "metadata": { 112 | "execution": { 113 | "iopub.execute_input": "2024-12-19T14:20:35.865423Z", 114 | "iopub.status.busy": "2024-12-19T14:20:35.864827Z", 115 | "iopub.status.idle": "2024-12-19T14:20:35.875125Z", 116 | "shell.execute_reply": "2024-12-19T14:20:35.872957Z", 117 | "shell.execute_reply.started": "2024-12-19T14:20:35.865378Z" 118 | } 119 | }, 120 | "outputs": [], 121 | "source": [ 122 | "ln = nn.LayerNorm(embedding_dim)" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": 45, 128 | "id": "a463ac73-23f4-4f5b-8746-e6b5642c9a97", 129 | "metadata": { 130 | "execution": { 131 | "iopub.execute_input": "2024-12-19T14:20:37.263538Z", 132 | "iopub.status.busy": "2024-12-19T14:20:37.262944Z", 133 | "iopub.status.idle": "2024-12-19T14:20:37.277615Z", 134 | "shell.execute_reply": "2024-12-19T14:20:37.275507Z", 135 | "shell.execute_reply.started": "2024-12-19T14:20:37.263494Z" 136 | } 137 | }, 138 | "outputs": [ 139 | { 140 | "data": { 141 | "text/plain": [ 142 | "[Parameter containing:\n", 143 | " tensor([1., 1., 1., 1.], requires_grad=True),\n", 144 | " Parameter containing:\n", 145 | " tensor([0., 0., 0., 0.], requires_grad=True)]" 146 | ] 147 | }, 148 | "execution_count": 45, 149 | "metadata": {}, 150 | "output_type": "execute_result" 151 | } 152 | ], 153 | "source": [ 154 | "list(ln.parameters())" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": 50, 160 | "id": "a5b43758-d1da-4dd8-a17d-62040bb894e1", 161 | "metadata": { 162 | "execution": { 163 | "iopub.execute_input": "2024-12-19T14:23:02.140722Z", 164 | "iopub.status.busy": "2024-12-19T14:23:02.140125Z", 165 | "iopub.status.idle": "2024-12-19T14:23:02.154222Z", 166 | "shell.execute_reply": "2024-12-19T14:23:02.152121Z", 167 | "shell.execute_reply.started": "2024-12-19T14:23:02.140678Z" 168 | } 169 | }, 170 | "outputs": [ 171 | { 172 | "data": { 173 | "text/plain": [ 174 | "(Parameter containing:\n", 175 | " tensor([1., 1., 1., 1.], requires_grad=True),\n", 176 | " Parameter containing:\n", 177 | " tensor([0., 0., 0., 0.], requires_grad=True))" 178 | ] 179 | }, 180 | "execution_count": 50, 181 | "metadata": {}, 182 | "output_type": "execute_result" 183 | } 184 | ], 185 | "source": [ 186 | "ln.weight, ln.bias" 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": 54, 192 | "id": "74a4438c-4d0c-462b-a285-6dd0daba2289", 193 | "metadata": { 194 | "execution": { 195 | "iopub.execute_input": "2024-12-19T14:25:10.690907Z", 196 | "iopub.status.busy": "2024-12-19T14:25:10.690240Z", 197 | "iopub.status.idle": "2024-12-19T14:25:10.702279Z", 198 | "shell.execute_reply": "2024-12-19T14:25:10.700908Z", 199 | "shell.execute_reply.started": "2024-12-19T14:25:10.690860Z" 200 | } 201 | }, 202 | "outputs": [ 203 | { 204 | "data": { 205 | "text/plain": [ 206 | "tensor([[[-0.2294, -0.2294, -1.1471, 1.6059],\n", 207 | " [-0.1525, -0.7625, -0.7625, 1.6775],\n", 208 | " [-1.6667, 0.3333, 0.3333, 1.0000]],\n", 209 | "\n", 210 | " [[-1.1832, 1.5213, -0.5071, 0.1690],\n", 211 | " [-0.9045, -0.9045, 1.5075, 0.3015],\n", 212 | " [ 1.3471, -0.9622, 0.5773, -0.9622]]],\n", 213 | " grad_fn=)" 214 | ] 215 | }, 216 | "execution_count": 54, 217 | "metadata": {}, 218 | "output_type": "execute_result" 219 | } 220 | ], 221 | "source": [ 222 | "ln(embedding)" 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": 56, 228 | "id": "badc0d85-3562-42c0-936b-08e7d9d1072b", 229 | "metadata": { 230 | "execution": { 231 | "iopub.execute_input": "2024-12-19T14:26:50.031528Z", 232 | "iopub.status.busy": "2024-12-19T14:26:50.030890Z", 233 | "iopub.status.idle": "2024-12-19T14:26:50.047679Z", 234 | "shell.execute_reply": "2024-12-19T14:26:50.045560Z", 235 | "shell.execute_reply.started": "2024-12-19T14:26:50.031467Z" 236 | } 237 | }, 238 | "outputs": [ 239 | { 240 | "data": { 241 | "text/plain": [ 242 | "tensor([-0.2294, -0.2294, -1.1471, 1.6059], grad_fn=)" 243 | ] 244 | }, 245 | "execution_count": 56, 246 | "metadata": {}, 247 | "output_type": "execute_result" 248 | } 249 | ], 250 | "source": [ 251 | "x = embedding[0, 0, :]\n", 252 | "x1 = x - torch.mean(x)\n", 253 | "x2 = x1 / torch.sqrt(torch.var(x1, unbiased=False))\n", 254 | "x3 = x2 * ln.weight + ln.bias\n", 255 | "x3" 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": 61, 261 | "id": "af2e56ec-9ac1-4365-ac29-92b9cd20e91d", 262 | "metadata": { 263 | "execution": { 264 | "iopub.execute_input": "2024-12-19T14:29:44.466924Z", 265 | "iopub.status.busy": "2024-12-19T14:29:44.466266Z", 266 | "iopub.status.idle": "2024-12-19T14:29:44.475613Z", 267 | "shell.execute_reply": "2024-12-19T14:29:44.473354Z", 268 | "shell.execute_reply.started": "2024-12-19T14:29:44.466876Z" 269 | } 270 | }, 271 | "outputs": [], 272 | "source": [ 273 | "gamma = ln.weight\n", 274 | "beta = ln.bias" 275 | ] 276 | }, 277 | { 278 | "cell_type": "code", 279 | "execution_count": 62, 280 | "id": "26fa9951-c58f-42ab-a86c-4ae43f8dea0d", 281 | "metadata": { 282 | "execution": { 283 | "iopub.execute_input": "2024-12-19T14:30:04.667088Z", 284 | "iopub.status.busy": "2024-12-19T14:30:04.666424Z", 285 | "iopub.status.idle": "2024-12-19T14:30:04.683057Z", 286 | "shell.execute_reply": "2024-12-19T14:30:04.681070Z", 287 | "shell.execute_reply.started": "2024-12-19T14:30:04.667040Z" 288 | } 289 | }, 290 | "outputs": [ 291 | { 292 | "data": { 293 | "text/plain": [ 294 | "tensor([[[-0.2294, -0.2294, -1.1471, 1.6059],\n", 295 | " [-0.1525, -0.7625, -0.7625, 1.6775],\n", 296 | " [-1.6667, 0.3333, 0.3333, 1.0000]],\n", 297 | "\n", 298 | " [[-1.1832, 1.5213, -0.5071, 0.1690],\n", 299 | " [-0.9045, -0.9045, 1.5075, 0.3015],\n", 300 | " [ 1.3471, -0.9622, 0.5773, -0.9622]]], grad_fn=)" 301 | ] 302 | }, 303 | "execution_count": 62, 304 | "metadata": {}, 305 | "output_type": "execute_result" 306 | } 307 | ], 308 | "source": [ 309 | "normalized = (embedding - embedding.mean(dim=-1, keepdim=True)) / torch.sqrt(embedding.var(dim=-1, unbiased=False, keepdim=True) + ln.eps) \n", 310 | "gamma * normalized + beta" 311 | ] 312 | } 313 | ], 314 | "metadata": { 315 | "kernelspec": { 316 | "display_name": "Python 3 (ipykernel)", 317 | "language": "python", 318 | "name": "python3" 319 | }, 320 | "language_info": { 321 | "codemirror_mode": { 322 | "name": "ipython", 323 | "version": 3 324 | }, 325 | "file_extension": ".py", 326 | "mimetype": "text/x-python", 327 | "name": "python", 328 | "nbconvert_exporter": "python", 329 | "pygments_lexer": "ipython3", 330 | "version": "3.10.13" 331 | } 332 | }, 333 | "nbformat": 4, 334 | "nbformat_minor": 5 335 | } 336 | -------------------------------------------------------------------------------- /tutorials/nn_basics/tricks_norms/nn_and_training_tricks.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 3, 6 | "id": "2f67f995", 7 | "metadata": { 8 | "ExecuteTime": { 9 | "end_time": "2023-11-28T13:06:58.026548Z", 10 | "start_time": "2023-11-28T13:06:58.020728Z" 11 | } 12 | }, 13 | "outputs": [], 14 | "source": [ 15 | "import torch\n", 16 | "from torch import nn" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "id": "405edb19", 22 | "metadata": {}, 23 | "source": [ 24 | "## 理解神经网络" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "id": "3d5d8c6d", 30 | "metadata": {}, 31 | "source": [ 32 | "- 从shape的含义，shape 的变化，可能是理解神经网络模型结构乃至处理过程的一个很实用的切入。" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "id": "0448091c", 38 | "metadata": {}, 39 | "source": [ 40 | "## `nn.utils.clip_grad_norm_`" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "id": "8c054c26", 46 | "metadata": {}, 47 | "source": [ 48 | "- 避免梯度爆炸\n", 49 | " \n", 50 | " - 计算所有梯度的范数：这一步骤涉及到计算网络中所有参数的梯度范数。通常使用 L2 范数，即平方和的平方根。\n", 51 | " - 也是 `nn.utils.clip_grad_norm_` 的返回值；\n", 52 | " - 比较梯度范数与最大值：将计算出的梯度范数与设定的最大范数值比较。\n", 53 | " - 裁剪梯度：如果梯度范数大于最大值，那么将所有梯度缩放到最大值以内。这是通过乘以一个缩放因子实现的，缩放因子是最大范数值除以梯度范数。\n", 54 | " - 更新梯度：使用裁剪后的梯度值更新网络参数。\n", 55 | "\n", 56 | "- 一般training过程中用到：\n", 57 | "\n", 58 | " ```\n", 59 | " optimizer.zero_grad()\n", 60 | " # 反向传播计算 parameters 的 grad\n", 61 | " loss.backward()\n", 62 | " # 计算完梯度之后，norm 所有参数的 grads\n", 63 | " nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip)\n", 64 | " # 基于更新后的 grad 值来更新参数\n", 65 | " optimizer.step()\n", 66 | " ```" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 13, 72 | "id": "c6a0eb31", 73 | "metadata": { 74 | "ExecuteTime": { 75 | "end_time": "2023-11-28T13:11:08.362707Z", 76 | "start_time": "2023-11-28T13:11:08.352229Z" 77 | } 78 | }, 79 | "outputs": [], 80 | "source": [ 81 | "# 定义一个简单的网络及其梯度\n", 82 | "class SimpleNet(torch.nn.Module):\n", 83 | " def __init__(self):\n", 84 | " super(SimpleNet, self).__init__()\n", 85 | " self.param1 = torch.nn.Parameter(torch.tensor([3.0, 4.0]))\n", 86 | " self.param2 = torch.nn.Parameter(torch.tensor([1.0, 2.0]))\n", 87 | "\n", 88 | "# 创建网络实例\n", 89 | "net = SimpleNet()\n", 90 | "\n", 91 | "# 设置梯度\n", 92 | "net.param1.grad = torch.tensor([3.0, 4.0])\n", 93 | "net.param2.grad = torch.tensor([1.0, 2.0])\n", 94 | "\n", 95 | "# 最大梯度范数\n", 96 | "max_norm = 5.0" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 14, 102 | "id": "c7b9ad30", 103 | "metadata": { 104 | "ExecuteTime": { 105 | "end_time": "2023-11-28T13:11:10.028971Z", 106 | "start_time": "2023-11-28T13:11:10.016953Z" 107 | } 108 | }, 109 | "outputs": [ 110 | { 111 | "data": { 112 | "text/plain": [ 113 | "(tensor(5.4772), [tensor([2.7386, 3.6515]), tensor([0.9129, 1.8257])])" 114 | ] 115 | }, 116 | "execution_count": 14, 117 | "metadata": {}, 118 | "output_type": "execute_result" 119 | } 120 | ], 121 | "source": [ 122 | "# 计算梯度范数\n", 123 | "total_norm = torch.sqrt(sum(p.grad.norm()**2 for p in net.parameters()))\n", 124 | "\n", 125 | "# 计算缩放因子\n", 126 | "scale = max_norm / (total_norm + 1e-6)\n", 127 | "\n", 128 | "# 应用梯度裁剪\n", 129 | "for p in net.parameters():\n", 130 | " p.grad.data.mul_(scale)\n", 131 | "\n", 132 | "# 更新后的梯度\n", 133 | "clipped_grads = [p.grad for p in net.parameters()]\n", 134 | "\n", 135 | "total_norm, clipped_grads" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": 15, 141 | "id": "d7143022", 142 | "metadata": { 143 | "ExecuteTime": { 144 | "end_time": "2023-11-28T13:11:11.532229Z", 145 | "start_time": "2023-11-28T13:11:11.522275Z" 146 | } 147 | }, 148 | "outputs": [], 149 | "source": [ 150 | "# 定义一个简单的网络及其梯度\n", 151 | "class SimpleNet(torch.nn.Module):\n", 152 | " def __init__(self):\n", 153 | " super(SimpleNet, self).__init__()\n", 154 | " self.param1 = torch.nn.Parameter(torch.tensor([3.0, 4.0]))\n", 155 | " self.param2 = torch.nn.Parameter(torch.tensor([1.0, 2.0]))\n", 156 | "\n", 157 | "# 创建网络实例\n", 158 | "net = SimpleNet()\n", 159 | "\n", 160 | "# 设置梯度\n", 161 | "net.param1.grad = torch.tensor([3.0, 4.0])\n", 162 | "net.param2.grad = torch.tensor([1.0, 2.0])\n", 163 | "\n", 164 | "# 最大梯度范数\n", 165 | "max_norm = 5.0" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": 16, 171 | "id": "15d074eb", 172 | "metadata": { 173 | "ExecuteTime": { 174 | "end_time": "2023-11-28T13:11:14.160926Z", 175 | "start_time": "2023-11-28T13:11:14.151739Z" 176 | } 177 | }, 178 | "outputs": [ 179 | { 180 | "data": { 181 | "text/plain": [ 182 | "tensor(5.4772)" 183 | ] 184 | }, 185 | "execution_count": 16, 186 | "metadata": {}, 187 | "output_type": "execute_result" 188 | } 189 | ], 190 | "source": [ 191 | "nn.utils.clip_grad_norm_(net.parameters(), max_norm)" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": 17, 197 | "id": "1a1bd7a7", 198 | "metadata": { 199 | "ExecuteTime": { 200 | "end_time": "2023-11-28T13:11:15.181539Z", 201 | "start_time": "2023-11-28T13:11:15.173888Z" 202 | } 203 | }, 204 | "outputs": [ 205 | { 206 | "name": "stdout", 207 | "output_type": "stream", 208 | "text": [ 209 | "tensor([2.7386, 3.6515])\n", 210 | "tensor([0.9129, 1.8257])\n" 211 | ] 212 | } 213 | ], 214 | "source": [ 215 | "for p in net.parameters():\n", 216 | " print(p.grad)" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": null, 222 | "id": "63fd5152", 223 | "metadata": {}, 224 | "outputs": [], 225 | "source": [] 226 | } 227 | ], 228 | "metadata": { 229 | "kernelspec": { 230 | "display_name": "Python 3 (ipykernel)", 231 | "language": "python", 232 | "name": "python3" 233 | }, 234 | "language_info": { 235 | "codemirror_mode": { 236 | "name": "ipython", 237 | "version": 3 238 | }, 239 | "file_extension": ".py", 240 | "mimetype": "text/x-python", 241 | "name": "python", 242 | "nbconvert_exporter": "python", 243 | "pygments_lexer": "ipython3", 244 | "version": "3.10.13" 245 | }, 246 | "toc": { 247 | "base_numbering": 1, 248 | "nav_menu": {}, 249 | "number_sections": true, 250 | "sideBar": true, 251 | "skip_h1_title": false, 252 | "title_cell": "Table of Contents", 253 | "title_sidebar": "Contents", 254 | "toc_cell": false, 255 | "toc_position": {}, 256 | "toc_section_display": true, 257 | "toc_window_display": false 258 | } 259 | }, 260 | "nbformat": 4, 261 | "nbformat_minor": 5 262 | } 263 | -------------------------------------------------------------------------------- /tutorials/nn_basics/tricks_norms/torch_dropout.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "058cc67c-76d3-4898-bd49-8196a9976535", 6 | "metadata": {}, 7 | "source": [ 8 | "> - 越基础的，越需要重视，熟悉其原理，越基础的越会更广泛地出现。\n", 9 | "> - forward 前向计算过程，backward 反向梯度计算；\n", 10 | "> - 基于 torch（autograd 框架）的求导练习；" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 18, 16 | "id": "76f78b47-162b-431c-ab07-92411ea7cbbb", 17 | "metadata": {}, 18 | "outputs": [ 19 | { 20 | "data": { 21 | "text/plain": [ 22 | "datetime.datetime(2024, 4, 19, 20, 28, 0, 71905)" 23 | ] 24 | }, 25 | "execution_count": 18, 26 | "metadata": {}, 27 | "output_type": "execute_result" 28 | } 29 | ], 30 | "source": [ 31 | "from datetime import datetime\n", 32 | "datetime.now()" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 1, 38 | "id": "261233b6-180f-4bb1-a15b-c30bc89bea11", 39 | "metadata": {}, 40 | "outputs": [ 41 | { 42 | "data": { 43 | "text/plain": [ 44 | "" 45 | ] 46 | }, 47 | "execution_count": 1, 48 | "metadata": {}, 49 | "output_type": "execute_result" 50 | } 51 | ], 52 | "source": [ 53 | "import torch\n", 54 | "from torch import nn\n", 55 | "\n", 56 | "torch.manual_seed(0)" 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "id": "ab6e5e2d-72d2-4716-a0b7-ed194ea768a1", 62 | "metadata": {}, 63 | "source": [ 64 | "## forward" 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "id": "2c4a1b66-e0ed-4388-bc35-50bfaa623b5a", 70 | "metadata": {}, 71 | "source": [ 72 | "- randomly zeroes some of the elements of the **input tensor** with probability $p$.\n", 73 | "- the outputs are scaled by a factor of $\\frac1{1-p}$ during training. " 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": 2, 79 | "id": "5f02fe7c-5e20-407b-a19f-8f294ae360fb", 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "m = nn.Dropout(0.4)\n", 84 | "input_ = torch.randn(5, requires_grad=True)" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 14, 90 | "id": "d6938b35-eea4-4df5-821c-950b94e8a0fc", 91 | "metadata": {}, 92 | "outputs": [ 93 | { 94 | "data": { 95 | "text/plain": [ 96 | "tensor([ 1.5410, -0.2934, -2.1788, 0.5684, -1.0845], requires_grad=True)" 97 | ] 98 | }, 99 | "execution_count": 14, 100 | "metadata": {}, 101 | "output_type": "execute_result" 102 | } 103 | ], 104 | "source": [ 105 | "input_" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": 3, 111 | "id": "0dbb3ecb-630a-4f25-b5b8-94436db56429", 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "out = m(input_)" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 6, 121 | "id": "1ee87871-0458-4878-81ee-0284eecd1388", 122 | "metadata": {}, 123 | "outputs": [ 124 | { 125 | "data": { 126 | "text/plain": [ 127 | "tensor([ 0.0000, -0.4890, -0.0000, 0.9474, -1.8075], grad_fn=)" 128 | ] 129 | }, 130 | "execution_count": 6, 131 | "metadata": {}, 132 | "output_type": "execute_result" 133 | } 134 | ], 135 | "source": [ 136 | "out" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": 21, 142 | "id": "18ce8cae-4277-4bdb-964d-f9afa34c7fb5", 143 | "metadata": {}, 144 | "outputs": [ 145 | { 146 | "data": { 147 | "text/plain": [ 148 | "-0.489" 149 | ] 150 | }, 151 | "execution_count": 21, 152 | "metadata": {}, 153 | "output_type": "execute_result" 154 | } 155 | ], 156 | "source": [ 157 | "-0.2934* (1/(1-0.4))" 158 | ] 159 | }, 160 | { 161 | "cell_type": "markdown", 162 | "id": "96c252f6-a1ef-4bf9-9af4-4aad4f637f17", 163 | "metadata": {}, 164 | "source": [ 165 | "## backward" 166 | ] 167 | }, 168 | { 169 | "cell_type": "markdown", 170 | "id": "180ab22f-617d-4a48-90c8-d6d36b2a0de2", 171 | "metadata": {}, 172 | "source": [ 173 | "$$\n", 174 | "\\begin{split}\n", 175 | "y=x * mask \\frac{1}{1-p}\\\\\n", 176 | "y'=mask\\frac{1}{1-p}\n", 177 | "\\end{split}\n", 178 | "$$" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": 7, 184 | "id": "ace7af32-223b-42a7-a630-aa3f4102806b", 185 | "metadata": {}, 186 | "outputs": [], 187 | "source": [ 188 | "y = out.sum()\n", 189 | "y.backward()" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": 8, 195 | "id": "74d6d0bf-94af-45bc-9118-66030d83b0fb", 196 | "metadata": {}, 197 | "outputs": [ 198 | { 199 | "data": { 200 | "text/plain": [ 201 | "tensor([0.0000, 1.6667, 0.0000, 1.6667, 1.6667])" 202 | ] 203 | }, 204 | "execution_count": 8, 205 | "metadata": {}, 206 | "output_type": "execute_result" 207 | } 208 | ], 209 | "source": [ 210 | "input_.grad" 211 | ] 212 | }, 213 | { 214 | "cell_type": "markdown", 215 | "id": "2072e792-a6b1-4553-9dce-8361ece35692", 216 | "metadata": {}, 217 | "source": [ 218 | "## training/eval mode" 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "execution_count": 9, 224 | "id": "086da38a-9bb6-42e5-a6b2-8c13224b7ae6", 225 | "metadata": {}, 226 | "outputs": [ 227 | { 228 | "data": { 229 | "text/plain": [ 230 | "True" 231 | ] 232 | }, 233 | "execution_count": 9, 234 | "metadata": {}, 235 | "output_type": "execute_result" 236 | } 237 | ], 238 | "source": [ 239 | "m.training" 240 | ] 241 | }, 242 | { 243 | "cell_type": "code", 244 | "execution_count": 23, 245 | "id": "ad6449f3-fce2-4d6c-be8d-eb5f8eb2ed7e", 246 | "metadata": {}, 247 | "outputs": [ 248 | { 249 | "data": { 250 | "text/plain": [ 251 | "Dropout(p=0.4, inplace=False)" 252 | ] 253 | }, 254 | "execution_count": 23, 255 | "metadata": {}, 256 | "output_type": "execute_result" 257 | } 258 | ], 259 | "source": [ 260 | "m.train()" 261 | ] 262 | }, 263 | { 264 | "cell_type": "code", 265 | "execution_count": 24, 266 | "id": "d1817900-c7b2-4753-85c3-2dd23df11549", 267 | "metadata": {}, 268 | "outputs": [ 269 | { 270 | "data": { 271 | "text/plain": [ 272 | "Dropout(p=0.4, inplace=False)" 273 | ] 274 | }, 275 | "execution_count": 24, 276 | "metadata": {}, 277 | "output_type": "execute_result" 278 | } 279 | ], 280 | "source": [ 281 | "m.eval()" 282 | ] 283 | }, 284 | { 285 | "cell_type": "code", 286 | "execution_count": 26, 287 | "id": "aa9a27b7-cb67-43dd-91e4-cf88e6ae8bb3", 288 | "metadata": {}, 289 | "outputs": [ 290 | { 291 | "data": { 292 | "text/plain": [ 293 | "False" 294 | ] 295 | }, 296 | "execution_count": 26, 297 | "metadata": {}, 298 | "output_type": "execute_result" 299 | } 300 | ], 301 | "source": [ 302 | "m.training" 303 | ] 304 | }, 305 | { 306 | "cell_type": "code", 307 | "execution_count": 13, 308 | "id": "f8ab03f4-e335-4c8e-a6d2-aac978ad4cca", 309 | "metadata": {}, 310 | "outputs": [ 311 | { 312 | "data": { 313 | "text/plain": [ 314 | "tensor([True, True, True, True, True])" 315 | ] 316 | }, 317 | "execution_count": 13, 318 | "metadata": {}, 319 | "output_type": "execute_result" 320 | } 321 | ], 322 | "source": [ 323 | "m(input_) == input_" 324 | ] 325 | } 326 | ], 327 | "metadata": { 328 | "kernelspec": { 329 | "display_name": "Python 3 (ipykernel)", 330 | "language": "python", 331 | "name": "python3" 332 | }, 333 | "language_info": { 334 | "codemirror_mode": { 335 | "name": "ipython", 336 | "version": 3 337 | }, 338 | "file_extension": ".py", 339 | "mimetype": "text/x-python", 340 | "name": "python", 341 | "nbconvert_exporter": "python", 342 | "pygments_lexer": "ipython3", 343 | "version": "3.10.13" 344 | } 345 | }, 346 | "nbformat": 4, 347 | "nbformat_minor": 5 348 | } 349 | -------------------------------------------------------------------------------- /tutorials/nn_basics/tricks_norms/why_norm.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "3f516499-ac5b-469d-8c9b-ff30c010e97c", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [] 10 | } 11 | ], 12 | "metadata": { 13 | "kernelspec": { 14 | "display_name": "Python 3 (ipykernel)", 15 | "language": "python", 16 | "name": "python3" 17 | }, 18 | "language_info": { 19 | "codemirror_mode": { 20 | "name": "ipython", 21 | "version": 3 22 | }, 23 | "file_extension": ".py", 24 | "mimetype": "text/x-python", 25 | "name": "python", 26 | "nbconvert_exporter": "python", 27 | "pygments_lexer": "ipython3", 28 | "version": "3.10.13" 29 | } 30 | }, 31 | "nbformat": 4, 32 | "nbformat_minor": 5 33 | } 34 | -------------------------------------------------------------------------------- /tutorials/nn_basics/utils/functional.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "58f53a9d", 7 | "metadata": { 8 | "ExecuteTime": { 9 | "end_time": "2023-07-31T13:49:32.773031Z", 10 | "start_time": "2023-07-31T13:49:31.784276Z" 11 | } 12 | }, 13 | "outputs": [], 14 | "source": [ 15 | "import torch.nn.functional as F" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "id": "334c8497", 21 | "metadata": {}, 22 | "source": [ 23 | "## basics" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "id": "f0d092d3", 29 | "metadata": {}, 30 | "source": [ 31 | "- `F.relu`\n", 32 | "- `F.linear`\n", 33 | " - `F.linear(input, weight, bias=None)`\n", 34 | " - input: $x\\in R^{m\\times i}$\n", 35 | " - weight: $W\\in R^{o\\times i}$\n", 36 | " - bias: $b\\in R^{o}$\n", 37 | "\n", 38 | "$$\n", 39 | "y=xW^T+b\n", 40 | "$$\n", 41 | "\n" 42 | ] 43 | } 44 | ], 45 | "metadata": { 46 | "kernelspec": { 47 | "display_name": "Python 3 (ipykernel)", 48 | "language": "python", 49 | "name": "python3" 50 | }, 51 | "language_info": { 52 | "codemirror_mode": { 53 | "name": "ipython", 54 | "version": 3 55 | }, 56 | "file_extension": ".py", 57 | "mimetype": "text/x-python", 58 | "name": "python", 59 | "nbconvert_exporter": "python", 60 | "pygments_lexer": "ipython3", 61 | "version": "3.10.13" 62 | }, 63 | "toc": { 64 | "base_numbering": 1, 65 | "nav_menu": {}, 66 | "number_sections": true, 67 | "sideBar": true, 68 | "skip_h1_title": false, 69 | "title_cell": "Table of Contents", 70 | "title_sidebar": "Contents", 71 | "toc_cell": false, 72 | "toc_position": {}, 73 | "toc_section_display": true, 74 | "toc_window_display": false 75 | } 76 | }, 77 | "nbformat": 4, 78 | "nbformat_minor": 5 79 | } 80 | -------------------------------------------------------------------------------- /tutorials/nn_basics/utils/torch_triangle_mask.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 18, 6 | "id": "29a152de", 7 | "metadata": { 8 | "ExecuteTime": { 9 | "end_time": "2023-10-12T14:27:54.511886Z", 10 | "start_time": "2023-10-12T14:27:54.507302Z" 11 | } 12 | }, 13 | "outputs": [], 14 | "source": [ 15 | "import torch" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "id": "a99751ea", 21 | "metadata": {}, 22 | "source": [ 23 | "## baiscs" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "id": "892c8d9b", 29 | "metadata": {}, 30 | "source": [ 31 | "- `torch.triu`: upper\n", 32 | "- `torch.tril`: lower" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "id": "70372582", 38 | "metadata": {}, 39 | "source": [ 40 | "## cases" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 12, 46 | "id": "58605127", 47 | "metadata": { 48 | "ExecuteTime": { 49 | "end_time": "2023-10-12T14:27:11.134112Z", 50 | "start_time": "2023-10-12T14:27:11.125536Z" 51 | } 52 | }, 53 | "outputs": [ 54 | { 55 | "data": { 56 | "text/plain": [ 57 | "tensor([[-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 58 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 59 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 60 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 61 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 62 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 63 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 64 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf]], device='cuda:0')" 65 | ] 66 | }, 67 | "execution_count": 12, 68 | "metadata": {}, 69 | "output_type": "execute_result" 70 | } 71 | ], 72 | "source": [ 73 | "seq_len = 8\n", 74 | "# mask = torch.full((seq_len, seq_len), float('-inf'), device='cuda:0')\n", 75 | "mask = torch.full((seq_len, seq_len), float('-inf'), device='cuda:0')\n", 76 | "mask" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 13, 82 | "id": "c86c43bf", 83 | "metadata": { 84 | "ExecuteTime": { 85 | "end_time": "2023-10-12T14:27:12.771374Z", 86 | "start_time": "2023-10-12T14:27:12.764122Z" 87 | } 88 | }, 89 | "outputs": [ 90 | { 91 | "data": { 92 | "text/plain": [ 93 | "tensor([[-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 94 | " [0., -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 95 | " [0., 0., -inf, -inf, -inf, -inf, -inf, -inf],\n", 96 | " [0., 0., 0., -inf, -inf, -inf, -inf, -inf],\n", 97 | " [0., 0., 0., 0., -inf, -inf, -inf, -inf],\n", 98 | " [0., 0., 0., 0., 0., -inf, -inf, -inf],\n", 99 | " [0., 0., 0., 0., 0., 0., -inf, -inf],\n", 100 | " [0., 0., 0., 0., 0., 0., 0., -inf]], device='cuda:0')" 101 | ] 102 | }, 103 | "execution_count": 13, 104 | "metadata": {}, 105 | "output_type": "execute_result" 106 | } 107 | ], 108 | "source": [ 109 | "# mask = torch.triu(mask, diagonal=0)\n", 110 | "torch.triu(mask)" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": 19, 116 | "id": "a6cec6aa", 117 | "metadata": { 118 | "ExecuteTime": { 119 | "end_time": "2023-10-12T14:27:59.967120Z", 120 | "start_time": "2023-10-12T14:27:59.958305Z" 121 | }, 122 | "scrolled": true 123 | }, 124 | "outputs": [ 125 | { 126 | "data": { 127 | "text/plain": [ 128 | "tensor([[0., 0., -inf, -inf, -inf, -inf, -inf, -inf],\n", 129 | " [0., 0., 0., -inf, -inf, -inf, -inf, -inf],\n", 130 | " [0., 0., 0., 0., -inf, -inf, -inf, -inf],\n", 131 | " [0., 0., 0., 0., 0., -inf, -inf, -inf],\n", 132 | " [0., 0., 0., 0., 0., 0., -inf, -inf],\n", 133 | " [0., 0., 0., 0., 0., 0., 0., -inf],\n", 134 | " [0., 0., 0., 0., 0., 0., 0., 0.],\n", 135 | " [0., 0., 0., 0., 0., 0., 0., 0.]], device='cuda:0')" 136 | ] 137 | }, 138 | "execution_count": 19, 139 | "metadata": {}, 140 | "output_type": "execute_result" 141 | } 142 | ], 143 | "source": [ 144 | "torch.triu(mask, diagonal=2)" 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": 23, 150 | "id": "da9d38de", 151 | "metadata": { 152 | "ExecuteTime": { 153 | "end_time": "2023-10-12T14:29:41.068374Z", 154 | "start_time": "2023-10-12T14:29:41.059374Z" 155 | } 156 | }, 157 | "outputs": [ 158 | { 159 | "data": { 160 | "text/plain": [ 161 | "tensor([[-inf, -inf, -inf, 0., 0., 0., 0., 0.],\n", 162 | " [-inf, -inf, -inf, -inf, 0., 0., 0., 0.],\n", 163 | " [-inf, -inf, -inf, -inf, -inf, 0., 0., 0.],\n", 164 | " [-inf, -inf, -inf, -inf, -inf, -inf, 0., 0.],\n", 165 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, 0.],\n", 166 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 167 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 168 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf]], device='cuda:0')" 169 | ] 170 | }, 171 | "execution_count": 23, 172 | "metadata": {}, 173 | "output_type": "execute_result" 174 | } 175 | ], 176 | "source": [ 177 | "torch.tril(mask, diagonal=2)" 178 | ] 179 | } 180 | ], 181 | "metadata": { 182 | "kernelspec": { 183 | "display_name": "Python 3 (ipykernel)", 184 | "language": "python", 185 | "name": "python3" 186 | }, 187 | "language_info": { 188 | "codemirror_mode": { 189 | "name": "ipython", 190 | "version": 3 191 | }, 192 | "file_extension": ".py", 193 | "mimetype": "text/x-python", 194 | "name": "python", 195 | "nbconvert_exporter": "python", 196 | "pygments_lexer": "ipython3", 197 | "version": "3.10.13" 198 | }, 199 | "toc": { 200 | "base_numbering": 1, 201 | "nav_menu": {}, 202 | "number_sections": true, 203 | "sideBar": true, 204 | "skip_h1_title": false, 205 | "title_cell": "Table of Contents", 206 | "title_sidebar": "Contents", 207 | "toc_cell": false, 208 | "toc_position": {}, 209 | "toc_section_display": true, 210 | "toc_window_display": false 211 | } 212 | }, 213 | "nbformat": 4, 214 | "nbformat_minor": 5 215 | } 216 | -------------------------------------------------------------------------------- /tutorials/nn_basics/utils/torch_utils_scripts.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 4, 6 | "id": "85662644", 7 | "metadata": { 8 | "ExecuteTime": { 9 | "end_time": "2023-11-24T13:17:04.814792Z", 10 | "start_time": "2023-11-24T13:17:03.320380Z" 11 | } 12 | }, 13 | "outputs": [], 14 | "source": [ 15 | "import torch\n", 16 | "from torch import nn" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "id": "9f4064f5-9309-476e-be5e-99a09aed8243", 22 | "metadata": {}, 23 | "source": [ 24 | "### flush" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "id": "d21b51e7-565e-489d-a114-9cb4925bd693", 30 | "metadata": {}, 31 | "source": [ 32 | "```\n", 33 | "def flush_model(self):\n", 34 | " # delete llm from gpu to save GPU memory\n", 35 | " if self.model is not None:\n", 36 | " self.model = None\n", 37 | " gc.collect()\n", 38 | " torch.cuda.empty_cache()\n", 39 | "```" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "id": "21f9626b-e5d0-432f-a2aa-510e7d655ad8", 45 | "metadata": {}, 46 | "source": [ 47 | "### parameters" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 3, 53 | "id": "b0c4f453", 54 | "metadata": { 55 | "ExecuteTime": { 56 | "end_time": "2023-11-24T13:16:53.035351Z", 57 | "start_time": "2023-11-24T13:16:53.029917Z" 58 | } 59 | }, 60 | "outputs": [], 61 | "source": [ 62 | "def num_parameters(model):\n", 63 | " return sum(p.numel() for p in model.parameters() if p.requires_grad)" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": 5, 69 | "id": "774de0cc", 70 | "metadata": { 71 | "ExecuteTime": { 72 | "end_time": "2023-11-24T13:17:12.603280Z", 73 | "start_time": "2023-11-24T13:17:12.596196Z" 74 | } 75 | }, 76 | "outputs": [ 77 | { 78 | "data": { 79 | "text/plain": [ 80 | "220" 81 | ] 82 | }, 83 | "execution_count": 5, 84 | "metadata": {}, 85 | "output_type": "execute_result" 86 | } 87 | ], 88 | "source": [ 89 | "m = nn.Linear(10, 20)\n", 90 | "num_parameters(m)" 91 | ] 92 | } 93 | ], 94 | "metadata": { 95 | "kernelspec": { 96 | "display_name": "Python 3 (ipykernel)", 97 | "language": "python", 98 | "name": "python3" 99 | }, 100 | "language_info": { 101 | "codemirror_mode": { 102 | "name": "ipython", 103 | "version": 3 104 | }, 105 | "file_extension": ".py", 106 | "mimetype": "text/x-python", 107 | "name": "python", 108 | "nbconvert_exporter": "python", 109 | "pygments_lexer": "ipython3", 110 | "version": "3.10.13" 111 | }, 112 | "toc": { 113 | "base_numbering": 1, 114 | "nav_menu": {}, 115 | "number_sections": true, 116 | "sideBar": true, 117 | "skip_h1_title": false, 118 | "title_cell": "Table of Contents", 119 | "title_sidebar": "Contents", 120 | "toc_cell": false, 121 | "toc_position": {}, 122 | "toc_section_display": true, 123 | "toc_window_display": false 124 | } 125 | }, 126 | "nbformat": 4, 127 | "nbformat_minor": 5 128 | } 129 | -------------------------------------------------------------------------------- /tutorials/nvidia_chat_with_rtx.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "ac12e0ef", 6 | "metadata": {}, 7 | "source": [ 8 | "- references\n", 9 | " - https://www.nvidia.com/en-us/ai-on-rtx/chat-with-rtx-generative-ai/\n", 10 | " - https://github.com/NVIDIA/trt-llm-rag-windows" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 1, 16 | "id": "195f1592", 17 | "metadata": { 18 | "ExecuteTime": { 19 | "end_time": "2024-02-15T02:29:34.494933Z", 20 | "start_time": "2024-02-15T02:29:34.485675Z" 21 | } 22 | }, 23 | "outputs": [], 24 | "source": [ 25 | "from IPython.display import Image" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "id": "361bd9e3", 31 | "metadata": {}, 32 | "source": [ 33 | "## 介绍下我的系统环境" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "id": "317a61b9", 39 | "metadata": {}, 40 | "source": [ 41 | "- 双系统\n", 42 | " - 一个 2T 的固态硬盘\n", 43 | " \n", 44 | " ```\n", 45 | " sudo apt install smartmontools -y\n", 46 | " sudo smartctl -a /dev/nvme0n1 | grep -i model\n", 47 | " ```" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 3, 53 | "id": "5b951bb1", 54 | "metadata": { 55 | "ExecuteTime": { 56 | "end_time": "2024-02-14T13:46:42.143816Z", 57 | "start_time": "2024-02-14T13:46:42.133527Z" 58 | } 59 | }, 60 | "outputs": [ 61 | { 62 | "data": { 63 | "text/html": [ 64 | "" 65 | ], 66 | "text/plain": [ 67 | "" 68 | ] 69 | }, 70 | "execution_count": 3, 71 | "metadata": {}, 72 | "output_type": "execute_result" 73 | } 74 | ], 75 | "source": [ 76 | "Image(url='../imgs/990_pro.png', width=500)" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 6, 82 | "id": "1437e2d6", 83 | "metadata": { 84 | "ExecuteTime": { 85 | "end_time": "2024-02-14T13:48:38.295898Z", 86 | "start_time": "2024-02-14T13:48:38.286073Z" 87 | }, 88 | "scrolled": true 89 | }, 90 | "outputs": [ 91 | { 92 | "data": { 93 | "text/html": [ 94 | "" 95 | ], 96 | "text/plain": [ 97 | "" 98 | ] 99 | }, 100 | "execution_count": 6, 101 | "metadata": {}, 102 | "output_type": "execute_result" 103 | } 104 | ], 105 | "source": [ 106 | "# /media/whaow: windows 系统位置\n", 107 | "# sudo mount /dev/nvme0n1p3 /media/whaow\n", 108 | "Image(url='../imgs/nvme_partition.png', width=500)" 109 | ] 110 | }, 111 | { 112 | "cell_type": "markdown", 113 | "id": "c636004b", 114 | "metadata": {}, 115 | "source": [ 116 | "## 关于 nvidia-chat-with-rtx" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": 3, 122 | "id": "8f0d64b3", 123 | "metadata": { 124 | "ExecuteTime": { 125 | "end_time": "2024-02-15T02:29:44.040187Z", 126 | "start_time": "2024-02-15T02:29:44.035092Z" 127 | } 128 | }, 129 | "outputs": [ 130 | { 131 | "data": { 132 | "text/html": [ 133 | "" 134 | ], 135 | "text/plain": [ 136 | "" 137 | ] 138 | }, 139 | "execution_count": 3, 140 | "metadata": {}, 141 | "output_type": "execute_result" 142 | } 143 | ], 144 | "source": [ 145 | "Image(url='../imgs/nv-chat-rtx-version.png', width=400)" 146 | ] 147 | }, 148 | { 149 | "cell_type": "markdown", 150 | "id": "3951c465", 151 | "metadata": {}, 152 | "source": [ 153 | "- 最大的特色是傻瓜式安装，以及本地化服务，离线可用；\n", 154 | " - 硬件需求较高（主要是需要一张较高配置的显卡，以及未来更大更快的存储空间用于存放格式各样的大模型）\n", 155 | " - 此外还有网络问题，嗨。\n", 156 | " - 类似 sd-webui；消费级显卡未来的需求可想而知；\n", 157 | " - 对于 nvidia，深度绑定软硬件服务；\n", 158 | "- 目前支持的模型是 Mistral 以及 Llama 4bit 量化，侧面也说明了目前较受认可的开源LLMs；\n", 159 | " - Llama 13B INT4 Engine\n", 160 | " - Mistral 7B INT4" 161 | ] 162 | }, 163 | { 164 | "cell_type": "markdown", 165 | "id": "07bedee7", 166 | "metadata": {}, 167 | "source": [ 168 | "### 启动或者安装问题" 169 | ] 170 | }, 171 | { 172 | "cell_type": "markdown", 173 | "id": "cbdebd23", 174 | "metadata": {}, 175 | "source": [ 176 | "- 安装时不建议修改默认的安装目录" 177 | ] 178 | }, 179 | { 180 | "cell_type": "markdown", 181 | "id": "e21c1905", 182 | "metadata": {}, 183 | "source": [ 184 | "`ValueError: When localhost is not accessible, a shareable link must be created. Please set share=True.`" 185 | ] 186 | }, 187 | { 188 | "cell_type": "markdown", 189 | "id": "58a85e5b", 190 | "metadata": {}, 191 | "source": [ 192 | "- 设置环境变量\n", 193 | " - `http_proxy`: `http://127.0.0.1:7890`\n", 194 | " - `https_proxy`: `http://127.0.0.1:7890`\n", 195 | "- 修改 app.py（`C:\\Users\\username\\AppData\\Local\\NVIDIA\\ChatWithRTX\\RAG\\trt-llm-rag-windows-main\\app.py`）\n", 196 | " - 添加一行 `os.environ['no_proxy'] = \"localhost,127.0.0.1,::1\"`" 197 | ] 198 | } 199 | ], 200 | "metadata": { 201 | "kernelspec": { 202 | "display_name": "Python 3 (ipykernel)", 203 | "language": "python", 204 | "name": "python3" 205 | }, 206 | "language_info": { 207 | "codemirror_mode": { 208 | "name": "ipython", 209 | "version": 3 210 | }, 211 | "file_extension": ".py", 212 | "mimetype": "text/x-python", 213 | "name": "python", 214 | "nbconvert_exporter": "python", 215 | "pygments_lexer": "ipython3", 216 | "version": "3.10.13" 217 | }, 218 | "toc": { 219 | "base_numbering": 1, 220 | "nav_menu": {}, 221 | "number_sections": true, 222 | "sideBar": true, 223 | "skip_h1_title": false, 224 | "title_cell": "Table of Contents", 225 | "title_sidebar": "Contents", 226 | "toc_cell": false, 227 | "toc_position": {}, 228 | "toc_section_display": true, 229 | "toc_window_display": false 230 | } 231 | }, 232 | "nbformat": 4, 233 | "nbformat_minor": 5 234 | } 235 | -------------------------------------------------------------------------------- /tutorials/oai/account_login.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "1fe193bf", 6 | "metadata": {}, 7 | "source": [ 8 | "- https://help.openai.com/\n", 9 | " - chat 申诉\n", 10 | "- https://zhuanlan.zhihu.com/p/621402195" 11 | ] 12 | } 13 | ], 14 | "metadata": { 15 | "kernelspec": { 16 | "display_name": "Python 3 (ipykernel)", 17 | "language": "python", 18 | "name": "python3" 19 | }, 20 | "language_info": { 21 | "codemirror_mode": { 22 | "name": "ipython", 23 | "version": 3 24 | }, 25 | "file_extension": ".py", 26 | "mimetype": "text/x-python", 27 | "name": "python", 28 | "nbconvert_exporter": "python", 29 | "pygments_lexer": "ipython3", 30 | "version": "3.10.13" 31 | }, 32 | "toc": { 33 | "base_numbering": 1, 34 | "nav_menu": {}, 35 | "number_sections": true, 36 | "sideBar": true, 37 | "skip_h1_title": false, 38 | "title_cell": "Table of Contents", 39 | "title_sidebar": "Contents", 40 | "toc_cell": false, 41 | "toc_position": {}, 42 | "toc_section_display": true, 43 | "toc_window_display": false 44 | } 45 | }, 46 | "nbformat": 4, 47 | "nbformat_minor": 5 48 | } 49 | -------------------------------------------------------------------------------- /tutorials/oai/async_demo.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import time 3 | 4 | async def make_coffee(): 5 | print("开始煮咖啡...") 6 | await asyncio.sleep(3) # 模拟煮咖啡的时间 7 | print("咖啡准备好了！") 8 | return "一杯咖啡" 9 | 10 | async def make_toast(): 11 | print("开始烤面包...") 12 | await asyncio.sleep(2) # 模拟烤面包的时间 13 | print("面包烤好了！") 14 | return "一片吐司" 15 | 16 | async def main(): 17 | # 同时开始准备咖啡和面包 18 | t0 = time.time() 19 | coffee_task = asyncio.create_task(make_coffee()) 20 | toast_task = asyncio.create_task(make_toast()) 21 | 22 | # 等待两个任务都完成 23 | coffee, toast = await asyncio.gather(coffee_task, toast_task) 24 | print(f"早餐准备完成：{coffee}和{toast}, 用时: {time.time() - t0:.2f}s") 25 | 26 | # 运行程序 27 | asyncio.run(main()) -------------------------------------------------------------------------------- /tutorials/oai/async_oai.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "4f3aebde-2527-4e9a-8121-a54acfc401c4", 6 | "metadata": {}, 7 | "source": [ 8 | "https://www.unite.ai/asynchronous-llm-api-calls-in-python-a-comprehensive-guide/\n", 9 | "- A guide to make asynchronous calls to OpenAI API with rate limiting.\n", 10 | "- Backoff is a python library that provides decorators that can used to wrap a function and retry until a specified condition is met." 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "id": "78eca8d0-6984-4d09-8481-b61037da20f5", 16 | "metadata": {}, 17 | "source": [ 18 | "### Asyncio" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "id": "c6151053-9157-46c6-bac9-2026f2274cd9", 24 | "metadata": {}, 25 | "source": [ 26 | "- Asyncio is a Python library that provides asynchronous I/O.\n", 27 | " - 协程（coroutine）：使用async def定义的函数\n", 28 | " - 可以在执行过程中暂停和恢复\n", 29 | " - 必须使用await来调用\n", 30 | " - await 关键字\n", 31 | " - 用于暂停协程执行，等待异步操作完成\n", 32 | " - 只能在async函数内使用\n", 33 | " - asyncio.run\n", 34 | " - 执行 async def 定义的事件循环（event loop）\n", 35 | " - asyncio.gather()同时处理多个异步任务\n", 36 | "- It relieves you of the burden of worrying about **locks and threads** when writing concurrent programs.\n", 37 | "- The foundation of asyncio is the idea of **coroutines**, or functions that can be interrupted and then resumed." 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "id": "2efff8fe-4aa2-4164-a78c-581b1ae39359", 43 | "metadata": {}, 44 | "source": [ 45 | "- io bound vs. cpu bound\n", 46 | " - I/O 操作指的是程序需要等待外部资源完成其工作的操作。这些外部资源通常比 CPU 慢得多。当程序执行 I/O 操作时，CPU 常常处于空闲等待状态。\n", 47 | " - time.sleep() (同步) 和 asyncio.sleep() (异步) 通常被归类为 I/O 操作，或者更准确地说，它们表现得像 I/O 操作。\n", 48 | " - 调用大型语言模型 (LLM) API 的操作绝对是 I/O 密集型 (I/O Bound) 操作。\n", 49 | " - 网络通信是核心：\n", 50 | " - 当你调用一个 LLM API 时，你的程序需要通过网络将你的请求（例如，prompt、参数等）发送到托管 LLM 模型的远程服务器。\n", 51 | " - 然后，你的程序必须等待远程服务器处理这个请求。这个处理过程可能涉及到模型加载、计算、生成文本等，这部分是在远程服务器上发生的，而不是在你的本地机器上。\n", 52 | " - 最后，远程服务器通过网络将响应（生成的文本、错误信息等）发送回你的程序。\n", 53 | " - 等待时间远大于本地计算时间：\n", 54 | " - 网络延迟 (Latency)：数据在互联网上传输需要时间。\n", 55 | " - 服务器处理时间：LLM 模型本身可能很大，推理计算也需要时间，尤其是在高负载情况下，服务器可能还需要排队处理你的请求。\n", 56 | " - 数据传输时间：请求和响应的数据量也可能影响传输时间。\n", 57 | " - 在整个 API 调用过程中，你的本地程序大部分时间都花在等待网络传输完成和远程服务器响应上。本地 CPU 在这段时间内基本是空闲的，或者只做一些非常轻量级的工作（比如序列化/反序列化数据）。\n", 58 | " - CPU 利用率低（本地）：\n", 59 | " - 当你的代码执行 `response = await client.chat.completions.create(...)` (或者类似的同步调用 `response = client.chat.completions.create(...)`) 时，你的程序会暂停执行，等待网络操作完成。\n", 60 | " - 在这段等待期间，你的本地 CPU 可以被操作系统或事件循环用于执行其他任务（如果是异步调用且有其他任务）或处于空闲状态。\n", 61 | " - CPU 密集型操作指的是程序主要时间消耗在执行计算任务上，CPU 持续高速运转。\n", 62 | " - CPU密集型任务不适合异步编程的原因\n", 63 | " - 异步编程的优势在于I/O等待时的切换\n", 64 | "\n", 65 | "```python\n", 66 | "# I/O密集型任务 - 适合异步\n", 67 | "async def io_task():\n", 68 | " await asyncio.sleep(1) # 在等待I/O时，可以切换到其他任务\n", 69 | " return \"完成\"\n", 70 | "\n", 71 | "# CPU密集型任务 - 不适合异步\n", 72 | "async def cpu_task():\n", 73 | " result = 0\n", 74 | " for i in range(1000000000): # CPU计算，无法切换\n", 75 | " result += i\n", 76 | " return result\n", 77 | "```" 78 | ] 79 | }, 80 | { 81 | "cell_type": "markdown", 82 | "id": "9868ffed-6e3f-4bb1-987a-df10d06b1cad", 83 | "metadata": {}, 84 | "source": [ 85 | "### Asynchronous LLM API Calls" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": null, 91 | "id": "3e72da5d-b6a5-4873-8f9c-11c888cba418", 92 | "metadata": {}, 93 | "outputs": [], 94 | "source": [ 95 | "from dotenv import load_dotenv, find_dotenv\n", 96 | "assert load_dotenv(find_dotenv())" 97 | ] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "id": "d7b73859-3212-4449-a2d2-2a94ca6c887d", 102 | "metadata": {}, 103 | "source": [ 104 | "```python\n", 105 | "import asyncio\n", 106 | "import aiohttp\n", 107 | "from openai import AsyncOpenAI\n", 108 | "\n", 109 | "async def generate_text(prompt, client):\n", 110 | " response = await client.chat.completions.create(\n", 111 | " model=\"gpt-4.1-mini-2025-04-14\",\n", 112 | " messages=[{\"role\": \"user\", \"content\": prompt}]\n", 113 | " )\n", 114 | " return response.choices[0].message.content\n", 115 | " \n", 116 | "async def main():\n", 117 | " prompts = [\n", 118 | " \"Explain quantum computing in simple terms.\",\n", 119 | " \"Write a haiku about artificial intelligence.\",\n", 120 | " \"Describe the process of photosynthesis.\"\n", 121 | " ]\n", 122 | " \n", 123 | " async with AsyncOpenAI() as client:\n", 124 | " tasks = [generate_text(prompt, client) for prompt in prompts]\n", 125 | " results = await asyncio.gather(*tasks)\n", 126 | " \n", 127 | " for prompt, result in zip(prompts, results):\n", 128 | " print(f\"Prompt: {prompt}\\nResponse: {result}\\n\")\n", 129 | " \n", 130 | "asyncio.run(main())\n", 131 | "```" 132 | ] 133 | }, 134 | { 135 | "cell_type": "markdown", 136 | "id": "9f13e52f-c17d-41cb-88fc-1eb11f0fff97", 137 | "metadata": {}, 138 | "source": [ 139 | "### Batching and Concurrency Control" 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "id": "77d8e98e-e215-4686-b53f-a5c9160ec66e", 145 | "metadata": {}, 146 | "source": [ 147 | "```python\n", 148 | "async def process_batch(batch, client):\n", 149 | " responses = await asyncio.gather(*[\n", 150 | " client.chat.completions.create(\n", 151 | " model=\"gpt-4.1-mini-2025-04-14\",\n", 152 | " messages=[{\"role\": \"user\", \"content\": prompt}]\n", 153 | " ) for prompt in batch\n", 154 | " ])\n", 155 | " return [response.choices[0].message.content for response in responses]\n", 156 | " \n", 157 | "async def main():\n", 158 | " prompts = [f\"Tell me a fact about number {i}\" for i in range(100)]\n", 159 | " batch_size = 10\n", 160 | " \n", 161 | " async with AsyncOpenAI() as client:\n", 162 | " results = []\n", 163 | " for i in range(0, len(prompts), batch_size):\n", 164 | " batch = prompts[i:i+batch_size]\n", 165 | " batch_results = await process_batch(batch, client)\n", 166 | " results.extend(batch_results)\n", 167 | " \n", 168 | " for prompt, result in zip(prompts, results):\n", 169 | " print(f\"Prompt: {prompt}\\nResponse: {result}\\n\")\n", 170 | "asyncio.run(main())\n", 171 | "```" 172 | ] 173 | }, 174 | { 175 | "cell_type": "markdown", 176 | "id": "0b84753c-b552-4cbe-8c03-6041ab50377d", 177 | "metadata": {}, 178 | "source": [ 179 | "### semaphore" 180 | ] 181 | }, 182 | { 183 | "cell_type": "markdown", 184 | "id": "fb59957d-e904-475a-b5b2-0891cf283e99", 185 | "metadata": {}, 186 | "source": [ 187 | "- asyncio.Semaphore：控制并发；\n", 188 | " - 用于限制同时运行的协程数量，从而防止资源过载或满足某些并发限制需求。\n", 189 | "- asyncio.as_completed：获取实时结果；\n", 190 | " - asyncio.as_completed 会在每个任务完成时生成一个迭代器，使您能够立即处理完成的任务，而不必等待所有任务结束。" 191 | ] 192 | }, 193 | { 194 | "cell_type": "markdown", 195 | "id": "b0eb31c0-b55e-4478-8af1-07564e99a4de", 196 | "metadata": {}, 197 | "source": [ 198 | "```python\n", 199 | "import asyncio\n", 200 | "from openai import AsyncOpenAI\n", 201 | "\n", 202 | "async def generate_text(prompt, client, semaphore):\n", 203 | " async with semaphore:\n", 204 | " response = await client.chat.completions.create(\n", 205 | " model=\"gpt-4.1-mini-2025-04-14\",\n", 206 | " messages=[{\"role\": \"user\", \"content\": prompt}]\n", 207 | " )\n", 208 | " return response.choices[0].message.content\n", 209 | " \n", 210 | "async def main():\n", 211 | " prompts = [f\"Tell me a fact about number {i}\" for i in range(100)]\n", 212 | " max_concurrent_requests = 5\n", 213 | " semaphore = asyncio.Semaphore(max_concurrent_requests)\n", 214 | " \n", 215 | " async with AsyncOpenAI() as client:\n", 216 | " tasks = [generate_text(prompt, client, semaphore) for prompt in prompts]\n", 217 | " results = await asyncio.gather(*tasks)\n", 218 | " \n", 219 | " for prompt, result in zip(prompts, results):\n", 220 | " print(f\"Prompt: {prompt}\\nResponse: {result}\\n\")\n", 221 | "asyncio.run(main())\n", 222 | "```" 223 | ] 224 | } 225 | ], 226 | "metadata": { 227 | "kernelspec": { 228 | "display_name": "Python 3 (ipykernel)", 229 | "language": "python", 230 | "name": "python3" 231 | }, 232 | "language_info": { 233 | "codemirror_mode": { 234 | "name": "ipython", 235 | "version": 3 236 | }, 237 | "file_extension": ".py", 238 | "mimetype": "text/x-python", 239 | "name": "python", 240 | "nbconvert_exporter": "python", 241 | "pygments_lexer": "ipython3", 242 | "version": "3.10.0" 243 | } 244 | }, 245 | "nbformat": 4, 246 | "nbformat_minor": 5 247 | } 248 | -------------------------------------------------------------------------------- /tutorials/oai/async_scripts/Semaphore.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | async def worker(name, semaphore): 4 | async with semaphore: 5 | print(f"{name} 获取到了信号量") 6 | await asyncio.sleep(2) # 模拟一个需要时间的操作 7 | print(f"{name} 释放了信号量") 8 | 9 | async def main(): 10 | # 创建一个信号量，最多允许3个协程同时运行 11 | semaphore = asyncio.Semaphore(3) 12 | 13 | tasks = [worker(f"任务{i}", semaphore) for i in range(10)] 14 | await asyncio.gather(*tasks) 15 | 16 | if __name__ == "__main__": 17 | asyncio.run(main()) 18 | -------------------------------------------------------------------------------- /tutorials/oai/async_scripts/async_call.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import openai 3 | from openai import AsyncOpenAI 4 | import backoff 5 | from dotenv import load_dotenv, find_dotenv 6 | import os 7 | 8 | assert load_dotenv(find_dotenv()) 9 | 10 | client = AsyncOpenAI(api_key=os.getenv('OPENAI_API_KEY')) 11 | 12 | @backoff.on_exception(backoff.expo, openai.RateLimitError) 13 | async def make_api_call_to_gpt(prompt, model="gpt-4o-mini"): 14 | messages = [{"role": "user", "content": prompt}] 15 | response = await client.chat.completions.create( 16 | model=model, 17 | messages=messages, 18 | temperature=0, 19 | ) 20 | 21 | return response.choices[0].message.content 22 | 23 | async def main(): 24 | prompts = ["What model are you?", 25 | "How many Rs in strawberry?", 26 | "Which is bigger, 0.9 or 0.11?" 27 | ] 28 | 29 | # Create a list to store the results of asynchronous calls 30 | results = [] 31 | 32 | # Asynchronously call the function for each prompt 33 | tasks = [make_api_call_to_gpt(prompt) for prompt in prompts] 34 | print(tasks) 35 | results = await asyncio.gather(*tasks) 36 | for prompt, result in zip(prompts, results): 37 | print(prompt) 38 | print(result) 39 | print('====================') 40 | 41 | asyncio.run(main()) 42 | -------------------------------------------------------------------------------- /tutorials/oai/async_scripts/async_call_semaphore.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import openai 3 | from openai import AsyncOpenAI 4 | import backoff 5 | from dotenv import load_dotenv, find_dotenv 6 | import os 7 | 8 | assert load_dotenv(find_dotenv()) 9 | 10 | client = AsyncOpenAI(api_key=os.getenv('OPENAI_API_KEY')) 11 | 12 | @backoff.on_exception(backoff.expo, openai.RateLimitError) 13 | async def make_api_call_to_gpt(prompt, model="gpt-4o-mini"): 14 | messages = [{"role": "user", "content": prompt}] 15 | response = await client.chat.completions.create( 16 | model=model, 17 | messages=messages, 18 | temperature=0, 19 | ) 20 | 21 | return response.choices[0].message.content 22 | 23 | async def make_api_call_with_semaphore(prompt, semaphore, model="gpt-4o-mini"): 24 | async with semaphore: 25 | result = await make_api_call_to_gpt(prompt, model) 26 | return prompt, result 27 | 28 | async def main(): 29 | prompts = [ 30 | "What model are you?", 31 | "How many Rs in strawberry?", 32 | "Which is bigger, 0.9 or 0.11?", 33 | # 假设这里有更多的 prompts 34 | ] 35 | 36 | # 设置最大并发数，例如 5 37 | max_concurrent_requests = 5 38 | semaphore = asyncio.Semaphore(max_concurrent_requests) 39 | 40 | # 创建任务列表，包含信号量控制 41 | tasks = [ 42 | make_api_call_with_semaphore(prompt, semaphore) 43 | for prompt in prompts 44 | ] 45 | 46 | # 使用 asyncio.as_completed 来实时处理完成的任务 47 | for coro in asyncio.as_completed(tasks): 48 | prompt, result = await coro 49 | print(f"Prompt: {prompt}") 50 | print(f"Response: {result}") 51 | print('====================') 52 | 53 | # 如果需要等待所有任务完成，可以使用以下方法 54 | # results = await asyncio.gather(*tasks) 55 | # for prompt, result in results: 56 | # print(prompt) 57 | # print(result) 58 | # print('====================') 59 | 60 | if __name__ == "__main__": 61 | asyncio.run(main()) 62 | -------------------------------------------------------------------------------- /tutorials/oai/async_scripts/async_llm_call.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import aiohttp 3 | from dotenv import load_dotenv, find_dotenv 4 | from openai import AsyncOpenAI 5 | 6 | async def generate_text(prompt, client): 7 | response = await client.chat.completions.create( 8 | model="gpt-4.1-mini-2025-04-14", 9 | messages=[{"role": "user", "content": prompt}] 10 | ) 11 | return response.choices[0].message.content 12 | 13 | async def process_batch(batch, client): 14 | responses = await asyncio.gather(*[ 15 | client.chat.completions.create( 16 | model="gpt-3.5-turbo", 17 | messages=[{"role": "user", "content": prompt}] 18 | ) for prompt in batch 19 | ]) 20 | return [response.choices[0].message.content for response in responses] 21 | 22 | async def main(): 23 | prompts = [f"Tell me a fact about number {i}" for i in range(100)] 24 | batch_size = 10 25 | 26 | async with AsyncOpenAI() as client: 27 | results = [] 28 | for i in range(0, len(prompts), batch_size): 29 | batch = prompts[i:i+batch_size] 30 | batch_results = await process_batch(batch, client) 31 | results.extend(batch_results) 32 | 33 | for prompt, result in zip(prompts, results): 34 | print(f"Prompt: {prompt}\nResponse: {result}\n") 35 | asyncio.run(main()) 36 | 37 | 38 | async def main(): 39 | prompts = [ 40 | "Explain quantum computing in simple terms.", 41 | "Write a haiku about artificial intelligence.", 42 | "Describe the process of photosynthesis." 43 | ] 44 | 45 | async with AsyncOpenAI() as client: 46 | tasks = [generate_text(prompt, client) for prompt in prompts] 47 | results = await asyncio.gather(*tasks) 48 | 49 | for prompt, result in zip(prompts, results): 50 | print(f"Prompt: {prompt}\nResponse: {result}\n") 51 | 52 | if __name__ == '__main__': 53 | assert load_dotenv(find_dotenv()) 54 | asyncio.run(main()) -------------------------------------------------------------------------------- /tutorials/oai/async_scripts/io_cpu.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import time 3 | 4 | # I/O密集型任务示例 5 | async def io_intensive(): 6 | print(f"开始I/O任务 {time.strftime('%H:%M:%S')}") 7 | await asyncio.sleep(2) # 模拟I/O操作 8 | print(f"完成I/O任务 {time.strftime('%H:%M:%S')}") 9 | 10 | # CPU密集型任务示例 11 | async def cpu_intensive(): 12 | print(f"开始CPU任务 {time.strftime('%H:%M:%S')}") 13 | for _ in range(50000000): # 执行大量计算 14 | _ = _ ** 2 15 | print(f"完成CPU任务 {time.strftime('%H:%M:%S')}") 16 | 17 | async def main(): 18 | # I/O密集型任务并发执行 19 | print("测试I/O密集型任务:") 20 | await asyncio.gather( 21 | io_intensive(), 22 | io_intensive(), 23 | io_intensive() 24 | ) # 总耗时约2秒 25 | 26 | print("\n测试CPU密集型任务:") 27 | await asyncio.gather( 28 | cpu_intensive(), 29 | cpu_intensive(), 30 | cpu_intensive() 31 | ) # 总耗时是三个任务的总和 32 | 33 | asyncio.run(main()) -------------------------------------------------------------------------------- /tutorials/oai/gpt_openai_生态.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "26a59777", 6 | "metadata": {}, 7 | "source": [ 8 | "## GPTs" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "35933c8e", 14 | "metadata": {}, 15 | "source": [ 16 | "- `site:chat.openai.com/g`\n", 17 | " - 谷歌搜索 gpts\n", 18 | "- GPTs\n", 19 | " - researchGPT：https://chat.openai.com/g/g-NgAcklHd8-researchgpt-official\n", 20 | " - 润色GPT：https://chat.openai.com/g/g-VX52iRD3r-ai-paper-polisher-pro" 21 | ] 22 | } 23 | ], 24 | "metadata": { 25 | "kernelspec": { 26 | "display_name": "Python 3 (ipykernel)", 27 | "language": "python", 28 | "name": "python3" 29 | }, 30 | "language_info": { 31 | "codemirror_mode": { 32 | "name": "ipython", 33 | "version": 3 34 | }, 35 | "file_extension": ".py", 36 | "mimetype": "text/x-python", 37 | "name": "python", 38 | "nbconvert_exporter": "python", 39 | "pygments_lexer": "ipython3", 40 | "version": "3.10.13" 41 | }, 42 | "toc": { 43 | "base_numbering": 1, 44 | "nav_menu": {}, 45 | "number_sections": true, 46 | "sideBar": true, 47 | "skip_h1_title": false, 48 | "title_cell": "Table of Contents", 49 | "title_sidebar": "Contents", 50 | "toc_cell": false, 51 | "toc_position": {}, 52 | "toc_section_display": true, 53 | "toc_window_display": false 54 | } 55 | }, 56 | "nbformat": 4, 57 | "nbformat_minor": 5 58 | } 59 | -------------------------------------------------------------------------------- /tutorials/oai/logP_supp.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "1547c023-b190-463f-9d3d-f0b0a4d71a39", 6 | "metadata": {}, 7 | "source": [ 8 | "- https://gist.github.com/vgel/55b845833af9f189ae1f834d5f7da8c3\n", 9 | " - logprob_inconsistency.py\n", 10 | " \n", 11 | "\n", 12 | "```python\n", 13 | "responses = []\n", 14 | "for _ in range(2):\n", 15 | " resp = client.chat.completions.create(\n", 16 | " model=model,\n", 17 | " messages=[\n", 18 | " {\"role\": \"user\", \"content\": question},\n", 19 | " ],\n", 20 | " temperature=1.0,\n", 21 | " max_tokens=3,\n", 22 | " logprobs=True,\n", 23 | " top_logprobs=3,\n", 24 | " )\n", 25 | " print(f\"{model} trial {_ + 1} chosen tokens:\", resp.choices[0].message.content)\n", 26 | " responses.append(resp.choices[0].logprobs.content)\n", 27 | "```" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "id": "dd2b9e4a-a215-41ac-bca5-f6e5a169e154", 33 | "metadata": {}, 34 | "source": [ 35 | "$$\n", 36 | "\\exp(\\log p)=p\n", 37 | "$$" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "id": "652a4147-70a0-4b01-8d1e-b85fda19ff1c", 43 | "metadata": {}, 44 | "source": [ 45 | "```\n", 46 | "gpt-4o-mini trial 1 chosen tokens: AABBA (A|AB|BA)\n", 47 | "gpt-4o-mini trial 2 chosen tokens: ABBAAB (AB|BA|AB)\n", 48 | "gpt-4o-mini trial 1 // gpt-4o-mini trial 2\n", 49 | "-----------------------------------//-----------------------------------\n", 50 | "- 'AB' | 51.31% | -0.66736 // 'AB' | 51.92% | -0.65544\n", 51 | "- 'BA' | 31.12% | -1.16736 // 'BA' | 31.49% | -1.15544\n", 52 | "- 'A' | 8.92% | -2.41736 // 'A' | 9.02% | -2.40544\n", 53 | "\n", 54 | "- 'AB' | 99.07% | -0.00932 // 'BA' | 81.25% | -0.20761\n", 55 | "- 'ABB' | 0.86% | -4.75932 // 'AB' | 18.13% | -1.70761\n", 56 | "- 'ABA' | 0.07% | -7.25932 // 'ABA' | 0.33% | -5.70761\n", 57 | "\n", 58 | "- 'BA' | 92.01% | -0.08323 // 'AB' | 88.97% | -0.11682\n", 59 | "- 'AB' | 7.55% | -2.58323 // 'A' | 9.38% | -2.36682\n", 60 | "- 'ABB' | 0.23% | -6.08323 // 'ABA' | 1.27% | -4.36682\n", 61 | "```\n", 62 | "\n", 63 | "```\n", 64 | "deepseek-chat trial 1 chosen tokens: ABABBA\n", 65 | "deepseek-chat trial 2 chosen tokens: ABABBA\n", 66 | "deepseek-chat trial 1 // deepseek-chat trial 2\n", 67 | "-----------------------------------//-----------------------------------\n", 68 | "- 'AB' | 52.90% | -0.63672 // 'AB' | 56.94% | -0.56308\n", 69 | "- 'Here' | 46.58% | -0.76399 // 'Here' | 42.72% | -0.85039\n", 70 | "- 'BA' | 0.33% | -5.70764 // 'ABA' | 0.16% | -6.42029\n", 71 | "\n", 72 | "- 'BA' | 59.17% | -0.52468 // 'BA' | 60.01% | -0.51058\n", 73 | "- 'AB' | 39.07% | -0.93970 // 'AB' | 38.48% | -0.95510\n", 74 | "- 'ABA' | 1.75% | -4.04636 // 'ABA' | 1.49% | -4.20826\n", 75 | "\n", 76 | "- 'BA' | 99.92% | -0.00077 // 'BA' | 99.73% | -0.00274\n", 77 | "- 'BB' | 0.08% | -7.16542 // 'BB' | 0.27% | -5.91770\n", 78 | "- 'AB' | 0.00% | -13.57515 // 'AB' | 0.00% | -10.23312\n", 79 | "```" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": null, 85 | "id": "cf292eb2-b590-42f3-8e1e-355cbbdc7d8e", 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [] 89 | } 90 | ], 91 | "metadata": { 92 | "kernelspec": { 93 | "display_name": "casual", 94 | "language": "python", 95 | "name": "casual" 96 | }, 97 | "language_info": { 98 | "codemirror_mode": { 99 | "name": "ipython", 100 | "version": 3 101 | }, 102 | "file_extension": ".py", 103 | "mimetype": "text/x-python", 104 | "name": "python", 105 | "nbconvert_exporter": "python", 106 | "pygments_lexer": "ipython3", 107 | "version": "3.10.0" 108 | } 109 | }, 110 | "nbformat": 4, 111 | "nbformat_minor": 5 112 | } 113 | -------------------------------------------------------------------------------- /tutorials/ocr/llm_ocr.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "b3aa3e94-d49b-4cf3-9430-8ec9dc50a526", 6 | "metadata": {}, 7 | "source": [ 8 | "- https://www.runpulse.com/blog/why-llms-suck-at-ocr\n", 9 | " - https://arxiv.org/pdf/2407.06581v1\n", 10 | " - Vision language models are blind" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "id": "9b7bbd8e-4397-4c84-b4e5-ea7e334d0cec", 16 | "metadata": {}, 17 | "source": [ 18 | "- intuitives\n", 19 | " - Original Text → Common LLM Substitutions‍\n", 20 | " - \"l1lI\" → \"1111\" or \"LLLL\"\n", 21 | " - \"O0o\" → \"000\" or \"OOO\"\n", 22 | " - \"vv\" → \"w\"\n", 23 | " - \"cl\" → \"d\"\n", 24 | "- 幻觉的根源是ViT对局部的不足 + 概率sampling\n", 25 | " - only ViT is not enough => cnn + vit ??\n", 26 | " - llm 建模（language modeling）的是联合概率，自回归的条件生成是：联合概率" 27 | ] 28 | } 29 | ], 30 | "metadata": { 31 | "kernelspec": { 32 | "display_name": "verl", 33 | "language": "python", 34 | "name": "verl" 35 | }, 36 | "language_info": { 37 | "codemirror_mode": { 38 | "name": "ipython", 39 | "version": 3 40 | }, 41 | "file_extension": ".py", 42 | "mimetype": "text/x-python", 43 | "name": "python", 44 | "nbconvert_exporter": "python", 45 | "pygments_lexer": "ipython3", 46 | "version": "3.9.0" 47 | } 48 | }, 49 | "nbformat": 4, 50 | "nbformat_minor": 5 51 | } 52 | -------------------------------------------------------------------------------- /tutorials/rag_in_action/chap1-simple-rag/simple_rag.py: -------------------------------------------------------------------------------- 1 | import os 2 | from dotenv import load_dotenv 3 | from llama_index.core import VectorStoreIndex, SimpleDirectoryReader 4 | from llama_index.embeddings.huggingface import HuggingFaceEmbedding 5 | # from llama_index.llms.deepseek import DeepSeek 6 | 7 | def simple_rag(embed_model): 8 | documents = SimpleDirectoryReader(input_files=["data/黑悟空/设定.txt"]).load_data() 9 | index = VectorStoreIndex.from_documents(documents, embed_model=embed_model) 10 | index.storage_context.persist("data/黑悟空/index") 11 | # query_engine = index.as_query_engine(llm=DeepSeek(api_key=os.getenv("DEEPSEEK_API_KEY"))) 12 | query_engine = index.as_query_engine() 13 | print(query_engine.query("黑神话悟空中有哪些战斗工具?")) 14 | 15 | 16 | if __name__ == "__main__": 17 | assert load_dotenv() 18 | embed_model = HuggingFaceEmbedding(model_name="BAAI/bge-m3") 19 | -------------------------------------------------------------------------------- /tutorials/rag_in_action/requirements.txt: -------------------------------------------------------------------------------- 1 | openai 2 | 3 | llama-index 4 | llama-index-embeddings-huggingface 5 | llama-index-llms-deepseek 6 | 7 | langchain 8 | langchain-openai 9 | langchain-deepseek 10 | langchain-community 11 | langchain-huggingface 12 | sentence-transformers 13 | 14 | beautifulsoup4 15 | 16 | python-dotenv 17 | 18 | pymilvus 19 | -------------------------------------------------------------------------------- /tutorials/rag_in_action/图文检索.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "97391b9e-7fc4-4f0f-b846-9aa8d6954377", 6 | "metadata": {}, 7 | "source": [ 8 | "- https://milvus.io/zh/blog/multimodal-semantic-search-with-images-and-text.md" 9 | ] 10 | } 11 | ], 12 | "metadata": { 13 | "kernelspec": { 14 | "display_name": "Python 3 (ipykernel)", 15 | "language": "python", 16 | "name": "python3" 17 | }, 18 | "language_info": { 19 | "codemirror_mode": { 20 | "name": "ipython", 21 | "version": 3 22 | }, 23 | "file_extension": ".py", 24 | "mimetype": "text/x-python", 25 | "name": "python", 26 | "nbconvert_exporter": "python", 27 | "pygments_lexer": "ipython3", 28 | "version": "3.10.0" 29 | } 30 | }, 31 | "nbformat": 4, 32 | "nbformat_minor": 5 33 | } 34 | -------------------------------------------------------------------------------- /tutorials/炼丹/炼丹&调参_tricks.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "d3134108", 6 | "metadata": {}, 7 | "source": [ 8 | "- reference\n", 9 | " - http://karpathy.github.io/2019/04/25/recipe/" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "id": "5c1ebce5", 15 | "metadata": {}, 16 | "source": [ 17 | "## batch size & learning rate" 18 | ] 19 | }, 20 | { 21 | "cell_type": "markdown", 22 | "id": "075979d9", 23 | "metadata": {}, 24 | "source": [ 25 | "- Adam & 3e-4\n", 26 | " - karpathy constant\n", 27 | "- batch size and learning rate" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "id": "ac4cfa19", 33 | "metadata": {}, 34 | "source": [ 35 | "## `tokenizer.padding_side`" 36 | ] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "id": "0ab430bc", 41 | "metadata": {}, 42 | "source": [ 43 | "If you want to fine-tune an LLM while using `FlashAttention`, you must set the padding side to “left”. Otherwise, it won’t work.\n", 44 | "\n", 45 | "```\n", 46 | "tokenizer.padding_side = 'left'\n", 47 | "```" 48 | ] 49 | } 50 | ], 51 | "metadata": { 52 | "kernelspec": { 53 | "display_name": "Python 3 (ipykernel)", 54 | "language": "python", 55 | "name": "python3" 56 | }, 57 | "language_info": { 58 | "codemirror_mode": { 59 | "name": "ipython", 60 | "version": 3 61 | }, 62 | "file_extension": ".py", 63 | "mimetype": "text/x-python", 64 | "name": "python", 65 | "nbconvert_exporter": "python", 66 | "pygments_lexer": "ipython3", 67 | "version": "3.10.13" 68 | }, 69 | "toc": { 70 | "base_numbering": 1, 71 | "nav_menu": {}, 72 | "number_sections": true, 73 | "sideBar": true, 74 | "skip_h1_title": false, 75 | "title_cell": "Table of Contents", 76 | "title_sidebar": "Contents", 77 | "toc_cell": false, 78 | "toc_position": {}, 79 | "toc_section_display": true, 80 | "toc_window_display": false 81 | } 82 | }, 83 | "nbformat": 4, 84 | "nbformat_minor": 5 85 | } 86 | --------------------------------------------------------------------------------