├── .gitignore ├── LICENSE ├── README.md ├── imgs ├── 990_pro.png ├── claude-sae.png ├── maxpool_grad.png ├── nv-chat-rtx-version.png ├── nvme_partition.png ├── rag-agent.png ├── rag-chain.png ├── self-corrective-rag.png └── tot_dfs_bfs.png ├── paper-reading └── openai_hallucination.ipynb └── tutorials ├── 01_image_caption.ipynb ├── 02_text2img.ipynb ├── agents ├── agent-运维.ipynb ├── agents_research.ipynb ├── applications │ ├── translation-agent-demo.ipynb │ └── utils.py ├── env_sandbox │ └── coding.ipynb ├── langchain │ ├── AgentExecutor.ipynb │ ├── LangGraph-MultiAgents.ipynb │ ├── LangGraph-rag.ipynb │ ├── LangGraph_basics.ipynb │ ├── advanced │ │ ├── branching.ipynb │ │ ├── building_effective_agents.ipynb │ │ ├── langgraph_advanced.ipynb │ │ ├── langgraph_studio.ipynb │ │ ├── memory.ipynb │ │ └── scripts │ │ │ ├── braching.py │ │ │ ├── memory.py │ │ │ └── wo_memory.py │ ├── agent_scratchpad_intermediate_steps.ipynb │ ├── agent_types.ipynb │ ├── conversational_agents.ipynb │ ├── examples │ │ ├── agent-simulation.ipynb │ │ ├── llm_compiler.ipynb │ │ ├── plan_and_execute.ipynb │ │ └── reflexion_agent.ipynb │ ├── imgs │ │ ├── dag.png │ │ ├── react-output.png │ │ ├── structure_output_option1.png │ │ └── structure_output_option2.png │ ├── langchain_agents.ipynb │ ├── langchain_graph_details.ipynb │ ├── langgraph │ │ ├── state_manage.ipynb │ │ └── 设计思想.ipynb │ ├── lc_lg_core_concepts.ipynb │ ├── output_parser.ipynb │ ├── scripts │ │ └── chat_react.py │ └── structured_output.ipynb ├── oai_function_call_rag.ipynb ├── planning │ ├── imgs │ │ └── mcts_3nodes.png │ ├── lats.ipynb │ ├── math_tools.py │ ├── mcts_tree.dot │ ├── mcts_tree.pdf │ ├── pddl │ │ ├── fast-downward.ipynb │ │ ├── pddlstream.ipynb │ │ ├── planning_pddl.ipynb │ │ └── solver_planutils.ipynb │ ├── reasoner_enhenced.ipynb │ └── tot-langgraph.ipynb └── tot_tree_of_thought.ipynb ├── ai_app ├── ai-products │ ├── NotebookLM.ipynb │ ├── copilot.ipynb │ ├── cursor-advanced.ipynb │ ├── cursor-composer.ipynb │ └── cursor.ipynb ├── audio_video │ ├── audio.wav │ ├── audio_notes.ipynb │ ├── imgs │ │ └── audio-notes-framework.png │ ├── speech_generation.ipynb │ ├── video.mp4 │ ├── videolingo.ipynb │ └── whisper_whisperx.ipynb ├── claude-code.ipynb ├── codex.ipynb ├── db │ └── postgresql.ipynb ├── gemini-cli.ipynb ├── imgs │ ├── 0_Parade_Parade_0_873.jpg │ ├── 0_Parade_marchingband_1_765.jpg │ ├── 11_Meeting_Meeting_11_Meeting_Meeting_11_176.jpg │ ├── 11_Meeting_Meeting_11_Meeting_Meeting_11_26.jpg │ ├── gh_copilot_suggest.png │ └── tab.jpeg ├── openrouter.ipynb ├── pipeline_工具.ipynb ├── streamlit(web).ipynb ├── tools │ ├── Monica.ipynb │ └── ai_tools.ipynb ├── vision │ ├── face_detect.ipynb │ ├── rembg-U2Net.ipynb │ └── 图像视频.ipynb └── 项目参考.ipynb ├── cv ├── PIL.ipynb ├── ViT.ipynb ├── cnn_3dcnn_se.ipynb ├── pytorch_cifar10.ipynb ├── timm.ipynb └── torchvision.ipynb ├── dnn ├── ae │ ├── ksparse-ae.ipynb │ └── ksparse-ae.py └── 对比学习 │ ├── SimCLR.ipynb │ ├── constrative_loss_infoNCE.ipynb │ └── 对比学习_contrastive_learning.ipynb ├── gnn ├── GTN_Transformer.ipynb ├── PyG.ipynb ├── gcn_gat.ipynb ├── gnn_basics.ipynb └── imgs │ ├── gnn_layers.png │ └── two-layers-gcn.png ├── mcp ├── about_mcp.ipynb ├── gemini-cli-mcp.ipynb ├── mcp_api.ipynb └── 实用 MCP.ipynb ├── nn_basics ├── 01_pytorch_wandb.ipynb ├── 02_gelu_grad.ipynb ├── 04_embedding_details.ipynb ├── activate_function │ └── swish.ipynb ├── bce_loss_cross_entropy.ipynb ├── data_prepare │ └── dataset_dataloader.ipynb ├── einsum_einops │ ├── einops.ipynb │ ├── einsum_bilinear.ipynb │ ├── einsum_numpy_torch.ipynb │ ├── einsum_supp_einops_初步.ipynb │ └── shape.ipynb ├── grad │ ├── gradient_autograd.ipynb │ ├── mask_grad.ipynb │ ├── max_pooling_grad.ipynb │ ├── readme.md │ ├── retain_grad_retatin_graph.ipynb │ ├── sae_topK.ipynb │ ├── sigmoid_grad.ipynb │ ├── tensorboard_graph.ipynb │ ├── torch_jacobian_hessian.ipynb │ ├── torch_jvp.ipynb │ ├── torch_常见错误.ipynb │ └── 可微才可学.ipynb ├── imgs │ ├── axies-flat.png │ ├── axis-swap-flat.png │ ├── axis-swap.png │ ├── axis0-1-2.png │ ├── boo.jpg │ ├── fan_in_fan_out.png │ ├── graph-legend.png │ ├── linear.png │ ├── multi_loss.PNG │ ├── numpy_einsum.png │ ├── shape_1x8.png │ ├── shape_1x8_2.png │ ├── shape_1x8_3.png │ ├── shape_1x8_4.png │ ├── shape_1x8_5.png │ ├── shape_2x4.png │ ├── shape_4x2.png │ ├── shape_4x2_2.png │ └── shape_4x2_3.png ├── loss │ ├── cross_entroy_loss_ignore_index.ipynb │ └── mse_crossentropy.ipynb ├── module_load_save.ipynb ├── multinomail_categorical.ipynb ├── optimizer │ ├── adadelta_adam.ipynb │ ├── adagrad_rmsprop.ipynb │ ├── airfoil_self_noise.dat │ ├── final_overall.ipynb │ ├── l2_reg_weight_decay_adaw.ipynb │ ├── lr_scheduler.ipynb │ ├── momentum_exp_avg.ipynb │ ├── pytorch_as_a_general_optimizer.ipynb │ └── pytorch_lbfgs.ipynb ├── performance_optimization │ └── pytorch_optimization.ipynb ├── quantization.ipynb ├── shape_axis_dim │ ├── axis_dim_2.ipynb │ ├── axis_dim_processing.ipynb │ ├── fan_in_fan_out_parameters.ipynb │ └── shape_modified.ipynb ├── torch_dtype.ipynb ├── tricks_norms │ ├── layer_norm.ipynb │ ├── nn_and_training_tricks.ipynb │ ├── normalization.ipynb │ ├── spectral_norm.ipynb │ ├── torch_dropout.ipynb │ └── why_norm.ipynb └── utils │ ├── functional.ipynb │ ├── softmax.ipynb │ ├── torch_triangle_mask.ipynb │ └── torch_utils_scripts.ipynb ├── oai ├── async_demo.py ├── async_oai.ipynb ├── async_scripts │ ├── Semaphore.py │ ├── async_call.py │ ├── async_call_semaphore.py │ ├── async_llm_call.py │ └── io_cpu.py ├── concurrent-api-threads.ipynb ├── gemini-2-5-pro.ipynb ├── gpt_openai_生态.ipynb ├── logP.ipynb ├── logP_supp.ipynb └── openai_api.ipynb ├── ocr └── llm_ocr.ipynb ├── rag_in_action ├── GraphRAG.ipynb ├── chap1-simple-rag │ └── simple_rag.py ├── overall.ipynb ├── requirements.txt └── 图文检索.ipynb ├── workflow ├── cipher_examples.py └── codeact.ipynb └── 炼丹 ├── batch_size_learning_rate.ipynb ├── 炼丹&调参_tricks.ipynb └── 超参搜索.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | wandb/ 132 | tutorials/nn_basics/data/ 133 | 134 | runs/ 135 | 136 | .idea/ 137 | data/ 138 | logs/ 139 | tutorials/rag/ 140 | 141 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # llm_aigc 2 | everything about llm & aigc 3 | -------------------------------------------------------------------------------- /imgs/990_pro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/imgs/990_pro.png -------------------------------------------------------------------------------- /imgs/claude-sae.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/imgs/claude-sae.png -------------------------------------------------------------------------------- /imgs/maxpool_grad.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/imgs/maxpool_grad.png -------------------------------------------------------------------------------- /imgs/nv-chat-rtx-version.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/imgs/nv-chat-rtx-version.png -------------------------------------------------------------------------------- /imgs/nvme_partition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/imgs/nvme_partition.png -------------------------------------------------------------------------------- /imgs/rag-agent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/imgs/rag-agent.png -------------------------------------------------------------------------------- /imgs/rag-chain.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/imgs/rag-chain.png -------------------------------------------------------------------------------- /imgs/self-corrective-rag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/imgs/self-corrective-rag.png -------------------------------------------------------------------------------- /imgs/tot_dfs_bfs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/imgs/tot_dfs_bfs.png -------------------------------------------------------------------------------- /tutorials/agents/agent-运维.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "889a35bd-4a0f-4f6f-aeeb-9824199bd0b9", 6 | "metadata": {}, 7 | "source": [ 8 | "- todo\n", 9 | " - 数据如何导出？" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "id": "d5ce5cd2-29d0-4445-94cf-8ae91a8ff32b", 15 | "metadata": {}, 16 | "source": [ 17 | "### langsmith" 18 | ] 19 | }, 20 | { 21 | "cell_type": "markdown", 22 | "id": "eaa67e70-4ccd-4dc9-b5f1-0bffe83a7dfc", 23 | "metadata": {}, 24 | "source": [ 25 | "- https://github.com/langchain-ai/langsmith-cookbook" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 1, 31 | "id": "7b44a667-5e69-4aa2-87c2-4db539066054", 32 | "metadata": {}, 33 | "outputs": [ 34 | { 35 | "data": { 36 | "text/plain": [ 37 | "True" 38 | ] 39 | }, 40 | "execution_count": 1, 41 | "metadata": {}, 42 | "output_type": "execute_result" 43 | } 44 | ], 45 | "source": [ 46 | "import os\n", 47 | "from dotenv import load_dotenv\n", 48 | "load_dotenv()" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 2, 54 | "id": "d012b497-7e6e-4509-96a9-76b66964cf76", 55 | "metadata": {}, 56 | "outputs": [], 57 | "source": [ 58 | "import os\n", 59 | "os.environ['LANGCHAIN_TRACING_V2'] = 'true'\n", 60 | "os.environ['LANGCHAIN_API_KEY'] = 'xx'" 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "id": "4abbe760-4f21-46a5-9f62-dd52e81b45b6", 66 | "metadata": {}, 67 | "source": [ 68 | "### langfuse" 69 | ] 70 | }, 71 | { 72 | "cell_type": "markdown", 73 | "id": "75a4fc00-1f5d-45d5-87e1-45f665bbaaf0", 74 | "metadata": {}, 75 | "source": [ 76 | "| 特性 | Langfuse | LangSmith |\n", 77 | "| :--- | :--- | :--- |\n", 78 | "| **开源性** | **开源 (MIT许可证)** | **闭源** |\n", 79 | "| **自托管** | **免费自托管**，所有核心功能无限制 | 需购买**企业版**才能自托管 |\n", 80 | "| **核心生态** | 独立于任何特定框架，提供广泛的集成 | 由LangChain团队开发，与LangChain和LangGraph深度集成 |\n", 81 | "| **商业模式** | 提供云服务和带商业附加功能的企业版自托管 | 提供云服务和企业版自托管/混合部署 |" 82 | ] 83 | } 84 | ], 85 | "metadata": { 86 | "kernelspec": { 87 | "display_name": "Python 3 (ipykernel)", 88 | "language": "python", 89 | "name": "python3" 90 | }, 91 | "language_info": { 92 | "codemirror_mode": { 93 | "name": "ipython", 94 | "version": 3 95 | }, 96 | "file_extension": ".py", 97 | "mimetype": "text/x-python", 98 | "name": "python", 99 | "nbconvert_exporter": "python", 100 | "pygments_lexer": "ipython3", 101 | "version": "3.13.5" 102 | } 103 | }, 104 | "nbformat": 4, 105 | "nbformat_minor": 5 106 | } 107 | -------------------------------------------------------------------------------- /tutorials/agents/agents_research.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "59ba6798-ab66-433f-903f-fc4cfabe7068", 6 | "metadata": {}, 7 | "source": [ 8 | "- https://sites.google.com/view/cs-159-2024/lectures" 9 | ] 10 | } 11 | ], 12 | "metadata": { 13 | "kernelspec": { 14 | "display_name": "Python 3 (ipykernel)", 15 | "language": "python", 16 | "name": "python3" 17 | }, 18 | "language_info": { 19 | "codemirror_mode": { 20 | "name": "ipython", 21 | "version": 3 22 | }, 23 | "file_extension": ".py", 24 | "mimetype": "text/x-python", 25 | "name": "python", 26 | "nbconvert_exporter": "python", 27 | "pygments_lexer": "ipython3", 28 | "version": "3.10.13" 29 | } 30 | }, 31 | "nbformat": 4, 32 | "nbformat_minor": 5 33 | } 34 | -------------------------------------------------------------------------------- /tutorials/agents/env_sandbox/coding.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "8de4e22e-44d8-4cc0-9cc1-ee285a0e4813", 6 | "metadata": {}, 7 | "source": [ 8 | "### ast (abstract syntax tree)" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 1, 14 | "id": "1c481af8-453d-43c9-b914-03007c0ade43", 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "import ast" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 2, 24 | "id": "a56b9b40-6bc5-4afb-b287-ad85a4ac1308", 25 | "metadata": {}, 26 | "outputs": [ 27 | { 28 | "data": { 29 | "text/plain": [ 30 | "" 31 | ] 32 | }, 33 | "execution_count": 2, 34 | "metadata": {}, 35 | "output_type": "execute_result" 36 | } 37 | ], 38 | "source": [ 39 | "ast.parse('(11 + 5) * 2 + 7 + 3', mode='eval').body" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "id": "587e6260-7b56-4733-86a0-b6e6b1650b91", 45 | "metadata": {}, 46 | "source": [ 47 | "```mermaid\n", 48 | "graph TD\n", 49 | " subgraph \"AST for (11 + 5) * 2 + 7 + 3\"\n", 50 | " A[\"BinOp
op = Add (+)\"] --> B[\"BinOp
op = Add (+)\"];\n", 51 | " A --> C[\"Constant
value = 3\"];\n", 52 | "\n", 53 | " B --> D[\"BinOp
op = Mult (*)\"];\n", 54 | " B --> E[\"Constant
value = 7\"];\n", 55 | "\n", 56 | " D --> F[\"BinOp
op = Add (+)\"];\n", 57 | " D --> G[\"Constant
value = 2\"];\n", 58 | "\n", 59 | " F --> H[\"Constant
value = 11\"];\n", 60 | " F --> I[\"Constant
value = 5\"];\n", 61 | " end\n", 62 | "\n", 63 | " style A fill:#f9f,stroke:#333,stroke-width:2px\n", 64 | " style B fill:#f9f,stroke:#333,stroke-width:2px\n", 65 | " style D fill:#f9f,stroke:#333,stroke-width:2px\n", 66 | " style F fill:#f9f,stroke:#333,stroke-width:2px\n", 67 | "\n", 68 | " style C fill:#bbf,stroke:#333,stroke-width:2px\n", 69 | " style E fill:#bbf,stroke:#333,stroke-width:2px\n", 70 | " style G fill:#bbf,stroke:#333,stroke-width:2px\n", 71 | " style H fill:#bbf,stroke:#333,stroke-width:2px\n", 72 | " style I fill:#bbf,stroke:#333,stroke-width:2px\n", 73 | "```" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": 6, 79 | "id": "625318ac-cf3f-4e9f-bc26-8d4a7ab3c2a9", 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "import operator as op\n", 84 | "_supported_operators = {\n", 85 | " ast.Add: op.add,\n", 86 | " ast.Sub: op.sub,\n", 87 | " ast.Mult: op.mul,\n", 88 | " ast.Div: op.truediv,\n", 89 | " ast.Pow: op.pow,\n", 90 | " ast.USub: op.neg,\n", 91 | "}\n" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": 7, 97 | "id": "d6c757bd-6ed0-4f6f-9a7b-7528190414f2", 98 | "metadata": {}, 99 | "outputs": [], 100 | "source": [ 101 | "def _eval_node(node):\n", 102 | " \"\"\"\n", 103 | " Recursively evaluates an AST node.\n", 104 | " This provides a safe alternative to eval().\n", 105 | " \"\"\"\n", 106 | " if isinstance(node, ast.Constant):\n", 107 | " return node.value\n", 108 | " elif isinstance(node, ast.BinOp):\n", 109 | " return _supported_operators[type(node.op)](_eval_node(node.left), _eval_node(node.right))\n", 110 | " elif isinstance(node, ast.UnaryOp):\n", 111 | " return _supported_operators[type(node.op)](_eval_node(node.operand))\n", 112 | " else:\n", 113 | " raise TypeError(f\"Unsupported node type: {node.__class__.__name__}\")" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": 8, 119 | "id": "c92fe474-696d-4d34-aff9-8f4e22291c3b", 120 | "metadata": {}, 121 | "outputs": [ 122 | { 123 | "data": { 124 | "text/plain": [ 125 | "42" 126 | ] 127 | }, 128 | "execution_count": 8, 129 | "metadata": {}, 130 | "output_type": "execute_result" 131 | } 132 | ], 133 | "source": [ 134 | "_eval_node(ast.parse('(11 + 5) * 2 + 7 + 3', mode='eval').body)" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": null, 140 | "id": "e7b7f80f-08c5-4dda-bd4b-7dc6b0c79371", 141 | "metadata": {}, 142 | "outputs": [], 143 | "source": [] 144 | } 145 | ], 146 | "metadata": { 147 | "kernelspec": { 148 | "display_name": "torch", 149 | "language": "python", 150 | "name": "verl" 151 | }, 152 | "language_info": { 153 | "codemirror_mode": { 154 | "name": "ipython", 155 | "version": 3 156 | }, 157 | "file_extension": ".py", 158 | "mimetype": "text/x-python", 159 | "name": "python", 160 | "nbconvert_exporter": "python", 161 | "pygments_lexer": "ipython3", 162 | "version": "3.10.0" 163 | } 164 | }, 165 | "nbformat": 4, 166 | "nbformat_minor": 5 167 | } 168 | -------------------------------------------------------------------------------- /tutorials/agents/langchain/advanced/langgraph_advanced.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "id": "4a23a9d9-5123-479f-b164-e1a0572bc861", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "from langchain_openai import ChatOpenAI\n", 11 | "from dotenv import load_dotenv\n", 12 | "assert load_dotenv()\n", 13 | "llm = ChatOpenAI(model='gpt-4o-mini')" 14 | ] 15 | }, 16 | { 17 | "cell_type": "markdown", 18 | "id": "cbefe874-c7c0-4c5b-bce6-b4111cadd7fa", 19 | "metadata": {}, 20 | "source": [ 21 | "### prebuilt" 22 | ] 23 | }, 24 | { 25 | "cell_type": "markdown", 26 | "id": "a140f952-5530-4fc6-b413-8af514237af9", 27 | "metadata": {}, 28 | "source": [ 29 | "```python\n", 30 | "from langgraph.prebuilt import create_react_agent\n", 31 | "\n", 32 | "# Pass in:\n", 33 | "# (1) the augmented LLM with tools\n", 34 | "# (2) the tools list (which is used to create the tool node)\n", 35 | "pre_built_agent = create_react_agent(llm, tools=tools)\n", 36 | "\n", 37 | "# Show the agent\n", 38 | "display(Image(pre_built_agent.get_graph().draw_mermaid_png()))\n", 39 | "\n", 40 | "# Invoke\n", 41 | "messages = [HumanMessage(content=\"Add 3 and 4.\")]\n", 42 | "messages = pre_built_agent.invoke({\"messages\": messages})\n", 43 | "for m in messages[\"messages\"]:\n", 44 | " m.pretty_print()\n", 45 | "```" 46 | ] 47 | } 48 | ], 49 | "metadata": { 50 | "kernelspec": { 51 | "display_name": "Python 3 (ipykernel)", 52 | "language": "python", 53 | "name": "python3" 54 | }, 55 | "language_info": { 56 | "codemirror_mode": { 57 | "name": "ipython", 58 | "version": 3 59 | }, 60 | "file_extension": ".py", 61 | "mimetype": "text/x-python", 62 | "name": "python", 63 | "nbconvert_exporter": "python", 64 | "pygments_lexer": "ipython3", 65 | "version": "3.10.0" 66 | } 67 | }, 68 | "nbformat": 4, 69 | "nbformat_minor": 5 70 | } 71 | -------------------------------------------------------------------------------- /tutorials/agents/langchain/advanced/langgraph_studio.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "cefc3aa1-0f6a-4546-9f60-a769abfaed65", 6 | "metadata": {}, 7 | "source": [ 8 | "- https://langchain-ai.github.io/langgraph/concepts/langgraph_studio/\n", 9 | " - https://langchain-ai.github.io/langgraph/concepts/application_structure/\n", 10 | " - LangGraph Studio: connect to a locally running agent" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "id": "5d49cb42-dde8-41f0-8620-bdef52b5bf3a", 16 | "metadata": {}, 17 | "source": [ 18 | "### application structure" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "id": "0842856d-ddae-46d1-bac5-e17033eb1030", 24 | "metadata": {}, 25 | "source": [ 26 | "- A LangGraph application consists of one or more graphs, a configuration file (`langgraph.json`), a file that specifies dependencies, and an optional .env file that specifies environment variables.\n", 27 | " - 约定优于配置；\n", 28 | "```json\n", 29 | "{\n", 30 | " \"dependencies\": [\".\"],\n", 31 | " \"graphs\": {\n", 32 | " \"agent\": \"./src/agent/graph.py:graph\"\n", 33 | " },\n", 34 | " \"http\": {\n", 35 | " \"app\": \"./src/agent/app.py:app\"\n", 36 | " },\n", 37 | " \"env\": \".env\"\n", 38 | "}\n", 39 | "\n", 40 | "```" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "id": "73585d6a-b250-4fd4-82b1-58f8b90dea0b", 46 | "metadata": {}, 47 | "source": [ 48 | "### `langgraph dev`" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "id": "1e262401-2d5f-45e7-8017-c23a491b6053", 54 | "metadata": {}, 55 | "source": [ 56 | "- The backend API will be available at `http://127.0.0.1:2024`" 57 | ] 58 | } 59 | ], 60 | "metadata": { 61 | "kernelspec": { 62 | "display_name": "Python 3 (ipykernel)", 63 | "language": "python", 64 | "name": "python3" 65 | }, 66 | "language_info": { 67 | "codemirror_mode": { 68 | "name": "ipython", 69 | "version": 3 70 | }, 71 | "file_extension": ".py", 72 | "mimetype": "text/x-python", 73 | "name": "python", 74 | "nbconvert_exporter": "python", 75 | "pygments_lexer": "ipython3", 76 | "version": "3.10.0" 77 | } 78 | }, 79 | "nbformat": 4, 80 | "nbformat_minor": 5 81 | } 82 | -------------------------------------------------------------------------------- /tutorials/agents/langchain/advanced/scripts/braching.py: -------------------------------------------------------------------------------- 1 | import operator 2 | from typing import Annotated 3 | from typing_extensions import TypedDict 4 | 5 | from langgraph.types import Send 6 | from langgraph.graph import END, StateGraph, START 7 | from langchain_openai import ChatOpenAI 8 | from langchain_core.prompts import ChatPromptTemplate 9 | from langchain_core.output_parsers import StrOutputParser, JsonOutputParser 10 | from dotenv import load_dotenv 11 | 12 | load_dotenv() 13 | 14 | # Instantiate the model 15 | model = ChatOpenAI(model="gpt-4o-mini", temperature=0) 16 | 17 | 18 | # This will be the overall state of the main graph. 19 | # It will contain a topic (which we expect the user to provide) 20 | # and then will generate a list of subjects, and then a joke for 21 | # each subject 22 | class OverallState(TypedDict): 23 | topic: str 24 | num_subjects: int 25 | subjects: list 26 | # Notice here we use the operator.add 27 | # This is because we want combine all the jokes we generate 28 | # from individual nodes back into one list - this is essentially 29 | # the "reduce" part 30 | jokes: Annotated[list, operator.add] 31 | best_selected_joke: str 32 | 33 | 34 | # This will be the state of the node that we will "map" all 35 | # subjects to in order to generate a joke 36 | class JokeState(TypedDict): 37 | subject: str 38 | 39 | 40 | # This is the function we will use to generate the subjects of the jokes. 41 | # In general the length of the list generated by this node could vary each run. 42 | def generate_topics(state: OverallState): 43 | prompt = ChatPromptTemplate.from_messages([ 44 | ("system", "You are a helpful assistant that generates a list of subjects based on a topic. Your output should be a JSON list of strings, for example: [\"subject1\", \"subject2\"]."), 45 | ("human", "Generate a list of {num_subjects} subjects related to the topic '{topic}'.") 46 | ]) 47 | chain = prompt | model | JsonOutputParser() 48 | subjects = chain.invoke({"topic": state["topic"], "num_subjects": state["num_subjects"]}) 49 | return {"subjects": subjects} 50 | 51 | 52 | # Here we generate a joke, given a subject 53 | def generate_joke(state: JokeState): 54 | prompt = ChatPromptTemplate.from_template("Tell me a short joke about {subject}") 55 | chain = prompt | model | StrOutputParser() 56 | joke = chain.invoke({"subject": state["subject"]}) 57 | return {"jokes": [joke]} 58 | 59 | 60 | # Here we define the logic to map out over the generated subjects 61 | # We will use this as an edge in the graph 62 | def continue_to_jokes(state: OverallState): 63 | # We will return a list of `Send` objects 64 | # Each `Send` object consists of the name of a node in the graph 65 | # as well as the state to send to that node 66 | return [Send("generate_joke", {"subject": s}) for s in state["subjects"]] 67 | 68 | 69 | # Here we will judge the best joke 70 | def best_joke(state: OverallState): 71 | jokes_str = "\n\n".join(state["jokes"]) 72 | prompt = ChatPromptTemplate.from_template( 73 | "Here are a few jokes:\n\n{jokes}\n\nWhich one is the best? Just return the joke itself, and nothing else." 74 | ) 75 | chain = prompt | model | StrOutputParser() 76 | best = chain.invoke({"jokes": jokes_str}) 77 | return {"best_selected_joke": best} 78 | 79 | 80 | # Construct the graph: here we put everything together to construct our graph 81 | builder = StateGraph(OverallState) 82 | builder.add_node("generate_topics", generate_topics) 83 | builder.add_node("generate_joke", generate_joke) 84 | builder.add_node("best_joke", best_joke) 85 | 86 | builder.add_edge(START, "generate_topics") 87 | builder.add_conditional_edges("generate_topics", continue_to_jokes, ["generate_joke"]) 88 | 89 | builder.add_edge("generate_joke", "best_joke") 90 | builder.add_edge("best_joke", END) 91 | graph = builder.compile() 92 | 93 | # Call the graph: here we call it to generate a list of jokes 94 | for step in graph.stream({"topic": "research", "num_subjects": 3}): 95 | print(step) -------------------------------------------------------------------------------- /tutorials/agents/langchain/advanced/scripts/memory.py: -------------------------------------------------------------------------------- 1 | from typing import Annotated 2 | 3 | from langchain_tavily import TavilySearch 4 | from langchain_core.messages import BaseMessage, HumanMessage 5 | from typing_extensions import TypedDict 6 | 7 | from langgraph.checkpoint.memory import MemorySaver 8 | from langgraph.graph import StateGraph 9 | from langgraph.graph.message import add_messages 10 | from langgraph.prebuilt import ToolNode, tools_condition 11 | 12 | from langchain.chat_models import init_chat_model 13 | 14 | import dotenv 15 | import uuid 16 | dotenv.load_dotenv() 17 | 18 | 19 | llm = init_chat_model("openai:gpt-4o-mini") 20 | 21 | class State(TypedDict): 22 | messages: Annotated[list, add_messages] 23 | 24 | graph_builder = StateGraph(State) 25 | 26 | tool = TavilySearch(max_results=2) 27 | tools = [tool] 28 | llm_with_tools = llm.bind_tools(tools) 29 | 30 | def chatbot(state: State): 31 | return {"messages": [llm_with_tools.invoke(state["messages"])]} 32 | 33 | graph_builder.add_node("chatbot", chatbot) 34 | 35 | tool_node = ToolNode(tools=[tool]) 36 | graph_builder.add_node("tools", tool_node) 37 | 38 | graph_builder.add_conditional_edges( 39 | "chatbot", 40 | tools_condition, 41 | ) 42 | graph_builder.add_edge("tools", "chatbot") 43 | graph_builder.set_entry_point("chatbot") 44 | memory = MemorySaver() 45 | # graph = graph_builder.compile() 46 | graph = graph_builder.compile(checkpointer=memory) 47 | 48 | png_graph = graph.get_graph().draw_mermaid_png() 49 | with open("langgraph_memory.png", "wb") as f: 50 | f.write(png_graph) 51 | 52 | if __name__ == "__main__": 53 | config = {"configurable": {"thread_id": str(uuid.uuid4())}} 54 | while True: 55 | user_input = input("User: ") 56 | if user_input.lower() in ["quit", "exit", "q"]: 57 | print("Goodbye!") 58 | break 59 | 60 | # The graph.stream() method returns an iterator. We can fully consume 61 | # it to get all events in a list. 62 | events = list(graph.stream( 63 | {"messages": [HumanMessage(content=user_input)]}, 64 | config, 65 | stream_mode="values", 66 | )) 67 | 68 | # The last event in the stream is the final state of the graph 69 | final_state = events[-1] 70 | 71 | # We can access the messages from the final state 72 | # and print them to see the full conversation history 73 | print("---- Full Conversation History ----") 74 | for message in final_state.get("messages", []): 75 | message.pretty_print() 76 | print("------------------------------------") -------------------------------------------------------------------------------- /tutorials/agents/langchain/advanced/scripts/wo_memory.py: -------------------------------------------------------------------------------- 1 | from typing import Annotated 2 | 3 | from langchain_tavily import TavilySearch 4 | from langchain_core.messages import BaseMessage, HumanMessage 5 | from typing_extensions import TypedDict 6 | 7 | from langgraph.checkpoint.memory import MemorySaver 8 | from langgraph.graph import StateGraph 9 | from langgraph.graph.message import add_messages 10 | from langgraph.prebuilt import ToolNode, tools_condition 11 | 12 | from langchain.chat_models import init_chat_model 13 | 14 | import dotenv 15 | import uuid 16 | dotenv.load_dotenv() 17 | 18 | 19 | llm = init_chat_model("openai:gpt-4o-mini") 20 | 21 | class State(TypedDict): 22 | messages: Annotated[list, add_messages] 23 | 24 | graph_builder = StateGraph(State) 25 | 26 | tool = TavilySearch(max_results=2) 27 | tools = [tool] 28 | llm_with_tools = llm.bind_tools(tools) 29 | 30 | def chatbot(state: State): 31 | return {"messages": [llm_with_tools.invoke(state["messages"])]} 32 | 33 | graph_builder.add_node("chatbot", chatbot) 34 | 35 | tool_node = ToolNode(tools=[tool]) 36 | graph_builder.add_node("tools", tool_node) 37 | 38 | graph_builder.add_conditional_edges( 39 | "chatbot", 40 | tools_condition, 41 | ) 42 | graph_builder.add_edge("tools", "chatbot") 43 | graph_builder.set_entry_point("chatbot") 44 | memory = MemorySaver() 45 | graph = graph_builder.compile() 46 | # graph = graph_builder.compile(checkpointer=memory) 47 | 48 | png_graph = graph.get_graph().draw_mermaid_png() 49 | with open("langgraph_memory.png", "wb") as f: 50 | f.write(png_graph) 51 | 52 | if __name__ == "__main__": 53 | config = {"configurable": {"thread_id": str(uuid.uuid4())}} 54 | while True: 55 | user_input = input("User: ") 56 | if user_input.lower() in ["quit", "exit", "q"]: 57 | print("Goodbye!") 58 | break 59 | 60 | # The graph.stream() method returns an iterator. We can fully consume 61 | # it to get all events in a list. 62 | events = list(graph.stream( 63 | {"messages": [HumanMessage(content=user_input)]}, 64 | # config, 65 | stream_mode="values", 66 | )) 67 | 68 | # The last event in the stream is the final state of the graph 69 | final_state = events[-1] 70 | 71 | # We can access the messages from the final state 72 | # and print them to see the full conversation history 73 | print("---- Full Conversation History ----") 74 | for message in final_state.get("messages", []): 75 | message.pretty_print() 76 | print("------------------------------------") -------------------------------------------------------------------------------- /tutorials/agents/langchain/examples/llm_compiler.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "8754839f-e9a0-4a25-9a67-303fd1447ed9", 6 | "metadata": {}, 7 | "source": [ 8 | "- https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/llm-compiler/LLMCompiler.ipynb" 9 | ] 10 | } 11 | ], 12 | "metadata": { 13 | "kernelspec": { 14 | "display_name": "agent", 15 | "language": "python", 16 | "name": "agent" 17 | }, 18 | "language_info": { 19 | "codemirror_mode": { 20 | "name": "ipython", 21 | "version": 3 22 | }, 23 | "file_extension": ".py", 24 | "mimetype": "text/x-python", 25 | "name": "python", 26 | "nbconvert_exporter": "python", 27 | "pygments_lexer": "ipython3", 28 | "version": "3.12.2" 29 | } 30 | }, 31 | "nbformat": 4, 32 | "nbformat_minor": 5 33 | } 34 | -------------------------------------------------------------------------------- /tutorials/agents/langchain/imgs/dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/agents/langchain/imgs/dag.png -------------------------------------------------------------------------------- /tutorials/agents/langchain/imgs/react-output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/agents/langchain/imgs/react-output.png -------------------------------------------------------------------------------- /tutorials/agents/langchain/imgs/structure_output_option1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/agents/langchain/imgs/structure_output_option1.png -------------------------------------------------------------------------------- /tutorials/agents/langchain/imgs/structure_output_option2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/agents/langchain/imgs/structure_output_option2.png -------------------------------------------------------------------------------- /tutorials/agents/langchain/langgraph/设计思想.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "94dc8bde-8d1e-48cd-920c-77c99bfb5c58", 6 | "metadata": {}, 7 | "source": [ 8 | "用“图”来定义 Agent 拥有哪些能力（节点）以及这些能力之间可能的跳转路径（边），再用一个全局的“状态”对象来承载流动在图中的信息。图的每一次执行，都是一个节点根据当前状态进行计算，然后更新状态，最后由边根据更新后的状态决定下一个要执行的节点。" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "d2781623-75ca-4f6f-b3de-c72a8c1a76be", 14 | "metadata": {}, 15 | "source": [ 16 | "### state machine (管理什么 what)" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "id": "9ff1a539-c02b-426d-bd94-78c5301a194a", 22 | "metadata": {}, 23 | "source": [ 24 | "状态机是理解 LangGraph 持久化和迭代能力的关键。一个经典的状态机包含三个核心元素：\n", 25 | "- 状态（State）: 系统在任何特定时间点的快照。\n", 26 | "- 转换（Transition）: 从一个状态变为另一个状态的动作或事件。\n", 27 | "- 最终状态（End State）: 系统停止运行的特定状态。\n", 28 | "\n", 29 | "在 LangGraph 中，这个模型被完美地映射了：\n", 30 | "- State (状态): LangGraph 的核心是一个**全局的、可读写**的状态对象。这通常是一个 Pydantic 模型或一个简单的字典。这个对象就是智能体（Agent）的 **“记忆”或“草稿纸”**，它包含了从开始到当前步骤的所有信息，比如用户的原始问题、中间生成的思考步骤、工具调用的结果、历史消息等。图中的每个节点都可以读取和修改这个状态对象。\n", 31 | "- Nodes (节点) as State Transformers (状态转换器): 图中的每个节点（Node）本质上都是一个函数或一个可调用对象。它的核心职责是接收当前的状态对象，执行某些计算（例如调用 LLM、执行工具、处理数据），然后返回一个更新了的对象，这个返回值会被用来更新全局状态。因此，每个节点都是一次状态转换（State Transition），它推动着系统从一个状态演进到下一个状态。\n", 32 | "- Edges (边) as Transition Logic (状态转移逻辑): 边决定了状态转换的方向。特别是条件边（Conditional Edges），它们扮演了状态机中“转换逻辑”的角色。一个节点执行完毕后，条件边会检查当前的状态对象（例如，检查工具调用的输出是什么，或者 LLM 是否决定要结束），然后根据这些信息决定接下来应该流向哪个节点。这就实现了状态机中“当处于状态 A 且发生事件 X 时，转移到状态 B”的逻辑。" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "id": "e93a58fe-f832-4fa0-b9b3-f7b1202c623d", 38 | "metadata": {}, 39 | "source": [ 40 | "### 从图（Graph）的角度理解：定义“如何” (How)" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "id": "93c4eb46-03da-4591-80eb-06778c14962b", 46 | "metadata": {}, 47 | "source": [ 48 | "> LangGraph 的真正威力在于它构建的是一个有向图（Directed Graph），并且允许包含循环（Cycles）。\n", 49 | "- DAG (有向无环图) 的局限: 在 LangGraph 出现之前，LangChain 主要使用 LCEL 来编排组件。LCEL 的本质就是一个 DAG，数据流清晰、单向、无循环。这对于构建简单的调用链（Chains）非常高效，例如“提问 -> 润色 Prompt -> 调用 LLM -> 解析输出”。但对于需要迭代、反思和修正的复杂 Agent 来说，DAG 就力不从心了。比如，Agent 调用一个工具失败了，它需要能够“回到”之前的规划步骤重新规划，这就形成了一个循环，而 DAG 是不允许循环的。\n", 50 | "- LangGraph 的图结构: LangGraph 将计算流程显式地定义为一个图。\n", 51 | " - Nodes (节点): 代表计算单元（“要做什么”）。每个节点都是一个独立的、可复用的功能模块，比如“规划”、“执行工具”、“生成最终答案”等。\n", 52 | " - Edges (边): 代表控制流（“下一步去哪里”）。边将节点连接起来，定义了执行的顺序。这使得整个 Agent 的逻辑流程变得可视化和模块化，非常清晰。\n", 53 | "- 循环（Cycles）是核心突破: LangGraph 的设计明确支持循环。这正是 Agent 实现复杂行为的关键。\n", 54 | " - ReAct 模式: Agent 需要在“思考”和“行动”之间循环。\n", 55 | " - 工具修正: 当工具调用失败或结果不理想时，Agent 可以循环回到“规划”节点。\n", 56 | " - 多步推理: 在解决一个复杂问题时，Agent 可能需要多次调用工具来收集信息，每次调用后都回到一个“评估”节点，判断信息是否足够，形成一个“调用-评估”的循环。" 57 | ] 58 | } 59 | ], 60 | "metadata": { 61 | "kernelspec": { 62 | "display_name": "torch", 63 | "language": "python", 64 | "name": "verl" 65 | }, 66 | "language_info": { 67 | "codemirror_mode": { 68 | "name": "ipython", 69 | "version": 3 70 | }, 71 | "file_extension": ".py", 72 | "mimetype": "text/x-python", 73 | "name": "python", 74 | "nbconvert_exporter": "python", 75 | "pygments_lexer": "ipython3", 76 | "version": "3.10.0" 77 | } 78 | }, 79 | "nbformat": 4, 80 | "nbformat_minor": 5 81 | } 82 | -------------------------------------------------------------------------------- /tutorials/agents/langchain/scripts/chat_react.py: -------------------------------------------------------------------------------- 1 | import os 2 | from langchain_openai import ChatOpenAI 3 | from langchain.agents import tool, initialize_agent, AgentType 4 | from langchain.memory import ConversationBufferMemory 5 | from langchain.callbacks.base import BaseCallbackHandler 6 | from typing import List, Dict, Any 7 | from langchain_core.messages import BaseMessage 8 | from dotenv import load_dotenv 9 | 10 | assert load_dotenv() 11 | 12 | # 确保你已经设置了 OpenAI API 密钥 13 | # os.environ["OPENAI_API_KEY"] = "YOUR_API_KEY" 14 | 15 | # --- 新增部分：自定义回调处理器 --- 16 | class PrintMessagesCallback(BaseCallbackHandler): 17 | """一个简单的回调处理器，用于在聊天模型调用开始时打印消息。""" 18 | def on_chat_model_start( 19 | self, serialized: Dict[str, Any], messages: List[List[BaseMessage]], **kwargs: Any 20 | ) -> Any: 21 | """当聊天模型即将被调用时触发。""" 22 | print("\n--- 发送给 LLM 的完整消息列表 ---") 23 | for role_messages in messages: 24 | for msg in role_messages: 25 | print(f"[{msg.type.upper()}]: {msg.content}") 26 | print("-------------------------------------------------\n") 27 | 28 | # 定义一个简单的工具 29 | @tool 30 | def get_weather(city: str) -> str: 31 | """当需要查询天气时，调用此工具。输入参数是城市名称。""" 32 | if city == "北京": 33 | return "北京今天晴，25摄氏度。" 34 | elif city == "上海": 35 | return "上海今天有雨，20摄氏度。" 36 | else: 37 | return f"抱歉，我没有 {city} 的天气信息。" 38 | 39 | # 1. 初始化 LLM 模型 40 | llm = ChatOpenAI(temperature=0, model_name="gpt-4.1-nano") 41 | 42 | # 2. 定义工具列表 43 | tools = [get_weather] 44 | 45 | # 3. 初始化记忆模块 46 | memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True) 47 | 48 | # 4. 初始化 Agent 49 | agent_chain = initialize_agent( 50 | tools, 51 | llm, 52 | agent=AgentType.CHAT_CONVERSATIONAL_REACT_DESCRIPTION, 53 | verbose=True, 54 | memory=memory, 55 | handle_parsing_errors=True 56 | ) 57 | 58 | # 5. 创建回调处理器的实例 59 | callbacks = [PrintMessagesCallback()] 60 | 61 | print("--- 第一次查询 (无历史记录) ---") 62 | # 在调用时传入 callbacks 63 | response1 = agent_chain.invoke( 64 | {"input": "你好，我想知道北京的天气怎么样？"}, 65 | config={"callbacks": callbacks} 66 | ) 67 | print(f"AI 回答: {response1['output']}") 68 | 69 | print("\n" + "="*50 + "\n") 70 | 71 | print("--- 第二次查询 (利用了第一次的对话历史) ---") 72 | # 再次传入 callbacks 73 | response2 = agent_chain.invoke( 74 | {"input": "那上海呢？"}, 75 | config={"callbacks": callbacks} 76 | ) 77 | print(f"AI 回答: {response2['output']}") -------------------------------------------------------------------------------- /tutorials/agents/planning/imgs/mcts_3nodes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/agents/planning/imgs/mcts_3nodes.png -------------------------------------------------------------------------------- /tutorials/agents/planning/math_tools.py: -------------------------------------------------------------------------------- 1 | from langchain_core.tools import tool 2 | import ast 3 | import operator as op 4 | 5 | # Supported operators mapping AST nodes to functions 6 | _supported_operators = { 7 | ast.Add: op.add, 8 | ast.Sub: op.sub, 9 | ast.Mult: op.mul, 10 | ast.Div: op.truediv, 11 | ast.Pow: op.pow, 12 | ast.USub: op.neg, 13 | } 14 | 15 | def _eval_node(node): 16 | """ 17 | Recursively evaluates an AST node. 18 | This provides a safe alternative to eval(). 19 | """ 20 | if isinstance(node, ast.Constant): 21 | return node.value 22 | elif isinstance(node, ast.BinOp): 23 | return _supported_operators[type(node.op)](_eval_node(node.left), _eval_node(node.right)) 24 | elif isinstance(node, ast.UnaryOp): 25 | return _supported_operators[type(node.op)](_eval_node(node.operand)) 26 | else: 27 | raise TypeError(f"Unsupported node type: {node.__class__.__name__}") 28 | 29 | def _eval_expr(expr: str): 30 | """ 31 | Safely evaluates a string expression using an Abstract Syntax Tree (AST). 32 | """ 33 | return _eval_node(ast.parse(expr, mode='eval').body) 34 | 35 | 36 | @tool 37 | def evaluate_expression(expression: str) -> str: 38 | """ 39 | Evaluates a mathematical expression string and returns the numerical result as a string. 40 | Only supports basic arithmetic operations: +, -, *, /, and ** (power). 41 | Example: evaluate_expression(expression="(2+3)*5") -> "25.0" 42 | """ 43 | try: 44 | result = _eval_expr(expression) 45 | return str(result) 46 | except (TypeError, SyntaxError, KeyError, ZeroDivisionError, RecursionError) as e: 47 | return f"Error: Invalid expression - {e}" 48 | -------------------------------------------------------------------------------- /tutorials/agents/planning/mcts_tree.dot: -------------------------------------------------------------------------------- 1 | digraph { 2 | "129649528035968" [label="Expression: (5*7) + 11 - 3 - 2\nValue: 0.52\nVisits: 6\nScore: 6", style=filled, fillcolor=lightgreen]; 3 | "129649528035968" -> "129649696601568"; 4 | "129649696601568" [label="Expression: (5*7) + 11 + 2 - 3\nValue: 0.50\nVisits: 5\nScore: 4", style=filled, fillcolor=lightgreen]; 5 | "129649696601568" -> "129648918388144"; 6 | "129648918388144" [label="Expression: (5 * 7) + 11 - (3 - 2)\nValue: 0.53\nVisits: 4\nScore: 4", style=filled, fillcolor=lightgreen]; 7 | "129648918388144" -> "129649094742832"; 8 | "129649094742832" [label="Expression: 7 * (5 + 2) - (11 + 3)\nValue: 0.57\nVisits: 3\nScore: 4", style=filled, fillcolor=lightgreen]; 9 | "129649094742832" -> "129648918496832"; 10 | "129648918496832" [label="Expression: 7 * (5 + 3) - (11 + 2)\nValue: 0.65\nVisits: 2\nScore: 3", style=filled, fillcolor=lightgreen]; 11 | "129648918496832" -> "129648918509408"; 12 | "129648918509408" [label="Expression: (7 * 3) + (5 * 2) + 11\nValue: 1.00\nVisits: 1\nScore: 10", style=filled, fillcolor=lightgreen]; 13 | } -------------------------------------------------------------------------------- /tutorials/agents/planning/mcts_tree.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/agents/planning/mcts_tree.pdf -------------------------------------------------------------------------------- /tutorials/agents/planning/pddl/fast-downward.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "2c1f4c17-7b04-4ae4-94f8-6e1f96f2a9e9", 6 | "metadata": {}, 7 | "source": [ 8 | "- exit code\n", 9 | " - https://github.com/aibasel/downward/blob/main/docs/exit-codes.md" 10 | ] 11 | } 12 | ], 13 | "metadata": { 14 | "kernelspec": { 15 | "display_name": "torch", 16 | "language": "python", 17 | "name": "verl" 18 | }, 19 | "language_info": { 20 | "codemirror_mode": { 21 | "name": "ipython", 22 | "version": 3 23 | }, 24 | "file_extension": ".py", 25 | "mimetype": "text/x-python", 26 | "name": "python", 27 | "nbconvert_exporter": "python", 28 | "pygments_lexer": "ipython3", 29 | "version": "3.10.0" 30 | } 31 | }, 32 | "nbformat": 4, 33 | "nbformat_minor": 5 34 | } 35 | -------------------------------------------------------------------------------- /tutorials/agents/planning/pddl/pddlstream.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "088c0d4b-1fb3-46f8-97fd-0f9283b0ff73", 6 | "metadata": {}, 7 | "source": [ 8 | "- pddlstream = pddl + stream\n", 9 | " - domain dynamics (domain.pddl): declare actions\n", 10 | " - stream properties (stream.pddl):\n", 11 | " - declare stream inputs, outputs, and certified facts\n", 12 | " - promblem & stream implementation (problem.py)\n", 13 | " - initial state, python constants, goal formula\n", 14 | " - stream implementation using python generators" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "id": "e9a320ca-9c8d-4c83-83ce-5a2fd740755a", 20 | "metadata": {}, 21 | "source": [ 22 | "### stream: a function to a generator" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 1, 28 | "id": "1b892eae-ae80-4194-a7f5-df0575b3f87f", 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "def stream(x1, x2, x3):\n", 33 | " i = 0\n", 34 | " while True:\n", 35 | " y1 = i * (x1 + x2)\n", 36 | " y2 = i * (x2 + x3)\n", 37 | " yield (y1, y2)\n", 38 | " i += 1" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 9, 44 | "id": "53730789-24d0-4759-aef2-f594f3805bf7", 45 | "metadata": {}, 46 | "outputs": [ 47 | { 48 | "name": "stdout", 49 | "output_type": "stream", 50 | "text": [ 51 | "(0, 0)\n", 52 | "(3, 5)\n", 53 | "(6, 10)\n", 54 | "(9, 15)\n", 55 | "(12, 20)\n", 56 | "(15, 25)\n", 57 | "(18, 30)\n", 58 | "(21, 35)\n", 59 | "(24, 40)\n", 60 | "(27, 45)\n" 61 | ] 62 | } 63 | ], 64 | "source": [ 65 | "gen = stream(1, 2, 3)\n", 66 | "for i in range(10):\n", 67 | " print(next(gen))" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": null, 73 | "id": "e47f109b-6bc4-45ac-ad03-aba7af38f215", 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [] 77 | } 78 | ], 79 | "metadata": { 80 | "kernelspec": { 81 | "display_name": "torch", 82 | "language": "python", 83 | "name": "verl" 84 | }, 85 | "language_info": { 86 | "codemirror_mode": { 87 | "name": "ipython", 88 | "version": 3 89 | }, 90 | "file_extension": ".py", 91 | "mimetype": "text/x-python", 92 | "name": "python", 93 | "nbconvert_exporter": "python", 94 | "pygments_lexer": "ipython3", 95 | "version": "3.10.0" 96 | } 97 | }, 98 | "nbformat": 4, 99 | "nbformat_minor": 5 100 | } 101 | -------------------------------------------------------------------------------- /tutorials/agents/planning/pddl/planning_pddl.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "id": "cea15f37-7143-408a-9ccc-81133e86cfd5", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "from IPython.display import Image" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "id": "3c2caa5c-4c12-48a9-b2a3-f77ebee21aa5", 16 | "metadata": {}, 17 | "source": [ 18 | "- docs\n", 19 | " - https://fareskalaboud.github.io/LearnPDDL/\n", 20 | "- examples\n", 21 | " - https://medium.com/correll-lab/thinking-fast-and-slow-with-llms-and-pddl-111699f9907e\n", 22 | "- A PDDL definition consists of two files:\n", 23 | " - A domain\n", 24 | " - A problem\n", 25 | " - `(: objects`\n", 26 | " - `(: init`\n", 27 | " - `(: goal`\n", 28 | "- online\n", 29 | " - https://editor.planning.domains/#\n", 30 | "- 可以解魔方(3*3*3)" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 3, 36 | "id": "c8f5a8b1-53e5-4346-bd40-0b4d559ce139", 37 | "metadata": {}, 38 | "outputs": [ 39 | { 40 | "data": { 41 | "text/html": [ 42 | "" 43 | ], 44 | "text/plain": [ 45 | "" 46 | ] 47 | }, 48 | "execution_count": 3, 49 | "metadata": {}, 50 | "output_type": "execute_result" 51 | } 52 | ], 53 | "source": [ 54 | "Image(url='https://pic1.zhimg.com/70/v2-ad49e572d846f336cf86c016ce9e3766_1440w.avis?source=172ae18b&biz_tag=Post', width=500)" 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "id": "c829696f-9899-4331-b3fa-73cf3261965b", 60 | "metadata": {}, 61 | "source": [ 62 | "### grammar" 63 | ] 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "id": "0ed02827-f770-4973-9a23-6c4cf34ab2b7", 68 | "metadata": {}, 69 | "source": [ 70 | "- `(:requirements :typing :fluents :negative-preconditions)`\n", 71 | " - :typing：表示领域中使用了 类型（Typing）。这意味着问题中的对象（objects）可以被区分为不同的类别。例如，在积木世界问题中，你可能会定义 block 和 table 两种类型，而不是把它们都当作通用的 object。这有助于更精确地定义动作（actions）和谓词（predicates），使得领域模型更加结构化和严谨。\n", 72 | " - :fluents：表示领域中使用了 流值（Fluents）。在经典的 STRIPS 规划中，状态由一组真/假的命题（谓词）描述。引入 :fluents 后，状态可以包含数值变量，这些变量的值可以通过动作来改变。例如，你可以用一个 fluent 来表示机器人的剩余电量、车辆的油量或者一个计数器的值。这大大增强了 PDDL 的表达能力，使其可以描述更复杂的问题。\n", 73 | " - :negative-preconditions：表示动作的先决条件（preconditions）中可以包含 否定式（Negative Preconditions）。这意味着一个动作的执行不仅可以要求某些条件为真，还可以要求某些条件为假。例如，pick-up 一个积木的动作，它的先决条件可以包括这个积木上没有其他积木 (not (on ?other ?block))。" 74 | ] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "id": "521f3459-8f9f-4e21-95d2-53ca4c214da6", 79 | "metadata": {}, 80 | "source": [ 81 | "- S-expression: https://en.wikipedia.org/wiki/S-expression\n", 82 | "- Predicates are neither variables nor functions, they are statements that are True.\n", 83 | "- `:objects` 列表定义了构成谓词的所有对象。\n", 84 | "- examples\n", 85 | " - https://zhuanlan.zhihu.com/p/647038300" 86 | ] 87 | }, 88 | { 89 | "cell_type": "markdown", 90 | "id": "86061cbb-c5de-4032-88d3-e9167747002b", 91 | "metadata": {}, 92 | "source": [ 93 | "```pddl\n", 94 | "(define (domain move)\n", 95 | " (:requirements :strips)\n", 96 | " (:predicates (robot-at ?x) (box-at ?x) (goal-at ?x))\n", 97 | " (:action move-robot\n", 98 | " :parameters (?from ?to)\n", 99 | " :precondition (robot-at ?from)\n", 100 | " :effect (and (not (robot-at ?from)) (robot-at ?to)))\n", 101 | " (:action move-box\n", 102 | " :parameters (?from ?to)\n", 103 | " :precondition (and (robot-at ?from) (box-at ?from))\n", 104 | " :effect (and (not (box-at ?from)) (box-at ?to) (not (robot-at ?from)) (robot-at ?to)))\n", 105 | ")\n", 106 | "\n", 107 | "\n", 108 | "(define (problem move-1)\n", 109 | " (:domain move)\n", 110 | " (:objects location1 location2 location3)\n", 111 | " (:init (robot-at location1) (box-at location2) (goal-at location3))\n", 112 | " (:goal (and (box-at location3)))\n", 113 | ")\n", 114 | "```" 115 | ] 116 | } 117 | ], 118 | "metadata": { 119 | "kernelspec": { 120 | "display_name": "torch", 121 | "language": "python", 122 | "name": "verl" 123 | }, 124 | "language_info": { 125 | "codemirror_mode": { 126 | "name": "ipython", 127 | "version": 3 128 | }, 129 | "file_extension": ".py", 130 | "mimetype": "text/x-python", 131 | "name": "python", 132 | "nbconvert_exporter": "python", 133 | "pygments_lexer": "ipython3", 134 | "version": "3.10.0" 135 | } 136 | }, 137 | "nbformat": 4, 138 | "nbformat_minor": 5 139 | } 140 | -------------------------------------------------------------------------------- /tutorials/agents/planning/pddl/solver_planutils.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "9d9491c0-f224-4e7d-a5a8-3fb8e156b3ad", 6 | "metadata": {}, 7 | "source": [ 8 | "### fast downward" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "741decd3-7cdf-46ba-8f57-6be2f2ae52e7", 14 | "metadata": {}, 15 | "source": [ 16 | "```\n", 17 | "brew install cmake python\n", 18 | "git clone https://github.com/aibasel/downward.git\n", 19 | "cd downward\n", 20 | "./build.py\n", 21 | "```" 22 | ] 23 | }, 24 | { 25 | "cell_type": "markdown", 26 | "id": "1203a7b9-d11c-4d4f-9eb0-becf2103ecf8", 27 | "metadata": {}, 28 | "source": [ 29 | "### planutils docker" 30 | ] 31 | }, 32 | { 33 | "cell_type": "markdown", 34 | "id": "badc215f-b6a4-45b6-b19f-1daec8844aad", 35 | "metadata": {}, 36 | "source": [ 37 | "```\n", 38 | "docker pull aiplanning/planutils:latest\n", 39 | "docker run -it --privileged aiplanning/planutils:latest bash\n", 40 | "\n", 41 | "planutils activate\n", 42 | "\n", 43 | "planutils install lama\n", 44 | "planutils install popf\n", 45 | "\n", 46 | "\n", 47 | "docker run -v ~/planning_problems:/root/data -it --privileged aiplanning/planutils:latest bash\n", 48 | "```" 49 | ] 50 | } 51 | ], 52 | "metadata": { 53 | "kernelspec": { 54 | "display_name": "torch", 55 | "language": "python", 56 | "name": "verl" 57 | }, 58 | "language_info": { 59 | "codemirror_mode": { 60 | "name": "ipython", 61 | "version": 3 62 | }, 63 | "file_extension": ".py", 64 | "mimetype": "text/x-python", 65 | "name": "python", 66 | "nbconvert_exporter": "python", 67 | "pygments_lexer": "ipython3", 68 | "version": "3.10.0" 69 | } 70 | }, 71 | "nbformat": 4, 72 | "nbformat_minor": 5 73 | } 74 | -------------------------------------------------------------------------------- /tutorials/agents/planning/reasoner_enhenced.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "49471438-c2f1-4699-b139-07944e3349ef", 6 | "metadata": {}, 7 | "source": [ 8 | "> reasoner 在 agent 中的必要性；\n", 9 | "\n", 10 | "- https://medium.com/@deansaco/reasoning-recursive-retrieval-with-deepseek-r1-tavily-and-langgraph-79b3336731e2\n", 11 | " - https://github.com/deansaco/r1-reasoning-rag\n", 12 | " - https://github.com/astordu/r1-reasoning-rag" 13 | ] 14 | } 15 | ], 16 | "metadata": { 17 | "kernelspec": { 18 | "display_name": "torch", 19 | "language": "python", 20 | "name": "verl" 21 | }, 22 | "language_info": { 23 | "codemirror_mode": { 24 | "name": "ipython", 25 | "version": 3 26 | }, 27 | "file_extension": ".py", 28 | "mimetype": "text/x-python", 29 | "name": "python", 30 | "nbconvert_exporter": "python", 31 | "pygments_lexer": "ipython3", 32 | "version": "3.10.0" 33 | } 34 | }, 35 | "nbformat": 4, 36 | "nbformat_minor": 5 37 | } 38 | -------------------------------------------------------------------------------- /tutorials/ai_app/ai-products/NotebookLM.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "b04f24e0-4e4f-40e6-9f60-c92e393126a6", 6 | "metadata": {}, 7 | "source": [ 8 | "- conversional learning：对话式文档学习\n", 9 | "- 基于 Gemini 2.5\n", 10 | "- 14 天免费试用\n", 11 | "- 局限与不足\n", 12 | " - pdf 转换为text时，还是丢了很多格式化信息\n", 13 | " - 比如 latex formula；" 14 | ] 15 | }, 16 | { 17 | "cell_type": "markdown", 18 | "id": "0a3c6572-4d74-466b-8088-9127d7d55c9b", 19 | "metadata": {}, 20 | "source": [ 21 | "- practical usecases\n", 22 | " - 先整理思维导图，形成一个大纲" 23 | ] 24 | } 25 | ], 26 | "metadata": { 27 | "kernelspec": { 28 | "display_name": "Python 3 (ipykernel)", 29 | "language": "python", 30 | "name": "python3" 31 | }, 32 | "language_info": { 33 | "codemirror_mode": { 34 | "name": "ipython", 35 | "version": 3 36 | }, 37 | "file_extension": ".py", 38 | "mimetype": "text/x-python", 39 | "name": "python", 40 | "nbconvert_exporter": "python", 41 | "pygments_lexer": "ipython3", 42 | "version": "3.13.5" 43 | } 44 | }, 45 | "nbformat": 4, 46 | "nbformat_minor": 5 47 | } 48 | -------------------------------------------------------------------------------- /tutorials/ai_app/ai-products/copilot.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "dfa1cb0a-b4d1-4110-b9fa-aaeaba916e52", 7 | "metadata": { 8 | "execution": { 9 | "iopub.execute_input": "2024-07-14T03:38:48.710895Z", 10 | "iopub.status.busy": "2024-07-14T03:38:48.710305Z", 11 | "iopub.status.idle": "2024-07-14T03:38:48.720499Z", 12 | "shell.execute_reply": "2024-07-14T03:38:48.718881Z", 13 | "shell.execute_reply.started": "2024-07-14T03:38:48.710849Z" 14 | } 15 | }, 16 | "outputs": [], 17 | "source": [ 18 | "from IPython.display import Image" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "id": "bbd22bd5", 24 | "metadata": {}, 25 | "source": [ 26 | "- 支持的 ide\n", 27 | " - vscode\n", 28 | " - jetbrains：暂不支持 chat\n", 29 | " - pycharm\n", 30 | " - idea\n", 31 | "\n", 32 | "- 主要作用\n", 33 | " - 协助阅读源码；\n", 34 | " - 自动代码补全；（非常智能）\n", 35 | "\n", 36 | "- 用法\n", 37 | " - `@workspace`：整个工作区的提问；" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "id": "61437bbc-c448-4776-8e6f-66cdf5031ccd", 43 | "metadata": {}, 44 | "source": [ 45 | "## gh cli" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "id": "2ae46090-c019-4187-aedf-6660faf99324", 51 | "metadata": {}, 52 | "source": [ 53 | "> 工作在 cli，也专注于命令行命令的解释、建议与输出；" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "id": "1d80e92e-a0ec-4580-8eea-a70940c279ef", 59 | "metadata": {}, 60 | "source": [ 61 | "### install" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "id": "5df9fc01-6ad7-4e7d-95a3-ccbe4fff6371", 67 | "metadata": {}, 68 | "source": [ 69 | "- install\n", 70 | " - https://github.com/cli/cli/blob/trunk/docs/install_linux.md\n", 71 | "\n", 72 | "```\n", 73 | "$ gh auth login\n", 74 | "\n", 75 | "$ gh pr status\n", 76 | "\n", 77 | "$ gh extension list\n", 78 | "$ gh extension install github/gh-copilot\n", 79 | "$ gh extension install yusukebe/gh-markdown-preview\n", 80 | "$ gh extension list\n", 81 | "```" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "id": "79a8556f-7e3b-45c8-bace-c437e4c49890", 87 | "metadata": {}, 88 | "source": [ 89 | "### usage" 90 | ] 91 | }, 92 | { 93 | "cell_type": "markdown", 94 | "id": "dedb55ff-e688-4482-88eb-3d9974c1c605", 95 | "metadata": {}, 96 | "source": [ 97 | "> 这里其实我们可以通过这样一个成熟的现代AI产品，学习如何对软件开发进行抽象的能力；\n", 98 | "\n", 99 | "- `explain`：解释\n", 100 | " - `$ gh copilot explain \"traceroute github.com\"`\n", 101 | "- `suggest`：交互模式（支持中文输入）\n", 102 | " - `$ gh copilot suggest \"Install git\"`\n", 103 | " - 其会生成一个 suggestion 的命令\n", 104 | " - 进一步可以选择 copy/explain/execute/revise(修改)/rate(评分，feedback)" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 3, 110 | "id": "bb63ceaf-2088-456e-87bc-9391db17572a", 111 | "metadata": { 112 | "execution": { 113 | "iopub.execute_input": "2024-07-14T03:39:55.856536Z", 114 | "iopub.status.busy": "2024-07-14T03:39:55.855834Z", 115 | "iopub.status.idle": "2024-07-14T03:39:55.862842Z", 116 | "shell.execute_reply": "2024-07-14T03:39:55.861601Z", 117 | "shell.execute_reply.started": "2024-07-14T03:39:55.856499Z" 118 | } 119 | }, 120 | "outputs": [ 121 | { 122 | "data": { 123 | "text/html": [ 124 | "" 125 | ], 126 | "text/plain": [ 127 | "" 128 | ] 129 | }, 130 | "execution_count": 3, 131 | "metadata": {}, 132 | "output_type": "execute_result" 133 | } 134 | ], 135 | "source": [ 136 | "Image(url='./imgs/gh_copilot_suggest.png', width=400)" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": null, 142 | "id": "4a45f2a6-64da-4b26-9a9c-69ac24fe4ade", 143 | "metadata": {}, 144 | "outputs": [], 145 | "source": [] 146 | } 147 | ], 148 | "metadata": { 149 | "kernelspec": { 150 | "display_name": "Python 3 (ipykernel)", 151 | "language": "python", 152 | "name": "python3" 153 | }, 154 | "language_info": { 155 | "codemirror_mode": { 156 | "name": "ipython", 157 | "version": 3 158 | }, 159 | "file_extension": ".py", 160 | "mimetype": "text/x-python", 161 | "name": "python", 162 | "nbconvert_exporter": "python", 163 | "pygments_lexer": "ipython3", 164 | "version": "3.10.13" 165 | }, 166 | "toc": { 167 | "base_numbering": 1, 168 | "nav_menu": {}, 169 | "number_sections": true, 170 | "sideBar": true, 171 | "skip_h1_title": false, 172 | "title_cell": "Table of Contents", 173 | "title_sidebar": "Contents", 174 | "toc_cell": false, 175 | "toc_position": {}, 176 | "toc_section_display": true, 177 | "toc_window_display": false 178 | } 179 | }, 180 | "nbformat": 4, 181 | "nbformat_minor": 5 182 | } 183 | -------------------------------------------------------------------------------- /tutorials/ai_app/ai-products/cursor-advanced.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "261b6f95-935b-4ee8-9201-1d7f4c9fe233", 6 | "metadata": {}, 7 | "source": [ 8 | "- ignore files\n", 9 | " - https://docs.cursor.com/context/ignore-files" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "id": "2202d527-d369-4186-aee8-cb56799e3be4", 15 | "metadata": {}, 16 | "source": [ 17 | "### `.cursorrules`" 18 | ] 19 | }, 20 | { 21 | "cell_type": "markdown", 22 | "id": "aced2015-7dab-494c-93da-b21f0b1eff2e", 23 | "metadata": {}, 24 | "source": [ 25 | "- https://cursor.directory/\n", 26 | " - https://cursor.directory/fastapi-python-microservices-serverless-cursor-rules\n", 27 | " - https://cursor.directory/learn\n", 28 | "\n", 29 | "```\n", 30 | "Copy and add a .cursorrules file in the root of your project.\n", 31 | "\n", 32 | "The instructions in the .cursorrules file will be included for features such as Cursor Chat and Ctrl/⌘ K.\n", 33 | "\n", 34 | "The more specific your rules for your project, the better.\n", 35 | "\n", 36 | "Feel free to create your own directory using our template on GitHub.\n", 37 | "```" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "id": "8c46fa26-c6ba-4298-b2c2-ae25d32ec85a", 43 | "metadata": {}, 44 | "source": [ 45 | "### MCP (model context protocol)" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "id": "78439003-97fa-4fef-98ba-f6be5e00feaf", 51 | "metadata": {}, 52 | "source": [ 53 | "> 一种特别的，又 general 的 function calling\n", 54 | "- 实现 agent（computer use）的基础；\n", 55 | " - 学习 mcp，除了更加的自动化外，也是一个再次接触前端的机会；\n", 56 | "- 经典基础 mcp: 配置好之后各自都会提供一组 available tools\n", 57 | " - fetch：网页爬取；\n", 58 | " - https://github.com/modelcontextprotocol/servers/tree/main/src/fetch\n", 59 | " - filesystem：文件读写；\n", 60 | " - https://github.com/modelcontextprotocol/servers/tree/main/src/filesystem" 61 | ] 62 | } 63 | ], 64 | "metadata": { 65 | "kernelspec": { 66 | "display_name": "Python 3 (ipykernel)", 67 | "language": "python", 68 | "name": "python3" 69 | }, 70 | "language_info": { 71 | "codemirror_mode": { 72 | "name": "ipython", 73 | "version": 3 74 | }, 75 | "file_extension": ".py", 76 | "mimetype": "text/x-python", 77 | "name": "python", 78 | "nbconvert_exporter": "python", 79 | "pygments_lexer": "ipython3", 80 | "version": "3.10.0" 81 | } 82 | }, 83 | "nbformat": 4, 84 | "nbformat_minor": 5 85 | } 86 | -------------------------------------------------------------------------------- /tutorials/ai_app/ai-products/cursor-composer.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "1b226f8c-f083-43d3-bf3b-6868940d84a8", 6 | "metadata": {}, 7 | "source": [ 8 | "- cmd + k：行级别\n", 9 | "- cmd + l：文件级别\n", 10 | "- cmd + i：跨文件；" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "id": "0d7c91e2-29df-4851-87f3-aab195a3dddd", 16 | "metadata": {}, 17 | "source": [ 18 | "### composer 与 checkpoint" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "id": "39eb9d8e-2faa-41a2-bb8e-4d432068042c", 24 | "metadata": {}, 25 | "source": [ 26 | "- 适合一个项目的 startup，以搭建起相对完善的模块及框架；\n", 27 | " - composer 直接跨文件编写似乎上下文是整个 codebase；\n", 28 | "- composer\n", 29 | " - normal vs. agent\n", 30 | " - agent: agent composers can use tools\n", 31 | " - 可以新建新的 composer\n", 32 | " - composer history" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "id": "f2f072a8-3d5e-4e16-b06d-85e2d10d2062", 38 | "metadata": {}, 39 | "source": [ 40 | "### new features" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "id": "1fb73684-d3f8-470f-a552-143af574abcb", 46 | "metadata": {}, 47 | "source": [ 48 | "- `Generate Commit Message`\n", 49 | " - 在 Source Control tab" 50 | ] 51 | } 52 | ], 53 | "metadata": { 54 | "kernelspec": { 55 | "display_name": "Python 3 (ipykernel)", 56 | "language": "python", 57 | "name": "python3" 58 | }, 59 | "language_info": { 60 | "codemirror_mode": { 61 | "name": "ipython", 62 | "version": 3 63 | }, 64 | "file_extension": ".py", 65 | "mimetype": "text/x-python", 66 | "name": "python", 67 | "nbconvert_exporter": "python", 68 | "pygments_lexer": "ipython3", 69 | "version": "3.10.13" 70 | } 71 | }, 72 | "nbformat": 4, 73 | "nbformat_minor": 5 74 | } 75 | -------------------------------------------------------------------------------- /tutorials/ai_app/ai-products/cursor.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "f65e2fbc-0bf6-4284-9141-6bb18ab5add7", 7 | "metadata": { 8 | "execution": { 9 | "iopub.execute_input": "2024-12-09T15:01:37.901339Z", 10 | "iopub.status.busy": "2024-12-09T15:01:37.900698Z", 11 | "iopub.status.idle": "2024-12-09T15:01:37.919635Z", 12 | "shell.execute_reply": "2024-12-09T15:01:37.917865Z", 13 | "shell.execute_reply.started": "2024-12-09T15:01:37.901291Z" 14 | } 15 | }, 16 | "outputs": [], 17 | "source": [ 18 | "from IPython.display import Image" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 2, 24 | "id": "dc33b224-8a35-46cf-9f11-bac50ed92def", 25 | "metadata": { 26 | "execution": { 27 | "iopub.execute_input": "2024-12-09T15:01:39.151916Z", 28 | "iopub.status.busy": "2024-12-09T15:01:39.151360Z", 29 | "iopub.status.idle": "2024-12-09T15:01:39.169286Z", 30 | "shell.execute_reply": "2024-12-09T15:01:39.167225Z", 31 | "shell.execute_reply.started": "2024-12-09T15:01:39.151872Z" 32 | } 33 | }, 34 | "outputs": [ 35 | { 36 | "data": { 37 | "text/html": [ 38 | "" 39 | ], 40 | "text/plain": [ 41 | "" 42 | ] 43 | }, 44 | "execution_count": 2, 45 | "metadata": {}, 46 | "output_type": "execute_result" 47 | } 48 | ], 49 | "source": [ 50 | "Image(url='../imgs/tab.jpeg', width=500)" 51 | ] 52 | }, 53 | { 54 | "cell_type": "markdown", 55 | "id": "7b0f672f-c703-4ee1-81bf-a8887876946c", 56 | "metadata": {}, 57 | "source": [ 58 | "### 如何基于 cursor 从 0-1 实现项目开发（1013）" 59 | ] 60 | }, 61 | { 62 | "cell_type": "markdown", 63 | "id": "7f0d8f55-6e68-490a-993d-9b88e48c2e5c", 64 | "metadata": {}, 65 | "source": [ 66 | "- 创建文件夹先写一个 pipeline.md 的文件；\n", 67 | " - 在这个文件中用自然语言写整个项目的目标，及开发上的模块和流程；\n", 68 | " - 思路必须清晰，考验架构、抽象及模块化的能力；" 69 | ] 70 | }, 71 | { 72 | "cell_type": "markdown", 73 | "id": "85d71a79-0874-45c9-96e1-8267b6a3337d", 74 | "metadata": {}, 75 | "source": [ 76 | "## cursor vs. (github) copilot" 77 | ] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "id": "12a814a7-6bd9-4734-b606-6703758745ae", 82 | "metadata": {}, 83 | "source": [ 84 | "> Claude 3.5 sonnect 卓越的代码能力的加持下，使得 Cursor 的代码能力得到显著地飞升；\n", 85 | "\n", 86 | "- 传统的补全\n", 87 | " - words 匹配\n", 88 | "- copilot 只是一个高级的代码补全\n", 89 | " - vscode等其他的 plugin\n", 90 | " - 局部的自动高级补全，基于代码语义\n", 91 | " - 只做补全，接下文；\n", 92 | "- cursor：AI-first code editor\n", 93 | " - features\n", 94 | " - https://www.cursor.com/features\n", 95 | " - 本身就是一个 app （AI-powered IDE），real 结对编程\n", 96 | " - 继承了 vscode（无缝迁移 vscode 的插件）\n", 97 | " - 继承自 vscode 的快捷键\n", 98 | " - ctrl + shift + o：当前文件中定位 symbol\n", 99 | " - ctrl + t：跨文件定位 symbol\n", 100 | " - ctrl + p：基于文件名搜索文件\n", 101 | " - 更全局（global）的 context window\n", 102 | " - context of the entire project\n", 103 | " - 不只是尾部做提示补全，还可在内部做提示，还可做修改；\n", 104 | " - chat tab 支持图片输入；\n", 105 | " - 不需要图片文件，截图的拷贝，右键的拷贝，都可以直接粘贴进对话框中；\n" 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "id": "8189c517-5167-4f45-9712-0462c7732a3b", 111 | "metadata": {}, 112 | "source": [ 113 | "### ctrl + k, ctrl + l" 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "id": "d0f2d5ab-ad3a-4b9a-94fb-4c7e65ea948c", 119 | "metadata": {}, 120 | "source": [ 121 | "- ctrl + l：chat 模式下生成的 code snippets 可以直接 apply\n", 122 | " - tab tab ..." 123 | ] 124 | }, 125 | { 126 | "cell_type": "markdown", 127 | "id": "a920786e-b432-4fe2-b93e-a9bc1b1b3417", 128 | "metadata": {}, 129 | "source": [ 130 | "### prompts" 131 | ] 132 | }, 133 | { 134 | "cell_type": "markdown", 135 | "id": "f201c290-4379-4e94-8c9b-f5c6c8f90b42", 136 | "metadata": {}, 137 | "source": [ 138 | "- 对话框中\n", 139 | " - 复制当前文件中的一段代码，\n", 140 | " - 解释这段代码；\n", 141 | " - 重构这段代码；\n", 142 | "- code editor 基于注释的自动补全：自然语言编程；\n", 143 | " - 先写注释\n", 144 | " - 换行，让其自动补全；\n", 145 | "- RAG\n", 146 | " - 读 api 文档（官方 doc），基于 api 或者官方 doc 写代码" 147 | ] 148 | }, 149 | { 150 | "cell_type": "markdown", 151 | "id": "65a84e5e-c9b1-42dd-aff5-b6cc3c52d5f8", 152 | "metadata": {}, 153 | "source": [ 154 | "### terminal" 155 | ] 156 | }, 157 | { 158 | "cell_type": "markdown", 159 | "id": "afc96c7c-d335-4081-935d-fae896eaf99f", 160 | "metadata": {}, 161 | "source": [ 162 | "- 输入时\n", 163 | " - ⌘ + k：generate a command\n", 164 | " - ⌘ + /：autocomplete\n", 165 | "- 执行命令时\n", 166 | " - 自动提取错误到右侧的 chat 对话框；\n", 167 | " - Please help me debug this code." 168 | ] 169 | }, 170 | { 171 | "cell_type": "markdown", 172 | "id": "cf0955da-f01c-4ead-94c2-b52d87b54786", 173 | "metadata": {}, 174 | "source": [ 175 | "### 3.5 sonnect artifact" 176 | ] 177 | }, 178 | { 179 | "cell_type": "markdown", 180 | "id": "af9e612e-b4cc-4da8-bf00-3920a0bd905e", 181 | "metadata": {}, 182 | "source": [ 183 | "\n", 184 | "- 基于 Mermaid 画流程图（而且是 svg），太秒了；\n", 185 | " - 后续可以基于此，也即是将一些复杂逻辑的代码丢进去，让其先画流程图出来，很好/很快地去理解和消化复杂的代码逻辑；" 186 | ] 187 | } 188 | ], 189 | "metadata": { 190 | "kernelspec": { 191 | "display_name": "Python 3 (ipykernel)", 192 | "language": "python", 193 | "name": "python3" 194 | }, 195 | "language_info": { 196 | "codemirror_mode": { 197 | "name": "ipython", 198 | "version": 3 199 | }, 200 | "file_extension": ".py", 201 | "mimetype": "text/x-python", 202 | "name": "python", 203 | "nbconvert_exporter": "python", 204 | "pygments_lexer": "ipython3", 205 | "version": "3.10.13" 206 | } 207 | }, 208 | "nbformat": 4, 209 | "nbformat_minor": 5 210 | } 211 | -------------------------------------------------------------------------------- /tutorials/ai_app/audio_video/audio.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/ai_app/audio_video/audio.wav -------------------------------------------------------------------------------- /tutorials/ai_app/audio_video/audio_notes.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "36f6b62e-5505-42e0-bc33-6119bb1e3b8e", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "from IPython.display import Image" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "id": "88ded059-af1a-4433-8356-1a806f8a8e2c", 16 | "metadata": {}, 17 | "source": [ 18 | "- https://github.com/harry0703/AudioNotes.git" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 2, 24 | "id": "32980ce2-581b-47dd-aa5e-884a2c80c232", 25 | "metadata": {}, 26 | "outputs": [ 27 | { 28 | "data": { 29 | "text/html": [ 30 | "" 31 | ], 32 | "text/plain": [ 33 | "" 34 | ] 35 | }, 36 | "execution_count": 2, 37 | "metadata": {}, 38 | "output_type": "execute_result" 39 | } 40 | ], 41 | "source": [ 42 | "Image(url='./imgs/audio-notes-framework.png', width=400)" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "id": "21f3d25d-d355-4dc9-bfd1-abd8a70572b8", 48 | "metadata": {}, 49 | "source": [ 50 | "```python\n", 51 | "@cl.on_chat_start\n", 52 | "async def on_chat_start():\n", 53 | " ...\n", 54 | " asr_result = await transcribe_file(file)\n", 55 | " await summarize_notes(asr_result)\n", 56 | " await msg.send()\n", 57 | " ...\n", 58 | "```" 59 | ] 60 | }, 61 | { 62 | "cell_type": "markdown", 63 | "id": "afdc3984-10fe-475e-9e3c-2d715b6d26b9", 64 | "metadata": {}, 65 | "source": [ 66 | "### transcribe_file" 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "id": "952f14be-b122-42ce-b6cf-924cbf80644e", 72 | "metadata": {}, 73 | "source": [ 74 | "```python\n", 75 | "from funasr import AutoModel\n", 76 | "...\n", 77 | " self.__model = AutoModel(model=\"paraformer-zh\",\n", 78 | " vad_model=\"fsmn-vad\",\n", 79 | " punc_model=\"ct-punc\",\n", 80 | " spk_model=\"cam++\",\n", 81 | " log_level=\"error\",\n", 82 | " hub=\"ms\" # hub：表示模型仓库，ms为选择modelscope下载，hf为选择huggingface下载。\n", 83 | " )\n", 84 | "```\n", 85 | "\n", 86 | "- `~/.cache/modelscope/hub/models`" 87 | ] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "id": "5aa79b0e-a282-4471-898a-f2f15b8bf703", 92 | "metadata": {}, 93 | "source": [ 94 | "### chat " 95 | ] 96 | }, 97 | { 98 | "cell_type": "markdown", 99 | "id": "2a024a88-ff8f-4bfb-80b2-3f7bcfdca6e4", 100 | "metadata": {}, 101 | "source": [ 102 | "```python\n", 103 | "async def summarize_notes(text):\n", 104 | " messages = [\n", 105 | " {\"role\": \"system\", \"content\": \"你是一名笔记整理专家，根据用户提供的内容，整理出一份内容详尽的结构化的笔记\"},\n", 106 | " {\"role\": \"user\", \"content\": text},\n", 107 | " ]\n", 108 | "```" 109 | ] 110 | } 111 | ], 112 | "metadata": { 113 | "kernelspec": { 114 | "display_name": "Python 3 (ipykernel)", 115 | "language": "python", 116 | "name": "python3" 117 | }, 118 | "language_info": { 119 | "codemirror_mode": { 120 | "name": "ipython", 121 | "version": 3 122 | }, 123 | "file_extension": ".py", 124 | "mimetype": "text/x-python", 125 | "name": "python", 126 | "nbconvert_exporter": "python", 127 | "pygments_lexer": "ipython3", 128 | "version": "3.10.0" 129 | } 130 | }, 131 | "nbformat": 4, 132 | "nbformat_minor": 5 133 | } 134 | -------------------------------------------------------------------------------- /tutorials/ai_app/audio_video/imgs/audio-notes-framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/ai_app/audio_video/imgs/audio-notes-framework.png -------------------------------------------------------------------------------- /tutorials/ai_app/audio_video/speech_generation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "7847bdbc-4bb8-4b5d-84be-60c37b11f3fb", 6 | "metadata": {}, 7 | "source": [ 8 | "- 声音克隆\n", 9 | " - https://github.com/FunAudioLLM/CosyVoice\n", 10 | " - https://funaudiollm.github.io/" 11 | ] 12 | } 13 | ], 14 | "metadata": { 15 | "kernelspec": { 16 | "display_name": "Python 3 (ipykernel)", 17 | "language": "python", 18 | "name": "python3" 19 | }, 20 | "language_info": { 21 | "codemirror_mode": { 22 | "name": "ipython", 23 | "version": 3 24 | }, 25 | "file_extension": ".py", 26 | "mimetype": "text/x-python", 27 | "name": "python", 28 | "nbconvert_exporter": "python", 29 | "pygments_lexer": "ipython3", 30 | "version": "3.10.13" 31 | } 32 | }, 33 | "nbformat": 4, 34 | "nbformat_minor": 5 35 | } 36 | -------------------------------------------------------------------------------- /tutorials/ai_app/audio_video/video.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/ai_app/audio_video/video.mp4 -------------------------------------------------------------------------------- /tutorials/ai_app/audio_video/whisper_whisperx.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "baa28a74-f874-42e4-a57d-b48a0af5b229", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "# pip install pytube moviepy openai-whisper" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 2, 16 | "id": "1f12058d-29cd-4d37-90ad-f0440436a971", 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "# !pip install you-get\n", 21 | "# !pip install yt-dlp" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "id": "5101bc6a-f847-4f53-bbc9-e8b5b6a3155a", 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "import pytube\n", 32 | "import moviepy.editor as mp\n", 33 | "import whisper\n", 34 | "import os\n", 35 | "import subprocess\n", 36 | "\n", 37 | "\n", 38 | "# 使用 yt-dlp 下载视频\n", 39 | "def download_video(url, output_path='video.mp4'):\n", 40 | " subprocess.run(['yt-dlp', '-o', output_path, url])\n", 41 | " return output_path\n", 42 | "\n", 43 | "# 从视频中提取音频\n", 44 | "def extract_audio(video_path, output_path='audio.wav'):\n", 45 | " video = mp.VideoFileClip(video_path)\n", 46 | " video.audio.write_audiofile(output_path)\n", 47 | " return output_path\n", 48 | "\n", 49 | "# 转录音频\n", 50 | "def transcribe_audio_with_whisper(audio_path):\n", 51 | " model = whisper.load_model(\"large\")\n", 52 | " # result = model.transcribe(audio_path, language='en')\n", 53 | " result = model.transcribe(audio_path)\n", 54 | " return result['text']" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 2, 60 | "id": "6912b1ae-ed9d-4947-8e7b-f16ccdbddfeb", 61 | "metadata": {}, 62 | "outputs": [ 63 | { 64 | "name": "stdout", 65 | "output_type": "stream", 66 | "text": [ 67 | "Downloading video...\n", 68 | "[BiliBili] Extracting URL: https://www.bilibili.com/video/BV12J4m1N7wU/\n", 69 | "[BiliBili] 12J4m1N7wU: Downloading webpage\n", 70 | "[BiliBili] BV12J4m1N7wU: Extracting videos in anthology\n", 71 | "[BiliBili] 1254605812: Extracting chapters\n", 72 | "[BiliBili] Format(s) 1080P 高码率, 1080P 高清, 720P 高清 are missing; you have to login or become premium member to download them. Use --cookies-from-browser or --cookies for the authentication. See https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for how to manually pass cookies\n", 73 | "[info] BV12J4m1N7wU: Downloading 1 format(s): 100110+30280\n", 74 | "[download] Destination: ./video.f100110.mp4\n", 75 | "[download] 100% of 425.01KiB in 00:00:00 at 7.64MiB/s \n", 76 | "[download] Destination: ./video.f30280.m4a\n", 77 | "[download] 100% of 274.45KiB in 00:00:00 at 10.31MiB/s \n", 78 | "[Merger] Merging formats into \"./video.mp4\"\n", 79 | "Deleting original file ./video.f30280.m4a (pass -k to keep)\n", 80 | "Deleting original file ./video.f100110.mp4 (pass -k to keep)\n", 81 | "Extracting audio...\n", 82 | "MoviePy - Writing audio in ./audio.wav\n" 83 | ] 84 | }, 85 | { 86 | "name": "stderr", 87 | "output_type": "stream", 88 | "text": [ 89 | " \r" 90 | ] 91 | }, 92 | { 93 | "name": "stdout", 94 | "output_type": "stream", 95 | "text": [ 96 | "MoviePy - Done.\n", 97 | "Transcribing audio with Whisper...\n", 98 | "Transcription:\n", 99 | " Saying goodbye, I don't think is ever nice, but saying goodbye without Feeling sad or feeling hurt or whatever that would that would just mean that the time we spent together was Right great and had a great time. So it was always clear that it will be tough and I know it will be tough\n" 100 | ] 101 | } 102 | ], 103 | "source": [ 104 | "\n", 105 | "video_url = 'https://www.bilibili.com/video/BV12J4m1N7wU/'\n", 106 | "video_path = './video.mp4'\n", 107 | "audio_path = './audio.wav'\n", 108 | "\n", 109 | "print(\"Downloading video...\")\n", 110 | "download_video(video_url, video_path)\n", 111 | "\n", 112 | "print(\"Extracting audio...\")\n", 113 | "extract_audio(video_path, audio_path)\n", 114 | "\n", 115 | "print(\"Transcribing audio with Whisper...\")\n", 116 | "transcription = transcribe_audio_with_whisper(audio_path)\n", 117 | "\n", 118 | "print(\"Transcription:\")\n", 119 | "print(transcription)\n", 120 | "\n", 121 | "# 清理临时文件\n", 122 | "# os.remove(video_path)\n", 123 | "# os.remove(audio_path)\n" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "id": "21bbd3ef-11f5-4238-b128-e626a60d9fed", 129 | "metadata": {}, 130 | "source": [ 131 | "### whisper" 132 | ] 133 | }, 134 | { 135 | "cell_type": "markdown", 136 | "id": "a38dddf6-8349-415a-863f-e46a4e8d3b22", 137 | "metadata": {}, 138 | "source": [ 139 | "- `--task`\n", 140 | " - transcribe：转录\n", 141 | " - translate：翻译；" 142 | ] 143 | }, 144 | { 145 | "cell_type": "markdown", 146 | "id": "311847fb-838b-4cc9-8446-42e0e1979f53", 147 | "metadata": {}, 148 | "source": [ 149 | "### whisperx" 150 | ] 151 | }, 152 | { 153 | "cell_type": "markdown", 154 | "id": "74aa0570-2c77-49d6-94ef-660b19e67db2", 155 | "metadata": {}, 156 | "source": [ 157 | "- https://github.com/m-bain/whisperX" 158 | ] 159 | }, 160 | { 161 | "cell_type": "markdown", 162 | "id": "f37ccfab-b771-476a-8eb2-03d01d0fd831", 163 | "metadata": {}, 164 | "source": [ 165 | "- 说话人分离（--diarize）\n", 166 | " - 注意一些 hf 的 models，也许需要在 hf 页面做申请，比如 `pyannote/segmentation-3.0`\n", 167 | " - 也许需要在环境变量中配置 `hf_token`\n", 168 | "- srt file：（--highlight_words）" 169 | ] 170 | } 171 | ], 172 | "metadata": { 173 | "kernelspec": { 174 | "display_name": "Python 3 (ipykernel)", 175 | "language": "python", 176 | "name": "python3" 177 | }, 178 | "language_info": { 179 | "codemirror_mode": { 180 | "name": "ipython", 181 | "version": 3 182 | }, 183 | "file_extension": ".py", 184 | "mimetype": "text/x-python", 185 | "name": "python", 186 | "nbconvert_exporter": "python", 187 | "pygments_lexer": "ipython3", 188 | "version": "3.10.13" 189 | } 190 | }, 191 | "nbformat": 4, 192 | "nbformat_minor": 5 193 | } 194 | -------------------------------------------------------------------------------- /tutorials/ai_app/claude-code.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "0cf33949-2cdb-4a1e-abd6-435230212a2e", 6 | "metadata": {}, 7 | "source": [ 8 | "```shell\n", 9 | "npm install -g @anthropic-ai/claude-code \n", 10 | "claude --version\n", 11 | "\n", 12 | "export ANTHROPIC_AUTH_TOKEN=你的月之暗面 API Key\n", 13 | "export ANTHROPIC_BASE_URL=https://api.moonshot.cn/anthropic\n", 14 | "claude\n", 15 | "```\n", 16 | "\n", 17 | "- 哪里对模型进行了选择：`kimi-k2-0711-preview`\n", 18 | " - `https://api.moonshot.cn/anthropic`跟这个端点有关？\n", 19 | "- kimi api\n", 20 | " - https://platform.moonshot.cn/console/api-keys\n", 21 | " - https://platform.moonshot.ai/docs/guide/agent-support#direct-api-usage-for-kimi-k2-0711-preview-model\n", 22 | " - https://api.moonshot.cn/v1\n", 23 | "- deepseek api\n", 24 | " - https://api-docs.deepseek.com/zh-cn/guides/anthropic_api\n", 25 | "- examples\n", 26 | " - https://mp.weixin.qq.com/s/S5SKpzQ8GFGzv6dO6efnFA" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "id": "ebf21fa5-8390-4fff-b8c4-caee256e5321", 32 | "metadata": {}, 33 | "source": [ 34 | "### 配置 mcp servers" 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "id": "53879969-e5c6-45ce-a5c2-947d24d26756", 40 | "metadata": {}, 41 | "source": [ 42 | "- `claude mcp list`: `~/.claude.json`\n", 43 | " - `claude mcp add fetch -- uvx mcp-server-fetch`\n", 44 | " - `claude mcp add filesystem -- npx -y @modelcontextprotocol/server-filesystem /Users/liyanyu/Downloads`\n", 45 | " - `claude mcp add context7 -- npx -y @upstash/context7-mcp`" 46 | ] 47 | } 48 | ], 49 | "metadata": { 50 | "kernelspec": { 51 | "display_name": "Python 3 (ipykernel)", 52 | "language": "python", 53 | "name": "python3" 54 | }, 55 | "language_info": { 56 | "codemirror_mode": { 57 | "name": "ipython", 58 | "version": 3 59 | }, 60 | "file_extension": ".py", 61 | "mimetype": "text/x-python", 62 | "name": "python", 63 | "nbconvert_exporter": "python", 64 | "pygments_lexer": "ipython3", 65 | "version": "3.13.5" 66 | } 67 | }, 68 | "nbformat": 4, 69 | "nbformat_minor": 5 70 | } 71 | -------------------------------------------------------------------------------- /tutorials/ai_app/codex.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "e1e73914-087a-4908-92cb-9ed5c5ea217a", 6 | "metadata": {}, 7 | "source": [ 8 | "- `npm i -g @openai/codex`\n", 9 | " - chatgpt plus 用户\n", 10 | "- slash\n", 11 | " - /model：select model\n", 12 | " - /init：创建 `AGENTS.md`" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "id": "7397ee41-1ed1-4f13-9d04-2decbd7496e3", 18 | "metadata": {}, 19 | "source": [ 20 | "### codex in cursor" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "id": "7b9c7e15-0565-420c-8d07-48cc12c51d0f", 26 | "metadata": {}, 27 | "source": [ 28 | "- 在 cursor 插件市场里搜索 openai codex\n", 29 | " - 在左侧面板里会看到 openai 的 logo" 30 | ] 31 | } 32 | ], 33 | "metadata": { 34 | "kernelspec": { 35 | "display_name": "torch", 36 | "language": "python", 37 | "name": "verl" 38 | }, 39 | "language_info": { 40 | "codemirror_mode": { 41 | "name": "ipython", 42 | "version": 3 43 | }, 44 | "file_extension": ".py", 45 | "mimetype": "text/x-python", 46 | "name": "python", 47 | "nbconvert_exporter": "python", 48 | "pygments_lexer": "ipython3", 49 | "version": "3.10.0" 50 | } 51 | }, 52 | "nbformat": 4, 53 | "nbformat_minor": 5 54 | } 55 | -------------------------------------------------------------------------------- /tutorials/ai_app/db/postgresql.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "82bfff79-6e0b-4617-b05a-5645d55c5a81", 6 | "metadata": {}, 7 | "source": [ 8 | "```sh\n", 9 | "psql -U user_name -d db_name\n", 10 | "\\list # list db\n", 11 | "\\c db_name # change db\n", 12 | "\\dt # 列出当前数据库中的所有表 \n", 13 | "\\d table_name # 查看特定表的详细结构\n", 14 | "\n", 15 | "\\q # quit\n", 16 | "```" 17 | ] 18 | } 19 | ], 20 | "metadata": { 21 | "kernelspec": { 22 | "display_name": "Python 3 (ipykernel)", 23 | "language": "python", 24 | "name": "python3" 25 | }, 26 | "language_info": { 27 | "codemirror_mode": { 28 | "name": "ipython", 29 | "version": 3 30 | }, 31 | "file_extension": ".py", 32 | "mimetype": "text/x-python", 33 | "name": "python", 34 | "nbconvert_exporter": "python", 35 | "pygments_lexer": "ipython3", 36 | "version": "3.10.0" 37 | } 38 | }, 39 | "nbformat": 4, 40 | "nbformat_minor": 5 41 | } 42 | -------------------------------------------------------------------------------- /tutorials/ai_app/gemini-cli.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "56b0a4df-78cf-4cf1-bc8b-70bd933cbb36", 6 | "metadata": {}, 7 | "source": [ 8 | "> Agent in the terminal ~\n", 9 | "\n", 10 | "- 安装与升级\n", 11 | "\n", 12 | "```shell\n", 13 | "npm upgrade -g @google/gemini-cli\n", 14 | "```\n", 15 | "\n", 16 | "- `~/.gemini`\n", 17 | " - `settings.json`：全局配置文件\n", 18 | " - mcpServers 在这里配置\n", 19 | "- `.env`\n", 20 | " - `GEMINI_API_KEY`\n", 21 | " - `GEMINI_MODEL`\n", 22 | "- 源码\n", 23 | " - gemini-cli prompts 设计\n", 24 | " - https://github.com/google-gemini/gemini-cli/blob/main/packages/core/src/core/prompts.ts\n", 25 | " - system prompt\n", 26 | " - https://github.com/google-gemini/gemini-cli/blob/main/packages/core/src/core/prompts.ts#L40\n", 27 | " - CompressionPrompt\n", 28 | " - https://github.com/google-gemini/gemini-cli/blob/main/packages/core/src/core/prompts.ts#L303C20-L361\n", 29 | " - tokenLimit\n", 30 | " - https://github.com/google-gemini/gemini-cli/blob/main/packages/core/src/core/tokenLimits.ts\n", 31 | "- sandbox 环境\n", 32 | " - gemini -s\n", 33 | " - gemini -s -p 'hello'" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "id": "6650406d-5a95-4adc-b548-3b845cc2eb40", 39 | "metadata": {}, 40 | "source": [ 41 | "- tasks\n", 42 | " - `> 请翻译 的内容为中文，并保存为 Markdown 文件`" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "id": "c0d18eac-f4f5-45fc-8af8-4d780f6f5943", 48 | "metadata": {}, 49 | "source": [ 50 | "- `/tools`：自动配置\n", 51 | " - Available Gemini CLI tools:\n", 52 | " - shell\n", 53 | " - google search\n", 54 | "- `/mcp`：手动配置" 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "id": "7fffb134-d674-48a1-b718-b83b69fdcbb9", 60 | "metadata": {}, 61 | "source": [ 62 | "### 命令行参数" 63 | ] 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "id": "e4946504-7142-43af-bdbf-b1f3bfb1a83c", 68 | "metadata": {}, 69 | "source": [ 70 | "- `--model`\n", 71 | "- `--prompt`：直接回答问题，启用非交互模式\n", 72 | "- `--sandbox`\n", 73 | "- `--all_files`：递归包含当前目录中的所有文件作为上下文\n", 74 | "- `--yolo`：you only live once，自动 approve 所有工具调用" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "id": "ba08f041-f78b-4b8e-a7df-dca61bcc8301", 80 | "metadata": {}, 81 | "source": [ 82 | "### GEMINI.md" 83 | ] 84 | }, 85 | { 86 | "cell_type": "markdown", 87 | "id": "ffad8945-1473-49a3-9d8a-10b85ad592a8", 88 | "metadata": {}, 89 | "source": [ 90 | "```\n", 91 | "1. 解释部分用中文，命令及代码用英文\n", 92 | "```\n", 93 | "- working directory\n", 94 | "- `~/.gemini/GEMINI.md`\n", 95 | " - memory, user facts" 96 | ] 97 | }, 98 | { 99 | "cell_type": "markdown", 100 | "id": "3357ab4b-c3ac-4934-99da-b490a342a626", 101 | "metadata": {}, 102 | "source": [ 103 | "### multi-modality" 104 | ] 105 | }, 106 | { 107 | "cell_type": "markdown", 108 | "id": "a414aea7-41ba-4705-8c1c-fdf50f01ceaf", 109 | "metadata": {}, 110 | "source": [ 111 | "- 通过工具解析文件（jpg、pdf、mp3...）\n", 112 | " - `ReadManyFiles`\n", 113 | " - 文本 / SVG → 直接合并为字符串，插入到 prompt\n", 114 | " - 图片 / PDF / 音视频 → 读取为 Base64，封装成\n", 115 | "```json\n", 116 | "{\n", 117 | " inlineData: { mimeType: '<对应 MIME>', data: '' }\n", 118 | "}\n", 119 | "```" 120 | ] 121 | }, 122 | { 123 | "cell_type": "markdown", 124 | "id": "4e2031c5-e794-452a-b9d5-4082cd51dbbc", 125 | "metadata": {}, 126 | "source": [ 127 | "### 核心源码" 128 | ] 129 | }, 130 | { 131 | "cell_type": "markdown", 132 | "id": "f1d26edc-4153-40d6-8833-1043b766a1cd", 133 | "metadata": {}, 134 | "source": [ 135 | "- config.ts\n", 136 | " - registerCoreTool 各种 tool" 137 | ] 138 | }, 139 | { 140 | "cell_type": "markdown", 141 | "id": "9910c48b-0ba2-46a6-b18a-05d4831e9caa", 142 | "metadata": {}, 143 | "source": [ 144 | "### memory tool" 145 | ] 146 | }, 147 | { 148 | "cell_type": "markdown", 149 | "id": "57b853fd-7c70-4a68-abed-14bfe536495c", 150 | "metadata": {}, 151 | "source": [ 152 | "```\n", 153 | "> remember that I'am 五道口纳什\n", 154 | "> Save Memory {\"fact\":\"My name is 五道口纳什\"}\n", 155 | "> 我是谁\n", 156 | "> 根据我记住的信息，你的名字是五道口纳什。\n", 157 | "```\n", 158 | "\n", 159 | "- https://github.com/google-gemini/gemini-cli/blob/main/packages/core/src/core/prompts.ts#L104\n", 160 | "- https://github.com/google-gemini/gemini-cli/blob/main/packages/core/src/tools/memoryTool.ts" 161 | ] 162 | }, 163 | { 164 | "cell_type": "markdown", 165 | "id": "b05a9643-a2ba-4b34-abba-e554469bc312", 166 | "metadata": {}, 167 | "source": [ 168 | "### plan mode & explain mode" 169 | ] 170 | }, 171 | { 172 | "cell_type": "markdown", 173 | "id": "504c96fc-b096-44d6-be2d-4ce077b76984", 174 | "metadata": {}, 175 | "source": [ 176 | "- https://gist.github.com/philschmid/379cf06d9d18a1ed67ff360118a575e5\n", 177 | " - `gemini.md`\n", 178 | "- https://gist.github.com/philschmid/64ed5dd32ce741b0f97f00e9abfa2a30" 179 | ] 180 | } 181 | ], 182 | "metadata": { 183 | "kernelspec": { 184 | "display_name": "Python 3 (ipykernel)", 185 | "language": "python", 186 | "name": "python3" 187 | }, 188 | "language_info": { 189 | "codemirror_mode": { 190 | "name": "ipython", 191 | "version": 3 192 | }, 193 | "file_extension": ".py", 194 | "mimetype": "text/x-python", 195 | "name": "python", 196 | "nbconvert_exporter": "python", 197 | "pygments_lexer": "ipython3", 198 | "version": "3.13.5" 199 | } 200 | }, 201 | "nbformat": 4, 202 | "nbformat_minor": 5 203 | } 204 | -------------------------------------------------------------------------------- /tutorials/ai_app/imgs/0_Parade_Parade_0_873.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/ai_app/imgs/0_Parade_Parade_0_873.jpg -------------------------------------------------------------------------------- /tutorials/ai_app/imgs/0_Parade_marchingband_1_765.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/ai_app/imgs/0_Parade_marchingband_1_765.jpg -------------------------------------------------------------------------------- /tutorials/ai_app/imgs/11_Meeting_Meeting_11_Meeting_Meeting_11_176.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/ai_app/imgs/11_Meeting_Meeting_11_Meeting_Meeting_11_176.jpg -------------------------------------------------------------------------------- /tutorials/ai_app/imgs/11_Meeting_Meeting_11_Meeting_Meeting_11_26.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/ai_app/imgs/11_Meeting_Meeting_11_Meeting_Meeting_11_26.jpg -------------------------------------------------------------------------------- /tutorials/ai_app/imgs/gh_copilot_suggest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/ai_app/imgs/gh_copilot_suggest.png -------------------------------------------------------------------------------- /tutorials/ai_app/imgs/tab.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/ai_app/imgs/tab.jpeg -------------------------------------------------------------------------------- /tutorials/ai_app/openrouter.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "c95eb1c2-1a8a-4ad6-95bc-23b9abe507ba", 6 | "metadata": {}, 7 | "source": [ 8 | "> A unified interface for LLMs。大模型的中间商，也可能是中间商的中间商；\n", 9 | "\n", 10 | "- 不是所有的模型都可以方便地本地部署 (*2 gpu memory)\n", 11 | " - qwen/qwen-2.5-72b-instruct\n", 12 | " - deepseek-v3；\n", 13 | " - llama3.1-405b；\n", 14 | "- 远端api快速验证，evaluate，科研或者工程；\n", 15 | "- 虚拟信用卡\n", 16 | " - https://bewildcard.com/i/CHUNHUI3\n", 17 | "- 关于一个模型的不同 provider 的路由策略（provider routing）\n", 18 | " - https://openrouter.ai/docs/provider-routing" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 3, 24 | "id": "8839a1bb-019b-42be-bbf3-5ed0ee5942d5", 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "from openai import OpenAI" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 4, 34 | "id": "449661f7-3031-4ccd-bb00-9d625f7ff440", 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "from dotenv import load_dotenv, find_dotenv\n", 39 | "import os" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 5, 45 | "id": "24dba6a5-9fc6-4e00-8fad-0b5c5c284c41", 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "assert load_dotenv(find_dotenv())" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 7, 55 | "id": "3cc1ad49-dfd8-44a8-8a8b-accad01b7b8d", 56 | "metadata": {}, 57 | "outputs": [ 58 | { 59 | "name": "stdout", 60 | "output_type": "stream", 61 | "text": [ 62 | "I am an instance of OpenAI's language model, specifically based on the GPT-4 architecture. My design allows me to understand and generate human-like text based on the input I receive. I can assist with a wide range of tasks, from answering questions and providing explanations to generating creative content and offering advice. Let me know how I can help you today!\n" 63 | ] 64 | } 65 | ], 66 | "source": [ 67 | "client = OpenAI(\n", 68 | " base_url=\"https://openrouter.ai/api/v1\",\n", 69 | " api_key=os.getenv('OPENROUTER_API_KEY'),\n", 70 | ")\n", 71 | "\n", 72 | "completion = client.chat.completions.create(\n", 73 | " model=\"deepseek/deepseek-chat\",\n", 74 | " messages=[\n", 75 | " {\n", 76 | " \"role\": \"user\",\n", 77 | " \"content\": \"what model are you?\"\n", 78 | " }\n", 79 | " ]\n", 80 | ")\n", 81 | "print(completion.choices[0].message.content)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "id": "08e42d1d-2faf-41ee-b867-f4786bfb6c5b", 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [] 91 | } 92 | ], 93 | "metadata": { 94 | "kernelspec": { 95 | "display_name": "Python 3 (ipykernel)", 96 | "language": "python", 97 | "name": "python3" 98 | }, 99 | "language_info": { 100 | "codemirror_mode": { 101 | "name": "ipython", 102 | "version": 3 103 | }, 104 | "file_extension": ".py", 105 | "mimetype": "text/x-python", 106 | "name": "python", 107 | "nbconvert_exporter": "python", 108 | "pygments_lexer": "ipython3", 109 | "version": "3.10.13" 110 | } 111 | }, 112 | "nbformat": 4, 113 | "nbformat_minor": 5 114 | } 115 | -------------------------------------------------------------------------------- /tutorials/ai_app/pipeline_工具.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "8f0e85be-7fb0-4027-b1cc-b80aa7c8912c", 6 | "metadata": {}, 7 | "source": [ 8 | "### 微信聊天记录导出" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "adfa7307-2681-4b57-9baf-94784dd47f46", 14 | "metadata": {}, 15 | "source": [ 16 | "- https://github.com/LC044/WeChatMsg\n", 17 | " - 甚至可以导出为支持 GLM 训练的 json 格式；" 18 | ] 19 | }, 20 | { 21 | "cell_type": "markdown", 22 | "id": "4af4610d-dfbf-4cad-a553-aea251628a20", 23 | "metadata": {}, 24 | "source": [ 25 | "### misc" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "id": "4621f7d2-179f-417b-aecb-f1c1de1b0697", 31 | "metadata": {}, 32 | "source": [ 33 | "- markitdown\n", 34 | " - https://github.com/microsoft/markitdown\n", 35 | " - 论文 pdf => markdown\n", 36 | " - latex/formula 转换效果不佳；\n", 37 | " - 轻量级，简单但也能力有限；\n", 38 | "- **MinerU**：挺惊艳的；\n", 39 | " - https://github.com/opendatalab/MinerU\n", 40 | " - 环境配置\n", 41 | " - model weights download\n", 42 | " - https://github.com/opendatalab/MinerU/blob/master/docs/how_to_download_models_en.md\n", 43 | " - 配置文件：`~/magic-pdf.json`\n", 44 | " - 命令行使用\n", 45 | " - https://mineru.readthedocs.io/en/latest/user_guide/quick_start/command_line.html" 46 | ] 47 | } 48 | ], 49 | "metadata": { 50 | "kernelspec": { 51 | "display_name": "Python 3 (ipykernel)", 52 | "language": "python", 53 | "name": "python3" 54 | }, 55 | "language_info": { 56 | "codemirror_mode": { 57 | "name": "ipython", 58 | "version": 3 59 | }, 60 | "file_extension": ".py", 61 | "mimetype": "text/x-python", 62 | "name": "python", 63 | "nbconvert_exporter": "python", 64 | "pygments_lexer": "ipython3", 65 | "version": "3.10.13" 66 | } 67 | }, 68 | "nbformat": 4, 69 | "nbformat_minor": 5 70 | } 71 | -------------------------------------------------------------------------------- /tutorials/ai_app/streamlit(web).ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "9c559a42-c664-44da-b22d-7f69a67d3f4f", 7 | "metadata": { 8 | "execution": { 9 | "iopub.execute_input": "2024-12-14T03:31:31.255262Z", 10 | "iopub.status.busy": "2024-12-14T03:31:31.254614Z", 11 | "iopub.status.idle": "2024-12-14T03:31:31.473634Z", 12 | "shell.execute_reply": "2024-12-14T03:31:31.472825Z", 13 | "shell.execute_reply.started": "2024-12-14T03:31:31.255226Z" 14 | } 15 | }, 16 | "outputs": [], 17 | "source": [ 18 | "import streamlit as st" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "id": "0f7df014-bb4b-48c8-97c8-5b155f4d907a", 24 | "metadata": {}, 25 | "source": [ 26 | "```\n", 27 | "st.set_page_config(\n", 28 | "st.title(\n", 29 | "st.markdown(\n", 30 | "user_query = st.text_input(\"Enter your query:\", placeholder=\"e.g., How many 'R's are in the word strawberry?\")\n", 31 | "```" 32 | ] 33 | }, 34 | { 35 | "cell_type": "markdown", 36 | "id": "66bac8e5-9c8f-419e-9508-43e3c2e96460", 37 | "metadata": {}, 38 | "source": [ 39 | "### 运行" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "id": "5388d98b-d544-4076-a88c-451569263d84", 45 | "metadata": {}, 46 | "source": [ 47 | "```\n", 48 | "streamlit run app.py\n", 49 | "```" 50 | ] 51 | } 52 | ], 53 | "metadata": { 54 | "kernelspec": { 55 | "display_name": "Python 3 (ipykernel)", 56 | "language": "python", 57 | "name": "python3" 58 | }, 59 | "language_info": { 60 | "codemirror_mode": { 61 | "name": "ipython", 62 | "version": 3 63 | }, 64 | "file_extension": ".py", 65 | "mimetype": "text/x-python", 66 | "name": "python", 67 | "nbconvert_exporter": "python", 68 | "pygments_lexer": "ipython3", 69 | "version": "3.10.13" 70 | } 71 | }, 72 | "nbformat": 4, 73 | "nbformat_minor": 5 74 | } 75 | -------------------------------------------------------------------------------- /tutorials/ai_app/tools/Monica.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "2bb10555-109d-4c43-8540-b0395415f758", 6 | "metadata": {}, 7 | "source": [ 8 | "- gpt 套壳\n", 9 | "- 浏览器插件：Modern AI tools\n", 10 | " - （任意）网页内容：几乎作为了浏览器任意网页的侧边栏及右键选项；\n", 11 | " - summary：带 index；\n", 12 | " - 选中之后的翻译\n", 13 | " - editor：扩写，润色等\n", 14 | " - （youtube）summary\n", 15 | " - chatbot" 16 | ] 17 | } 18 | ], 19 | "metadata": { 20 | "kernelspec": { 21 | "display_name": "Python 3 (ipykernel)", 22 | "language": "python", 23 | "name": "python3" 24 | }, 25 | "language_info": { 26 | "codemirror_mode": { 27 | "name": "ipython", 28 | "version": 3 29 | }, 30 | "file_extension": ".py", 31 | "mimetype": "text/x-python", 32 | "name": "python", 33 | "nbconvert_exporter": "python", 34 | "pygments_lexer": "ipython3", 35 | "version": "3.10.13" 36 | } 37 | }, 38 | "nbformat": 4, 39 | "nbformat_minor": 5 40 | } 41 | -------------------------------------------------------------------------------- /tutorials/ai_app/tools/ai_tools.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "cae47a17", 6 | "metadata": {}, 7 | "source": [ 8 | "- ocr\n", 9 | " - paddleocr\n", 10 | "- asr\n", 11 | " - whisper" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "id": "b3e0d1d9-18e7-4f1a-9c75-2cdd95638aa0", 17 | "metadata": {}, 18 | "source": [ 19 | "### whisper" 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "id": "daa28c8f-26e7-47ae-8da4-94bf7d30fa7a", 25 | "metadata": {}, 26 | "source": [ 27 | "- https://github.com/chidiwilliams/buzz\n", 28 | " - release：https://github.com/chidiwilliams/buzz/releases\n", 29 | " - 使用：https://www.youtube.com/watch?v=qZ3T5hunOuQ" 30 | ] 31 | } 32 | ], 33 | "metadata": { 34 | "kernelspec": { 35 | "display_name": "Python 3 (ipykernel)", 36 | "language": "python", 37 | "name": "python3" 38 | }, 39 | "language_info": { 40 | "codemirror_mode": { 41 | "name": "ipython", 42 | "version": 3 43 | }, 44 | "file_extension": ".py", 45 | "mimetype": "text/x-python", 46 | "name": "python", 47 | "nbconvert_exporter": "python", 48 | "pygments_lexer": "ipython3", 49 | "version": "3.10.13" 50 | }, 51 | "toc": { 52 | "base_numbering": 1, 53 | "nav_menu": {}, 54 | "number_sections": true, 55 | "sideBar": true, 56 | "skip_h1_title": false, 57 | "title_cell": "Table of Contents", 58 | "title_sidebar": "Contents", 59 | "toc_cell": false, 60 | "toc_position": {}, 61 | "toc_section_display": true, 62 | "toc_window_display": false 63 | } 64 | }, 65 | "nbformat": 4, 66 | "nbformat_minor": 5 67 | } 68 | -------------------------------------------------------------------------------- /tutorials/ai_app/vision/rembg-U2Net.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "ac82e26e-c981-40c5-992c-4e698a04af42", 6 | "metadata": {}, 7 | "source": [ 8 | "## u2net" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "bdeec8d8-6e0e-40ce-9173-ac87499058c6", 14 | "metadata": {}, 15 | "source": [ 16 | "- https://github.com/xuebinqin/U-2-Net" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "id": "327a78a1-96e8-48b0-bfce-8f8135718bd6", 22 | "metadata": {}, 23 | "source": [ 24 | "### human portrait" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "id": "dcda27c5-d2b7-4795-8871-e9a8597571c6", 30 | "metadata": {}, 31 | "source": [ 32 | "- `python u2net_portrait_composite.py -s 20 -a 0.5`\n", 33 | " - -s indicates the sigma of gaussian function for blurring the orignal image\n", 34 | " - -a denotes the alpha weights of the orignal image when fusing them.\n", 35 | " - 代码层面要修改\n", 36 | " - `im_comp = im_comp.astype(np.uint8)`（增加这一步处理，适配新版本的 pillow？）" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "id": "298c3c49-4566-4295-90b5-e2c7c3ecf55d", 42 | "metadata": {}, 43 | "source": [ 44 | "### human segmentation" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "id": "0450941b-d491-445e-8b49-df32925166ac", 50 | "metadata": {}, 51 | "source": [ 52 | "- `python u2net_human_seg_test.py`" 53 | ] 54 | }, 55 | { 56 | "cell_type": "markdown", 57 | "id": "14fa93d2-1c9d-4323-b870-512bfe9c400b", 58 | "metadata": {}, 59 | "source": [ 60 | "## rembg" 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "id": "3e69eda4-2d17-4995-94e6-dcc8622e039d", 66 | "metadata": {}, 67 | "source": [ 68 | "- `pip install \"rembg[gpu,cli]\"`\n", 69 | " - 注意 onnxruntime-gpu 的版本兼容性问题\n", 70 | " - `pip install onnxruntime-gpu==1.11`\n", 71 | " - `pip install onnxruntime-gpu==1.12`\n", 72 | " - https://onnxruntime.ai/docs/execution-providers/TensorRT-ExecutionProvider.html#requirements\n", 73 | "- https://github.com/danielgatis/rembg\n", 74 | " - Rembg is a tool to remove images background " 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "id": "a8e8309b-4be6-4167-a54d-009144203cff", 80 | "metadata": {}, 81 | "source": [ 82 | "- rembg i path/to/input.png path/to/output.png\n", 83 | "- rembg i -m u2netp path/to/input.png path/to/output.png\n", 84 | " - 内部使用的也是 u2net" 85 | ] 86 | } 87 | ], 88 | "metadata": { 89 | "kernelspec": { 90 | "display_name": "Python 3 (ipykernel)", 91 | "language": "python", 92 | "name": "python3" 93 | }, 94 | "language_info": { 95 | "codemirror_mode": { 96 | "name": "ipython", 97 | "version": 3 98 | }, 99 | "file_extension": ".py", 100 | "mimetype": "text/x-python", 101 | "name": "python", 102 | "nbconvert_exporter": "python", 103 | "pygments_lexer": "ipython3", 104 | "version": "3.10.13" 105 | } 106 | }, 107 | "nbformat": 4, 108 | "nbformat_minor": 5 109 | } 110 | -------------------------------------------------------------------------------- /tutorials/ai_app/vision/图像视频.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "9a42d307-b1ef-4ef9-9f61-1e7092caba21", 6 | "metadata": {}, 7 | "source": [ 8 | "### 图像" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "cf14d1d8-985a-47b7-b164-f1efb0c76bb3", 14 | "metadata": {}, 15 | "source": [ 16 | "- textbehindimage" 17 | ] 18 | } 19 | ], 20 | "metadata": { 21 | "kernelspec": { 22 | "display_name": "Python 3 (ipykernel)", 23 | "language": "python", 24 | "name": "python3" 25 | }, 26 | "language_info": { 27 | "codemirror_mode": { 28 | "name": "ipython", 29 | "version": 3 30 | }, 31 | "file_extension": ".py", 32 | "mimetype": "text/x-python", 33 | "name": "python", 34 | "nbconvert_exporter": "python", 35 | "pygments_lexer": "ipython3", 36 | "version": "3.10.13" 37 | } 38 | }, 39 | "nbformat": 4, 40 | "nbformat_minor": 5 41 | } 42 | -------------------------------------------------------------------------------- /tutorials/ai_app/项目参考.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "178d3861-5fe5-4ea3-b9e1-d8cbc6e783f4", 6 | "metadata": {}, 7 | "source": [ 8 | "- https://github.com/LYiHub/mad-professor-public\n", 9 | " - https://www.bilibili.com/video/BV18j5jz2Eco/" 10 | ] 11 | } 12 | ], 13 | "metadata": { 14 | "kernelspec": { 15 | "display_name": "casual", 16 | "language": "python", 17 | "name": "casual" 18 | }, 19 | "language_info": { 20 | "codemirror_mode": { 21 | "name": "ipython", 22 | "version": 3 23 | }, 24 | "file_extension": ".py", 25 | "mimetype": "text/x-python", 26 | "name": "python", 27 | "nbconvert_exporter": "python", 28 | "pygments_lexer": "ipython3", 29 | "version": "3.10.0" 30 | } 31 | }, 32 | "nbformat": 4, 33 | "nbformat_minor": 5 34 | } 35 | -------------------------------------------------------------------------------- /tutorials/cv/PIL.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "8116ee9e", 7 | "metadata": { 8 | "ExecuteTime": { 9 | "end_time": "2024-02-07T13:55:21.637323Z", 10 | "start_time": "2024-02-07T13:55:20.304107Z" 11 | } 12 | }, 13 | "outputs": [], 14 | "source": [ 15 | "from einops import rearrange\n", 16 | "from torchvision import transforms\n", 17 | "from PIL import Image\n", 18 | "\n", 19 | "transform = transforms.Compose(\n", 20 | " [\n", 21 | " transforms.Resize((224, 224)),\n", 22 | "# transforms.RandomResizedCrop(224),\n", 23 | "# transforms.RandomHorizontalFlip(),\n", 24 | " transforms.ToTensor(),\n", 25 | " ]\n", 26 | ")" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": 2, 32 | "id": "deff7797", 33 | "metadata": { 34 | "ExecuteTime": { 35 | "end_time": "2024-02-08T12:24:38.743729Z", 36 | "start_time": "2024-02-08T12:24:38.591058Z" 37 | } 38 | }, 39 | "outputs": [], 40 | "source": [ 41 | "import numpy as np" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "id": "58989dc8", 47 | "metadata": {}, 48 | "source": [ 49 | "## Image" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 3, 55 | "id": "cbbac838", 56 | "metadata": { 57 | "ExecuteTime": { 58 | "end_time": "2024-02-07T14:02:25.305528Z", 59 | "start_time": "2024-02-07T14:02:25.298925Z" 60 | } 61 | }, 62 | "outputs": [], 63 | "source": [ 64 | "from PIL import Image" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 4, 70 | "id": "6894b6b4", 71 | "metadata": { 72 | "ExecuteTime": { 73 | "end_time": "2024-02-07T14:02:39.646406Z", 74 | "start_time": "2024-02-07T14:02:39.626204Z" 75 | } 76 | }, 77 | "outputs": [], 78 | "source": [ 79 | "image = Image.open('../nn_basics/imgs/boo.jpg')" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": 5, 85 | "id": "b0185f3f", 86 | "metadata": { 87 | "ExecuteTime": { 88 | "end_time": "2024-02-07T14:02:45.354535Z", 89 | "start_time": "2024-02-07T14:02:45.338394Z" 90 | } 91 | }, 92 | "outputs": [ 93 | { 94 | "data": { 95 | "text/plain": [ 96 | "(1500, 1500)" 97 | ] 98 | }, 99 | "execution_count": 5, 100 | "metadata": {}, 101 | "output_type": "execute_result" 102 | } 103 | ], 104 | "source": [ 105 | "image.size" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": 11, 111 | "id": "5038a39a", 112 | "metadata": { 113 | "ExecuteTime": { 114 | "end_time": "2024-02-07T14:23:11.461043Z", 115 | "start_time": "2024-02-07T14:23:11.450438Z" 116 | } 117 | }, 118 | "outputs": [ 119 | { 120 | "data": { 121 | "text/plain": [ 122 | "'RGB'" 123 | ] 124 | }, 125 | "execution_count": 11, 126 | "metadata": {}, 127 | "output_type": "execute_result" 128 | } 129 | ], 130 | "source": [ 131 | "# channel\n", 132 | "\n", 133 | "# \"L\": 8位灰度图像。\n", 134 | "# \"RGB\": 24位真彩色图像（8位×3）。\n", 135 | "# \"RGBA\": 32位真彩色图像+透明通道（8位×4）。\n", 136 | "image.mode" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": 1, 142 | "id": "d79e2cf3", 143 | "metadata": { 144 | "ExecuteTime": { 145 | "end_time": "2024-02-08T12:24:10.167103Z", 146 | "start_time": "2024-02-08T12:24:10.128469Z" 147 | } 148 | }, 149 | "outputs": [], 150 | "source": [ 151 | "from PIL.Image import fromarray" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": null, 157 | "id": "0691c1d9", 158 | "metadata": {}, 159 | "outputs": [], 160 | "source": [] 161 | } 162 | ], 163 | "metadata": { 164 | "kernelspec": { 165 | "display_name": "Python 3 (ipykernel)", 166 | "language": "python", 167 | "name": "python3" 168 | }, 169 | "language_info": { 170 | "codemirror_mode": { 171 | "name": "ipython", 172 | "version": 3 173 | }, 174 | "file_extension": ".py", 175 | "mimetype": "text/x-python", 176 | "name": "python", 177 | "nbconvert_exporter": "python", 178 | "pygments_lexer": "ipython3", 179 | "version": "3.10.13" 180 | }, 181 | "toc": { 182 | "base_numbering": 1, 183 | "nav_menu": {}, 184 | "number_sections": true, 185 | "sideBar": true, 186 | "skip_h1_title": false, 187 | "title_cell": "Table of Contents", 188 | "title_sidebar": "Contents", 189 | "toc_cell": false, 190 | "toc_position": {}, 191 | "toc_section_display": true, 192 | "toc_window_display": false 193 | } 194 | }, 195 | "nbformat": 4, 196 | "nbformat_minor": 5 197 | } 198 | -------------------------------------------------------------------------------- /tutorials/cv/ViT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "0b5abd4b", 6 | "metadata": {}, 7 | "source": [ 8 | "- 不再需要词表维度的 embedding lookup table；\n", 9 | " - 直接通过 learnable 的线性映射；\n", 10 | " - 或者 pretrained ResNet 提取；" 11 | ] 12 | } 13 | ], 14 | "metadata": { 15 | "kernelspec": { 16 | "display_name": "Python 3 (ipykernel)", 17 | "language": "python", 18 | "name": "python3" 19 | }, 20 | "language_info": { 21 | "codemirror_mode": { 22 | "name": "ipython", 23 | "version": 3 24 | }, 25 | "file_extension": ".py", 26 | "mimetype": "text/x-python", 27 | "name": "python", 28 | "nbconvert_exporter": "python", 29 | "pygments_lexer": "ipython3", 30 | "version": "3.10.13" 31 | }, 32 | "toc": { 33 | "base_numbering": 1, 34 | "nav_menu": {}, 35 | "number_sections": true, 36 | "sideBar": true, 37 | "skip_h1_title": false, 38 | "title_cell": "Table of Contents", 39 | "title_sidebar": "Contents", 40 | "toc_cell": false, 41 | "toc_position": {}, 42 | "toc_section_display": true, 43 | "toc_window_display": false 44 | } 45 | }, 46 | "nbformat": 4, 47 | "nbformat_minor": 5 48 | } 49 | -------------------------------------------------------------------------------- /tutorials/cv/cnn_3dcnn_se.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "79cbc3d8", 6 | "metadata": {}, 7 | "source": [ 8 | "- SE module:\n", 9 | " - Squeeze-and-Excitation 模块\n", 10 | " - Squeeze：压缩；\n", 11 | " - Excitation：激励；" 12 | ] 13 | } 14 | ], 15 | "metadata": { 16 | "kernelspec": { 17 | "display_name": "Python 3 (ipykernel)", 18 | "language": "python", 19 | "name": "python3" 20 | }, 21 | "language_info": { 22 | "codemirror_mode": { 23 | "name": "ipython", 24 | "version": 3 25 | }, 26 | "file_extension": ".py", 27 | "mimetype": "text/x-python", 28 | "name": "python", 29 | "nbconvert_exporter": "python", 30 | "pygments_lexer": "ipython3", 31 | "version": "3.10.13" 32 | }, 33 | "toc": { 34 | "base_numbering": 1, 35 | "nav_menu": {}, 36 | "number_sections": true, 37 | "sideBar": true, 38 | "skip_h1_title": false, 39 | "title_cell": "Table of Contents", 40 | "title_sidebar": "Contents", 41 | "toc_cell": false, 42 | "toc_position": {}, 43 | "toc_section_display": true, 44 | "toc_window_display": false 45 | } 46 | }, 47 | "nbformat": 4, 48 | "nbformat_minor": 5 49 | } 50 | -------------------------------------------------------------------------------- /tutorials/cv/timm.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "8d612bef", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [] 10 | } 11 | ], 12 | "metadata": { 13 | "kernelspec": { 14 | "display_name": "Python 3 (ipykernel)", 15 | "language": "python", 16 | "name": "python3" 17 | }, 18 | "language_info": { 19 | "codemirror_mode": { 20 | "name": "ipython", 21 | "version": 3 22 | }, 23 | "file_extension": ".py", 24 | "mimetype": "text/x-python", 25 | "name": "python", 26 | "nbconvert_exporter": "python", 27 | "pygments_lexer": "ipython3", 28 | "version": "3.10.9" 29 | }, 30 | "toc": { 31 | "base_numbering": 1, 32 | "nav_menu": {}, 33 | "number_sections": true, 34 | "sideBar": true, 35 | "skip_h1_title": false, 36 | "title_cell": "Table of Contents", 37 | "title_sidebar": "Contents", 38 | "toc_cell": false, 39 | "toc_position": {}, 40 | "toc_section_display": true, 41 | "toc_window_display": false 42 | } 43 | }, 44 | "nbformat": 4, 45 | "nbformat_minor": 5 46 | } 47 | -------------------------------------------------------------------------------- /tutorials/dnn/ae/ksparse-ae.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.optim as optim 4 | from torch.utils.data import DataLoader 5 | from torchvision import datasets, transforms 6 | import matplotlib.pyplot as plt 7 | import numpy as np 8 | from tqdm import tqdm 9 | from torch.nn import functional as F 10 | 11 | # 设置随机种子和设备 12 | torch.manual_seed(42) 13 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 14 | 15 | # 数据加载和预处理 16 | transform = transforms.Compose([ 17 | # 0-1 之间 18 | transforms.ToTensor(), 19 | # 这种运行不会出错的，数据/逻辑bug，很难排查 20 | # transforms.Normalize((0.1307,), (0.3081,)) 21 | ]) 22 | 23 | train_dataset = datasets.MNIST('./data', train=True, download=True, transform=transform) 24 | test_dataset = datasets.MNIST('./data', train=False, transform=transform) 25 | 26 | train_loader = DataLoader(train_dataset, batch_size=1024, shuffle=True) 27 | test_loader = DataLoader(test_dataset, batch_size=128, shuffle=False) 28 | 29 | class KSparseAutoencoder(nn.Module): 30 | def __init__(self, input_dim=784, hidden_dim=256, k=60): 31 | super(KSparseAutoencoder, self).__init__() 32 | self.k = k 33 | 34 | # Encoder 35 | self.encoder = nn.Sequential( 36 | nn.Linear(input_dim, hidden_dim), 37 | # nn.ReLU() 38 | ) 39 | 40 | # Decoder 41 | self.decoder = nn.Sequential( 42 | nn.Linear(hidden_dim, input_dim), 43 | nn.Sigmoid() 44 | ) 45 | 46 | def k_sparse(self, x): 47 | # 实现k-sparse约束 48 | topk, indices = torch.topk(x, self.k, dim=1) 49 | mask = torch.zeros_like(x).scatter_(1, indices, 1) 50 | return x * mask 51 | 52 | def forward(self, x): 53 | # 编码 54 | x = x.view(x.size(0), -1) 55 | encoded = self.encoder(x) 56 | # 应用k-sparse约束 57 | sparse_encoded = self.k_sparse(encoded) 58 | # 解码 59 | decoded = self.decoder(sparse_encoded) 60 | return decoded, sparse_encoded 61 | 62 | def train_model(model, train_loader, num_epochs=20): 63 | criterion = nn.MSELoss() 64 | optimizer = optim.Adam(model.parameters(), lr=1e-3) 65 | 66 | train_losses = [] 67 | 68 | for epoch in range(num_epochs): 69 | model.train() 70 | total_loss = 0 71 | 72 | for data, _ in tqdm(train_loader, desc=f'Epoch {epoch+1}/{num_epochs}'): 73 | data = data.to(device) 74 | optimizer.zero_grad() 75 | 76 | decoded, _ = model(data) 77 | loss = criterion(decoded, data.view(data.size(0), -1)) 78 | 79 | loss.backward() 80 | optimizer.step() 81 | 82 | total_loss += loss.item() 83 | 84 | avg_loss = total_loss / len(train_loader) 85 | train_losses.append(avg_loss) 86 | print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {avg_loss:.6f}') 87 | 88 | return train_losses 89 | 90 | def visualize_results(model, test_loader, num_images=10, k=20): 91 | model.eval() 92 | with torch.no_grad(): 93 | data = next(iter(test_loader))[0][:num_images].to(device) 94 | labels = next(iter(test_loader))[1][:num_images] 95 | decoded, encoded = model(data) 96 | 97 | # 可视化原始图像和重构图像 98 | fig, axes = plt.subplots(2, num_images, figsize=(20, 4)) 99 | 100 | for i in range(num_images): 101 | # 原始图像 102 | err = F.mse_loss(data[i].cpu().squeeze(), decoded[i].cpu().view(28, 28)) 103 | print(labels[i].item(), err.item()) 104 | axes[0, i].imshow(data[i].cpu().squeeze(), cmap='gray') 105 | axes[0, i].axis('off') 106 | axes[0, i].set_title('Original') 107 | 108 | # 重构图像 109 | axes[1, i].imshow(decoded[i].cpu().view(28, 28), cmap='gray') 110 | axes[1, i].axis('off') 111 | 112 | axes[1, i].set_title(f'Reconstructed, {err.item():.2f}') 113 | 114 | plt.tight_layout() 115 | # plt.show() 116 | plt.savefig(f'./figs/ksae-reconstructed-{k}.png') 117 | 118 | # 可视化隐层激活 119 | plt.figure(figsize=(10, 4)) 120 | plt.imshow(encoded.cpu().T, aspect='auto', cmap='viridis') 121 | plt.colorbar() 122 | plt.title('Hidden Layer Activations') 123 | plt.xlabel('Sample') 124 | plt.ylabel('Hidden Unit') 125 | # plt.show() 126 | plt.savefig(f'./figs/ksae-activations-{k}.png') 127 | 128 | def plot_training_loss(losses, k): 129 | plt.figure(figsize=(10, 4)) 130 | plt.plot(losses) 131 | plt.title(f'Training Loss Over Time (k={k})') 132 | plt.xlabel('Epoch') 133 | plt.ylabel('Loss') 134 | plt.grid(True) 135 | # plt.show() 136 | plt.savefig(f'./figs/ksae-loss-{k}.png') 137 | 138 | # 主函数 139 | def main(): 140 | k = 20 141 | model = KSparseAutoencoder(k=k).to(device) 142 | losses = train_model(model, train_loader) 143 | plot_training_loss(losses, k) 144 | visualize_results(model, test_loader, k=k) 145 | 146 | if __name__ == "__main__": 147 | main() 148 | -------------------------------------------------------------------------------- /tutorials/dnn/对比学习/对比学习_contrastive_learning.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "65cb11f2", 6 | "metadata": {}, 7 | "source": [ 8 | "- https://lilianweng.github.io/posts/2021-05-31-contrastive/\n", 9 | "- siamese-triplet\n", 10 | " - https://github.com/adambielski/siamese-triplet/tree/master\n", 11 | "- contrastive-image-text\n", 12 | " - CLIP：https://github.com/huggingface/transformers/tree/main/examples/pytorch/contrastive-image-text" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "id": "e0083f42-32c4-4466-887c-7a4f51729825", 18 | "metadata": {}, 19 | "source": [ 20 | "### constrastive learning & reward modeling" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "id": "c955783d-6ac2-4f13-b967-8e609897ad95", 26 | "metadata": {}, 27 | "source": [ 28 | "> contrastive learning 学习表示（representation），基于表示计算距离，远近；\n", 29 | "> Reward modelling：学习reward function，学习 好坏/高低；\n", 30 | "\n", 31 | "- constrastive learning\n", 32 | " - 一种通过对样本对进行比较来学习有用表征的方法。目标是使相似样本的表示更接近，不相似样本的表示更远离。\n", 33 | " - representation learning\n", 34 | " - datapoint_i => representation_i\n", 35 | " - a measure of distance between data points.\n", 36 | " - embedding learning：学习的是表示；\n", 37 | " - 传统对比学习主要关注学习数据的表示（embedding），使得相似样本的表示更接近，以便在下游任务中使用。\n", 38 | "- learning to rank（learning from preference）\n", 39 | " - 从类别到偏序\n", 40 | "- contrastive learning vs. reward modeling\n", 41 | " - contrastive learning 学到的是一个表示（representation），或者说基于表示还可进一步算距离 $\\|f(x)-f(x_+)\\|$；\n", 42 | " - Reward modelling 你最终学到的是一个 reward/value function：$r_\\theta(x,y_w)$；\n", 43 | "- 数据集（样本对，正例、负例，或者偏序对）的构造，以及损失函数的设计；" 44 | ] 45 | }, 46 | { 47 | "cell_type": "markdown", 48 | "id": "775bcad4-d0c6-4d81-9062-9678f8a46243", 49 | "metadata": {}, 50 | "source": [ 51 | "### supervised vs. un-supervised" 52 | ] 53 | }, 54 | { 55 | "cell_type": "markdown", 56 | "id": "93bcb46e-7db3-4d68-b4e6-cf1161eb077f", 57 | "metadata": {}, 58 | "source": [ 59 | "\n", 60 | "> 监督跟无监督的一个直观的区别就是有没有/用不用 label 信息；\n", 61 | "\n", 62 | "- supervised\n", 63 | " - 比如用真实的 label 做监督\n", 64 | " - dog vs. \n", 65 | "- unsupervised\n", 66 | " - 对 input 做增广（augmentation）" 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "id": "703c9121-a94f-426e-99a4-46bdc7cfae57", 72 | "metadata": {}, 73 | "source": [ 74 | "### CLIP (Contrastive Language–Image Pretraining)" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "id": "9a95e624-1fe6-48a9-a516-53af5986feb6", 80 | "metadata": {}, 81 | "source": [ 82 | "> 相应的图像和文本在特征空间中接近，而不相关的图文对则远离\n", 83 | "\n", 84 | "- 在训练过程中，CLIP 处理一批（batch）图文对。例如，在一个批次中有 N 个图像和 N 个对应的文本描述。模型将每个图像和文本分别编码为向量表示。然后，它计算每个图像向量和每个文本向量之间的相似度，形成一个 N x N 的相似度矩阵。\n", 85 | " - 对于这个矩阵中的每一行，模型希望对应的正确匹配（即第 i 个图像和第 i 个文本）的相似度最大，而其他不匹配的相似度较小。为此，CLIP 使用了一种基于交叉熵的对比损失函数，它考虑了图像到文本和文本到图像的匹配，这意味着模型同时优化图像检索文本和文本检索图像的任务。\n", 86 | " - 这种损失函数的设计使得模型学会在高维特征空间中，将相关的图像和文本映射到相近的位置，从而实现跨模态的检索和匹配。\n", 87 | "- 设在一个批次中，有 N 个图像和 N 个对应的文本描述，形成 N 个匹配的图文对。\n", 88 | " - 令 $I_i$ 表示第 $i$ 个图像，$T_i$ 表示第 $i$ 个文本\n", 89 | " - 定义图像编码器 $f$ 和文本编码器 $g$，将图像和文本映射到一个共同的特征空间。（表征学习 representation learning）\n", 90 | " - 获取归一化的图像和文本特征向量：\n", 91 | "\n", 92 | " $$\n", 93 | " \\mathbf v_i=\\frac{f(I_i)}{\\|f(I_t)\\|}, \\mathbf u_i=\\frac{g(T_i)}{\\|g(T_i)\\|}\n", 94 | " $$\n", 95 | " - 相似度矩阵 $S$ 的元素定义为（其中，$\\tau$ 是可学习的温度（temperature）参数，用于控制分布的锐化程度。）\n", 96 | " - 注意 $v_i$ 和 $u_i$ 维度未必一致，也许还需要一层映射，映射到一个共同的表征空间；\n", 97 | " $$\n", 98 | " s_{ij}=\\frac{v_i^Tu_i}{\\tau}\n", 99 | " $$\n", 100 | " - 对于图像到文本的方向，定义预测的概率分布：\n", 101 | " $$\n", 102 | " p_{ij}=\\frac{\\exp(s_{ij})}{\\sum_{k=1}^N\\exp(s_{ik})}\n", 103 | " $$\n", 104 | " - 相应的损失函数为：$\\mathcal L_{\\text{img2txt}}=-\\frac1N\\sum_{i=1}^N\\log p_{i,i}$\n", 105 | " - 对于文本到图像的方向，定义预测的概率分布为：\n", 106 | " $$\n", 107 | " q_{ij}=\\frac{\\exp(s_{ij})}{\\sum_{k=1}^N\\exp(s_{k,j})}\n", 108 | " $$\n", 109 | " - 相应的损失函数为：$\\mathcal L_{\\text{txt2img}}=-\\frac1N\\sum_{i=1}^N\\log q_{i,i}$\n", 110 | " - 总的损失\n", 111 | " $$\n", 112 | " \\mathcal L=\\frac12({L_{\\text{img2txt}}+L_{\\text{txt2img}}})\n", 113 | " $$ " 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "id": "908edeb4-8070-4851-a935-affc9f6682ec", 119 | "metadata": {}, 120 | "source": [ 121 | "### reward modeling" 122 | ] 123 | }, 124 | { 125 | "cell_type": "markdown", 126 | "id": "962b4a5d-9530-439f-9c0d-8ed5cc2c5aea", 127 | "metadata": {}, 128 | "source": [ 129 | "- 而 instruct GPT 开创的 reward modeling 追求的是学习 reward function，即给定一个 input, response pair，输出一个好坏的分数（scalar output）\n", 130 | " $$\n", 131 | " \\text{loss}(\\theta) = -\\frac{1}{\\left(\\frac{K}{2}\\right)} \\mathbb{E}_{(x, y_w, y_l) \\sim \\mathcal{D}} \\left[ \\log \\left( \\sigma \\left( r_\\theta(x, y_w) - r_\\theta(x, y_l) \\right) \\right) \\right]\n", 132 | " $$ \n", 133 | "- 对比学习，强调的是 $f(I_i),g(T_i)$ 将原始的输入，映射到 hidden space，得到的 feature，即为表征学习（representation learning）；" 134 | ] 135 | } 136 | ], 137 | "metadata": { 138 | "kernelspec": { 139 | "display_name": "Python 3 (ipykernel)", 140 | "language": "python", 141 | "name": "python3" 142 | }, 143 | "language_info": { 144 | "codemirror_mode": { 145 | "name": "ipython", 146 | "version": 3 147 | }, 148 | "file_extension": ".py", 149 | "mimetype": "text/x-python", 150 | "name": "python", 151 | "nbconvert_exporter": "python", 152 | "pygments_lexer": "ipython3", 153 | "version": "3.10.13" 154 | }, 155 | "toc": { 156 | "base_numbering": 1, 157 | "nav_menu": {}, 158 | "number_sections": true, 159 | "sideBar": true, 160 | "skip_h1_title": false, 161 | "title_cell": "Table of Contents", 162 | "title_sidebar": "Contents", 163 | "toc_cell": false, 164 | "toc_position": {}, 165 | "toc_section_display": true, 166 | "toc_window_display": false 167 | } 168 | }, 169 | "nbformat": 4, 170 | "nbformat_minor": 5 171 | } 172 | -------------------------------------------------------------------------------- /tutorials/gnn/GTN_Transformer.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "b8f5b88b-70fe-4c9a-87c9-c89932db593e", 6 | "metadata": {}, 7 | "source": [ 8 | "- cs224w\n", 9 | " - https://web.stanford.edu/class/cs224w/slides/14-graph-transformer.pdf\n", 10 | "\n", 11 | "| **Graph** | **Transformers** |\n", 12 | "|-----------------------------|---------------------------|\n", 13 | "| (1) Node features? | (1) Tokenizing |\n", 14 | "| (2) Adjacency information? | (2) Positional encoding |\n", 15 | "| (3) Edge features? | (3) Self-attention |" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "id": "632aad59-dfa3-49b7-8809-26ff3b61948c", 21 | "metadata": {}, 22 | "source": [ 23 | "### 拉普拉斯矩阵" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "id": "27dc50f6-33f1-4e70-9fcf-44a556a45b94", 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [] 33 | } 34 | ], 35 | "metadata": { 36 | "kernelspec": { 37 | "display_name": "Python 3 (ipykernel)", 38 | "language": "python", 39 | "name": "python3" 40 | }, 41 | "language_info": { 42 | "codemirror_mode": { 43 | "name": "ipython", 44 | "version": 3 45 | }, 46 | "file_extension": ".py", 47 | "mimetype": "text/x-python", 48 | "name": "python", 49 | "nbconvert_exporter": "python", 50 | "pygments_lexer": "ipython3", 51 | "version": "3.10.13" 52 | } 53 | }, 54 | "nbformat": 4, 55 | "nbformat_minor": 5 56 | } 57 | -------------------------------------------------------------------------------- /tutorials/gnn/PyG.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 5, 6 | "id": "9cfcbc26-2f73-4893-b4ad-70153466678c", 7 | "metadata": { 8 | "execution": { 9 | "iopub.execute_input": "2024-10-14T15:43:53.563630Z", 10 | "iopub.status.busy": "2024-10-14T15:43:53.562997Z", 11 | "iopub.status.idle": "2024-10-14T15:43:53.572555Z", 12 | "shell.execute_reply": "2024-10-14T15:43:53.570259Z", 13 | "shell.execute_reply.started": "2024-10-14T15:43:53.563584Z" 14 | } 15 | }, 16 | "outputs": [], 17 | "source": [ 18 | "from IPython.display import Image" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 6, 24 | "id": "2fb2db2b-effd-46fe-8c53-22f7cca1795c", 25 | "metadata": { 26 | "execution": { 27 | "iopub.execute_input": "2024-10-14T15:43:54.968278Z", 28 | "iopub.status.busy": "2024-10-14T15:43:54.967636Z", 29 | "iopub.status.idle": "2024-10-14T15:43:54.980845Z", 30 | "shell.execute_reply": "2024-10-14T15:43:54.978750Z", 31 | "shell.execute_reply.started": "2024-10-14T15:43:54.968222Z" 32 | } 33 | }, 34 | "outputs": [ 35 | { 36 | "data": { 37 | "text/html": [ 38 | "" 39 | ], 40 | "text/plain": [ 41 | "" 42 | ] 43 | }, 44 | "execution_count": 6, 45 | "metadata": {}, 46 | "output_type": "execute_result" 47 | } 48 | ], 49 | "source": [ 50 | "Image(url='./imgs/gnn_layers.png', width=500)" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 7, 56 | "id": "8a01bc67-10d2-42c7-ab15-5566f383cd1b", 57 | "metadata": { 58 | "execution": { 59 | "iopub.execute_input": "2024-10-14T15:45:04.557978Z", 60 | "iopub.status.busy": "2024-10-14T15:45:04.557351Z", 61 | "iopub.status.idle": "2024-10-14T15:45:08.682441Z", 62 | "shell.execute_reply": "2024-10-14T15:45:08.680521Z", 63 | "shell.execute_reply.started": "2024-10-14T15:45:04.557932Z" 64 | } 65 | }, 66 | "outputs": [], 67 | "source": [ 68 | "import torch_geometric as pyg" 69 | ] 70 | }, 71 | { 72 | "cell_type": "markdown", 73 | "id": "b7bea46c-52cc-4f6a-87f8-39a6de5de259", 74 | "metadata": {}, 75 | "source": [ 76 | "- 2708 nodes, 10556 edges\n", 77 | "- Each node describes a document.\n", 78 | "- Each node is described by a FEATURE TENSOR.\n", 79 | " - x_1, ..., x_1433\n", 80 | "- Two documents are connected (with an edge) if there exists a citation link in CORA between them." 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "id": "6bdca450-8147-4843-b17c-6634da63be66", 86 | "metadata": {}, 87 | "source": [ 88 | "### InMemoryDataset" 89 | ] 90 | }, 91 | { 92 | "cell_type": "markdown", 93 | "id": "fb731a18-4ddb-4149-87dd-ddd02ba3d028", 94 | "metadata": {}, 95 | "source": [ 96 | "- Dataset base class for creating graph datasets which easily fit into CPU memory." 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 3, 102 | "id": "dae55b20-1c01-4143-a4bb-28f7075d1427", 103 | "metadata": { 104 | "execution": { 105 | "iopub.execute_input": "2024-10-19T03:03:50.870103Z", 106 | "iopub.status.busy": "2024-10-19T03:03:50.869429Z", 107 | "iopub.status.idle": "2024-10-19T03:03:50.878801Z", 108 | "shell.execute_reply": "2024-10-19T03:03:50.876627Z", 109 | "shell.execute_reply.started": "2024-10-19T03:03:50.870053Z" 110 | } 111 | }, 112 | "outputs": [], 113 | "source": [ 114 | "# !pip install rdkit" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 7, 120 | "id": "621ffa4e-9029-4131-b37f-2e09d164b9d1", 121 | "metadata": { 122 | "execution": { 123 | "iopub.execute_input": "2024-10-19T03:05:12.239026Z", 124 | "iopub.status.busy": "2024-10-19T03:05:12.238693Z", 125 | "iopub.status.idle": "2024-10-19T03:05:12.246822Z", 126 | "shell.execute_reply": "2024-10-19T03:05:12.244862Z", 127 | "shell.execute_reply.started": "2024-10-19T03:05:12.239004Z" 128 | } 129 | }, 130 | "outputs": [], 131 | "source": [ 132 | "from torch_geometric.datasets import QM9" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": 10, 138 | "id": "d6bf3868-d4ca-45b8-9679-347de10160f9", 139 | "metadata": { 140 | "execution": { 141 | "iopub.execute_input": "2024-10-19T03:08:34.780446Z", 142 | "iopub.status.busy": "2024-10-19T03:08:34.779792Z", 143 | "iopub.status.idle": "2024-10-19T03:10:15.343084Z", 144 | "shell.execute_reply": "2024-10-19T03:10:15.342250Z", 145 | "shell.execute_reply.started": "2024-10-19T03:08:34.780397Z" 146 | } 147 | }, 148 | "outputs": [ 149 | { 150 | "name": "stderr", 151 | "output_type": "stream", 152 | "text": [ 153 | "Processing...\n", 154 | "100%|██████████| 133885/133885 [01:34<00:00, 1413.15it/s]\n", 155 | "Done!\n" 156 | ] 157 | } 158 | ], 159 | "source": [ 160 | "dp = QM9(root='./data/QM9/').to_datapipe()" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 21, 166 | "id": "3bc0d371-c824-40c3-9621-dabc6996fb0b", 167 | "metadata": { 168 | "execution": { 169 | "iopub.execute_input": "2024-10-19T03:12:37.817601Z", 170 | "iopub.status.busy": "2024-10-19T03:12:37.817102Z", 171 | "iopub.status.idle": "2024-10-19T03:12:37.828669Z", 172 | "shell.execute_reply": "2024-10-19T03:12:37.826502Z", 173 | "shell.execute_reply.started": "2024-10-19T03:12:37.817566Z" 174 | } 175 | }, 176 | "outputs": [ 177 | { 178 | "data": { 179 | "text/plain": [ 180 | "Batcher" 181 | ] 182 | }, 183 | "execution_count": 21, 184 | "metadata": {}, 185 | "output_type": "execute_result" 186 | } 187 | ], 188 | "source": [ 189 | "dp = dp.batch_graphs(batch_size=16, drop_last=True)\n", 190 | "dp" 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": 22, 196 | "id": "6825ac76-675f-4a77-8ead-3c03243b5f15", 197 | "metadata": { 198 | "execution": { 199 | "iopub.execute_input": "2024-10-19T03:12:39.340635Z", 200 | "iopub.status.busy": "2024-10-19T03:12:39.340037Z", 201 | "iopub.status.idle": "2024-10-19T03:12:40.342465Z", 202 | "shell.execute_reply": "2024-10-19T03:12:40.340739Z", 203 | "shell.execute_reply.started": "2024-10-19T03:12:39.340590Z" 204 | } 205 | }, 206 | "outputs": [ 207 | { 208 | "data": { 209 | "text/plain": [ 210 | "DataBatch(x=[12622, 11], edge_index=[2, 24802], edge_attr=[24802, 4], y=[1024, 19], pos=[12622, 3], z=[12622], smiles=[16], name=[16], idx=[1024], batch=[12622])" 211 | ] 212 | }, 213 | "execution_count": 22, 214 | "metadata": {}, 215 | "output_type": "execute_result" 216 | } 217 | ], 218 | "source": [ 219 | "batch = next(iter(dp))\n", 220 | "batch" 221 | ] 222 | } 223 | ], 224 | "metadata": { 225 | "kernelspec": { 226 | "display_name": "Python 3 (ipykernel)", 227 | "language": "python", 228 | "name": "python3" 229 | }, 230 | "language_info": { 231 | "codemirror_mode": { 232 | "name": "ipython", 233 | "version": 3 234 | }, 235 | "file_extension": ".py", 236 | "mimetype": "text/x-python", 237 | "name": "python", 238 | "nbconvert_exporter": "python", 239 | "pygments_lexer": "ipython3", 240 | "version": "3.10.13" 241 | } 242 | }, 243 | "nbformat": 4, 244 | "nbformat_minor": 5 245 | } 246 | -------------------------------------------------------------------------------- /tutorials/gnn/imgs/gnn_layers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/gnn/imgs/gnn_layers.png -------------------------------------------------------------------------------- /tutorials/gnn/imgs/two-layers-gcn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/gnn/imgs/two-layers-gcn.png -------------------------------------------------------------------------------- /tutorials/mcp/gemini-cli-mcp.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "eca012f3-02ad-48c5-a13a-19e10e5ab53d", 6 | "metadata": {}, 7 | "source": [ 8 | "- `httpUrl` → `StreamableHTTPClientTransport`\n", 9 | "- `url` → `SSEClientTransport`\n", 10 | "- `command` → `StdioClientTransport`" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "id": "ce0d5567-4078-4fcc-99f7-a49f5156d003", 16 | "metadata": {}, 17 | "source": [ 18 | "### client.ts" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": null, 24 | "id": "6799507c-21b2-4ccf-900e-595f4355e54a", 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [] 28 | } 29 | ], 30 | "metadata": { 31 | "kernelspec": { 32 | "display_name": "Python 3 (ipykernel)", 33 | "language": "python", 34 | "name": "python3" 35 | }, 36 | "language_info": { 37 | "codemirror_mode": { 38 | "name": "ipython", 39 | "version": 3 40 | }, 41 | "file_extension": ".py", 42 | "mimetype": "text/x-python", 43 | "name": "python", 44 | "nbconvert_exporter": "python", 45 | "pygments_lexer": "ipython3", 46 | "version": "3.10.0" 47 | } 48 | }, 49 | "nbformat": 4, 50 | "nbformat_minor": 5 51 | } 52 | -------------------------------------------------------------------------------- /tutorials/mcp/mcp_api.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "a6c5a210-c07a-4c34-ad46-dbd8d4c9246b", 6 | "metadata": {}, 7 | "source": [ 8 | "- https://mp.weixin.qq.com/s/CLCWHYrZgr_bIwjpyWGAFg" 9 | ] 10 | } 11 | ], 12 | "metadata": { 13 | "kernelspec": { 14 | "display_name": "Python 3 (ipykernel)", 15 | "language": "python", 16 | "name": "python3" 17 | }, 18 | "language_info": { 19 | "codemirror_mode": { 20 | "name": "ipython", 21 | "version": 3 22 | }, 23 | "file_extension": ".py", 24 | "mimetype": "text/x-python", 25 | "name": "python", 26 | "nbconvert_exporter": "python", 27 | "pygments_lexer": "ipython3", 28 | "version": "3.10.0" 29 | } 30 | }, 31 | "nbformat": 4, 32 | "nbformat_minor": 5 33 | } 34 | -------------------------------------------------------------------------------- /tutorials/mcp/实用 MCP.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "754e82fe-ade2-458c-be72-521ef9bf8643", 6 | "metadata": {}, 7 | "source": [ 8 | "- 爬虫类\n", 9 | " - fetch\n", 10 | " - 抓网页\n", 11 | " - firecrawl\n", 12 | "- 开发类\n", 13 | " - context7： `use context7`\n", 14 | " - https://context7.com/: 追逐一些库/框架的最新的用法；" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "id": "a1831b7b-e0ca-464a-af6e-ae8adf2a0e43", 20 | "metadata": {}, 21 | "source": [ 22 | "### 基础" 23 | ] 24 | }, 25 | { 26 | "cell_type": "markdown", 27 | "id": "fe6932be-2a7b-4830-a9b1-94af69d870e0", 28 | "metadata": {}, 29 | "source": [ 30 | "- `npx -y @modelcontextprotocol/server-sequential-thinking`\n", 31 | " - https://github.com/modelcontextprotocol/servers/blob/main/src/sequentialthinking/index.ts#L140" 32 | ] 33 | }, 34 | { 35 | "cell_type": "markdown", 36 | "id": "d4ead8dd-e1de-46b3-a96f-23f92a44e325", 37 | "metadata": {}, 38 | "source": [ 39 | "### 浏览器" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "id": "a3d17501-a477-4d22-8146-114d4c852d38", 45 | "metadata": {}, 46 | "source": [ 47 | "- https://github.com/hangwin/mcp-chrome/blob/master/README_zh.md\n", 48 | " - stdio\n", 49 | " - `npm list -g mcp-chrome-bridge`：查询本地安装路径\n", 50 | "\n", 51 | "```\n", 52 | "/usr/local/lib\n", 53 | "└── mcp-chrome-bridge@1.0.29\n", 54 | "\n", 55 | "/usr/local/lib/node_modules/mcp-chrome-bridge\n", 56 | "```" 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "id": "130602f9-c19c-44bc-8d2e-b53bb041c7b2", 62 | "metadata": {}, 63 | "source": [ 64 | "### 可视化" 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "id": "9cb01a4f-1a84-4443-b448-f98bfee06ecb", 70 | "metadata": {}, 71 | "source": [ 72 | "- mcp-server-chart\n", 73 | "- excalidraw\n", 74 | " - https://playbooks.com/mcp/i-tozer-excalidraw" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "id": "37ca105b-fd80-4442-9de9-8219a511d15e", 80 | "metadata": {}, 81 | "source": [ 82 | "### 应用类" 83 | ] 84 | }, 85 | { 86 | "cell_type": "markdown", 87 | "id": "74e619be-92ae-4779-aa19-67fda12ed2f3", 88 | "metadata": {}, 89 | "source": [ 90 | "- Location\n", 91 | " - 高德 amap server\n", 92 | " - 旅行规划" 93 | ] 94 | } 95 | ], 96 | "metadata": { 97 | "kernelspec": { 98 | "display_name": "Python 3 (ipykernel)", 99 | "language": "python", 100 | "name": "python3" 101 | }, 102 | "language_info": { 103 | "codemirror_mode": { 104 | "name": "ipython", 105 | "version": 3 106 | }, 107 | "file_extension": ".py", 108 | "mimetype": "text/x-python", 109 | "name": "python", 110 | "nbconvert_exporter": "python", 111 | "pygments_lexer": "ipython3", 112 | "version": "3.10.0" 113 | } 114 | }, 115 | "nbformat": 4, 116 | "nbformat_minor": 5 117 | } 118 | -------------------------------------------------------------------------------- /tutorials/nn_basics/einsum_einops/einops.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "id": "9cc62bc7-5408-4dfe-a9e5-cc773a1595ed", 7 | "metadata": { 8 | "execution": { 9 | "iopub.execute_input": "2024-11-23T10:06:06.583433Z", 10 | "iopub.status.busy": "2024-11-23T10:06:06.583022Z", 11 | "iopub.status.idle": "2024-11-23T10:06:06.590218Z", 12 | "shell.execute_reply": "2024-11-23T10:06:06.588478Z", 13 | "shell.execute_reply.started": "2024-11-23T10:06:06.583410Z" 14 | } 15 | }, 16 | "outputs": [], 17 | "source": [ 18 | "import torch\n", 19 | "import einops\n", 20 | "from einops import repeat" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "id": "1b9275d2-1c60-4914-8701-df5497814309", 26 | "metadata": {}, 27 | "source": [ 28 | "#### repeat" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "id": "22bfd1db-1cb7-4a41-be90-15e7986d49da", 34 | "metadata": {}, 35 | "source": [ 36 | "- 模式 \"i -> (i j)\" 的含义：\n", 37 | " - 输入维度：i，即原始的一维张量。\n", 38 | " - 输出维度：(i j)，表示将每个元素 i 重复 j 次后展平成一个新的维度。\n", 39 | "- 参数 j=p 指定每个元素重复的次数为 p。" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 3, 45 | "id": "59b0e5ad-dffa-4831-bbfa-8a14c971c899", 46 | "metadata": { 47 | "execution": { 48 | "iopub.execute_input": "2024-11-23T10:06:39.266087Z", 49 | "iopub.status.busy": "2024-11-23T10:06:39.265444Z", 50 | "iopub.status.idle": "2024-11-23T10:06:40.299609Z", 51 | "shell.execute_reply": "2024-11-23T10:06:40.297826Z", 52 | "shell.execute_reply.started": "2024-11-23T10:06:39.266042Z" 53 | } 54 | }, 55 | "outputs": [ 56 | { 57 | "data": { 58 | "text/plain": [ 59 | "tensor([1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5,\n", 60 | " 5])" 61 | ] 62 | }, 63 | "execution_count": 3, 64 | "metadata": {}, 65 | "output_type": "execute_result" 66 | } 67 | ], 68 | "source": [ 69 | "repeat(torch.tensor([1, 2, 3, 4, 5]), 'i -> (i j)', j=5)" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 5, 75 | "id": "c16cdedd-eefd-47ba-a9dd-9be4fc2e09be", 76 | "metadata": { 77 | "execution": { 78 | "iopub.execute_input": "2024-11-23T10:09:08.088125Z", 79 | "iopub.status.busy": "2024-11-23T10:09:08.087482Z", 80 | "iopub.status.idle": "2024-11-23T10:09:08.102022Z", 81 | "shell.execute_reply": "2024-11-23T10:09:08.099919Z", 82 | "shell.execute_reply.started": "2024-11-23T10:09:08.088080Z" 83 | } 84 | }, 85 | "outputs": [ 86 | { 87 | "data": { 88 | "text/plain": [ 89 | "(tensor([10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10]), torch.Size([12]))" 90 | ] 91 | }, 92 | "execution_count": 5, 93 | "metadata": {}, 94 | "output_type": "execute_result" 95 | } 96 | ], 97 | "source": [ 98 | "p = repeat(torch.tensor(10), '-> (i j)', i=3, j=4)\n", 99 | "p, p.shape" 100 | ] 101 | } 102 | ], 103 | "metadata": { 104 | "kernelspec": { 105 | "display_name": "Python 3 (ipykernel)", 106 | "language": "python", 107 | "name": "python3" 108 | }, 109 | "language_info": { 110 | "codemirror_mode": { 111 | "name": "ipython", 112 | "version": 3 113 | }, 114 | "file_extension": ".py", 115 | "mimetype": "text/x-python", 116 | "name": "python", 117 | "nbconvert_exporter": "python", 118 | "pygments_lexer": "ipython3", 119 | "version": "3.10.13" 120 | } 121 | }, 122 | "nbformat": 4, 123 | "nbformat_minor": 5 124 | } 125 | -------------------------------------------------------------------------------- /tutorials/nn_basics/grad/gradient_autograd.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "25aba748", 6 | "metadata": {}, 7 | "source": [ 8 | "## grad fn" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "f16be65e", 14 | "metadata": {}, 15 | "source": [ 16 | "- MulBackward0\n", 17 | "- MeanBackward0\n", 18 | "- ..." 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "id": "2a8799d5", 24 | "metadata": {}, 25 | "source": [ 26 | "## `requires_grad = True`" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": 9, 32 | "id": "3aec21fe", 33 | "metadata": { 34 | "ExecuteTime": { 35 | "end_time": "2023-11-14T13:12:52.080492Z", 36 | "start_time": "2023-11-14T13:12:52.067867Z" 37 | } 38 | }, 39 | "outputs": [ 40 | { 41 | "name": "stdout", 42 | "output_type": "stream", 43 | "text": [ 44 | "tensor([8.], grad_fn=)\n", 45 | "tensor(8., grad_fn=)\n", 46 | "tensor([2.], requires_grad=True) True\n" 47 | ] 48 | } 49 | ], 50 | "source": [ 51 | "import torch\n", 52 | "\n", 53 | "# 创建一个可跟踪梯度的张量\n", 54 | "x = torch.tensor([2.0], requires_grad=True)\n", 55 | "\n", 56 | "# 对张量进行一些操作\n", 57 | "# 2x^2\n", 58 | "y = x * x * 2\n", 59 | "print(y)\n", 60 | "z = y.mean()\n", 61 | "print(z)\n", 62 | "print(x, x.is_leaf)" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 7, 68 | "id": "86b1b9dd", 69 | "metadata": { 70 | "ExecuteTime": { 71 | "end_time": "2023-11-14T13:12:14.120433Z", 72 | "start_time": "2023-11-14T13:12:14.111021Z" 73 | } 74 | }, 75 | "outputs": [ 76 | { 77 | "name": "stdout", 78 | "output_type": "stream", 79 | "text": [ 80 | "tensor([8.])\n" 81 | ] 82 | } 83 | ], 84 | "source": [ 85 | "# 执行反向传播\n", 86 | "z.backward()\n", 87 | "\n", 88 | "# 输出梯度, 4x\n", 89 | "print(x.grad)" 90 | ] 91 | } 92 | ], 93 | "metadata": { 94 | "kernelspec": { 95 | "display_name": "Python 3 (ipykernel)", 96 | "language": "python", 97 | "name": "python3" 98 | }, 99 | "language_info": { 100 | "codemirror_mode": { 101 | "name": "ipython", 102 | "version": 3 103 | }, 104 | "file_extension": ".py", 105 | "mimetype": "text/x-python", 106 | "name": "python", 107 | "nbconvert_exporter": "python", 108 | "pygments_lexer": "ipython3", 109 | "version": "3.10.13" 110 | }, 111 | "toc": { 112 | "base_numbering": 1, 113 | "nav_menu": {}, 114 | "number_sections": true, 115 | "sideBar": true, 116 | "skip_h1_title": false, 117 | "title_cell": "Table of Contents", 118 | "title_sidebar": "Contents", 119 | "toc_cell": false, 120 | "toc_position": {}, 121 | "toc_section_display": true, 122 | "toc_window_display": false 123 | } 124 | }, 125 | "nbformat": 4, 126 | "nbformat_minor": 5 127 | } 128 | -------------------------------------------------------------------------------- /tutorials/nn_basics/grad/readme.md: -------------------------------------------------------------------------------- 1 | 2 | - 标量关于谁求导，返回的 shape，就跟谁 shape 保持一致； 3 | - $y=x^TAx$ 4 | - $\nabla_xy=(A+A^T)x$ 5 | - $\nabla_Ay=xx^T$ -------------------------------------------------------------------------------- /tutorials/nn_basics/grad/torch_常见错误.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "078e2471", 7 | "metadata": { 8 | "ExecuteTime": { 9 | "end_time": "2024-03-18T13:18:37.760538Z", 10 | "start_time": "2024-03-18T13:18:35.712909Z" 11 | } 12 | }, 13 | "outputs": [], 14 | "source": [ 15 | "import torch" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "id": "a81d8a5c", 21 | "metadata": { 22 | "ExecuteTime": { 23 | "end_time": "2024-03-18T13:25:54.748902Z", 24 | "start_time": "2024-03-18T13:25:54.742465Z" 25 | } 26 | }, 27 | "source": [ 28 | "## no_grad & enable_grad" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "id": "61774502", 34 | "metadata": {}, 35 | "source": [ 36 | "- torch module 前向构建计算图的过程中，不能使用 `torch.enable_grad()`\n", 37 | " - 因为外部在推理的时候，大概率还会有 `torch.no_grad` 的 context" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": 3, 43 | "id": "7cdf2f49", 44 | "metadata": { 45 | "ExecuteTime": { 46 | "end_time": "2024-03-18T13:18:42.334361Z", 47 | "start_time": "2024-03-18T13:18:42.326367Z" 48 | } 49 | }, 50 | "outputs": [ 51 | { 52 | "data": { 53 | "text/plain": [ 54 | "True" 55 | ] 56 | }, 57 | "execution_count": 3, 58 | "metadata": {}, 59 | "output_type": "execute_result" 60 | } 61 | ], 62 | "source": [ 63 | "x = torch.tensor([1., 2.], requires_grad=True)\n", 64 | "y = x ** 2\n", 65 | "y.requires_grad" 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": 4, 71 | "id": "2467ef40", 72 | "metadata": { 73 | "ExecuteTime": { 74 | "end_time": "2024-03-18T13:18:55.687029Z", 75 | "start_time": "2024-03-18T13:18:55.679115Z" 76 | } 77 | }, 78 | "outputs": [ 79 | { 80 | "data": { 81 | "text/plain": [ 82 | "True" 83 | ] 84 | }, 85 | "execution_count": 4, 86 | "metadata": {}, 87 | "output_type": "execute_result" 88 | } 89 | ], 90 | "source": [ 91 | "with torch.no_grad():\n", 92 | " with torch.enable_grad():\n", 93 | " y = x**2\n", 94 | "y.requires_grad" 95 | ] 96 | } 97 | ], 98 | "metadata": { 99 | "kernelspec": { 100 | "display_name": "Python 3 (ipykernel)", 101 | "language": "python", 102 | "name": "python3" 103 | }, 104 | "language_info": { 105 | "codemirror_mode": { 106 | "name": "ipython", 107 | "version": 3 108 | }, 109 | "file_extension": ".py", 110 | "mimetype": "text/x-python", 111 | "name": "python", 112 | "nbconvert_exporter": "python", 113 | "pygments_lexer": "ipython3", 114 | "version": "3.10.13" 115 | }, 116 | "toc": { 117 | "base_numbering": 1, 118 | "nav_menu": {}, 119 | "number_sections": true, 120 | "sideBar": true, 121 | "skip_h1_title": false, 122 | "title_cell": "Table of Contents", 123 | "title_sidebar": "Contents", 124 | "toc_cell": false, 125 | "toc_position": {}, 126 | "toc_section_display": true, 127 | "toc_window_display": false 128 | } 129 | }, 130 | "nbformat": 4, 131 | "nbformat_minor": 5 132 | } 133 | -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/axies-flat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/axies-flat.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/axis-swap-flat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/axis-swap-flat.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/axis-swap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/axis-swap.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/axis0-1-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/axis0-1-2.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/boo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/boo.jpg -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/fan_in_fan_out.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/fan_in_fan_out.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/graph-legend.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/graph-legend.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/linear.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/linear.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/multi_loss.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/multi_loss.PNG -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/numpy_einsum.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/numpy_einsum.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/shape_1x8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/shape_1x8.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/shape_1x8_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/shape_1x8_2.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/shape_1x8_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/shape_1x8_3.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/shape_1x8_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/shape_1x8_4.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/shape_1x8_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/shape_1x8_5.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/shape_2x4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/shape_2x4.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/shape_4x2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/shape_4x2.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/shape_4x2_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/shape_4x2_2.png -------------------------------------------------------------------------------- /tutorials/nn_basics/imgs/shape_4x2_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chunhuizhang/llm_aigc/5f631ac192ae93610aa177059ca4bd0b38bc0ec9/tutorials/nn_basics/imgs/shape_4x2_3.png -------------------------------------------------------------------------------- /tutorials/nn_basics/module_load_save.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "33a38d29", 7 | "metadata": { 8 | "ExecuteTime": { 9 | "end_time": "2023-11-14T14:52:10.373949Z", 10 | "start_time": "2023-11-14T14:52:09.182936Z" 11 | } 12 | }, 13 | "outputs": [], 14 | "source": [ 15 | "import torch\n", 16 | "from torch import nn" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "id": "cd46809a-ae0b-4a49-89ee-7b346b3faabe", 22 | "metadata": {}, 23 | "source": [ 24 | "## basics" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "id": "32da4ec6-bb98-4f5b-8390-3b6311853aa1", 30 | "metadata": {}, 31 | "source": [ 32 | "- `for name, child in module.named_children():`\n", 33 | " - module 本质上维护了一个树结构；\n", 34 | " - 参考 SyncBatchNorm（https://pytorch.org/docs/stable/_modules/torch/nn/modules/batchnorm.html#SyncBatchNorm）\n", 35 | " - `def convert_sync_batchnorm`：这个函数递归地处理所有的 module\n", 36 | "- PyTorch Module — quick reference：https://medium.com/howsofcoding/pytorch-module-quick-reference-b35d5a0f9a00" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "id": "eb2f75ea-2d67-49b9-b3ef-1bd1093af56b", 42 | "metadata": {}, 43 | "source": [ 44 | "## `named_children` vs. `named_modules`" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 6, 50 | "id": "206bf950-cc6e-4cf0-acb7-d30237390fb3", 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "class MyModule(nn.Module):\n", 55 | " def __init__(self):\n", 56 | " super(MyModule, self).__init__()\n", 57 | " self.layer1 = nn.Sequential(\n", 58 | " nn.Conv2d(16, 32, 3, 1),\n", 59 | " nn.ReLU(inplace=True)\n", 60 | " )\n", 61 | " self.layer2 = nn.Sequential(\n", 62 | " nn.Linear(32,10)\n", 63 | " )\n", 64 | " self.misc = nn.Linear(1, 1)\n", 65 | " \n", 66 | " def forward(self,x):\n", 67 | " x = self.layer1(x)\n", 68 | " x = self.layer2(x)\n", 69 | " \n", 70 | "model = MyModule()" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 7, 76 | "id": "4c782cac-2a5b-4e87-968b-7556c6fe4557", 77 | "metadata": {}, 78 | "outputs": [ 79 | { 80 | "name": "stdout", 81 | "output_type": "stream", 82 | "text": [ 83 | "layer1 -> Sequential(\n", 84 | " (0): Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1))\n", 85 | " (1): ReLU(inplace=True)\n", 86 | ")\n", 87 | "layer2 -> Sequential(\n", 88 | " (0): Linear(in_features=32, out_features=10, bias=True)\n", 89 | ")\n", 90 | "misc -> Linear(in_features=1, out_features=1, bias=True)\n" 91 | ] 92 | } 93 | ], 94 | "source": [ 95 | "for name, module in model.named_children():\n", 96 | " print(name, '->', module)" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 8, 102 | "id": "cf93557e-23fd-4cc1-9d20-735711922a1d", 103 | "metadata": {}, 104 | "outputs": [ 105 | { 106 | "name": "stdout", 107 | "output_type": "stream", 108 | "text": [ 109 | " -> MyModule(\n", 110 | " (layer1): Sequential(\n", 111 | " (0): Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1))\n", 112 | " (1): ReLU(inplace=True)\n", 113 | " )\n", 114 | " (layer2): Sequential(\n", 115 | " (0): Linear(in_features=32, out_features=10, bias=True)\n", 116 | " )\n", 117 | " (misc): Linear(in_features=1, out_features=1, bias=True)\n", 118 | ")\n", 119 | "layer1 -> Sequential(\n", 120 | " (0): Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1))\n", 121 | " (1): ReLU(inplace=True)\n", 122 | ")\n", 123 | "layer1.0 -> Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1))\n", 124 | "layer1.1 -> ReLU(inplace=True)\n", 125 | "layer2 -> Sequential(\n", 126 | " (0): Linear(in_features=32, out_features=10, bias=True)\n", 127 | ")\n", 128 | "layer2.0 -> Linear(in_features=32, out_features=10, bias=True)\n", 129 | "misc -> Linear(in_features=1, out_features=1, bias=True)\n" 130 | ] 131 | } 132 | ], 133 | "source": [ 134 | "for name, module in model.named_modules():\n", 135 | " print(name, '->', module)" 136 | ] 137 | }, 138 | { 139 | "cell_type": "markdown", 140 | "id": "0ee438f1", 141 | "metadata": {}, 142 | "source": [ 143 | "## `model.parameters()`" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": 2, 149 | "id": "19c5fba6-2f0d-4f0d-9185-c5dcd3143d7b", 150 | "metadata": {}, 151 | "outputs": [], 152 | "source": [ 153 | "model = nn.Linear(5, 3)" 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": 9, 159 | "id": "29d04b31", 160 | "metadata": { 161 | "ExecuteTime": { 162 | "end_time": "2023-11-14T14:52:57.383206Z", 163 | "start_time": "2023-11-14T14:52:57.369346Z" 164 | } 165 | }, 166 | "outputs": [ 167 | { 168 | "data": { 169 | "text/plain": [ 170 | "[Parameter containing:\n", 171 | " tensor([[-0.2888, -0.1333, 0.2464, -0.2843, -0.2926],\n", 172 | " [-0.1179, -0.3022, -0.4258, 0.4373, 0.0226],\n", 173 | " [ 0.1486, -0.1183, -0.3256, -0.3780, -0.4289]], requires_grad=True),\n", 174 | " Parameter containing:\n", 175 | " tensor([-0.2695, -0.1645, -0.4121], requires_grad=True)]" 176 | ] 177 | }, 178 | "execution_count": 9, 179 | "metadata": {}, 180 | "output_type": "execute_result" 181 | } 182 | ], 183 | "source": [ 184 | "list(model.parameters())" 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": 10, 190 | "id": "a1168a17", 191 | "metadata": { 192 | "ExecuteTime": { 193 | "end_time": "2023-11-14T14:53:01.334448Z", 194 | "start_time": "2023-11-14T14:53:01.321081Z" 195 | } 196 | }, 197 | "outputs": [ 198 | { 199 | "data": { 200 | "text/plain": [ 201 | "[('weight',\n", 202 | " Parameter containing:\n", 203 | " tensor([[-0.2888, -0.1333, 0.2464, -0.2843, -0.2926],\n", 204 | " [-0.1179, -0.3022, -0.4258, 0.4373, 0.0226],\n", 205 | " [ 0.1486, -0.1183, -0.3256, -0.3780, -0.4289]], requires_grad=True)),\n", 206 | " ('bias',\n", 207 | " Parameter containing:\n", 208 | " tensor([-0.2695, -0.1645, -0.4121], requires_grad=True))]" 209 | ] 210 | }, 211 | "execution_count": 10, 212 | "metadata": {}, 213 | "output_type": "execute_result" 214 | } 215 | ], 216 | "source": [ 217 | "list(model.named_parameters())" 218 | ] 219 | } 220 | ], 221 | "metadata": { 222 | "kernelspec": { 223 | "display_name": "Python 3 (ipykernel)", 224 | "language": "python", 225 | "name": "python3" 226 | }, 227 | "language_info": { 228 | "codemirror_mode": { 229 | "name": "ipython", 230 | "version": 3 231 | }, 232 | "file_extension": ".py", 233 | "mimetype": "text/x-python", 234 | "name": "python", 235 | "nbconvert_exporter": "python", 236 | "pygments_lexer": "ipython3", 237 | "version": "3.10.13" 238 | }, 239 | "toc": { 240 | "base_numbering": 1, 241 | "nav_menu": {}, 242 | "number_sections": true, 243 | "sideBar": true, 244 | "skip_h1_title": false, 245 | "title_cell": "Table of Contents", 246 | "title_sidebar": "Contents", 247 | "toc_cell": false, 248 | "toc_position": {}, 249 | "toc_section_display": true, 250 | "toc_window_display": false 251 | } 252 | }, 253 | "nbformat": 4, 254 | "nbformat_minor": 5 255 | } 256 | -------------------------------------------------------------------------------- /tutorials/nn_basics/optimizer/momentum_exp_avg.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "dbed347c-181f-4178-ab67-915064a44985", 6 | "metadata": {}, 7 | "source": [ 8 | "\n", 9 | "- 基于 Momentum 的 SGD\n", 10 | "\n", 11 | " $$\n", 12 | " \\begin{split}\n", 13 | " v_t&=\\beta v_{t-1}+g_t\\\\\n", 14 | " \\theta_{t+1}&=\\theta_t-\\alpha v_t\n", 15 | " \\end{split}\n", 16 | " $$\n", 17 | " \n", 18 | "\n", 19 | " - 对于 $v_t$ 部分递归的结果：$v_t=\\sum_{i=0}^{t-i}\\beta^ig_{t-i}$ （$v_0$ 初始化为 0）" 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "id": "ee28e34f-3c8d-4815-a457-b65f203db2d2", 25 | "metadata": {}, 26 | "source": [ 27 | "- EMA (Exponential Moving Average)\n", 28 | "\n", 29 | "$$\n", 30 | "\\begin{split}\n", 31 | "\\text{EMA}_t&=\\alpha x_t+(1-\\alpha) \\text{EMA}_{t-1}\\\\\n", 32 | "&=\\alpha x_t+(1-\\alpha)\\alpha x_{t-1}+(1-\\alpha)^2\\alpha x_{t-2}+\\cdots+(1-\\alpha)^{t-1}\\alpha x_1\n", 33 | "\\end{split}\n", 34 | "$$" 35 | ] 36 | } 37 | ], 38 | "metadata": { 39 | "kernelspec": { 40 | "display_name": "Python 3 (ipykernel)", 41 | "language": "python", 42 | "name": "python3" 43 | }, 44 | "language_info": { 45 | "codemirror_mode": { 46 | "name": "ipython", 47 | "version": 3 48 | }, 49 | "file_extension": ".py", 50 | "mimetype": "text/x-python", 51 | "name": "python", 52 | "nbconvert_exporter": "python", 53 | "pygments_lexer": "ipython3", 54 | "version": "3.10.13" 55 | } 56 | }, 57 | "nbformat": 4, 58 | "nbformat_minor": 5 59 | } 60 | -------------------------------------------------------------------------------- /tutorials/nn_basics/tricks_norms/nn_and_training_tricks.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 3, 6 | "id": "2f67f995", 7 | "metadata": { 8 | "ExecuteTime": { 9 | "end_time": "2023-11-28T13:06:58.026548Z", 10 | "start_time": "2023-11-28T13:06:58.020728Z" 11 | } 12 | }, 13 | "outputs": [], 14 | "source": [ 15 | "import torch\n", 16 | "from torch import nn" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "id": "405edb19", 22 | "metadata": {}, 23 | "source": [ 24 | "## 理解神经网络" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "id": "3d5d8c6d", 30 | "metadata": {}, 31 | "source": [ 32 | "- 从shape的含义，shape 的变化，可能是理解神经网络模型结构乃至处理过程的一个很实用的切入。" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "id": "0448091c", 38 | "metadata": {}, 39 | "source": [ 40 | "## `nn.utils.clip_grad_norm_`" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "id": "8c054c26", 46 | "metadata": {}, 47 | "source": [ 48 | "- 避免梯度爆炸\n", 49 | " \n", 50 | " - 计算所有梯度的范数：这一步骤涉及到计算网络中所有参数的梯度范数。通常使用 L2 范数，即平方和的平方根。\n", 51 | " - 也是 `nn.utils.clip_grad_norm_` 的返回值；\n", 52 | " - 比较梯度范数与最大值：将计算出的梯度范数与设定的最大范数值比较。\n", 53 | " - 裁剪梯度：如果梯度范数大于最大值，那么将所有梯度缩放到最大值以内。这是通过乘以一个缩放因子实现的，缩放因子是最大范数值除以梯度范数。\n", 54 | " - 更新梯度：使用裁剪后的梯度值更新网络参数。\n", 55 | "\n", 56 | "- 一般training过程中用到：\n", 57 | "\n", 58 | " ```\n", 59 | " optimizer.zero_grad()\n", 60 | " # 反向传播计算 parameters 的 grad\n", 61 | " loss.backward()\n", 62 | " # 计算完梯度之后，norm 所有参数的 grads\n", 63 | " nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip)\n", 64 | " # 基于更新后的 grad 值来更新参数\n", 65 | " optimizer.step()\n", 66 | " ```" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 13, 72 | "id": "c6a0eb31", 73 | "metadata": { 74 | "ExecuteTime": { 75 | "end_time": "2023-11-28T13:11:08.362707Z", 76 | "start_time": "2023-11-28T13:11:08.352229Z" 77 | } 78 | }, 79 | "outputs": [], 80 | "source": [ 81 | "# 定义一个简单的网络及其梯度\n", 82 | "class SimpleNet(torch.nn.Module):\n", 83 | " def __init__(self):\n", 84 | " super(SimpleNet, self).__init__()\n", 85 | " self.param1 = torch.nn.Parameter(torch.tensor([3.0, 4.0]))\n", 86 | " self.param2 = torch.nn.Parameter(torch.tensor([1.0, 2.0]))\n", 87 | "\n", 88 | "# 创建网络实例\n", 89 | "net = SimpleNet()\n", 90 | "\n", 91 | "# 设置梯度\n", 92 | "net.param1.grad = torch.tensor([3.0, 4.0])\n", 93 | "net.param2.grad = torch.tensor([1.0, 2.0])\n", 94 | "\n", 95 | "# 最大梯度范数\n", 96 | "max_norm = 5.0" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 14, 102 | "id": "c7b9ad30", 103 | "metadata": { 104 | "ExecuteTime": { 105 | "end_time": "2023-11-28T13:11:10.028971Z", 106 | "start_time": "2023-11-28T13:11:10.016953Z" 107 | } 108 | }, 109 | "outputs": [ 110 | { 111 | "data": { 112 | "text/plain": [ 113 | "(tensor(5.4772), [tensor([2.7386, 3.6515]), tensor([0.9129, 1.8257])])" 114 | ] 115 | }, 116 | "execution_count": 14, 117 | "metadata": {}, 118 | "output_type": "execute_result" 119 | } 120 | ], 121 | "source": [ 122 | "# 计算梯度范数\n", 123 | "total_norm = torch.sqrt(sum(p.grad.norm()**2 for p in net.parameters()))\n", 124 | "\n", 125 | "# 计算缩放因子\n", 126 | "scale = max_norm / (total_norm + 1e-6)\n", 127 | "\n", 128 | "# 应用梯度裁剪\n", 129 | "for p in net.parameters():\n", 130 | " p.grad.data.mul_(scale)\n", 131 | "\n", 132 | "# 更新后的梯度\n", 133 | "clipped_grads = [p.grad for p in net.parameters()]\n", 134 | "\n", 135 | "total_norm, clipped_grads" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": 15, 141 | "id": "d7143022", 142 | "metadata": { 143 | "ExecuteTime": { 144 | "end_time": "2023-11-28T13:11:11.532229Z", 145 | "start_time": "2023-11-28T13:11:11.522275Z" 146 | } 147 | }, 148 | "outputs": [], 149 | "source": [ 150 | "# 定义一个简单的网络及其梯度\n", 151 | "class SimpleNet(torch.nn.Module):\n", 152 | " def __init__(self):\n", 153 | " super(SimpleNet, self).__init__()\n", 154 | " self.param1 = torch.nn.Parameter(torch.tensor([3.0, 4.0]))\n", 155 | " self.param2 = torch.nn.Parameter(torch.tensor([1.0, 2.0]))\n", 156 | "\n", 157 | "# 创建网络实例\n", 158 | "net = SimpleNet()\n", 159 | "\n", 160 | "# 设置梯度\n", 161 | "net.param1.grad = torch.tensor([3.0, 4.0])\n", 162 | "net.param2.grad = torch.tensor([1.0, 2.0])\n", 163 | "\n", 164 | "# 最大梯度范数\n", 165 | "max_norm = 5.0" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": 16, 171 | "id": "15d074eb", 172 | "metadata": { 173 | "ExecuteTime": { 174 | "end_time": "2023-11-28T13:11:14.160926Z", 175 | "start_time": "2023-11-28T13:11:14.151739Z" 176 | } 177 | }, 178 | "outputs": [ 179 | { 180 | "data": { 181 | "text/plain": [ 182 | "tensor(5.4772)" 183 | ] 184 | }, 185 | "execution_count": 16, 186 | "metadata": {}, 187 | "output_type": "execute_result" 188 | } 189 | ], 190 | "source": [ 191 | "nn.utils.clip_grad_norm_(net.parameters(), max_norm)" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": 17, 197 | "id": "1a1bd7a7", 198 | "metadata": { 199 | "ExecuteTime": { 200 | "end_time": "2023-11-28T13:11:15.181539Z", 201 | "start_time": "2023-11-28T13:11:15.173888Z" 202 | } 203 | }, 204 | "outputs": [ 205 | { 206 | "name": "stdout", 207 | "output_type": "stream", 208 | "text": [ 209 | "tensor([2.7386, 3.6515])\n", 210 | "tensor([0.9129, 1.8257])\n" 211 | ] 212 | } 213 | ], 214 | "source": [ 215 | "for p in net.parameters():\n", 216 | " print(p.grad)" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": null, 222 | "id": "63fd5152", 223 | "metadata": {}, 224 | "outputs": [], 225 | "source": [] 226 | } 227 | ], 228 | "metadata": { 229 | "kernelspec": { 230 | "display_name": "Python 3 (ipykernel)", 231 | "language": "python", 232 | "name": "python3" 233 | }, 234 | "language_info": { 235 | "codemirror_mode": { 236 | "name": "ipython", 237 | "version": 3 238 | }, 239 | "file_extension": ".py", 240 | "mimetype": "text/x-python", 241 | "name": "python", 242 | "nbconvert_exporter": "python", 243 | "pygments_lexer": "ipython3", 244 | "version": "3.10.13" 245 | }, 246 | "toc": { 247 | "base_numbering": 1, 248 | "nav_menu": {}, 249 | "number_sections": true, 250 | "sideBar": true, 251 | "skip_h1_title": false, 252 | "title_cell": "Table of Contents", 253 | "title_sidebar": "Contents", 254 | "toc_cell": false, 255 | "toc_position": {}, 256 | "toc_section_display": true, 257 | "toc_window_display": false 258 | } 259 | }, 260 | "nbformat": 4, 261 | "nbformat_minor": 5 262 | } 263 | -------------------------------------------------------------------------------- /tutorials/nn_basics/tricks_norms/why_norm.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "3f516499-ac5b-469d-8c9b-ff30c010e97c", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [] 10 | } 11 | ], 12 | "metadata": { 13 | "kernelspec": { 14 | "display_name": "Python 3 (ipykernel)", 15 | "language": "python", 16 | "name": "python3" 17 | }, 18 | "language_info": { 19 | "codemirror_mode": { 20 | "name": "ipython", 21 | "version": 3 22 | }, 23 | "file_extension": ".py", 24 | "mimetype": "text/x-python", 25 | "name": "python", 26 | "nbconvert_exporter": "python", 27 | "pygments_lexer": "ipython3", 28 | "version": "3.10.13" 29 | } 30 | }, 31 | "nbformat": 4, 32 | "nbformat_minor": 5 33 | } 34 | -------------------------------------------------------------------------------- /tutorials/nn_basics/utils/functional.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "58f53a9d", 7 | "metadata": { 8 | "ExecuteTime": { 9 | "end_time": "2023-07-31T13:49:32.773031Z", 10 | "start_time": "2023-07-31T13:49:31.784276Z" 11 | } 12 | }, 13 | "outputs": [], 14 | "source": [ 15 | "import torch.nn.functional as F" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "id": "334c8497", 21 | "metadata": {}, 22 | "source": [ 23 | "## basics" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "id": "f0d092d3", 29 | "metadata": {}, 30 | "source": [ 31 | "- `F.relu`\n", 32 | "- `F.linear`\n", 33 | " - `F.linear(input, weight, bias=None)`\n", 34 | " - input: $x\\in R^{m\\times i}$\n", 35 | " - weight: $W\\in R^{o\\times i}$\n", 36 | " - bias: $b\\in R^{o}$\n", 37 | "\n", 38 | "$$\n", 39 | "y=xW^T+b\n", 40 | "$$\n", 41 | "\n" 42 | ] 43 | } 44 | ], 45 | "metadata": { 46 | "kernelspec": { 47 | "display_name": "Python 3 (ipykernel)", 48 | "language": "python", 49 | "name": "python3" 50 | }, 51 | "language_info": { 52 | "codemirror_mode": { 53 | "name": "ipython", 54 | "version": 3 55 | }, 56 | "file_extension": ".py", 57 | "mimetype": "text/x-python", 58 | "name": "python", 59 | "nbconvert_exporter": "python", 60 | "pygments_lexer": "ipython3", 61 | "version": "3.10.13" 62 | }, 63 | "toc": { 64 | "base_numbering": 1, 65 | "nav_menu": {}, 66 | "number_sections": true, 67 | "sideBar": true, 68 | "skip_h1_title": false, 69 | "title_cell": "Table of Contents", 70 | "title_sidebar": "Contents", 71 | "toc_cell": false, 72 | "toc_position": {}, 73 | "toc_section_display": true, 74 | "toc_window_display": false 75 | } 76 | }, 77 | "nbformat": 4, 78 | "nbformat_minor": 5 79 | } 80 | -------------------------------------------------------------------------------- /tutorials/nn_basics/utils/torch_triangle_mask.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 18, 6 | "id": "29a152de", 7 | "metadata": { 8 | "ExecuteTime": { 9 | "end_time": "2023-10-12T14:27:54.511886Z", 10 | "start_time": "2023-10-12T14:27:54.507302Z" 11 | } 12 | }, 13 | "outputs": [], 14 | "source": [ 15 | "import torch" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "id": "a99751ea", 21 | "metadata": {}, 22 | "source": [ 23 | "## baiscs" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "id": "892c8d9b", 29 | "metadata": {}, 30 | "source": [ 31 | "- `torch.triu`: upper\n", 32 | "- `torch.tril`: lower" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "id": "70372582", 38 | "metadata": {}, 39 | "source": [ 40 | "## cases" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 12, 46 | "id": "58605127", 47 | "metadata": { 48 | "ExecuteTime": { 49 | "end_time": "2023-10-12T14:27:11.134112Z", 50 | "start_time": "2023-10-12T14:27:11.125536Z" 51 | } 52 | }, 53 | "outputs": [ 54 | { 55 | "data": { 56 | "text/plain": [ 57 | "tensor([[-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 58 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 59 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 60 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 61 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 62 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 63 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 64 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf]], device='cuda:0')" 65 | ] 66 | }, 67 | "execution_count": 12, 68 | "metadata": {}, 69 | "output_type": "execute_result" 70 | } 71 | ], 72 | "source": [ 73 | "seq_len = 8\n", 74 | "# mask = torch.full((seq_len, seq_len), float('-inf'), device='cuda:0')\n", 75 | "mask = torch.full((seq_len, seq_len), float('-inf'), device='cuda:0')\n", 76 | "mask" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 13, 82 | "id": "c86c43bf", 83 | "metadata": { 84 | "ExecuteTime": { 85 | "end_time": "2023-10-12T14:27:12.771374Z", 86 | "start_time": "2023-10-12T14:27:12.764122Z" 87 | } 88 | }, 89 | "outputs": [ 90 | { 91 | "data": { 92 | "text/plain": [ 93 | "tensor([[-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 94 | " [0., -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 95 | " [0., 0., -inf, -inf, -inf, -inf, -inf, -inf],\n", 96 | " [0., 0., 0., -inf, -inf, -inf, -inf, -inf],\n", 97 | " [0., 0., 0., 0., -inf, -inf, -inf, -inf],\n", 98 | " [0., 0., 0., 0., 0., -inf, -inf, -inf],\n", 99 | " [0., 0., 0., 0., 0., 0., -inf, -inf],\n", 100 | " [0., 0., 0., 0., 0., 0., 0., -inf]], device='cuda:0')" 101 | ] 102 | }, 103 | "execution_count": 13, 104 | "metadata": {}, 105 | "output_type": "execute_result" 106 | } 107 | ], 108 | "source": [ 109 | "# mask = torch.triu(mask, diagonal=0)\n", 110 | "torch.triu(mask)" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": 19, 116 | "id": "a6cec6aa", 117 | "metadata": { 118 | "ExecuteTime": { 119 | "end_time": "2023-10-12T14:27:59.967120Z", 120 | "start_time": "2023-10-12T14:27:59.958305Z" 121 | }, 122 | "scrolled": true 123 | }, 124 | "outputs": [ 125 | { 126 | "data": { 127 | "text/plain": [ 128 | "tensor([[0., 0., -inf, -inf, -inf, -inf, -inf, -inf],\n", 129 | " [0., 0., 0., -inf, -inf, -inf, -inf, -inf],\n", 130 | " [0., 0., 0., 0., -inf, -inf, -inf, -inf],\n", 131 | " [0., 0., 0., 0., 0., -inf, -inf, -inf],\n", 132 | " [0., 0., 0., 0., 0., 0., -inf, -inf],\n", 133 | " [0., 0., 0., 0., 0., 0., 0., -inf],\n", 134 | " [0., 0., 0., 0., 0., 0., 0., 0.],\n", 135 | " [0., 0., 0., 0., 0., 0., 0., 0.]], device='cuda:0')" 136 | ] 137 | }, 138 | "execution_count": 19, 139 | "metadata": {}, 140 | "output_type": "execute_result" 141 | } 142 | ], 143 | "source": [ 144 | "torch.triu(mask, diagonal=2)" 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": 23, 150 | "id": "da9d38de", 151 | "metadata": { 152 | "ExecuteTime": { 153 | "end_time": "2023-10-12T14:29:41.068374Z", 154 | "start_time": "2023-10-12T14:29:41.059374Z" 155 | } 156 | }, 157 | "outputs": [ 158 | { 159 | "data": { 160 | "text/plain": [ 161 | "tensor([[-inf, -inf, -inf, 0., 0., 0., 0., 0.],\n", 162 | " [-inf, -inf, -inf, -inf, 0., 0., 0., 0.],\n", 163 | " [-inf, -inf, -inf, -inf, -inf, 0., 0., 0.],\n", 164 | " [-inf, -inf, -inf, -inf, -inf, -inf, 0., 0.],\n", 165 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, 0.],\n", 166 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 167 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n", 168 | " [-inf, -inf, -inf, -inf, -inf, -inf, -inf, -inf]], device='cuda:0')" 169 | ] 170 | }, 171 | "execution_count": 23, 172 | "metadata": {}, 173 | "output_type": "execute_result" 174 | } 175 | ], 176 | "source": [ 177 | "torch.tril(mask, diagonal=2)" 178 | ] 179 | } 180 | ], 181 | "metadata": { 182 | "kernelspec": { 183 | "display_name": "Python 3 (ipykernel)", 184 | "language": "python", 185 | "name": "python3" 186 | }, 187 | "language_info": { 188 | "codemirror_mode": { 189 | "name": "ipython", 190 | "version": 3 191 | }, 192 | "file_extension": ".py", 193 | "mimetype": "text/x-python", 194 | "name": "python", 195 | "nbconvert_exporter": "python", 196 | "pygments_lexer": "ipython3", 197 | "version": "3.10.13" 198 | }, 199 | "toc": { 200 | "base_numbering": 1, 201 | "nav_menu": {}, 202 | "number_sections": true, 203 | "sideBar": true, 204 | "skip_h1_title": false, 205 | "title_cell": "Table of Contents", 206 | "title_sidebar": "Contents", 207 | "toc_cell": false, 208 | "toc_position": {}, 209 | "toc_section_display": true, 210 | "toc_window_display": false 211 | } 212 | }, 213 | "nbformat": 4, 214 | "nbformat_minor": 5 215 | } 216 | -------------------------------------------------------------------------------- /tutorials/nn_basics/utils/torch_utils_scripts.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 4, 6 | "id": "85662644", 7 | "metadata": { 8 | "ExecuteTime": { 9 | "end_time": "2023-11-24T13:17:04.814792Z", 10 | "start_time": "2023-11-24T13:17:03.320380Z" 11 | } 12 | }, 13 | "outputs": [], 14 | "source": [ 15 | "import torch\n", 16 | "from torch import nn" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "id": "9f4064f5-9309-476e-be5e-99a09aed8243", 22 | "metadata": {}, 23 | "source": [ 24 | "### flush" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "id": "d21b51e7-565e-489d-a114-9cb4925bd693", 30 | "metadata": {}, 31 | "source": [ 32 | "```\n", 33 | "def flush_model(self):\n", 34 | " # delete llm from gpu to save GPU memory\n", 35 | " if self.model is not None:\n", 36 | " self.model = None\n", 37 | " gc.collect()\n", 38 | " torch.cuda.empty_cache()\n", 39 | "```" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "id": "21f9626b-e5d0-432f-a2aa-510e7d655ad8", 45 | "metadata": {}, 46 | "source": [ 47 | "### parameters" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 3, 53 | "id": "b0c4f453", 54 | "metadata": { 55 | "ExecuteTime": { 56 | "end_time": "2023-11-24T13:16:53.035351Z", 57 | "start_time": "2023-11-24T13:16:53.029917Z" 58 | } 59 | }, 60 | "outputs": [], 61 | "source": [ 62 | "def num_parameters(model):\n", 63 | " return sum(p.numel() for p in model.parameters() if p.requires_grad)" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": 5, 69 | "id": "774de0cc", 70 | "metadata": { 71 | "ExecuteTime": { 72 | "end_time": "2023-11-24T13:17:12.603280Z", 73 | "start_time": "2023-11-24T13:17:12.596196Z" 74 | } 75 | }, 76 | "outputs": [ 77 | { 78 | "data": { 79 | "text/plain": [ 80 | "220" 81 | ] 82 | }, 83 | "execution_count": 5, 84 | "metadata": {}, 85 | "output_type": "execute_result" 86 | } 87 | ], 88 | "source": [ 89 | "m = nn.Linear(10, 20)\n", 90 | "num_parameters(m)" 91 | ] 92 | } 93 | ], 94 | "metadata": { 95 | "kernelspec": { 96 | "display_name": "Python 3 (ipykernel)", 97 | "language": "python", 98 | "name": "python3" 99 | }, 100 | "language_info": { 101 | "codemirror_mode": { 102 | "name": "ipython", 103 | "version": 3 104 | }, 105 | "file_extension": ".py", 106 | "mimetype": "text/x-python", 107 | "name": "python", 108 | "nbconvert_exporter": "python", 109 | "pygments_lexer": "ipython3", 110 | "version": "3.10.13" 111 | }, 112 | "toc": { 113 | "base_numbering": 1, 114 | "nav_menu": {}, 115 | "number_sections": true, 116 | "sideBar": true, 117 | "skip_h1_title": false, 118 | "title_cell": "Table of Contents", 119 | "title_sidebar": "Contents", 120 | "toc_cell": false, 121 | "toc_position": {}, 122 | "toc_section_display": true, 123 | "toc_window_display": false 124 | } 125 | }, 126 | "nbformat": 4, 127 | "nbformat_minor": 5 128 | } 129 | -------------------------------------------------------------------------------- /tutorials/oai/async_demo.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import time 3 | 4 | async def make_coffee(): 5 | print("开始煮咖啡...") 6 | await asyncio.sleep(3) # 模拟煮咖啡的时间 7 | print("咖啡准备好了！") 8 | return "一杯咖啡" 9 | 10 | async def make_toast(): 11 | print("开始烤面包...") 12 | await asyncio.sleep(2) # 模拟烤面包的时间 13 | print("面包烤好了！") 14 | return "一片吐司" 15 | 16 | async def main(): 17 | # 同时开始准备咖啡和面包 18 | t0 = time.time() 19 | coffee_task = asyncio.create_task(make_coffee()) 20 | toast_task = asyncio.create_task(make_toast()) 21 | 22 | # 等待两个任务都完成 23 | coffee, toast = await asyncio.gather(coffee_task, toast_task) 24 | print(f"早餐准备完成：{coffee}和{toast}, 用时: {time.time() - t0:.2f}s") 25 | 26 | # 运行程序 27 | asyncio.run(main()) -------------------------------------------------------------------------------- /tutorials/oai/async_scripts/Semaphore.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | async def worker(name, semaphore): 4 | async with semaphore: 5 | print(f"{name} 获取到了信号量") 6 | await asyncio.sleep(2) # 模拟一个需要时间的操作 7 | print(f"{name} 释放了信号量") 8 | 9 | async def main(): 10 | # 创建一个信号量，最多允许3个协程同时运行 11 | semaphore = asyncio.Semaphore(3) 12 | 13 | tasks = [worker(f"任务{i}", semaphore) for i in range(10)] 14 | await asyncio.gather(*tasks) 15 | 16 | if __name__ == "__main__": 17 | asyncio.run(main()) 18 | -------------------------------------------------------------------------------- /tutorials/oai/async_scripts/async_call.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import openai 3 | from openai import AsyncOpenAI 4 | import backoff 5 | from dotenv import load_dotenv, find_dotenv 6 | import os 7 | 8 | assert load_dotenv(find_dotenv()) 9 | 10 | client = AsyncOpenAI(api_key=os.getenv('OPENAI_API_KEY')) 11 | 12 | @backoff.on_exception(backoff.expo, openai.RateLimitError) 13 | async def make_api_call_to_gpt(prompt, model="gpt-4o-mini"): 14 | messages = [{"role": "user", "content": prompt}] 15 | response = await client.chat.completions.create( 16 | model=model, 17 | messages=messages, 18 | temperature=0, 19 | ) 20 | 21 | return response.choices[0].message.content 22 | 23 | async def main(): 24 | prompts = ["What model are you?", 25 | "How many Rs in strawberry?", 26 | "Which is bigger, 0.9 or 0.11?" 27 | ] 28 | 29 | # Create a list to store the results of asynchronous calls 30 | results = [] 31 | 32 | # Asynchronously call the function for each prompt 33 | tasks = [make_api_call_to_gpt(prompt) for prompt in prompts] 34 | print(tasks) 35 | results = await asyncio.gather(*tasks) 36 | for prompt, result in zip(prompts, results): 37 | print(prompt) 38 | print(result) 39 | print('====================') 40 | 41 | asyncio.run(main()) 42 | -------------------------------------------------------------------------------- /tutorials/oai/async_scripts/async_call_semaphore.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import openai 3 | from openai import AsyncOpenAI 4 | import backoff 5 | from dotenv import load_dotenv, find_dotenv 6 | import os 7 | 8 | assert load_dotenv(find_dotenv()) 9 | 10 | client = AsyncOpenAI(api_key=os.getenv('OPENAI_API_KEY')) 11 | 12 | @backoff.on_exception(backoff.expo, openai.RateLimitError) 13 | async def make_api_call_to_gpt(prompt, model="gpt-4o-mini"): 14 | messages = [{"role": "user", "content": prompt}] 15 | response = await client.chat.completions.create( 16 | model=model, 17 | messages=messages, 18 | temperature=0, 19 | ) 20 | 21 | return response.choices[0].message.content 22 | 23 | async def make_api_call_with_semaphore(prompt, semaphore, model="gpt-4o-mini"): 24 | async with semaphore: 25 | result = await make_api_call_to_gpt(prompt, model) 26 | return prompt, result 27 | 28 | async def main(): 29 | prompts = [ 30 | "What model are you?", 31 | "How many Rs in strawberry?", 32 | "Which is bigger, 0.9 or 0.11?", 33 | # 假设这里有更多的 prompts 34 | ] 35 | 36 | # 设置最大并发数，例如 5 37 | max_concurrent_requests = 5 38 | semaphore = asyncio.Semaphore(max_concurrent_requests) 39 | 40 | # 创建任务列表，包含信号量控制 41 | tasks = [ 42 | make_api_call_with_semaphore(prompt, semaphore) 43 | for prompt in prompts 44 | ] 45 | 46 | # 使用 asyncio.as_completed 来实时处理完成的任务 47 | for coro in asyncio.as_completed(tasks): 48 | prompt, result = await coro 49 | print(f"Prompt: {prompt}") 50 | print(f"Response: {result}") 51 | print('====================') 52 | 53 | # 如果需要等待所有任务完成，可以使用以下方法 54 | # results = await asyncio.gather(*tasks) 55 | # for prompt, result in results: 56 | # print(prompt) 57 | # print(result) 58 | # print('====================') 59 | 60 | if __name__ == "__main__": 61 | asyncio.run(main()) 62 | -------------------------------------------------------------------------------- /tutorials/oai/async_scripts/async_llm_call.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import aiohttp 3 | from dotenv import load_dotenv, find_dotenv 4 | from openai import AsyncOpenAI 5 | 6 | async def generate_text(prompt, client): 7 | response = await client.chat.completions.create( 8 | model="gpt-4.1-mini-2025-04-14", 9 | messages=[{"role": "user", "content": prompt}] 10 | ) 11 | return response.choices[0].message.content 12 | 13 | async def process_batch(batch, client): 14 | responses = await asyncio.gather(*[ 15 | client.chat.completions.create( 16 | model="gpt-3.5-turbo", 17 | messages=[{"role": "user", "content": prompt}] 18 | ) for prompt in batch 19 | ]) 20 | return [response.choices[0].message.content for response in responses] 21 | 22 | async def main(): 23 | prompts = [f"Tell me a fact about number {i}" for i in range(100)] 24 | batch_size = 10 25 | 26 | async with AsyncOpenAI() as client: 27 | results = [] 28 | for i in range(0, len(prompts), batch_size): 29 | batch = prompts[i:i+batch_size] 30 | batch_results = await process_batch(batch, client) 31 | results.extend(batch_results) 32 | 33 | for prompt, result in zip(prompts, results): 34 | print(f"Prompt: {prompt}\nResponse: {result}\n") 35 | asyncio.run(main()) 36 | 37 | 38 | async def main(): 39 | prompts = [ 40 | "Explain quantum computing in simple terms.", 41 | "Write a haiku about artificial intelligence.", 42 | "Describe the process of photosynthesis." 43 | ] 44 | 45 | async with AsyncOpenAI() as client: 46 | tasks = [generate_text(prompt, client) for prompt in prompts] 47 | results = await asyncio.gather(*tasks) 48 | 49 | for prompt, result in zip(prompts, results): 50 | print(f"Prompt: {prompt}\nResponse: {result}\n") 51 | 52 | if __name__ == '__main__': 53 | assert load_dotenv(find_dotenv()) 54 | asyncio.run(main()) -------------------------------------------------------------------------------- /tutorials/oai/async_scripts/io_cpu.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import time 3 | 4 | # I/O密集型任务示例 5 | async def io_intensive(): 6 | print(f"开始I/O任务 {time.strftime('%H:%M:%S')}") 7 | await asyncio.sleep(2) # 模拟I/O操作 8 | print(f"完成I/O任务 {time.strftime('%H:%M:%S')}") 9 | 10 | # CPU密集型任务示例 11 | async def cpu_intensive(): 12 | print(f"开始CPU任务 {time.strftime('%H:%M:%S')}") 13 | for _ in range(50000000): # 执行大量计算 14 | _ = _ ** 2 15 | print(f"完成CPU任务 {time.strftime('%H:%M:%S')}") 16 | 17 | async def main(): 18 | # I/O密集型任务并发执行 19 | print("测试I/O密集型任务:") 20 | await asyncio.gather( 21 | io_intensive(), 22 | io_intensive(), 23 | io_intensive() 24 | ) # 总耗时约2秒 25 | 26 | print("\n测试CPU密集型任务:") 27 | await asyncio.gather( 28 | cpu_intensive(), 29 | cpu_intensive(), 30 | cpu_intensive() 31 | ) # 总耗时是三个任务的总和 32 | 33 | asyncio.run(main()) -------------------------------------------------------------------------------- /tutorials/oai/gpt_openai_生态.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "26a59777", 6 | "metadata": {}, 7 | "source": [ 8 | "## GPTs" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "35933c8e", 14 | "metadata": {}, 15 | "source": [ 16 | "- `site:chat.openai.com/g`\n", 17 | " - 谷歌搜索 gpts\n", 18 | "- GPTs\n", 19 | " - researchGPT：https://chat.openai.com/g/g-NgAcklHd8-researchgpt-official\n", 20 | " - 润色GPT：https://chat.openai.com/g/g-VX52iRD3r-ai-paper-polisher-pro" 21 | ] 22 | } 23 | ], 24 | "metadata": { 25 | "kernelspec": { 26 | "display_name": "Python 3 (ipykernel)", 27 | "language": "python", 28 | "name": "python3" 29 | }, 30 | "language_info": { 31 | "codemirror_mode": { 32 | "name": "ipython", 33 | "version": 3 34 | }, 35 | "file_extension": ".py", 36 | "mimetype": "text/x-python", 37 | "name": "python", 38 | "nbconvert_exporter": "python", 39 | "pygments_lexer": "ipython3", 40 | "version": "3.10.13" 41 | }, 42 | "toc": { 43 | "base_numbering": 1, 44 | "nav_menu": {}, 45 | "number_sections": true, 46 | "sideBar": true, 47 | "skip_h1_title": false, 48 | "title_cell": "Table of Contents", 49 | "title_sidebar": "Contents", 50 | "toc_cell": false, 51 | "toc_position": {}, 52 | "toc_section_display": true, 53 | "toc_window_display": false 54 | } 55 | }, 56 | "nbformat": 4, 57 | "nbformat_minor": 5 58 | } 59 | -------------------------------------------------------------------------------- /tutorials/oai/logP_supp.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "1547c023-b190-463f-9d3d-f0b0a4d71a39", 6 | "metadata": {}, 7 | "source": [ 8 | "- https://gist.github.com/vgel/55b845833af9f189ae1f834d5f7da8c3\n", 9 | " - logprob_inconsistency.py\n", 10 | " \n", 11 | "\n", 12 | "```python\n", 13 | "responses = []\n", 14 | "for _ in range(2):\n", 15 | " resp = client.chat.completions.create(\n", 16 | " model=model,\n", 17 | " messages=[\n", 18 | " {\"role\": \"user\", \"content\": question},\n", 19 | " ],\n", 20 | " temperature=1.0,\n", 21 | " max_tokens=3,\n", 22 | " logprobs=True,\n", 23 | " top_logprobs=3,\n", 24 | " )\n", 25 | " print(f\"{model} trial {_ + 1} chosen tokens:\", resp.choices[0].message.content)\n", 26 | " responses.append(resp.choices[0].logprobs.content)\n", 27 | "```" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "id": "dd2b9e4a-a215-41ac-bca5-f6e5a169e154", 33 | "metadata": {}, 34 | "source": [ 35 | "$$\n", 36 | "\\exp(\\log p)=p\n", 37 | "$$" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "id": "652a4147-70a0-4b01-8d1e-b85fda19ff1c", 43 | "metadata": {}, 44 | "source": [ 45 | "```\n", 46 | "gpt-4o-mini trial 1 chosen tokens: AABBA (A|AB|BA)\n", 47 | "gpt-4o-mini trial 2 chosen tokens: ABBAAB (AB|BA|AB)\n", 48 | "gpt-4o-mini trial 1 // gpt-4o-mini trial 2\n", 49 | "-----------------------------------//-----------------------------------\n", 50 | "- 'AB' | 51.31% | -0.66736 // 'AB' | 51.92% | -0.65544\n", 51 | "- 'BA' | 31.12% | -1.16736 // 'BA' | 31.49% | -1.15544\n", 52 | "- 'A' | 8.92% | -2.41736 // 'A' | 9.02% | -2.40544\n", 53 | "\n", 54 | "- 'AB' | 99.07% | -0.00932 // 'BA' | 81.25% | -0.20761\n", 55 | "- 'ABB' | 0.86% | -4.75932 // 'AB' | 18.13% | -1.70761\n", 56 | "- 'ABA' | 0.07% | -7.25932 // 'ABA' | 0.33% | -5.70761\n", 57 | "\n", 58 | "- 'BA' | 92.01% | -0.08323 // 'AB' | 88.97% | -0.11682\n", 59 | "- 'AB' | 7.55% | -2.58323 // 'A' | 9.38% | -2.36682\n", 60 | "- 'ABB' | 0.23% | -6.08323 // 'ABA' | 1.27% | -4.36682\n", 61 | "```\n", 62 | "\n", 63 | "```\n", 64 | "deepseek-chat trial 1 chosen tokens: ABABBA\n", 65 | "deepseek-chat trial 2 chosen tokens: ABABBA\n", 66 | "deepseek-chat trial 1 // deepseek-chat trial 2\n", 67 | "-----------------------------------//-----------------------------------\n", 68 | "- 'AB' | 52.90% | -0.63672 // 'AB' | 56.94% | -0.56308\n", 69 | "- 'Here' | 46.58% | -0.76399 // 'Here' | 42.72% | -0.85039\n", 70 | "- 'BA' | 0.33% | -5.70764 // 'ABA' | 0.16% | -6.42029\n", 71 | "\n", 72 | "- 'BA' | 59.17% | -0.52468 // 'BA' | 60.01% | -0.51058\n", 73 | "- 'AB' | 39.07% | -0.93970 // 'AB' | 38.48% | -0.95510\n", 74 | "- 'ABA' | 1.75% | -4.04636 // 'ABA' | 1.49% | -4.20826\n", 75 | "\n", 76 | "- 'BA' | 99.92% | -0.00077 // 'BA' | 99.73% | -0.00274\n", 77 | "- 'BB' | 0.08% | -7.16542 // 'BB' | 0.27% | -5.91770\n", 78 | "- 'AB' | 0.00% | -13.57515 // 'AB' | 0.00% | -10.23312\n", 79 | "```" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": null, 85 | "id": "cf292eb2-b590-42f3-8e1e-355cbbdc7d8e", 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [] 89 | } 90 | ], 91 | "metadata": { 92 | "kernelspec": { 93 | "display_name": "Python 3 (ipykernel)", 94 | "language": "python", 95 | "name": "python3" 96 | }, 97 | "language_info": { 98 | "codemirror_mode": { 99 | "name": "ipython", 100 | "version": 3 101 | }, 102 | "file_extension": ".py", 103 | "mimetype": "text/x-python", 104 | "name": "python", 105 | "nbconvert_exporter": "python", 106 | "pygments_lexer": "ipython3", 107 | "version": "3.10.0" 108 | } 109 | }, 110 | "nbformat": 4, 111 | "nbformat_minor": 5 112 | } 113 | -------------------------------------------------------------------------------- /tutorials/ocr/llm_ocr.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "b3aa3e94-d49b-4cf3-9430-8ec9dc50a526", 6 | "metadata": {}, 7 | "source": [ 8 | "- https://www.runpulse.com/blog/why-llms-suck-at-ocr\n", 9 | " - https://arxiv.org/pdf/2407.06581v1\n", 10 | " - Vision language models are blind" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "id": "9b7bbd8e-4397-4c84-b4e5-ea7e334d0cec", 16 | "metadata": {}, 17 | "source": [ 18 | "- intuitives\n", 19 | " - Original Text → Common LLM Substitutions‍\n", 20 | " - \"l1lI\" → \"1111\" or \"LLLL\"\n", 21 | " - \"O0o\" → \"000\" or \"OOO\"\n", 22 | " - \"vv\" → \"w\"\n", 23 | " - \"cl\" → \"d\"\n", 24 | "- 幻觉的根源是ViT对局部的不足 + 概率sampling\n", 25 | " - only ViT is not enough => cnn + vit ??\n", 26 | " - llm 建模（language modeling）的是联合概率，自回归的条件生成是：联合概率" 27 | ] 28 | } 29 | ], 30 | "metadata": { 31 | "kernelspec": { 32 | "display_name": "verl", 33 | "language": "python", 34 | "name": "verl" 35 | }, 36 | "language_info": { 37 | "codemirror_mode": { 38 | "name": "ipython", 39 | "version": 3 40 | }, 41 | "file_extension": ".py", 42 | "mimetype": "text/x-python", 43 | "name": "python", 44 | "nbconvert_exporter": "python", 45 | "pygments_lexer": "ipython3", 46 | "version": "3.9.0" 47 | } 48 | }, 49 | "nbformat": 4, 50 | "nbformat_minor": 5 51 | } 52 | -------------------------------------------------------------------------------- /tutorials/rag_in_action/GraphRAG.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "f489b0aa-0f2a-4d6a-a4bd-e2adfeea1b53", 6 | "metadata": {}, 7 | "source": [ 8 | "- 将非结构化文本转化为结构化的知识图谱\n", 9 | " - 利用图数据库中的关联性检索以及社区发现等算法，提升回答的准确性和可解释性；\n", 10 | "- 两阶段实现\n", 11 | " - 索引阶段：index\n", 12 | " - 层次聚类及社区总结，基于 Leiden 算法对图谱进行层次聚类，形成紧密关联的社区结构，通过自底向上的方式生成社区摘要；\n", 13 | " - 学术论文库中，数学和物理领域各自形成独立的社区，每个社区都会生成关于该核心主题的总结 \n", 14 | " - 查询阶段：query\n", 15 | " - 全局搜索：涉及数据集整体的查询，通过社区摘要进行多批次的信息整合，生成聚合响应\n", 16 | " - 局部搜索：专注于特定实体，通过向量数据库检索相关实体，并结合社区报告生成精准答案；" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "id": "4cb46b19-0ab3-49e2-a700-663a8b47ca1d", 22 | "metadata": {}, 23 | "source": [ 24 | "## GraphRAG" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "id": "d0857959-b2fb-4c54-a9bb-3cac3750a54f", 30 | "metadata": {}, 31 | "source": [ 32 | "- 传统 RAG\n", 33 | " - 文本 chunks 间的复杂关系与层次结构；\n", 34 | " - 固定数量的文本 chunks\n", 35 | " - “这本小说的主旨”\n", 36 | "- GraphRAG 的核心：“全局性”\n", 37 | " - 提取实体（Entity）\n", 38 | " - 提取关系（Relation）" 39 | ] 40 | } 41 | ], 42 | "metadata": { 43 | "kernelspec": { 44 | "display_name": "Python 3 (ipykernel)", 45 | "language": "python", 46 | "name": "python3" 47 | }, 48 | "language_info": { 49 | "codemirror_mode": { 50 | "name": "ipython", 51 | "version": 3 52 | }, 53 | "file_extension": ".py", 54 | "mimetype": "text/x-python", 55 | "name": "python", 56 | "nbconvert_exporter": "python", 57 | "pygments_lexer": "ipython3", 58 | "version": "3.13.5" 59 | } 60 | }, 61 | "nbformat": 4, 62 | "nbformat_minor": 5 63 | } 64 | -------------------------------------------------------------------------------- /tutorials/rag_in_action/chap1-simple-rag/simple_rag.py: -------------------------------------------------------------------------------- 1 | import os 2 | from dotenv import load_dotenv 3 | from llama_index.core import VectorStoreIndex, SimpleDirectoryReader 4 | from llama_index.embeddings.huggingface import HuggingFaceEmbedding 5 | # from llama_index.llms.deepseek import DeepSeek 6 | 7 | def simple_rag(embed_model): 8 | documents = SimpleDirectoryReader(input_files=["data/黑悟空/设定.txt"]).load_data() 9 | index = VectorStoreIndex.from_documents(documents, embed_model=embed_model) 10 | index.storage_context.persist("data/黑悟空/index") 11 | # query_engine = index.as_query_engine(llm=DeepSeek(api_key=os.getenv("DEEPSEEK_API_KEY"))) 12 | query_engine = index.as_query_engine() 13 | print(query_engine.query("黑神话悟空中有哪些战斗工具?")) 14 | 15 | 16 | if __name__ == "__main__": 17 | assert load_dotenv() 18 | embed_model = HuggingFaceEmbedding(model_name="BAAI/bge-m3") 19 | -------------------------------------------------------------------------------- /tutorials/rag_in_action/overall.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "5a6e64cf-6093-4dcf-b0be-9eb242123a71", 6 | "metadata": {}, 7 | "source": [ 8 | "### retrieval" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "cab719b4-c775-4eb6-ae1c-2633da64706a", 14 | "metadata": {}, 15 | "source": [ 16 | "- embedding-based\n", 17 | " - BGE: BAAI General Embedding\n", 18 | " - BERT + 对抗样本训练\n", 19 | " - 锚点样本 ($p$)\n", 20 | " - 普通正样本，硬正样本 ($q$)\n", 21 | " - 普通负样本，硬负样本 ($q'$)\n", 22 | "- term-based\n", 23 | " - bm25：tf(词频)-idf(稀有性，log)\n", 24 | " - 倒排索引（inverted index）" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "id": "249bfbe6-7d8d-43d9-876f-1133e55638ec", 30 | "metadata": {}, 31 | "source": [ 32 | "- info-nce loss\n", 33 | " - -log(`[0-1]`): 这是个数学技巧。我们想让括号里的值越大越好（越接近1）\n", 34 | "$$\n", 35 | "\\min \\sum_{(p, q)} - \\log \\frac{\\exp(\\langle \\mathbf{e}_p, \\mathbf{e}_q \\rangle / \\tau)}{\\exp(\\langle \\mathbf{e}_p, \\mathbf{e}_q \\rangle / \\tau) + \\sum_{Q'} \\exp(\\langle \\mathbf{e}_p, \\mathbf{e}_{q'} \\rangle / \\tau)}\n", 36 | "$$\n", 37 | "\n", 38 | "- 硬负样本挖掘 (Hard Negative Mining), 其中 $\\mathcal{H}_p$ 是从所有负样本中挑选出来的硬负样本集合。\n", 39 | " - 硬正样本挖掘 (Hard Positive Mining): 这通常不是通过修改损失函数本身，而是通过数据增强策略来实现的。在为锚点 p 生成其正样本 q 时，我们刻意使用非常强的增强手段，使得 p 和 q 在像素层面差异巨大，从而创造出一个“硬正样本”。损失函数的形式保持不变，但输入的样本对本身已经具备了“硬”的属性。\n", 40 | "$$\n", 41 | "\\min \\sum_{(p, q)} - \\log \\frac{\\exp(\\langle \\mathbf{e}_p, \\mathbf{e}_q \\rangle / \\tau)}{\\exp(\\langle \\mathbf{e}_p, \\mathbf{e}_q \\rangle / \\tau) + \\sum_{\\mathbf{e}_{q'} \\in \\mathcal{H}_p} \\exp(\\langle \\mathbf{e}_p, \\mathbf{e}_{q'} \\rangle / \\tau)}\n", 42 | "$$" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "id": "213d882a-4961-4713-9a30-0d92bfd4416a", 49 | "metadata": {}, 50 | "outputs": [], 51 | "source": [] 52 | } 53 | ], 54 | "metadata": { 55 | "kernelspec": { 56 | "display_name": "Python 3 (ipykernel)", 57 | "language": "python", 58 | "name": "python3" 59 | }, 60 | "language_info": { 61 | "codemirror_mode": { 62 | "name": "ipython", 63 | "version": 3 64 | }, 65 | "file_extension": ".py", 66 | "mimetype": "text/x-python", 67 | "name": "python", 68 | "nbconvert_exporter": "python", 69 | "pygments_lexer": "ipython3", 70 | "version": "3.10.0" 71 | } 72 | }, 73 | "nbformat": 4, 74 | "nbformat_minor": 5 75 | } 76 | -------------------------------------------------------------------------------- /tutorials/rag_in_action/requirements.txt: -------------------------------------------------------------------------------- 1 | openai 2 | 3 | llama-index 4 | llama-index-embeddings-huggingface 5 | llama-index-llms-deepseek 6 | 7 | langchain 8 | langchain-openai 9 | langchain-deepseek 10 | langchain-community 11 | langchain-huggingface 12 | sentence-transformers 13 | 14 | beautifulsoup4 15 | 16 | python-dotenv 17 | 18 | pymilvus 19 | -------------------------------------------------------------------------------- /tutorials/rag_in_action/图文检索.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "97391b9e-7fc4-4f0f-b846-9aa8d6954377", 6 | "metadata": {}, 7 | "source": [ 8 | "- https://milvus.io/zh/blog/multimodal-semantic-search-with-images-and-text.md" 9 | ] 10 | } 11 | ], 12 | "metadata": { 13 | "kernelspec": { 14 | "display_name": "Python 3 (ipykernel)", 15 | "language": "python", 16 | "name": "python3" 17 | }, 18 | "language_info": { 19 | "codemirror_mode": { 20 | "name": "ipython", 21 | "version": 3 22 | }, 23 | "file_extension": ".py", 24 | "mimetype": "text/x-python", 25 | "name": "python", 26 | "nbconvert_exporter": "python", 27 | "pygments_lexer": "ipython3", 28 | "version": "3.10.0" 29 | } 30 | }, 31 | "nbformat": 4, 32 | "nbformat_minor": 5 33 | } 34 | -------------------------------------------------------------------------------- /tutorials/workflow/cipher_examples.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import builtins 3 | import contextlib 4 | import io 5 | from typing import Any 6 | from dotenv import load_dotenv, find_dotenv 7 | 8 | from langchain.chat_models import init_chat_model 9 | from langchain_core.runnables import RunnableConfig 10 | from langgraph.checkpoint.memory import MemorySaver 11 | 12 | from langgraph_codeact import create_codeact, create_default_prompt 13 | 14 | 15 | def eval(code: str, _locals: dict[str, Any]) -> tuple[str, dict[str, Any]]: 16 | # Store original keys before execution 17 | original_keys = set(_locals.keys()) 18 | 19 | print(f"++++++++++ 💻 Executing Code ++++++++++") 20 | print(code) 21 | print(f"++++++++++++++++++++++++++++++++++++++") 22 | 23 | try: 24 | with contextlib.redirect_stdout(io.StringIO()) as f: 25 | exec(code, builtins.__dict__, _locals) 26 | 27 | print(f"---------- 📄 Code Output ----------") 28 | result = f.getvalue() 29 | print(result) 30 | print(f"------------------------------------\n") 31 | if not result: 32 | result = "" 33 | except Exception as e: 34 | result = f"Error during execution: {repr(e)}" 35 | 36 | # Determine new variables created during execution 37 | new_keys = set(_locals.keys()) - original_keys 38 | new_vars = {key: _locals[key] for key in new_keys} 39 | return result, new_vars 40 | 41 | 42 | def caesar_shift_decode(text: str, shift: int) -> str: 43 | """Decode text that was encoded using Caesar shift. 44 | 45 | Args: 46 | text: The encoded text to decode 47 | shift: The number of positions to shift back (positive number) 48 | 49 | Returns: 50 | The decoded text 51 | """ 52 | result = "" 53 | for char in text: 54 | if char.isalpha(): 55 | # Determine the case and base ASCII value 56 | ascii_base = ord("A") if char.isupper() else ord("a") 57 | # Shift the character back and wrap around if needed 58 | shifted = (ord(char) - ascii_base - shift) % 26 59 | result += chr(ascii_base + shifted) 60 | else: 61 | result += char 62 | return result 63 | 64 | 65 | def base64_decode(text: str) -> str: 66 | """Decode text that was encoded using base64. 67 | 68 | Args: 69 | text: The base64 encoded text to decode 70 | 71 | Returns: 72 | The decoded text as a string 73 | 74 | Raises: 75 | Exception: If the input is not valid base64 76 | """ 77 | # Add padding if needed 78 | padding = 4 - (len(text) % 4) 79 | if padding != 4: 80 | text += "=" * padding 81 | 82 | # Decode the base64 string 83 | decoded_bytes = base64.b64decode(text) 84 | return decoded_bytes.decode("utf-8") 85 | 86 | 87 | def caesar_shift_encode(text: str, shift: int) -> str: 88 | """Encode text using Caesar shift. 89 | 90 | Args: 91 | text: The text to encode 92 | shift: The number of positions to shift forward (positive number) 93 | 94 | Returns: 95 | The encoded text 96 | """ 97 | result = "" 98 | for char in text: 99 | if char.isalpha(): 100 | # Determine the case and base ASCII value 101 | ascii_base = ord("A") if char.isupper() else ord("a") 102 | # Shift the character forward and wrap around if needed 103 | shifted = (ord(char) - ascii_base + shift) % 26 104 | result += chr(ascii_base + shifted) 105 | else: 106 | result += char 107 | return result 108 | 109 | 110 | def base64_encode(text: str) -> str: 111 | """Encode text using base64. 112 | 113 | Args: 114 | text: The text to encode 115 | 116 | Returns: 117 | The base64 encoded text as a string 118 | """ 119 | # Convert text to bytes and encode 120 | text_bytes = text.encode("utf-8") 121 | encoded_bytes = base64.b64encode(text_bytes) 122 | return encoded_bytes.decode("utf-8") 123 | 124 | 125 | assert load_dotenv(find_dotenv()) 126 | # List of available tools 127 | tools = [ 128 | caesar_shift_decode, 129 | base64_decode, 130 | caesar_shift_encode, 131 | base64_encode, 132 | ] 133 | 134 | model = init_chat_model("gemini-2.5-flash", model_provider="google_genai") 135 | code_act = create_codeact( 136 | model, 137 | tools, 138 | eval, 139 | prompt=create_default_prompt( 140 | tools, 141 | "Once you have the final answer, respond to the user with plain text, do not respond with a code snippet.", 142 | ), 143 | ) 144 | agent = code_act.compile(checkpointer=MemorySaver()) 145 | 146 | if __name__ == "__main__": 147 | 148 | def stream_from_agent(messages: list[dict], config: RunnableConfig): 149 | for typ, chunk in agent.stream( 150 | {"messages": messages}, 151 | stream_mode=["values", "messages"], 152 | config=config, 153 | ): 154 | if typ == "messages": 155 | print(chunk[0].content, end="") 156 | elif typ == "values": 157 | print("\n\n---answer---\n\n", chunk) 158 | 159 | # first turn 160 | config = {"configurable": {"thread_id": 1}} 161 | stream_from_agent( 162 | [ 163 | { 164 | "role": "user", 165 | "content": "Decipher this text: 'VGhybCB6dnRsYW9wdW4gZHZ1a2x5bWJz'", 166 | } 167 | ], 168 | config, 169 | ) 170 | 171 | # second turn 172 | # stream_from_agent( 173 | # [ 174 | # { 175 | # "role": "user", 176 | # "content": "Using the same cipher as the original text, encode this text: 'The work is mysterious and important'", 177 | # } 178 | # ], 179 | # config, 180 | # ) -------------------------------------------------------------------------------- /tutorials/炼丹/炼丹&调参_tricks.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "d3134108", 6 | "metadata": {}, 7 | "source": [ 8 | "- reference\n", 9 | " - http://karpathy.github.io/2019/04/25/recipe/" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "id": "5c1ebce5", 15 | "metadata": {}, 16 | "source": [ 17 | "## batch size & learning rate" 18 | ] 19 | }, 20 | { 21 | "cell_type": "markdown", 22 | "id": "075979d9", 23 | "metadata": {}, 24 | "source": [ 25 | "- Adam & 3e-4\n", 26 | " - karpathy constant\n", 27 | "- batch size and learning rate" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "id": "ac4cfa19", 33 | "metadata": {}, 34 | "source": [ 35 | "## `tokenizer.padding_side`" 36 | ] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "id": "0ab430bc", 41 | "metadata": {}, 42 | "source": [ 43 | "If you want to fine-tune an LLM while using `FlashAttention`, you must set the padding side to “left”. Otherwise, it won’t work.\n", 44 | "\n", 45 | "```\n", 46 | "tokenizer.padding_side = 'left'\n", 47 | "```" 48 | ] 49 | } 50 | ], 51 | "metadata": { 52 | "kernelspec": { 53 | "display_name": "Python 3 (ipykernel)", 54 | "language": "python", 55 | "name": "python3" 56 | }, 57 | "language_info": { 58 | "codemirror_mode": { 59 | "name": "ipython", 60 | "version": 3 61 | }, 62 | "file_extension": ".py", 63 | "mimetype": "text/x-python", 64 | "name": "python", 65 | "nbconvert_exporter": "python", 66 | "pygments_lexer": "ipython3", 67 | "version": "3.10.0" 68 | }, 69 | "toc": { 70 | "base_numbering": 1, 71 | "nav_menu": {}, 72 | "number_sections": true, 73 | "sideBar": true, 74 | "skip_h1_title": false, 75 | "title_cell": "Table of Contents", 76 | "title_sidebar": "Contents", 77 | "toc_cell": false, 78 | "toc_position": {}, 79 | "toc_section_display": true, 80 | "toc_window_display": false 81 | } 82 | }, 83 | "nbformat": 4, 84 | "nbformat_minor": 5 85 | } 86 | -------------------------------------------------------------------------------- /tutorials/炼丹/超参搜索.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "2219f2f0-2ce7-4810-9758-56459f10b3c4", 6 | "metadata": {}, 7 | "source": [ 8 | "- optuna: 一个超参数优化框架\n", 9 | " - 可以可视化超参的重要性" 10 | ] 11 | } 12 | ], 13 | "metadata": { 14 | "kernelspec": { 15 | "display_name": "Python 3 (ipykernel)", 16 | "language": "python", 17 | "name": "python3" 18 | }, 19 | "language_info": { 20 | "codemirror_mode": { 21 | "name": "ipython", 22 | "version": 3 23 | }, 24 | "file_extension": ".py", 25 | "mimetype": "text/x-python", 26 | "name": "python", 27 | "nbconvert_exporter": "python", 28 | "pygments_lexer": "ipython3", 29 | "version": "3.10.0" 30 | } 31 | }, 32 | "nbformat": 4, 33 | "nbformat_minor": 5 34 | } 35 | --------------------------------------------------------------------------------