├── .gitignore
├── DISCLAIMER
├── LICENSE
├── README.md
├── README_CN.md
├── cli_demo.py
├── eval_qwen_wisdomvast.py
├── images
├── image.png
└── logo.png
├── merge_lora.py
├── requirements.txt
├── vllm_web_demo.py
└── web_demo.py
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | share/python-wheels/
24 | *.egg-info/
25 | .installed.cfg
26 | *.egg
27 | MANIFEST
28 |
29 | # PyInstaller
30 | # Usually these files are written by a python script from a template
31 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
32 | *.manifest
33 | *.spec
34 |
35 | # Installer logs
36 | pip-log.txt
37 | pip-delete-this-directory.txt
38 |
39 | # Unit test / coverage reports
40 | htmlcov/
41 | .tox/
42 | .nox/
43 | .coverage
44 | .coverage.*
45 | .cache
46 | nosetests.xml
47 | coverage.xml
48 | *.cover
49 | *.py,cover
50 | .hypothesis/
51 | .pytest_cache/
52 | cover/
53 |
54 | # Translations
55 | *.mo
56 | *.pot
57 |
58 | # Django stuff:
59 | *.log
60 | local_settings.py
61 | db.sqlite3
62 | db.sqlite3-journal
63 |
64 | # Flask stuff:
65 | instance/
66 | .webassets-cache
67 |
68 | # Scrapy stuff:
69 | .scrapy
70 |
71 | # Sphinx documentation
72 | docs/_build/
73 |
74 | # PyBuilder
75 | .pybuilder/
76 | target/
77 |
78 | # Jupyter Notebook
79 | .ipynb_checkpoints
80 |
81 | # IPython
82 | profile_default/
83 | ipython_config.py
84 |
85 | # pyenv
86 | # For a library or package, you might want to ignore these files since the code is
87 | # intended to run in multiple environments; otherwise, check them in:
88 | # .python-version
89 |
90 | # pipenv
91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
94 | # install all needed dependencies.
95 | #Pipfile.lock
96 |
97 | # poetry
98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99 | # This is especially recommended for binary packages to ensure reproducibility, and is more
100 | # commonly ignored for libraries.
101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102 | #poetry.lock
103 |
104 | # pdm
105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106 | #pdm.lock
107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108 | # in version control.
109 | # https://pdm.fming.dev/#use-with-ide
110 | .pdm.toml
111 |
112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
113 | __pypackages__/
114 |
115 | # Celery stuff
116 | celerybeat-schedule
117 | celerybeat.pid
118 |
119 | # SageMath parsed files
120 | *.sage.py
121 |
122 | # Environments
123 | .env
124 | .venv
125 | env/
126 | venv/
127 | ENV/
128 | env.bak/
129 | venv.bak/
130 |
131 | # Spyder project settings
132 | .spyderproject
133 | .spyproject
134 |
135 | # Rope project settings
136 | .ropeproject
137 |
138 | # mkdocs documentation
139 | /site
140 |
141 | # mypy
142 | .mypy_cache/
143 | .dmypy.json
144 | dmypy.json
145 |
146 | # Pyre type checker
147 | .pyre/
148 |
149 | # pytype static type analyzer
150 | .pytype/
151 |
152 | # Cython debug symbols
153 | cython_debug/
154 |
155 | # PyCharm
156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
158 | # and can be added to the global gitignore or merged into this file. For a more nuclear
159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder.
160 | #.idea/
161 | train_log.md
162 | upload_modelscope.py
--------------------------------------------------------------------------------
/DISCLAIMER:
--------------------------------------------------------------------------------
1 | The software project, data, and models provided by our GitHub project are provided "as is," without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose, and non-infringement.
2 |
3 | In no event shall the project owners or contributors be liable for any direct, indirect, incidental, special, exemplary, or consequential damages (including, but not limited to, procurement of substitute goods or services; loss of use, data, or profits; or business interruption) however caused and on any theory of liability, whether in contract, strict liability, or tort (including negligence or otherwise) arising in any way out of the use of this software project, data, or models, even if advised of the possibility of such damage.
4 |
5 | Users of this software project, data, and models are solely responsible for any consequences of their use. The project owners and contributors shall not be held responsible for any subsequent or potential harm caused by the use of this software project, data, or models.
6 |
7 | By using this software project, data, or models, users accept and agree to this disclaimer. If users do not agree to the terms of this disclaimer, they should not use this software project, data, or models.
8 |
9 | It is important to note that this software project, data, and models are still in the research phase and are provided for experimental purposes only. As such, the project owners and contributors do not guarantee the accuracy, completeness, or usefulness of the software project, data, or models.
10 |
11 | Furthermore, due to the experimental nature of this software project, data, and models, it is possible that they may contain or generate inappropriate responses, errors, or inconsistencies. Users should exercise caution when using this software project, data, or models, and should not rely solely on them for any critical or sensitive tasks.
12 |
13 | The project owners and contributors shall not be held responsible for any damages, losses, or liabilities arising from the use of this software project, data, or models, including but not limited to, any inappropriate responses generated by the software project, data, or models.
14 |
15 | By using this software project, data, or models, users acknowledge and accept the experimental nature of the software project, data, and models, and understand the potential risks and limitations associated with their use. If users do not agree to the terms of this disclaimer, they should not use this software project, data, or models.
16 |
17 | The software project, data, and models provided by our GitHub project are intended for research purposes only. They should not be used for any commercial, business, or legal purposes, and should not be relied upon as a substitute for professional advice or judgment.
18 |
19 | Users of this software project, data, and models are strictly prohibited from using them for any commercial purposes, including but not limited to, selling, licensing, or distributing the software project, data, or models to third parties.
20 |
21 | The project owners and contributors shall not be held responsible for any damages, losses, or liabilities arising from the use of this software project, data, or models for any commercial or business purposes.
22 |
23 | By using this software project, data, or models, users agree to use them for research purposes only, and not for any commercial or business purposes. If users do not agree to the terms of this disclaimer, they should not use this software project, data, or models.
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright [yyyy] [name of copyright owner]
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 | 中文 ｜ English
3 |
4 |
5 |
6 |
7 |
8 | 
9 |
10 |
11 |
12 |
32 |
33 |
34 | ## Introduce
35 |
36 | **Qwen-WisdomVast** is a large model trained on 1 million high-quality Chinese multi-turn SFT data, 200,000 English multi-turn SFT data, and 2,000 single-turn self-cognition data, using the training methods of [DORA](https://arxiv.org/pdf/2402.09353.pdf) and [LORA+](https://arxiv.org/pdf/2402.12354.pdf) based on **Qwen1.5-7B** as the base. Compared to Qwen1.5-7B-Chat, it has improved **mathematical abilities** by **5.16%**, **12.8%** on the **HumanEval** dataset, **11.6%** on the **MBPP** dataset, and **12.44%** on the **BBH** dataset. The performance on all evaluations is shown in the table below.
37 |
38 | 
39 |
40 |
41 | ## Evaluation Results
42 |
43 | | Model | MMLU | C-Eval | GSM8K | MATH | HumanEval | MBPP | BBH |
44 | |-------------------|-------|--------|-------|-------|-----------|-------|-------|
45 | | **Qwen1.5-7B-Chat** | 60.88 | 70.18 | 54.13 | 7.96 | 31.10 | 15.00 | 31.67 |
46 | | **Qwen-WisdomVast** | 57.09 | **70.82** | 51.93 | **13.12** | **43.90** | **26.60** | **44.11** |
47 |
48 |
49 | Explanation:
50 |
51 | Since the official evaluation performance of Qwen1.5-7B-Chat has not been disclosed, we conducted our own testing using [opencompass](https://github.com/open-compass/opencompass) and obtained the above results.
52 |
53 | Qwen-WisdomVast was tested using the same parameters as Qwen1.5-7B-Chat.
54 |
55 |
56 | ## Download Model
57 |
58 | | Model | Download |
59 | |:-------------------:|:-----------:|
60 | | Qwen1.5-7B |[ 🤗 HuggingFace](https://huggingface.co/Qwen/Qwen1.5-7B) [ 🤖 ModelScope](https://modelscope.cn/models/qwen/Qwen1.5-7B)|
61 | | Qwen-WisdomVast-Lora |[ 🤗 HuggingFace](https://huggingface.co/zhichen/Qwen-WisdomVast-Lora) [ 🤖 ModelScope](https://modelscope.cn/models/seanzhang/Qwen-WisdomVast-Lora)|
62 | | Qwen-WisdomVast (Merged Model) |[ 🤗 HuggingFace](https://huggingface.co/zhichen/Qwen-WisdomVast) [ 🤖 ModelScope](https://modelscope.cn/models/seanzhang/Qwen-WisdomVast)|
63 |
64 |
65 | ## Merge LORA Model (Skippable)
66 |
67 | 1、Download [Qwen1.5-7B](https://modelscope.cn/models/qwen/Qwen1.5-7B)
68 |
69 | ```bash
70 | git clone https://www.modelscope.cn/qwen/Qwen1.5-7B.git
71 | ```
72 |
73 | 2、Download [Qwen-WisdomVast-Lora](https://www.modelscope.cn/models/seanzhang/Qwen-WisdomVast-Lora)
74 |
75 | **From ModelScope**
76 | ```bash
77 | git lfs install
78 | git clone https://www.modelscope.cn/seanzhang/Qwen-WisdomVast-Lora.git
79 |
80 | ```
81 |
82 | **From HuggingFace**
83 | ```bash
84 | git lfs install
85 | git clone https://huggingface.co/zhichen/Qwen-WisdomVast-Lora
86 | ```
87 |
88 | 3、Merge Model
89 |
90 | ```bash
91 | python merge_lora.py \
92 | --base_model path/to/qwen/Qwen1.5-7B \
93 | --lora_model path/to/lora/Qwen-WisdomVast-Lora \
94 | --output_dir ./Qwen-WisdomVast
95 | ```
96 |
97 |
98 | ## Download Qwen-WisdomVast (Merged Model)
99 |
100 | **From ModelScope**
101 | ```bash
102 | git lfs install
103 | git clone https://www.modelscope.cn/seanzhang/Qwen-WisdomVast.git
104 |
105 | ```
106 |
107 | **From HuggingFace**
108 | ```bash
109 | git lfs install
110 | git clone https://huggingface.co/zhichen/Qwen-WisdomVast
111 | ```
112 |
113 | ## CLI DEMO
114 |
115 | ```bash
116 | python cli_demo.py --model_path ./Qwen-WisdomVast(Replace it with your own merged model path)
117 | ```
118 |
119 | ## WEB DEMO
120 |
121 | ```bash
122 | python web_demo.py --model_path ./Qwen-WisdomVast(Replace it with your own merged model path)
123 | ```
124 |
125 |
126 | ## VLLM WEB DEMO
127 |
128 | 1、Use [vllm](https://github.com/vllm-project/vllm) deploy model
129 |
130 | ```bash
131 | python -m vllm.entrypoints.openai.api_server --served-model-name Qwen-WisdomVast --model ./Qwen-WisdomVast(Replace it with your own merged model path)
132 | ```
133 |
134 | 2、This command is executed on the CLI
135 |
136 | ```bash
137 | python vllm_web_demo.py --model Qwen-WisdomVast
138 | ```
139 |
140 |
141 | ## Repeat the evaluation results
142 |
143 | 1、Use [vllm](https://github.com/vllm-project/vllm) deploy `openai api server`
144 |
145 | deploy command:
146 |
147 | ```bash
148 | python -m vllm.entrypoints.openai.api_server --served-model-name Qwen-WisdomVast --model ./Qwen-WisdomVast(Replace it with your own merged model path)
149 | ```
150 |
151 | 2、Use [opencompass](https://github.com/open-compass/opencompass) framework to eval
152 |
153 | Reference: [Verify model effects using opencompass](https://blog.csdn.net/qq_44193969/article/details/134979054)
154 |
155 | After modifying as described above, copy the `eval_qwen_wisdomvast.py` file in the `opencompass/configs` folder
156 |
157 |
158 | 3、Execute test script
159 |
160 | ```bash
161 | python run.py configs/eval_qwen_wisdomvast.py -w outputs/Qwen-WisdomVast
162 | ```
163 |
164 |
165 | ## LICENSE
166 |
167 | This project can only be used for research purposes, and the project developer shall not bear any harm or loss caused by the use of this project (including but not limited to data, models, codes, etc.). For details, please refer to [DISCLAIMER](https://github.com/seanzhang-zhichen/Qwen-WisdomVast/blob/main/DISCLAIMER)。
168 |
169 | The License agreement of the Qwen-WisdomVast project code is the [Apache License 2.0](./LICENSE). The code is free for commercial use, and the model weights and data can only be used for research purposes. Please attach a link to Qwen-WisdomVast and the licensing agreement in the product description.
170 |
171 |
172 | ## Citation
173 |
174 | If you used Qwen-WisdomVast in your research, cite it in the following format:
175 |
176 | ```latex
177 | @misc{Qwen-WisdomVast,
178 | title={Qwen-WisdomVast},
179 | author={Zhichen Zhang, Weihan Huang},
180 | year={2024},
181 | howpublished={\url{https://github.com/seanzhang-zhichen/Qwen-WisdomVast}},
182 | }
183 | ```
184 |
185 | ## Acknowledgement
186 |
187 | [QwenLM/Qwen1.5](https://github.com/QwenLM/Qwen1.5)
188 |
189 | [hiyouga/LLaMA-Factory](https://github.com/hiyouga/LLaMA-Factory)
190 |
191 | [shibing624/MedicalGPT](https://github.com/shibing624/MedicalGPT)
192 |
193 | [modelscope/swift](https://github.com/modelscope/swift)
194 |
195 | ## Star History
196 |
197 | [](https://star-history.com/#seanzhang-zhichen/Qwen-WisdomVast&Date)
--------------------------------------------------------------------------------
/README_CN.md:
--------------------------------------------------------------------------------
1 |
2 | 中文 ｜ English
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
32 |
33 |
34 | ## 介绍
35 |
36 | **Qwen-WisdomVast**是**以Qwen1.5-7B为底座**，使用 [DORA](https://arxiv.org/pdf/2402.09353.pdf) + [LORA+](https://arxiv.org/pdf/2402.12354.pdf) 的训练方法，在100w高质量中文多轮SFT数据 + 20w英文多轮SFT数据 + 2000单轮自我认知数据训练而来的大模型，**数学能力**相比Qwen1.5-7B-Chat**提升了5.16%**，在**HumanEval**数据集上相比Qwen1.5-7B-Chat**提升了12.8**，在**MBPP**数据集上**提升了11.6%**，在**BBH**数据集上**提升了12.44%**，全部评测表现见下表。
37 |
38 | 
39 |
40 | ## 评测表现
41 |
42 | | Model | MMLU | C-Eval | GSM8K | MATH | HumanEval | MBPP | BBH |
43 | |-------------------|-------|--------|-------|-------|-----------|-------|-------|
44 | | **Qwen1.5-7B-Chat** | 60.88 | 70.18 | 54.13 | 7.96 | 31.10 | 15.00 | 31.67 |
45 | | **Qwen-WisdomVast** | 57.09 | **70.82** | 51.93 | **13.12** | **43.90** | **26.60** | **44.11** |
46 |
47 | 说明：
48 |
49 | 由于官方并未公布Qwen1.5-7B-Chat的评测表现，所以我们自己使用[opencompass](https://github.com/open-compass/opencompass)测试得到以上结果
50 |
51 | Qwen-WisdomVast使用和Qwen1.5-7B-Chat一样的参数进行测试
52 |
53 | ## 模型下载
54 |
55 | | Model | Download |
56 | |:-------------------:|:-----------:|
57 | | Qwen1.5-7B |[ 🤗 HuggingFace](https://huggingface.co/Qwen/Qwen1.5-7B) [ 🤖 ModelScope](https://modelscope.cn/models/qwen/Qwen1.5-7B)|
58 | | Qwen-WisdomVast-Lora |[ 🤗 HuggingFace](https://huggingface.co/zhichen/Qwen-WisdomVast-Lora) [ 🤖 ModelScope](https://modelscope.cn/models/seanzhang/Qwen-WisdomVast-Lora)|
59 | | Qwen-WisdomVast (合并好的模型) |[ 🤗 HuggingFace](https://huggingface.co/zhichen/Qwen-WisdomVast) [ 🤖 ModelScope](https://modelscope.cn/models/seanzhang/Qwen-WisdomVast)|
60 |
61 |
62 |
63 | ## 合并LORA模型（可跳过）
64 |
65 | 1、下载 [Qwen1.5-7B](https://modelscope.cn/models/qwen/Qwen1.5-7B)
66 |
67 | ```bash
68 | git clone https://www.modelscope.cn/qwen/Qwen1.5-7B.git
69 | ```
70 |
71 | 2、下载[Qwen-WisdomVast-Lora](https://www.modelscope.cn/models/seanzhang/Qwen-WisdomVast-Lora)
72 |
73 | **From ModelScope**
74 | ```bash
75 | git lfs install
76 | git clone https://www.modelscope.cn/seanzhang/Qwen-WisdomVast-Lora.git
77 |
78 | ```
79 |
80 | **From HuggingFace**
81 | ```bash
82 | git lfs install
83 | git clone https://huggingface.co/zhichen/Qwen-WisdomVast-Lora
84 | ```
85 |
86 | 3、合并模型
87 |
88 | ```bash
89 | python merge_lora.py \
90 | --base_model path/to/qwen/Qwen1.5-7B \
91 | --lora_model path/to/lora/Qwen-WisdomVast-Lora \
92 | --output_dir ./Qwen-WisdomVast
93 | ```
94 |
95 | ## 下载 Qwen-WisdomVast（合并好的模型）
96 |
97 | **From ModelScope**
98 | ```bash
99 | git lfs install
100 | git clone https://www.modelscope.cn/seanzhang/Qwen-WisdomVast.git
101 |
102 | ```
103 |
104 | **From HuggingFace**
105 | ```bash
106 | git lfs install
107 | git clone https://huggingface.co/zhichen/Qwen-WisdomVast
108 | ```
109 |
110 |
111 | ## 命令行推理
112 |
113 | ```bash
114 | python cli_demo.py --model_path ./Qwen-WisdomVast(换成你自己的合并后的模型路径)
115 | ```
116 |
117 | ## web 推理
118 |
119 | ```bash
120 | python web_demo.py --model_path ./Qwen-WisdomVast(换成你自己的合并后的模型路径)
121 | ```
122 |
123 |
124 | ## vllm web 推理
125 |
126 | 1、使用[vllm](https://github.com/vllm-project/vllm)部署模型
127 |
128 | ```bash
129 | python -m vllm.entrypoints.openai.api_server --served-model-name Qwen-WisdomVast --model ./Qwen-WisdomVast(换成你自己的合并后的模型路径)
130 | ```
131 |
132 | 2、在命令行执行
133 |
134 | ```bash
135 | python vllm_web_demo.py --model Qwen-WisdomVast
136 | ```
137 |
138 |
139 | ## 复现测试结果
140 |
141 | 1、使用[vllm](https://github.com/vllm-project/vllm)部署`openai api server`
142 |
143 | 部署命令:
144 |
145 | ```bash
146 | python -m vllm.entrypoints.openai.api_server --served-model-name Qwen-WisdomVast --model ./Qwen-WisdomVast(换成你自己的合并后的模型路径)
147 | ```
148 |
149 | 2、使用[opencompass](https://github.com/open-compass/opencompass)框架进行测试
150 |
151 | 参考：[使用opencompass验证模型效果](https://blog.csdn.net/qq_44193969/article/details/134979054)
152 |
153 | 按照以上文章修改好后，将`eval_qwen_wisdomvast.py`文件复制到 `opencompass/configs`文件夹下
154 |
155 |
156 | 3、执行测试脚本
157 |
158 | ```bash
159 | python run.py configs/eval_qwen_wisdomvast.py -w outputs/Qwen-WisdomVast
160 | ```
161 |
162 | ## LICENSE
163 |
164 | 本项目仅可应用于研究目的，项目开发者不承担任何因使用本项目（包含但不限于数据、模型、代码等）导致的危害或损失。详细请参考[免责声明](https://github.com/seanzhang-zhichen/Qwen-WisdomVast/blob/main/DISCLAIMER)。
165 |
166 | Qwen-WisdomVast项目代码的授权协议为 [The Apache License 2.0](./LICENSE)，代码可免费用做商业用途，模型权重和数据只能用于研究目的。请在产品说明中附加Qwen-WisdomVast的链接和授权协议。
167 |
168 | ## Citation
169 |
170 | 如果你在研究中使用了Qwen-WisdomVast，请按如下格式引用：
171 |
172 | ```latex
173 | @misc{Qwen-WisdomVast,
174 | title={Qwen-WisdomVast},
175 | author={Zhichen Zhang, Weihan Huang},
176 | year={2024},
177 | howpublished={\url{https://github.com/seanzhang-zhichen/Qwen-WisdomVast}},
178 | }
179 | ```
180 |
181 |
182 | ## Acknowledgement
183 |
184 | [QwenLM/Qwen1.5](https://github.com/QwenLM/Qwen1.5)
185 |
186 | [hiyouga/LLaMA-Factory](https://github.com/hiyouga/LLaMA-Factory)
187 |
188 | [shibing624/MedicalGPT](https://github.com/shibing624/MedicalGPT)
189 |
190 | [modelscope/swift](https://github.com/modelscope/swift)
191 |
192 | ## Star History
193 |
194 | [](https://star-history.com/#seanzhang-zhichen/Qwen-WisdomVast&Date)
--------------------------------------------------------------------------------
/cli_demo.py:
--------------------------------------------------------------------------------
1 |
2 | import argparse
3 | import os
4 | import platform
5 | import shutil
6 | import torch
7 | from copy import deepcopy
8 | from threading import Thread
9 | from transformers import GenerationConfig
10 | from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
11 | from transformers.trainer_utils import set_seed
12 |
13 |
14 | _WELCOME_MSG = '''\
15 | Welcome to use Qwen-WisdomVast model, type text to start chat, type :h to show command help.
16 | (欢迎使用 Qwen-WisdomVast 模型，输入内容即可进行对话，:h 显示命令帮助。)
17 |
18 | Note: This demo is governed by the original license of Qwen1.5.
19 | We strongly advise users not to knowingly generate or allow others to knowingly generate harmful content, including hate speech, violence, pornography, deception, etc.
20 | (注：本演示受Qwen1.5的许可协议限制。我们强烈建议，用户不应传播及不应允许他人传播以下内容，包括但不限于仇恨言论、暴力、色情、欺诈相关的有害信息。)
21 | '''
22 | _HELP_MSG = '''\
23 | Commands:
24 | :help / :h Show this help message 显示帮助信息
25 | :exit / :quit / :q Exit the demo 退出Demo
26 | :clear / :cl Clear screen 清屏
27 | :clear-history / :clh Clear history 清除对话历史
28 | :history / :his Show history 显示对话历史
29 | :seed Show current random seed 显示当前随机种子
30 | :seed Set random seed to 设置随机种子
31 | :conf Show current generation config 显示生成配置
32 | :conf = Change generation config 修改生成配置
33 | :reset-conf Reset generation config 重置生成配置
34 | '''
35 | _ALL_COMMAND_NAMES = [
36 | 'help', 'h', 'exit', 'quit', 'q', 'clear', 'cl', 'clear-history', 'clh', 'history', 'his',
37 | 'seed', 'conf', 'reset-conf',
38 | ]
39 |
40 |
41 | def _setup_readline():
42 | try:
43 | import readline
44 | except ImportError:
45 | return
46 |
47 | _matches = []
48 |
49 | def _completer(text, state):
50 | nonlocal _matches
51 |
52 | if state == 0:
53 | _matches = [cmd_name for cmd_name in _ALL_COMMAND_NAMES if cmd_name.startswith(text)]
54 | if 0 <= state < len(_matches):
55 | return _matches[state]
56 | return None
57 |
58 | readline.set_completer(_completer)
59 | readline.parse_and_bind('tab: complete')
60 |
61 |
62 | def _load_model_tokenizer(args):
63 | tokenizer = AutoTokenizer.from_pretrained(
64 | args.model_path, resume_download=True,
65 | )
66 |
67 | if args.cpu_only:
68 | device_map = "cpu"
69 | else:
70 | device_map = "auto"
71 |
72 | model = AutoModelForCausalLM.from_pretrained(
73 | args.model_path,
74 | device_map=device_map,
75 | resume_download=True,
76 | ).eval()
77 |
78 | model.generation_config = GenerationConfig(
79 | bos_token_id = 151643,
80 | do_sample = True,
81 | eos_token_id = [
82 | 151645,
83 | 151643
84 | ],
85 | max_new_tokens = 2048,
86 | repetition_penalty = 1.05,
87 | temperature = 0.7,
88 | top_p = 0.8,
89 | top_k = 20,
90 | )
91 |
92 | return model, tokenizer
93 |
94 |
95 | def _gc():
96 | import gc
97 | gc.collect()
98 | if torch.cuda.is_available():
99 | torch.cuda.empty_cache()
100 |
101 |
102 | def _clear_screen():
103 | if platform.system() == "Windows":
104 | os.system("cls")
105 | else:
106 | os.system("clear")
107 |
108 |
109 | def _print_history(history):
110 | terminal_width = shutil.get_terminal_size()[0]
111 | print(f'History ({len(history)})'.center(terminal_width, '='))
112 | for index, (query, response) in enumerate(history):
113 | print(f'User[{index}]: {query}')
114 | print(f'QWen[{index}]: {response}')
115 | print('=' * terminal_width)
116 |
117 |
118 | def _get_input() -> str:
119 | while True:
120 | try:
121 | message = input('User> ').strip()
122 | except UnicodeDecodeError:
123 | print('[ERROR] Encoding error in input')
124 | continue
125 | except KeyboardInterrupt:
126 | exit(1)
127 | if message:
128 | return message
129 | print('[ERROR] Query is empty')
130 |
131 |
132 | def _chat_stream(model, tokenizer, query, history):
133 | conversation = [
134 | {'role': 'system', 'content': 'You are a helpful assistant.'},
135 | ]
136 | for query_h, response_h in history:
137 | conversation.append({'role': 'user', 'content': query_h})
138 | conversation.append({'role': 'assistant', 'content': response_h})
139 | conversation.append({'role': 'user', 'content': query})
140 | inputs = tokenizer.apply_chat_template(
141 | conversation,
142 | add_generation_prompt=True,
143 | return_tensors='pt',
144 | )
145 | inputs = inputs.to(model.device)
146 | streamer = TextIteratorStreamer(tokenizer=tokenizer, skip_prompt=True, timeout=60.0, skip_special_tokens=True)
147 | generation_kwargs = dict(
148 | input_ids=inputs,
149 | streamer=streamer,
150 | )
151 | thread = Thread(target=model.generate, kwargs=generation_kwargs)
152 | thread.start()
153 |
154 | for new_text in streamer:
155 | yield new_text
156 |
157 |
158 | def main():
159 | parser = argparse.ArgumentParser(
160 | description='Qwen-WisdomVast command-line interactive chat demo.')
161 | parser.add_argument("--model_path", type=str, help="Checkpoint name or path")
162 | parser.add_argument("-s", "--seed", type=int, default=1234, help="Random seed")
163 | parser.add_argument("--cpu-only", action="store_true", help="Run demo with CPU only")
164 | args = parser.parse_args()
165 |
166 | history, response = [], ''
167 |
168 | model, tokenizer = _load_model_tokenizer(args)
169 | orig_gen_config = deepcopy(model.generation_config)
170 |
171 | _setup_readline()
172 |
173 | _clear_screen()
174 | print(_WELCOME_MSG)
175 |
176 | seed = args.seed
177 |
178 | while True:
179 | query = _get_input()
180 |
181 | # Process commands.
182 | if query.startswith(':'):
183 | command_words = query[1:].strip().split()
184 | if not command_words:
185 | command = ''
186 | else:
187 | command = command_words[0]
188 |
189 | if command in ['exit', 'quit', 'q']:
190 | break
191 | elif command in ['clear', 'cl']:
192 | _clear_screen()
193 | print(_WELCOME_MSG)
194 | _gc()
195 | continue
196 | elif command in ['clear-history', 'clh']:
197 | print(f'[INFO] All {len(history)} history cleared')
198 | history.clear()
199 | _gc()
200 | continue
201 | elif command in ['help', 'h']:
202 | print(_HELP_MSG)
203 | continue
204 | elif command in ['history', 'his']:
205 | _print_history(history)
206 | continue
207 | elif command in ['seed']:
208 | if len(command_words) == 1:
209 | print(f'[INFO] Current random seed: {seed}')
210 | continue
211 | else:
212 | new_seed_s = command_words[1]
213 | try:
214 | new_seed = int(new_seed_s)
215 | except ValueError:
216 | print(f'[WARNING] Fail to change random seed: {new_seed_s!r} is not a valid number')
217 | else:
218 | print(f'[INFO] Random seed changed to {new_seed}')
219 | seed = new_seed
220 | continue
221 | elif command in ['conf']:
222 | if len(command_words) == 1:
223 | print(model.generation_config)
224 | else:
225 | for key_value_pairs_str in command_words[1:]:
226 | eq_idx = key_value_pairs_str.find('=')
227 | if eq_idx == -1:
228 | print('[WARNING] format: =')
229 | continue
230 | conf_key, conf_value_str = key_value_pairs_str[:eq_idx], key_value_pairs_str[eq_idx + 1:]
231 | try:
232 | conf_value = eval(conf_value_str)
233 | except Exception as e:
234 | print(e)
235 | continue
236 | else:
237 | print(f'[INFO] Change config: model.generation_config.{conf_key} = {conf_value}')
238 | setattr(model.generation_config, conf_key, conf_value)
239 | continue
240 | elif command in ['reset-conf']:
241 | print('[INFO] Reset generation config')
242 | model.generation_config = deepcopy(orig_gen_config)
243 | print(model.generation_config)
244 | continue
245 | else:
246 | # As normal query.
247 | pass
248 |
249 | # Run chat.
250 | set_seed(seed)
251 | print(f"\nQwen-WisdomVast: ", end="")
252 | try:
253 | partial_text = ''
254 | for new_text in _chat_stream(model, tokenizer, query, history):
255 | print(new_text, end='', flush=True)
256 | partial_text += new_text
257 | response = partial_text
258 | print()
259 |
260 | except KeyboardInterrupt:
261 | print('[WARNING] Generation interrupted')
262 | continue
263 |
264 | history.append((query, response))
265 |
266 |
267 | if __name__ == "__main__":
268 | main()
--------------------------------------------------------------------------------
/eval_qwen_wisdomvast.py:
--------------------------------------------------------------------------------
1 | from mmengine.config import read_base
2 | from opencompass.models import OpenAI
3 | from opencompass.partitioners import NaivePartitioner
4 | from opencompass.runners import LocalRunner
5 | from opencompass.tasks import OpenICLInferTask
6 |
7 | with read_base():
8 | # choose a list of datasets
9 | from .datasets.ceval.ceval_gen import ceval_datasets
10 | from .datasets.cmmlu.cmmlu_gen import cmmlu_datasets
11 | from .datasets.mmlu.mmlu_gen import mmlu_datasets
12 | from .datasets.gsm8k.gsm8k_gen import gsm8k_datasets
13 | from .datasets.bbh.bbh_gen import bbh_datasets
14 | from .datasets.math.math_gen import math_datasets
15 | from .datasets.mbpp.mbpp_gen import mbpp_datasets
16 | from .datasets.humaneval.humaneval_gen import humaneval_datasets
17 |
18 | # and output the results in a choosen format
19 | from .summarizers.medium import summarizer
20 |
21 |
22 | datasets = [*ceval_datasets, *cmmlu_datasets, *mmlu_datasets,
23 | *gsm8k_datasets, *bbh_datasets, *math_datasets,
24 | *humaneval_datasets, *mbpp_datasets]
25 |
26 |
27 | api_meta_template = dict(
28 | round=[
29 | dict(role='HUMAN', api_role='HUMAN'),
30 | dict(role='BOT', api_role='BOT', generate=True),
31 | ],
32 | )
33 |
34 | models = [
35 | dict(abbr='Qwen-WisdomVast',
36 | type=OpenAI, path='Qwen-WisdomVast',
37 | key='EMPTY', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
38 | meta_template=api_meta_template,
39 | query_per_second=10,
40 | max_out_len=2048, max_seq_len=4096, batch_size=8),
41 | ]
42 |
43 | infer = dict(
44 | partitioner=dict(type=NaivePartitioner),
45 | runner=dict(
46 | type=LocalRunner,
47 | max_num_workers=8,
48 | task=dict(type=OpenICLInferTask)),
49 | )
50 |
--------------------------------------------------------------------------------
/images/image.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/seanzhang-zhichen/Qwen-WisdomVast/0da272eb24352cdd87c9899c324c1fd8972c79a8/images/image.png
--------------------------------------------------------------------------------
/images/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/seanzhang-zhichen/Qwen-WisdomVast/0da272eb24352cdd87c9899c324c1fd8972c79a8/images/logo.png
--------------------------------------------------------------------------------
/merge_lora.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | import torch
4 | import argparse
5 | from peft import PeftModel
6 | from transformers import AutoModelForCausalLM, AutoTokenizer
7 |
8 |
9 | def main():
10 | parser = argparse.ArgumentParser()
11 | parser.add_argument('--base_model', default=None, required=True, type=str,
12 | help="Base model name or path")
13 | parser.add_argument('--lora_model', default=None, required=True, type=str,
14 | help="Please specify LoRA model to be merged.")
15 | parser.add_argument('--output_dir', default='./merged', type=str)
16 | args = parser.parse_args()
17 |
18 | base_model_path = args.base_model
19 | lora_model_path = args.lora_model
20 | output_dir = args.output_dir
21 | print(f"Base model: {base_model_path}")
22 | print(f"LoRA model: {lora_model_path}")
23 |
24 | print("Loading LoRA for causal language model")
25 | base_model = AutoModelForCausalLM.from_pretrained(
26 | base_model_path,
27 | torch_dtype=torch.bfloat16,
28 | device_map="auto",
29 | )
30 |
31 | tokenizer = AutoTokenizer.from_pretrained(base_model_path)
32 |
33 |
34 | new_model = PeftModel.from_pretrained(
35 | base_model,
36 | lora_model_path,
37 | device_map="auto",
38 | torch_dtype=torch.bfloat16,
39 | safe_serialization=False
40 | )
41 |
42 | print(f"Merging with merge_and_unload...")
43 | base_model = new_model.merge_and_unload()
44 |
45 | print("Saving to Hugging Face format...")
46 | tokenizer.save_pretrained(output_dir)
47 | base_model.save_pretrained(output_dir, safe_serialization=False) # max_shard_size='10GB'
48 | print(f"Done! model saved to {output_dir}")
49 |
50 |
51 | if __name__ == '__main__':
52 | main()
53 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | transformers==4.39.3
2 | vllm==0.4.0.post1
3 | torch==2.1.2
--------------------------------------------------------------------------------
/vllm_web_demo.py:
--------------------------------------------------------------------------------
1 | import argparse
2 |
3 | import gradio as gr
4 | from openai import OpenAI
5 |
6 | # Argument parser setup
7 | parser = argparse.ArgumentParser(
8 | description='Chatbot Interface with Customizable Parameters')
9 | parser.add_argument('--model-url',
10 | type=str,
11 | default='http://localhost:8000/v1',
12 | help='Model URL')
13 | parser.add_argument('-m',
14 | '--model',
15 | type=str,
16 | required=True,
17 | help='Model name for the chatbot')
18 | parser.add_argument('--temp',
19 | type=float,
20 | default=0.8,
21 | help='Temperature for text generation')
22 | parser.add_argument('--stop-token-ids',
23 | type=str,
24 | default='',
25 | help='Comma-separated stop token IDs')
26 | parser.add_argument("--host", type=str, default=None)
27 | parser.add_argument("--port", type=int, default=8001)
28 |
29 | # Parse the arguments
30 | args = parser.parse_args()
31 |
32 | # Set OpenAI's API key and API base to use vLLM's API server.
33 | openai_api_key = "EMPTY"
34 | openai_api_base = args.model_url
35 |
36 | # Create an OpenAI client to interact with the API server
37 | client = OpenAI(
38 | api_key=openai_api_key,
39 | base_url=openai_api_base,
40 | )
41 |
42 |
43 | def predict(message, history):
44 | # Convert chat history to OpenAI format
45 | history_openai_format = [{
46 | "role": "system",
47 | "content": "You are a great ai assistant."
48 | }]
49 | for human, assistant in history:
50 | history_openai_format.append({"role": "user", "content": human})
51 | history_openai_format.append({
52 | "role": "assistant",
53 | "content": assistant
54 | })
55 | history_openai_format.append({"role": "user", "content": message})
56 |
57 | # Create a chat completion request and send it to the API server
58 | stream = client.chat.completions.create(
59 | model=args.model, # Model name to use
60 | messages=history_openai_format, # Chat history
61 | temperature=args.temp, # Temperature for text generation
62 | stream=True, # Stream response
63 | extra_body={
64 | 'repetition_penalty':
65 | 1,
66 | 'stop_token_ids': [
67 | int(id.strip()) for id in args.stop_token_ids.split(',')
68 | if id.strip()
69 | ] if args.stop_token_ids else []
70 | })
71 |
72 | # Read and return generated text from response stream
73 | partial_message = ""
74 | for chunk in stream:
75 | partial_message += (chunk.choices[0].delta.content or "")
76 | yield partial_message
77 |
78 | # Create and launch a chat interface with Gradio
79 | gr.ChatInterface(predict).queue().launch(server_name=args.host,
80 | server_port=args.port,
81 | share=True)
--------------------------------------------------------------------------------
/web_demo.py:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | import torch
5 | import gradio as gr
6 | from threading import Thread
7 | from argparse import ArgumentParser
8 | from transformers import GenerationConfig
9 | from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
10 |
11 |
12 | def _get_args():
13 | parser = ArgumentParser()
14 | parser.add_argument("--model_path", type=str, help="Checkpoint name or path")
15 | parser.add_argument("--cpu-only", action="store_true", help="Run demo with CPU only")
16 | parser.add_argument("--share", action="store_true", default=False,
17 | help="Create a publicly shareable link for the interface.")
18 | parser.add_argument("--inbrowser", action="store_true", default=False,
19 | help="Automatically launch the interface in a new tab on the default browser.")
20 | parser.add_argument("--server-port", type=int, default=8000,
21 | help="Demo server port.")
22 | parser.add_argument("--server-name", type=str, default="127.0.0.1",
23 | help="Demo server name.")
24 |
25 | args = parser.parse_args()
26 | return args
27 |
28 |
29 | def _load_model_tokenizer(args):
30 | tokenizer = AutoTokenizer.from_pretrained(
31 | args.model_path, resume_download=True,
32 | )
33 |
34 | if args.cpu_only:
35 | device_map = "cpu"
36 | else:
37 | device_map = "auto"
38 |
39 | model = AutoModelForCausalLM.from_pretrained(
40 | args.model_path,
41 | device_map=device_map,
42 | resume_download=True,
43 | ).eval()
44 |
45 | model.generation_config = GenerationConfig(
46 | bos_token_id = 151643,
47 | do_sample = True,
48 | eos_token_id = [
49 | 151645,
50 | 151643
51 | ],
52 | max_new_tokens = 2048,
53 | repetition_penalty = 1.05,
54 | temperature = 0.7,
55 | top_p = 0.8,
56 | top_k = 20,
57 | )
58 |
59 | return model, tokenizer
60 |
61 |
62 | def _chat_stream(model, tokenizer, query, history):
63 | conversation = [
64 | {'role': 'system', 'content': 'You are a helpful assistant.'},
65 | ]
66 | for query_h, response_h in history:
67 | conversation.append({'role': 'user', 'content': query_h})
68 | conversation.append({'role': 'assistant', 'content': response_h})
69 | conversation.append({'role': 'user', 'content': query})
70 | inputs = tokenizer.apply_chat_template(
71 | conversation,
72 | add_generation_prompt=True,
73 | return_tensors='pt',
74 | )
75 | inputs = inputs.to(model.device)
76 | streamer = TextIteratorStreamer(tokenizer=tokenizer, skip_prompt=True, timeout=60.0, skip_special_tokens=True)
77 | generation_kwargs = dict(
78 | input_ids=inputs,
79 | streamer=streamer,
80 | )
81 | thread = Thread(target=model.generate, kwargs=generation_kwargs)
82 | thread.start()
83 |
84 | for new_text in streamer:
85 | yield new_text
86 |
87 |
88 | def _gc():
89 | import gc
90 | gc.collect()
91 | if torch.cuda.is_available():
92 | torch.cuda.empty_cache()
93 |
94 |
95 | def _launch_demo(args, model, tokenizer):
96 |
97 | def predict(_query, _chatbot, _task_history):
98 | print(f"User: {_query}")
99 | _chatbot.append((_query, ""))
100 | full_response = ""
101 | response = ""
102 | for new_text in _chat_stream(model, tokenizer, _query, history=_task_history):
103 | response += new_text
104 | _chatbot[-1] = (_query, response)
105 |
106 | yield _chatbot
107 | full_response = response
108 |
109 | print(f"History: {_task_history}")
110 | _task_history.append((_query, full_response))
111 | print(f"Qwen1.5-Chat: {full_response}")
112 |
113 | def regenerate(_chatbot, _task_history):
114 | if not _task_history:
115 | yield _chatbot
116 | return
117 | item = _task_history.pop(-1)
118 | _chatbot.pop(-1)
119 | yield from predict(item[0], _chatbot, _task_history)
120 |
121 | def reset_user_input():
122 | return gr.update(value="")
123 |
124 | def reset_state(_chatbot, _task_history):
125 | _task_history.clear()
126 | _chatbot.clear()
127 | _gc()
128 | return _chatbot
129 |
130 | with gr.Blocks() as demo:
131 | gr.Markdown("""Qwen-WisdomVast""")
132 | gr.Markdown("""\
133 |
134 | Qwen-WisdomVast 🤖 ModelScope |
135 | 🤗 HuggingFace ｜
136 | Github""")
137 |
138 | chatbot = gr.Chatbot(label='Qwen-WisdomVast', elem_classes="control-height")
139 | query = gr.Textbox(lines=2, label='Input')
140 | task_history = gr.State([])
141 |
142 | with gr.Row():
143 | empty_btn = gr.Button("🧹 Clear History (清除历史)")
144 | submit_btn = gr.Button("🚀 Submit (发送)")
145 | regen_btn = gr.Button("🤔️ Regenerate (重试)")
146 |
147 | submit_btn.click(predict, [query, chatbot, task_history], [chatbot], show_progress=True)
148 | submit_btn.click(reset_user_input, [], [query])
149 | empty_btn.click(reset_state, [chatbot, task_history], outputs=[chatbot], show_progress=True)
150 | regen_btn.click(regenerate, [chatbot, task_history], [chatbot], show_progress=True)
151 |
152 | gr.Markdown("""\
153 | Note: This demo is governed by the original license of Qwen-WisdomVast. \
154 | We strongly advise users not to knowingly generate or allow others to knowingly generate harmful content, \
155 | including hate speech, violence, pornography, deception, etc. \
156 |
157 | (注：本演示受Qwen-WisdomVast的许可协议限制。我们强烈建议，用户不应传播及不应允许他人传播以下内容，\
158 | 包括但不限于仇恨言论、暴力、色情、欺诈相关的有害信息。)""")
159 |
160 | demo.queue().launch(
161 | share=args.share,
162 | inbrowser=args.inbrowser,
163 | server_port=args.server_port,
164 | server_name=args.server_name,
165 | )
166 |
167 |
168 | def main():
169 | args = _get_args()
170 |
171 | model, tokenizer = _load_model_tokenizer(args)
172 |
173 | _launch_demo(args, model, tokenizer)
174 |
175 |
176 | if __name__ == '__main__':
177 | main()
--------------------------------------------------------------------------------
29 |
30 |