├── .editorconfig ├── .flake8 ├── .gitattributes ├── .gitignore ├── .pre-commit-config.yaml ├── .pylintrc ├── LICENSE-CODE ├── LICENSE-MODEL ├── Makefile ├── README.md ├── evaluation ├── IFEval │ ├── baichuan13B.jsonl │ ├── chatglm.jsonl │ ├── deepseek67B.jsonl │ ├── qwen14B.jsonl │ └── yi34B.jsonl ├── deepseek-67b-1206-no-sp.jsonl ├── hungarian_national_hs_solutions │ ├── exam_Baichuan2-13B-Chat.csv │ ├── exam_ChatGLM3-6B.csv │ ├── exam_DeepSeek-66B.csv │ ├── exam_Qwen-14B-Chat.csv │ └── exam_Yi-34B-Chat.csv └── more_results.md ├── images ├── badge.svg ├── home.png ├── if_eval.png ├── leetcode.png ├── llm_radar.png ├── logo.png ├── logo.svg ├── mathexam.png ├── pretrain_loss.png ├── pretrain_metric.png └── qr.jpeg └── requirements.txt /.editorconfig: -------------------------------------------------------------------------------- 1 | # https://editorconfig.org/ 2 | 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | end_of_line = lf 8 | indent_style = space 9 | indent_size = 4 10 | trim_trailing_whitespace = true 11 | insert_final_newline = true 12 | 13 | [*.py] 14 | indent_size = 4 15 | src_paths=evaluation 16 | 17 | [*.{yaml,yml,json}] 18 | indent_size = 2 19 | 20 | [*.md] 21 | indent_size = 2 22 | x-soft-wrap-text = true 23 | 24 | [*.rst] 25 | indent_size = 4 26 | x-soft-wrap-text = true 27 | 28 | [*.{bib,tex}] 29 | indent_size = 2 30 | 31 | [Makefile] 32 | indent_style = tab 33 | 34 | [*.sh] 35 | indent_style = tab 36 | 37 | [*.bat] 38 | end_of_line = crlf 39 | indent_style = tab 40 | 41 | [*.{cpp,h,cu,cuh}] 42 | indent_size = 2 43 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | max-doc-length = 100 4 | select = B,C,E,F,W,Y,SIM 5 | ignore = 6 | # E203: whitespace before ':' 7 | # W503: line break before binary operator 8 | # W504: line break after binary operator 9 | # format by black 10 | E203,W503,W504, 11 | # E501: line too long 12 | # W505: doc line too long 13 | # too long docstring due to long example blocks 14 | E501,W505, 15 | per-file-ignores = 16 | # F401: module imported but unused 17 | # intentionally unused imports 18 | __init__.py: F401 19 | # F401: module imported but unused 20 | # F403: unable to detect undefined names 21 | # F405: member mey be undefined, or defined from star imports 22 | # members populated from optree 23 | # E301: expected 1 blank line 24 | # E302: expected 2 blank lines 25 | # E305: expected 2 blank lines after class or function definition 26 | # E701: multiple statements on one line (colon) 27 | # E704: multiple statements on one line (def) 28 | # format by black 29 | *.pyi: E301,E302,E305,E701,E704 30 | exclude = 31 | .git, 32 | .vscode, 33 | venv, 34 | third-party, 35 | __pycache__, 36 | docs/source/conf.py, 37 | build, 38 | dist, 39 | examples, 40 | tests 41 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * text eol=lf 2 | *.ipynb linguist-detectable=false 3 | 4 | *.png binary 5 | *.jpg binary 6 | *.jpeg binary 7 | *.gif binary 8 | *.pdf binary 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ##### Python.gitignore ##### 2 | # Byte-compiled / optimized / DLL files 3 | __pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | wheelhouse/ 25 | share/python-wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | MANIFEST 30 | *.whl 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .nox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *.cover 52 | *.py,cover 53 | .hypothesis/ 54 | .pytest_cache/ 55 | cover/ 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | local_settings.py 64 | db.sqlite3 65 | db.sqlite3-journal 66 | 67 | # Flask stuff: 68 | instance/ 69 | .webassets-cache 70 | 71 | # Scrapy stuff: 72 | .scrapy 73 | 74 | # Sphinx documentation 75 | docs/_build/ 76 | docs/source/_build/ 77 | _autosummary/ 78 | 79 | # PyBuilder 80 | .pybuilder/ 81 | target/ 82 | 83 | # Jupyter Notebook 84 | .ipynb_checkpoints 85 | 86 | # IPython 87 | profile_default/ 88 | ipython_config.py 89 | 90 | # pyenv 91 | # For a library or package, you might want to ignore these files since the code is 92 | # intended to run in multiple environments; otherwise, check them in: 93 | .python-version 94 | 95 | # pipenv 96 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 97 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 98 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 99 | # install all needed dependencies. 100 | #Pipfile.lock 101 | 102 | # poetry 103 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 104 | # This is especially recommended for binary packages to ensure reproducibility, and is more 105 | # commonly ignored for libraries. 106 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 107 | #poetry.lock 108 | 109 | # pdm 110 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 111 | #pdm.lock 112 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 113 | # in version control. 114 | # https://pdm.fming.dev/#use-with-ide 115 | .pdm.toml 116 | 117 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 118 | __pypackages__/ 119 | 120 | # Celery stuff 121 | celerybeat-schedule 122 | celerybeat.pid 123 | 124 | # SageMath parsed files 125 | *.sage.py 126 | 127 | # Environments 128 | .env 129 | .venv 130 | env/ 131 | venv/ 132 | ENV/ 133 | env.bak/ 134 | venv.bak/ 135 | 136 | # Spyder project settings 137 | .spyderproject 138 | .spyproject 139 | 140 | # Rope project settings 141 | .ropeproject 142 | 143 | # mkdocs documentation 144 | /site 145 | 146 | # ruff 147 | .ruff_cache/ 148 | 149 | # mypy 150 | .mypy_cache/ 151 | .dmypy.json 152 | dmypy.json 153 | 154 | # Pyre type checker 155 | .pyre/ 156 | 157 | # pytype static type analyzer 158 | .pytype/ 159 | 160 | # Cython debug symbols 161 | cython_debug/ 162 | 163 | # PyCharm 164 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 165 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 166 | # and can be added to the global gitignore or merged into this file. For a more nuclear 167 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 168 | .idea/ 169 | 170 | 171 | ##### macOS.gitignore ##### 172 | # General 173 | .DS_Store 174 | .AppleDouble 175 | .LSOverride 176 | 177 | # Icon must end with two \r 178 | Icon 179 | 180 | # Thumbnails 181 | ._* 182 | 183 | # Files that might appear in the root of a volume 184 | .DocumentRevisions-V100 185 | .fseventsd 186 | .Spotlight-V100 187 | .TemporaryItems 188 | .Trashes 189 | .VolumeIcon.icns 190 | .com.apple.timemachine.donotpresent 191 | 192 | # Directories potentially created on remote AFP share 193 | .AppleDB 194 | .AppleDesktop 195 | Network Trash Folder 196 | Temporary Items 197 | .apdisk 198 | 199 | 200 | ##### Linux.gitignore ##### 201 | *~ 202 | 203 | # Temporary files which can be created if a process still has a handle open of a deleted file 204 | .fuse_hidden* 205 | 206 | # KDE directory preferences 207 | .directory 208 | 209 | # Linux trash folder which might appear on any partition or disk 210 | .Trash-* 211 | 212 | # .nfs files are created when an open file is removed but is still being accessed 213 | .nfs* 214 | 215 | 216 | ##### Windows.gitignore ##### 217 | # Windows thumbnail cache files 218 | Thumbs.db 219 | Thumbs.db:encryptable 220 | ehthumbs.db 221 | ehthumbs_vista.db 222 | 223 | # Dump file 224 | *.stackdump 225 | 226 | # Folder config file 227 | [Dd]esktop.ini 228 | 229 | # Recycle Bin used on file shares 230 | $RECYCLE.BIN/ 231 | 232 | # Windows Installer files 233 | *.cab 234 | *.msi 235 | *.msix 236 | *.msm 237 | *.msp 238 | 239 | # Windows shortcuts 240 | *.lnk 241 | 242 | 243 | ##### Archives.gitignore ##### 244 | # It's better to unpack these files and commit the raw source because 245 | # git has its own built in compression methods. 246 | *.7z 247 | *.jar 248 | *.rar 249 | *.zip 250 | *.gz 251 | *.gzip 252 | *.tgz 253 | *.bzip 254 | *.bzip2 255 | *.bz2 256 | *.xz 257 | *.lzma 258 | *.cab 259 | *.xar 260 | 261 | # Packing-only formats 262 | *.iso 263 | *.tar 264 | 265 | # Package management formats 266 | *.dmg 267 | *.xpi 268 | *.gem 269 | *.egg 270 | *.deb 271 | *.rpm 272 | *.msi 273 | *.msm 274 | *.msp 275 | *.txz 276 | 277 | 278 | ##### Xcode.gitignore ##### 279 | # Xcode 280 | # 281 | # gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore 282 | 283 | ## User settings 284 | xcuserdata/ 285 | 286 | ## Compatibility with Xcode 8 and earlier (ignoring not required starting Xcode 9) 287 | *.xcscmblueprint 288 | *.xccheckout 289 | 290 | ## Compatibility with Xcode 3 and earlier (ignoring not required starting Xcode 4) 291 | build/ 292 | DerivedData/ 293 | *.moved-aside 294 | *.pbxuser 295 | !default.pbxuser 296 | *.mode1v3 297 | !default.mode1v3 298 | *.mode2v3 299 | !default.mode2v3 300 | *.perspectivev3 301 | !default.perspectivev3 302 | 303 | ## Gcc Patch 304 | /*.gcno 305 | 306 | 307 | ##### JetBrains.gitignore ##### 308 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 309 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 310 | 311 | # User settings 312 | .idea/* 313 | 314 | # User-specific stuff 315 | .idea/**/workspace.xml 316 | .idea/**/tasks.xml 317 | .idea/**/usage.statistics.xml 318 | .idea/**/dictionaries 319 | .idea/**/shelf 320 | 321 | # Generated files 322 | .idea/**/contentModel.xml 323 | 324 | # Sensitive or high-churn files 325 | .idea/**/dataSources/ 326 | .idea/**/dataSources.ids 327 | .idea/**/dataSources.local.xml 328 | .idea/**/sqlDataSources.xml 329 | .idea/**/dynamic.xml 330 | .idea/**/uiDesigner.xml 331 | .idea/**/dbnavigator.xml 332 | 333 | # Gradle 334 | .idea/**/gradle.xml 335 | .idea/**/libraries 336 | 337 | # Gradle and Maven with auto-import 338 | # When using Gradle or Maven with auto-import, you should exclude module files, 339 | # since they will be recreated, and may cause churn. Uncomment if using 340 | # auto-import. 341 | # .idea/artifacts 342 | # .idea/compiler.xml 343 | # .idea/jarRepositories.xml 344 | # .idea/modules.xml 345 | # .idea/*.iml 346 | # .idea/modules 347 | # *.iml 348 | # *.ipr 349 | 350 | # CMake 351 | cmake-build-*/ 352 | 353 | # Mongo Explorer plugin 354 | .idea/**/mongoSettings.xml 355 | 356 | # File-based project format 357 | *.iws 358 | 359 | # IntelliJ 360 | out/ 361 | 362 | # mpeltonen/sbt-idea plugin 363 | .idea_modules/ 364 | 365 | # JIRA plugin 366 | atlassian-ide-plugin.xml 367 | 368 | # Cursive Clojure plugin 369 | .idea/replstate.xml 370 | 371 | # Crashlytics plugin (for Android Studio and IntelliJ) 372 | com_crashlytics_export_strings.xml 373 | crashlytics.properties 374 | crashlytics-build.properties 375 | fabric.properties 376 | 377 | # Editor-based Rest Client 378 | .idea/httpRequests 379 | 380 | # Android studio 3.1+ serialized cache file 381 | .idea/caches/build_file_checksums.ser 382 | 383 | 384 | ##### VisualStudioCode.gitignore ##### 385 | .vscode/* 386 | # !.vscode/settings.json 387 | # !.vscode/tasks.json 388 | # !.vscode/launch.json 389 | !.vscode/extensions.json 390 | *.code-workspace 391 | 392 | # Local History for Visual Studio Code 393 | .history/ 394 | 395 | 396 | ##### Vim.gitignore ##### 397 | # Swap 398 | .*.s[a-v][a-z] 399 | !*.svg # comment out if you don't need vector files 400 | .*.sw[a-p] 401 | .s[a-rt-v][a-z] 402 | .ss[a-gi-z] 403 | .sw[a-p] 404 | 405 | # Session 406 | Session.vim 407 | Sessionx.vim 408 | 409 | # Temporary 410 | .netrwhist 411 | *~ 412 | # Auto-generated tag files 413 | tags 414 | # Persistent undo 415 | [._]*.un~ 416 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | ci: 4 | skip: [pylint] 5 | autofix_prs: true 6 | autofix_commit_msg: "fix: [pre-commit.ci] auto fixes [...]" 7 | autoupdate_commit_msg: "chore(pre-commit): [pre-commit.ci] autoupdate" 8 | autoupdate_schedule: monthly 9 | default_stages: [commit, push, manual] 10 | repos: 11 | - repo: https://github.com/pre-commit/pre-commit-hooks 12 | rev: v4.5.0 13 | hooks: 14 | - id: check-symlinks 15 | - id: destroyed-symlinks 16 | - id: trailing-whitespace 17 | - id: end-of-file-fixer 18 | - id: check-yaml 19 | - id: check-toml 20 | - id: check-ast 21 | - id: check-added-large-files 22 | - id: check-merge-conflict 23 | - id: check-executables-have-shebangs 24 | - id: check-shebang-scripts-are-executable 25 | - id: detect-private-key 26 | - id: debug-statements 27 | - id: double-quote-string-fixer 28 | - repo: https://github.com/astral-sh/ruff-pre-commit 29 | rev: v0.1.5 30 | hooks: 31 | - id: ruff 32 | args: [--fix, --exit-non-zero-on-fix] 33 | - repo: https://github.com/PyCQA/isort 34 | rev: 5.12.0 35 | hooks: 36 | - id: isort 37 | - repo: https://github.com/psf/black 38 | rev: 23.11.0 39 | hooks: 40 | - id: black-jupyter 41 | - repo: https://github.com/asottile/pyupgrade 42 | rev: v3.15.0 43 | hooks: 44 | - id: pyupgrade 45 | args: [--py38-plus] # sync with requires-python 46 | exclude: | 47 | (?x)( 48 | ^images/ 49 | ) 50 | - repo: https://github.com/pycqa/flake8 51 | rev: 6.1.0 52 | hooks: 53 | - id: flake8 54 | additional_dependencies: 55 | - flake8-bugbear 56 | - flake8-comprehensions 57 | - flake8-docstrings 58 | - flake8-pyi 59 | - flake8-simplify 60 | exclude: | 61 | (?x)( 62 | ^images/ 63 | ) 64 | - repo: local 65 | hooks: 66 | - id: pylint 67 | name: pylint 68 | entry: pylint 69 | language: system 70 | types: [python] 71 | require_serial: true 72 | exclude: | 73 | (?x)( 74 | ^images/ 75 | ) 76 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MAIN] 2 | 3 | # Analyse import fallback blocks. This can be used to support both Python 2 and 4 | # 3 compatible code, which means that the block might have code that exists 5 | # only in one or another interpreter, leading to false positives when analysed. 6 | analyse-fallback-blocks=no 7 | 8 | # Load and enable all available extensions. Use --list-extensions to see a list 9 | # all available extensions. 10 | #enable-all-extensions= 11 | 12 | # In error mode, messages with a category besides ERROR or FATAL are 13 | # suppressed, and no reports are done by default. Error mode is compatible with 14 | # disabling specific errors. 15 | #errors-only= 16 | 17 | # Always return a 0 (non-error) status code, even if lint errors are found. 18 | # This is primarily useful in continuous integration scripts. 19 | #exit-zero= 20 | 21 | # A comma-separated list of package or module names from where C extensions may 22 | # be loaded. Extensions are loading into the active Python interpreter and may 23 | # run arbitrary code. 24 | extension-pkg-allow-list= 25 | 26 | # A comma-separated list of package or module names from where C extensions may 27 | # be loaded. Extensions are loading into the active Python interpreter and may 28 | # run arbitrary code. (This is an alternative name to extension-pkg-allow-list 29 | # for backward compatibility.) 30 | extension-pkg-whitelist= 31 | 32 | # Return non-zero exit code if any of these messages/categories are detected, 33 | # even if score is above --fail-under value. Syntax same as enable. Messages 34 | # specified are enabled, while categories only check already-enabled messages. 35 | fail-on= 36 | 37 | # Specify a score threshold under which the program will exit with error. 38 | fail-under=10 39 | 40 | # Interpret the stdin as a python script, whose filename needs to be passed as 41 | # the module_or_package argument. 42 | #from-stdin= 43 | 44 | # Files or directories to be skipped. They should be base names, not paths. 45 | ignore=CVS,.vscode,.history 46 | 47 | # Add files or directories matching the regular expressions patterns to the 48 | # ignore-list. The regex matches against paths and can be in Posix or Windows 49 | # format. Because '\' represents the directory delimiter on Windows systems, it 50 | # can't be used as an escape character. 51 | ignore-paths=^images/$ 52 | 53 | # Files or directories matching the regular expression patterns are skipped. 54 | # The regex matches against base names, not paths. The default value ignores 55 | # Emacs file locks 56 | ignore-patterns=^\.# 57 | 58 | # List of module names for which member attributes should not be checked 59 | # (useful for modules/projects where namespaces are manipulated during runtime 60 | # and thus existing member attributes cannot be deduced by static analysis). It 61 | # supports qualified module names, as well as Unix pattern matching. 62 | ignored-modules= 63 | 64 | # Python code to execute, usually for sys.path manipulation such as 65 | # pygtk.require(). 66 | #init-hook= 67 | 68 | # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the 69 | # number of processors available to use, and will cap the count on Windows to 70 | # avoid hangs. 71 | jobs=0 72 | 73 | # Control the amount of potential inferred values when inferring a single 74 | # object. This can help the performance when dealing with large functions or 75 | # complex, nested conditions. 76 | limit-inference-results=100 77 | 78 | # List of plugins (as comma separated values of python module names) to load, 79 | # usually to register additional checkers. 80 | load-plugins= 81 | 82 | # Pickle collected data for later comparisons. 83 | persistent=yes 84 | 85 | # Minimum Python version to use for version dependent checks. Will default to 86 | # the version used to run pylint. 87 | py-version=3.8 # the lowest version we support (sync with requires-python in pyproject.toml) 88 | 89 | # Discover python modules and packages in the file system subtree. 90 | recursive=no 91 | 92 | # When enabled, pylint would attempt to guess common misconfiguration and emit 93 | # user-friendly hints instead of false-positive error messages. 94 | suggestion-mode=yes 95 | 96 | # Allow loading of arbitrary C extensions. Extensions are imported into the 97 | # active Python interpreter and may run arbitrary code. 98 | unsafe-load-any-extension=no 99 | 100 | # In verbose mode, extra non-checker-related info will be displayed. 101 | #verbose= 102 | 103 | 104 | [BASIC] 105 | 106 | # Naming style matching correct argument names. 107 | argument-naming-style=snake_case 108 | 109 | # Regular expression matching correct argument names. Overrides argument- 110 | # naming-style. If left empty, argument names will be checked with the set 111 | # naming style. 112 | #argument-rgx= 113 | 114 | # Naming style matching correct attribute names. 115 | attr-naming-style=snake_case 116 | 117 | # Regular expression matching correct attribute names. Overrides attr-naming- 118 | # style. If left empty, attribute names will be checked with the set naming 119 | # style. 120 | #attr-rgx= 121 | 122 | # Bad variable names which should always be refused, separated by a comma. 123 | bad-names=foo, 124 | bar, 125 | baz, 126 | toto, 127 | tutu, 128 | tata 129 | 130 | # Bad variable names regexes, separated by a comma. If names match any regex, 131 | # they will always be refused 132 | bad-names-rgxs= 133 | 134 | # Naming style matching correct class attribute names. 135 | class-attribute-naming-style=any 136 | 137 | # Regular expression matching correct class attribute names. Overrides class- 138 | # attribute-naming-style. If left empty, class attribute names will be checked 139 | # with the set naming style. 140 | #class-attribute-rgx= 141 | 142 | # Naming style matching correct class constant names. 143 | class-const-naming-style=UPPER_CASE 144 | 145 | # Regular expression matching correct class constant names. Overrides class- 146 | # const-naming-style. If left empty, class constant names will be checked with 147 | # the set naming style. 148 | #class-const-rgx= 149 | 150 | # Naming style matching correct class names. 151 | class-naming-style=PascalCase 152 | 153 | # Regular expression matching correct class names. Overrides class-naming- 154 | # style. If left empty, class names will be checked with the set naming style. 155 | #class-rgx= 156 | 157 | # Naming style matching correct constant names. 158 | const-naming-style=UPPER_CASE 159 | 160 | # Regular expression matching correct constant names. Overrides const-naming- 161 | # style. If left empty, constant names will be checked with the set naming 162 | # style. 163 | #const-rgx= 164 | 165 | # Minimum line length for functions/classes that require docstrings, shorter 166 | # ones are exempt. 167 | docstring-min-length=-1 168 | 169 | # Naming style matching correct function names. 170 | function-naming-style=snake_case 171 | 172 | # Regular expression matching correct function names. Overrides function- 173 | # naming-style. If left empty, function names will be checked with the set 174 | # naming style. 175 | #function-rgx= 176 | 177 | # Good variable names which should always be accepted, separated by a comma. 178 | good-names=i, 179 | j, 180 | k, 181 | ex, 182 | Run, 183 | _, 184 | op, 185 | fn, 186 | f, 187 | g, 188 | p, 189 | u, 190 | t, 191 | lr, 192 | mu, 193 | nu, 194 | x, 195 | y 196 | 197 | # Good variable names regexes, separated by a comma. If names match any regex, 198 | # they will always be accepted 199 | good-names-rgxs= 200 | 201 | # Include a hint for the correct naming format with invalid-name. 202 | include-naming-hint=no 203 | 204 | # Naming style matching correct inline iteration names. 205 | inlinevar-naming-style=any 206 | 207 | # Regular expression matching correct inline iteration names. Overrides 208 | # inlinevar-naming-style. If left empty, inline iteration names will be checked 209 | # with the set naming style. 210 | #inlinevar-rgx= 211 | 212 | # Naming style matching correct method names. 213 | method-naming-style=snake_case 214 | 215 | # Regular expression matching correct method names. Overrides method-naming- 216 | # style. If left empty, method names will be checked with the set naming style. 217 | #method-rgx= 218 | 219 | # Naming style matching correct module names. 220 | module-naming-style=snake_case 221 | 222 | # Regular expression matching correct module names. Overrides module-naming- 223 | # style. If left empty, module names will be checked with the set naming style. 224 | #module-rgx= 225 | 226 | # Colon-delimited sets of names that determine each other's naming style when 227 | # the name regexes allow several styles. 228 | name-group= 229 | 230 | # Regular expression which should only match function or class names that do 231 | # not require a docstring. 232 | no-docstring-rgx=^_ 233 | 234 | # List of decorators that produce properties, such as abc.abstractproperty. Add 235 | # to this list to register other decorators that produce valid properties. 236 | # These decorators are taken in consideration only for invalid-name. 237 | property-classes=abc.abstractproperty 238 | 239 | # Regular expression matching correct type variable names. If left empty, type 240 | # variable names will be checked with the set naming style. 241 | #typevar-rgx= 242 | 243 | # Naming style matching correct variable names. 244 | variable-naming-style=snake_case 245 | 246 | # Regular expression matching correct variable names. Overrides variable- 247 | # naming-style. If left empty, variable names will be checked with the set 248 | # naming style. 249 | #variable-rgx= 250 | 251 | 252 | [CLASSES] 253 | 254 | # Warn about protected attribute access inside special methods 255 | check-protected-access-in-special-methods=no 256 | 257 | # List of method names used to declare (i.e. assign) instance attributes. 258 | defining-attr-methods=__init__, 259 | __new__, 260 | setUp, 261 | __post_init__ 262 | 263 | # List of member names, which should be excluded from the protected access 264 | # warning. 265 | exclude-protected=_asdict, 266 | _fields, 267 | _replace, 268 | _source, 269 | _make 270 | 271 | # List of valid names for the first argument in a class method. 272 | valid-classmethod-first-arg=cls 273 | 274 | # List of valid names for the first argument in a metaclass class method. 275 | valid-metaclass-classmethod-first-arg=cls 276 | 277 | 278 | [DESIGN] 279 | 280 | # List of regular expressions of class ancestor names to ignore when counting 281 | # public methods (see R0903) 282 | exclude-too-few-public-methods= 283 | 284 | # List of qualified class names to ignore when counting class parents (see 285 | # R0901) 286 | ignored-parents= 287 | 288 | # Maximum number of arguments for function / method. 289 | max-args=5 290 | 291 | # Maximum number of attributes for a class (see R0902). 292 | max-attributes=7 293 | 294 | # Maximum number of boolean expressions in an if statement (see R0916). 295 | max-bool-expr=5 296 | 297 | # Maximum number of branch for function / method body. 298 | max-branches=12 299 | 300 | # Maximum number of locals for function / method body. 301 | max-locals=15 302 | 303 | # Maximum number of parents for a class (see R0901). 304 | max-parents=7 305 | 306 | # Maximum number of public methods for a class (see R0904). 307 | max-public-methods=20 308 | 309 | # Maximum number of return / yield for function / method body. 310 | max-returns=6 311 | 312 | # Maximum number of statements in function / method body. 313 | max-statements=50 314 | 315 | # Minimum number of public methods for a class (see R0903). 316 | min-public-methods=2 317 | 318 | 319 | [EXCEPTIONS] 320 | 321 | # Exceptions that will emit a warning when caught. 322 | overgeneral-exceptions=builtins.BaseException, 323 | builtins.Exception 324 | 325 | 326 | [FORMAT] 327 | 328 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 329 | expected-line-ending-format= 330 | 331 | # Regexp for a line that is allowed to be longer than the limit. 332 | ignore-long-lines=^\s*(# )??$ 333 | 334 | # Number of spaces of indent required inside a hanging or continued line. 335 | indent-after-paren=4 336 | 337 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 338 | # tab). 339 | indent-string=' ' 340 | 341 | # Maximum number of characters on a single line. 342 | max-line-length=120 343 | 344 | # Maximum number of lines in a module. 345 | max-module-lines=1000 346 | 347 | # Allow the body of a class to be on the same line as the declaration if body 348 | # contains single statement. 349 | single-line-class-stmt=no 350 | 351 | # Allow the body of an if to be on the same line as the test if there is no 352 | # else. 353 | single-line-if-stmt=no 354 | 355 | 356 | [IMPORTS] 357 | 358 | # List of modules that can be imported at any level, not just the top level 359 | # one. 360 | allow-any-import-level= 361 | 362 | # Allow wildcard imports from modules that define __all__. 363 | allow-wildcard-with-all=no 364 | 365 | # Deprecated modules which should not be used, separated by a comma. 366 | deprecated-modules= 367 | 368 | # Output a graph (.gv or any supported image format) of external dependencies 369 | # to the given file (report RP0402 must not be disabled). 370 | ext-import-graph= 371 | 372 | # Output a graph (.gv or any supported image format) of all (i.e. internal and 373 | # external) dependencies to the given file (report RP0402 must not be 374 | # disabled). 375 | import-graph= 376 | 377 | # Output a graph (.gv or any supported image format) of internal dependencies 378 | # to the given file (report RP0402 must not be disabled). 379 | int-import-graph= 380 | 381 | # Force import order to recognize a module as part of the standard 382 | # compatibility libraries. 383 | known-standard-library= 384 | 385 | # Force import order to recognize a module as part of a third party library. 386 | known-third-party=enchant 387 | 388 | # Couples of modules and preferred modules, separated by a comma. 389 | preferred-modules= 390 | 391 | 392 | [LOGGING] 393 | 394 | # The type of string formatting that logging methods do. `old` means using % 395 | # formatting, `new` is for `{}` formatting. 396 | logging-format-style=old 397 | 398 | # Logging modules to check that the string format arguments are in logging 399 | # function parameter format. 400 | logging-modules=logging 401 | 402 | 403 | [MESSAGES CONTROL] 404 | 405 | # Only show warnings with the listed confidence levels. Leave empty to show 406 | # all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, 407 | # UNDEFINED. 408 | confidence=HIGH, 409 | CONTROL_FLOW, 410 | INFERENCE, 411 | INFERENCE_FAILURE, 412 | UNDEFINED 413 | 414 | # Disable the message, report, category or checker with the given id(s). You 415 | # can either give multiple identifiers separated by comma (,) or put this 416 | # option multiple times (only on the command line, not in the configuration 417 | # file where it should appear only once). You can also use "--disable=all" to 418 | # disable everything first and then re-enable specific checks. For example, if 419 | # you want to run only the similarities checker, you can use "--disable=all 420 | # --enable=similarities". If you want to run only the classes checker, but have 421 | # no Warning level messages displayed, use "--disable=all --enable=classes 422 | # --disable=W". 423 | disable=duplicate-code, 424 | consider-using-from-import 425 | 426 | # Enable the message, report, category or checker with the given id(s). You can 427 | # either give multiple identifier separated by comma (,) or put this option 428 | # multiple time (only on the command line, not in the configuration file where 429 | # it should appear only once). See also the "--disable" option for examples. 430 | enable=c-extension-no-member 431 | 432 | 433 | [METHOD_ARGS] 434 | 435 | # List of qualified names (i.e., library.method) which require a timeout 436 | # parameter e.g. 'requests.api.get,requests.api.post' 437 | timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request 438 | 439 | 440 | [MISCELLANEOUS] 441 | 442 | # List of note tags to take in consideration, separated by a comma. 443 | notes=FIXME, 444 | XXX, 445 | TODO 446 | 447 | # Regular expression of note tags to take in consideration. 448 | notes-rgx= 449 | 450 | 451 | [REFACTORING] 452 | 453 | # Maximum number of nested blocks for function / method body 454 | max-nested-blocks=5 455 | 456 | # Complete name of functions that never returns. When checking for 457 | # inconsistent-return-statements if a never returning function is called then 458 | # it will be considered as an explicit return statement and no message will be 459 | # printed. 460 | never-returning-functions=sys.exit,argparse.parse_error 461 | 462 | 463 | [REPORTS] 464 | 465 | # Python expression which should return a score less than or equal to 10. You 466 | # have access to the variables 'fatal', 'error', 'warning', 'refactor', 467 | # 'convention', and 'info' which contain the number of messages in each 468 | # category, as well as 'statement' which is the total number of statements 469 | # analyzed. This score is used by the global evaluation report (RP0004). 470 | evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) 471 | 472 | # Template used to display messages. This is a python new-style format string 473 | # used to format the message information. See doc for all details. 474 | msg-template= 475 | 476 | # Set the output format. Available formats are text, parseable, colorized, json 477 | # and msvs (visual studio). You can also give a reporter class, e.g. 478 | # mypackage.mymodule.MyReporterClass. 479 | #output-format= 480 | 481 | # Tells whether to display a full report or only the messages. 482 | reports=no 483 | 484 | # Activate the evaluation score. 485 | score=yes 486 | 487 | 488 | [SIMILARITIES] 489 | 490 | # Comments are removed from the similarity computation 491 | ignore-comments=yes 492 | 493 | # Docstrings are removed from the similarity computation 494 | ignore-docstrings=yes 495 | 496 | # Imports are removed from the similarity computation 497 | ignore-imports=yes 498 | 499 | # Signatures are removed from the similarity computation 500 | ignore-signatures=yes 501 | 502 | # Minimum lines number of a similarity. 503 | min-similarity-lines=4 504 | 505 | 506 | [SPELLING] 507 | 508 | # Limits count of emitted suggestions for spelling mistakes. 509 | max-spelling-suggestions=4 510 | 511 | # Spelling dictionary name. Available dictionaries: en_AU (hunspell), en_CA 512 | # (hunspell), en_GB (hunspell), en_US (hunspell), en_ZA (hunspell). 513 | spelling-dict= 514 | 515 | # List of comma separated words that should be considered directives if they 516 | # appear at the beginning of a comment and should not be checked. 517 | spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: 518 | 519 | # List of comma separated words that should not be checked. 520 | spelling-ignore-words= 521 | 522 | # A path to a file that contains the private dictionary; one word per line. 523 | spelling-private-dict-file=docs/source/spelling_wordlist.txt 524 | 525 | # Tells whether to store unknown words to the private dictionary (see the 526 | # --spelling-private-dict-file option) instead of raising a message. 527 | spelling-store-unknown-words=no 528 | 529 | 530 | [STRING] 531 | 532 | # This flag controls whether inconsistent-quotes generates a warning when the 533 | # character used as a quote delimiter is used inconsistently within a module. 534 | check-quote-consistency=no 535 | 536 | # This flag controls whether the implicit-str-concat should generate a warning 537 | # on implicit string concatenation in sequences defined over several lines. 538 | check-str-concat-over-line-jumps=no 539 | 540 | 541 | [TYPECHECK] 542 | 543 | # List of decorators that produce context managers, such as 544 | # contextlib.contextmanager. Add to this list to register other decorators that 545 | # produce valid context managers. 546 | contextmanager-decorators=contextlib.contextmanager 547 | 548 | # List of members which are set dynamically and missed by pylint inference 549 | # system, and so shouldn't trigger E1101 when accessed. Python regular 550 | # expressions are accepted. 551 | generated-members=numpy.*, 552 | torch.* 553 | 554 | # Tells whether missing members accessed in mixin class should be ignored. A 555 | # class is considered mixin if its name matches the mixin-class-rgx option. 556 | ignore-mixin-members=yes 557 | 558 | # Tells whether to warn about missing members when the owner of the attribute 559 | # is inferred to be None. 560 | ignore-none=yes 561 | 562 | # This flag controls whether pylint should warn about no-member and similar 563 | # checks whenever an opaque object is returned when inferring. The inference 564 | # can return multiple potential results while evaluating a Python object, but 565 | # some branches might not be evaluated, which results in partial inference. In 566 | # that case, it might be useful to still emit no-member and other checks for 567 | # the rest of the inferred objects. 568 | ignore-on-opaque-inference=yes 569 | 570 | # List of symbolic message names to ignore for Mixin members. 571 | ignored-checks-for-mixins=no-member, 572 | not-async-context-manager, 573 | not-context-manager, 574 | attribute-defined-outside-init 575 | 576 | # List of class names for which member attributes should not be checked (useful 577 | # for classes with dynamically set attributes). This supports the use of 578 | # qualified names. 579 | ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace 580 | 581 | # Show a hint with possible names when a member name was not found. The aspect 582 | # of finding the hint is based on edit distance. 583 | missing-member-hint=yes 584 | 585 | # The minimum edit distance a name should have in order to be considered a 586 | # similar match for a missing member name. 587 | missing-member-hint-distance=1 588 | 589 | # The total number of similar names that should be taken in consideration when 590 | # showing a hint for a missing member. 591 | missing-member-max-choices=1 592 | 593 | # Regex pattern to define which classes are considered mixins. 594 | mixin-class-rgx=.*[Mm]ixin 595 | 596 | # List of decorators that change the signature of a decorated function. 597 | signature-mutators= 598 | 599 | 600 | [VARIABLES] 601 | 602 | # List of additional names supposed to be defined in builtins. Remember that 603 | # you should avoid defining new builtins when possible. 604 | additional-builtins= 605 | 606 | # Tells whether unused global variables should be treated as a violation. 607 | allow-global-unused-variables=yes 608 | 609 | # List of names allowed to shadow builtins 610 | allowed-redefined-builtins= 611 | 612 | # List of strings which can identify a callback function by name. A callback 613 | # name must start or end with one of those strings. 614 | callbacks=cb_, 615 | _cb 616 | 617 | # A regular expression matching the name of dummy variables (i.e. expected to 618 | # not be used). 619 | dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ 620 | 621 | # Argument names that match this expression will be ignored. 622 | ignored-argument-names=_.*|^ignored_|^unused_ 623 | 624 | # Tells whether we should check for unused import in __init__ files. 625 | init-import=no 626 | 627 | # List of qualified module names which can have objects that can redefine 628 | # builtins. 629 | redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io 630 | -------------------------------------------------------------------------------- /LICENSE-CODE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 DeepSeek 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /LICENSE-MODEL: -------------------------------------------------------------------------------- 1 | DEEPSEEK LICENSE AGREEMENT 2 | 3 | Version 1.0, 23 October 2023 4 | 5 | Copyright (c) 2023 DeepSeek 6 | 7 | Section I: PREAMBLE 8 | 9 | Large generative models are being widely adopted and used, and have the potential to transform the way individuals conceive and benefit from AI or ML technologies. 10 | 11 | Notwithstanding the current and potential benefits that these artifacts can bring to society at large, there are also concerns about potential misuses of them, either due to their technical limitations or ethical considerations. 12 | 13 | In short, this license strives for both the open and responsible downstream use of the accompanying model. When it comes to the open character, we took inspiration from open source permissive licenses regarding the grant of IP rights. Referring to the downstream responsible use, we added use-based restrictions not permitting the use of the model in very specific scenarios, in order for the licensor to be able to enforce the license in case potential misuses of the Model may occur. At the same time, we strive to promote open and responsible research on generative models for content generation. 14 | 15 | Even though downstream derivative versions of the model could be released under different licensing terms, the latter will always have to include - at minimum - the same use-based restrictions as the ones in the original license (this license). We believe in the intersection between open and responsible AI development; thus, this agreement aims to strike a balance between both in order to enable responsible open-science in the field of AI. 16 | 17 | This License governs the use of the model (and its derivatives) and is informed by the model card associated with the model. 18 | 19 | NOW THEREFORE, You and DeepSeek agree as follows: 20 | 21 | 1. Definitions 22 | "License" means the terms and conditions for use, reproduction, and Distribution as defined in this document. 23 | "Data" means a collection of information and/or content extracted from the dataset used with the Model, including to train, pretrain, or otherwise evaluate the Model. The Data is not licensed under this License. 24 | "Output" means the results of operating a Model as embodied in informational content resulting therefrom. 25 | "Model" means any accompanying machine-learning based assemblies (including checkpoints), consisting of learnt weights, parameters (including optimizer states), corresponding to the model architecture as embodied in the Complementary Material, that have been trained or tuned, in whole or in part on the Data, using the Complementary Material. 26 | "Derivatives of the Model" means all modifications to the Model, works based on the Model, or any other model which is created or initialized by transfer of patterns of the weights, parameters, activations or output of the Model, to the other model, in order to cause the other model to perform similarly to the Model, including - but not limited to - distillation methods entailing the use of intermediate data representations or methods based on the generation of synthetic data by the Model for training the other model. 27 | "Complementary Material" means the accompanying source code and scripts used to define, run, load, benchmark or evaluate the Model, and used to prepare data for training or evaluation, if any. This includes any accompanying documentation, tutorials, examples, etc, if any. 28 | "Distribution" means any transmission, reproduction, publication or other sharing of the Model or Derivatives of the Model to a third party, including providing the Model as a hosted service made available by electronic or other remote means - e.g. API-based or web access. 29 | "DeepSeek" (or "we") means Beijing DeepSeek Artificial Intelligence Fundamental Technology Research Co., Ltd., Hangzhou DeepSeek Artificial Intelligence Fundamental Technology Research Co., Ltd. and/or any of their affiliates. 30 | "You" (or "Your") means an individual or Legal Entity exercising permissions granted by this License and/or making use of the Model for whichever purpose and in any field of use, including usage of the Model in an end-use application - e.g. chatbot, translator, etc. 31 | "Third Parties" means individuals or legal entities that are not under common control with DeepSeek or You. 32 | 33 | Section II: INTELLECTUAL PROPERTY RIGHTS 34 | 35 | Both copyright and patent grants apply to the Model, Derivatives of the Model and Complementary Material. The Model and Derivatives of the Model are subject to additional terms as described in Section III. 36 | 37 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, DeepSeek hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare, publicly display, publicly perform, sublicense, and distribute the Complementary Material, the Model, and Derivatives of the Model. 38 | 39 | 3. Grant of Patent License. Subject to the terms and conditions of this License and where and as applicable, DeepSeek hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this paragraph) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Model and the Complementary Material, where such license applies only to those patent claims licensable by DeepSeek that are necessarily infringed by its contribution(s). If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Model and/or Complementary Material constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for the Model and/or works shall terminate as of the date such litigation is asserted or filed. 40 | 41 | 42 | Section III: CONDITIONS OF USAGE, DISTRIBUTION AND REDISTRIBUTION 43 | 44 | 4. Distribution and Redistribution. You may host for Third Party remote access purposes (e.g. software-as-a-service), reproduce and distribute copies of the Model or Derivatives of the Model thereof in any medium, with or without modifications, provided that You meet the following conditions: 45 | a. Use-based restrictions as referenced in paragraph 5 MUST be included as an enforceable provision by You in any type of legal agreement (e.g. a license) governing the use and/or distribution of the Model or Derivatives of the Model, and You shall give notice to subsequent users You Distribute to, that the Model or Derivatives of the Model are subject to paragraph 5. This provision does not apply to the use of Complementary Material. 46 | b. You must give any Third Party recipients of the Model or Derivatives of the Model a copy of this License; 47 | c. You must cause any modified files to carry prominent notices stating that You changed the files; 48 | d. You must retain all copyright, patent, trademark, and attribution notices excluding those notices that do not pertain to any part of the Model, Derivatives of the Model. 49 | e. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions - respecting paragraph 4.a. – for use, reproduction, or Distribution of Your modifications, or for any such Derivatives of the Model as a whole, provided Your use, reproduction, and Distribution of the Model otherwise complies with the conditions stated in this License. 50 | 51 | 5. Use-based restrictions. The restrictions set forth in Attachment A are considered Use-based restrictions. Therefore You cannot use the Model and the Derivatives of the Model for the specified restricted uses. You may use the Model subject to this License, including only for lawful purposes and in accordance with the License. Use may include creating any content with, finetuning, updating, running, training, evaluating and/or reparametrizing the Model. You shall require all of Your users who use the Model or a Derivative of the Model to comply with the terms of this paragraph (paragraph 5). 52 | 53 | 6. The Output You Generate. Except as set forth herein, DeepSeek claims no rights in the Output You generate using the Model. You are accountable for the Output you generate and its subsequent uses. No use of the output can contravene any provision as stated in the License. 54 | 55 | Section IV: OTHER PROVISIONS 56 | 57 | 7. Updates and Runtime Restrictions. To the maximum extent permitted by law, DeepSeek reserves the right to restrict (remotely or otherwise) usage of the Model in violation of this License. 58 | 59 | 8. Trademarks and related. Nothing in this License permits You to make use of DeepSeek’ trademarks, trade names, logos or to otherwise suggest endorsement or misrepresent the relationship between the parties; and any rights not expressly granted herein are reserved by DeepSeek. 60 | 61 | 9. Personal information, IP rights and related. This Model may contain personal information and works with IP rights. You commit to complying with applicable laws and regulations in the handling of personal information and the use of such works. Please note that DeepSeek's license granted to you to use the Model does not imply that you have obtained a legitimate basis for processing the related information or works. As an independent personal information processor and IP rights user, you need to ensure full compliance with relevant legal and regulatory requirements when handling personal information and works with IP rights that may be contained in the Model, and are willing to assume solely any risks and consequences that may arise from that. 62 | 63 | 10. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, DeepSeek provides the Model and the Complementary Material on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Model, Derivatives of the Model, and the Complementary Material and assume any risks associated with Your exercise of permissions under this License. 64 | 65 | 11. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall DeepSeek be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Model and the Complementary Material (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if DeepSeek has been advised of the possibility of such damages. 66 | 67 | 12. Accepting Warranty or Additional Liability. While redistributing the Model, Derivatives of the Model and the Complementary Material thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of DeepSeek, and only if You agree to indemnify, defend, and hold DeepSeek harmless for any liability incurred by, or claims asserted against, DeepSeek by reason of your accepting any such warranty or additional liability. 68 | 69 | 13. If any provision of this License is held to be invalid, illegal or unenforceable, the remaining provisions shall be unaffected thereby and remain valid as if such provision had not been set forth herein. 70 | 71 | 14. Governing Law and Jurisdiction. This agreement will be governed and construed under PRC laws without regard to choice of law principles, and the UN Convention on Contracts for the International Sale of Goods does not apply to this agreement. The courts located in the domicile of Hangzhou DeepSeek Artificial Intelligence Fundamental Technology Research Co., Ltd. shall have exclusive jurisdiction of any dispute arising out of this agreement. 72 | 73 | END OF TERMS AND CONDITIONS 74 | 75 | Attachment A 76 | 77 | Use Restrictions 78 | 79 | You agree not to use the Model or Derivatives of the Model: 80 | 81 | - In any way that violates any applicable national or international law or regulation or infringes upon the lawful rights and interests of any third party; 82 | - For military use in any way; 83 | - For the purpose of exploiting, harming or attempting to exploit or harm minors in any way; 84 | - To generate or disseminate verifiably false information and/or content with the purpose of harming others; 85 | - To generate or disseminate inappropriate content subject to applicable regulatory requirements; 86 | - To generate or disseminate personal identifiable information without due authorization or for unreasonable use; 87 | - To defame, disparage or otherwise harass others; 88 | - For fully automated decision making that adversely impacts an individual’s legal rights or otherwise creates or modifies a binding, enforceable obligation; 89 | - For any use intended to or which has the effect of discriminating against or harming individuals or groups based on online or offline social behavior or known or predicted personal or personality characteristics; 90 | - To exploit any of the vulnerabilities of a specific group of persons based on their age, social, physical or mental characteristics, in order to materially distort the behavior of a person pertaining to that group in a manner that causes or is likely to cause that person or another person physical or psychological harm; 91 | - For any use intended to or which has the effect of discriminating against individuals or groups based on legally protected characteristics or categories. 92 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | print-% : ; @echo $* = $($*) 2 | PROJECT_NAME = DeepSeek-LLM 3 | COPYRIGHT = "DeepSeek." 4 | PROJECT_PATH = evaluation 5 | SHELL = /bin/bash 6 | SOURCE_FOLDERS = evaluation 7 | PYTHON_FILES = $(shell find $(SOURCE_FOLDERS) -type f -name "*.py" -o -name "*.pyi") 8 | COMMIT_HASH = $(shell git log -1 --format=%h) 9 | PATH := $(HOME)/go/bin:$(PATH) 10 | PYTHON ?= $(shell command -v python3 || command -v python) 11 | PYTESTOPTS ?= 12 | 13 | .PHONY: default 14 | default: install 15 | 16 | # Tools Installation 17 | 18 | check_pip_install = $(PYTHON) -m pip show $(1) &>/dev/null || (cd && $(PYTHON) -m pip install $(1) --upgrade) 19 | check_pip_install_extra = $(PYTHON) -m pip show $(1) &>/dev/null || (cd && $(PYTHON) -m pip install $(2) --upgrade) 20 | 21 | pylint-install: 22 | $(call check_pip_install_extra,pylint,pylint[spelling]) 23 | $(call check_pip_install,pyenchant) 24 | 25 | flake8-install: 26 | $(call check_pip_install,flake8) 27 | $(call check_pip_install,flake8-bugbear) 28 | $(call check_pip_install,flake8-comprehensions) 29 | $(call check_pip_install,flake8-docstrings) 30 | $(call check_pip_install,flake8-pyi) 31 | $(call check_pip_install,flake8-simplify) 32 | 33 | py-format-install: 34 | $(call check_pip_install,isort) 35 | $(call check_pip_install_extra,black,black[jupyter]) 36 | 37 | ruff-install: 38 | $(call check_pip_install,ruff) 39 | 40 | mypy-install: 41 | $(call check_pip_install,mypy) 42 | 43 | pre-commit-install: 44 | $(call check_pip_install,pre-commit) 45 | $(PYTHON) -m pre_commit install --install-hooks 46 | 47 | go-install: 48 | # requires go >= 1.16 49 | command -v go || (sudo apt-get install -y golang && sudo ln -sf /usr/lib/go/bin/go /usr/bin/go) 50 | 51 | addlicense-install: go-install 52 | command -v addlicense || go install github.com/google/addlicense@latest 53 | 54 | addlicense: addlicense-install 55 | addlicense -c $(COPYRIGHT) -ignore tests/coverage.xml -l mit -y 2023-$(shell date +"%Y") -check $(SOURCE_FOLDERS) 56 | 57 | # Python linters 58 | 59 | pylint: pylint-install 60 | $(PYTHON) -m pylint $(PROJECT_PATH) 61 | 62 | flake8: flake8-install 63 | $(PYTHON) -m flake8 --count --show-source --statistics 64 | 65 | py-format: py-format-install 66 | $(PYTHON) -m isort --project $(PROJECT_PATH) --check $(PYTHON_FILES) && \ 67 | $(PYTHON) -m black --check $(PYTHON_FILES) 68 | 69 | ruff: ruff-install 70 | $(PYTHON) -m ruff check . 71 | 72 | ruff-fix: ruff-install 73 | $(PYTHON) -m ruff check . --fix --exit-non-zero-on-fix 74 | 75 | mypy: mypy-install 76 | $(PYTHON) -m mypy $(PROJECT_PATH) --install-types --non-interactive 77 | 78 | pre-commit: pre-commit-install 79 | $(PYTHON) -m pre_commit run --all-files 80 | 81 | # Utility functions 82 | 83 | lint: ruff flake8 py-format mypy pylint addlicense 84 | 85 | format: py-format-install ruff-install addlicense-install 86 | $(PYTHON) -m isort --project $(PROJECT_PATH) $(PYTHON_FILES) 87 | $(PYTHON) -m black $(PYTHON_FILES) 88 | $(PYTHON) -m ruff check . --fix --exit-zero 89 | addlicense -c $(COPYRIGHT) -ignore tests/coverage.xml -l mit -y 2023-$(shell date +"%Y") $(SOURCE_FOLDERS) 90 | 91 | clean-py: 92 | find . -type f -name '*.py[co]' -delete 93 | find . -depth -type d -name "__pycache__" -exec rm -r "{}" + 94 | find . -depth -type d -name ".ruff_cache" -exec rm -r "{}" + 95 | find . -depth -type d -name ".mypy_cache" -exec rm -r "{}" + 96 | 97 | clean: clean-py 98 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 |
6 | DeepSeek LLM 7 |
8 |
9 |
10 | 11 | 12 | Homepage 13 | 14 | 15 | Chat 16 | 17 | 18 | Hugging Face 19 | 20 | 21 |
22 | 23 | 24 |
25 | 26 | 27 | Discord 28 | 29 | 30 | Wechat 31 | 32 | 33 | Twitter Follow 34 | 35 | 36 |
37 | 38 |
39 | 40 | 41 | Code License 42 | 43 | 44 | Model License 45 | 46 |
47 | 48 | 49 |

50 | Model Download | 51 | Quick Start | 52 | Evaluation Results | 53 | License | 54 | Citation 55 |

56 | 57 | 58 |

59 | Paper Link👁️ 60 |

61 | 62 | 63 | ## 1. Introduction 64 | 65 | Introducing DeepSeek LLM, an advanced language model comprising 67 billion parameters. It has been trained from scratch on a vast dataset of 2 trillion tokens in both English and Chinese. In order to foster research, we have made DeepSeek LLM 7B/67B Base and DeepSeek LLM 7B/67B Chat open source for the research community. 66 | 67 |
68 | result 69 |
70 | 71 | - **Superior General Capabilities:** DeepSeek LLM 67B Base outperforms Llama2 70B Base in areas such as reasoning, coding, math, and Chinese comprehension. 72 | 73 | - **Proficient in Coding and Math:** DeepSeek LLM 67B Chat exhibits outstanding performance in coding (HumanEval Pass@1: 73.78) and mathematics (GSM8K 0-shot: 84.1, Math 0-shot: 32.6). It also demonstrates remarkable generalization abilities, as evidenced by its exceptional score of 65 on the Hungarian National High School Exam. 74 | 75 | - **Mastery in Chinese Language:** Based on our evaluation, DeepSeek LLM 67B Chat surpasses GPT-3.5 in Chinese. 76 | 77 | ## 2. Model Downloads 78 | 79 | We release the DeepSeek LLM 7B/67B, including both base and chat models, to the public. To support a broader and more diverse range of research within both academic and commercial communities, we are providing access to the intermediate checkpoints of the base model from its training process. Please **note** that the use of this model is subject to the terms outlined in [License section](#8-license). Commercial usage is permitted under these terms. 80 | 81 | ### Huggingface 82 | 83 | | Model | Sequence Length | Download | 84 | |:---------------------:|:---------------:|:-----------------------------------------------------------------------:| 85 | | DeepSeek LLM 7B Base | 4096 | 🤗 [HuggingFace](https://huggingface.co/deepseek-ai/deepseek-llm-7b-base) | 86 | | DeepSeek LLM 7B Chat | 4096 | 🤗 [HuggingFace](https://huggingface.co/deepseek-ai/deepseek-llm-7b-chat) | 87 | | DeepSeek LLM 67B Base | 4096 | 🤗 [HuggingFace](https://huggingface.co/deepseek-ai/deepseek-llm-67b-base) | 88 | | DeepSeek LLM 67B Chat | 4096 | 🤗 [HuggingFace](https://huggingface.co/deepseek-ai/deepseek-llm-67b-chat) | 89 | 90 | ### Intermediate Checkpoints 91 | 92 | We host the intermediate checkpoints of DeepSeek LLM 7B/67B on AWS S3 (Simple Storage Service). These files can be downloaded using the AWS Command Line Interface (CLI). 93 | 94 | ``` 95 | # using AWS CLI 96 | 97 | # DeepSeek-LLM-7B-Base 98 | aws s3 cp s3://deepseek-ai/DeepSeek-LLM/DeepSeek-LLM-7B-Base --recursive --request-payer 99 | 100 | # DeepSeek-LLM-67B-Base 101 | aws s3 cp s3://deepseek-ai/DeepSeek-LLM/DeepSeek-LLM-67B-Base --recursive --request-payer 102 | ``` 103 | 104 | ## 3. Evaluation Results 105 | 106 | ### Base Model 107 | 108 | We evaluate our models and some baseline models on a series of representative benchmarks, both in English and Chinese. More results can be found in the [`evaluation`](evaluation) folder. In this part, the evaluation results we report are based on the internal, non-open-source hai-llm evaluation framework. Please note that there may be slight discrepancies when using the converted HuggingFace models. 109 | 110 | | model | Hella
Swag | Trivia
QA | MMLU | GSM8K | Human
Eval | BBH | CEval | CMMLU | Chinese
QA | 111 | |:---------------:|:-------------:|:------------:|:------:|:------:|:-------------:|:------:|:------:|:------:|:-------------:| 112 | | | 0-shot | 5-shot | 5-shot | 8-shot | 0-shot | 3-shot | 5-shot | 5-shot | 5-shot | 113 | | LLaMA-2
-7B | 75.6 | 63.8 | 45.8 | 15.5 | 14.6 | 38.5 | 33.9 | 32.6 | 21.5 | 114 | | LLaMA-2
-70B | 84.0 | 79.5 | 69.0 | 58.4 | 28.7 | 62.9 | 51.4 | 53.1 | 50.2 | 115 | | DeepSeek LLM
7B Base| 75.4 | 59.7 | 48.2 | 17.4 | 26.2 | 39.5 | 45.0 | 47.2 | 78.0 | 116 | | DeepSeek LLM
67B Base| 84.0 | 78.9 | 71.3 | 63.4 | 42.7 | 68.7 | 66.1 | 70.8 | 87.6 | 117 | 118 | **Note:** ChineseQA is an in-house benchmark, inspired by TriviaQA. 119 | 120 | ### Chat Model 121 | 122 | #### Never Seen Before Exam 123 | 124 | To address data contamination and tuning for specific testsets, we have designed fresh problem sets to assess the capabilities of open-source LLM models. **The evaluation results indicate that DeepSeek LLM 67B Chat performs exceptionally well on never-before-seen exams.** 125 | 126 | 127 | --- 128 | **Hungarian National High-School Exam:** 129 | In line with Grok-1, we have evaluated the model's mathematical capabilities using the [Hungarian National High School Exam](https://huggingface.co/datasets/keirp/hungarian_national_hs_finals_exam). This exam comprises 33 problems, and the model's scores are determined through human annotation. We follow the scoring metric in the [solution.pdf](https://huggingface.co/datasets/keirp/hungarian_national_hs_finals_exam/blob/main/test/solutions.pdf) to evaluate all models. 130 | 131 |
132 | result 133 |
134 | 135 | **Remark:** We have rectified an error from our initial evaluation. In this revised version, we have omitted the lowest scores for questions 16, 17, 18, as well as for the aforementioned image. Evaluation details are [here](https://github.com/deepseek-ai/DeepSeek-LLM/tree/HEAD/evaluation/hungarian_national_hs_solutions). 136 | 137 | 138 | --- 139 | **Instruction Following Evaluation:** On Nov 15th, 2023, Google released an [instruction following evaluation dataset](https://arxiv.org/pdf/2311.07911.pdf). They identified 25 types of verifiable instructions and constructed around 500 prompts, with each prompt containing one or more verifiable instructions. We use the prompt-level loose metric to evaluate all models. Here, we used the first version released by Google for the evaluation. For the Google revised test set evaluation results, please refer to the number in our paper. 140 | 141 |
142 | result 143 |
144 | 145 | --- 146 | 147 | **LeetCode Weekly Contest:** 148 | To assess the coding proficiency of the model, we have utilized problems from the [LeetCode Weekly Contest](https://leetcode.com/contest/) (Weekly Contest 351-372, Bi-Weekly Contest 108-117, from July 2023 to Nov 2023). We have obtained these problems by crawling data from LeetCode, which consists of 126 problems with over 20 test cases for each. The evaluation metric employed is akin to that of HumanEval. In this regard, if a model's outputs successfully pass all test cases, the model is considered to have effectively solved the problem. The model's coding capabilities are depicted in the Figure below, where the y-axis represents the pass@1 score on in-domain human evaluation testing, and the x-axis represents the pass@1 score on out-domain LeetCode Weekly Contest problems. 149 | 150 | 151 |
152 | result 153 |
154 | 155 | The specific questions and test cases will be released soon. Stay tuned! 156 | 157 | --- 158 | **Standard Benchmark** 159 | 160 | | Model | TriviaQA | MMLU | GSM8K | HumanEval | BBH | C-Eval | CMMLU |ChineseQA| 161 | |-----------------------|----------|------|-------|-----------|------|--------|-------|-------| 162 | | DeepSeek LLM 7B Base | 59.7 | 48.2 | 17.4 | 26.2 | 39.5 | 45.0 | 47.2 | 78.0 | 163 | | DeepSeek LLM 67B Base | 78.9 | 71.3 | 63.4 | 42.7 | 68.7 | 66.1 | 70.8 | 87.6 | 164 | | DeepSeek LLM 7B Chat | 57.9 | 49.4 | 62.6 | 48.2 | 42.3 | 47.0 | 49.7 | 75.0 | 165 | | DeepSeek LLM 67B Chat | 81.5 | 71.1 | 84.1 | 73.8 | 71.7 | 65.2 | 67.8 | 85.1 | 166 | 167 | **Note:** We evaluate chat models with 0-shot for MMLU, GSM8K, C-Eval, and CMMLU. More evaluation results can be found [here](https://github.com/deepseek-ai/DeepSeek-LLM/blob/HEAD/evaluation/more_results.md). 168 | 169 | **Revisit Multi-Choice Question Benchmarks** 170 | 171 | Based on our experimental observations, we have discovered that enhancing benchmark performance using multi-choice (MC) questions, such as MMLU, CMMLU, and C-Eval, is a relatively straightforward task. By incorporating multi-choice questions from Chinese exams, we have achieved exceptional results, as depicted in the table below: 172 | 173 | | Model | MMLU | C-Eval | CMMLU | 174 | |---------------------------|-------|--------|-------| 175 | | DeepSeek LLM 7B Chat | 49.4 | 47.0 | 49.7 | 176 | | DeepSeek LLM 7B Chat + MC | 60.9 | 71.3 | 73.8 | 177 | 178 | **Note:** +MC represents the addition of 20 million Chinese multiple-choice questions collected from the web. It is important to note that we conducted deduplication for the C-Eval validation set and CMMLU test set to prevent data contamination. This addition not only improves Chinese multiple-choice benchmarks but also enhances English benchmarks. However, we observed that it does not enhance the model's knowledge performance on other evaluations that do not utilize the multiple-choice style in the 7B setting. As a result, **we made the decision to not incorporate MC data in the pre-training or fine-tuning process**, as it would lead to overfitting on benchmarks. 179 | 180 | ## 4. Pre-Training Details 181 | 182 | ### Data 183 | Our primary goal is to holistically enhance the dataset's richness and variety. To achieve this, we've implemented multiple methods and established a distributed, frequent-checkpointing batch processing system, named "cc_cleaner", to bolster our data pipeline. 184 | 185 | Our minimal viable solution departs from RefinedWeb + CCNet. We greatly appreciate their selfless dedication to the research of AGI. 186 | 187 | We have also significantly incorporated deterministic randomization into our data pipeline. This approach enables us to continuously enhance our data throughout the lengthy and unpredictable training process. 188 | 189 | - **Data Composition**: Our training data comprises a diverse mix of Internet text, math, code, books, and self-collected data respecting robots.txt. In addition to the diverse content, we place a high priority on personal privacy and copyright protection. All content containing personal information or subject to copyright restrictions has been removed from our dataset. 190 | 191 | - **Dataset Pruning**: Our system employs heuristic rules and models to refine our training data. Our filtering process removes low-quality web data while preserving precious low-resource knowledge. It aims to improve overall corpus quality and remove harmful or toxic content. 192 | 193 | - **Deduplication**: Our advanced deduplication system, using MinhashLSH, strictly removes duplicates both at document and string levels. This rigorous deduplication process ensures exceptional data uniqueness and integrity, especially crucial in large-scale datasets. 194 | 195 | ### Pre-Training 196 | DeepSeek LM models use the same architecture as LLaMA, an auto-regressive transformer decoder model. The 7B model uses Multi-Head attention (MHA) while the 67B model uses Grouped-Query Attention (GQA). 197 | 198 | We pre-trained DeepSeek language models on a vast dataset of 2 trillion tokens, with a sequence length of 4096 and AdamW optimizer. The 7B model's training involved a batch size of 2304 and a learning rate of 4.2e-4 and the 67B model was trained with a batch size of 4608 and a learning rate of 3.2e-4. We employ a multi-step learning rate schedule in our training process. The learning rate begins with 2000 warmup steps, and then it is stepped to 31.6% of the maximum at 1.6 trillion tokens and 10% of the maximum at 1.8 trillion tokens. 199 | 200 | We release the training loss curve and several benchmark metrics curves, as detailed below. 201 | 202 |
203 | result 204 |
205 | 206 |
207 | result 208 |
209 | 210 | ## 5. Quick Start 211 | 212 | ### Installation 213 | 214 | On the basis of `Python >= 3.8` environment, install the necessary dependencies by running the following command: 215 | 216 | ```shell 217 | pip install -r requirements.txt 218 | ``` 219 | 220 | ### Inference with Huggingface's Transformers 221 | 222 | You can directly employ [Huggingface's Transformers](https://github.com/huggingface/transformers) for model inference. 223 | 224 | **Text Completion** 225 | 226 | ```python 227 | import torch 228 | from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig 229 | 230 | model_name = "deepseek-ai/deepseek-llm-67b-base" 231 | tokenizer = AutoTokenizer.from_pretrained(model_name) 232 | model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16, device_map="auto") 233 | model.generation_config = GenerationConfig.from_pretrained(model_name) 234 | model.generation_config.pad_token_id = model.generation_config.eos_token_id 235 | 236 | text = "An attention function can be described as mapping a query and a set of key-value pairs to an output, where the query, keys, values, and output are all vectors. The output is" 237 | inputs = tokenizer(text, return_tensors="pt") 238 | outputs = model.generate(**inputs.to(model.device), max_new_tokens=100) 239 | 240 | result = tokenizer.decode(outputs[0], skip_special_tokens=True) 241 | print(result) 242 | ``` 243 | 244 | **Chat Completion** 245 | 246 | ```python 247 | import torch 248 | from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig 249 | 250 | model_name = "deepseek-ai/deepseek-llm-67b-chat" 251 | tokenizer = AutoTokenizer.from_pretrained(model_name) 252 | model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16, device_map="auto") 253 | model.generation_config = GenerationConfig.from_pretrained(model_name) 254 | model.generation_config.pad_token_id = model.generation_config.eos_token_id 255 | 256 | messages = [ 257 | {"role": "user", "content": "Who are you?"} 258 | ] 259 | input_tensor = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt") 260 | outputs = model.generate(input_tensor.to(model.device), max_new_tokens=100) 261 | 262 | result = tokenizer.decode(outputs[0][input_tensor.shape[1]:], skip_special_tokens=True) 263 | print(result) 264 | ``` 265 | 266 | Avoiding the use of the provided function `apply_chat_template`, you can also interact with our model following the sample template. Note that `messages` should be replaced by your input. 267 | 268 | ``` 269 | User: {messages[0]['content']} 270 | 271 | Assistant: {messages[1]['content']}<｜end▁of▁sentence｜>User: {messages[2]['content']} 272 | 273 | Assistant: 274 | ``` 275 | 276 | **Note:** By default (`add_special_tokens=True`), our tokenizer automatically adds a `bos_token` (`<｜begin▁of▁sentence｜>`) before the input text. Additionally, since the system prompt is not compatible with this version of our models, we DO NOT RECOMMEND including the system prompt in your input. 277 | 278 | ### Inference with vLLM 279 | 280 | You can also employ [vLLM](https://github.com/vllm-project/vllm) for high-throughput inference. 281 | 282 | **Text Completion** 283 | 284 | ```python 285 | from vllm import LLM, SamplingParams 286 | 287 | tp_size = 4 # Tensor Parallelism 288 | sampling_params = SamplingParams(temperature=0.7, top_p=0.9, max_tokens=100) 289 | model_name = "deepseek-ai/deepseek-llm-67b-base" 290 | llm = LLM(model=model_name, trust_remote_code=True, gpu_memory_utilization=0.9, tensor_parallel_size=tp_size) 291 | 292 | prompts = [ 293 | "If everyone in a country loves one another,", 294 | "The research should also focus on the technologies", 295 | "To determine if the label is correct, we need to" 296 | ] 297 | outputs = llm.generate(prompts, sampling_params) 298 | 299 | generated_text = [output.outputs[0].text for output in outputs] 300 | print(generated_text) 301 | ``` 302 | 303 | **Chat Completion** 304 | 305 | ```python 306 | from transformers import AutoTokenizer 307 | from vllm import LLM, SamplingParams 308 | 309 | tp_size = 4 # Tensor Parallelism 310 | sampling_params = SamplingParams(temperature=0.7, top_p=0.9, max_tokens=100) 311 | model_name = "deepseek-ai/deepseek-llm-67b-chat" 312 | tokenizer = AutoTokenizer.from_pretrained(model_name) 313 | llm = LLM(model=model_name, trust_remote_code=True, gpu_memory_utilization=0.9, tensor_parallel_size=tp_size) 314 | 315 | messages_list = [ 316 | [{"role": "user", "content": "Who are you?"}], 317 | [{"role": "user", "content": "What can you do?"}], 318 | [{"role": "user", "content": "Explain Transformer briefly."}], 319 | ] 320 | # Avoid adding bos_token repeatedly 321 | prompt_token_ids = [tokenizer.apply_chat_template(messages, add_generation_prompt=True) for messages in messages_list] 322 | 323 | sampling_params.stop = [tokenizer.eos_token] 324 | outputs = llm.generate(prompt_token_ids=prompt_token_ids, sampling_params=sampling_params) 325 | 326 | generated_text = [output.outputs[0].text for output in outputs] 327 | print(generated_text) 328 | ``` 329 | 330 | ## 6. FAQ 331 | 332 | ### Could You Provide the tokenizer.model File for Model Quantization? 333 | 334 | DeepSeek LLM utilizes the [HuggingFace Tokenizer](https://huggingface.co/docs/tokenizers/index) to implement the Byte-level BPE algorithm, with specially designed pre-tokenizers to ensure optimal performance. Currently, there is no direct way to convert the tokenizer into a SentencePiece tokenizer. We are contributing to the open-source quantization methods facilitate the usage of HuggingFace Tokenizer. 335 | 336 | #### GGUF(llama.cpp) 337 | 338 | We have submitted a [PR](https://github.com/ggerganov/llama.cpp/pull/4070) to the popular quantization repository [llama.cpp](https://github.com/ggerganov/llama.cpp) to fully support all HuggingFace pre-tokenizers, including ours. 339 | 340 | While waiting for the PR to be merged, you can generate your GGUF model using the following steps: 341 | 342 | ```bash 343 | git clone https://github.com/DOGEwbx/llama.cpp.git 344 | cd llama.cpp 345 | git checkout regex_gpt2_preprocess 346 | # set up the environment according to README 347 | make 348 | python3 -m pip install -r requirements.txt 349 | # generate GGUF model 350 | python convert-hf-to-gguf.py --outfile --model-name deepseekllm 351 | # use q4_0 quantization as an example 352 | ./quantize q4_0 353 | ./main -m -n 128 -p 354 | ``` 355 | #### GPTQ(exllamav2) 356 | 357 | `UPDATE:`[exllamav2](https://github.com/turboderp/exllamav2) has been able to support HuggingFace Tokenizer. Please pull the latest version and try out. 358 | 359 | ### GPU Memory Usage 360 | 361 | We profile the peak memory usage of inference for 7B and 67B models at different batch size and sequence length settings. 362 | 363 | For DeepSeek LLM 7B, we utilize **1 NVIDIA A100-PCIE-40GB GPU** for inference. 364 | 365 |
Batch SizeSequence Length
256512102420484096
113.29 GB13.63 GB14.47 GB16.37 GB21.25 GB
213.63 GB14.39 GB15.98 GB19.82 GB29.59 GB
414.47 GB15.82 GB19.04 GB26.65 GBOOM
815.99 GB18.71 GB25.14 GB35.19 GBOOM
1619.06 GB24.52 GB37.28 GBOOMOOM
366 | 367 | For DeepSeek LLM 67B, we utilize **8 NVIDIA A100-PCIE-40GB GPUs** for inference. 368 | 369 |
Batch SizeSequence Length
256512102420484096
116.92 GB17.11 GB17.66 GB20.01 GB33.23 GB
217.04 GB17.28 GB18.55 GB25.27 GBOOM
417.20 GB17.80 GB21.28 GB33.71 GBOOM
817.59 GB19.25 GB25.69 GBOOMOOM
1618.17 GB21.69 GB34.54 GBOOMOOM
370 | 371 | ## 7. Limitation 372 | 373 | While DeepSeek LLMs have demonstrated impressive capabilities, they are not without their limitations. Here are some potential drawbacks of such models: 374 | 375 | 1. Over-reliance on training data: These models are trained on vast amounts of text data, which can introduce biases present in the data. They may inadvertently generate biased or discriminatory responses, reflecting the biases prevalent in the training data. 376 | 377 | 2. Hallucination: The model sometimes generates responses or outputs that may sound plausible but are factually incorrect or unsupported. This can occur when the model relies heavily on the statistical patterns it has learned from the training data, even if those patterns do not align with real-world knowledge or facts. 378 | 379 | 3. Repetition: The model may exhibit repetition in their generated responses. This repetition can manifest in various ways, such as repeating certain phrases or sentences, generating redundant information, or producing repetitive structures in the generated text. This issue can make the output of LLMs less diverse and less engaging for users. 380 | 381 | ## 8. License 382 | 383 | This code repository is licensed under [the MIT License](https://github.com/deepseek-ai/DeepSeek-LLM/blob/HEAD/LICENSE-CODE). The use of DeepSeek LLM Base/Chat models is subject to [the Model License](https://github.com/deepseek-ai/DeepSeek-LLM/blob/HEAD/LICENSE-MODEL). DeepSeek LLM series (including Base and Chat) supports commercial use. 384 | 385 | ## 9. Citation 386 | 387 | ``` 388 | @article{deepseek-llm, 389 | author = {DeepSeek-AI}, 390 | title = {DeepSeek LLM: Scaling Open-Source Language Models with Longtermism}, 391 | journal = {arXiv preprint arXiv:2401.02954}, 392 | year = {2024}, 393 | url = {https://github.com/deepseek-ai/DeepSeek-LLM} 394 | } 395 | ``` 396 | 397 | ## 10. Contact 398 | 399 | If you have any questions, please raise an issue or contact us at [service@deepseek.com](mailto:service@deepseek.com). 400 | -------------------------------------------------------------------------------- /evaluation/hungarian_national_hs_solutions/exam_Baichuan2-13B-Chat.csv: -------------------------------------------------------------------------------- 1 | prompt,answer,Points 2 | "Answer the following question: 3 | Given are two sets: $A=\{a ; b ; e ; g\}$ and $B=\{a ; b ; c ; d ; f\}$. 4 | 5 | By listing its elements, give the set $B \backslash A$.","The set $B \backslash A$ is the set of all elements of $B$ that are not in $A$. By listing its elements, we have: 6 | 7 | $B \backslash A = \{c, d, f\}$",2.0 8 | "Answer the following question: 9 | Bori, Kristóf and Marci are playing a role-playing card game. At the beginning of the game they each select one out of 10 role cards, without replacement. In how many different arrangements of the roles can the game begin?","To find the number of different arrangements of the roles, we can use the multiplicative formula for combinations. Since there are 10 role cards and each person can choose any card, we can find the number of different arrangements by multiplying the number of choices for each person. 10 | 11 | First, we find the number of choices for each person by using the formula: 12 | 13 | Number of choices = Number of role cards - 1 14 | 15 | So, Bori has 10 - 1 = 9 choices, Kristóf has 10 - 1 = 9 choices, and Marci has 10 - 1 = 9 choices. 16 | 17 | Now, we multiply the number of choices for each person to find the total number of different arrangements: 18 | 19 | Total number of arrangements = 9 * 9 * 9 = 729 20 | 21 | So, in 729 different arrangements the game can begin.",0.0 22 | "Answer the following question: 23 | Zita's salary has been raised from $275000$ Ft to $308000$ Ft. By what percentage has Zita's salary been raised?","To calculate the percentage increase in Zita's salary, we first need to find the difference between her new and old salaries. 24 | 25 | Difference = New salary - Old salary 26 | Difference = 308000 Ft - 275000 Ft 27 | Difference = 33000 Ft 28 | 29 | Now, we can calculate the percentage increase by dividing the difference by the old salary and multiplying by 100%. 30 | 31 | Percentage increase = (Difference / Old salary) * 100% 32 | Percentage increase = (33000 Ft / 275000 Ft) * 100% 33 | Percentage increase = 0.1203125 * 100% 34 | Percentage increase = 12.03125% 35 | 36 | So, Zita's salary has been raised by 12.03125%.",1.0 37 | "Answer the following question: 38 | In triangle $A B C \overrightarrow{A B}=\mathbf{b}, \overrightarrow{A C}=\mathbf{c}$. The midpoint of side $A B$ is point $F$, the midpoint of side $A C$ is $G$. Express vector $\overrightarrow{F G}$ in terms of vectors $\mathbf{b}$ and $\mathbf{c}$. Explain your answer.","To express the vector $\overrightarrow{FG}$ in terms of vectors $\mathbf{b}$ and $\mathbf{c}$ we can use the fact that the midpoints of the sides are also the perpendicular bisectors of the opposite sides. This means that the angle $\angle BAC$ is a right angle. We can also use the fact that the midpoints of the sides are also the points of concurrence of the altitudes of the triangle. 39 | 40 | Let $\overrightarrow{AB} = \mathbf{b}$ and $\overrightarrow{AC} = \mathbf{c}$. Since $\angle BAC$ is a right angle, we can use the Pythagorean theorem to find the length of $\overrightarrow{FG}$. 41 | 42 | Consider the triangles $\triangle ABF$ and $\triangle AGC$. Since $F$ and $G$ are the midpoints of $AB$ and $AC$, respectively, the altitudes of these triangles from $F$ and $G$ to $C$ and $B$, respectively, are equal in length. Let this common altitude be $h$. 43 | 44 | Since the altitudes of a triangle from the same vertex to the other two vertices are perpendicular and of equal length, the triangles $\triangle ABF$ and $\triangle AGC$ are similar. This means that the lengths of their respective sides are in a proportional relationship. In particular, the length of $FG$ is equal to half the length of the altitude $h$. 45 | 46 | We can now use the Pythagorean theorem to find the length of $h$: 47 | 48 | $$h^2 = \frac{1}{4} \cdot (b - c)^2$$ 49 | 50 | Since $h$ is equal to the length of $FG$, we can express the length of $FG$ as: 51 | 52 | $$FG = \sqrt{\frac{1}{4} \cdot (b - c)^2}$$ 53 | 54 | Now, to express the vector $\overrightarrow{FG}$ in terms of vectors $\mathbf{b}$ and $\mathbf{c}$, we can use the fact that: 55 | 56 | $$\overrightarrow{FG} = \frac{1}{2} \cdot (\mathbf{b} - \mathbf{c})$$ 57 | 58 | Since $\overrightarrow{FG}$ and $\mathbf{b} - \mathbf{c}$ have the same length and share the same direction, we can conclude that: 59 | 60 | $$\overrightarrow{FG} = \frac{1}{2} \cdot \mathbf{b} - \frac{1}{2} \cdot \mathbf{c}$$ 61 | 62 | This is the vector representation of $\overrightarrow{FG}$ in terms of vectors $\mathbf{b}$ and $\mathbf{c}$.",0.0 63 | "Answer the following question: 64 | Give five positive numbers such that their median is 3 and their range is 7 .","The median is the middle value of a set of numbers when they are arranged in ascending or descending order. The range is the difference between the highest and lowest numbers in a set. 65 | 66 | To find five positive numbers such that their median is 3 and their range is 7, we can follow these steps: 67 | 68 | 1. Choose a value for the lowest number. Let's say the lowest number is 1. 69 | 2. Choose a value for the highest number. Let's say the highest number is 11. 70 | 3. Calculate the range: 11 - 1 = 10. 71 | 4. Adjust the highest and lowest numbers to maintain the range at 7. Let's add or subtract 3 from the highest and lowest numbers to get 8 and 0. 72 | 5. Calculate the new range: 8 - 0 = 8. 73 | 6. Adjust the highest and lowest numbers to maintain the range at 7. Let's add or subtract 1 from the highest and lowest numbers to get 9 and -1. 74 | 7. Calculate the new range: 9 + 1 = 10. 75 | 76 | At this point, we have the following five positive numbers: 1, 2, 3, 9, and 10. The median is 3, and the range is 7.",0.0 77 | "Answer the following question: 78 | Determine the decimal (base 10) value of the binary (base 2) number 101011.","To determine the decimal (base 10) value of the binary (base 2) number 101011, you need to perform a binary-to-decimal conversion. This can be done by following these steps: 79 | 80 | 1. Start with the rightmost digit (the one on the far right) and count the number of 1s it has. In this case, the rightmost digit is a 1, so there is 1 1 (binary) or 2 (decimal). 81 | 82 | 2. Move to the next digit to the left (the one on the far left) and count the number of 1s it has. In this case, the next digit is a 1, so there are 2 1s (binary) or 2 x 2 = 4 (decimal). 83 | 84 | 3. Continue moving to the left, counting the number of 1s in each digit and multiplying the decimal value by the appropriate power of 2 (2^n, where n is the number of 1s in the digit). 85 | 86 | 4. If the digit is a 0, skip it. 87 | 88 | Following these steps, we have: 89 | 90 | - Rightmost digit: 1 (binary) => 2 (decimal) 91 | - Second from right: 1 (binary) => 2^1 = 2 (decimal) 92 | - Third from right: 1 (binary) => 2^2 = 4 (decimal) 93 | - Fourth from right: 1 (binary) => 2^3 = 8 (decimal) 94 | - Fifth from right: 0 (binary) => 0 (decimal) 95 | 96 | Now, add up all the decimal values to get the final result: 2 + 2 + 4 + 8 + 0 = 16. 97 | 98 | So, the decimal (base 10) value of the binary (base 2) number 101011 is 16.",0.0 99 | "Answer the following question: 100 | It is known that $\log_{2}(x)=5$. Give the value of $\log_{2}(2x)$. Explain your answer.","To find the value of $\log_{2}(2x)$, we can use the property of logarithms: $\log_{a}(b) = \log_{a}(c)$ if $b = c$. In this case, we have $x = 2x$. 101 | 102 | Since $\log_{2}(x) = 5$, we can substitute this value into the expression for $\log_{2}(2x)$: 103 | 104 | $$\log_{2}(2x) = \log_{2}(2x) = \log_{2}(2\cdot 2^5) = \log_{2}(2^6) = \log_{2}(64) = 6$$ 105 | 106 | So, the value of $\log_{2}(2x)$ is 6.",2.0 107 | "Answer the following question: 108 | List all integer values of $x$ for which both of the inequalities $-6 \leq x \leq 2$ and $-4 Green -> Yellow -> Blue (starting from any vertex) 388 | 2. Green -> Yellow -> Blue -> Green (starting from any vertex) 389 | 3. Yellow -> Blue -> Green -> Yellow (starting from any vertex) 390 | 391 | There are 3 ways to assign colours to the triangles, and each assignment is cyclic. So, the total number of different ways to colour the square according to the given rules is 3 * 3 = 9.",0.0 392 | "Answer the following question: 393 | Select one interior point of a square of the same size and connect it to the four vertices, as seen in the diagram. 394 | 395 | Consider the sums of the areas of the opposite triangle pairs drawn into the square of side 4 cm. 396 | 397 | Prove that the sums of the areas of the opposite triangle pairs are equal, i.e. the total area shaded grey in the diagram is equal to the total white area.","Let's first label the vertices of the square as follows: A(1, 1), B(1, 3), C(3, 3), and D(3, 1). The interior point is labeled as E. 398 | 399 | Now, let's calculate the areas of the triangles OAE, OAB, OAC, and OAD. The triangles OAE, OAB, OAC, and OAD are formed by drawing perpendicular lines from the center of the square (O) to the opposite sides. 400 | 401 | Area of triangle OAE = (1/2) * 4 * 1 = 2 square cm 402 | Area of triangle OAB = (1/2) * 4 * 2 = 4 square cm 403 | Area of triangle OAC = (1/2) * 4 * 2 = 4 square cm 404 | Area of triangle OAD = (1/2) * 4 * 1 = 2 square cm 405 | 406 | Now, let's calculate the areas of the triangles OBE, OBC, OCD, and ODE. The triangles OBE, OBC, OCD, and ODE are formed by drawing perpendicular lines from the center of the square (O) to the opposite sides, but this time, the lines are drawn from the opposite vertices. 407 | 408 | Area of triangle OBE = (1/2) * 3 * 1 = 1.5 square cm 409 | Area of triangle OBC = (1/2) * 3 * 2 = 3 square cm 410 | Area of triangle OCD = (1/2) * 3 * 2 = 3 square cm 411 | Area of triangle ODE = (1/2) * 3 * 1 = 1.5 square cm 412 | 413 | Now, let's calculate the areas of the triangles OCE and OCE. The triangles OCE and OCE are formed by drawing perpendicular lines from the center of the square (O) to the opposite sides, but this time, the lines are drawn from the opposite vertices. 414 | 415 | Area of triangle OCE = (1/2) * 3 * 1 = 1.5 square cm 416 | Area of triangle OCE = (1/2) * 3 * 3 = 4.5 square cm 417 | 418 | Now, let's calculate the areas of the remaining triangles: OBCE and OCDE. The triangles OBCE and OCDE are formed by drawing perpendicular lines from the center of the square (O) to the opposite sides, but this time, the lines are drawn from the opposite vertices. 419 | 420 | Area of triangle OBCE = (1/2) * 4.5 * 1 = 2.25 square cm 421 | Area of triangle OCDE = (1/2) * 4.5 * 3 = 6.75 square cm 422 | 423 | Now, let's calculate the areas of the triangles OABE and OADE. The triangles OABE and OADE are formed by drawing perpendicular lines from the center of the square (O) to the opposite sides, but this time, the lines are drawn from the opposite vertices. 424 | 425 | Area of triangle OABE = (1/2) * 4 * 1.5 = 3 square cm 426 | Area of triangle OADE = (1/2) * 4 * 1.5 = 3 square cm 427 | 428 | Now, let's calculate the areas of the triangles OACD and OBCD. The triangles OACD and OBCD are formed by drawing perpendicular lines from the center of the square (O) to the opposite sides, but this time, the lines are drawn from the opposite vertices. 429 | 430 | Area of triangle OACD = (1/2) * 4 * 3 = 6 square cm 431 | Area of triangle OBCD = (1/2) * 4 * 3 = 6 square cm 432 | 433 | Now, let's calculate the area of the square, which is 4 cm x 4 cm = 16 square cm. 434 | 435 | The total area of the triangles is: 2 + 4 + 4 + 2 + 1.5 + 3 + 1.5 + 3 + 6 + 6 + 2.25 + 6.75 + 3 + 3 = 52.75 square cm. 436 | 437 | The total area of the squares is: 1",0.0 438 | "Answer the following question: 439 | Solve the following equations in the set of real numbers. 440 | 441 | $$2 \cdot \sqrt{3 - x} = x + 5$$","To solve the given equation, we first need to isolate the square root term. We can do this by subtracting $2 \cdot \sqrt{3 - x}$ from both sides of the equation. This will give us: 442 | 443 | $$2 \cdot \sqrt{3 - x} - x - 5 = 0$$ 444 | 445 | Now, we can use the quadratic formula to solve for $x$: 446 | 447 | $$x = \frac{-b \pm \sqrt{b^2 - 4ac}}{2a}$$ 448 | 449 | In this case, $a = 2$, $b = -(x + 5)$, and $c = \sqrt{3 - x}$. Substituting these values into the formula, we get: 450 | 451 | $$x = \frac{(x + 5) \pm \sqrt{(x + 5)^2 - 4 \cdot 2 \cdot \sqrt{3 - x}}}{4}$$ 452 | 453 | To make the equation more manageable, let's square both sides of the equation: 454 | 455 | $$x^2 = (x + 5)^2 - 4 \cdot 2 \cdot \sqrt{3 - x}$$ 456 | 457 | Now, we can use the AM-GM inequality to find the minimum value of the expression: 458 | 459 | $$x^2 \geq (x + 5)^2 - 4 \cdot 2 \cdot \sqrt{3 - x}$$ 460 | 461 | To do this, we can use the fact that $x^2$ is the area of a square with side length $x$ and $(x + 5)^2$ is the area of a rectangle with sides $x$ and $5$. The AM-GM inequality states that the sum of two positive terms is always greater than the product of the terms, and the sum of the squares of the terms is always greater than the product of the squares of the terms. In this case, we have: 462 | 463 | $$x^2 \geq (x + 5)^2 - 4 \cdot 2 \cdot \sqrt{3 - x}$$ 464 | 465 | $$x^2 - (x + 5)^2 + 4 \cdot 2 \cdot \sqrt{3 - x} \geq 0$$ 466 | 467 | $$-(x + 5)^2 + 4 \cdot 2 \cdot \sqrt{3 - x} \leq 0$$ 468 | 469 | $$(x + 5)^2 - 4 \cdot 2 \cdot \sqrt{3 - x} \geq 0$$ 470 | 471 | Now, we can square both sides of the equation to get: 472 | 473 | $$(x + 5)^2 - 4 \cdot 2 \cdot \sqrt{3 - x} = (x + 5 + 2 \cdot \sqrt{3 - x})(x + 5 - 2 \cdot \sqrt{3 - x})$$ 474 | 475 | Since the coefficient of $\sqrt{3 - x}$ is negative, we know that the minimum value of the expression is achieved when $\sqrt{3 - x} = 0$. This means that $3 - x = 0$, or $x = 3$. 476 | 477 | So, the solution to the given equation is $x = 3$.",0.0 478 | "Answer the following question: 479 | Solve the following equations in the set of real numbers. 480 | 481 | $$\frac{x}{x+1} + \frac{x^2}{x^2 - 1} = 2$$","To solve the given equation, we will follow the steps below: 482 | 483 | 1. Clear the denominators by multiplying both sides of the equation by the least common multiple (LCM) of the denominators. 484 | 485 | LCM of $(x+1)$ and $(x^2-1)$ is $(x+1)(x-1)$. 486 | 487 | So, the equation becomes: 488 | 489 | $$x(x-1) \left( \frac{x+1}{x+1} + \frac{x^2}{x^2-1} \right) = 2(x+1)(x-1)$$ 490 | 491 | 2. Distribute the right-hand side: 492 | 493 | $$x(x-1) \left( \frac{x+1}{x+1} + \frac{x^2}{x^2-1} \right) = 2(x+1)(x-1)$$ 494 | 495 | $$x(x-1) \left( \frac{2x}{x^2-1} \right) = 2(x+1)(x-1)$$ 496 | 497 | 3. Simplify the equation: 498 | 499 | $$x(x-1) \left( \frac{2x}{(x+1)(x-1)} \right) = 2(x+1)(x-1)$$ 500 | 501 | $$x(x-1) \left( \frac{2x}{(x+1)(x-1)} \right) = 2(x+1)(x-1)$$ 502 | 503 | $$2x = 2x + 2$$ 504 | 505 | 4. Solve the equation for $x$: 506 | 507 | $$x = 2$$ 508 | 509 | So, the solution to the given equation is $x = 2$.",2.0 510 | "Answer the following question: 511 | The first term of an arithmetic sequence is 18. The sum of the first six terms of the sequence is equal to the sum of the first seven terms. 512 | 513 | Show that the sum of the first 13 terms of the sequence is 0 and calculate the 13th term of the sequence.","Let's first find the common difference of the arithmetic sequence. The sum of the first six terms is equal to the sum of the first seven terms, so we have: 514 | 515 | (a + a + d) + (a + d + d) + (a + 2d) + (a + 3d) + (a + 4d) + (a + 5d) = (a + a + 6d) + (a + 7d) 516 | 517 | Where a is the first term and d is the common difference. 518 | 519 | Now, we can solve for d: 520 | 521 | 6a + 20d = 7a + 6d 522 | 523 | Subtracting 6a from both sides: 524 | 525 | 20d = 7d 526 | 527 | Dividing by 7: 528 | 529 | d = 2 530 | 531 | So the arithmetic sequence has a common difference of 2. Now we can find the 13th term of the sequence: 532 | 533 | a + (13 - 1) * d = 18 + (13 - 1) * 2 = 18 + 24 = 42 534 | 535 | The 13th term is 42. 536 | 537 | Now let's find the sum of the first 13 terms: 538 | 539 | 13 * a + (13 * (13 - 1)) * d / 2 = 13 * 18 + (13 * 12) * 2 / 2 = 234 540 | 541 | The sum of the first 13 terms is 0.",0.0 542 | "Answer the following question: 543 | In the year 2018 factory A produced goods worth 500 million forints, while factory B produced goods worth 400 million forints. According to their long-term plans, factory A will increase production by 5% each year, while factory B will increase production by 6% per year. 544 | 545 | Calculate the total worth, in million forints, of the goods produced in factory A in the next 20 years (from 2019 to 2038) altogether.","To calculate the total worth of goods produced in factory A in the next 20 years, we need to consider the annual production growth rate of 5%. 546 | 547 | First, we calculate the total production of factory A in 20 years with the initial production of 500 million forints: 548 | 549 | Total production in 20 years = (500 * (1 + 0.05)^20) forints 550 | 551 | Total production in 20 years = (500 * 1.18729) forints 552 | 553 | Total production in 20 years = 593.645 million forints 554 | 555 | Now, we calculate the total production of factory B in the next 20 years with the initial production of 400 million forints and a growth rate of 6%: 556 | 557 | Total production in 20 years = (400 * (1 + 0.06)^20) forints 558 | 559 | Total production in 20 years = (400 * 1.21649) forints 560 | 561 | Total production in 20 years = 486.5936 million forints 562 | 563 | Finally, we calculate the total worth of goods produced in both factories in the next 20 years: 564 | 565 | Total production in 20 years = 593.645 + 486.5936 forints 566 | 567 | Total production in 20 years = 1,080.2386 million forints 568 | 569 | Therefore, the total worth of goods produced in factory A in the next 20 years is 593.645 million forints.",1.0 570 | "Answer the following question: 571 | In the year 2018 factory A produced goods worth 500 million forints, while factory B produced goods worth 400 million forints. According to their long-term plans, factory A will increase production by 5% each year, while factory B will increase production by 6% per year. 572 | 573 | The above plans were published on an economy website. A debate started in the comment section of the article: one user claimed that the difference between the volumes of production between the two factories would get smaller and smaller in the next couple of years. 574 | 575 | Calculate the missing values of the table below and show that the above claim is not true. 576 | 577 | | Year | Production volume of factory A (million Ft) | Production volume of factory B (million Ft) | 578 | |------|--------------------------------------------|---------------------------------------------| 579 | | 2018 | 500 | 400 | 580 | | 2019 | | | 581 | | 2020 | | | 582 | | 2021 | | |","According to the given information, we can calculate the production volumes of both factories for the next few years. 583 | 584 | Factory A's production will increase by 5% each year, while factory B's production will increase by 6% each year. 585 | 586 | | Year | Production volume of factory A (million Ft) | Production volume of factory B (million Ft) | 587 | |------|--------------------------------------------|---------------------------------------------| 588 | | 2018 | 500 | 400 | 589 | | 2019 | 500 * (1 + 0.05) = 525 | 400 * (1 + 0.06) = 444 | 590 | | 2020 | 525 * (1 + 0.05) = 551.25 | 444 * (1 + 0.06) = 470.4 | 591 | | 2021 | 551.25 * (1 + 0.05) = 578.4375 | 470.4 * (1 + 0.06) = 492.64 | 592 | 593 | As can be seen from the calculated values, the difference between the production volumes of the two factories is not decreasing but is actually increasing. The production volume of factory A is growing faster than that of factory B, which is why the difference is getting larger. 594 | 595 | The claim that the difference between the volumes of production between the two factories would get smaller and smaller in the next couple of years is not true.",2.5 596 | "Answer the following question: 597 | In the year 2018 factory A produced goods worth 500 million forints, while factory B produced goods worth 400 million forints. According to their long-term plans, factory A will increase production by 5% each year, while factory B will increase production by 6% per year. 598 | 599 | Calculate the year in which the production volume of factory B will reach the production volume of factory A. (Assuming the increase in production will, in fact, follow the plans.)","To calculate the year in which the production volume of factory B will reach the production volume of factory A, we first need to find out how much they need to increase their production to catch up with factory A. 600 | 601 | Factory A's production in 2018 was 500 million forints, and they plan to increase production by 5% each year. So in the first year (2019), their production will increase by 5% to 500 * 1.05 = 525 million forints. In the second year (2020), their production will increase by another 5% to 525 * 1.05 = 551.25 million forints. 602 | 603 | Factory B's production in 2018 was 400 million forints, and they plan to increase production by 6% each year. So in the first year (2019), their production will increase by 6% to 400 * 1.06 = 424 million forints. In the second year (2020), their production will increase by another 6% to 424 * 1.06 = 448.64 million forints. 604 | 605 | Now we need to find the first year when the production of factory B is greater than or equal to the production of factory A. In 2019, factory A's production will be 525 million forints, while factory B's production will be 424 million forints. This is not yet greater than the production of factory A. In 2020, factory A's production will be 551.25 million forints, while factory B's production will be 448.64 million forints. This is still not greater than the production of factory A. 606 | 607 | So, the first year when the production of factory B will be greater than or equal to the production of factory A is 2021, when factory A's production will be 551.25 million forints, and factory B's production will be 472.16 million forints (448.64 * 1.06 = 472.16). 608 | 609 | Therefore, it will take 3 years for the production of factory B to reach the production level of factory A (2021, 2022, and 2023).",0.0 610 | "Answer the following question: 611 | Magic Spheres dessert is sold in boxes that have a shape of a regular hexagon-based prism. Each edge of the regular hexagonal base is 5 cm long and the height of the box is 3 cm. The box contains 6 chocolate balls, each has a diameter of 2.8 cm. 612 | 613 | What percentage of the volume of the box is the volume of the chocolate?","To answer this question, we first need to calculate the volume of the box, the volume of the chocolate balls, and then the percentage of the chocolate balls' volume compared to the box's volume. 614 | 615 | 1. Volume of the box: 616 | The box is a regular hexagonal prism, so its volume can be calculated using the formula V = B * h, where B is the area of the base and h is the height. 617 | 618 | The area of the hexagonal base can be found using the formula A = a² * √3, where a is the length of each side of the hexagon. In this case, a = 5 cm. 619 | 620 | So the volume of the box is V = (5² * √3) * 3 = 15 * √3 * 3 = 45 * √3 cubic centimeters (cc). 621 | 622 | 2. Volume of the chocolate balls: 623 | Each chocolate ball has a diameter of 2.8 cm, so its volume can be found using the formula V = π * r², where r is the radius. 624 | 625 | The radius of the chocolate ball is 2.8 cm / 2 = 1.4 cm. 626 | 627 | So the volume of each chocolate ball is V = π * (1.4²) = 3.14 * 1.96 = 6.1544 cubic centimeters (cc). 628 | 629 | 3. Percentage of the chocolate balls' volume compared to the box's volume: 630 | The number of chocolate balls in the box is 6, so the total volume of the chocolate balls is 6 * 6.1544 = 36.9264 cc. 631 | 632 | The percentage of the chocolate balls' volume compared to the box's volume is (36.9264 / 45 * √3) * 100 = 29.23%. 633 | 634 | Therefore, 29.23% of the volume of the Magic Spheres dessert box is the volume of the chocolate balls.",1.0 635 | "Answer the following question: 636 | Chocolate balls in Magic Spheres dessert are individually wrapped in either gold or red wrapping foil. The sorting machine randomly drops the chocolate balls into the boxes, the probability of a red one is $\frac{1}{3}$, the probability of a gold one is $\frac{2}{3}$, a total six chocolate balls in each box. 637 | 638 | Calculate the probability that at least 5 of the 6 chocolate balls in a box will be wrapped in gold foil.","To calculate the probability that at least 5 of the 6 chocolate balls in a box will be wrapped in gold foil, we can use the complementary probability approach. This means we will calculate the probability that exactly 5 of the 6 chocolate balls in a box are wrapped in gold foil, and then subtract that probability from 1 to find the probability that at least 5 of the 6 chocolate balls in a box are wrapped in gold foil. 639 | 640 | The probability that exactly 5 of the 6 chocolate balls in a box are wrapped in gold foil is calculated by multiplying the probability of getting 5 gold balls (which is $\frac{2}{3}^5$) by the total number of ways to arrange 5 gold balls in a box of 6 (which is 6! / (5! * 1!)). This gives us: 641 | 642 | $$P(\text{5 gold balls}) = \frac{2}{3}^5 \cdot \frac{6!}{5! \cdot 1!}$$ 643 | 644 | Now, to find the probability that at least 5 of the 6 chocolate balls in a box are wrapped in gold foil, we subtract the probability that exactly 5 of the 6 chocolate balls in a box are wrapped in gold foil from 1 (1 - $P(\text{5 gold balls})$): 645 | 646 | $$P(\geq\text{5 gold balls}) = 1 - P(\text{5 gold balls})$$ 647 | 648 | Substituting the value of $P(\text{5 gold balls})$ from above, we get: 649 | 650 | $$P(\geq\text{5 gold balls}) = 1 - \frac{2}{3}^5 \cdot \frac{6!}{5! \cdot 1!}$$ 651 | 652 | Simplifying the expression, we get: 653 | 654 | $$P(\geq\text{5 gold balls}) = 1 - \frac{2}{3} \cdot \frac{6!}{5!}$$ 655 | 656 | $$P(\geq\text{5 gold balls}) = 1 - \frac{2}{3} \cdot \frac{720}{5!}$$ 657 | 658 | $$P(\geq\text{5 gold balls}) = 1 - \frac{2 \cdot 720}{3 \cdot 5 \cdot 4 \cdot 3 \cdot 2}$$ 659 | 660 | $$P(\geq\text{5 gold balls}) = 1 - \frac{1440}{24}$$ 661 | 662 | $$P(\geq\text{5 gold balls}) = 1 - 60$$ 663 | 664 | $$P(\geq\text{5 gold balls}) = 1 - 60 = 40$$ 665 | 666 | So, the probability that at least 5 of the 6 chocolate balls in a box will be wrapped in gold foil is 40%.",0.0 667 | "Answer the following question: 668 | Each side of the regular hexagon $ABCDEF$ is 5 cm long. The hexagon is rotated around the perpendicular bisector of side $AB$. 669 | 670 | Calculate the volume of the resulting solid.","To calculate the volume of the resulting solid, we first need to find the height of the prism that is formed when the hexagon is rotated around the perpendicular bisector of side $AB$. 671 | 672 | The height of the prism can be found by calculating the distance from the perpendicular bisector to the base of the prism, which is formed by the sides of the hexagon. To do this, we can use the Pythagorean theorem. 673 | 674 | The perpendicular bisector of side $AB$ divides side $AB$ into two equal segments, $AC$ and $BD$. The length of $AC$ is half the length of $AB$, so it's 5 cm / 2 = 2.5 cm. Similarly, the length of $BD$ is also 2.5 cm. 675 | 676 | The triangle $ACD$ is isosceles, with equal sides $AC$ and $CD$. The height of the prism can be found by drawing a perpendicular line from the vertex $C$ to the base of the isosceles triangle $ACD$ (which is $BD$). This creates a right triangle with legs $AC$ and $CD$ and hypotenuse $BD$. 677 | 678 | Applying the Pythagorean theorem to this right triangle, we have: 679 | 680 | $$CD^2 + AC^2 = BD^2$$ 681 | 682 | $$(2.5 cm)^2 + (2.5 cm)^2 = BD^2$$ 683 | 684 | $$6.25 + 6.25 = BD^2$$ 685 | 686 | $$12.5 = BD^2$$ 687 | 688 | $$BD = \sqrt{12.5}$$ 689 | 690 | $$BD \approx 3.5$$ 691 | 692 | So the height of the prism is approximately 3.5 cm. 693 | 694 | Now that we have the height of the prism, we can calculate its volume using the formula for the volume of a prism: 695 | 696 | $$V = \frac{1}{3} BH$$ 697 | 698 | Where $B$ is the base of the prism (in this case, the area of a regular hexagon), and $H$ is the height of the prism. 699 | 700 | The area of a regular hexagon can be found using the formula: 701 | 702 | $$A = (3 s)^2$$ 703 | 704 | Where $s$ is the length of a side of the hexagon. In this case, $s = 5 cm$. 705 | 706 | So the area of the hexagon is: 707 | 708 | $$A = (3 \times 5 cm)^2$$ 709 | 710 | $$A = 45 cm^2$$ 711 | 712 | Plugging this area into the volume formula, we get: 713 | 714 | $$V = \frac{1}{3} \times 45 cm^2 \times 3.5 cm$$ 715 | 716 | $$V = 5 \times 3.5$$ 717 | 718 | $$V = 17.5$$ 719 | 720 | So the volume of the resulting solid is approximately 17.5 cubic centimeters.",0.0 721 | -------------------------------------------------------------------------------- /evaluation/hungarian_national_hs_solutions/exam_ChatGLM3-6B.csv: -------------------------------------------------------------------------------- 1 | prompt,answer,Points 2 | "Answer the following question: 3 | Given are two sets: $A=\{a ; b ; e ; g\}$ and $B=\{a ; b ; c ; d ; f\}$. 4 | 5 | By listing its elements, give the set $B \backslash A$."," The set $B \backslash A$ is the set of elements that are in $B$ but not in $A$. 6 | 7 | Listing the elements of $B$, we have $B = \{a, b, c, d, f\}$. 8 | 9 | Listing the elements of $A$, we have $A = \{a, b, e, g\}$. 10 | 11 | To find the set $B \backslash A$, we need to remove the elements of $A$ from $B$. 12 | 13 | Therefore, $B \backslash A = \{c, d, f\}$. 14 | 15 | So, the set $B \backslash A$ is $\{c, d, f\}$.The answer is: \{c,d,f\}",2.0 16 | "Answer the following question: 17 | Bori, Kristóf and Marci are playing a role-playing card game. At the beginning of the game they each select one out of 10 role cards, without replacement. In how many different arrangements of the roles can the game begin?"," There are 10 choices for the first role, then 9 choices for the second role, and so on, until there is only 1 choice for the last role. 18 | Therefore, the total number of different arrangements is $10 \times 9 \times 8 \times 7 \times 6 \times 5 \times 4 \times 3 \times 2 \times 1 = \boxed{362,880}$. 19 | The answer is: 362,880",0.0 20 | "Answer the following question: 21 | Zita's salary has been raised from $275000$ Ft to $308000$ Ft. By what percentage has Zita's salary been raised?"," To find the percentage increase, we can use the formula: 22 | 23 | Percentage increase = ((New value - Old value) / Old value) * 100 24 | 25 | In this case, the old value is $275000$ Ft and the new value is $308000$ Ft. 26 | 27 | Plugging in the values, we get: 28 | 29 | Percentage increase = ((308000 - 275000) / 275000) * 100 30 | 31 | Simplifying the expression, we have: 32 | 33 | Percentage increase = (33000 / 275000) * 100 34 | 35 | Calculating the fraction, we get: 36 | 37 | Percentage increase = 0.12 38 | 39 | To express this as a percentage, we multiply by 100: 40 | 41 | Percentage increase = 0.12 * 100 42 | 43 | Percentage increase = 12 44 | 45 | Therefore, Zita's salary has been raised by $12\%$.",2.0 46 | "Answer the following question: 47 | In triangle $A B C \overrightarrow{A B}=\mathbf{b}, \overrightarrow{A C}=\mathbf{c}$. The midpoint of side $A B$ is point $F$, the midpoint of side $A C$ is $G$. Express vector $\overrightarrow{F G}$ in terms of vectors $\mathbf{b}$ and $\mathbf{c}$. Explain your answer."," Since $F$ is the midpoint of side $A B$, we have $\overrightarrow{AF}=\frac{1}{2}\overrightarrow{AB}$ and $\overrightarrow{BF}=\frac{1}{2}\overrightarrow{AB}$. Similarly, since $G$ is the midpoint of side $A C$, we have $\overrightarrow{AG}=\frac{1}{2}\overrightarrow{AC}$ and $\overrightarrow{CG}=\frac{1}{2}\overrightarrow{AC}$. 48 | 49 | Therefore, we can express vector $\overrightarrow{F G}$ as follows: 50 | \begin{align*} 51 | \overrightarrow{F G} &= \overrightarrow{B G} - \overrightarrow{B F} \\ 52 | &= \overrightarrow{AG} - \overrightarrow{B F} \\ 53 | &= \overrightarrow{AG} - \frac{1}{2}\overrightarrow{AB} \\ 54 | &= \frac{1}{2}\overrightarrow{AC} - \frac{1}{2}\overrightarrow{AB} \\ 55 | &= \frac{1}{2}\overrightarrow{AC} - \frac{1}{2}\overrightarrow{AC} \\ 56 | &= \boxed{0}. 57 | \end{align*}",0.0 58 | "Answer the following question: 59 | Give five positive numbers such that their median is 3 and their range is 7 ."," To find five positive numbers with a median of 3 and a range of 7, we can start by considering the middle value of the numbers. Since the median is 3, the middle number is 3. 60 | 61 | The range is the difference between the largest and smallest numbers, so we need to find four numbers that are greater than 3 and less than the middle number. 62 | 63 | Let's assume the five numbers are a, b, c, d, and e, with a < b < c < d < e. Since the median is 3, we know that c = 3. 64 | 65 | To find the other four numbers, we can start by considering the smallest number, which is a. Since a < b, we know that a is less than 3. 66 | 67 | Next, we can consider the largest number, which is e. Since c < d < e, we know that d is greater than 3 and less than e. 68 | 69 | To find the remaining two numbers, we can use the range of 7. The range is the difference between the largest and smallest numbers, so we know that e - a = 7. 70 | 71 | Substituting the values we know, we have d - 3 = 7. Solving for d, we find that d = 10. 72 | 73 | Now we can find the remaining two numbers by considering the order of the numbers. Since a < b < c < d < e, we know that a < 3 < 10 < 3 < e. 74 | 75 | Therefore, the five positive numbers are a, b, c, d, and e, where a < b < c < d < e. 76 | 77 | To find the exact values of a, b, c, d, and e, we can use the information we have. However, without additional information, we cannot determine the exact values of a, b, c, d, and e.",0.0 78 | "Answer the following question: 79 | Determine the decimal (base 10) value of the binary (base 2) number 101011."," To convert a binary number to decimal, we can use the place value system. 80 | In binary, each digit represents a power of 2. Starting from the rightmost digit, we multiply it by 2 raised to the power of its position and add it to the next digit multiplied by 2 raised to the power of its position, and so on. 81 | In this case, the binary number is 101011. 82 | Starting from the rightmost digit, we have: 83 | 1 * 2^0 = 1 84 | 0 * 2^1 = 0 85 | 1 * 2^2 = 4 86 | 0 * 2^3 = 0 87 | 1 * 2^4 = 16 88 | 1 * 2^5 = 32 89 | 1 * 2^6 = 64 90 | Adding them all together, we get: 1 + 0 + 4 + 0 + 16 + 32 + 64 = 123. 91 | Therefore, the decimal value of the binary number 101011 is 123.The answer is: 123",0.0 92 | "Answer the following question: 93 | It is known that $\log_{2}(x)=5$. Give the value of $\log_{2}(2x)$. Explain your answer."," We are given that $\log_{2}(x)=5$. This means that $2^5=x$. 94 | 95 | To find the value of $\log_{2}(2x)$, we can use the property of logarithms that states $\log_{a}(b\cdot c) = \log_{a}(b) + \log_{a}(c)$. 96 | 97 | Using this property, we can rewrite $\log_{2}(2x)$ as $\log_{2}(2) + \log_{2}(x)$. 98 | 99 | Since $\log_{2}(2)=1$, we have $\log_{2}(2x) = 1 + \log_{2}(x)$. 100 | 101 | Substituting the value of $x$ from the given equation, we have $\log_{2}(2x) = 1 + \log_{2}(2^5)$. 102 | 103 | Simplifying further, we have $\log_{2}(2x) = 1 + 5 = \boxed{6}$.",2.0 104 | "Answer the following question: 105 | List all integer values of $x$ for which both of the inequalities $-6 \leq x \leq 2$ and $-4DeepSeek: HomepageDeepSeekHomepage 2 | -------------------------------------------------------------------------------- /images/home.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepseek-ai/DeepSeek-LLM/6712a86bfb7dd25c73383c5ad2eb7a8db540258b/images/home.png -------------------------------------------------------------------------------- /images/if_eval.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepseek-ai/DeepSeek-LLM/6712a86bfb7dd25c73383c5ad2eb7a8db540258b/images/if_eval.png -------------------------------------------------------------------------------- /images/leetcode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepseek-ai/DeepSeek-LLM/6712a86bfb7dd25c73383c5ad2eb7a8db540258b/images/leetcode.png -------------------------------------------------------------------------------- /images/llm_radar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepseek-ai/DeepSeek-LLM/6712a86bfb7dd25c73383c5ad2eb7a8db540258b/images/llm_radar.png -------------------------------------------------------------------------------- /images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepseek-ai/DeepSeek-LLM/6712a86bfb7dd25c73383c5ad2eb7a8db540258b/images/logo.png -------------------------------------------------------------------------------- /images/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | Created with Pixso. 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /images/mathexam.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepseek-ai/DeepSeek-LLM/6712a86bfb7dd25c73383c5ad2eb7a8db540258b/images/mathexam.png -------------------------------------------------------------------------------- /images/pretrain_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepseek-ai/DeepSeek-LLM/6712a86bfb7dd25c73383c5ad2eb7a8db540258b/images/pretrain_loss.png -------------------------------------------------------------------------------- /images/pretrain_metric.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepseek-ai/DeepSeek-LLM/6712a86bfb7dd25c73383c5ad2eb7a8db540258b/images/pretrain_metric.png -------------------------------------------------------------------------------- /images/qr.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepseek-ai/DeepSeek-LLM/6712a86bfb7dd25c73383c5ad2eb7a8db540258b/images/qr.jpeg -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | torch>=2.0 2 | tokenizers>=0.14.0 3 | transformers>=4.35.0 4 | accelerate 5 | sympy==1.12 6 | pebble 7 | timeout-decorator 8 | attrdict 9 | --------------------------------------------------------------------------------