├── .dockerignore ├── .github └── workflows │ └── Build.yml ├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── Dockerfile ├── LICENSE ├── README.md ├── docker-compose.yaml ├── docs ├── benchmark.md ├── faq.md ├── fastllm_pytools.md └── llama_cookbook.md ├── example ├── Android │ └── LLMAssistant │ │ ├── .gitignore │ │ ├── .idea │ │ ├── .gitignore │ │ ├── .name │ │ ├── compiler.xml │ │ ├── dbnavigator.xml │ │ ├── deploymentTargetDropDown.xml │ │ ├── gradle.xml │ │ ├── misc.xml │ │ └── vcs.xml │ │ ├── app │ │ ├── .gitignore │ │ ├── build.gradle │ │ ├── libs │ │ │ ├── arm64-v8a │ │ │ │ └── libassistant.so │ │ │ └── armeabi-v7a │ │ │ │ └── libassistant.so │ │ ├── proguard-rules.pro │ │ ├── release │ │ │ ├── app-arm64-v8a-release-unsigned.apk │ │ │ ├── app-armeabi-v7a-release-unsigned.apk │ │ │ ├── app-universal-release-unsigned.apk │ │ │ └── app-x86-release-unsigned.apk │ │ └── src │ │ │ ├── androidTest │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── doujiao │ │ │ │ └── xiaozhihuiassistant │ │ │ │ └── ExampleInstrumentedTest.java │ │ │ ├── main │ │ │ ├── AndroidManifest.xml │ │ │ ├── cpp │ │ │ │ ├── CMakeLists.txt │ │ │ │ ├── LLMChat.cpp │ │ │ │ ├── LLMChat.h │ │ │ │ ├── main.cpp │ │ │ │ └── native-lib.cpp │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── doujiao │ │ │ │ │ ├── core │ │ │ │ │ └── AssistantCore.java │ │ │ │ │ └── xiaozhihuiassistant │ │ │ │ │ ├── ChatMessage.java │ │ │ │ │ ├── MainActivity.java │ │ │ │ │ ├── adapter │ │ │ │ │ ├── BaseViewHolder.java │ │ │ │ │ └── MyAdapter.java │ │ │ │ │ ├── utils │ │ │ │ │ ├── PrefUtil.java │ │ │ │ │ ├── StatusBarUtils.java │ │ │ │ │ └── UriUtils.java │ │ │ │ │ └── widget │ │ │ │ │ ├── ChatPromptViewManager.java │ │ │ │ │ ├── Location.java │ │ │ │ │ ├── PromptView.java │ │ │ │ │ ├── PromptViewHelper.java │ │ │ │ │ └── location │ │ │ │ │ ├── BottomCenterLocation.java │ │ │ │ │ ├── ICalculateLocation.java │ │ │ │ │ ├── TopCenterLocation.java │ │ │ │ │ ├── TopLeftLocation.java │ │ │ │ │ └── TopRightLocation.java │ │ │ └── res │ │ │ │ ├── drawable-v24 │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ ├── drawable │ │ │ │ ├── btnbg.xml │ │ │ │ ├── editbg.xml │ │ │ │ └── ic_launcher_background.xml │ │ │ │ ├── layout │ │ │ │ ├── activity_item_left.xml │ │ │ │ ├── activity_item_right.xml │ │ │ │ └── activity_main.xml │ │ │ │ ├── mipmap-anydpi-v26 │ │ │ │ ├── ic_launcher.xml │ │ │ │ └── ic_launcher_round.xml │ │ │ │ ├── mipmap-hdpi │ │ │ │ ├── ic_launcher.webp │ │ │ │ └── ic_launcher_round.webp │ │ │ │ ├── mipmap-mdpi │ │ │ │ ├── ic_launcher.webp │ │ │ │ └── ic_launcher_round.webp │ │ │ │ ├── mipmap-xhdpi │ │ │ │ ├── ic_launcher.webp │ │ │ │ └── ic_launcher_round.webp │ │ │ │ ├── mipmap-xxhdpi │ │ │ │ ├── glm.png │ │ │ │ ├── ic_launcher.webp │ │ │ │ ├── ic_launcher_round.webp │ │ │ │ └── me.png │ │ │ │ ├── mipmap-xxxhdpi │ │ │ │ ├── ic_launcher.webp │ │ │ │ └── ic_launcher_round.webp │ │ │ │ ├── values-night │ │ │ │ └── themes.xml │ │ │ │ └── values │ │ │ │ ├── colors.xml │ │ │ │ ├── strings.xml │ │ │ │ └── themes.xml │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── doujiao │ │ │ └── xiaozhihuiassistant │ │ │ └── ExampleUnitTest.java │ │ ├── build.gradle │ │ ├── gradle.properties │ │ ├── gradle │ │ └── wrapper │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ └── settings.gradle ├── Qui │ ├── FastLLM.cpp │ ├── bin │ │ ├── Qt5Core.dll │ │ ├── Qt5Gui.dll │ │ ├── Qt5Widgets.dll │ │ ├── Qui.exe │ │ ├── fastllm_cpu.exe │ │ ├── fastllm_cuda.exe │ │ ├── path.txt │ │ ├── platforms │ │ │ └── qwindows.dll │ │ ├── qui_cn.qm │ │ └── styles │ │ │ └── qwindowsvistastyle.dll │ └── src │ │ ├── Qui.cpp │ │ ├── Qui.h │ │ ├── Qui.pro │ │ ├── Qui.ui │ │ ├── main.cpp │ │ └── qui_cn.ts ├── README.md ├── Win32Demo │ ├── StringUtils.h │ ├── Win32Demo.cpp │ ├── Win32Demo.sln │ ├── Win32Demo.vcxproj │ ├── bin │ │ └── web │ │ │ ├── css │ │ │ ├── github-markdown-light.min.css │ │ │ ├── github.min.css │ │ │ ├── katex.min.css │ │ │ └── texmath.css │ │ │ ├── index.html │ │ │ └── js │ │ │ ├── highlight.min.js │ │ │ ├── katex.min.js │ │ │ ├── markdown-it-link-attributes.min.js │ │ │ ├── markdown-it.min.js │ │ │ └── texmath.js │ ├── fastllm-gpu.vcxproj │ ├── fastllm-gpu.vcxproj.filters │ ├── fastllm.vcxproj │ ├── fastllm.vcxproj.filters │ └── httplib.h ├── apiserver │ ├── apiserver.cpp │ ├── json11.cpp │ └── json11.hpp ├── benchmark │ ├── benchmark.cpp │ └── prompts │ │ ├── beijing.txt │ │ └── hello.txt └── webui │ ├── httplib.h │ ├── web │ ├── css │ │ ├── github-markdown-light.min.css │ │ ├── github.min.css │ │ ├── katex.min.css │ │ └── texmath.css │ ├── index.html │ └── js │ │ ├── highlight.min.js │ │ ├── katex.min.js │ │ ├── markdown-it-link-attributes.min.js │ │ ├── markdown-it.min.js │ │ └── texmath.js │ └── webui.cpp ├── include ├── device.h ├── devices │ ├── cpu │ │ ├── cpudevice.h │ │ └── cputhreadpool.h │ └── cuda │ │ ├── cudadevice.h │ │ └── fastllm-cuda.cuh ├── executor.h ├── fastllm.h ├── model.h ├── models │ ├── basellm.h │ ├── chatglm.h │ ├── factoryllm.h │ ├── glm.h │ ├── internlm2.h │ ├── llama.h │ ├── minicpm.h │ ├── moss.h │ └── qwen.h └── utils │ ├── armMath.h │ └── utils.h ├── main.cpp ├── pyfastllm ├── README.md ├── examples │ ├── cli_low_level.py │ ├── cli_simple.py │ ├── convert_model.py │ ├── test_chatglm2.py │ ├── test_chatglm2_cpp.py │ ├── test_chatglm2_func.py │ ├── test_ops.py │ ├── web_api.py │ └── web_api_client.py ├── fastllm │ ├── __init__.py │ ├── convert.py │ ├── functions │ │ ├── __init__.py │ │ ├── custom_ops.py │ │ ├── fastllm_ops.py │ │ ├── numpy_ops.py │ │ └── util.py │ ├── hub │ │ ├── __init__.py │ │ └── chatglm2.py │ ├── models.py │ ├── nn │ │ ├── __init__.py │ │ ├── base_module.py │ │ └── modules.py │ └── utils │ │ ├── __init__.py │ │ ├── converter.py │ │ ├── quantizer.py │ │ └── writer.py ├── install.sh └── setup.py ├── src ├── device.cpp ├── devices │ ├── cpu │ │ ├── cpudevice.cpp │ │ └── cpudevicebatch.cpp │ └── cuda │ │ ├── cudadevice.cpp │ │ ├── cudadevicebatch.cpp │ │ └── fastllm-cuda.cu ├── executor.cpp ├── fastllm.cpp ├── model.cpp ├── models │ ├── basellm.cpp │ ├── chatglm.cpp │ ├── glm.cpp │ ├── internlm2.cpp │ ├── llama.cpp │ ├── minicpm.cpp │ ├── moss.cpp │ └── qwen.cpp └── pybinding.cpp ├── test ├── cmmlu │ ├── README.md │ ├── baichuan.py │ ├── categories.py │ ├── chatglm.py │ ├── eval.py │ └── qwen.py └── ops │ └── cppOps.cpp └── tools ├── fastllm_pytools ├── __init__.py ├── hf_model.py ├── llm.py └── torch2flm.py ├── scripts ├── alpaca2flm.py ├── baichuan2_2flm.py ├── baichuan2flm.py ├── chatglm_export.py ├── cli_demo.py ├── glm_export.py ├── llamalike2flm.py ├── minicpm2flm.py ├── moss_export.py ├── qwen2flm.py ├── setup.py └── web_demo.py └── src ├── pytools.cpp └── quant.cpp /.dockerignore: -------------------------------------------------------------------------------- 1 | ./models 2 | ./build/ -------------------------------------------------------------------------------- /.github/workflows/Build.yml: -------------------------------------------------------------------------------- 1 | name: Action Build 2 | on: [push] 3 | 4 | jobs: 5 | build: 6 | runs-on: ubuntu-latest 7 | 8 | steps: 9 | - name: Set up JDK 10 | uses: actions/setup-java@v1 11 | with: 12 | java-version: '11' 13 | 14 | - name: Checkout code 15 | uses: actions/checkout@v2 16 | 17 | - name: Build with arm64-v8a 18 | run: | 19 | wget https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip 20 | unzip android-ndk-r21d-linux-x86_64.zip 21 | export NDK=$GITHUB_WORKSPACE/android-ndk-r21d 22 | mkdir build-android 23 | cd build-android 24 | #ls ${NDK}/build/cmake/android.toolchain.cmake 25 | cmake -DCMAKE_MAKE_PROGRAM=/usr/bin/make -DCMAKE_CXX_COMPILER=/usr/bin/g++ -DCMAKE_TOOLCHAIN_FILE=${NDK}/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=android-23 -DCMAKE_CXX_FLAGS=-march=armv8.2a+dotprod .. 26 | make -j -B 27 | cp main fastllm-main-android 28 | 29 | - name: Build with x86 30 | run: | 31 | mkdir build-x86 32 | cd build-x86 33 | cmake .. -DUSE_CUDA=OFF 34 | make -j 35 | cp main fastllm-main-x86_64 36 | 37 | - name: Export and Upload Artifact 38 | uses: actions/upload-artifact@v2 39 | with: 40 | name: Output 41 | path: | 42 | build-android/fastllm-main-android 43 | build-x86/fastllm-main-x86_64 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.log 2 | *.pyc 3 | token 4 | /cmake-build-debug/ 5 | /build-tfacc/ 6 | /build-android/ 7 | /build-py/ 8 | /build/ 9 | /pyfastllm/build/ 10 | /pyfastllm/dist/ 11 | /.idea/ 12 | /.vscode/ 13 | /example/Win32Demo/bin/*.* 14 | /example/Win32Demo/Win32 15 | /example/Win32Demo/x64 16 | /example/Win32Demo/*.filters 17 | /example/Win32Demo/*.user 18 | /example/Win32Demo/.vs 19 | /example/Android/LLMAssistant/*.iml 20 | /example/Android/LLMAssistant/.gradle 21 | /example/Android/LLMAssistant/local.properties 22 | /example/Android/LLMAssistant/.idea/caches 23 | /example/Android/LLMAssistant/.idea/libraries 24 | /example/Android/LLMAssistant/.idea/modules.xml 25 | /example/Android/LLMAssistant/.idea/workspace.xml 26 | /example/Android/LLMAssistant/.idea/navEditor.xml 27 | /example/Android/LLMAssistant/.idea/assetWizardSettings.xml 28 | /example/Android/LLMAssistant/.DS_Store 29 | /example/Android/LLMAssistant/build 30 | /example/Android/LLMAssistant/captures 31 | /example/Android/LLMAssistant/.externalNativeBuild 32 | /example/Android/LLMAssistant/.cxx 33 | /example/Android/LLMAssistant/local.properties 34 | /test/cmmlu/results/ 35 | /models/ 36 | /localtest/ -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "third_party/pybind11"] 2 | path = third_party/pybind11 3 | url = https://github.com/pybind/pybind11.git 4 | branch = v2.10.5 5 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1-labs 2 | FROM nvidia/cuda:12.1.0-devel-ubuntu22.04 3 | 4 | # Update Apt repositories 5 | RUN apt-get update 6 | 7 | # Install and configure Python 8 | RUN apt-get -y --no-install-recommends install wget build-essential python3.10 python3-pip 9 | RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1 10 | RUN pip install setuptools streamlit-chat 11 | 12 | ENV WORKDIR /fastllm 13 | 14 | # Install cmake 15 | RUN wget -c https://cmake.org/files/LatestRelease/cmake-3.28.3-linux-x86_64.sh && bash ./cmake-3.28.3-linux-x86_64.sh --skip-license --prefix=/usr/ 16 | 17 | WORKDIR $WORKDIR 18 | ADD . $WORKDIR/ 19 | 20 | RUN mkdir $WORKDIR/build && cd build && cmake .. -DUSE_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=native && make -j && cd tools && python setup.py install 21 | 22 | CMD /fastllm/build/webui -p /models/chatglm2-6b-int8.flm 23 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | services: 3 | fastllm: 4 | build: 5 | context: . 6 | args: 7 | DOCKER_BUILDKIT: 0 8 | # privileged: true 9 | platforms: 10 | - "linux/amd64" 11 | tags: 12 | - "fastllm:v0.9" 13 | restart: always 14 | ports: 15 | - 11234:8081 16 | volumes: 17 | - ./models/:/models/ 18 | command: /fastllm/build/webui -p /models/chatglm2-6b-int8.flm -w ./example/webui/web 19 | 20 | -------------------------------------------------------------------------------- /docs/benchmark.md: -------------------------------------------------------------------------------- 1 | ## 推理速度 2 | 3 | 可以使用benchmark程序进行测速，根据不同配置、不同输入，推理速度也会有一些差别 4 | 5 | 例如: 6 | 7 | ``` sh 8 | ./benchmark -p ~/chatglm-6b-int4.flm -f ../example/benchmark/prompts/beijing.txt -b 1 9 | ./benchmark -p ~/chatglm-6b-int8.flm -f ../example/benchmark/prompts/beijing.txt -b 1 10 | ./benchmark -p ~/chatglm-6b-fp16.flm -f ../example/benchmark/prompts/hello.txt -b 512 -l 18 11 | ``` 12 | 13 | | 模型 | Data精度 | 平台 | Batch | 最大推理速度(token / s) | 14 | |-----------------:|---------|--------------------|-----------|---------------------:| 15 | | ChatGLM-6b-int4 | float32 | RTX 4090 | 1 | 176 | 16 | | ChatGLM-6b-int8 | float32 | RTX 4090 | 1 | 121 | 17 | | ChatGLM-6b-fp16 | float32 | RTX 4090 | 64 | 2919 | 18 | | ChatGLM-6b-fp16 | float32 | RTX 4090 | 256 | 7871 | 19 | | ChatGLM-6b-fp16 | float32 | RTX 4090 | 512 | 10209 | 20 | | ChatGLM-6b-int4 | float32 | Xiaomi 10 Pro - 4 Threads | 1 | 4 ~ 5 | 21 | -------------------------------------------------------------------------------- /docs/faq.md: -------------------------------------------------------------------------------- 1 | # 常见问题 2 | 3 | ## CMAKE 4 | 5 | ### CMAKE_CUDA_ARCHITECTURES must be non-empty if set. 6 | 7 | **现象：** 8 | 9 | > CMake Error at cmake/Modules/CMakeDetermineCUDACompiler.cmake:277 (message): 10 | > CMAKE_CUDA_ARCHITECTURES must be non-empty if set. 11 | > Call Stack (most recent call first): 12 | > CMakeLists.txt:39 (enable_language) 13 | 14 | **解决办法：** 15 | 16 | 部分版本cmake存在该问题，需手动指定`CMAKE_CUDA_ARCHITECTURES`。执行： 17 | 18 | ```shell 19 | cmake .. -DUSE_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=native 20 | ``` 21 | 22 | ### Unsupported gpu architecture 'compute_native' 23 | 24 | **现象：** 25 | 26 | > nvcc fatal : Unsupported gpu architecture 'compute_native' 27 | 28 | **解决办法：** 29 | 30 | 手动修改 CMakeLists.txt，根据GPU型号手动指定GPU的[Compute Capability](https://developer.nvidia.com/cuda-gpus)。如： 31 | 32 | ``` diff 33 | --- a/CMakeLists.txt 34 | +++ b/CMakeLists.txt 35 | @@ -52,7 +52,7 @@ 36 | #message(${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES}) 37 | set(FASTLLM_CUDA_SOURCES src/devices/cuda/cudadevice.cpp src/devices/cuda/cudadevicebatch.cpp src/devices/cuda/fastllm-cuda.cu) 38 | set(FASTLLM_LINKED_LIBS ${FASTLLM_LINKED_LIBS} cublas) 39 | - set(CMAKE_CUDA_ARCHITECTURES "native") 40 | + set(CMAKE_CUDA_ARCHITECTURES 61 75 86 89) 41 | endif() 42 | 43 | if (PY_API) 44 | ``` 45 | 46 | ### identifier "__hdiv" is undefined 47 | 48 | **现象：** 49 | 50 | > src/devices/cuda/fastllm-cuda.cu(247): error: identifier "hexp" is undefined 51 | > src/devices/cuda/fastllm-cuda.cu(247): error: identifier "__hdiv" is undefined 52 | > ... 53 | 54 | **原因：** [计算能力（Compute Capability）](https://developer.nvidia.com/cuda-gpus) <= 5.3 的GPU不支持半精度计算。 55 | 56 | **解决办法：** 如需要支持这些GPU，执行cmake时使用编译选项`CUDA_NO_TENSOR_CORE`： 57 | 58 | ```shell 59 | cmake .. -DUSE_CUDA=ON -DCUDA_NO_TENSOR_CORE=ON 60 | ``` 61 | 62 | ## Windows 63 | 64 | ### fastllm.h error 65 | 66 | **现象：** 67 | 68 | > include\fastllm.h(50): error : identifier "top_k" is undefined 69 | > include\fastllm.h(172): error : expected a "}" 70 | > include\fastllm.h(234): error : identifier "DataDevice" is undefined 71 | > .... 72 | 73 | **解决办法：** 参考 [example\README.md](/example/README.md)。签出代码后，**修改 include/fastllm.h**，Visual Studio中点击”文件“ -> "高级保存选项"，在编码中选择”Unicode (UTF-8 **带签名**) -代码页 65001“，或在其他文本编辑器中转为”UTF-8 BOM“编码。（由于linux下gcc不识别BOM头，MSVC依赖BOM判断文件编码，该修改只能手动处理。） 74 | 75 | ### main.exe 无法识别中文输入 76 | 77 | **原因：** Windows下cmd不支持UTF-8编码， 78 | 79 | **解决办法：** 编译[Win32Demo](/example/README.md#win32demo-windows平台) 或使用 [WebUI](/example/README.md#web-ui) 80 | 81 | ### Windows（MSVC）编译下，int4出现乱码 82 | 83 | **原因：** MSVC编译器优化选项 "`/Ob2`"、"`/Ob3`"与的现有代码冲突， 84 | 85 | **解决办法：** 编译时，在”属性“中找到"C/C++" -> "优化" -> "内联函数扩展" 中选择“只适用于 __inline (/Ob1)”。 86 | 87 | ### 导入提示 FileNotFoundError 88 | 89 | **现象：** 90 | 91 | > File "...Python\lib\ctypes\_\_init\_\_.py", line 374, in \_\_init\_\_ 92 | > self._handle = _dlopen(self._name, mode) 93 | > FileNotFoundError: Could not find module 'tools\fastllm_pytools\fastllm_tools.dll' (or one of its dependencies). Try using the full path with constructor syntax. 94 | 95 | **解决办法：** 非CPU编译时，部分版本的python存在这一问题。 96 | 97 | GPU编译时，根据使用的CUDA版本，将cudart cublas的相关dll文件复制到fastllm_tools同一目录下，例如： 98 | 99 | * CUDA 9.2 100 | * %CUDA_PATH%\bin\cublas64_92.dll 101 | * %CUDA_PATH%\bin\cudart64_92.dll 102 | * CUDA 11.x 103 | * %CUDA_PATH%\bin\cudart64_110.dll 104 | * %CUDA_PATH%\bin\cublas64_11.dll 105 | * %CUDA_PATH%\bin\cublasLt64_11.dll 106 | * CUDA 12.x 107 | * %CUDA_PATH%\bin\cudart64_12.dll 108 | * %CUDA_PATH%\bin\cublas64_12.dll 109 | * %CUDA_PATH%\bin\cublasLt64_12.dll 110 | 111 | ## fastllm_pytools 112 | 113 | ### 释放内存报错： CUDA error when release memory 114 | 115 | **现象：** 116 | 退出时报错： 117 | > Error: CUDA error when release memory! 118 | > CUDA error = 4, cudaErrorCudartUnloading at fastllm/src/devices/cuda/fastllm-cuda.cu:1493 119 | > 'driver shutting down' 120 | 121 | **原因：** python解释器在终止时常常会优先终止自己的进程，而没有现先析构调用的第三方库，因此在退出python时CUDA Runtime已关闭，释放显存操作失败。由于大多数时候显存已释放，并不会引起问题。 122 | 123 | **解决办法：** python程序退出时，先显式调用 `llm.release_memory()`方法。 124 | -------------------------------------------------------------------------------- /docs/fastllm_pytools.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TylunasLi/fastllm/f3cfc63dc9efb995670461b6e0d64d2c55cfa0a3/docs/fastllm_pytools.md -------------------------------------------------------------------------------- /example/Android/LLMAssistant/.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | .gradle 3 | /local.properties 4 | /.idea/caches 5 | /.idea/libraries 6 | /.idea/modules.xml 7 | /.idea/workspace.xml 8 | /.idea/navEditor.xml 9 | /.idea/assetWizardSettings.xml 10 | .DS_Store 11 | /build 12 | /captures 13 | .externalNativeBuild 14 | .cxx 15 | local.properties 16 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/.idea/.name: -------------------------------------------------------------------------------- 1 | XiaoZhihuiAssistant -------------------------------------------------------------------------------- /example/Android/LLMAssistant/.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/.idea/deploymentTargetDropDown.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/.idea/gradle.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 19 | 20 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 10 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/.gitignore: -------------------------------------------------------------------------------- 1 | /build -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'com.android.application' 3 | } 4 | 5 | android { 6 | compileSdk 30 7 | 8 | defaultConfig { 9 | applicationId "com.doujiao.xiaozhihuiassistant" 10 | minSdk 21 11 | targetSdk 26 12 | versionCode 1 13 | versionName "1.0" 14 | 15 | testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" 16 | externalNativeBuild { 17 | cmake { 18 | cppFlags '-std=c++11' 19 | } 20 | } 21 | ndk { 22 | abiFilters 'arm64-v8a','armeabi-v7a','x86' 23 | } 24 | } 25 | 26 | buildTypes { 27 | release { 28 | minifyEnabled false 29 | proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' 30 | } 31 | } 32 | compileOptions { 33 | sourceCompatibility JavaVersion.VERSION_1_8 34 | targetCompatibility JavaVersion.VERSION_1_8 35 | } 36 | externalNativeBuild { 37 | cmake { 38 | path file('src/main/cpp/CMakeLists.txt') 39 | version '3.18.1' 40 | } 41 | } 42 | // sourceSets { 43 | // main { 44 | // // jnilib 45 | // jniLibs.srcDirs = ['libs'] 46 | // } 47 | // } 48 | splits { 49 | abi { 50 | enable true 51 | reset() 52 | include 'arm64-v8a', 'armeabi-v7a','x86' 53 | universalApk true 54 | } 55 | } 56 | buildFeatures { 57 | viewBinding true 58 | } 59 | } 60 | 61 | dependencies { 62 | 63 | implementation 'com.android.support:appcompat-v7:28.0.0' 64 | implementation 'com.android.support:recyclerview-v7:28.0.0' 65 | implementation 'com.android.support.constraint:constraint-layout:2.0.4' 66 | testImplementation 'junit:junit:4.13.2' 67 | androidTestImplementation 'com.android.support.test:runner:1.0.2' 68 | androidTestImplementation 'com.android.support.test.espresso:espresso-core:3.0.2' 69 | } -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/libs/arm64-v8a/libassistant.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TylunasLi/fastllm/f3cfc63dc9efb995670461b6e0d64d2c55cfa0a3/example/Android/LLMAssistant/app/libs/arm64-v8a/libassistant.so -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/libs/armeabi-v7a/libassistant.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TylunasLi/fastllm/f3cfc63dc9efb995670461b6e0d64d2c55cfa0a3/example/Android/LLMAssistant/app/libs/armeabi-v7a/libassistant.so -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/proguard-rules.pro: -------------------------------------------------------------------------------- 1 | # Add project specific ProGuard rules here. 2 | # You can control the set of applied configuration files using the 3 | # proguardFiles setting in build.gradle. 4 | # 5 | # For more details, see 6 | # http://developer.android.com/guide/developing/tools/proguard.html 7 | 8 | # If your project uses WebView with JS, uncomment the following 9 | # and specify the fully qualified class name to the JavaScript interface 10 | # class: 11 | #-keepclassmembers class fqcn.of.javascript.interface.for.webview { 12 | # public *; 13 | #} 14 | 15 | # Uncomment this to preserve the line number information for 16 | # debugging stack traces. 17 | #-keepattributes SourceFile,LineNumberTable 18 | 19 | # If you keep the line number information, uncomment this to 20 | # hide the original source file name. 21 | #-renamesourcefileattribute SourceFile -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/release/app-arm64-v8a-release-unsigned.apk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TylunasLi/fastllm/f3cfc63dc9efb995670461b6e0d64d2c55cfa0a3/example/Android/LLMAssistant/app/release/app-arm64-v8a-release-unsigned.apk -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/release/app-armeabi-v7a-release-unsigned.apk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TylunasLi/fastllm/f3cfc63dc9efb995670461b6e0d64d2c55cfa0a3/example/Android/LLMAssistant/app/release/app-armeabi-v7a-release-unsigned.apk -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/release/app-universal-release-unsigned.apk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TylunasLi/fastllm/f3cfc63dc9efb995670461b6e0d64d2c55cfa0a3/example/Android/LLMAssistant/app/release/app-universal-release-unsigned.apk -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/release/app-x86-release-unsigned.apk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TylunasLi/fastllm/f3cfc63dc9efb995670461b6e0d64d2c55cfa0a3/example/Android/LLMAssistant/app/release/app-x86-release-unsigned.apk -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/androidTest/java/com/doujiao/xiaozhihuiassistant/ExampleInstrumentedTest.java: -------------------------------------------------------------------------------- 1 | package com.doujiao.xiaozhihuiassistant; 2 | 3 | import android.content.Context; 4 | import android.support.test.InstrumentationRegistry; 5 | import android.support.test.runner.AndroidJUnit4; 6 | 7 | import org.junit.Test; 8 | import org.junit.runner.RunWith; 9 | 10 | import static org.junit.Assert.*; 11 | 12 | /** 13 | * Instrumented test, which will execute on an Android device. 14 | * 15 | * @see Testing documentation 16 | */ 17 | @RunWith(AndroidJUnit4.class) 18 | public class ExampleInstrumentedTest { 19 | @Test 20 | public void useAppContext() { 21 | // Context of the app under test. 22 | Context appContext = InstrumentationRegistry.getInstrumentation().getTargetContext(); 23 | assertEquals("com.doujiao.xiaozhihuiassistant", appContext.getPackageName()); 24 | } 25 | } -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 16 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/cpp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # For more information about using CMake with Android Studio, read the 2 | # documentation: https://d.android.com/studio/projects/add-native-code.html 3 | 4 | # Sets the minimum version of CMake required to build the native library. 5 | 6 | cmake_minimum_required(VERSION 3.10.2) 7 | 8 | # Declares and names the project. 9 | 10 | project("assistant") 11 | set(CMAKE_BUILD_TYPE "Release") 12 | 13 | option(USE_CUDA "use cuda" OFF) 14 | 15 | option(PY_API "python api" OFF) 16 | 17 | #可以注释掉下面优化选项 18 | #add_definitions(${CMAKE_CXX_FLAGS} "${CMAKE_CXX_FLAGS} -march=armv8.2a+dotprod") 19 | # 20 | #file(GLOB_RECURSE NANODET_SOURCE ../../../../../../../src/*.cpp 21 | # ../../../../../../../src/devices/cpu/*.cpp 22 | # ../../../../../../../src/models/*.cpp) 23 | # 24 | #set(PROJECT_SOURCE 25 | # ${NANODET_SOURCE} 26 | # ) 27 | 28 | set(PROJECT_SOURCE 29 | ../../../../../../../src/fastllm.cpp 30 | ../../../../../../../src/device.cpp 31 | ../../../../../../../src/model.cpp 32 | ../../../../../../../src/executor.cpp 33 | ../../../../../../../src/devices/cpu/cpudevice.cpp 34 | ../../../../../../../src/devices/cpu/cpudevicebatch.cpp 35 | ../../../../../../../src/models/basellm.cpp 36 | ../../../../../../../src/models/chatglm.cpp 37 | ../../../../../../../src/models/moss.cpp 38 | ../../../../../../../src/models/llama.cpp 39 | ../../../../../../../src/models/qwen.cpp 40 | ../../../../../../../src/models/glm.cpp 41 | ../../../../../../../src/models/minicpm.cpp 42 | ) 43 | 44 | include_directories( 45 | ./ 46 | ../../../../../../../include 47 | ../../../../../../../include/models 48 | ../../../../../../../include/utils 49 | ../../../../../../../include/devices/cpu) 50 | 51 | add_library( # Sets the name of the library. 52 | assistant 53 | # Sets the library as a shared library. 54 | SHARED 55 | # Provides a relative path to your source file(s). 56 | ${PROJECT_SOURCE} LLMChat.cpp native-lib.cpp) 57 | 58 | # Searches for a specified prebuilt library and stores the path as a 59 | # variable. Because CMake includes system libraries in the search path by 60 | # default, you only need to specify the name of the public NDK library 61 | # you want to add. CMake verifies that the library exists before 62 | # completing its build. 63 | 64 | find_library( # Sets the name of the path variable. 65 | log-lib 66 | # Specifies the name of the NDK library that 67 | # you want CMake to locate. 68 | log) 69 | 70 | # Specifies libraries CMake should link to your target library. You 71 | # can link multiple libraries, such as libraries you define in this 72 | # build script, prebuilt third-party libraries, or system libraries. 73 | 74 | target_link_libraries( # Specifies the target library. 75 | assistant 76 | # Links the target library to the log library 77 | # included in the NDK. 78 | ${log-lib}) 79 | 80 | #add_executable(main main.cpp ../../../../../../../src/fastllm.cpp 81 | # ../../../../../../../src/chatglm.cpp 82 | # ../../../../../../../src/moss.cpp) -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/cpp/LLMChat.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "LLMChat.h" 6 | 7 | #include "model.h" 8 | //void(^ __nonnull RuntimeChat)(int index,const char* _Nonnull content) = NULL;//实时回调 9 | 10 | static int modeltype = 0; 11 | static char* modelpath = NULL; 12 | static std::unique_ptr chatGlm = NULL; 13 | static int sRound = 0; 14 | static std::string history; 15 | static RuntimeResultMobile g_callback = NULL; 16 | 17 | std::string initGptConf(const char* modelPath,int threads) { 18 | fastllm::SetThreads(threads); 19 | LOG_Debug("@@init llmpath:%s\n",modelPath); 20 | chatGlm = fastllm::CreateLLMModelFromFile(modelPath); 21 | if(chatGlm != NULL) 22 | { 23 | std::string modelName = chatGlm->model_type; 24 | LOG_Debug("@@model name:%s\n",modelName.c_str()); 25 | return modelName; 26 | } 27 | LOG_Debug("@@CreateLLMModelFromFile failed."); 28 | return ""; 29 | } 30 | 31 | int chat(const char* prompt, RuntimeResultMobile chatCallback) { 32 | std::string ret = ""; 33 | g_callback = chatCallback; 34 | LOG_Debug("@@init llm:type:%d,prompt:%s\n",modeltype,prompt); 35 | std::string input(prompt); 36 | 37 | if (input == "reset") { 38 | history = ""; 39 | sRound = 0; 40 | g_callback(0,"Done!"); 41 | g_callback(-1,""); 42 | return 0; 43 | } 44 | 45 | ret = chatGlm->Response(chatGlm->MakeInput(history, sRound, input), [](int index, const char* content) { 46 | g_callback(index,content); 47 | }); 48 | history = chatGlm->MakeHistory(history, sRound, input, ret); 49 | sRound++; 50 | 51 | long len = ret.length(); 52 | return len; 53 | } 54 | 55 | void uninitLLM() 56 | { 57 | } 58 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/cpp/LLMChat.h: -------------------------------------------------------------------------------- 1 | // 2 | // LLMChat.h 3 | // LLMChat 4 | // 5 | // Created by 胡其斌 on 2023/5/18. 6 | // 7 | 8 | #ifdef __cplusplus 9 | extern "C" { 10 | #endif 11 | 12 | #include 13 | #define LOG_Debug(...) __android_log_print(ANDROID_LOG_DEBUG, "Assistant", __VA_ARGS__) 14 | 15 | typedef void(* RuntimeResultMobile)(int index,const char* content); 16 | 17 | std::string initGptConf(const char* modelPath,int threads); 18 | int chat(const char* prompt, RuntimeResultMobile chatCallback); 19 | void uninitLLM(); 20 | 21 | #ifdef __cplusplus 22 | } 23 | #endif 24 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/cpp/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "factoryllm.h" 7 | 8 | static factoryllm fllm; 9 | static int modeltype = 0; 10 | static char* modelpath = NULL; 11 | static fastllm::basellm* chatGlm = fllm.createllm(LLM_TYPE_CHATGLM); 12 | static fastllm::basellm* moss = fllm.createllm(LLM_TYPE_MOSS); 13 | static int sRound = 0; 14 | static std::string history; 15 | 16 | struct RunConfig { 17 | int model = LLM_TYPE_CHATGLM; // 模型类型, LLM_TYPE_CHATGLM:chatglm, LLM_TYPE_MOSS:moss 18 | std::string path = "/sdcard/chatglm-6b-int4.bin"; // 模型文件路径 19 | int threads = 6; // 使用的线程数 20 | }; 21 | 22 | static struct option long_options[] = { 23 | {"help", no_argument, nullptr, 'h'}, 24 | {"model", required_argument, nullptr, 'm'}, 25 | {"path", required_argument, nullptr, 'p'}, 26 | {"threads", required_argument, nullptr, 't'}, 27 | {nullptr, 0, nullptr, 0}, 28 | }; 29 | 30 | void Usage() { 31 | std::cout << "Usage:" << std::endl; 32 | std::cout << "[-h|--help]: 显示帮助" << std::endl; 33 | std::cout << "<-m|--model> : 模型类型，默认为chatglm, 可以设置为0, moss:1" << std::endl; 34 | std::cout << "<-p|--path> : 模型文件的路径" << std::endl; 35 | std::cout << "<-t|--threads> : 使用的线程数量" << std::endl; 36 | } 37 | 38 | void ParseArgs(int argc, char **argv, RunConfig &config) { 39 | int opt; 40 | int option_index = 0; 41 | const char *opt_string = "h:m:p:t:"; 42 | 43 | while ((opt = getopt_long_only(argc, argv, opt_string, long_options, &option_index)) != -1) { 44 | switch (opt) { 45 | case 'h': 46 | Usage(); 47 | exit (0); 48 | case 'm': 49 | config.model = atoi(argv[optind - 1]); 50 | break; 51 | case 'p': 52 | config.path = argv[optind - 1]; 53 | break; 54 | case 't': 55 | config.threads = atoi(argv[optind - 1]); 56 | break; 57 | default: 58 | Usage(); 59 | exit (-1); 60 | } 61 | } 62 | } 63 | 64 | int initLLMConf(int model,const char* modelPath,int threads) { 65 | fastllm::SetThreads(threads); 66 | modeltype = model; 67 | // printf("@@init llm:type:%d,path:%s\n",model,modelPath); 68 | if (modeltype == 0) { 69 | chatGlm->LoadFromFile(modelPath); 70 | } 71 | if (modeltype == 1) { 72 | moss->LoadFromFile(modelPath); 73 | } 74 | return 0; 75 | } 76 | 77 | int chat(const char* prompt) { 78 | std::string ret = ""; 79 | //printf("@@init llm:type:%d,prompt:%s\n",modeltype,prompt); 80 | std::string input(prompt); 81 | if (modeltype == 0) { 82 | if (input == "reset") { 83 | history = ""; 84 | sRound = 0; 85 | return 0; 86 | } 87 | history += ("[Round " + std::to_string(sRound++) + "]\n问：" + input); 88 | auto prompt = sRound > 1 ? history : input; 89 | ret = chatGlm->Response(prompt,[](int index,const char* content){ 90 | 91 | if(index == 0) { 92 | printf("ChatGLM:"); 93 | } 94 | printf("%s", content); 95 | if (index == -1) { 96 | printf("\n"); 97 | } 98 | 99 | }); 100 | history += ("\n答：" + ret + "\n"); 101 | } 102 | 103 | if (modeltype == 1) { 104 | auto prompt = "You are an AI assistant whose name is MOSS. <|Human|>: " + input + ""; 105 | ret = moss->Response(prompt,[](int index,const char* content){ 106 | if(index == 0) { 107 | printf("MOSS:"); 108 | } 109 | printf("%s", content); 110 | if (index == -1) { 111 | printf("\n"); 112 | } 113 | }); 114 | } 115 | long len = ret.length(); 116 | return len; 117 | } 118 | 119 | void uninitLLM() 120 | { 121 | if (chatGlm) 122 | { 123 | delete chatGlm; 124 | chatGlm = NULL; 125 | } 126 | if (moss) 127 | { 128 | delete moss; 129 | moss = NULL; 130 | } 131 | } 132 | 133 | 134 | int main(int argc, char **argv) { 135 | RunConfig config; 136 | ParseArgs(argc, argv, config); 137 | 138 | initLLMConf(config.model, config.path.c_str(), config.threads); 139 | 140 | if (config.model == LLM_TYPE_MOSS) { 141 | 142 | while (true) { 143 | printf("用户: "); 144 | std::string input; 145 | std::getline(std::cin, input); 146 | if (input == "stop") { 147 | break; 148 | } 149 | chat(input.c_str()); 150 | } 151 | } else if (config.model == LLM_TYPE_CHATGLM) { 152 | while (true) { 153 | printf("用户: "); 154 | std::string input; 155 | std::getline(std::cin, input); 156 | if (input == "stop") { 157 | break; 158 | } 159 | chat(input.c_str()); 160 | } 161 | 162 | } else { 163 | Usage(); 164 | exit(-1); 165 | } 166 | 167 | return 0; 168 | } 169 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/cpp/native-lib.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "LLMChat.h" 4 | 5 | 6 | JavaVM *g_javaVM = NULL; 7 | jobject g_obj; 8 | 9 | void initGvm(JNIEnv *env,jobject thiz) { 10 | if(g_javaVM == NULL) { 11 | env->GetJavaVM(&g_javaVM); 12 | g_obj = env->NewGlobalRef(thiz); 13 | } 14 | } 15 | 16 | void chatCb(int index,const char* content) { 17 | JNIEnv *env = NULL; 18 | int mNeedDetach = 0; 19 | //获取当前native线程是否有没有被附加到jvm环境中 20 | int getEnvStat = g_javaVM->GetEnv((void **)&env,JNI_VERSION_1_6); 21 | if (getEnvStat == JNI_EDETACHED) { 22 | //如果没有， 主动附加到jvm环境中，获取到env 23 | if (g_javaVM->AttachCurrentThread( &env, NULL) != 0) { 24 | LOG_Debug("Unable to AttachCurrentThread"); 25 | return; 26 | } 27 | mNeedDetach = 1; 28 | } 29 | //通过全局变量g_obj 获取到要回调的类 30 | jclass javaClass = env->GetObjectClass(g_obj);//env->FindClass("com/doujiao/core/AssistantCore");// 31 | if (javaClass == 0) { 32 | LOG_Debug("Unable to find class"); 33 | if(mNeedDetach) { 34 | g_javaVM->DetachCurrentThread(); 35 | } 36 | return; 37 | } 38 | jmethodID jgetDBpathMethod = env->GetMethodID(javaClass, "reportChat", "(Ljava/lang/String;I)V"); 39 | if (jgetDBpathMethod == NULL) { 40 | LOG_Debug("Unable to find method:jgetDBpathMethod"); 41 | return; 42 | } 43 | jobject bb = env->NewDirectByteBuffer((void *) content, strlen(content)); 44 | jclass cls_Charset = env->FindClass("java/nio/charset/Charset"); 45 | jmethodID mid_Charset_forName = env->GetStaticMethodID(cls_Charset, "forName", "(Ljava/lang/String;)Ljava/nio/charset/Charset;"); 46 | jobject charset = env->CallStaticObjectMethod(cls_Charset, mid_Charset_forName, env->NewStringUTF("UTF-8")); 47 | 48 | jmethodID mid_Charset_decode = env->GetMethodID(cls_Charset, "decode", "(Ljava/nio/ByteBuffer;)Ljava/nio/CharBuffer;"); 49 | jobject cb = env->CallObjectMethod(charset, mid_Charset_decode, bb); 50 | env->DeleteLocalRef(bb); 51 | 52 | jclass cls_CharBuffer = env->FindClass("java/nio/CharBuffer"); 53 | jmethodID mid_CharBuffer_toString = env->GetMethodID(cls_CharBuffer, "toString", "()Ljava/lang/String;"); 54 | jstring str = static_cast(env->CallObjectMethod(cb, mid_CharBuffer_toString)); 55 | env->CallVoidMethod(g_obj, jgetDBpathMethod,str,index); 56 | env->DeleteLocalRef(javaClass); 57 | //释放当前线程 58 | if(mNeedDetach) { 59 | g_javaVM->DetachCurrentThread(); 60 | } 61 | env = NULL; 62 | } 63 | 64 | extern "C" JNIEXPORT jstring JNICALL 65 | Java_com_doujiao_core_AssistantCore_initLLMConfig( 66 | JNIEnv* env, 67 | jobject obj, 68 | jstring modelpath, 69 | jint threads) { 70 | initGvm(env,obj); 71 | const char *path = env->GetStringUTFChars(modelpath, NULL); 72 | std::string ret = initGptConf(path,threads); 73 | LOG_Debug("@@@initLLMConfig:%s",ret.c_str()); 74 | env->ReleaseStringUTFChars( modelpath, path); 75 | return env->NewStringUTF(ret.c_str()); 76 | } 77 | 78 | extern "C" JNIEXPORT jint JNICALL 79 | Java_com_doujiao_core_AssistantCore_chat( 80 | JNIEnv* env, 81 | jobject obj, 82 | jstring prompt) { 83 | initGvm(env,obj); 84 | const char *question = env->GetStringUTFChars(prompt, NULL); 85 | chat(question,[](int index,const char* content){ 86 | chatCb(index,content); 87 | }); 88 | // chatCb(1,"content"); 89 | return 0; 90 | } 91 | 92 | extern "C" JNIEXPORT void JNICALL 93 | Java_com_doujiao_core_AssistantCore_uninitLLM( 94 | JNIEnv* env, 95 | jobject /* this */) { 96 | uninitLLM(); 97 | } -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/java/com/doujiao/core/AssistantCore.java: -------------------------------------------------------------------------------- 1 | package com.doujiao.core; 2 | 3 | import android.support.annotation.Keep; 4 | import android.util.Log; 5 | 6 | public class AssistantCore { 7 | 8 | private static AssistantCore instance = null; 9 | private static runtimeResult mRuntimeRes = null; 10 | 11 | static { 12 | System.loadLibrary("assistant"); 13 | } 14 | 15 | /*静态对象*/ 16 | public static AssistantCore getInstance(){ 17 | if(instance == null){ 18 | instance = new AssistantCore(); 19 | } 20 | 21 | return instance; 22 | } 23 | 24 | public String initLLM(String path,runtimeResult callback) { 25 | mRuntimeRes = callback; 26 | return initLLMConfig(path,8); 27 | } 28 | 29 | @Keep 30 | public void reportChat(String content,int index) { 31 | Log.d("@@@","recv:"+content+",index:"+index); 32 | if (mRuntimeRes != null) { 33 | mRuntimeRes.callbackResult(index,content); 34 | } 35 | } 36 | 37 | public interface runtimeResult { 38 | void callbackResult(int index,String content); 39 | } 40 | 41 | private native String initLLMConfig(String path,int threads); 42 | public native int chat(String prompt); 43 | public native int uninitLLM(); 44 | } 45 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/java/com/doujiao/xiaozhihuiassistant/ChatMessage.java: -------------------------------------------------------------------------------- 1 | package com.doujiao.xiaozhihuiassistant; 2 | 3 | /** 4 | * Created by chenpengfei on 2016/10/27. 5 | */ 6 | public class ChatMessage { 7 | 8 | private String content; 9 | 10 | private int type; 11 | 12 | public ChatMessage(String content, int type) { 13 | this.content = content; 14 | this.type = type; 15 | } 16 | 17 | public ChatMessage(String content) { 18 | this(content, 1); 19 | } 20 | 21 | 22 | public String getContent() { 23 | return content; 24 | } 25 | 26 | public void setContent(String content) { 27 | this.content = content; 28 | } 29 | 30 | public int getType() { 31 | return type; 32 | } 33 | 34 | public void setType(int type) { 35 | this.type = type; 36 | } 37 | 38 | 39 | } 40 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/java/com/doujiao/xiaozhihuiassistant/adapter/BaseViewHolder.java: -------------------------------------------------------------------------------- 1 | package com.doujiao.xiaozhihuiassistant.adapter; 2 | 3 | import android.support.v7.widget.RecyclerView; 4 | import android.view.View; 5 | 6 | /** 7 | * Created by chenpengfei on 2016/10/27. 8 | */ 9 | public class BaseViewHolder extends RecyclerView.ViewHolder { 10 | 11 | private View iv; 12 | 13 | public BaseViewHolder(View itemView) { 14 | super(itemView); 15 | iv = itemView; 16 | } 17 | 18 | public View findViewById(int id) { 19 | return iv.findViewById(id); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/java/com/doujiao/xiaozhihuiassistant/adapter/MyAdapter.java: -------------------------------------------------------------------------------- 1 | package com.doujiao.xiaozhihuiassistant.adapter; 2 | 3 | import android.app.Activity; 4 | import android.support.v7.widget.RecyclerView; 5 | import android.view.View; 6 | import android.view.ViewGroup; 7 | import android.widget.TextView; 8 | import android.widget.Toast; 9 | 10 | 11 | import com.doujiao.xiaozhihuiassistant.ChatMessage; 12 | import com.doujiao.xiaozhihuiassistant.R; 13 | import com.doujiao.xiaozhihuiassistant.utils.StatusBarUtils; 14 | import com.doujiao.xiaozhihuiassistant.widget.ChatPromptViewManager; 15 | import com.doujiao.xiaozhihuiassistant.widget.Location; 16 | import com.doujiao.xiaozhihuiassistant.widget.PromptViewHelper; 17 | 18 | import java.util.List; 19 | 20 | public class MyAdapter extends RecyclerView.Adapter { 21 | 22 | private List mChatMessageList = null; 23 | private Activity mActivity; 24 | 25 | public MyAdapter(Activity activity) { 26 | mActivity = activity; 27 | } 28 | 29 | public void setMessages(List chatMessageList) { 30 | mChatMessageList = chatMessageList; 31 | } 32 | 33 | @Override 34 | public BaseViewHolder onCreateViewHolder(ViewGroup parent, int viewType) { 35 | if(viewType == 1) { 36 | return new LeftViewHolder(View.inflate(mActivity, R.layout.activity_item_left, null)); 37 | } else { 38 | return new RightViewHolder(View.inflate(mActivity, R.layout.activity_item_right, null)); 39 | } 40 | } 41 | 42 | @Override 43 | public int getItemCount() { 44 | return mChatMessageList.size(); 45 | } 46 | 47 | @Override 48 | public int getItemViewType(int position) { 49 | return mChatMessageList.get(position).getType(); 50 | } 51 | 52 | @Override 53 | public void onBindViewHolder(BaseViewHolder holder, int position) { 54 | PromptViewHelper pvHelper = new PromptViewHelper(mActivity); 55 | ChatMessage chatMessage = mChatMessageList.get(position); 56 | if(holder instanceof LeftViewHolder) { 57 | LeftViewHolder leftViewHolder = (LeftViewHolder) holder; 58 | leftViewHolder.tv.setText(chatMessage.getContent()); 59 | pvHelper.setPromptViewManager(new ChatPromptViewManager(mActivity)); 60 | } 61 | if(holder instanceof RightViewHolder) { 62 | RightViewHolder rightViewHolder = (RightViewHolder) holder; 63 | rightViewHolder.tv.setText(chatMessage.getContent()); 64 | pvHelper.setPromptViewManager(new ChatPromptViewManager(mActivity, Location.TOP_RIGHT)); 65 | } 66 | pvHelper.addPrompt(holder.itemView.findViewById(R.id.textview_content)); 67 | pvHelper.setOnItemClickListener(new PromptViewHelper.OnItemClickListener() { 68 | @Override 69 | public void onItemClick(int position) { 70 | String str = ""; 71 | switch (position) { 72 | case 0: 73 | str = "已复制到剪贴板!"; 74 | TextView tv = holder.itemView.findViewById(R.id.textview_content); 75 | StatusBarUtils.copyStr2ClibBoard(mActivity.getApplicationContext(), tv.getText().toString()); 76 | break; 77 | } 78 | Toast.makeText(mActivity, str, Toast.LENGTH_SHORT).show(); 79 | } 80 | }); 81 | } 82 | 83 | class LeftViewHolder extends BaseViewHolder { 84 | 85 | TextView tv; 86 | 87 | public LeftViewHolder(View view) { 88 | super(view); 89 | tv = (TextView) findViewById(R.id.textview_content); 90 | } 91 | } 92 | 93 | class RightViewHolder extends BaseViewHolder { 94 | 95 | TextView tv; 96 | 97 | public RightViewHolder(View view) { 98 | super(view); 99 | tv = (TextView) findViewById(R.id.textview_content); 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/java/com/doujiao/xiaozhihuiassistant/utils/PrefUtil.java: -------------------------------------------------------------------------------- 1 | package com.doujiao.xiaozhihuiassistant.utils; 2 | 3 | import android.content.Context; 4 | import android.content.SharedPreferences; 5 | 6 | public class PrefUtil { 7 | private static final String SF_NAME = "com.doujiao.llm.config"; 8 | private static final String MOLE_PATH = "llm_path"; 9 | private static SharedPreferences mPref; 10 | 11 | public static void initPref(Context context) { 12 | if (mPref == null) { 13 | mPref = context.getSharedPreferences(SF_NAME, Context.MODE_PRIVATE); 14 | } 15 | } 16 | 17 | public static void setModelPath(String path) { 18 | if (mPref != null) { 19 | mPref.edit().putString(MOLE_PATH,path).apply(); 20 | } 21 | } 22 | 23 | public static String getModelPath() { 24 | if (mPref != null) { 25 | return mPref.getString(MOLE_PATH,""); 26 | } 27 | return ""; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/java/com/doujiao/xiaozhihuiassistant/utils/StatusBarUtils.java: -------------------------------------------------------------------------------- 1 | package com.doujiao.xiaozhihuiassistant.utils; 2 | 3 | import android.content.ClipData; 4 | import android.content.ClipboardManager; 5 | import android.content.Context; 6 | import android.graphics.Color; 7 | import android.os.Build; 8 | import android.support.v7.app.ActionBar; 9 | import android.support.v7.app.AppCompatActivity; 10 | import android.view.View; 11 | import android.view.Window; 12 | import android.view.WindowManager; 13 | 14 | public class StatusBarUtils { 15 | public static void setTranslucentStatus(AppCompatActivity activity) { 16 | if (Build.VERSION.SDK_INT >= 21) { 17 | View decorView = activity.getWindow().getDecorView(); 18 | int option = View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN 19 | | View.SYSTEM_UI_FLAG_LAYOUT_STABLE; 20 | decorView.setSystemUiVisibility(option); 21 | activity.getWindow().setStatusBarColor(Color.TRANSPARENT); 22 | } 23 | ActionBar actionBar = activity.getSupportActionBar(); 24 | actionBar.hide(); 25 | } 26 | 27 | public static void hideStatusBar(Window window, boolean darkText) { 28 | window.clearFlags(WindowManager.LayoutParams.FLAG_TRANSLUCENT_STATUS); 29 | window.addFlags(WindowManager.LayoutParams.FLAG_DRAWS_SYSTEM_BAR_BACKGROUNDS); 30 | window.setStatusBarColor(Color.TRANSPARENT); 31 | 32 | int flag = View.SYSTEM_UI_FLAG_LAYOUT_STABLE; 33 | if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && darkText) { 34 | flag = View.SYSTEM_UI_FLAG_LIGHT_STATUS_BAR; 35 | } 36 | 37 | window.getDecorView().setSystemUiVisibility(flag | 38 | View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN); 39 | } 40 | 41 | public static boolean copyStr2ClibBoard(Context context, String copyStr) { 42 | try { 43 | //获取剪贴板管理器 44 | ClipboardManager cm = (ClipboardManager) context.getSystemService(Context.CLIPBOARD_SERVICE); 45 | // 创建普通字符型ClipData 46 | ClipData mClipData = ClipData.newPlainText("Label", copyStr); 47 | // 将ClipData内容放到系统剪贴板里。 48 | cm.setPrimaryClip(mClipData); 49 | return true; 50 | } catch (Exception e) { 51 | return false; 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/java/com/doujiao/xiaozhihuiassistant/widget/ChatPromptViewManager.java: -------------------------------------------------------------------------------- 1 | package com.doujiao.xiaozhihuiassistant.widget; 2 | 3 | import android.app.Activity; 4 | import android.view.View; 5 | 6 | /** 7 | * Created by chenpengfei on 2016/11/2. 8 | */ 9 | public class ChatPromptViewManager extends PromptViewHelper.PromptViewManager { 10 | 11 | public ChatPromptViewManager(Activity activity, String[] dataArray, Location location) { 12 | super(activity, dataArray, location); 13 | } 14 | 15 | public ChatPromptViewManager(Activity activity) { 16 | this(activity, new String[]{"复制"}, Location.TOP_LEFT); 17 | } 18 | 19 | public ChatPromptViewManager(Activity activity, Location location) { 20 | this(activity, new String[]{"复制"}, location); 21 | } 22 | 23 | 24 | @Override 25 | public View inflateView() { 26 | return new PromptView(activity); 27 | } 28 | 29 | @Override 30 | public void bindData(View view, String[] dataArray) { 31 | if(view instanceof PromptView) { 32 | PromptView promptView = (PromptView) view; 33 | promptView.setContentArray(dataArray); 34 | promptView.setOnItemClickListener(new PromptView.OnItemClickListener() { 35 | @Override 36 | public void onItemClick(int position) { 37 | if(onItemClickListener != null) onItemClickListener.onItemClick(position); 38 | } 39 | }); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/java/com/doujiao/xiaozhihuiassistant/widget/Location.java: -------------------------------------------------------------------------------- 1 | package com.doujiao.xiaozhihuiassistant.widget; 2 | 3 | import com.doujiao.xiaozhihuiassistant.widget.location.BottomCenterLocation; 4 | import com.doujiao.xiaozhihuiassistant.widget.location.ICalculateLocation; 5 | import com.doujiao.xiaozhihuiassistant.widget.location.TopCenterLocation; 6 | import com.doujiao.xiaozhihuiassistant.widget.location.TopLeftLocation; 7 | import com.doujiao.xiaozhihuiassistant.widget.location.TopRightLocation; 8 | 9 | /** 10 | * Created by chenpengfei on 2016/11/2. 11 | */ 12 | public enum Location { 13 | 14 | TOP_LEFT(1), 15 | TOP_CENTER(2), 16 | TOP_RIGHT(3), 17 | BOTTOM_LEFT(4), 18 | BOTTOM_CENTER(5), 19 | BOTTOM_RIGHT(6); 20 | 21 | ICalculateLocation calculateLocation; 22 | 23 | private Location(int type) { 24 | switch (type) { 25 | case 1: 26 | calculateLocation = new TopLeftLocation(); 27 | break; 28 | case 2: 29 | calculateLocation = new TopCenterLocation(); 30 | break; 31 | case 3: 32 | calculateLocation = new TopRightLocation(); 33 | break; 34 | case 4: 35 | calculateLocation = new TopLeftLocation(); 36 | break; 37 | case 5: 38 | calculateLocation = new BottomCenterLocation(); 39 | break; 40 | case 6: 41 | calculateLocation = new TopLeftLocation(); 42 | break; 43 | } 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/java/com/doujiao/xiaozhihuiassistant/widget/PromptViewHelper.java: -------------------------------------------------------------------------------- 1 | package com.doujiao.xiaozhihuiassistant.widget; 2 | 3 | import android.app.Activity; 4 | import android.graphics.Color; 5 | import android.graphics.drawable.ColorDrawable; 6 | import android.view.Gravity; 7 | import android.view.View; 8 | import android.view.ViewGroup; 9 | import android.view.ViewTreeObserver; 10 | import android.widget.PopupWindow; 11 | 12 | /** 13 | * Created by chenpengfei on 2016/11/2. 14 | */ 15 | public class PromptViewHelper { 16 | 17 | private PromptViewManager promptViewManager; 18 | private Activity activity; 19 | private PopupWindow popupWindow; 20 | private boolean isShow; 21 | private OnItemClickListener onItemClickListener; 22 | 23 | public PromptViewHelper(Activity activity) { 24 | this.activity = activity; 25 | } 26 | 27 | public void setPromptViewManager(PromptViewManager promptViewManager) { 28 | this.promptViewManager = promptViewManager; 29 | this.promptViewManager.setOnItemClickListener(new OnItemClickListener() { 30 | @Override 31 | public void onItemClick(int position) { 32 | if(onItemClickListener != null && popupWindow != null) { 33 | onItemClickListener.onItemClick(position); 34 | popupWindow.dismiss(); 35 | } 36 | } 37 | }); 38 | } 39 | 40 | public void addPrompt(View srcView) { 41 | srcView.setOnLongClickListener(new View.OnLongClickListener() { 42 | @Override 43 | public boolean onLongClick(View v) { 44 | createPrompt(v); 45 | return true; 46 | } 47 | }); 48 | } 49 | 50 | private void createPrompt(final View srcView) { 51 | final View promptView = promptViewManager.getPromptView(); 52 | if(popupWindow == null) 53 | popupWindow = new PopupWindow(activity); 54 | popupWindow.setWindowLayoutMode(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT); 55 | popupWindow.setTouchable(true); 56 | popupWindow.setOutsideTouchable(true); 57 | popupWindow.setBackgroundDrawable( new ColorDrawable(Color.TRANSPARENT)); 58 | popupWindow.setContentView(promptView); 59 | final int[] location = new int[2]; 60 | promptView.getViewTreeObserver().addOnGlobalLayoutListener(new ViewTreeObserver.OnGlobalLayoutListener() { 61 | @Override 62 | public void onGlobalLayout() { 63 | if(!isShow && popupWindow.isShowing()) { 64 | popupWindow.dismiss(); 65 | show(srcView, promptView, location); 66 | isShow = true; 67 | } 68 | } 69 | }); 70 | srcView.getLocationOnScreen(location); 71 | show(srcView, promptView, location); 72 | } 73 | 74 | public void show(View srcView, View promptView, int[] srcViewLocation) { 75 | int[] xy = promptViewManager.getLocation().calculateLocation.calculate(srcViewLocation, srcView, promptView); 76 | popupWindow.showAtLocation(srcView, Gravity.NO_GRAVITY, xy[0], xy[1]); 77 | } 78 | 79 | 80 | public static abstract class PromptViewManager { 81 | 82 | private View promptView; 83 | protected Activity activity; 84 | private String[] dataArray; 85 | private Location location; 86 | public OnItemClickListener onItemClickListener; 87 | 88 | public PromptViewManager(Activity activity, String[] dataArray, Location location) { 89 | this.activity = activity; 90 | this.dataArray = dataArray; 91 | this.location = location; 92 | init(); 93 | } 94 | 95 | public void setOnItemClickListener(OnItemClickListener onItemClickListener) { 96 | this.onItemClickListener = onItemClickListener; 97 | } 98 | 99 | public void init() { 100 | promptView = inflateView(); 101 | bindData(promptView, dataArray); 102 | } 103 | 104 | public abstract View inflateView(); 105 | 106 | public abstract void bindData(View view, String[] dataArray); 107 | 108 | public View getPromptView() { 109 | return promptView; 110 | } 111 | 112 | public Location getLocation() { 113 | return location; 114 | } 115 | } 116 | 117 | public void setOnItemClickListener(OnItemClickListener onItemClickListener) { 118 | this.onItemClickListener = onItemClickListener; 119 | } 120 | 121 | public interface OnItemClickListener { 122 | void onItemClick(int position); 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/java/com/doujiao/xiaozhihuiassistant/widget/location/BottomCenterLocation.java: -------------------------------------------------------------------------------- 1 | package com.doujiao.xiaozhihuiassistant.widget.location; 2 | 3 | import android.view.View; 4 | 5 | /** 6 | * Created by chenpengfei on 2016/11/2. 7 | */ 8 | public class BottomCenterLocation implements ICalculateLocation { 9 | 10 | @Override 11 | public int[] calculate(int[] srcViewLocation, View srcView, View promptView) { 12 | int[] location = new int[2]; 13 | int offset = (promptView.getWidth() - srcView.getWidth()) / 2; 14 | location[0] = srcViewLocation[0] - offset; 15 | location[1] = srcViewLocation[1] + promptView.getHeight(); 16 | return location; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/java/com/doujiao/xiaozhihuiassistant/widget/location/ICalculateLocation.java: -------------------------------------------------------------------------------- 1 | package com.doujiao.xiaozhihuiassistant.widget.location; 2 | 3 | import android.view.View; 4 | 5 | /** 6 | * Created by chenpengfei on 2016/11/2. 7 | */ 8 | public interface ICalculateLocation { 9 | 10 | int[] calculate(int[] srcViewLocation, View srcView, View promptView); 11 | 12 | } 13 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/java/com/doujiao/xiaozhihuiassistant/widget/location/TopCenterLocation.java: -------------------------------------------------------------------------------- 1 | package com.doujiao.xiaozhihuiassistant.widget.location; 2 | 3 | import android.view.View; 4 | 5 | /** 6 | * Created by chenpengfei on 2016/11/2. 7 | */ 8 | public class TopCenterLocation implements ICalculateLocation { 9 | 10 | @Override 11 | public int[] calculate(int[] srcViewLocation, View srcView, View promptView) { 12 | int[] location = new int[2]; 13 | int offset = (promptView.getWidth() - srcView.getWidth()) / 2; 14 | location[0] = srcViewLocation[0] - offset; 15 | location[1] = srcViewLocation[1] - promptView.getHeight(); 16 | return location; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/java/com/doujiao/xiaozhihuiassistant/widget/location/TopLeftLocation.java: -------------------------------------------------------------------------------- 1 | package com.doujiao.xiaozhihuiassistant.widget.location; 2 | 3 | import android.view.View; 4 | 5 | /** 6 | * Created by chenpengfei on 2016/11/2. 7 | */ 8 | public class TopLeftLocation implements ICalculateLocation { 9 | 10 | @Override 11 | public int[] calculate(int[] srcViewLocation, View srcView, View promptView) { 12 | int[] location = new int[2]; 13 | location[0] = srcViewLocation[0]; 14 | location[1] = srcViewLocation[1] - promptView.getHeight(); 15 | return location; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/java/com/doujiao/xiaozhihuiassistant/widget/location/TopRightLocation.java: -------------------------------------------------------------------------------- 1 | package com.doujiao.xiaozhihuiassistant.widget.location; 2 | 3 | import android.view.View; 4 | 5 | /** 6 | * Created by chenpengfei on 2016/11/2. 7 | */ 8 | public class TopRightLocation implements ICalculateLocation { 9 | 10 | @Override 11 | public int[] calculate(int[] srcViewLocation, View srcView, View promptView) { 12 | int[] location = new int[2]; 13 | int offset = promptView.getWidth() - srcView.getWidth(); 14 | location[0] = srcViewLocation[0] - offset; 15 | location[1] = srcViewLocation[1] - promptView.getHeight(); 16 | return location; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/res/drawable-v24/ic_launcher_foreground.xml: -------------------------------------------------------------------------------- 1 | 7 | 8 | 9 | 15 | 18 | 21 | 22 | 23 | 24 | 30 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/res/drawable/btnbg.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 7 | 8 | 9 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/res/drawable/editbg.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 7 | 8 | 9 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/res/layout/activity_item_left.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 11 | 12 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/res/layout/activity_item_right.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 13 | 14 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /example/Android/LLMAssistant/app/src/main/res/layout/activity_main.xml: -------------------------------------------------------------------------------- 1 | 2 | 9 | 10 | 17 | 21 | 26 |