├── make_ndk_project.bat ├── all_build.bat ├── jnihead.bat ├── .gitmodules ├── cmake ├── config.cmake.in ├── Modules │ ├── FindFse.cmake │ ├── FindCassdk.cmake │ └── utils.cmake └── custom_feature_compare.h.in ├── jnihead.sh ├── all_build.sh ├── test ├── test.cpp ├── test_c.c ├── make_msvc_cassdk_project.bat ├── make_gcc_project.sh └── CMakeLists.txt ├── include └── cassdk_ndk_feacomp.h ├── jni ├── jni_mgr.h ├── jni_mgr.cpp ├── JNIContext.h ├── CMakeLists.txt ├── net_gdface_sdk_fse_FseJniBridge.h ├── BeanUtilits.h ├── net_gdface_sdk_fse_FseJniBridge.cpp └── BeanUtilits.cpp ├── UNINSTALL.bat.in ├── INSTALL.bat.in ├── feature_se ├── MiniThreadPool.h ├── CodeManagerFactory.cpp ├── CodeManagerFactory.h ├── MD5SetImpl.h ├── config.h ├── feature_compare_cas.h ├── TopKCodeBean.cpp ├── ICodeManager.cpp ├── cross_define.h ├── TopKCodeBean.h ├── hashset_cl.h ├── feature_compare.h ├── hashtable_core.h ├── HashMapCl.h ├── OverTableManager.cpp ├── hashmap_abstract.h ├── HashMapCl.cpp ├── ICodeManager.h ├── feature_data_types.h ├── FeatureCompareSpeedTest.cpp ├── feature_se.h ├── ICodeManagerCPU.h ├── hash_fun.h ├── codemgr.h ├── OverTableManager.h ├── ICodeManagerCPU.cpp ├── feature_se.cpp ├── dot_product.h ├── CMakeLists.txt ├── FeatureCompareVeracityTest.cpp ├── topk_base.h └── hashtable_abstract.h ├── make_gcc_project.sh ├── UNINSTALL.sh.in ├── .gitignore ├── make_msvc_cassdk_project.bat ├── make_ndk_project.sh ├── INSTALL.sh.in ├── LICENSE ├── make_cassdk_ndk_project.bat ├── make_mtfsdk_ndk_project.bat ├── make_msvc_custom_project.bat ├── CMakeLists.txt └── README.md /make_ndk_project.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10km/feature_se/HEAD/make_ndk_project.bat -------------------------------------------------------------------------------- /all_build.bat: -------------------------------------------------------------------------------- 1 | set sh_folder=%~dp0 2 | pushd %sh_folder% 3 | if exist release rmdir release /s/q 4 | call build debug 5 | call build 6 | popd -------------------------------------------------------------------------------- /jnihead.bat: -------------------------------------------------------------------------------- 1 | ECHO OFF 2 | pushd %~sdp0 3 | javah -v -classpath ..\faceapi\faceapi-base\target\classes -d jni net.gdface.sdk.fse.FseJniBridge 4 | popd -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "common_source_cpp"] 2 | path = common_source_cpp 3 | url = https://l0km:gyd86471910@gitee.com/l0km/common_source_cpp.git 4 | -------------------------------------------------------------------------------- /cmake/config.cmake.in: -------------------------------------------------------------------------------- 1 | @PACKAGE_INIT@ 2 | 3 | include (CMakeFindDependencyMacro) 4 | 5 | @fse_DEPENDENCY@ 6 | 7 | include ("${CMAKE_CURRENT_LIST_DIR}/fse-targets.cmake") -------------------------------------------------------------------------------- /jnihead.sh: -------------------------------------------------------------------------------- 1 | src_path=`cd "$(dirname $0)";pwd` 2 | javah -v -classpath $src_path/../faceapi/faceapi-base/target/classes -d $src_path/jni net.gdface.sdk.fse.FseJniBridge 3 | 4 | -------------------------------------------------------------------------------- /all_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sh_folder=$(dirname $(readlink -f $0)) 3 | pushd $sh_folder 4 | [ -d release ] && rm -fr release 5 | ./build.sh debug 6 | ./build.sh release 7 | popd 8 | -------------------------------------------------------------------------------- /test/test.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "feature_se/feature_se.h" 3 | int main() { 4 | fse_init(16777216L,0.75f,1024L); 5 | face_code code; 6 | code_bean out[20]; 7 | fse_searchCode(&code, 0.75, 20, out, NULL, 0); 8 | code_bean added; 9 | fse_addFeature(&added); 10 | MD5 md5; 11 | fse_removeFeature(&md5); 12 | fse_release(); 13 | } -------------------------------------------------------------------------------- /include/cassdk_ndk_feacomp.h: -------------------------------------------------------------------------------- 1 | #ifndef CASSDK_NDK_FEACOMP_H_ 2 | #define CASSDK_NDK_FEACOMP_H_ 3 | typedef double (*FF_Similarity)(double* f1, double* f2); 4 | extern char* FF_GetVersion(); 5 | static FF_Similarity ff_similarity; 6 | inline void feacomp_init() { 7 | ff_similarity = (FF_Similarity)((long)FF_GetVersion - 0x104); 8 | } 9 | #endif /* CASSDK_NDK_FEACOMP_H_ */ 10 | -------------------------------------------------------------------------------- /test/test_c.c: -------------------------------------------------------------------------------- 1 | #include 2 | //#include 3 | #include "feature_se/feature_se.h" 4 | int main() { 5 | fse_init(16777216L,0.75f,1024L); 6 | face_code code; 7 | code_bean out[20]; 8 | fse_searchCode(&code, 0.75, 20, out, NULL, 0); 9 | code_bean added; 10 | fse_addFeature(&added); 11 | MD5 md5; 12 | fse_removeFeature(&md5); 13 | fse_release(); 14 | } -------------------------------------------------------------------------------- /jni/jni_mgr.h: -------------------------------------------------------------------------------- 1 | /* 2 | * jni_mgr.h 3 | * 4 | * Created on: 2015年11月26日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef JNI_JNI_MGR_H_ 9 | #define JNI_JNI_MGR_H_ 10 | #include 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved); 15 | JNIEXPORT void JNICALL JNI_OnUnload(JavaVM* vm, void* reserved); 16 | 17 | #ifdef __cplusplus 18 | } 19 | #endif 20 | #endif /* JNI_JNI_MGR_H_ */ 21 | -------------------------------------------------------------------------------- /UNINSTALL.bat.in: -------------------------------------------------------------------------------- 1 | ECHO 2 | ECHO OFF 3 | ECHO uninstall...... 4 | pushd %~dp0 5 | set full_path=%cd% 6 | set full_path=%full_path:\= % 7 | for %%a in (%full_path%) do set sub_name=%%a 8 | pushd .. 9 | set full_path=%cd% 10 | set full_path=%full_path:\= % 11 | for %%a in (%full_path%) do set folder_name=%%a 12 | popd 13 | popd 14 | SET FSE_HOME=%SystemDrive%\%folder_name% 15 | IF exist %FSE_HOME% rmdir %FSE_HOME% /S /Q 16 | ECHO please manual remove the %FSE_HOME_LIB% from environment variable 'PATH' 17 | ECHO uninstall finished 18 | :END 19 | 20 | -------------------------------------------------------------------------------- /jni/jni_mgr.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * jni_mgr.cpp 3 | * 4 | * Created on: 2015年11月26日 5 | * Author: guyadong 6 | */ 7 | #include 8 | #include "jni_mgr.h" 9 | #include "jni_utilits.h" 10 | #include "sample_log.h" 11 | using namespace gdface; 12 | JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved) { 13 | jni_utilits::setJVM(vm); 14 | SAMPLE_OUT("feature_se jni loaded"); 15 | return JNI_VERSION_1_6; 16 | } 17 | 18 | JNIEXPORT void JNICALL JNI_OnUnload(JavaVM* vm, void* reserved){ 19 | SAMPLE_OUT("feature_se jni unloaded"); 20 | } 21 | 22 | 23 | -------------------------------------------------------------------------------- /INSTALL.bat.in: -------------------------------------------------------------------------------- 1 | ECHO OFF 2 | ECHO install...... 3 | pushd %~dp0 4 | set full_path=%cd% 5 | set full_path=%full_path:\= % 6 | for %%a in (%full_path%) do set sub_name=%%a 7 | pushd .. 8 | set full_path=%cd% 9 | set full_path=%full_path:\= % 10 | for %%a in (%full_path%) do set folder_name=%%a 11 | popd 12 | popd 13 | SET FSE_HOME=%SystemDrive%\%folder_name% 14 | SET FSE_HOME_LIB=%FSE_HOME%\%sub_name% 15 | pushd %~sdp0 16 | xcopy .. %FSE_HOME% /S /I 17 | popd 18 | ECHO install finished 19 | ECHO please manual add the %FSE_HOME_LIB% to environment variable 'PATH' 20 | :END 21 | -------------------------------------------------------------------------------- /jni/JNIContext.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JNIContext.h 3 | * 4 | * Created on: 2015年11月26日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef JNI_JNICONTEXT_H_ 9 | #define JNI_JNICONTEXT_H_ 10 | #include "raii.h" 11 | #include "jni_utilits.h" 12 | namespace gdface { 13 | 14 | class JNIContext { 15 | public: 16 | jni_utilits::JavaClassMirror m_code_bean_mirror; 17 | JNIContext(jni_utilits::JavaClassMirror&& jcm) : 18 | m_code_bean_mirror(std::move(jcm)) { 19 | } 20 | ~JNIContext()=default; 21 | }; 22 | 23 | } /* namespace gdface */ 24 | 25 | #endif /* JNI_JNICONTEXT_H_ */ 26 | -------------------------------------------------------------------------------- /feature_se/MiniThreadPool.h: -------------------------------------------------------------------------------- 1 | /* 2 | * MiniThreadPool.h 3 | * 4 | * Created on: 2015年11月24日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef FEATURE_SE_MINITHREADPOOL_H_ 9 | #define FEATURE_SE_MINITHREADPOOL_H_ 10 | #include "ThreadPool.h" 11 | namespace gdface { 12 | /* 线程池 */ 13 | class MiniThreadPool:public ThreadPool { 14 | public: 15 | /* 线程池构造函数,线程数默认初始化为与CPU核数相同 */ 16 | MiniThreadPool(size_t threads=0):ThreadPool(0==threads?std::thread::hardware_concurrency():threads) { 17 | } 18 | virtual ~MiniThreadPool()=default; 19 | }; 20 | 21 | } /* namespace gdface */ 22 | 23 | #endif /* FEATURE_SE_MINITHREADPOOL_H_ */ 24 | -------------------------------------------------------------------------------- /make_gcc_project.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sh_folder=$(dirname $(readlink -f $0)) 4 | folder_name=$(basename $sh_folder) 5 | # 定义编译的版本类型(DEBUG|RELEASE) 6 | build_type=DEBUG 7 | typeset -u arg1=$1 8 | [ "$arg1" = "DEBUG" ] && build_type=$arg1 9 | echo build_type=$build_type 10 | 11 | pushd $sh_folder/.. 12 | 13 | [ -d $folder_name.prj ] && rm -fr $folder_name.prj 14 | mkdir $folder_name.prj 15 | 16 | pushd $folder_name.prj 17 | cmake "$sh_folder" -G "Eclipse CDT4 - Unix Makefiles" -DCMAKE_BUILD_TYPE=$build_type \ 18 | -DCMAKE_INSTALL_PREFIX=$sh_folder/release/fse_linux_x86_64 \ 19 | -DEXT_SDK_TYPE=CASSDK 20 | 21 | popd 22 | 23 | popd 24 | -------------------------------------------------------------------------------- /feature_se/CodeManagerFactory.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * CodeManagerFactory.cpp 3 | * 4 | * Created on: 2015年12月6日 5 | * Author: guyadong 6 | */ 7 | 8 | #include "CodeManagerFactory.h" 9 | #include "ICodeManagerCPU.h" 10 | namespace gdface { 11 | 12 | CodeManagerFactory::CodeManagerFactory()=default; 13 | 14 | CodeManagerFactory::~CodeManagerFactory() =default; 15 | 16 | ICodeManager * CodeManagerFactory::getICodeManagerCPU(HASH_TABLE_SIZE_TYPE initCapacity, bool isCopy, float loadFactor, size_t overBlockCapacity){ 17 | return &ICodeManagerCPU::getInstance(initCapacity,isCopy,loadFactor,overBlockCapacity); 18 | } 19 | } /* namespace gdface */ 20 | -------------------------------------------------------------------------------- /feature_se/CodeManagerFactory.h: -------------------------------------------------------------------------------- 1 | /* 2 | * CodeManagerFactory.h 3 | * 4 | * Created on: 2015年12月6日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef FEATURE_SE_CODEMANAGERFACTORY_H_ 9 | #define FEATURE_SE_CODEMANAGERFACTORY_H_ 10 | #include "ICodeManager.h" 11 | namespace gdface { 12 | 13 | class FSE_DLL_DECL CodeManagerFactory { 14 | public: 15 | CodeManagerFactory(); 16 | virtual ~CodeManagerFactory(); 17 | static ICodeManager * getICodeManagerCPU(uint32_t initCapacity=DEFAULT_FACETABLE_CAPACITY, bool isCopy=true, float loadFactor=0, size_t overBlockCapacity=0); 18 | 19 | }; 20 | 21 | } /* namespace gdface */ 22 | 23 | #endif /* FEATURE_SE_CODEMANAGERFACTORY_H_ */ 24 | -------------------------------------------------------------------------------- /feature_se/MD5SetImpl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * MD5SetImpl1.h 3 | * 4 | * Created on: 2015年12月7日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef FEATURE_SE_MD5SETIMPL_H_ 9 | #define FEATURE_SE_MD5SETIMPL_H_ 10 | #include "hashset_cl.h" 11 | #include "codemgr.h" 12 | namespace gdface { 13 | 14 | class MD5SetImpl:public HashSetCl{ 15 | public: 16 | MD5SetImpl(uint32_t initCapacity=0, bool isCopy=false, float loadFactor=0.0, size_t overBlockCapacity=0):HashSetCl(initCapacity, isCopy, loadFactor, overBlockCapacity){} 17 | MD5SetImpl(MD5SetImpl&&)=default; 18 | virtual ~MD5SetImpl()=default; 19 | }; 20 | 21 | } /* namespace gdface */ 22 | 23 | #endif /* FEATURE_SE_MD5SETIMPL_H_ */ 24 | -------------------------------------------------------------------------------- /feature_se/config.h: -------------------------------------------------------------------------------- 1 | /* 2 | * config.h 3 | * 4 | * Created on: 2015年12月6日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef FEATURE_SE_CONFIG_H_ 9 | #define FEATURE_SE_CONFIG_H_ 10 | #ifndef FSE_IS_A_DLL 11 | #define FSE_IS_A_DLL 1 /* not set if you're statically linking */ 12 | #endif // !FSE_IS_A_DLL 13 | 14 | #ifndef FSE_DLL_DECL 15 | #if defined( _WIN32) && FSE_IS_A_DLL 16 | #ifdef FSE_EXPORTS 17 | #define FSE_DLL_DECL __declspec(dllexport) 18 | #define FSE_API __declspec(dllexport) 19 | #else 20 | #define FSE_DLL_DECL __declspec(dllimport) 21 | #define FSE_API __declspec(dllimport) 22 | #endif 23 | #else 24 | #define FSE_DLL_DECL 25 | #define FSE_API 26 | #endif 27 | #endif 28 | 29 | #endif /* FEATURE_SE_CONFIG_H_ */ 30 | -------------------------------------------------------------------------------- /feature_se/feature_compare_cas.h: -------------------------------------------------------------------------------- 1 | /* 2 | * feature_compare_cas.h 3 | * 4 | * Created on: 2016年7月12日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef FEATURE_SE_FEATURE_COMPARE_CAS_H_ 9 | #define FEATURE_SE_FEATURE_COMPARE_CAS_H_ 10 | #if __linux 11 | #include "FiStdDefEx.h" 12 | #elif _WIN32 13 | #include 14 | #endif 15 | #include "THFeature_i.h" 16 | #include "codemgr.h" 17 | namespace gdface{ 18 | namespace feature{ 19 | /* CASSDK特征码比对 */ 20 | inline float compare(const face_code &f1,const face_code&f2){ 21 | // 本函数被调用前必须确保已经执行 EF_Init() 22 | return EF_Compare((BYTE*)f1.element,(BYTE*)f2.element); 23 | } 24 | } /* namespace feature*/ 25 | } /* namespace gdface*/ 26 | 27 | #endif /* FEATURE_SE_FEATURE_COMPARE_CAS_H_ */ 28 | -------------------------------------------------------------------------------- /cmake/Modules/FindFse.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find FEATURE_SE 2 | # 3 | # The following variables are optionally searched for defaults 4 | # FSE_ROOT_DIR: Base directory where all FEATURE_SE components are found 5 | # 6 | # The following are set after configuration is done: 7 | set(FSE_ROOT_DIR "" CACHE PATH "Folder contains feature_se") 8 | if(FSE_ROOT_DIR) 9 | if(WIN32 AND NOT CYGWIN) 10 | set (_CONFIG_INSTALL_DIR cmake) 11 | else() 12 | set (_CONFIG_INSTALL_DIR lib/cmake/fse) 13 | endif() 14 | get_filename_component(Fse_DIR "${FSE_ROOT_DIR}/${_CONFIG_INSTALL_DIR}" ABSOLUTE) 15 | unset(_CONFIG_INSTALL_DIR) 16 | endif(FSE_ROOT_DIR) 17 | if(Fse_DIR) 18 | message(STATUS "Fse_DIR = ${Fse_DIR}") 19 | endif() 20 | find_package(Fse CONFIG) 21 | -------------------------------------------------------------------------------- /feature_se/TopKCodeBean.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * TopKCodeBean.cpp 3 | * 4 | * Created on: 2015年10月23日 5 | * Author: guyadong 6 | */ 7 | 8 | #include "TopKCodeBean.h" 9 | 10 | namespace gdface { 11 | auto TopKCodeBean::result_to(code_bean* out) noexcept -> decltype(declval().size()) 12 | { 13 | assert(nullptr!=out); 14 | auto& sort = result(); 15 | for (auto i = size(); i > 0; --i) { 16 | out[i-1] = *sort[i - 1].t; 17 | out[i-1].similarity = sort[i - 1].n; 18 | } 19 | return size(); 20 | } 21 | 22 | auto TopKCodeBean::result_vector() noexcept -> decltype(make_unique >(0)) 23 | { 24 | auto v = make_unique >(size()); 25 | result_to(v.get()->data()); 26 | return v; 27 | } 28 | 29 | } /* namespace gdface */ 30 | -------------------------------------------------------------------------------- /UNINSTALL.sh.in: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | if [ -n "$1" ] 3 | then 4 | FSE_HOME=$1 5 | else 6 | ##没有指定安装路径时，指定默认的路径 7 | FSE_HOME="/usr/local/feature_se" 8 | fi 9 | 10 | FSE_HOME_LIB="$FSE_HOME/bin" 11 | ##目录不存在则退出 12 | if [[ ! -d $FSE_HOME_LIB ]] 13 | then 14 | echo "not exist dir: $FSE_HOME_LIB" 15 | exit 0 16 | fi 17 | echo "uninstall from $FSE_HOME.............................." 18 | # 删除安装目录 19 | rm -frv $FSE_HOME 20 | ## 删除conf 21 | rm -fv /etc/ld.so.conf.d/feature_se_lib.conf 22 | /sbin/ldconfig 23 | ##更新profile 24 | reg_path=`echo $FSE_HOME_LIB|sed -r 's/([\/\.])/\\\\\1/g'` 25 | echo delete the line if exist: 26 | echo \"export PATH=$FSE_HOME_LIB:\$PATH\" 27 | sed -i_bak -r "/^\s*export\s*PATH=$reg_path:\\\$PATH/d" /etc/profile 28 | echo "uninstall finished" 29 | 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | build/ 34 | build.gcc/ 35 | /release 36 | Debug/ 37 | Release/ 38 | *.tlog 39 | *.pdb 40 | *.opendb 41 | *.log 42 | *.db 43 | *.ilk 44 | *.cache 45 | .suo 46 | install_manifest.txt 47 | /CMakeFiles 48 | /cmake_install.cmake 49 | /CMakeCache.txt 50 | /project.gcc 51 | project.vs2015/ 52 | project.custom.vs2015/ 53 | .project 54 | .cproject 55 | /test.prj/ 56 | -------------------------------------------------------------------------------- /feature_se/ICodeManager.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * ICodeManager.cpp 3 | * 4 | * Created on: 2015年12月7日 5 | * Author: guyadong 6 | */ 7 | #include "ICodeManager.h" 8 | #include "MD5SetImpl.h" 9 | #include "gf_utility.h" 10 | namespace gdface { 11 | MD5Set::MD5Set(uint32_t initCapacity, bool isCopy, float loadFactor, size_t overBlockCapacity) 12 | :m_set(new MD5SetImpl(initCapacity,isCopy,loadFactor,overBlockCapacity)){ 13 | } 14 | MD5Set::~MD5Set(){ 15 | delete m_set; 16 | } 17 | MD5Set::MD5Set(MD5Set&&rv):m_set(rv.m_set){ 18 | modify_const(rv.m_set,(decltype(rv.m_set))nullptr); 19 | } 20 | bool MD5Set::has(const MD5&key){ 21 | return m_set->has(key); 22 | } 23 | const MD5& MD5Set::insert(const MD5&key){ 24 | return m_set->insert(key); 25 | } 26 | } /* namespace gdface */ 27 | 28 | -------------------------------------------------------------------------------- /feature_se/cross_define.h: -------------------------------------------------------------------------------- 1 | /* 2 | * cross_platform_def.h 3 | * 4 | * Created on: 2015年11月6日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef FEATURE_SE_CROSS_DEFINE_H_ 9 | #define FEATURE_SE_CROSS_DEFINE_H_ 10 | 11 | #ifdef _MSC_VER 12 | #define _ALIGN_(n) __declspec( align(n) ) 13 | #elif defined(__GNUC__) 14 | #define _ALIGN_(n) __attribute__((aligned(n))) 15 | #elif __cplusplus>=201103L 16 | #define _ALIGN_(n) alignas(n) 17 | #else 18 | // 不支持的编译器类型 19 | #error unexpected c complier (msc/gcc) 20 | #endif /*_MSC_VER*/ 21 | #define _ALIGN_16 _ALIGN_(16) 22 | 23 | #if __cplusplus<201103L 24 | #ifdef _MSC_VER 25 | #define thread_local __declspec( thread ) 26 | #elif defined(__GNUC__) 27 | #define thread_local __thread 28 | #else 29 | // 不支持的编译器类型 30 | #error unexpected c complier (msc/gcc) 31 | #endif 32 | #endif /*__cplusplus>=201103L*/ 33 | 34 | #endif /* FEATURE_SE_CROSS_DEFINE_H_ */ 35 | -------------------------------------------------------------------------------- /test/make_msvc_cassdk_project.bat: -------------------------------------------------------------------------------- 1 | echo off 2 | echo make test VS2015 project 3 | if not defined VS140COMNTOOLS ( 4 | echo vs2015 NOT FOUND. 5 | exit /B -1 6 | ) 7 | echo vs2015 found. 8 | where cmake 9 | if errorlevel 1 ( 10 | echo cmake NOT FOUND. 11 | exit /B -1 12 | ) 13 | echo cmake found 14 | set sh_folder=%~dp0 15 | 16 | pushd %sh_folder% 17 | 18 | rem 需要先编译feature_se 19 | if exist project.vs2015 rmdir project.vs2015 /s/q 20 | mkdir project.vs2015 21 | pushd project.vs2015 22 | if not defined VisualStudioVersion ( 23 | echo make MSVC environment ... 24 | call "%VS140COMNTOOLS%..\..\vc/vcvarsall" x86_amd64 25 | ) 26 | echo creating x86_64 Project for Visual Studio 2015 ... 27 | set sh_folder=%sh_folder:\=/% 28 | 29 | cmake -G "Visual Studio 14 2015 Win64" ^ 30 | -DCMAKE_MODULE_PATH=%sh_folder%../cmake/Modules ^ 31 | -DFSE_ROOT_DIR=..\..\release\fse_windows_x86_64 .. 32 | 33 | popd 34 | popd -------------------------------------------------------------------------------- /make_msvc_cassdk_project.bat: -------------------------------------------------------------------------------- 1 | echo off 2 | echo make feature_se VS2015 project 3 | if not defined VS140COMNTOOLS ( 4 | echo vs2015 NOT FOUND. 5 | exit /B -1 6 | ) 7 | echo vs2015 found. 8 | where cmake 9 | if errorlevel 1 ( 10 | echo cmake NOT FOUND. 11 | exit /B -1 12 | ) 13 | echo cmake found 14 | where java 15 | if errorlevel 1 ( 16 | echo java NOT FOUND. 17 | exit /B -1 18 | ) 19 | echo java found 20 | set sh_folder=%~dp0 21 | pushd %sh_folder% 22 | 23 | rem 需要先编译feature_se 24 | if exist project.vs2015 rmdir project.vs2015 /s/q 25 | mkdir project.vs2015 26 | pushd project.vs2015 27 | if not defined VisualStudioVersion ( 28 | echo make MSVC environment ... 29 | call "%VS140COMNTOOLS%..\..\vc/vcvarsall" x86_amd64 30 | ) 31 | echo creating x86_64 Project for Visual Studio 2015 ... 32 | cmake -G "Visual Studio 14 2015 Win64" -DCMAKE_INSTALL_PREFIX=..\release\fse_windows_x86_64 .. ^ 33 | -DEXT_SDK_TYPE=CASSDK 34 | 35 | popd 36 | popd -------------------------------------------------------------------------------- /cmake/Modules/FindCassdk.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find CASSDK 2 | # 3 | # The following variables are optionally searched for defaults 4 | # CASSDK_ROOT_DIR: Base directory where all CASSDK components are found 5 | # 6 | # The following are set after configuration is done: 7 | #定义CASSDK位置 8 | set(CASSDK_ROOT_DIR "" CACHE PATH "Folder contains CASSDK") 9 | if(WIN32 AND NOT CYGWIN) 10 | set (_CONFIG_INSTALL_DIR cmake) 11 | else() 12 | set (_CONFIG_INSTALL_DIR lib/cmake/cassdk) 13 | endif() 14 | if(CASSDK_ROOT_DIR) 15 | get_filename_component(Cassdk_DIR "${CASSDK_ROOT_DIR}/${_CONFIG_INSTALL_DIR}" ABSOLUTE) 16 | elseif(DEFINED ENV{CASSDK_HOME}) 17 | # 使用环境变量CASSDK_HOME指定的位置 18 | get_filename_component(Cassdk_DIR "$ENV{CASSDK_HOME}/${_CONFIG_INSTALL_DIR}" ABSOLUTE) 19 | endif() 20 | if(Cassdk_DIR) 21 | message(STATUS "Cassdk_DIR = ${Cassdk_DIR}") 22 | endif() 23 | unset(_CONFIG_INSTALL_DIR) 24 | 25 | find_package(Cassdk CONFIG) 26 | -------------------------------------------------------------------------------- /test/make_gcc_project.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | GXX_PATH= 3 | if [ `/usr/bin/g++ -dumpversion` != "5.2.0" ] 4 | then 5 | if [ `/usr/local/bin/g++ -dumpversion` = "5.2.0" ] 6 | then 7 | GXX_PATH="-DCMAKE_CXX_COMPILER:FILEPATH=/usr/local/bin/g++ -DCMAKE_C_COMPILER:FILEPATH=/usr/local/bin/gcc" 8 | else 9 | echo "g++ compiler required version 5.2.0" 10 | exit -1 11 | fi 12 | fi 13 | 14 | sh_folder=$(dirname $(readlink -f $0)) 15 | folder_name=$(basename $sh_folder) 16 | # 定义编译的版本类型(DEBUG|RELEASE) 17 | build_type=DEBUG 18 | typeset -u arg1=$1 19 | [ "$arg1" = "DEBUG" ] && build_type=$arg1 20 | echo build_type=$build_type 21 | 22 | pushd $sh_folder/.. 23 | 24 | [ -d $folder_name.prj ] && rm -fr $folder_name.prj 25 | mkdir $folder_name.prj 26 | pushd $folder_name.prj 27 | cmake "$sh_folder" $GXX_PATH -G "Eclipse CDT4 - Unix Makefiles" -DCMAKE_BUILD_TYPE=$build_type \ 28 | -DCMAKE_MODULE_PATH=$sh_folder/../cmake/Modules \ 29 | -DFSE_ROOT_DIR=$sh_folder/../release/fse_linux_x86_64 30 | popd 31 | popd 32 | -------------------------------------------------------------------------------- /feature_se/TopKCodeBean.h: -------------------------------------------------------------------------------- 1 | /* 2 | * TopKCodeBean.h 3 | * 4 | * Created on: 2015年10月23日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef FEATURE_SE_TOPKCODEBEAN_H_ 9 | #define FEATURE_SE_TOPKCODEBEAN_H_ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "codemgr.h" 15 | #include "topk_base.h" 16 | #include "gf_utility.h" 17 | namespace gdface { 18 | class TopKCodeBean: public topk_base { 19 | public: 20 | // 比较值类型 与code_bean::similarity类型保持同步 21 | using cmp_type = decltype(code_bean::similarity); 22 | // 基类型 23 | using base_type=topk_base; 24 | using base_type::base_type; // 继承基类构造函数 25 | /* 返回包含排序结果到out数组,并将相似度值填到 code_bean.similarity 26 | * 参见父类result()说明 27 | */ 28 | auto result_to(code_bean* out) noexcept -> decltype(size()); 29 | /* 返回包含排序结果的vector 30 | * 参见父类result_to()说明 31 | */ 32 | auto result_vector() noexcept -> decltype(make_unique >(0)); 33 | 34 | }; 35 | 36 | 37 | } /* namespace gdface */ 38 | 39 | #endif /* FEATURE_SE_TOPKCODEBEAN_H_ */ 40 | -------------------------------------------------------------------------------- /make_ndk_project.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Android NDK 交叉编译脚本,创建Makefile 3 | # author guyadong 4 | 5 | # 检测是否安装NDK,没有安装NDK则报错退出 6 | [ -z "$ANDROID_NDK" ] && echo "ERROR:environment variable ANDROID_NDK not define" && exit -1 7 | 8 | sh_folder=$(dirname $(readlink -f $0)) 9 | folder_name=$(basename $sh_folder) 10 | # 定义编译的版本类型(DEBUG|RELEASE) 11 | build_type=RELEASE 12 | typeset -u arg1=$1 13 | [ "$arg1" = "DEBUG" ] && build_type=$arg1 14 | echo build_type=$build_type 15 | 16 | pushd $sh_folder/.. 17 | 18 | [ -d $folder_name.ndk.prj ] && rm -fr $folder_name.ndk.prj 19 | mkdir $folder_name.ndk.prj 20 | 21 | pushd $folder_name.ndk.prj 22 | cmake "$sh_folder" -G "Eclipse CDT4 - Unix Makefiles" -DCMAKE_BUILD_TYPE=$build_type \ 23 | -DEXT_SDK_TYPE=EUCLIDEAN \ 24 | -DDEFAULT_ELEM_TYPE=double \ 25 | -DDEFAULT_ELEM_LEN=512 \ 26 | -DDEFAULT_CODE_END_WITH_SUM=OFF \ 27 | -DCMAKE_INSTALL_PREFIX=$sh_folder/release/fse_android_armeabi-v7a \ 28 | -DCMAKE_SYSTEM_NAME=Android \ 29 | -DCMAKE_SYSTEM_VERSION=17 \ 30 | -DANDROID_ARM_NEON=ON \ 31 | -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake 32 | 33 | popd 34 | popd 35 | -------------------------------------------------------------------------------- /feature_se/hashset_cl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * hashsetm_cl.h 3 | * 4 | * Created on: 2015年10月12日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef FEATURE_SE_HASHSET_CL_H_ 9 | #define FEATURE_SE_HASHSET_CL_H_ 10 | #include "hashtable_abstract.h" 11 | namespace gdface { 12 | /* hashset实现模板*/ 13 | template 14 | class HashSetCl: public HashTableAbstract { 15 | public: 16 | HashSetCl(HASH_TABLE_SIZE_TYPE initCapacity=0, bool isCopy=false, float loadFactor=0.0, size_t overBlockCapacity=0) : 17 | HashTableAbstract::HashTableAbstract(initCapacity, isCopy, loadFactor, overBlockCapacity) { 18 | } 19 | HashSetCl(HASH_TABLE_SIZE_TYPE initCapacity, bool isCopy) : 20 | HashSetCl(initCapacity, isCopy, 0.0, 0) { 21 | 22 | } 23 | HashSetCl(HashSetCl&&rv)=default; 24 | virtual ~HashSetCl()=default; 25 | /* 26 | * 向表中插入一个key 27 | */ 28 | const K& insert(const K &key){ 29 | return *(K*)HashTableAbstract::insert(key,key).obj; 30 | } 31 | bool remove(const K &key) { 32 | return HashTableAbstract::removeKey(key); 33 | } 34 | protected: 35 | virtual const K& keyFrom(const K& value)const noexcept{ 36 | return value; 37 | } 38 | }; 39 | 40 | } 41 | 42 | #endif /* FEATURE_SE_HASHSET_CL_H_ */ 43 | -------------------------------------------------------------------------------- /feature_se/feature_compare.h: -------------------------------------------------------------------------------- 1 | /* 2 | * feature_compare.h 3 | * 4 | * Created on: 2015年12月16日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef FEATURE_SE_FEATURE_COMPARE_H_ 9 | #define FEATURE_SE_FEATURE_COMPARE_H_ 10 | 11 | #include 12 | #include "dot_product.h" 13 | #include "codemgr.h" 14 | #ifndef CODE_FLOAT_NUM 15 | // 特征码默认长度，调用层应该在外部根据实际长度定义该宏 16 | #define CODE_FLOAT_NUM 210 17 | #endif 18 | 19 | namespace gdface{ 20 | namespace feature{ 21 | /* V2版本特征码比对 */ 22 | template 23 | inline double compareV2(const E *f1, const E *f2, double sssum1, double sssum2) { 24 | return dot_product(f1, f2) / sqrt(sssum1 * sssum2); 25 | } 26 | /* V1版本特征码比对 */ 27 | template 28 | inline double compareV1(const E *f1, const E *f2) { 29 | return compareV2(f1,f2,dot_product(f1,f1),dot_product(f2,f2)); 30 | } 31 | /* V2版本特征码比对 */ 32 | template 33 | inline double compare(const face_code &f1,const face_code&f2){ 34 | return compareV2(f1.element,f2.element,f1.sum,f2.sum); 35 | } 36 | } /* namespace feature*/ 37 | } /* namespace gdface*/ 38 | 39 | #endif /* FEATURE_SE_FEATURE_COMPARE_H_ */ 40 | -------------------------------------------------------------------------------- /INSTALL.sh.in: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ -n "$1" ] 4 | then 5 | FSE_HOME="$1" 6 | else 7 | ##没有指定安装路径时，指定默认的路径 8 | FSE_HOME="/usr/local/feature_se" 9 | fi 10 | mkdir_if_no_exist(){ 11 | if [ $# -eq 1 ] 12 | then 13 | if [[ -d $1 ]] 14 | then 15 | return 0 16 | 17 | else 18 | ##安装路径不存在时创建目录 19 | mkdir $1 20 | fi 21 | else 22 | echo invalid argument 23 | return -1 24 | fi 25 | } 26 | FSE_HOME_LIB="$FSE_HOME/bin" 27 | if mkdir_if_no_exist $FSE_HOME && mkdir_if_no_exist $FSE_HOME_LIB 28 | then 29 | echo "install to $FSE_HOME........................." 30 | src_path=$(cd "$(dirname "$0")"; pwd) 31 | cd "$(dirname "$0")" 32 | echo copy to $FSE_HOME 33 | chmod +x lib* 34 | cd .. 35 | cp -r * $FSE_HOME 36 | echo "$FSE_HOME_LIB">/etc/ld.so.conf.d/feature_se_lib.conf 37 | /sbin/ldconfig 38 | # modify /etc/profile,add FSE_HOME_LIB to PATH 39 | new_export="export PATH=$FSE_HOME_LIB:\$PATH" 40 | reg_str="^\s*export\s*PATH=$FSE_HOME_LIB:\$PATH" 41 | if grep -q $reg_str /etc/profile 42 | then 43 | echo $FSE_HOME_LIB exists in PATH,skip 44 | else 45 | echo defined PATH variable 46 | echo $new_export >> /etc/profile 47 | fi 48 | # display new line 49 | grep $reg_str /etc/profile 50 | echo variable PATH in /etc/profile updated 51 | source /etc/profile 52 | echo "install finished" 53 | fi 54 | 55 | -------------------------------------------------------------------------------- /feature_se/hashtable_core.h: -------------------------------------------------------------------------------- 1 | /* 2 | * hashtable_core.h 3 | * 4 | * Created on: 2015年10月6日 5 | * Author: guyadong 6 | */ 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | #ifndef FEATURE_SE_HASHTABLE_CORE_H_ 11 | #define FEATURE_SE_HASHTABLE_CORE_H_ 12 | #include 13 | #include 14 | #define DEFAULT_HASHTABLE_INITCAPACITY 16 15 | #define MAX_HASHTABLE_CAPACITY (1UL<<27) 16 | #define DEFAULT_LOADFACTOR 0.75f 17 | /* 哈希表节点数据结构 */ 18 | typedef struct _HashNode { 19 | void * obj; 20 | struct _HashNode *next; 21 | _HashNode(void *obj,struct _HashNode *next):obj(obj),next(next){} 22 | _HashNode():_HashNode(nullptr,nullptr){} 23 | _HashNode(const _HashNode&)=default; 24 | _HashNode(_HashNode&&)=default; 25 | _HashNode& operator=(const _HashNode&)=default; 26 | _HashNode& operator=(_HashNode&&)=default; 27 | 28 | } HashNode; 29 | typedef HashNode* HASH_NODE_PTR; 30 | /* 哈希表核心数据结构 */ 31 | typedef struct _HashtableCore { 32 | uint32_t size; // hash 表中的元素个数 33 | uint32_t capacity; // hash 表容量(数组大小) 34 | uint32_t obj_size; //hash_node 中 obj大小 35 | HASH_NODE_PTR nodes; // hash表(动态数组) 36 | } HashtableCore; 37 | uint32_t default_hashcode(const void *obj, size_t size); 38 | int default_equals(const void *obj1, const void *obj2, size_t size); 39 | #endif /* FEATURE_SE_HASHTABLE_CORE_H_ */ 40 | 41 | #ifdef __cplusplus 42 | } 43 | #endif 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018, 10km 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | list of conditions and the following disclaimer in the documentation and/or 12 | other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 18 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 21 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /cmake/custom_feature_compare.h.in: -------------------------------------------------------------------------------- 1 | /* 2 | * custom_feature_compare.h 3 | * Description: 由custom_feature_compare.h.in模板自动生成, 4 | * 用于将第三方特征比对函数封装为统一的接口供搜索比对调用 5 | * Created on: 2019年2月22日 6 | * Author: guyadong 7 | */ 8 | 9 | #ifndef CUSTOM_FEACOMP_H_ 10 | #define CUSTOM_FEACOMP_H_ 11 | #include 12 | #include 13 | #include 14 | 15 | @CUSTOM_INCLUDE_STATEMENTS@ 16 | #include "codemgr.h" 17 | namespace gdface{ 18 | namespace feature{ 19 | template 20 | struct function_traits; 21 | 22 | template 23 | struct function_traits> 24 | { 25 | static const size_t nargs = sizeof...(Args); 26 | 27 | typedef R result_type; 28 | 29 | template 30 | struct arg 31 | { 32 | typedef typename std::tuple_element>::type type; 33 | }; 34 | }; 35 | inline double compare(const face_code &f1,const face_code&f2){ 36 | static std::once_flag oc; 37 | std::call_once(oc, [&]() { 38 | @CUSTOM_FEACOMP_INIT@ 39 | }); 40 | typedef std::function<@CUSTOM_FEACOMP_FUNTYPE@> feacomp_fun; 41 | return (double)@CUSTOM_FEACOMP_FUNNAME@((function_traits::arg<0>::type)f1.element,(function_traits::arg<1>::type)f2.element); 42 | } 43 | } /* namespace feature*/ 44 | } /* namespace gdface*/ 45 | 46 | #endif /* CUSTOM_FEACOMP_H_ */ 47 | -------------------------------------------------------------------------------- /make_cassdk_ndk_project.bat: -------------------------------------------------------------------------------- 1 | echo off 2 | rem Android NDK 交叉编译脚本,创建Makefile 3 | rem author guyadong 4 | 5 | rem 检测是否安装NDK,没有安装NDK则报错退出 6 | if not DEFINED ANDROID_NDK ( 7 | echo "ERROR:environment variable ANDROID_NDK not define" && exit /B -1 8 | ) 9 | where cmake 10 | if errorlevel 1 ( 11 | echo cmake NOT FOUND. 12 | exit /B -1 13 | ) 14 | 15 | set sh_folder=%~dp0 16 | rem 定义编译的版本类型(DEBUG|RELEASE) 17 | set build_type=RELEASE 18 | if "%1" == "DEBUG" set build_type=%1 19 | echo build_type=%build_type% 20 | 21 | pushd %sh_folder%.. 22 | if exist feature_se.cassdk.ndk.prj rmdir feature_se.cassdk.ndk.prj /s/q 23 | mkdir feature_se.cassdk.ndk.prj 24 | 25 | pushd feature_se.cassdk.ndk.prj 26 | 27 | cmake %sh_folder% -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=%build_type% ^ 28 | -DJNI_FSE_LIBNAME=FS_FaceFeatureCompare ^ 29 | -DEXT_SDK_TYPE=CUSTOM ^ 30 | -DCUSTOM_FEACOMP_INCLUDE=%sh_folder%include ^ 31 | -DCUSTOM_FEACOMP_LIBRARY=J:\cassdk_jna\cassdk-android\src\main\resources\lib\armeabi-v7a\libFaceFeature.so ^ 32 | -DCUSTOM_FEACOMP_HEADERS=cassdk_ndk_feacomp.h ^ 33 | -DCUSTOM_FEACOMP_FUNNAME=ff_similarity ^ 34 | -DCUSTOM_FEACOMP_INIT="feacomp_init();" ^ 35 | -DCUSTOM_FEACOMP_FUNTYPE="double(double*, double*)" ^ 36 | -DCUSTOM_CODE_BYTE_LEN=4096 ^ 37 | -DCMAKE_SYSTEM_VERSION=17 ^ 38 | -DCMAKE_INSTALL_PREFIX=%sh_folder%release\fse_android_armeabi-v7a ^ 39 | -DCMAKE_TOOLCHAIN_FILE=%ANDROID_NDK%\build\cmake\android.toolchain.cmake 40 | 41 | 42 | popd 43 | popd 44 | -------------------------------------------------------------------------------- /feature_se/HashMapCl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * HashMapCl.h 3 | * 4 | * Created on: 2015年10月13日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef FEATURE_SE_HASHMAPCL_H_ 9 | #define FEATURE_SE_HASHMAPCL_H_ 10 | #include 11 | #include 12 | #include "hashmap_abstract.h" 13 | #include "hashset_cl.h" 14 | #include "TopKCodeBean.h" 15 | namespace gdface { 16 | class HashMapCl: public HashMap_Abstract { 17 | protected: 18 | virtual inline const MD5& keyFrom(const code_bean& value)const noexcept{ 19 | return value.id; 20 | } 21 | /* 计算MD5哈希值 */ 22 | virtual inline HASH_TABLE_SIZE_TYPE hashcode(MD5 *obj) const noexcept{ 23 | auto hashcode =obj->l[0]+obj->l[1]; 24 | return (hashcode>>32)^(hashcode&0xffffffff); 25 | } 26 | public: 27 | using MD5Set=HashSetCl; 28 | HashMapCl(HASH_TABLE_SIZE_TYPE initCapacity); 29 | HashMapCl(HashMapCl&&)=default; 30 | HashMapCl(HASH_TABLE_SIZE_TYPE initCapacity=0, bool isCopy=true, float loadFactor=0, size_t overBlockCapacity=0); 31 | virtual ~HashMapCl()=default; 32 | TopKCodeBean searchCode(const face_code &code, double threshold, size_t rows, const MD5Set *md5set=nullptr) const; 33 | auto searchCode(atomic_uint & countDown,const face_code &code, double threshold, size_t rows, const MD5Set *md5set=nullptr) 34 | ->decltype(searchCode(code,threshold,rows,md5set)) const{ 35 | raii guard([&countDown]{--countDown;}); 36 | return searchCode(code,threshold,rows,md5set); 37 | } 38 | HASH_TABLE_SIZE_TYPE removeBeansByImgMD5(const MD5 &imgMD5); 39 | std::vector getBeansByImgMD5(const MD5 &imgMD5)const; 40 | }; 41 | 42 | } /* namespace gdface */ 43 | 44 | #endif /* FEATURE_SE_HASHMAPCL_H_ */ 45 | -------------------------------------------------------------------------------- /feature_se/OverTableManager.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * OverTableManager1.cpp 3 | * 4 | * Created on: 2015年10月12日 5 | * Author: guyadong 6 | */ 7 | #include 8 | #include "OverTableManager.h" 9 | 10 | namespace gdface { 11 | 12 | OverTableManager::OverTableManager(size_t tableCapacity):TABLECAPACITY(tableCapacity) { 13 | } 14 | /* 15 | * 分配一个新的节点空间 16 | */ 17 | HASH_NODE_PTR OverTableManager::newNode() { 18 | HASH_NODE_PTR node=nullptr; 19 | // 遍历内存表，寻找空闲的内存块 20 | for(auto count=m_overList.size();count>0;--count){ 21 | auto &front=m_overList.front(); 22 | node=front.newNode(); 23 | if(front.full()){ 24 | //将全满的对象放到链表最后,减少下次查找次数 25 | m_overList.push_back(std::move(front)); 26 | m_overList.pop_front(); 27 | } 28 | if(nullptr!=node) 29 | return node; 30 | } 31 | 32 | //cout << "new block:"<deleteNode(node); 46 | if(itor->empty()){ 47 | if(hasEmpty){ 48 | // 存在超过一个空块，就删除当前块，始终保持只有一个空块 49 | itor=m_overList.erase(itor); 50 | //cout << "empty block "<<"deleted,left"<print_info(); 63 | return sucess; 64 | } 65 | 66 | } /* namespace gdface */ 67 | -------------------------------------------------------------------------------- /make_mtfsdk_ndk_project.bat: -------------------------------------------------------------------------------- 1 | echo off 2 | rem Android NDK 交叉编译脚本,创建Makefile 3 | rem author guyadong 4 | 5 | rem 检测是否安装NDK,没有安装NDK则报错退出 6 | if not DEFINED ANDROID_NDK ( 7 | echo "ERROR:environment variable ANDROID_NDK not define" && exit /B -1 8 | ) 9 | where cmake 10 | if errorlevel 1 ( 11 | echo cmake NOT FOUND. 12 | exit /B -1 13 | ) 14 | 15 | set sh_folder=%~dp0 16 | rem 定义编译的版本类型(DEBUG|RELEASE) 17 | set build_type=RELEASE 18 | if "%1" == "DEBUG" set build_type=%1 19 | echo build_type=%build_type% 20 | 21 | pushd %sh_folder%.. 22 | if exist feature_se.mtfsdk.ndk.prj rmdir feature_se.mtfsdk.ndk.prj /s/q 23 | mkdir feature_se.mtfsdk.ndk.prj 24 | 25 | pushd feature_se.mtfsdk.ndk.prj 26 | 27 | @rem EXT_SDK_TYPE 指定算法类型可选值： 28 | @rem CASSDK(默认值) 29 | @rem EUCLIDEAN 默认使用欧氏距离计算相似度 30 | @rem CUSTOM 使用自定义算法提的供相似度比较函数，此方式暂时未支持 31 | @rem 如果EXT_SDK_TYPE指定为EUCLIDEAN,下列参数需要设置: 32 | @rem EUCLIDEAN_ELEM_TYPE 定义特征值数组类型(double/float)，如果不指定，默认值为double 33 | @rem EUCLIDEAN_ELEM_LEN 定义特征值数组长度 34 | @rem EUCLIDEAN_CODE_END_WITH_SUM 定义特征值数组最后是否有一个double保存特征值数组的点积和，默认为OFF 35 | @rem ============================下列为通用参数与EXT_SDK_TYPE无关 36 | @rem FSE_LIBNAME 指定生成动态库名,不指定则使用默认值 37 | @rem JNI_FSE_LIBNAME 指定生成jni动态库名,不指定则使用默认值 38 | 39 | cmake %sh_folder% -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=%build_type% ^ 40 | -DJNI_FSE_LIBNAME=FS_FaceFeatureCompare ^ 41 | -DEXT_SDK_TYPE=EUCLIDEAN ^ 42 | -DEUCLIDEAN_ELEM_TYPE=float ^ 43 | -DEUCLIDEAN_ELEM_LEN=128 ^ 44 | -DEUCLIDEAN_CODE_END_WITH_SUM=OFF ^ 45 | -DCMAKE_SYSTEM_VERSION=17 ^ 46 | -DCMAKE_INSTALL_PREFIX=%sh_folder%release\fse_mtfsdk_android_armeabi-v7a ^ 47 | -DCMAKE_TOOLCHAIN_FILE=%ANDROID_NDK%\build\cmake\android.toolchain.cmake 48 | 49 | @rem 加入NEON指令优化后 会导致release版本执行异常 50 | @rem -DANDROID_ARM_NEON=ON 51 | 52 | popd 53 | popd 54 | -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #cmake file for test fse 2 | #author:guyadong 3 | #created:2018/04/16 4 | cmake_minimum_required( VERSION 3.0 ) 5 | 6 | cmake_policy(SET CMP0048 NEW) 7 | project( testfse VERSION 1.0.0 LANGUAGES C CXX) 8 | # 查找feature_se库,需要设置 CMAKE_PREIX_PATH 指定 feature_se 的安装路径 9 | # 需要指定CMAKE_MODULE_PATH 指定 FindFse.cmake 的位置 10 | # 本例中的位置在项目根目录下/cmake/Modules 11 | find_package(Fse REQUIRED) 12 | # include utils.cmake,location /cmake/Modules 13 | include (utils) 14 | cxx11_support() 15 | ##############设置目标文件生成位置##################### 16 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/bin") 17 | set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/bin") 18 | set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/lib") 19 | if(MSVC) 20 | #关闭C4819警告 21 | add_definitions("/wd4819") 22 | message(STATUS "optional:/wd4819") 23 | endif(MSVC) 24 | # 定义所有的源文件列表 25 | set(_CPP_SOURCE_FILES 26 | test.cpp 27 | ) 28 | set(_C_SOURCE_FILES 29 | test_c.c 30 | ) 31 | # C++调用测试 32 | add_executable(test_fse ${_CPP_SOURCE_FILES}) 33 | add_executable(test_fse_static ${_CPP_SOURCE_FILES}) 34 | target_link_libraries(test_fse gdface::fse_cas) 35 | target_link_libraries(test_fse_static gdface::fse_cas_static) 36 | if(MSVC) 37 | add_executable(test_fse_static_mt ${_CPP_SOURCE_FILES}) 38 | target_link_libraries(test_fse_static_mt gdface::fse_cas_static_mt) 39 | endif() 40 | 41 | # 标准C调用测试 42 | add_executable(test_fse_c ${_C_SOURCE_FILES}) 43 | add_executable(test_fse_c_static ${_C_SOURCE_FILES}) 44 | 45 | target_link_libraries(test_fse_c gdface::fse_cas) 46 | target_link_libraries(test_fse_c_static gdface::fse_cas_static) 47 | if(MSVC) 48 | add_executable(test_fse_c_static_mt ${_C_SOURCE_FILES}) 49 | target_link_libraries(test_fse_c_static_mt gdface::fse_cas_static_mt) 50 | endif() 51 | 52 | # Cleanup temporary variables. 53 | set(_CPP_SOURCE_FILES) 54 | set(_C_SOURCE_FILES) 55 | -------------------------------------------------------------------------------- /feature_se/hashmap_abstract.h: -------------------------------------------------------------------------------- 1 | /* 2 | * hashmap_abstract.h 3 | * 4 | * Created on: 2015年10月13日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef FEATURE_SE_HASHMAP_ABSTRACT_H_ 9 | #define FEATURE_SE_HASHMAP_ABSTRACT_H_ 10 | #include "hashtable_abstract.h" 11 | namespace gdface { 12 | /* 13 | * 基于 HashTableAbstract实现hashmap基本功能的抽象类 14 | * 要求子类提供从V中可以获取K的虚函数 keyFrom 15 | */ 16 | template 17 | class HashMap_Abstract: public HashTableAbstract { 18 | protected: 19 | virtual const K& keyFrom(const V& value)const noexcept=0; 20 | public: 21 | HashMap_Abstract(HASH_TABLE_SIZE_TYPE initCapacity=0, bool isCopy=false, float loadFactor=0, size_t overBlockCapacity=0) : 22 | HashTableAbstract::HashTableAbstract(initCapacity, isCopy, loadFactor, overBlockCapacity) { 23 | } 24 | HashMap_Abstract(HashMap_Abstract&&)=default; 25 | virtual ~HashMap_Abstract()=default; 26 | /* 27 | * 根据key向表中插入一个value，返回为插入的对象指针(非复制模式下[isCopy==false],等于value), 28 | */ 29 | const V & put(const K &key, const V& value) { 30 | return *(V*) HashTableAbstract::insert(key, value).obj; 31 | } 32 | /* 33 | * 向表中插入一个value，返回为插入的对象指针(非复制模式下[isCopy==false],等于value), 34 | * value为nullptr则返回nullptr 35 | */ 36 | const V & put(const V& value) { 37 | return put(keyFrom(value), value); 38 | } 39 | bool remove(const K &key) { 40 | return HashTableAbstract::removeKey(key); 41 | } 42 | bool remove(const V &value) { 43 | return HashTableAbstract::removeValue(value); 44 | } 45 | bool hasValue(const V &value)const noexcept { 46 | return HashTableAbstract::has(keyFrom(value)); 47 | } 48 | const V* get(const K& key)const noexcept { 49 | auto node = HashTableAbstract::findNode(key); 50 | return nullptr == node ? nullptr : (V*) node->obj; 51 | } 52 | }; 53 | 54 | } 55 | 56 | #endif /* FEATURE_SE_HASHMAP_ABSTRACT_H_ */ 57 | -------------------------------------------------------------------------------- /make_msvc_custom_project.bat: -------------------------------------------------------------------------------- 1 | echo off 2 | echo make feature_se VS2015 project 3 | if not defined VS140COMNTOOLS ( 4 | echo vs2015 NOT FOUND. 5 | exit /B -1 6 | ) 7 | echo vs2015 found. 8 | where cmake 9 | if errorlevel 1 ( 10 | echo cmake NOT FOUND. 11 | exit /B -1 12 | ) 13 | echo cmake found 14 | where java 15 | if errorlevel 1 ( 16 | echo java NOT FOUND. 17 | exit /B -1 18 | ) 19 | echo java found 20 | set sh_folder=%~dp0 21 | pushd %sh_folder% 22 | 23 | rem 需要先编译feature_se 24 | if exist project.custom.vs2015 rmdir project.custom.vs2015 /s/q 25 | mkdir project.custom.vs2015 26 | pushd project.custom.vs2015 27 | if not defined VisualStudioVersion ( 28 | echo make MSVC environment ... 29 | call "%VS140COMNTOOLS%..\..\vc/vcvarsall" x86_amd64 30 | ) 31 | 32 | @rem 使用第三方识别库提供的特征比对函数 33 | @rem EXT_SDK_TYPE 识别函数类型 CUSTOM 使用第三方库提供的特征比对函数 34 | @rem 如果EXT_SDK_TYPE指定为CUSTOM,下列参数需要设置: 35 | @rem CUSTOM_FEACOMP_INCLUDE 指定比对函数所在头文件的位置(文件夹全路径) 36 | @rem CUSTOM_FEACOMP_LIBRARY 指定比对函数所在库文件(全路径) 37 | @rem CUSTOM_FEACOMP_HEADERS 指定引用比对函数所需要的头文件名列表,';'分隔,按顺序引用 38 | @rem CUSTOM_FEACOMP_FUNNAME 指定比对函数名, 39 | @rem CUSTOM_FEACOMP_FUNTYPE 指定比对函数类型定义, 40 | @rem 格式:return_type(intput_type0,intput_type1),如果不指定则默认为double(unsigned char*,unsigned char*) 41 | @rem CUSTOM_SYS_HEADERS 指定需要引用的系统头文件名,如windows.h,可不设置 42 | @rem CUSTOM_CODE_BYTE_LEN 特征数据长度(byte) 43 | @rem CUSTOM_FEACOMP_INIT 比对函数初始化代码，可不指定 44 | 45 | echo creating x86_64 Project for Visual Studio 2015 ... 46 | cmake -G "Visual Studio 14 2015 Win64" -DCMAKE_INSTALL_PREFIX=..\release\fse_custom_windows_x86_64 .. ^ 47 | -DEXT_SDK_TYPE=CUSTOM ^ 48 | -DCUSTOM_FEACOMP_INCLUDE=J:\workspace.neon\cassdk54\FSFaceSDK\FSFaceSDK-windows-x86_64\include ^ 49 | -DCUSTOM_FEACOMP_LIBRARY=J:\workspace.neon\cassdk54\FSFaceSDK\FSFaceSDK-windows-x86_64\lib\FSFaceSDK.lib ^ 50 | -DCUSTOM_FEACOMP_HEADERS=FSFaceSDK.h ^ 51 | -DCUSTOM_SYS_HEADERS=windows.h ^ 52 | -DCUSTOM_FEACOMP_FUNNAME=FSCompare ^ 53 | -DCUSTOM_FEACOMP_FUNTYPE="double(unsigned char*, unsigned char*)" ^ 54 | -DCUSTOM_CODE_BYTE_LEN=2560 55 | 56 | popd 57 | popd -------------------------------------------------------------------------------- /feature_se/HashMapCl.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * HashMapCl.cpp 3 | * 4 | * Created on: 2015年10月13日 5 | * Author: guyadong 6 | */ 7 | #include 8 | #include 9 | #include "HashMapCl.h" 10 | #ifdef CASSDK 11 | #include "feature_compare_cas.h" 12 | #elif defined(CUSTOM_FEACOMP) 13 | #include "custom_feature_compare.h" 14 | #else 15 | #include "feature_compare.h" 16 | #endif 17 | namespace gdface { 18 | 19 | HashMapCl::HashMapCl(HASH_TABLE_SIZE_TYPE initCapacity, bool isCopy, float loadFactor, size_t overBlockCapacity) 20 | :HashMap_Abstract(initCapacity, isCopy, loadFactor, overBlockCapacity){ 21 | } 22 | HashMapCl::HashMapCl(HASH_TABLE_SIZE_TYPE initCapacity) 23 | :HashMapCl(initCapacity, true, 0, (size_t)(initCapacity*0.1f)){ 24 | } 25 | /* 通过特征码比对查找表中与code相似度大于阀值threshold(>0&&<=1.0)的前rows个code_bean, 26 | * imgMD5Filter不为nullptr时,只对code_bean.imgMD5包含在imgMD5Filter集合的对象进行特征值比对,否则全表查找 27 | * 返回的对象中包含搜索结果 28 | */ 29 | TopKCodeBean HashMapCl::searchCode(const face_code &code, double threshold, size_t rows, const MD5Set *imgMD5Filter)const{ 30 | if(!(rows>0&&threshold>0.0&&threshold<=1.0)) 31 | throw invalid_argument("argument not match with rows>0&&threshold>0.0&&threshold<=1.0"); 32 | TopKCodeBean top(rows, threshold); 33 | if(nullptr==imgMD5Filter||imgMD5Filter->empty()){ 34 | for_each_break_if([&](const code_bean &node)->bool{ 35 | top.insert(node, feature::compare(node.code, code)); 36 | return false; 37 | }); 38 | }else{ 39 | for_each_break_if([&](const code_bean &node)->bool{ 40 | if(imgMD5Filter->has(node.id)) 41 | top.insert(node, feature::compare(node.code, code)); 42 | return false; 43 | }); 44 | } 45 | return top; 46 | } 47 | HASH_TABLE_SIZE_TYPE HashMapCl::removeBeansByImgMD5(const MD5 &imgMD5){ 48 | return erase_if([&imgMD5](code_bean& bean) {return imgMD5==bean.imgMD5; }); 49 | } 50 | std::vector HashMapCl::getBeansByImgMD5(const MD5 &imgMD5)const{ 51 | vector v; 52 | for_each_break_if([&](const code_bean&bean){ 53 | if(imgMD5==bean.imgMD5) 54 | v.push_back(bean); 55 | return false; 56 | }); 57 | return v; 58 | } 59 | } /* namespace gdface */ 60 | -------------------------------------------------------------------------------- /jni/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 2.6) 2 | # We are only interested in finding jni.h: we do not care about extended JVM 3 | # functionality or the AWT library. 4 | set(JAVA_AWT_LIBRARY NotNeeded) 5 | set(JAVA_JVM_LIBRARY NotNeeded) 6 | if(CMAKE_SYSTEM_NAME STREQUAL "Android") 7 | # android 平台没有jni_md.h 8 | set(JAVA_INCLUDE_PATH2 NotNeeded) 9 | endif() 10 | 11 | set(JAVA_AWT_INCLUDE_PATH NotNeeded) 12 | find_package(JNI REQUIRED) 13 | message(STATUS JAVA_INCLUDE_PATH=${JAVA_INCLUDE_PATH}) 14 | 15 | # 加入jni支持 16 | include_directories(${JAVA_INCLUDE_PATH}) 17 | if(NOT CMAKE_SYSTEM_NAME STREQUAL "Android") 18 | include_directories(${JAVA_INCLUDE_PATH2}) 19 | endif() 20 | 21 | include_directories("${PROJECT_SOURCE_DIR}/feature_se") 22 | #设置变量，表示所有的源文件列表 23 | set(SOURCE_FILES 24 | ${DEPENDENT_SOURCE_DIR}/jni_utilits.cpp 25 | ${DEPENDENT_SOURCE_DIR}/md5/md5.cpp 26 | net_gdface_sdk_fse_FseJniBridge.cpp 27 | jni_mgr.cpp 28 | BeanUtilits.cpp 29 | ) 30 | # CASSDK版本的JNI接口 31 | add_library(${JNI_FSE_LIBNAME} SHARED ${SOURCE_FILES}) 32 | link_static_libstdcxx_if_linux(${JNI_FSE_LIBNAME}) 33 | # 加入依赖库 MSVC编译时使用/MT静态库 34 | target_link_libraries(${JNI_FSE_LIBNAME} fse_cas_static$<$:_mt> $<$:log>) 35 | 36 | 37 | target_include_directories (${JNI_FSE_LIBNAME} PRIVATE ${DEPENDENT_SOURCE_DIR}) 38 | if(EXT_SDK_TYPE STREQUAL "CUSTOM") 39 | target_include_directories(${JNI_FSE_LIBNAME} PRIVATE "${PROJECT_BINARY_DIR}/generated" ${CUSTOM_FEACOMP_INCLUDE}) 40 | endif() 41 | #############设置动态库版本号################### 42 | set_target_properties(${JNI_FSE_LIBNAME} PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR} DEBUG_POSTFIX _d) 43 | message(STATUS "feature_se jni VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR}") 44 | 45 | #######Windows 下DLL 文件名前缀为空 ####### 46 | if(CMAKE_SYSTEM_NAME MATCHES "Windows") 47 | set_property(TARGET ${JNI_FSE_LIBNAME} PROPERTY PREFIX "") 48 | endif(CMAKE_SYSTEM_NAME MATCHES "Windows") 49 | ########安装脚本########## 50 | install(TARGETS ${JNI_FSE_LIBNAME} 51 | RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR} 52 | LIBRARY DESTINATION ${LIBRARY_INSTALL_DIR} 53 | ) 54 | if(MSVC) 55 | # 安装pdb文件 56 | install(FILES $ DESTINATION bin OPTIONAL) 57 | endif(MSVC) 58 | -------------------------------------------------------------------------------- /feature_se/ICodeManager.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ICodeManager.h 3 | * 4 | * Created on: 2015年11月24日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef FEATURE_SE_ICODEMANAGER_H_ 9 | #define FEATURE_SE_ICODEMANAGER_H_ 10 | #include 11 | #include 12 | #include "config.h" 13 | #include "codemgr.h" 14 | 15 | namespace gdface { 16 | #define DEFAULT_FACETABLE_CAPACITY (1024*1024) 17 | class MD5SetImpl; 18 | class FSE_DLL_DECL MD5Set{ 19 | public: 20 | MD5SetImpl * const m_set; 21 | public: 22 | MD5Set(uint32_t initCapacity=0, bool isCopy=false, float loadFactor=0.0, size_t overBlockCapacity=0); 23 | MD5Set(MD5Set&&rv); 24 | bool has(const MD5&); 25 | const MD5& insert(const MD5&key); 26 | virtual ~MD5Set(); 27 | }; 28 | /* 特征码管理对象接口 */ 29 | class FSE_DLL_DECL ICodeManager { 30 | public: 31 | using out_fun=std::function; 32 | /* 用于确定特征码所在并行表位置的字节数下标 */ 33 | static const uint8_t INDEX_CORE = 0; 34 | /* 用于确定特征码所在主机位置的字节数下标 */ 35 | static const uint8_t INDEX_HOST = (INDEX_CORE+1); 36 | ICodeManager()=default; 37 | virtual ~ICodeManager()=default; 38 | /* 释放所有资源 */ 39 | virtual void release()=0; 40 | /* 返回表中数据总数 */ 41 | virtual uint32_t size()const noexcept=0; 42 | /* 向表中加入一个特征码对象code */ 43 | virtual const code_bean& addBean(const code_bean& bean)=0; 44 | /* 向表中获取一个特征码对象code */ 45 | virtual const code_bean* getBean(const MD5 &md5)const noexcept=0; 46 | /* 返回所有属于校验码为imgMD5的图片的特征码对象数组 */ 47 | virtual size_t getBeansByImgMD5(const MD5 &imgMD5,out_fun out)const=0; 48 | /* 从表中删除一个特征码对象 */ 49 | virtual bool removeBean(const MD5 &md5)=0; 50 | /* 从表中删除所有属于imgMD5的特征码对象 返回删除的记录数目 */ 51 | virtual uint32_t removeBeansByImgMD5(const MD5 &imgMD5)=0; 52 | /* 返回哈希表统计信息 */ 53 | virtual std::string statInfo()=0; 54 | /* 通过特征码比对查找表中与code相似度大于阀值threshold(>0&&<=1.0)的前rows个code_bean, 55 | * md5set不为nullptr时,只对code_bean.imgMD5包含在md5set集合的对象进行特征值比对,否则全表查找 56 | * 返回结果数目,out中返回的搜索结果数组 57 | */ 58 | virtual size_t searchCode(const face_code &code, double threshold, size_t rows, code_bean*out, const MD5Set *md5set=nullptr) const=0; 59 | inline bool removeBean(const code_bean &bean) {return removeBean(bean.id);} 60 | inline bool hasBean(const MD5 &md5) const noexcept{return nullptr != getBean(md5);} 61 | inline bool empty(){return 0==size();} 62 | }; 63 | 64 | } /* namespace gdface */ 65 | 66 | #endif /* FEATURE_SE_ICODEMANAGER_H_ */ 67 | -------------------------------------------------------------------------------- /feature_se/feature_data_types.h: -------------------------------------------------------------------------------- 1 | /* 2 | * data_types.h 3 | * 4 | * Created on: 2018年4月14日 5 | * Author: 10km 6 | */ 7 | 8 | #ifndef FEATURE_SE_FEATURE_DATA_TYPES_H_ 9 | #define FEATURE_SE_FEATURE_DATA_TYPES_H_ 10 | #include 11 | #include "cross_define.h" 12 | #ifdef CASSDK 13 | // CASSDK x86/64平台可见光算法 14 | // 定义 CASSDK的特征码长度(byte) 15 | #define CODE_BYTE_LENGTH 2560 16 | #define CODE_END_WITH_SUM 0 17 | #elif defined CUSTOM_FEACOMP 18 | // 使用自定义算法特征比较函数,暂时不支持 19 | #ifndef CODE_BYTE_LENGTH 20 | #error undefine CODE_BYTE_LENGTH 21 | #endif 22 | #define CODE_END_WITH_SUM 0 23 | #else 24 | // 基于欧氏距离计算相似度的内置比较算法 25 | // 定义特征码数据类型,如果未定义默认为double 26 | #ifndef CODE_ELEM_TYPE 27 | #define CODE_ELEM_TYPE double 28 | #endif 29 | #ifndef CODE_ELEM_LEN 30 | // 定义 特征码总长度(浮点类型) 31 | #define CODE_FLOAT_NUM 210 32 | // 旧版本face sdk 33 | #define OLD_FACE_SDK 1 34 | #else 35 | #define CODE_FLOAT_NUM CODE_ELEM_LEN 36 | #endif 37 | // 定义特征值末尾是否有点积和(double),默认没有 38 | #ifndef CODE_END_WITH_SUM 39 | #define CODE_END_WITH 0 40 | #endif 41 | // 特征码长度(byte) 42 | #if CODE_END_WITH_SUM 43 | #define CODE_BYTE_LENGTH CODE_FLOAT_NUM*sizeof(CODE_ELEM_TYPE) + sizeof(double) 44 | #else 45 | #define CODE_BYTE_LENGTH CODE_FLOAT_NUM*sizeof(CODE_ELEM_TYPE) 46 | #endif 47 | #endif 48 | #define MD5_SIZE 16 49 | typedef union _MD5{ 50 | uint8_t c[MD5_SIZE]; 51 | uint32_t i[MD5_SIZE/sizeof(uint32_t)]; 52 | uint64_t l[MD5_SIZE/sizeof(uint64_t)]; 53 | #ifdef __cplusplus 54 | bool operator==(const _MD5 &src)const noexcept{ 55 | return l[0] == src.l[0] && l[1] == src.l[1]; 56 | } 57 | bool operator!=(const _MD5 &src)const noexcept{ 58 | return l[0] != src.l[0] || l[1] != src.l[1]; 59 | } 60 | bool is_null()const noexcept { 61 | return 0 == l[0] && 0 == l[1]; 62 | } 63 | #endif 64 | }MD5; 65 | typedef char MD5_STR[MD5_SIZE*2+1]; 66 | #if defined(CUSTOM_FEACOMP) || defined(CASSDK) 67 | // CASSDK 或其他自定义算法特征码类型 68 | typedef struct _ALIGN_16 _face_code{ 69 | unsigned char element[CODE_BYTE_LENGTH]; 70 | }face_code; 71 | #else 72 | // 默认基于点积(欧氏距离)计算相似度的特征码类型 73 | typedef struct _ALIGN_16 _face_code { 74 | CODE_ELEM_TYPE element[CODE_FLOAT_NUM]; 75 | double sum;/** element的点积和 */ 76 | }face_code; 77 | #endif 78 | typedef struct _code_bean { 79 | MD5 id; 80 | MD5 imgMD5; 81 | face_code code; 82 | double similarity; 83 | }code_bean; 84 | 85 | #endif /* FEATURE_SE_FEATURE_DATA_TYPES_H_ */ 86 | -------------------------------------------------------------------------------- /feature_se/FeatureCompareSpeedTest.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "cross_define.h" 7 | #include "feature_compare.h" 8 | using namespace gdface; 9 | int main() { 10 | //float feature[210]; 11 | //float f=0.75f; 12 | //for(int i=0;i<224;++i)feature[i]=0.75f; 13 | //auto farrays=std::make_shared(100*210); 14 | float _ALIGN_(16) farrays[224*100]; 15 | for(int i=0;i<100*224;++i){ 16 | farrays[i]=(float)(rand()); 17 | } 18 | clock_t start, end; 19 | const int count = 1000 * 10000; 20 | double sq1=1.4,sq2=1.5; 21 | //double res; 22 | std::vector res(count); 23 | start = clock(); 24 | for (int i = 0; i < count; ++i) { 25 | res[i]=feature::compareV2(farrays+(i%100)*224,farrays+(i%100)*224,sq1,sq2); 26 | } 27 | end = clock(); 28 | printf("dot_product_default:\nV2 res=%lf %d takes %lf s \n", res[0],count, ((double) (end - start) / CLOCKS_PER_SEC)); 29 | printf("speed: %lf ten thousand /s \n", count/10000 / ((double) (end - start) / CLOCKS_PER_SEC)); 30 | 31 | start = clock(); 32 | for (int i = 0; i < count; ++i) { 33 | res[i]=feature::compareV2(farrays+(i%100)*224,farrays+(i%100)*224,sq1,sq2); 34 | } 35 | end = clock(); 36 | printf("dot_product_default[recursive]:\nV2 res=%lf %d takes %lf s \n", res[0],count, ((double) (end - start) / CLOCKS_PER_SEC)); 37 | printf("speed: %lf ten thousand /s \n", count/10000 / ((double) (end - start) / CLOCKS_PER_SEC)); 38 | 39 | start = clock(); 40 | for (int i = 0; i < count; ++i) { 41 | res[i]=feature::compareV2(farrays+(i%100)*224,farrays+(i%100)*224,sq1,sq2); 42 | } 43 | end = clock(); 44 | printf("dot_product_simd:\nV2 res=%lf %d takes %lf s \n", res[0],count, ((double) (end - start) / CLOCKS_PER_SEC)); 45 | printf("speed: %lf ten thousand /s \n", count/10000 / ((double) (end - start) / CLOCKS_PER_SEC)); 46 | 47 | start = clock(); 48 | for (int i = 0; i < count; ++i) { 49 | res[i]=feature::compareV2(farrays+(i%100)*224,farrays+(i%100)*224,sq1,sq2); 50 | } 51 | end = clock(); 52 | printf("dot_product_simd[recursive]:\nV2 res=%lf %d takes %lf s \n", res[0],count, ((double) (end - start) / CLOCKS_PER_SEC)); 53 | printf("speed: %lf ten thousand /s \n", count/10000 / ((double) (end - start) / CLOCKS_PER_SEC)); 54 | 55 | 56 | return 0; 57 | } 58 | -------------------------------------------------------------------------------- /feature_se/feature_se.h: -------------------------------------------------------------------------------- 1 | /* 2 | * feature_se.h 3 | * (人脸)特征搜索引擎(Feature Search Engine)C接口定义 4 | * 所有返回int类型的函数返回值<0,代表异常，这里可以通过fse_error_msg()获取错误信息,只在当前线程有效 5 | * Created on: 2018年4月14日 6 | * Author: guyadong 7 | */ 8 | 9 | #ifndef FEATURE_SE_FEATURE_SE_H_ 10 | #define FEATURE_SE_FEATURE_SE_H_ 11 | #include "config.h" 12 | #include "feature_data_types.h" 13 | #ifdef __cplusplus 14 | extern "C" { 15 | #endif 16 | 17 | /* 返回当前线程的错误信息 */ 18 | FSE_API const char* fse_error_msg(); 19 | /* (人脸)特征搜索引擎初始化 20 | * initCapacity 哈希表初始容量 21 | * loadFactor 哈希表扩容因子 22 | * overBlockCapacity 过载表容量 23 | * 正常返回0, <0 出错 24 | */ 25 | FSE_API int fse_init(uint32_t initCapacity, float loadFactor, size_t overBlockCapacity); 26 | /* 释放所有资源 27 | * 正常返回0, <0 出错 28 | */ 29 | FSE_API int fse_release(); 30 | /* 在内存表中根据比对相似度进行特征码搜索 31 | * code 要比对的特征码 32 | * threshold 相似度阀值 33 | * rows 最大返回的记录数 34 | * out 搜索结果输出缓冲区,长度由rows定义 35 | * imgMD5Array 比对的图像范围(图像集),为null搜索所有特征 36 | * md5Count 图像范围大小 37 | * 返回 搜索结果数目,< 0出错 38 | */ 39 | FSE_API int fse_searchCode(const face_code *code, double threshold, size_t rows, code_bean out[], const MD5 imgMD5Array[], size_t md5Count); 40 | /* 根据特征码ID在表中查找指定的记录 41 | * md5 特征码ID,为null时返回0 42 | * out 结果输出,为null时出错 43 | * 返回 0没有找到,1找到,<0 出错 44 | */ 45 | FSE_API int fse_getFeature(const MD5 *md5, code_bean* out); 46 | /* 添加一组特征码到内存表,beans为null返回0 47 | * 正常返回添加的特征记录数,beans为null时返回0,< 0 出错 48 | */ 49 | FSE_API int fse_addFeatures(const code_bean beans[], size_t beanCount); 50 | /* 添加一条特征码到内存表,bean为null返回0 51 | * 正常返回1,< 0 出错 52 | */ 53 | FSE_API int fse_addFeature(const code_bean* bean); 54 | /* 删除md5s指定的一组特征,md5s为null返回0 55 | * 删除成功返回删除的记录数,< 0出错 56 | */ 57 | FSE_API int fse_removeFeatures(const MD5 md5s[], size_t md5Count); 58 | /* 删除md5指定的特征,md5为null返回0 59 | * 删除成功返回1,失败返回0,< 0出错 60 | */ 61 | FSE_API int fse_removeFeature(MD5 *md5); 62 | /* 根据图像imgMD5数组删除所有指定的记录 63 | * imgMD5s 特征码所在图像的MD5校验码数组,为null返回0 64 | * 返回返回删除的记录数,< 0 出错 65 | */ 66 | FSE_API int fse_removeFeaturesByImgMD5s(const MD5 imgMD5s[], size_t md5Count); 67 | /* 根据图像imgMD5删除所有指定的记录 68 | * imgMD5 特征码所在图像的MD5校验码 69 | * 返回返回删除的记录数,< 0 出错 70 | */ 71 | FSE_API int fse_removeFeaturesByImgMD5(const MD5 *imgMD5); 72 | /* 返回内存表中元素个数 */ 73 | FSE_API int fse_size(); 74 | /* 返回哈希表统计信息字符串指针(当前线程有效,用于debug测试),出错返回null */ 75 | FSE_API const char* fse_statInfo(); 76 | #ifdef __cplusplus 77 | } 78 | #endif 79 | #endif /* FEATURE_SE_FEATURE_SE_H_ */ 80 | -------------------------------------------------------------------------------- /feature_se/ICodeManagerCPU.h: -------------------------------------------------------------------------------- 1 | /* 2 | * FaceCodeManager.h 3 | * 4 | * Created on: 2015年10月14日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef FEATURE_SE_ICODEMANAGERCPU_H_ 9 | #define FEATURE_SE_ICODEMANAGERCPU_H_ 10 | #include 11 | #include 12 | #include 13 | #include "HashMapCl.h" 14 | #include "ICodeManager.h" 15 | #include "MiniThreadPool.h" 16 | namespace gdface { 17 | 18 | class ICodeManagerCPU:public ICodeManager { 19 | private: 20 | static ICodeManagerCPU* m_instance; 21 | vector m_maps; 22 | mutable MiniThreadPool m_pool; 23 | /* CPU核数 */ 24 | static const decltype(std::thread::hardware_concurrency()) CONCURRENCY; 25 | ICodeManagerCPU(HASH_TABLE_SIZE_TYPE initCapacity, bool isCopy, float loadFactor, size_t overBlockCapacity); 26 | class GC { 27 | public: 28 | ~GC() noexcept{ 29 | // singleton模式下自动销毁实例 30 | _release(); 31 | } 32 | }; 33 | /* 计算指定MD5所在的表的下标 */ 34 | inline static decltype(CONCURRENCY) mapIndexOf(const MD5 &md5)noexcept{ 35 | return md5.c[INDEX_CORE]%CONCURRENCY; 36 | } 37 | static const GC gc; 38 | public: 39 | static ICodeManagerCPU& getInstance(HASH_TABLE_SIZE_TYPE initCapacity=DEFAULT_FACETABLE_CAPACITY, bool isCopy=true, float loadFactor=0, size_t overBlockCapacity=0) { 40 | init(initCapacity, isCopy, loadFactor, overBlockCapacity); 41 | return *m_instance; 42 | } 43 | static void init(HASH_TABLE_SIZE_TYPE initCapacity=DEFAULT_FACETABLE_CAPACITY, bool isCopy=true, float loadFactor=0, size_t overBlockCapacity=0); 44 | static void _release(); 45 | // 禁止复制构造函数 46 | ICodeManagerCPU(const ICodeManagerCPU&) = delete; 47 | virtual ~ICodeManagerCPU(); 48 | virtual void release(); 49 | virtual uint32_t size()const noexcept; 50 | /* 向表中加入一个特征码对象code */ 51 | virtual const code_bean& addBean(const code_bean& bean); 52 | virtual const code_bean* getBean(const MD5 &md5)const noexcept; 53 | virtual size_t getBeansByImgMD5(const MD5 &imgMD5,out_fun out)const; 54 | virtual bool removeBean(const MD5 &md5); 55 | virtual HASH_TABLE_SIZE_TYPE removeBeansByImgMD5(const MD5 &imgMD5); 56 | virtual std::string statInfo(); 57 | virtual size_t searchCode(const face_code &code, double threshold, size_t rows, code_bean*out, const MD5Set *md5set=nullptr)const; 58 | TopKCodeBean searchCode(const face_code &code, TopKCodeBean::cmp_type threshold, size_t rows, const MD5Set *imgMD5Filter=nullptr)const; 59 | }; 60 | 61 | 62 | 63 | } /* namespace gdface */ 64 | 65 | #endif /* FEATURE_SE_ICODEMANAGERCPU_H_ */ 66 | -------------------------------------------------------------------------------- /feature_se/hash_fun.h: -------------------------------------------------------------------------------- 1 | /* 2 | * hash_fun.h 3 | * 4 | * Created on: 2015年12月14日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef FEATURE_SE_HASH_FUN_H_ 9 | #define FEATURE_SE_HASH_FUN_H_ 10 | #include 11 | #include 12 | namespace gdface{ 13 | template 14 | typename std::enable_if< (TAIL==0)>::type hash_code_tail(uint64_t &hash, const uint64_t* u,uint64_t seed) noexcept { 15 | } 16 | template 17 | typename std::enable_if< (TAIL>0)>::type hash_code_tail(uint64_t &hash, const uint64_t* u,uint64_t seed) noexcept { 18 | #if defined( _MSC_VER) || (defined(__GNUC__) && __BYTE_ORDER__ ==__ORDER_LITTLE_ENDIAN__) 19 | hash = hash * seed + ((*u) & ((1ULL << (TAIL << 3)) - 1)); // 小端模式 20 | #elif defined(__GNUC__) && __BYTE_ORDER__ ==__ORDER_BIG_ENDIAN__ 21 | hash = hash * seed + ((*u) >> ((sizeof(uint64_t) - TAIL) << 3)); // 大端模式 22 | #else 23 | #error unexpected c complier (msc/gcc) 24 | #endif 25 | } 26 | /* 哈希函数实现 */ 27 | template 28 | size_t hash_code( const T& t) noexcept { 29 | uint64_t hash = 0; 30 | auto p = reinterpret_cast(const_cast(std::addressof(t))); 31 | auto const u = p + (sizeof(T)>>3);//sizeof(uint64_t); 32 | uint64_t seed = 131; // 31 131 1313 13131 131313 etc..BKDRHash 33 | for (; p < u; p ++) hash = hash * seed + *p; 34 | hash_code_tail(hash,u,seed); 35 | return (size_t) hash; 36 | } 37 | /*template 38 | size_t hash_code( const T& t) noexcept { 39 | size_t hash = 0; 40 | auto p = reinterpret_cast(std::addressof(t)); 41 | size_t seed = 131; // 31 131 1313 13131 131313 etc..BKDRHash 42 | for (auto u = p +sizeof(T); p < u; ++p) hash = hash * seed + *p; 43 | return (size_t) hash; 44 | }*/ 45 | /* 返回获取hash值的一元函数, 46 | * 如果T有std::hash特例实现返回std::hash,否则提供缺省的hash实现 47 | */ 48 | template 49 | struct hash_fun{ 50 | /* 缺省的hash实现 */ 51 | struct default_hash { 52 | typedef T argument_type; 53 | typedef std::size_t result_type; 54 | result_type operator()(argument_type const& t) const noexcept {return hash_code(t); } 55 | }; 56 | /* SFINAE 判断T有没有std::hash特例实现 */ 57 | template static std::hash test(decltype(std::declval>().operator()(std::declval()))); 58 | template static default_hash test(...); 59 | using type =decltype(test(0)); 60 | type fun; 61 | }; 62 | /* 判断有没有std::hash实现 */ 63 | template 64 | struct has_hash_specific{ 65 | template static auto test(int)-> decltype(std::declval>().operator()(std::declval())); 66 | template static void test(...); 67 | enum{value=!std::is_void(0))>::value}; 68 | //通过判断test(0)返回值是否为void来判断是否有hash特例 69 | }; 70 | 71 | } /* namespace gdface*/ 72 | 73 | 74 | 75 | #endif /* FEATURE_SE_HASH_FUN_H_ */ 76 | -------------------------------------------------------------------------------- /jni/net_gdface_sdk_fse_FseJniBridge.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class net_gdface_sdk_fse_FseJniBridge */ 4 | 5 | #ifndef _Included_net_gdface_sdk_fse_FseJniBridge 6 | #define _Included_net_gdface_sdk_fse_FseJniBridge 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | #undef net_gdface_sdk_fse_FseJniBridge_DEFAULT_INITIAL_CAPACITY 11 | #define net_gdface_sdk_fse_FseJniBridge_DEFAULT_INITIAL_CAPACITY 16777216L 12 | #undef net_gdface_sdk_fse_FseJniBridge_DEFAULT_LOAD_FACTOR 13 | #define net_gdface_sdk_fse_FseJniBridge_DEFAULT_LOAD_FACTOR 0.75f 14 | #undef net_gdface_sdk_fse_FseJniBridge_DEFAULT_OVERBLOCK_CAPACITY 15 | #define net_gdface_sdk_fse_FseJniBridge_DEFAULT_OVERBLOCK_CAPACITY 1024L 16 | #undef net_gdface_sdk_fse_FseJniBridge_DEFAULT_MODE 17 | #define net_gdface_sdk_fse_FseJniBridge_DEFAULT_MODE 0L 18 | /* 19 | * Class: net_gdface_sdk_fse_FseJniBridge 20 | * Method: init 21 | * Signature: (IIFI)V 22 | */ 23 | JNIEXPORT void JNICALL Java_net_gdface_sdk_fse_FseJniBridge_init 24 | (JNIEnv *, jclass, jint, jint, jfloat, jint); 25 | 26 | /* 27 | * Class: net_gdface_sdk_fse_FseJniBridge 28 | * Method: release 29 | * Signature: ()V 30 | */ 31 | JNIEXPORT void JNICALL Java_net_gdface_sdk_fse_FseJniBridge_release 32 | (JNIEnv *, jclass); 33 | 34 | /* 35 | * Class: net_gdface_sdk_fse_FseJniBridge 36 | * Method: searchCode 37 | * Signature: ([BDI[Ljava/lang/String;)[Lnet/gdface/sdk/fse/CodeBean; 38 | */ 39 | JNIEXPORT jobjectArray JNICALL Java_net_gdface_sdk_fse_FseJniBridge_searchCode 40 | (JNIEnv *, jclass, jbyteArray, jdouble, jint, jobjectArray); 41 | 42 | /* 43 | * Class: net_gdface_sdk_fse_FseJniBridge 44 | * Method: getFeature 45 | * Signature: ([B)Lnet/gdface/sdk/fse/CodeBean; 46 | */ 47 | JNIEXPORT jobject JNICALL Java_net_gdface_sdk_fse_FseJniBridge_getFeature 48 | (JNIEnv *, jclass, jbyteArray); 49 | 50 | /* 51 | * Class: net_gdface_sdk_fse_FseJniBridge 52 | * Method: addFeature 53 | * Signature: ([B[BLjava/lang/String;)Z 54 | */ 55 | JNIEXPORT jboolean JNICALL Java_net_gdface_sdk_fse_FseJniBridge_addFeature 56 | (JNIEnv *, jclass, jbyteArray, jbyteArray, jstring); 57 | 58 | /* 59 | * Class: net_gdface_sdk_fse_FseJniBridge 60 | * Method: removeFeature 61 | * Signature: ([B)Z 62 | */ 63 | JNIEXPORT jboolean JNICALL Java_net_gdface_sdk_fse_FseJniBridge_removeFeature 64 | (JNIEnv *, jclass, jbyteArray); 65 | 66 | /* 67 | * Class: net_gdface_sdk_fse_FseJniBridge 68 | * Method: size 69 | * Signature: ()I 70 | */ 71 | JNIEXPORT jint JNICALL Java_net_gdface_sdk_fse_FseJniBridge_size 72 | (JNIEnv *, jclass); 73 | 74 | /* 75 | * Class: net_gdface_sdk_fse_FseJniBridge 76 | * Method: statInfo 77 | * Signature: ()Ljava/lang/String; 78 | */ 79 | JNIEXPORT jstring JNICALL Java_net_gdface_sdk_fse_FseJniBridge_statInfo 80 | (JNIEnv *, jclass); 81 | #ifdef __cplusplus 82 | } 83 | #endif 84 | #endif 85 | -------------------------------------------------------------------------------- /feature_se/codemgr.h: -------------------------------------------------------------------------------- 1 | /* 2 | * codemgr.h 3 | * 4 | * Created on: 2015年10月1日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef FEATURE_SE_CODEMGR_H_ 9 | #define FEATURE_SE_CODEMGR_H_ 10 | #include 11 | #include 12 | #include 13 | #include "common_utilits.h" 14 | #include "feature_data_types.h" 15 | #include "intrinsic_wrapper.h" 16 | namespace gdface{ 17 | typedef signed char FACE_CODE[sizeof(face_code)]; 18 | 19 | inline string MD5toMD5_STR(MD5 const &md5char) { 20 | return com_utilits::bytes_to_hex_string(&md5char,sizeof(MD5)); 21 | } 22 | 23 | inline void PSTRtoMD5(const char * pStr,MD5 &md5) { 24 | com_utilits::hex_to_bytes(pStr,&md5,sizeof(MD5)); 25 | } 26 | 27 | inline bool is_null_MD5(const MD5 *md5) { 28 | return nullptr == md5 || md5->is_null(); 29 | } 30 | 31 | #ifdef CASSDK 32 | #define FACE_CODE_CONVERT(code) code 33 | #elif ((_M_X64||_M_AMD64||_M_IX86) && _MSC_VER) || (__GNUC__&& (__BYTE_ORDER__ ==__ORDER_LITTLE_ENDIAN__)) 34 | #define FACE_CODE_CONVERT(code) code 35 | #elif __GNUC__ && __BYTE_ORDER__ ==__ORDER_BIG_ENDIAN__ 36 | inline face_code _code_reverse(const face_code& code) { 37 | // 当系统为大端时把字节序转换为小端 38 | // gcc下用内置函数转换 39 | face_code new_code; 40 | for(int i=0;i 51 | inline typename std::enable_if<2==sizeof(T),T>::type _reverse_bytes(const T &src) { 52 | T dst=src; 53 | char*ptr=reinterpret_cast(std::addressof(dst)); 54 | std::swap(*ptr,*(ptr+1)); 55 | return dst; 56 | } 57 | template 58 | inline typename std::enable_if<4==sizeof(T),T>::type _reverse_bytes(const T &src) { 59 | T dst=src; 60 | char*ptr=reinterpret_cast(std::addressof(dst)); 61 | std::swap(*ptr,*(ptr+3)); 62 | std::swap(*(ptr+1),*(ptr+2)); 63 | return dst; 64 | } 65 | template 66 | inline typename std::enable_if<8==sizeof(T),T>::type _reverse_bytes(const T &src) { 67 | T dst=src; 68 | char*ptr=reinterpret_cast(std::addressof(dst)); 69 | std::swap(*ptr,*(ptr+7)); 70 | std::swap(*(ptr+1),*(ptr+6)); 71 | std::swap(*(ptr+2),*(ptr+5)); 72 | std::swap(*(ptr+3),*(ptr+4)); 73 | return dst; 74 | } 75 | 76 | inline face_code _code_reverse(const face_code& code) { 77 | //当系统为大端时把字节序转换为小端 78 | face_code new_code; 79 | for(int i=0;i特例化实现 */ 90 | namespace std 91 | { 92 | template<> 93 | struct hash { 94 | typedef MD5 argument_type; 95 | typedef std::size_t result_type; 96 | result_type operator()(argument_type const& md5) const noexcept { 97 | return (result_type)(md5.l[0]^md5.l[1]); 98 | } 99 | }; 100 | } /* namespace std */ 101 | 102 | #endif /* FEATURE_SE_CODEMGR_H_ */ 103 | -------------------------------------------------------------------------------- /feature_se/OverTableManager.h: -------------------------------------------------------------------------------- 1 | /* 2 | * OverTableManager1.h 3 | * 4 | * Created on: 2015年10月12日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef FEATURE_SE_OVERTABLEMANAGER_H_ 9 | #define FEATURE_SE_OVERTABLEMANAGER_H_ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include "gf_utility.h" 19 | #include "hashtable_core.h" 20 | #include "intrinsic_wrapper.h" 21 | using namespace std; 22 | namespace gdface { 23 | 24 | #define MIN_BLOCK_BIT 6 25 | #define MIN_BLOCK_SIZE (1U< m_overTable; 34 | size_t m_free; // 空闲块数目 35 | vector m_occupyMap; // overTable的空闲/占用状态表,每一位代表一个HashNode,1代表空闲，0,代表占用 36 | // 调整capacity尺寸为MIN_BLOCK_SIZE的整数倍 37 | size_t FORMAT_CAPACITY(size_t capacity)const noexcept{ 38 | return capacity > DEFAULT_TABLE_CAPACITY ? 39 | (capacity + MIN_BLOCK_SIZE - 1) 40 | & (~(MIN_BLOCK_SIZE - 1)) : 41 | DEFAULT_TABLE_CAPACITY; 42 | } 43 | public: 44 | explicit OverTableInfo(size_t capacity) : 45 | m_overTable(FORMAT_CAPACITY(capacity))// 分配的内存块初始化为0 46 | ,m_free(m_overTable.size()) 47 | ,m_occupyMap(m_overTable.size() >> MIN_BLOCK_BIT){ 48 | // map内存块所有位初始化为1 49 | memset(m_occupyMap.data(), -1, m_occupyMap.size() * sizeof(OM_ELEMENT)); 50 | } 51 | /* 允许移动构造 */ 52 | OverTableInfo(OverTableInfo&& rv)=default; 53 | /* 54 | * 返回第一个找到的空闲内存块,并将该内存块置为占用 55 | * 找不到返回nullptr 56 | */ 57 | HASH_NODE_PTR newNode()noexcept { 58 | if(!full()){ 59 | auto offset = -1; 60 | for (size_t seg=0,size=m_occupyMap.size(); seg < size; ++seg) { 61 | auto &om=m_occupyMap[seg]; 62 | if ((offset = intrinsics::_bsr(om)) >= 0) { 63 | assert(offset<(sizeof(OM_ELEMENT)<<3)); 64 | om &= ~(UINT64_C(1) << offset); //将该位置0(占用) 65 | //om=bitset<64>(om).set(offset,false).to_ullong(); 66 | assert(((seg << 6) | offset) < m_overTable.size()); 67 | assert(m_free>0); 68 | --m_free; 69 | return m_overTable.data()+((seg << 6) | offset); 70 | } 71 | } 72 | } 73 | return nullptr; 74 | } 75 | bool deleteNode(HashNode& node)noexcept { 76 | assert(nullptr!=std::addressof(node)); 77 | if(!empty()){ 78 | // 判断指针是否在本块内存 79 | auto dist=std::addressof(node) - m_overTable.data(); 80 | if (dist>=0 &&(size_t)dist > MIN_BLOCK_BIT; 82 | auto offset = dist & (MIN_BLOCK_SIZE - 1); 83 | assert(offset<64&&offset>=0); 84 | auto &om=m_occupyMap[seg]; 85 | om |= om | (UINT64_C(1) << offset); // 将该位置1(空闲) 86 | node.next=nullptr,node.obj=nullptr; 87 | ++m_free; 88 | assert(m_free<=m_overTable.size()); 89 | return true; 90 | } 91 | } 92 | return false; 93 | } 94 | bool full()const noexcept {return 0==m_free;} 95 | bool empty()const noexcept{return m_overTable.size()==m_free;} 96 | size_t free()const noexcept {return m_free;} 97 | }; 98 | const size_t TABLECAPACITY; 99 | list m_overList; // 公共溢出区 100 | public: 101 | HASH_NODE_PTR newNode(); 102 | bool deleteNode(HashNode& node)noexcept; 103 | OverTableManager(size_t capacity); 104 | void print_info()const{ 105 | int count=0; 106 | cout<<"om:["< 11 | #include 12 | #include 13 | #include 14 | #include "jni_utilits.h" 15 | #include "codemgr.h" 16 | namespace gdface { 17 | class BeanUtilits { 18 | public: 19 | #define JAVA_STRING "Ljava/lang/String;" 20 | #define CODEBEAN "net/gdface/sdk/fse/CodeBean" 21 | #define CODEBEAN_ID "id" 22 | #define CODEBEAN_CODE "code" 23 | #define CODEBEAN_IMGMD5 "imgMD5" 24 | #define CODEBEAN_SIMILARITY "similarity" 25 | static raii_var toJCodeBean(const code_bean& bean, jni_utilits::JavaClassMirror& mirror,jboolean full=true); 26 | // 将id,code,imgMD5填充到bean对应的字段 27 | static bool tocodeBean(code_bean& bean, jbyteArray id, jbyteArray code, jstring imgMD5); 28 | 29 | static bool tocodeBean(code_bean& bean,jobject obj, jni_utilits::JavaClassMirror& mirror); 30 | static bool jstringToMD5(jstring jstr, MD5& md5); 31 | static bool jbytearraytoMD5(jbyteArray bytes, MD5& md5); 32 | static raii_var MD5toJString(const MD5&md5) { 33 | return jni_utilits::raii_NewStringUTF(MD5toMD5_STR(md5).data()); 34 | } 35 | static raii_varMD5tojbyteArray(const MD5&md5){ 36 | return jni_utilits::tojbytearray((jbyte*) (&md5), (jsize)sizeof(MD5)); 37 | } 38 | static raii_varface_codetojbyteArray(const face_code &code){ 39 | #if defined(CUSTOM_FEACOMP) || defined(CASSDK) 40 | return jni_utilits::tojbytearray((jbyte*)(&code), (jsize)sizeof(face_code)); 41 | #elif CODE_END_WITH_SUM 42 | return jni_utilits::tojbytearray((jbyte*)(&code), (jsize)sizeof(face_code)); 43 | #else 44 | // 特征码中不包含点积和时只将element数组转为java array 45 | return jni_utilits::tojbytearray((jbyte*)(&code.element), (jsize)sizeof(code.element)); 46 | #endif 47 | } 48 | static bool jbytearraytoface_code(jbyteArray bytes, face_code& code); 49 | static bool jdoublearraytoface_code(jdoubleArray bytes, face_code& code); 50 | /* 生成 code_bean镜像描述 */ 51 | static jni_utilits::JavaClassMirror JavaClassMirrorForCodeBean(){ 52 | return jni_utilits::JavaClassMirror( 53 | CODEBEAN, 54 | { "", "()V" }, 55 | { {CODEBEAN_ID, "[B"}, 56 | {CODEBEAN_CODE, "[B"}, 57 | {CODEBEAN_IMGMD5, JAVA_STRING}, 58 | {CODEBEAN_SIMILARITY, "D"} 59 | }); 60 | } 61 | 62 | static std::shared_ptr > jmd5settoMD5Vector(jobjectArray jset); 63 | 64 | static void output_md5(const MD5& md5, const char* prefix = "") { 65 | cout << prefix << " [" << MD5toMD5_STR(md5) << "]" << endl; 66 | } 67 | static void output_bean(const code_bean& bean, const char* prefix = "") { 68 | cout << prefix << " id[" << MD5toMD5_STR(bean.id) << "]img[" << MD5toMD5_STR(bean.imgMD5).data() << "]" << endl; 69 | } 70 | static raii_var code_bean_ptr_tojobjectArray(const code_bean* src, jsize size, jni_utilits::JavaClassMirror& mirror, jboolean full); 71 | /*static jobject toJCodeBean(JNIEnv* env, const code_bean& bean) { 72 | auto code_bean_class =jni_utilits::raii_FindClass_LocalRef("Lnet/gdface/facedbsdk/local/CodeCacheManager$CodeBean;"); 73 | auto constructor = env->GetMethodID(code_bean_class.get(), "", "()V"); 74 | auto obj = env->NewObject(code_bean_class.get(), constructor); 75 | auto field_id = env->GetFieldID(code_bean_class.get(), "id", "[B"); 76 | env->SetObjectField(obj, field_id, jni_utilits::tojbytearray((jbyte*) (&bean.id), (jsize) sizeof(bean.id)).get()); 77 | auto field_code = env->GetFieldID(code_bean_class.get(), "code", "[B"); 78 | env->SetObjectField(obj, field_code, 79 | jni_utilits::tojbytearray((jbyte*) &(FACE_CODE_CONVERT(bean.code)), (jsize) sizeof(bean.code)).get()); 80 | auto field_imgMD5 = env->GetFieldID(code_bean_class.get(), "imgMD5", "Ljava/lang/String;"); 81 | env->SetObjectField(obj, field_imgMD5, MD5toJString(bean.imgMD5).get()); 82 | auto field_similarity = env->GetFieldID(code_bean_class.get(), "similarity", "D"); 83 | env->SetFloatField(code_bean_class.get(), field_similarity, (jfloat) (((bean.similarity)))); 84 | return obj; 85 | }*/ 86 | }; 87 | 88 | 89 | } /* namespace gdface */ 90 | 91 | #endif /* JNI_BEANUTILITS_H_ */ 92 | -------------------------------------------------------------------------------- /feature_se/ICodeManagerCPU.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * ICodeManagerCPU.cpp 3 | * 4 | * Created on: 2015年10月14日 5 | * Author: guyadong 6 | */ 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "ICodeManagerCPU.h" 12 | #include "MD5SetImpl.h" 13 | 14 | namespace gdface { 15 | 16 | ICodeManagerCPU::ICodeManagerCPU(HASH_TABLE_SIZE_TYPE initCapacity, bool isCopy, float loadFactor, size_t overBlockCapacity):m_pool(){ 17 | //初始化所有并行表 18 | for(auto i = CONCURRENCY;i>0;--i){ 19 | m_maps.emplace_back(initCapacity/CONCURRENCY,isCopy, loadFactor, overBlockCapacity); 20 | } 21 | } 22 | ICodeManagerCPU::~ICodeManagerCPU()=default; 23 | void ICodeManagerCPU::_release(){ 24 | static std::once_flag oc; 25 | if(nullptr != m_instance){ 26 | std::call_once(oc, [] { delete m_instance;m_instance=nullptr;}); 27 | } 28 | } 29 | 30 | void ICodeManagerCPU::init(HASH_TABLE_SIZE_TYPE initCapacity, bool isCopy, float loadFactor, size_t overBlockCapacity) { 31 | static std::once_flag oc; 32 | std::call_once(oc, [&] { m_instance = new ICodeManagerCPU(initCapacity, isCopy, loadFactor, overBlockCapacity);}); 33 | } 34 | 35 | uint32_t ICodeManagerCPU::size()const noexcept{ 36 | assert(m_maps.size()>0); 37 | auto count=m_maps[0].size(); 38 | for(auto i=m_maps.size();i>1;--i){ 39 | count+=m_maps[i-1].size(); 40 | } 41 | return count; 42 | } 43 | inline void ICodeManagerCPU::release() { 44 | _release(); 45 | } 46 | inline const code_bean& ICodeManagerCPU::addBean(const code_bean& bean){ 47 | return m_maps[mapIndexOf(bean.id)].put(bean); 48 | } 49 | 50 | inline const code_bean* ICodeManagerCPU::getBean(const MD5 &md5)const noexcept{ 51 | return m_maps[mapIndexOf(md5)].get(md5); 52 | } 53 | size_t ICodeManagerCPU::getBeansByImgMD5(const MD5 &imgMD5,out_fun out)const{ 54 | if(nullptr == out){ 55 | throw std::invalid_argument("the argument fun must not be nullptr"); 56 | } 57 | using return_type=decltype(m_maps[0].getBeansByImgMD5(imgMD5)); 58 | std::vector< std::future > futures(m_maps.size()); 59 | // 向线程池加入任务，并行查找 60 | for (auto i = futures.size(); i > 0; --i){ 61 | futures[i - 1] = m_pool.enqueue([this,imgMD5,i](){ return m_maps[i - 1].getBeansByImgMD5(imgMD5);}); 62 | } 63 | size_t count=0; 64 | for (auto i = futures.size(); i > 0; --i){ 65 | auto v=futures[i - 1].get(); 66 | for( auto const b:v)out(b); 67 | count+=v.size(); 68 | } 69 | return count; 70 | } 71 | inline bool ICodeManagerCPU::removeBean(const MD5 &md5){ 72 | return m_maps[mapIndexOf(md5)].remove(md5); 73 | } 74 | HASH_TABLE_SIZE_TYPE ICodeManagerCPU::removeBeansByImgMD5(const MD5 &imgMD5){ 75 | std::vector< std::future > futures(m_maps.size()); 76 | // 向线程池加入任务，并行删除 77 | for (auto i = futures.size(); i > 0; --i){ 78 | futures[i - 1] = m_pool.enqueue([this,imgMD5,i](){ return m_maps[i - 1].removeBeansByImgMD5(imgMD5);}); 79 | } 80 | HASH_TABLE_SIZE_TYPE count=0; 81 | for (auto i = futures.size(); i > 0; --i){ 82 | count+=futures[i - 1].get(); 83 | } 84 | return count; 85 | } 86 | std::string ICodeManagerCPU::statInfo(){ 87 | stringstream out; 88 | for(auto i=m_maps.size();i>0;--i){ 89 | out<<"map["< > futures(m_maps.size()); 96 | // 向线程池加入搜索任务，并行搜索 97 | for (auto i = futures.size(); i > 0; --i) { 98 | futures[i - 1] = const_cast(m_pool).enqueue( 99 | [code, threshold, rows, md5set, i, this]() { 100 | //cout<<"working m_maps["<m_maps[i - 1].searchCode(code, threshold, rows, 102 | nullptr == md5set ? nullptr : static_cast(md5set->m_set)); 103 | }); 104 | 105 | } 106 | // 合并排序结果 107 | TopKCodeBean top(rows, threshold), buf(rows, threshold); 108 | for (auto i = futures.size(); i > 0; --i) { 109 | top.merge(futures[i - 1].get(), std::addressof(buf)); 110 | } 111 | return top; 112 | } 113 | size_t ICodeManagerCPU::searchCode(const face_code &code, double threshold, size_t rows, code_bean*out, const MD5Set *md5set)const{ 114 | assert(nullptr!=out); 115 | auto v=searchCode(code,threshold,rows,md5set); 116 | v.result_to(out); 117 | return v.size(); 118 | } 119 | ICodeManagerCPU* ICodeManagerCPU::m_instance=nullptr; 120 | const ICodeManagerCPU::GC ICodeManagerCPU::gc; 121 | const decltype(std::thread::hardware_concurrency()) ICodeManagerCPU::CONCURRENCY=std::thread::hardware_concurrency(); 122 | } /* namespace gdface */ 123 | -------------------------------------------------------------------------------- /jni/net_gdface_sdk_fse_FseJniBridge.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * net_gdface_facedbsdk_local_CodeCacheManager.cpp 3 | * 4 | * Created on: 2015年10月2日 5 | * Author: guyadong 6 | */ 7 | #include "net_gdface_sdk_fse_FseJniBridge.h" 8 | #include "jni_utilits.h" 9 | #include "BeanUtilits.h" 10 | #include "feature_se.h" 11 | #include "JNIContext.h" 12 | #include "sample_log.h" 13 | #include "md5/md5.h" 14 | 15 | using namespace gdface; 16 | using namespace std; 17 | // JNI 上下文对象全局指针 18 | JNIContext* context; 19 | void throwJNIException(const char* msg) { 20 | jni_utilits::throwByName("net/gdface/sdk/fse/JNIException", msg); 21 | } 22 | JNIEXPORT void JNICALL Java_net_gdface_sdk_fse_FseJniBridge_init 23 | (JNIEnv *env, jclass,jint mode, jint initCapacity, jfloat loadFactor, jint overBlockCapacity){ 24 | // 初始化表对象 25 | auto result = fse_init(initCapacity, loadFactor, overBlockCapacity); 26 | if (result >= 0) { 27 | context = new JNIContext(BeanUtilits::JavaClassMirrorForCodeBean()); 28 | } 29 | else { 30 | throwJNIException(fse_error_msg()); 31 | } 32 | 33 | } 34 | 35 | JNIEXPORT void JNICALL Java_net_gdface_sdk_fse_FseJniBridge_release 36 | (JNIEnv *, jclass){ 37 | SAMPLE_OUT("feature_se jni release..."); 38 | delete context; 39 | auto result = fse_release(); 40 | if (result < 0) { 41 | throwJNIException(fse_error_msg()); 42 | } 43 | } 44 | 45 | JNIEXPORT jobjectArray JNICALL Java_net_gdface_sdk_fse_FseJniBridge_searchCode 46 | (JNIEnv *env, jclass, jbyteArray jfeature, jdouble threshold, jint rows, jobjectArray imgMD5Set){ 47 | try{ 48 | auto feature = std::make_shared(); 49 | if(!BeanUtilits::jbytearraytoface_code(jfeature,*feature)){ 50 | throw invalid_argument("fail to jbytearraytoface_code"); 51 | } 52 | auto v = BeanUtilits::jmd5settoMD5Vector(imgMD5Set); 53 | vector result(rows); 54 | auto size=fse_searchCode(feature.get(),threshold,rows,result.data(),v->data(), v->size()); 55 | if (size >= 0) { 56 | return BeanUtilits::code_bean_ptr_tojobjectArray(result.data(), (jsize)size, context->m_code_bean_mirror, true).norelease().get(); 57 | } 58 | throwJNIException(fse_error_msg()); 59 | }catch(invalid_argument &e){ 60 | if (!env->ExceptionOccurred()) { 61 | jni_utilits::throwIllegalArgumentException(e.what()); 62 | } 63 | }catch(exception &e){ 64 | if (!env->ExceptionOccurred()) { 65 | throwJNIException(e.what()); 66 | } 67 | } 68 | return nullptr; 69 | } 70 | 71 | JNIEXPORT jobject JNICALL Java_net_gdface_sdk_fse_FseJniBridge_getFeature 72 | (JNIEnv *env, jclass, jbyteArray jmd5){ 73 | try{ 74 | MD5 md5; 75 | if(BeanUtilits::jbytearraytoMD5(jmd5,md5)){ 76 | auto bean = std::make_shared(); 77 | auto result=fse_getFeature(std::addressof(md5),bean.get()); 78 | switch (result) { 79 | case 1: { 80 | return BeanUtilits::toJCodeBean(*bean, context->m_code_bean_mirror, true).norelease().get(); 81 | } 82 | case 0: { 83 | return nullptr; 84 | } 85 | default: 86 | throwJNIException(fse_error_msg()); 87 | break; 88 | } 89 | }else if(!env->ExceptionOccurred()){ 90 | jni_utilits::throwIllegalArgumentException("fail to jbytearraytoMD5"); 91 | } 92 | }catch(exception &e){ 93 | if (!env->ExceptionOccurred()) { 94 | throwJNIException(e.what()); 95 | } 96 | } 97 | return nullptr; 98 | } 99 | 100 | JNIEXPORT jboolean JNICALL Java_net_gdface_sdk_fse_FseJniBridge_addFeature 101 | (JNIEnv *env, jclass, jbyteArray featureId, jbyteArray feature, jstring imgMD5){ 102 | try{ 103 | auto bean = std::make_shared(); 104 | if(BeanUtilits::tocodeBean(*bean,featureId,feature,imgMD5)){ 105 | //BeanUtilits::output_bean(bean,"jni add "); 106 | auto result = fse_addFeature(bean.get()); 107 | if(1==result){ 108 | return JNI_TRUE; 109 | } 110 | else if (0 == result) { 111 | return JNI_FALSE; 112 | } 113 | throwJNIException(fse_error_msg()); 114 | }else if(!env->ExceptionOccurred()){ 115 | jni_utilits::throwIllegalArgumentException("fail to tocodeBean"); 116 | } 117 | }catch(exception &e){ 118 | if (!env->ExceptionOccurred()) { 119 | throwJNIException(e.what()); 120 | } 121 | } 122 | return JNI_FALSE; 123 | } 124 | 125 | JNIEXPORT jboolean JNICALL Java_net_gdface_sdk_fse_FseJniBridge_removeFeature 126 | (JNIEnv *env, jclass, jbyteArray featureId){ 127 | try { 128 | MD5 md5; 129 | if (BeanUtilits::jbytearraytoMD5(featureId, md5)) { 130 | auto result = fse_removeFeature(std::addressof(md5)); 131 | switch (result) { 132 | case 0: { 133 | return JNI_FALSE; 134 | } 135 | case 1: { 136 | return JNI_TRUE; 137 | } 138 | default: 139 | throwJNIException(fse_error_msg()); 140 | break; 141 | } 142 | } 143 | else if (!env->ExceptionOccurred()) { 144 | jni_utilits::throwIllegalArgumentException("fail to jbytearraytoMD5"); 145 | } 146 | } catch (exception &e) { 147 | if (!env->ExceptionOccurred()){ 148 | throwJNIException(e.what()); 149 | } 150 | } 151 | return JNI_FALSE; 152 | } 153 | 154 | JNIEXPORT jint JNICALL Java_net_gdface_sdk_fse_FseJniBridge_size 155 | (JNIEnv *, jclass){ 156 | auto result = fse_size(); 157 | if (result < 0) { 158 | throwJNIException(fse_error_msg()); 159 | } 160 | return (jint)result; 161 | } 162 | JNIEXPORT jstring JNICALL Java_net_gdface_sdk_fse_FseJniBridge_statInfo 163 | (JNIEnv *, jclass){ 164 | auto info=fse_statInfo(); 165 | if (nullptr == info) { 166 | throwJNIException(fse_error_msg()); 167 | return nullptr; 168 | } 169 | return jni_utilits::raii_NewStringUTF(info).norelease().get(); 170 | } -------------------------------------------------------------------------------- /feature_se/feature_se.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * feature_se.cpp 3 | * 4 | * Created on: 2018年4月14日 5 | * Author: 10km 6 | */ 7 | #include 8 | #include "feature_se.h" 9 | #include "CodeManagerFactory.h" 10 | #include "sample_log.h" 11 | using namespace gdface; 12 | // 搜索引擎对象，需要调用初始化函数初始化才有效 13 | static ICodeManager* instance = nullptr; 14 | // 当前线程的错误消息 15 | static thread_local std::string error_msg; 16 | static string INVALID_ARGUMENT = "Invalid argument:"; 17 | static string NULL_POINTER = "null pointer:"; 18 | 19 | static void fill_error_msg(const std::string& msg) { 20 | error_msg = msg; 21 | } 22 | 23 | static void fill_error_msg(const char* msg) { 24 | if(nullptr != msg){ 25 | error_msg = msg; 26 | } 27 | } 28 | // 将MD5数组转为MD5Set对象 29 | static MD5Set toMD5Set(const MD5 md5Arrays[], size_t md5Count) { 30 | MD5Set md5_set; 31 | if(nullptr != md5Arrays){ 32 | for (auto i = md5Count; i > 0; --i) { 33 | md5_set.insert(md5Arrays[i]); 34 | } 35 | } 36 | return std::move(md5_set); 37 | } 38 | // 检查 instance是否初始化 39 | static bool valid_instance() { 40 | if (nullptr != instance) { 41 | return true; 42 | } 43 | else { 44 | fill_error_msg("uninitialized instance,must call init(...) firstly to initialize feature search engine"); 45 | return false; 46 | } 47 | } 48 | const char* fse_error_msg() { 49 | return error_msg.c_str(); 50 | } 51 | 52 | int fse_init(uint32_t initCapacity, float loadFactor, size_t overBlockCapacity){ 53 | // 初始化表对象 54 | try { 55 | if(nullptr == instance){ 56 | SAMPLE_OUT("feature_se jni initialized:\ninitCapacity:{};loadFactor:{};overBlockCapacity:{}", 57 | initCapacity, loadFactor, overBlockCapacity); 58 | 59 | instance = CodeManagerFactory::getICodeManagerCPU(initCapacity, true, loadFactor, overBlockCapacity); 60 | } 61 | return 0; 62 | } 63 | catch (exception &e) { 64 | fill_error_msg(e.what()); 65 | return -1; 66 | } 67 | } 68 | int fse_release() { 69 | if (!valid_instance()) { 70 | return -1; 71 | } 72 | SAMPLE_OUT("ICodeManager instance release"); 73 | try { 74 | instance->release(); 75 | instance = nullptr; 76 | return 0; 77 | } 78 | catch (exception &e) { 79 | fill_error_msg(e.what()); 80 | return -1; 81 | } 82 | } 83 | 84 | int fse_searchCode(const face_code *code, double threshold, size_t rows, code_bean out[], const MD5 imgMD5Array[], size_t md5Count) { 85 | if (nullptr == code) { 86 | fill_error_msg(NULL_POINTER + "code"); 87 | return -1; 88 | } 89 | if (nullptr == out) { 90 | fill_error_msg(NULL_POINTER + "out"); 91 | return -1; 92 | } 93 | if (!valid_instance()) { 94 | return -1; 95 | } 96 | try { 97 | MD5Set md5_set = toMD5Set(imgMD5Array, md5Count); 98 | return (int)instance->searchCode(*code, threshold, rows, out, std::addressof(md5_set)); 99 | } 100 | catch (exception &e) { 101 | fill_error_msg(e.what()); 102 | return -1; 103 | } 104 | } 105 | 106 | int fse_getFeature(const MD5 *md5, code_bean* out){ 107 | if (is_null_MD5(md5)) { 108 | return 0; 109 | } 110 | if (nullptr == out) { 111 | fill_error_msg(NULL_POINTER + "out"); 112 | return -1; 113 | } 114 | if (!valid_instance()) { 115 | return -1; 116 | } 117 | try{ 118 | auto result = instance->getBean(*md5); 119 | if (nullptr != result) { 120 | *out = *result; 121 | return 1; 122 | } 123 | return 0; 124 | } 125 | catch (exception &e) { 126 | fill_error_msg(e.what()); 127 | return -1; 128 | } 129 | } 130 | int fse_addFeatures(const code_bean beans[], size_t beanCount) { 131 | if (nullptr == beans) { 132 | return 0; 133 | } 134 | if (!valid_instance()) { 135 | return -1; 136 | } 137 | int count = 0; 138 | try { 139 | for (decltype(beanCount) i = 0; i < beanCount; ++i) { 140 | if (!beans[i].id.is_null()) { 141 | instance->addBean(beans[i]); 142 | ++count; 143 | } 144 | } 145 | } 146 | catch (exception &e) { 147 | fill_error_msg(e.what()); 148 | return -1; 149 | } 150 | return count; 151 | } 152 | int fse_addFeature(const code_bean* bean) { 153 | return fse_addFeatures(bean, 1); 154 | } 155 | int fse_removeFeatures(const MD5 md5s[], size_t md5Count) { 156 | if (nullptr == md5s) { 157 | return 0; 158 | } 159 | if (!valid_instance()) { 160 | return -1; 161 | } 162 | int count = 0; 163 | bool result; 164 | try { 165 | for (decltype(md5Count) i = 0; i < md5Count; ++i) { 166 | if (!md5s[i].is_null()) { 167 | result = instance->removeBean(md5s[i]); 168 | count += result ? 1 : 0; 169 | } 170 | } 171 | } 172 | catch (exception &e) { 173 | fill_error_msg(e.what()); 174 | return -1; 175 | } 176 | return count; 177 | } 178 | int fse_removeFeature(MD5 *md5) { 179 | return fse_removeFeatures(md5, 1); 180 | } 181 | int fse_removeFeaturesByImgMD5s(const MD5 imgMD5s[], size_t md5Count) { 182 | if (nullptr == imgMD5s) { 183 | return 0; 184 | } 185 | if (!valid_instance()) { 186 | return -1; 187 | } 188 | try { 189 | int count = 0; 190 | for (decltype(md5Count) i = 0; i < md5Count; ++i) { 191 | if(!imgMD5s[i].is_null()){ 192 | count += instance->removeBeansByImgMD5(imgMD5s[i]); 193 | } 194 | } 195 | return count; 196 | } 197 | catch (exception &e) { 198 | fill_error_msg(e.what()); 199 | return -1; 200 | } 201 | } 202 | int fse_removeFeaturesByImgMD5(const MD5 *imgMD5) { 203 | return fse_removeFeaturesByImgMD5s(imgMD5, 1); 204 | } 205 | 206 | int fse_size() { 207 | if (!valid_instance()) { 208 | return -1; 209 | } 210 | return instance->size(); 211 | } 212 | 213 | static thread_local std::string tls_str_statinfo; 214 | const char* fse_statInfo() { 215 | if (!valid_instance()) { 216 | return nullptr; 217 | } 218 | try { 219 | tls_str_statinfo = instance->statInfo(); 220 | return tls_str_statinfo.c_str(); 221 | } 222 | catch (exception &e) { 223 | fill_error_msg(e.what()); 224 | return nullptr; 225 | } 226 | } 227 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #cmake file for project codemgr 2 | #author:guyadong 3 | #created:2015/10/05 4 | ############################################################################ 5 | cmake_minimum_required( VERSION 3.0 ) 6 | cmake_policy(SET CMP0048 NEW) 7 | # 3.0以上版本才允许使用VERSION option 8 | project( feature_se VERSION 1.0.0 LANGUAGES CXX) 9 | message(STATUS "Project: ${PROJECT_NAME} ${PROJECT_VERSION}") 10 | message(STATUS "Project Directory: ${PROJECT_SOURCE_DIR}") 11 | message(STATUS "Project Binary Directory: ${PROJECT_BINARY_DIR}") 12 | message(STATUS "CMAKE_COMPILER=${CMAKE_CXX_COMPILER}") 13 | message(STATUS "CMAKE_CXX_COMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION}") 14 | 15 | # 设置外部提供人脸识别SDK类型,可选值CASSDK,CUSTOM,EUCLIDEAN 16 | # 当设置为CUSTOM时，必须指定 CUSTOM_SDK_INCLUDE CUSTOM_SDK_LIBRARY 17 | # 设置为EUCLIDEAN时，使用默认的点积比对函数 18 | if(NOT EXT_SDK_TYPE) 19 | ## 默认值EUCLIDEAN 20 | set(EXT_SDK_TYPE EUCLIDEAN) 21 | endif() 22 | if(EXT_SDK_TYPE STREQUAL "CASSDK") 23 | set(CASSDK_USED TRUE) 24 | endif() 25 | if(EXT_SDK_TYPE STREQUAL "CUSTOM") 26 | # CUSTOM_FEACOMP_INCLUDE 比较函数头文件所在文件夹 27 | # CUSTOM_FEACOMP_HEADERS 比较函数头文件名列表,';'分隔 28 | # CUSTOM_FEACOMP_LIBRARY 比较函数库文件位置 29 | # CUSTOM_FEACOMP_FUNNAME 比较函数名 30 | # CUSTOM_CODE_BYTE_LEN 特征数据长度(字节) 31 | if(NOT CUSTOM_FEACOMP_INCLUDE OR NOT CUSTOM_FEACOMP_HEADERS OR NOT CUSTOM_FEACOMP_LIBRARY OR NOT CUSTOM_FEACOMP_FUNNAME OR NOT CUSTOM_CODE_BYTE_LEN) 32 | message(FATAL_ERROR "MUST DEFINE 33 | CUSTOM_FEACOMP_INCLUDE(face sdk include folder) 34 | CUSTOM_FEACOMP_HEADERS(feature compare function's header file full path) 35 | CUSTOM_FEACOMP_LIBRARY(feature compare function's library full path), 36 | CUSTOM_FEACOMP_FUNNAME(function name) 37 | CUSTOM_CODE_BYTE_LEN(feature date length(byte))") 38 | endif() 39 | # 没有定义 CUSTOM_FEACOMP_FUNTYPE 则使用默认值 40 | if(NOT CUSTOM_FEACOMP_FUNTYPE) 41 | set(CUSTOM_FEACOMP_FUNTYPE "double(unsigned char*,unsigned char*)") 42 | endif() 43 | endif() 44 | if(EXT_SDK_TYPE STREQUAL "EUCLIDEAN") 45 | # EUCLIDEAN 使用系统内置的特征值比对函数,需要指定特征值数据类型和特征值长度 46 | # 特征值数据类型如果没指定，默认使用double 47 | # 特征值数据长度如果没指定，则报错退出 48 | if(NOT EUCLIDEAN_ELEM_TYPE) 49 | set(EUCLIDEAN_ELEM_TYPE double) 50 | message(STATUS "EUCLIDEAN_ELEM_TYPE use default:double,available value:(double|float)") 51 | endif() 52 | if(NOT EUCLIDEAN_ELEM_TYPE STREQUAL "double" OR NOT EUCLIDEAN_ELEM_TYPE STREQUAL "float") 53 | message(STATUS "invalid EUCLIDEAN_ELEM_TYPE: ${EUCLIDEAN_ELEM_TYPE},available value:(double|float)") 54 | endif() 55 | # 默认特征值数组后面没有double类型的点积和 56 | if(NOT EUCLIDEAN_CODE_END_WITH_SUM) 57 | set(EUCLIDEAN_CODE_END_WITH_SUM OFF) 58 | message(STATUS "EUCLIDEAN_CODE_END_WITH_SUM: OFF") 59 | endif() 60 | if(NOT EUCLIDEAN_ELEM_LEN) 61 | message(FATAL_ERROR "MUST DEFINE EUCLIDEAN_ELEM_LEN") 62 | endif() 63 | endif() 64 | # 定义动态库名 65 | if(NOT FSE_LIBNAME) 66 | set(FSE_LIBNAME fse_cas) 67 | endif() 68 | # 定义JNI动态库名 69 | if(NOT JNI_FSE_LIBNAME) 70 | set(JNI_FSE_LIBNAME fse_cas_jni) 71 | endif() 72 | #判断编译类型和版本是否满足编译要求 73 | if(MSVC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 19) 74 | message(STATUS "CMAKE_CXX_COMPILER: ${CMAKE_CXX_COMPILER}") 75 | message(STATUS "CMAKE_CXX_COMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION}" ) 76 | elseif(CMAKE_COMPILER_IS_GNUCXX) 77 | include(CheckCXXCompilerFlag) 78 | ## 检测编译器是否支持C++11 79 | check_cxx_compiler_flag("-std=c++11" COMPILER_SUPPORTS_CXX11) 80 | if(COMPILER_SUPPORTS_CXX11) 81 | message(STATUS "CMAKE_CXX_COMPILER: ${CMAKE_CXX_COMPILER}") 82 | message(STATUS "CMAKE_CXX_COMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION}" ) 83 | else() 84 | message(FATAL_ERROR "compiler required: Visual Studio 2015 OR surrpoted -std=c++11" ) 85 | endif() 86 | if(CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5.0.0" ) 87 | message(FATAL_ERROR "compiler required: gcc 5.0.0" ) 88 | endif() 89 | 90 | endif() 91 | # includes utils.cmake module 92 | set (CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/Modules") 93 | include (utils) 94 | # 依赖代码位置 95 | set(DEPENDENT_SOURCE_DIR "${PROJECT_BINARY_DIR}/dependent_sources" CACHE STRINGS "dependent source folder" FORCE) 96 | #定义公共代码位置 97 | set( COMMONS_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/common_source_cpp) 98 | if(NOT EXISTS ${COMMONS_SOURCE_DIR}/CMakeLists.txt ) 99 | message( FATAL_ERROR "Not exists or Empty folder: ${COMMONS_SOURCE_DIR},the submodule must be init and pull" ) 100 | endif() 101 | cxx11_support() 102 | if(MSVC) 103 | #关闭C4819警告 104 | add_definitions("/wd4819") 105 | message(STATUS "optional:/wd4819") 106 | #关闭CRT_SECURE警告 107 | add_definitions(-D_CRT_SECURE_NO_WARNINGS) 108 | message(STATUS "optional:-D_CRT_SECURE_NO_WARNINGS") 109 | endif(MSVC) 110 | ##############设置目标文件生成位置##################### 111 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/bin") 112 | set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/bin") 113 | set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/lib") 114 | # define RUNTIME_INSTALL_DIR LIBRARY_INSTALL_DIR INCLUDE_INSTALL_DIR CONFIG_INSTALL_DIR 115 | set (RUNTIME_INSTALL_DIR bin) 116 | set (LIBRARY_INSTALL_DIR lib) 117 | set (INCLUDE_INSTALL_DIR include) 118 | set( SUBDIRECTORIES common_source_cpp jni feature_se) 119 | foreach( subdir ${SUBDIRECTORIES} ) 120 | if( IS_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}/${subdir}" ) 121 | MESSAGE( STATUS "Found sub project ${subdir}, adding it" ) 122 | add_subdirectory( ${subdir} ) 123 | endif() 124 | endforeach( subdir ) 125 | # 复制安装脚本 126 | if(WIN32) 127 | install(FILES ${PROJECT_SOURCE_DIR}/INSTALL.bat.in DESTINATION bin RENAME INSTALL.bat) 128 | install(FILES ${PROJECT_SOURCE_DIR}/UNINSTALL.bat.in DESTINATION bin RENAME UNINSTALL.bat) 129 | elseif(CMAKE_SYSTEM_NAME MATCHES "Linux") 130 | install(PROGRAMS ${PROJECT_SOURCE_DIR}/INSTALL.sh.in DESTINATION bin 131 | PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE 132 | RENAME INSTALL.sh 133 | ) 134 | install(PROGRAMS ${PROJECT_SOURCE_DIR}/UNINSTALL.sh.in DESTINATION bin 135 | PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE 136 | RENAME UNINSTALL.sh 137 | ) 138 | endif() 139 | -------------------------------------------------------------------------------- /jni/BeanUtilits.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * beanutilits.cpp 3 | * 4 | * Created on: 2015年11月28日 5 | * Author: guyadong 6 | */ 7 | #include 8 | #include "BeanUtilits.h" 9 | #include "dot_product.h" 10 | #include "md5/md5.h" 11 | 12 | namespace gdface { 13 | bool BeanUtilits::tocodeBean(code_bean& bean, jobject obj, jni_utilits::JavaClassMirror& mirror) { 14 | if (nullptr == obj) { 15 | return false; 16 | } 17 | auto success = tocodeBean(bean, 18 | mirror.GetField(obj, CODEBEAN_ID).get(), 19 | mirror.GetField(obj, CODEBEAN_CODE).get(), 20 | mirror.GetField(obj, CODEBEAN_IMGMD5).get()); 21 | static_assert(std::is_same::value,"different type with code_bean::similarity and jdouble"); 22 | bean.similarity = mirror.GetField(obj, CODEBEAN_SIMILARITY); 23 | return success; 24 | } 25 | bool BeanUtilits::jstringToMD5(jstring jstr, MD5& md5) { 26 | if (nullptr == jstr){ 27 | // 为null时填0 28 | md5.l[0] = 0,md5.l[1] = 0; 29 | return false; 30 | } 31 | auto env = jni_utilits::getJNIEnv(); 32 | auto len = env->GetStringUTFLength(jstr); 33 | if (len == (sizeof(MD5) << 1)) { 34 | auto bytes = jni_utilits::raii_GetStringUTFChars(jstr); 35 | PSTRtoMD5(bytes.get(), md5); 36 | return true; 37 | } 38 | return false; 39 | } 40 | 41 | bool BeanUtilits::jbytearraytoMD5(jbyteArray bytes, MD5& md5) { 42 | if (nullptr == bytes) { 43 | return false; 44 | } 45 | auto env = jni_utilits::getJNIEnv(); 46 | if (env->GetArrayLength(bytes) == sizeof(MD5)) { 47 | auto byte_ptr = jni_utilits::raii_GetByteArrayElements(bytes); 48 | md5 = *((MD5*) ((byte_ptr.get()))); 49 | return true; 50 | } 51 | return false; 52 | } 53 | 54 | bool BeanUtilits::jbytearraytoface_code(jbyteArray bytes, face_code& code) { 55 | if (nullptr == bytes) { 56 | return false; 57 | } 58 | auto env = jni_utilits::getJNIEnv(); 59 | if (env->GetArrayLength(bytes) == sizeof(face_code)) { 60 | auto byte_ptr = jni_utilits::raii_GetByteArrayElements(bytes); 61 | code = *((face_code*)((byte_ptr.get()))); 62 | return true; 63 | } 64 | #if EUCLIDEAN_FEACOMP && ! CODE_END_WITH_SUM 65 | else if (env->GetArrayLength(bytes) == sizeof(code.element)) { 66 | auto byte_ptr = jni_utilits::raii_GetByteArrayElements(bytes); 67 | // 复制所有特征数据到element 68 | std::memcpy(code.element, byte_ptr.get(), sizeof(code.element)); 69 | // 计算特征值数组的点积和保存在sum 70 | code.sum = dot_product(code.element, code.element); 71 | return true; 72 | } 73 | #endif 74 | return false; 75 | } 76 | bool BeanUtilits::jdoublearraytoface_code(jdoubleArray bytes, face_code& code) { 77 | if (nullptr == bytes) { 78 | return false; 79 | } 80 | auto env = jni_utilits::getJNIEnv(); 81 | if (env->GetArrayLength(bytes) == sizeof(face_code) / sizeof(double)) { 82 | auto byte_ptr = jni_utilits::raii_GetDoubleArrayElements(bytes); 83 | code = *((face_code*)((byte_ptr.get()))); 84 | return true; 85 | } 86 | #if EUCLIDEAN_FEACOMP && ! CODE_END_WITH_SUM 87 | else if (env->GetArrayLength(bytes) == sizeof(code.element)) { 88 | auto byte_ptr = jni_utilits::raii_GetDoubleArrayElements(bytes); 89 | // 复制所有特征数据到element 90 | std::memcpy(code.element, byte_ptr.get(), sizeof(code.element)); 91 | // 计算特征值数组的点积和保存在sum 92 | code.sum = dot_product(code.element, code.element); 93 | return true; 94 | } 95 | #endif 96 | return false; 97 | } 98 | raii_var BeanUtilits::toJCodeBean(const code_bean& bean, jni_utilits::JavaClassMirror& mirror,jboolean full) { 99 | auto var = jni_utilits::raii_NewObject(mirror.javaclass.get(), mirror.constructor); 100 | auto obj = *var; 101 | if (nullptr != obj) { 102 | mirror.SetField(obj, CODEBEAN_ID, MD5tojbyteArray(bean.id).get()); 103 | if (full) { 104 | mirror.SetField(obj, CODEBEAN_CODE, face_codetojbyteArray(FACE_CODE_CONVERT(bean.code)).get()); 105 | } 106 | // 如果imgMD5不为全0则转为java string,否则为null 107 | if ( ! bean.imgMD5.is_null() ) { 108 | mirror.SetField(obj, CODEBEAN_IMGMD5, MD5toJString(bean.imgMD5).get()); 109 | } 110 | mirror.SetField(obj, CODEBEAN_SIMILARITY, (jdouble) (bean.similarity)); 111 | } 112 | return var; 113 | } 114 | bool BeanUtilits::tocodeBean(code_bean& bean, jbyteArray id, jbyteArray code, jstring imgMD5) { 115 | auto result = jbytearraytoface_code(code, bean.code); 116 | // 允许id,imgMD5为null 117 | if (result) { 118 | if (nullptr == id) { 119 | #if EUCLIDEAN_FEACOMP && ! CODE_END_WITH_SUM 120 | auto codeMD5 = md5::digest(std::addressof(bean.code), (unsigned int)sizeof(bean.code.element)); 121 | bean.id = *(::MD5*)codeMD5.data(); 122 | #else 123 | auto codeMD5 = md5::digest(std::addressof(bean.code), (unsigned int)sizeof(bean.code)); 124 | bean.id = *(::MD5*)codeMD5.data(); 125 | #endif 126 | } 127 | else { 128 | result &= jbytearraytoMD5(id, bean.id); 129 | } 130 | result &= jstringToMD5(imgMD5, bean.imgMD5) || nullptr == imgMD5; 131 | } 132 | return result; 133 | 134 | } 135 | std::shared_ptr > BeanUtilits::jmd5settoMD5Vector(jobjectArray jset) { 136 | decltype(jni_utilits::getJNIEnv()->GetArrayLength(jset)) len = 137 | nullptr != jset ? jni_utilits::getJNIEnv()->GetArrayLength(jset) : 0; 138 | auto v = std::make_shared >(len); 139 | for (auto i = len; i > 0; --i) { 140 | auto jstr = jni_utilits::raii_GetObjectArrayElement(jset, i - 1); 141 | if (!BeanUtilits::jstringToMD5(*jstr, (*v)[i])) { 142 | throw invalid_argument("invalid jstring argument,cant convert to MD5"); 143 | break; 144 | } 145 | } 146 | return std::move(v); 147 | } 148 | raii_var BeanUtilits::code_bean_ptr_tojobjectArray(const code_bean* src, jsize size, jni_utilits::JavaClassMirror& mirror, 149 | jboolean full) { 150 | auto env = jni_utilits::getJNIEnv(); 151 | auto var=jni_utilits::raii_NewObjectArray(size, *mirror.javaclass, nullptr); 152 | auto joarray = *var; 153 | if (nullptr != joarray) { 154 | for (auto i = size; i > 0; --i) { 155 | auto obj = toJCodeBean(src[i - 1], mirror, full); 156 | env->SetObjectArrayElement(joarray, i - 1, obj.get()); 157 | } 158 | } 159 | return var; 160 | } 161 | } /* namespace gdface */ 162 | -------------------------------------------------------------------------------- /feature_se/dot_product.h: -------------------------------------------------------------------------------- 1 | /* 2 | * dot_product.h 3 | * 4 | * Created on: 2015年12月15日 5 | * Author: guyadong 6 | */ 7 | 8 | #ifndef FEATURE_SE_DOT_PRODUCT_H_ 9 | #define FEATURE_SE_DOT_PRODUCT_H_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #if defined(__x86_64)|| defined(_M_X64) // x64 体系结构 使用SSE指令优化 gcc 和 MinGW 有定义 __x86_64, MinGW 和 msc 有定义 _M_X64 17 | #ifdef _MSC_VER 18 | #include 19 | #elif defined(__GNUC__) 20 | #include 21 | #else 22 | // 不支持的编译器类型 23 | #error unexpected c complier (msc/gcc) 24 | #endif 25 | #endif 26 | 27 | namespace gdface{ 28 | /* 定义点积函数类型 */ 29 | enum dot_product_type{ 30 | /**默认循环累加*/DOT_DEFAULT, 31 | /** 宏递归 */DOT_DEFAULT_RECURSIVE, 32 | /** x64 SIMD 循环累加 */DOT_SIMD_X64, 33 | /** x64 SIMD 宏递归 */DOT_SIMD_X64_RECURSIVE}; 34 | 35 | #if defined(__x86_64)|| defined(_M_X64) // x64 体系结构 使用SSE指令优化 gcc 和 MinGW 有定义 __x86_64, MinGW 和 msc 有定义 _M_X64 36 | #ifdef _MSC_VER 37 | const dot_product_type DOT_TYPE_CONST= DOT_DEFAULT;//DOT_SIMD_X64; 38 | inline double _sum__m128(const __m128 & m1){ 39 | return m1.m128_f32[0]+m1.m128_f32[1]+m1.m128_f32[2]+m1.m128_f32[3]; 40 | } 41 | #define _SIMD_X64_ 42 | #elif defined(__GNUC__) 43 | const dot_product_type DOT_TYPE_CONST= DOT_DEFAULT;//DOT_SIMD_X64; 44 | inline double _sum__m128(const __m128 & m1){ 45 | return m1[0]+m1[1]+m1[2]+m1[3]; 46 | } 47 | #define _SIMD_X64_ 48 | #else 49 | const dot_product_type DOT_TYPE_CONST = DOT_DEFAULT; 50 | #endif 51 | #else 52 | const dot_product_type DOT_TYPE_CONST=DOT_DEFAULT; 53 | #endif 54 | 55 | #ifdef _SIMD_X64_ 56 | 57 | template 58 | inline typename std::enable_if::type tail(__m128& m1,const __m128& src1, const __m128& src2) {} 59 | 60 | template 61 | inline typename std::enable_if::type tail(__m128& m1,const __m128& src1, const __m128& src2) { 62 | __m128 t1=_mm_load_ss((float*)std::addressof(src1)); 63 | __m128 t2=_mm_load_ss((float*)std::addressof(src2)); 64 | m1=_mm_add_ps(m1,_mm_mul_ps(t1,t2)); 65 | } 66 | template 67 | inline typename std::enable_if::type tail(__m128& m1,const __m128&src1, const __m128& src2) { 68 | __m128 z=_mm_setzero_ps(); 69 | //__m128 t1=_mm_shuffle_ps(*src1,z,_MM_SHUFFLE(0,0,1,0)); 70 | //__m128 t2=_mm_shuffle_ps(*src2,z,_MM_SHUFFLE(0,0,1,0)); 71 | //m1=_mm_add_ps(m1,_mm_mul_ps(t1,t2)); 72 | m1=_mm_add_ps(m1,_mm_mul_ps(_mm_shuffle_ps(src1,z,_MM_SHUFFLE(0,0,1,0)),_mm_shuffle_ps(src2,z,_MM_SHUFFLE(0,0,1,0)))); 73 | } 74 | template 75 | inline typename std::enable_if::type tail(__m128& m1,const __m128& src1, const __m128& src2) { 76 | __m128 t1=src1; 77 | __m128 t2=src2; 78 | #ifdef _MSC_VER 79 | t1.m128_f32[3]=0; 80 | t2.m128_f32[3]=0; 81 | #elif defined(__GNUC__) 82 | t1[3]=0; 83 | t2[3]=0; 84 | #else 85 | // 暂不支持的编译器类型 86 | #error unexpected c complier (msc/gcc) 87 | #endif 88 | m1=_mm_add_ps(m1,_mm_mul_ps(t1,t2)); 89 | } 90 | 91 | /* 递归方式计算__m128点积 */ 92 | template 93 | inline typename std::enable_if::type _dot_product__m128(const __m128* src1,const __m128* src2) { 94 | return _mm_mul_ps(*src1,*src2); 95 | } 96 | template 97 | inline typename std::enable_if<(N>1),__m128>::type _dot_product__m128(const __m128* src1,const __m128* src2) { 98 | return _mm_add_ps(_dot_product__m128<1>(src1,src2),_dot_product__m128(src1+1,src2+1)); 99 | } 100 | 101 | /* SIMD版点积函数(递归版) f1,f2地址必须16字节对齐 */ 102 | template 103 | inline typename std::enable_if::type dot_product(const float* f1, const float* f2) { 104 | assert(nullptr!=f1&&nullptr!=f2); 105 | __m128* src1=(__m128*)f1; 106 | __m128* src2=(__m128*)f2; 107 | __m128 m1=_dot_product__m128<(N>>2)>(src1,src2); 108 | tail(m1,src1[N>>2],src2[N>>2]); 109 | return _sum__m128(m1); 110 | } 111 | /* SIMD版点积函数(循环版) f1,f2地址必须16字节对齐 */ 112 | template 113 | inline typename std::enable_if::type dot_product(const float* f1, const float* f2) { 114 | assert(nullptr!=f1&&nullptr!=f2); 115 | __m128* src1=(__m128*)f1; 116 | __m128* src2=(__m128*)f2; 117 | __m128 m1=_mm_setzero_ps(); 118 | for (size_t i = 0; i < N>>2; ++i){ 119 | m1=_mm_add_ps(m1,_mm_mul_ps(src1[i],src2[i])); 120 | } 121 | tail(m1,src1[N>>2],src2[N>>2]); 122 | return _sum__m128(m1); 123 | } 124 | #endif /* _SIMD_X64_ */ 125 | 126 | /* float类型默认点积函数(循环版) */ 127 | template 128 | inline typename std::enable_if::type dot_product(const float* f1, const float* f2) { 129 | assert(nullptr!=f1&&nullptr!=f2); 130 | float sum = 0.0; 131 | for (size_t i = 0; i < N; ++i) 132 | sum += f1[i] * f2[i]; 133 | return (double)sum; 134 | } 135 | /* float类型默认点积函数(递归版) */ 136 | template 137 | inline typename std::enable_if<(N==1)&&(TYPE==DOT_DEFAULT_RECURSIVE),double>::type dot_product(const float* f1, const float* f2) { 138 | assert(nullptr!=f1&&nullptr!=f2); 139 | return (*f1) * (*f1); 140 | } 141 | template 142 | inline typename std::enable_if<(N>1)&&(TYPE==DOT_DEFAULT_RECURSIVE),double>::type dot_product(const float* f1, const float* f2) { 143 | assert(nullptr!=f1&&nullptr!=f2); 144 | return dot_product<1,TYPE>(f1,f2)+dot_product(f1+1,f2+1); 145 | } 146 | /* double类型默认点积函数(循环版) */ 147 | template 148 | inline typename std::enable_if::type dot_product(const double* f1, const double* f2) { 149 | assert(nullptr != f1&&nullptr != f2); 150 | double sum = 0.0; 151 | for (size_t i = 0; i < N; ++i) 152 | sum += f1[i] * f2[i]; 153 | return (double)sum; 154 | } 155 | /* double类型默认点积函数(递归版) */ 156 | template 157 | inline typename std::enable_if<(N == 1) && (TYPE == DOT_DEFAULT_RECURSIVE), double>::type dot_product(const double* f1, const double* f2) { 158 | assert(nullptr != f1&&nullptr != f2); 159 | return (*f1) * (*f1); 160 | } 161 | template 162 | inline typename std::enable_if<(N>1) && (TYPE == DOT_DEFAULT_RECURSIVE), double>::type dot_product(const double* f1, const double* f2) { 163 | assert(nullptr != f1&&nullptr != f2); 164 | return dot_product<1, TYPE>(f1, f2) + dot_product(f1 + 1, f2 + 1); 165 | } 166 | } /* namespace gdface */ 167 | 168 | #endif /* FEATURE_SE_DOT_PRODUCT_H_ */ 169 | -------------------------------------------------------------------------------- /feature_se/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #cmake file for library fse(feature search engine) 2 | #author:guyadong 3 | #created:2018/04/13 4 | include (CMakePackageConfigHelpers) 5 | if(WIN32 AND NOT CYGWIN) 6 | set (CONFIG_INSTALL_DIR cmake) 7 | else() 8 | set (CONFIG_INSTALL_DIR ${LIBRARY_INSTALL_DIR}/cmake/fse) 9 | endif() 10 | if(EXT_SDK_TYPE STREQUAL "CASSDK") 11 | find_package(Cassdk REQUIRED) 12 | elseif(EXT_SDK_TYPE STREQUAL "EUCLIDEAN") 13 | elseif(EXT_SDK_TYPE STREQUAL "CUSTOM") 14 | if(NOT IS_DIRECTORY ${CUSTOM_FEACOMP_INCLUDE}) 15 | message(FATAL_ERROR "NOT FOUND include directory ${CUSTOM_FEACOMP_INCLUDE}") 16 | endif() 17 | set(CUSTOM_INCLUDE_STATEMENTS ) 18 | foreach(_name ${CUSTOM_SYS_HEADERS}) 19 | set(CUSTOM_INCLUDE_STATEMENTS "${CUSTOM_INCLUDE_STATEMENTS} 20 | #include <${_name}>") 21 | endforeach() 22 | message(STATUS CUSTOM_FEACOMP_HEADERS=${CUSTOM_FEACOMP_HEADERS}) 23 | foreach(_name ${CUSTOM_FEACOMP_HEADERS}) 24 | set(CUSTOM_INCLUDE_STATEMENTS "${CUSTOM_INCLUDE_STATEMENTS} 25 | #include \"${_name}\" 26 | ") 27 | endforeach() 28 | # 生成比较函数代码 29 | configure_file(${PROJECT_SOURCE_DIR}/cmake/custom_feature_compare.h.in 30 | ${PROJECT_BINARY_DIR}/generated/custom_feature_compare.h @ONLY ) 31 | else() 32 | message(STATUS "unsupported operation") 33 | endif() 34 | # 定义所有的源文件列表 35 | set(_SOURCE_FILES 36 | OverTableManager.cpp 37 | HashMapCl.cpp 38 | ICodeManager.cpp 39 | ICodeManagerCPU.cpp 40 | CodeManagerFactory.cpp 41 | TopKCodeBean.cpp 42 | feature_se.cpp 43 | ${DEPENDENT_SOURCE_DIR}/RWLock.cpp 44 | ${DEPENDENT_SOURCE_DIR}/ThreadPool.cpp 45 | ) 46 | # 用于外部调用的公共头文件 47 | set (_PUBLIC_HEADERS 48 | config.h 49 | feature_data_types.h 50 | cross_define.h 51 | feature_se.h 52 | ) 53 | set (_FSE_TARGETS ${FSE_LIBNAME}_static ${FSE_LIBNAME}) 54 | # CASSDK版本的CodeMgr实现 55 | add_library(${FSE_LIBNAME}_static STATIC ${_SOURCE_FILES}) 56 | add_library(${FSE_LIBNAME} SHARED ${_SOURCE_FILES}) 57 | 58 | use_static_library(${FSE_LIBNAME}) 59 | 60 | if(MSVC) 61 | add_library(${FSE_LIBNAME}_static_mt STATIC ${_SOURCE_FILES}) 62 | with_mt_if_msvc(${FSE_LIBNAME}_static_mt) 63 | list(APPEND _FSE_TARGETS ${FSE_LIBNAME}_static_mt) 64 | endif(MSVC) 65 | 66 | #########################3 67 | # 设置项目属性 68 | function (define_fse_target target) 69 | if(NOT TARGET ${target}) 70 | return () 71 | endif() 72 | # MSVC Debug 编译时生成包含用于调试器的完整符号调试信息的 .obj 文件 73 | # see also https://msdn.microsoft.com/zh-cn/library/958x11bc.aspx 74 | target_compile_options(${target} PRIVATE $<$,$>:/Z7>) 75 | 76 | get_target_property(_type ${target} TYPE) 77 | if(${_type} STREQUAL "SHARED_LIBRARY") 78 | set(_is_shared TRUE) 79 | set(_is_shared_str "1") 80 | else() 81 | set(_is_shared FALSE) 82 | set(_is_shared_str "0") 83 | endif() 84 | target_include_directories (${target} 85 | PRIVATE ${CMAKE_CURRENT_LIST_DIR} 86 | INTERFACE "$" 87 | ) 88 | # 编译库时必须加FSE_EXPORTS定义 89 | target_compile_definitions (${target} 90 | PRIVATE FSE_EXPORTS 91 | PUBLIC FSE_IS_A_DLL=${_is_shared_str} 92 | _SL_USE_BYTE_STREAM 93 | ) 94 | set_target_properties (${target} PROPERTIES 95 | PUBLIC_HEADER "${_PUBLIC_HEADERS}" 96 | VERSION ${PROJECT_VERSION} 97 | SOVERSION ${PROJECT_VERSION_MAJOR} 98 | DEBUG_POSTFIX _d 99 | ) 100 | set(_link_libraries ) 101 | if(CMAKE_SYSTEM_NAME MATCHES "Linux") 102 | list(APPEND _link_libraries -pthread -ldl) 103 | endif() 104 | #################### 105 | if(CASSDK_USED) 106 | # 加入CASSDK 版本定义 107 | target_compile_definitions (${target} 108 | PUBLIC CASSDK=1 109 | ) 110 | # 加入CASSDK　import target 111 | list(APPEND _link_libraries cassdk_THFeature) 112 | endif(CASSDK_USED) 113 | if(EXT_SDK_TYPE STREQUAL "EUCLIDEAN") 114 | # 加入特征长度和类型定义 115 | target_compile_definitions (${target} 116 | PUBLIC EUCLIDEAN_FEACOMP=1 117 | PUBLIC CODE_ELEM_TYPE=${EUCLIDEAN_ELEM_TYPE} 118 | PUBLIC CODE_ELEM_LEN=${EUCLIDEAN_ELEM_LEN} 119 | ) 120 | if(EUCLIDEAN_CODE_END_WITH_SUM) 121 | # 指定特征值数组后以double类型的点积和结尾 122 | target_compile_definitions (${target} PUBLIC CODE_END_WITH_SUM=1) 123 | endif() 124 | endif() 125 | if(EXT_SDK_TYPE STREQUAL "CUSTOM") 126 | # 加入自定义的包含特征比较函数的算法库 127 | target_compile_definitions(${target} 128 | PUBLIC CUSTOM_FEACOMP=1 129 | PUBLIC CODE_BYTE_LENGTH=${CUSTOM_CODE_BYTE_LEN}) 130 | target_link_libraries(${target} PRIVATE ${CUSTOM_FEACOMP_LIBRARY}) 131 | target_include_directories(${target} PRIVATE "${PROJECT_BINARY_DIR}/generated" ${CUSTOM_FEACOMP_INCLUDE}) 132 | endif() 133 | target_link_libraries(${target} PRIVATE $ $<$:log>) 134 | if(_is_shared) 135 | target_link_libraries(${target} PRIVATE ${_link_libraries}) 136 | else() 137 | target_link_libraries(${target} PUBLIC ${_link_libraries}) 138 | set_target_properties (${target} PROPERTIES INTERFACE_POSITION_INDEPENDENT_CODE ON 139 | POSITION_INDEPENDENT_CODE ON) 140 | endif() 141 | endfunction() 142 | ##################### 143 | 144 | define_fse_target(${FSE_LIBNAME}_static) 145 | define_fse_target(${FSE_LIBNAME}_static_mt) 146 | define_fse_target(${FSE_LIBNAME}) 147 | 148 | # 测试程序 149 | add_executable(fctest EXCLUDE_FROM_ALL FeatureCompareSpeedTest.cpp) 150 | add_executable(fctest2 EXCLUDE_FROM_ALL FeatureCompareVeracityTest.cpp) 151 | 152 | target_link_libraries(fctest common_source) 153 | target_link_libraries(fctest2 common_source $<$:-lpthread>) 154 | target_compile_options(fctest2 PRIVATE $<$:/utf-8>) 155 | # ---------------------------------------------------------------------------- 156 | set (fse_DEPENDENCY "find_dependency(Cassdk)") 157 | configure_package_config_file (${PROJECT_SOURCE_DIR}/cmake/config.cmake.in 158 | ${CMAKE_CURRENT_BINARY_DIR}/fse-config.cmake 159 | INSTALL_DESTINATION ${CONFIG_INSTALL_DIR} 160 | NO_CHECK_REQUIRED_COMPONENTS_MACRO) 161 | write_basic_package_version_file (fse-config-version.cmake VERSION 162 | ${PROJECT_VERSION} COMPATIBILITY SameMajorVersion) 163 | ########安装脚本########## 164 | install(TARGETS ${_FSE_TARGETS} EXPORT fse-targets 165 | RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR} 166 | LIBRARY DESTINATION ${LIBRARY_INSTALL_DIR} 167 | ARCHIVE DESTINATION ${LIBRARY_INSTALL_DIR} 168 | PUBLIC_HEADER DESTINATION ${INCLUDE_INSTALL_DIR}/feature_se 169 | ) 170 | 171 | if(MSVC) 172 | foreach(_target ${_FSE_TARGETS}) 173 | # 如果是动态库则安装对应的pdb文件 174 | install(FILES $<$>:$> DESTINATION ${RUNTIME_INSTALL_DIR} OPTIONAL) 175 | endforeach(_target) 176 | endif(MSVC) 177 | 178 | install (FILES 179 | ${CMAKE_CURRENT_BINARY_DIR}/fse-config.cmake 180 | ${CMAKE_CURRENT_BINARY_DIR}/fse-config-version.cmake 181 | DESTINATION ${CONFIG_INSTALL_DIR} 182 | ) 183 | install(EXPORT fse-targets NAMESPACE gdface:: DESTINATION ${CONFIG_INSTALL_DIR}) 184 | 185 | # Cleanup temporary variables. 186 | set(_PUBLIC_HEADERS) 187 | set(_SOURCE_FILES) 188 | set(_FSE_TARGETS) 189 | -------------------------------------------------------------------------------- /cmake/Modules/utils.cmake: -------------------------------------------------------------------------------- 1 | ## Utility CMake functions. 2 | 3 | # ---------------------------------------------------------------------------- 4 | ## Convert boolean value to 0 or 1 5 | macro (bool_to_int VAR) 6 | if (${VAR}) 7 | set (${VAR} 1) 8 | else () 9 | set (${VAR} 0) 10 | endif () 11 | endmacro () 12 | ## 对给定的library_name,在lib_dir中查找指定类型(library_type)的库(library_type不可包含空格)， 13 | ## 定义为{library_name}(大写)${library_type} 全局变量 14 | function(defineLibraryVariable library_name library_type lib_dir ) 15 | ##计算${library_name}的静态库名 16 | string(TOUPPER ${library_name} var_project_name) 17 | ##查找${library_name}的库 18 | string(REPLACE _ " " doc_str "${library_name} ${library_type}") 19 | string(TOLOWER ${doc_str} doc_str) 20 | set (lib_var_name ${var_project_name}${library_type}) 21 | unset(${lib_var_name} CACHE) 22 | find_library( ${lib_var_name} 23 | NAMES ${library_name} 24 | PATHS ${lib_dir} 25 | DOC ${doc_str} 26 | NO_DEFAULT_PATH 27 | ) 28 | message(STATUS "${lib_var_name}=${${lib_var_name}}" ) 29 | endfunction() 30 | # ---------------------------------------------------------------------------- 31 | ## Extract version numbers from version string 32 | function (version_numbers version major minor patch) 33 | if (version MATCHES "([0-9]+)(\\.[0-9]+)?(\\.[0-9]+)?(rc[1-9][0-9]*|[a-z]+)?") 34 | if (CMAKE_MATCH_1) 35 | set (_major ${CMAKE_MATCH_1}) 36 | else () 37 | set (_major 0) 38 | endif () 39 | if (CMAKE_MATCH_2) 40 | set (_minor ${CMAKE_MATCH_2}) 41 | string (REGEX REPLACE "^\\." "" _minor "${_minor}") 42 | else () 43 | set (_minor 0) 44 | endif () 45 | if (CMAKE_MATCH_3) 46 | set (_patch ${CMAKE_MATCH_3}) 47 | string (REGEX REPLACE "^\\." "" _patch "${_patch}") 48 | else () 49 | set (_patch 0) 50 | endif () 51 | else () 52 | set (_major 0) 53 | set (_minor 0) 54 | set (_patch 0) 55 | endif () 56 | set ("${major}" "${_major}" PARENT_SCOPE) 57 | set ("${minor}" "${_minor}" PARENT_SCOPE) 58 | set ("${patch}" "${_patch}" PARENT_SCOPE) 59 | endfunction () 60 | 61 | # ---------------------------------------------------------------------------- 62 | ## Configure public header files 63 | function (configure_headers out) 64 | set (tmp) 65 | foreach (src IN LISTS ARGN) 66 | if (IS_ABSOLUTE "${src}") 67 | list (APPEND tmp "${src}") 68 | elseif (EXISTS "${PROJECT_SOURCE_DIR}/src/${src}.in") 69 | configure_file ("${PROJECT_SOURCE_DIR}/src/${src}.in" "${PROJECT_BINARY_DIR}/include/${GFLAGS_INCLUDE_DIR}/${src}" @ONLY) 70 | list (APPEND tmp "${PROJECT_BINARY_DIR}/include/${GFLAGS_INCLUDE_DIR}/${src}") 71 | else () 72 | configure_file ("${PROJECT_SOURCE_DIR}/src/${src}" "${PROJECT_BINARY_DIR}/include/${GFLAGS_INCLUDE_DIR}/${src}" COPYONLY) 73 | list (APPEND tmp "${PROJECT_BINARY_DIR}/include/${GFLAGS_INCLUDE_DIR}/${src}") 74 | endif () 75 | endforeach () 76 | set (${out} "${tmp}" PARENT_SCOPE) 77 | endfunction () 78 | 79 | # ---------------------------------------------------------------------------- 80 | ## Configure source files with .in suffix 81 | function (configure_sources out) 82 | set (tmp) 83 | foreach (src IN LISTS ARGN) 84 | if (src MATCHES ".h$" AND EXISTS "${PROJECT_SOURCE_DIR}/src/${src}.in") 85 | configure_file ("${PROJECT_SOURCE_DIR}/src/${src}.in" "${PROJECT_BINARY_DIR}/include/${GFLAGS_INCLUDE_DIR}/${src}" @ONLY) 86 | list (APPEND tmp "${PROJECT_BINARY_DIR}/include/${GFLAGS_INCLUDE_DIR}/${src}") 87 | else () 88 | list (APPEND tmp "${PROJECT_SOURCE_DIR}/src/${src}") 89 | endif () 90 | endforeach () 91 | set (${out} "${tmp}" PARENT_SCOPE) 92 | endfunction () 93 | 94 | function(fatal_if_not_target target) 95 | if(NOT TARGET ${target}) 96 | message(FATAL_ERROR "INVALID TARGET :${target}") 97 | endif() 98 | endfunction() 99 | 100 | function (link_static_libstdcxx_if_linux target ) 101 | ############linux下静态链接c++库################### 102 | if(CMAKE_SYSTEM_NAME MATCHES "Linux") 103 | fatal_if_not_target(${target}) 104 | get_target_property(_type ${target} TYPE) 105 | if(_type STREQUAL "STATIC_LIBRARY") 106 | message(STATUS "WARNING:${target} is STATIC_LIBRARY,skip") 107 | else() 108 | get_target_property(_options ${target} LINK_FLAGS) 109 | if(_options) 110 | set_target_properties(${target} PROPERTIES LINK_FLAGS "-static-libstdc++ ${_options}" ) 111 | else() 112 | set_target_properties(${target} PROPERTIES LINK_FLAGS "-static-libstdc++" ) 113 | endif() 114 | message(STATUS "target ${target} add link option:-static-libstdc++") 115 | endif() 116 | unset(_type) 117 | endif() 118 | endfunction () 119 | 120 | # Use the static C library for all build types to a target 121 | function (with_mt_if_msvc target ) 122 | fatal_if_not_target(${target}) 123 | if(MSVC) 124 | set(_mt "/MT$<$:d>") 125 | get_target_property(_options ${target} COMPILE_OPTIONS) 126 | if(_options) 127 | # message(STATUS "${target} COMPILE_OPTIONS=${_options}") 128 | if(${_options} MATCHES "/MD") 129 | string(REGEX REPLACE "/MD" "/MT" _options "${_options}") 130 | else() 131 | set(_options "${_options} ${_mt}") 132 | endif() 133 | else() 134 | set(_options "${_mt}") 135 | endif() 136 | get_target_property(_type ${target} TYPE) 137 | if(_type STREQUAL "STATIC_LIBRARY") 138 | # 静态库将/MT选项加入INTERFACE_COMPILE_OPTIONS 139 | target_compile_options( ${target} PUBLIC "${_options}") 140 | else() 141 | target_compile_options( ${target} PRIVATE "${_options}") 142 | endif() 143 | # Cleanup temporary variables. 144 | unset(_mt) 145 | unset(_options) 146 | unset(_type) 147 | message(STATUS "target ${target} use static runtime /MT") 148 | endif(MSVC) 149 | endfunction() 150 | # Use the static C library for all build types to all target 151 | function(with_mt_if_msvc_for_all) 152 | foreach(var 153 | CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE 154 | CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO 155 | CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE 156 | CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO 157 | ) 158 | if(${var} MATCHES "/MD") 159 | string(REGEX REPLACE "/MD" "/MT" ${var} "${${var}}") 160 | endif() 161 | endforeach() 162 | endfunction() 163 | 164 | # 对executeable和shared library target有效 165 | function (use_static_library target ) 166 | fatal_if_not_target(${target}) 167 | get_target_property(_type ${target} TYPE) 168 | if(_type STREQUAL "STATIC_LIBRARY") 169 | message(STATUS "WARNING:${target} is STATIC_LIBRARY,skip") 170 | else() 171 | link_static_libstdcxx_if_linux(${target}) 172 | with_mt_if_msvc(${target}) 173 | endif() 174 | unset(_type) 175 | endfunction() 176 | 177 | function(cxx11_support) 178 | #判断编译器类型,如果是gcc编译器,则在编译选项中加入c++11支持 179 | if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") 180 | set(CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS}" PARENT_SCOPE) 181 | message(STATUS "optional:-std=c++11") 182 | endif() 183 | endfunction() 184 | 185 | 186 | ##############根据编译条件定义project_name 安装文件夹名############## 187 | #定义变量_folder 188 | function(define_project_folder_name project_name with_static_crt) 189 | if(NOT project_name) 190 | message(FATAL_ERROR "INVALID ARGUMENT :project_name") 191 | endif() 192 | set(varname "${project_name}_folder") 193 | unset(${varname} PARENT_SCOPE) 194 | if( WIN32) 195 | set(_os_name "windows") 196 | elseif(CMAKE_SYSTEM_NAME MATCHES "Linux") 197 | set(_os_name "linux") 198 | else() 199 | string(REPLACE " " "_" _os_name "${CMAKE_SYSTEM_NAME}" ) 200 | endif() 201 | if(WIN32) 202 | if(MSVC) 203 | set(_compiler "vc") 204 | elseif(CMAKE_COMPILER_IS_GNUCXX) 205 | set(_compiler "gcc") 206 | else() 207 | string(REPLACE " " "_" _compiler "${CMAKE_C_COMPILER_ID}" ) 208 | endif() 209 | else() 210 | set(_compiler "") 211 | endif() 212 | if(CMAKE_SYSTEM_PROCESSOR MATCHES "AMD64|x86_64") 213 | set(_processor "x86_64") 214 | elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86") 215 | set(_processor "x86") 216 | else() 217 | string(REPLACE " " "_" _processor "${CMAKE_SYSTEM_PROCESSOR}" ) 218 | endif() 219 | set(_folder "${project_name}-${_os_name}") 220 | if(_compiler) 221 | set(_folder "${_folder}-${_compiler}") 222 | endif() 223 | set(_folder "${_folder}-${_processor}") 224 | if(MSVC AND with_static_crt ) 225 | set(_folder "${_folder}-mt") 226 | endif() 227 | set(${varname} ${_folder} PARENT_SCOPE) 228 | endfunction() 229 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # feature_se(feature search engine) 2 | 3 | (人脸)特征内存搜索引擎(feature search engine)，提供高速的人脸特征相似度比对搜索/排序,支持多线程并行搜索，适用于百万级以上人脸库的快速搜索。(C++11实现) 4 | 5 | ## clone代码 6 | 7 | git clone --recursive https://gitee.com/l0km/feature_se.git 8 | # 因为项目中有submodule，克隆代码时必须加上 --recursive 参数 9 | 10 | ## feature_se 11 | 12 | 实现在内存中进行大规模特征比对的人脸搜索功能。 13 | 14 | 人脸搜索引擎的核心是一个允许并发读写的自定义哈希表(参见 [hashtable_abstract.h](feature_se/hashtable_abstract.h),[ICodeManagerCPU.cpp](feature_se/ICodeManagerCPU.cpp)),特征搜索过程是就是对哈希表中的(人脸)特征多线程分段比对排序/合并结果(类似map/reduce)的过程。 15 | 16 | 大规模人脸搜索的速度最终取决于人脸识别算法提供的比对函数的计算速度和服务器内存读写速度。本库提供的搜索算法将所有的特征加载到内存，完全基于内存比对，最大限度减少了比对函数之外的运行时间开销，基于CASSDK人脸识别算法在500万条记录的环境下，(24核处理器)搜索速度在0.1~0.2秒。 17 | 18 | 当前项目支持CASSDK人脸识别算法，也可以支持其他第三方算法，搜索算法本身已经高度抽象，独立于人脸识别算法，如果需要支持其他人脸识别算法，只需要替换对应的特征比对函数即可,参见: 19 | 20 | [feature_se/feature_compare_cas.h](feature_se/feature_compare_cas.h) 21 | 22 | [feature_se/HashMapCl.cpp](feature_se/HashMapCl.cpp) 23 | 24 | 25 | cmake生成的工程文件可以同时生成动态库(fse_cas)，静态库(fse_cas_static)。windows下还提供不依赖msvcrt.dll的MT静态库(fse_cas_static_mt)。 26 | 27 | 参见cmake脚本：[feature_se/CMakeLists.txt](feature_se/CMakeLists.txt) 28 | 29 | **接口描述** 30 | 31 | 标准C接口，参见 [feature_se/feature_se.h](feature_se/feature_se.h) 32 | 33 | ## jni 34 | 35 | feature_se的jni接口动态库实现，提供java访问能力。 36 | 37 | jni接口定义参见[jni/net_gdface_sdk_fse_FseJniBridge.h](jni/net_gdface_sdk_fse_FseJniBridge.h) 38 | 39 | 对应的java接口定义参见[java.net.gdface.sdk.fse.FseJniBridge.java](https://gitee.com/l0km/faceapi/blob/master/faceapi-base/src/main/java/net/gdface/sdk/fse/FseJniBridge.java) 40 | 41 | ## 编译 42 | 43 | 编译要求 44 | 45 | 46 | - cmake 3.0以上版本 47 | - JDK 1.7 48 | 49 | 50 | ### windows 51 | 52 | window下编译要求Visual Studio 2015 53 | 54 | Windows下命令行编译过程: 55 | 56 | rem 清除 build 文件夹 57 | if exist build rmdir build /s/q 58 | mkdir build 59 | cd build 60 | rem 创建VS2015编译环境,只需要执行一次 61 | call "%VS140COMNTOOLS%..\..\vc/vcvarsall" x86_amd64 62 | rem 生成Makefile 63 | cmake -G "NMake Makefiles" ^ 64 | -DCMAKE_BUILD_TYPE=RELEASE ^ 65 | -DCASSDK_ROOT_DIR=%CASSDK算法SDK安装位置% ^ 66 | -DCMAKE_INSTALL_PREFIX=..\release\fse_windows_x86_64 .. 67 | rem 编译并安装到CMAKE_INSTALL_PREFIX指定的位置 68 | nmake install 69 | 70 | 参见编译脚本：[build.bat](build.bat) 71 | 72 | 执行[all_build.bat](all_build.bat)同时编译debug,release版 73 | 74 | ### vs2015工程编译 75 | 76 | 可以用 [make_msvc_cassdk_project.bat](make_msvc_cassdk_project.bat) 创建Visuao Studio 2015工程 77 | 78 | ### Linux 79 | 80 | linux下编译要求支持c++11的gcc 5.2.0 81 | 82 | linux下命令行编译过程: 83 | 84 | # 清除 build.gcc 文件夹 85 | [ -d build.gcc ] && rm -fr build.gcc 86 | mkdir build.gcc 87 | pushd build.gcc 88 | # 生成release 版 Makefile 89 | cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=RELEASE \ 90 | -DCASSDK_ROOT_DIR=${CASSDK算法SDK安装位置} \ 91 | -DCMAKE_INSTALL_PREFIX=../release/fse_linux_x86_64 .. 92 | # 编译并安装到CMAKE_INSTALL_PREFIX指定的位置 93 | make install 94 | popd 95 | 96 | 参见编译脚本 [build.sh](build.sh) 97 | 98 | ### Android NDK 99 | 100 | 编译要求 101 | 102 | - Android NDK 版本不限，但Clang要求支持C++11编译，15c编译通过 103 | 104 | Android NDK交叉编译方法如下(参见 [make_ndk_project.bat](make_ndk_project.bat)). 105 | 106 | 107 | 108 | @rem EXT_SDK_TYPE 指定算法类型可选值： 109 | @rem CASSDK(默认值) 110 | @rem EUCLIDEAN 默认使用欧氏距离计算相似度 111 | @rem CUSTOM 使用自定义算法提的供相似度比较函数，此方式暂时未支持 112 | @rem 如果EXT_SDK_TYPE指定为EUCLIDEAN,下列参数需要设置: 113 | @rem EUCLIDEAN_ELEM_TYPE 定义特征值数组类型(double/float)，如果不指定，默认值为double 114 | @rem EUCLIDEAN_ELEM_LEN 定义特征值数组长度 115 | @rem EUCLIDEAN_CODE_END_WITH_SUM 定义特征值数组最后是否有一个double保存特征值数组的点积和，默认为OFF 116 | @rem ============================下列为通用参数与EXT_SDK_TYPE无关 117 | @rem FSE_LIBNAME 指定生成动态库名,不指定则使用默认值 118 | @rem JNI_FSE_LIBNAME 指定生成jni动态库名,不指定则使用默认值 119 | 120 | cmake %sh_folder% -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=%build_type% ^ 121 | -DJNI_FSE_LIBNAME=FS_FaceFeatureCompare ^ 122 | -DEXT_SDK_TYPE=EUCLIDEAN ^ 123 | -DEUCLIDEAN_ELEM_TYPE=double ^ 124 | -DEUCLIDEAN_ELEM_LEN=512 ^ 125 | -DEUCLIDEAN_CODE_END_WITH_SUM=OFF ^ 126 | -DCMAKE_SYSTEM_VERSION=17 ^ 127 | -DANDROID_ARM_NEON=ON ^ 128 | -DCMAKE_INSTALL_PREFIX=%sh_folder%release\fse_android_armeabi-v7a ^ 129 | -DCMAKE_TOOLCHAIN_FILE=%ANDROID_NDK%\build\cmake\android.toolchain.cmake 130 | 131 | ### 指定第三方库的特征比对函数 132 | 133 | 本项目独立于具体人脸识别算法，所以在实际应用中需要指定人脸特征比对函数，可以如下示例使用第三方库提供的比对函数，无需修改本项目代码,完整脚本参见 [make_msvc_custom_project.bat](make_msvc_custom_project.bat) 134 | 135 | @rem 使用第三方识别库提供的特征比对函数 136 | @rem EXT_SDK_TYPE 识别函数类型 CUSTOM 使用第三方库提供的特征比对函数 137 | @rem 如果EXT_SDK_TYPE指定为CUSTOM,下列参数需要设置: 138 | @rem CUSTOM_FEACOMP_INCLUDE 指定比对函数所在头文件的位置(文件夹全路径) 139 | @rem CUSTOM_FEACOMP_LIBRARY 指定比对函数所在库文件(全路径) 140 | @rem CUSTOM_FEACOMP_HEADERS 指定引用比对函数所需要的头文件名列表,';'分隔,按顺序引用 141 | @rem CUSTOM_FEACOMP_FUNNAME 指定比对函数名, 142 | @rem CUSTOM_FEACOMP_FUNTYPE 指定比对函数类型定义, 143 | @rem 格式:return_type(intput_type0,intput_type1),如果不指定则默认为double(unsigned char*,unsigned char*) 144 | @rem CUSTOM_SYS_HEADERS 指定需要引用的系统头文件名,如windows.h,可不设置 145 | @rem CUSTOM_CODE_BYTE_LEN 特征数据长度(byte) 146 | 147 | echo creating x86_64 Project for Visual Studio 2015 ... 148 | cmake -G "Visual Studio 14 2015 Win64" -DCMAKE_INSTALL_PREFIX=..\release\fse_custom_windows_x86_64 .. ^ 149 | -DEXT_SDK_TYPE=CUSTOM ^ 150 | -DCUSTOM_FEACOMP_INCLUDE=J:\workspace.neon\cassdk54\FSFaceSDK\FSFaceSDK-windows-x86_64\include ^ 151 | -DCUSTOM_FEACOMP_LIBRARY=J:\workspace.neon\cassdk54\FSFaceSDK\FSFaceSDK-windows-x86_64\lib\FSFaceSDK.lib ^ 152 | -DCUSTOM_FEACOMP_HEADERS=FSFaceSDK.h ^ 153 | -DCUSTOM_SYS_HEADERS=windows.h ^ 154 | -DCUSTOM_FEACOMP_FUNNAME=FSCompare ^ 155 | -DCUSTOM_FEACOMP_FUNTYPE="double(unsigned char*, unsigned char*)" ^ 156 | -DCUSTOM_CODE_BYTE_LEN=2560 157 | 158 | 根据以上提供的信息,cmake脚本会自动生成如下的临时代码文件`custom_feature_compare.h`,并加入项目中，在实现内存特征搜索时会调用代码中的`compare`函数从而实现调用第三方比对函数完成特征比对： 159 | 160 | /* 161 | * custom_feature_compare.h 162 | * Description: 由custom_feature_compare.h.in模板自动生成, 163 | * 用于将第三方特征比对函数封装为统一的接口供搜索比对调用 164 | * Created on: 2019年2月22日 165 | * Author: guyadong 166 | */ 167 | 168 | #ifndef CUSTOM_FEACOMP_H_ 169 | #define CUSTOM_FEACOMP_H_ 170 | #include 171 | #include 172 | 173 | #include 174 | #include "FSFaceSDK.h" 175 | 176 | #include "codemgr.h" 177 | namespace gdface{ 178 | namespace feature{ 179 | template 180 | struct function_traits; 181 | 182 | template 183 | struct function_traits> 184 | { 185 | static const size_t nargs = sizeof...(Args); 186 | 187 | typedef R result_type; 188 | 189 | template 190 | struct arg 191 | { 192 | typedef typename std::tuple_element>::type type; 193 | }; 194 | }; 195 | inline double compare(const face_code &f1,const face_code&f2){ 196 | 197 | typedef std::function feacomp_fun; 198 | return (double)FSCompare((function_traits::arg<0>::type)f1.element,(function_traits::arg<1>::type)f2.element); 199 | } 200 | } /* namespace feature*/ 201 | } /* namespace gdface*/ 202 | 203 | #endif /* CUSTOM_FEACOMP_H_ */ 204 | 205 | 206 | ## 调用示例 207 | 208 | cmake查找feature_se库的示例： 209 | 210 | # 调用cmake/Modules/FindFse.cmake查找fse依赖库 211 | # 需要在 CMAKE_MODULE_PATH 指定 FindFse.cmake的位置 212 | find_package(Fse REQUIRED) 213 | 214 | cmake脚本中引用feature_se库的示例： 215 | 216 | # 引用fse_cas动态库 217 | target_link_libraries(test_fse gdface::fse_cas) 218 | # 引用fse_cas静态库 219 | target_link_libraries(test_fse_static gdface::fse_cas_static) 220 | # 引用fse_cas静态库/MT 221 | target_link_libraries(test_fse_static_mt gdface::fse_cas_static_mt) 222 | 223 | cmake脚本中引用feature_se库的完整示例参见 [test/CMakeLists.txt](test/CMakeLists.txt) 224 | 225 | 创建调用示例的VS2015工程: 226 | 227 | mkdir build 228 | cd build 229 | call "%VS140COMNTOOLS%..\..\vc/vcvarsall" x86_amd64 230 | rem 生成64位工程 231 | cmake -G "Visual Studio 14 2015 Win64" ^ 232 | -DCASSDK_ROOT_DIR=%CASSDK算法SDK安装位置% ^ 233 | -DFSE_ROOT_DIR=..\..\release\fse_windows_x86_64 .. 234 | rem 需要先编译feature_se 235 | rem FSE_ROOT_DIR用于指定 feature_se 安装位置 236 | 237 | 完整脚本参见 [test/make_msvc_cassdk_project.bat](test/make_msvc_cassdk_project.bat) 238 | 239 | 生成unix Makefile过程参见： 240 | [test/make_gcc_project.sh](test/make_gcc_project.sh) -------------------------------------------------------------------------------- /feature_se/FeatureCompareVeracityTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * FeatureCompareVeracityTest.cpp 3 | * 4 | * Created on: 2016年1月7日 5 | * Author: hadoop 6 | */ 7 | 8 | #include 9 | #include 10 | #include 11 | #include "feature_compare.h" 12 | #include "common_utilits.h" 13 | #include "sample_log.h" 14 | using namespace gdface; 15 | using namespace std; 16 | 17 | inline face_code to_face_code(const char*hex){ 18 | return *reinterpret_cast(hex_to_bytes(hex).data()); 19 | } 20 | inline face_code to_face_code(const string &hex){ 21 | return *reinterpret_cast(hex_to_bytes(hex).data()); 22 | } 23 | 24 | int main() { 25 | // setlocale(LC_CTYPE, ""); 26 | //std::locale::global(std::locale("")); 27 | 28 | //locale lc("zh_CN.UTF-8"); 29 | 30 | //std::wcout.imbue(std::locale(std::locale(), "", LC_CTYPE)); 31 | //std::cout.imbue(std::locale(std::locale(), "", LC_CTYPE)); 32 | 33 | const wchar_t * wcp = L"[char pointer汉字]"; 34 | double pi = 3.14159265358979323846; 35 | // string,wstring,pointer,number类型测试 36 | SAMPLE_OUT("{}std::wcout输出测试 wchar_t*:{} pointer = {} double:{} chinese char*:{}", "hello,", wcp, &pi, pi,"error程序猿"); 37 | // 当输入参数多于{} 占位符时，多余的参数不显示 38 | SAMPLE_OUT("{}std::wcout输出测试 wchar_t*:{} ", "hello,", wcp, &pi, pi); 39 | // 当输入参数少于{} 占位符时，显示多余的占位符 40 | SAMPLE_OUT("{}std::wcout输出测试 wchar_t*:{} pointer = {} double:{} chinese char*:{}", "hello,", wcp); 41 | SAMPLE_OUT("ERROR: {}", "std::wcerr输出测试"); 42 | SAMPLE_LOG("LOG: {}", "std::wclog输出测试"); 43 | 44 | std::wcout << gdface::tolower(std::wstring(L"字符串转小写TEST HELLO WORD 测试")) << std::endl; 45 | std::wcout << gdface::toupper(std::wstring(L"字符串转大写test hello word 测试")) << std::endl; 46 | 47 | std::wcout << to_wide_string("string到wstring转换测试") << std::endl; 48 | std::cout << to_byte_string(L"wstring到string转换测试") << std::endl; 49 | 50 | auto g_result1 = split(std::string("hello,do you ;know the word?"), std::string("[\\s,;?]+")); 51 | std::copy(g_result1.begin(), g_result1.end(), std::ostream_iterator(std::cout, "<->")); 52 | 53 | std::cout << "=============" << std::endl; 54 | 55 | auto g_result2 = split(std::wstring(L"lao ban 老板，来份，小龙虾，快点啊！？"), std::wstring(L"[\\s，？！]+")); 56 | std::copy(g_result2.begin(), g_result2.end(), std::ostream_iterator(std::wcout, L"<->")); 57 | 58 | std::cout << "=============" << std::endl; 59 | 60 | auto s_result = split("{}hello{}world{}", "\\{\\}"); 61 | std::copy(s_result.begin(), s_result.end(), std::ostream_iterator(std::cout, "<->")); 62 | std::cout << "=============" << std::endl; 63 | auto c_result = split("hello,do you ;know the word?", "[\\s,;?]+"); 64 | std::copy(c_result.begin(), c_result.end(), std::ostream_iterator(std::cout, "<->")); 65 | std::cout << "=============" << std::endl; 66 | 67 | auto ws_result = split(L"lao ban 老板，来份，小龙虾，快点啊！？", L"[\\s，？！]+"); 68 | std::copy(ws_result.begin(), ws_result.end(), std::ostream_iterator(std::wcout, L"<->")); 69 | 70 | //const char *hex_code1 = "f48cb1bf7d0958c0dcdb3a40ac589140bf0b813e34f45740fca3adbf4c3dd6bd2837154091b8b2bf9a4443bf6768b73f99ec483fa312873e6c7ea4bf458290bfef41cebe6bcb1f3e7b40a5bec8847ebfd091f9bf67a344bfc68343bd8e6b2f3e8aa7953f3c160d3f258c76bff0c41dbe6c327fbfa06c53bf93e8fabdc8221bbfb657d9bfef57183e47ca4ebcdff91e3e192c9ebf1b21813fdfeb95bf2e5a523e8557483e72442dbfb593a43e800c5dbe247609bfc8b5a1be728aac3ff319a83e34d959bfd92c933e47f7b5bd6c0b8dbf0229403f022b3a3e069d8f3f2a1513be6fb30e3f89d822bed9719f3f6370bfbe1d1b32be92db10bf7288233f8bf14d3f549f21bfa683983e0440c0bf5eb0323f0654b2bdbb13e7bfb02dc43d7bffe6be600e223fac2e18bf4031b4bf6409c73d8de28dbd2f519dbe51cac83f66328cbe48a4a13ed9d96bbf7d885cbf9f63c53eddf5763fb815583e0333573f49dc3d3e1d7566bfc021b83ed2506dbed43fe6bffb684a3e0c89933f9efebfbe25433bbf7230ef3db4d7b93f868539be8c8d883e90c265bceb83173ea5b9bbbfa43a6d3d0ab403be240b05bf3de240bf95cbb43d1c180d3e1b74b6be4df0d5beb831913e939c2c3e9eaa8e3eb87a423df73d3e3cea20e6bc0fad1ebf2fc45f3fa4c9a3be5658c93f0e3687bf8659913dc50e0d3fa335b03e91b7833e56d5ad3e054f283f563c493f0ce34a3f860367bfde6e813e73b2573f0c7b25be7683a9be4ea4cf3ec659243f995874bf6976043eaae9363f17b9263ffb7e8b3d133617be4873893e803bdebe270f35bf8a0d35bec542ad3e534b7e3f342636bf4555723fbb1f92bdddb09fbf5cc2d93dbb60603d9585713f2f5876be67e006becc6d53bfec4d3bbd76ac2dbf5f66a5bda73e93bf7e59933ef055f33e40981e3f95a977bea843913dbbc38fbe3d8095bff338b0be95edad3e5fb3dabed54041bc787b8b3ea396a43f00bc61bf4103dfbea39f08beaa3c413ef1876abf598c24bfea710fbf5daeb5be70ac12bff5cbbcbe8119f33e8cb7273f6c05dfbedd454fbf8c6d6a3ff346033fde1fc23d3a7dd93e0c73c33effa2263f197070bd94703cbecadb803eee316ebebcc167bf62c49dbed17e04bf0ca316be0a00cabd4633863d1bc700bfc2a42bbf77d5a83d783a63bd00e63f1e53a86340"; 71 | //const char *hex_code2 = "f9b10fc00712f4c000c31d405a5ba240d77c0bc0d6ed533f71f680c00048f03f855703c0653b1d3fcd2acdbfc11f6c3f10b0c7bfb7f2b1bfdcaa8abf068602c03dee02bf2de7b1be0ad18bbf6a39e0bf94a1bc3e671664c06884b7bffbb4e73f8b24a6bfdfe9e53d94e2d93e48b462402db4ba3fd6a1c63e4fab41bf750e10c01cb02840218bf53f1f43a2bfc9ea10bf53d960bfa6666e3f4f3c163ffc99623f908f42c059a719403817c1bfddc209c022b03d40cc2f0f3dc5b3b5bedb83cb3e695615c0368f873e068ebdbfe1fa37bf5fc261bf1b613540366227c08595f93d463682bf40f0833dad1ba7bf064f40c0106317bfcbd28dbf0b2c0c40925d7a3fb5be4640ae231340645f5fc054e4863f1c80febf064981bf13b5e6bfc3120cc0643a9fbfac3e9dbfb1a8c2be44293ac0b50ffdbf6eb9e93e214c67bf0d6f05405576b3bd0ad3154077521ebe74edb43f58209e3f1af0983ffe6d8abefd16d9bf8d299bbdd528294056d2ca3e518f95bf6f162e3f13606bbf5539b13f861f5bbf1b90ab3d7d29af3f1dd0e7beef210a3f2a2ce4bf92001440c490413f7d6629bf84cc53be327775bf4f8a683f23c14e3fab2fbdbfc70620bf5b00b1be9cb0e93fc9a654bd11751c3f595e10403ef8eb3e81678abf98a9373f3e228f3f29019e3f86a002c0cac3043fe0ae8abf0446c8bfddddc93fd2ea02bffa23b0bdc82ba2bf41328cbf9b4b8ebe6fa7494016ebfc3e879237bf7583ddbec40675bf8ccb0fbf958b1abf31cd0240d5ececbe3fa82c3f6083933f328223bfe17ad13ec6d2ff3f712ba63df7b16dc075698fbfbbe8913d06f2ed3fc8fe3c3f41e548bfce130dbfd44fafbd648eafbe41ca043de88b2e3f4148a7bff3c0b9bf4228413f6af98cbf182fddbebe53c93cdb6a0bbf134bbabe697934bf4965913e4e731bbf4606e03e973da33e77b0c43ead6d51bfe1d8bb3eba211ebf77bd0740446c373e12aaeebe3b6e9b3f6c83d0bfc07d90bfa99d613f763be2bfab35a3be5e23e13e91ecdabef2ea1340a2720d3f0062333ef28fb23dd70582bd2cce423f9d438ebe1daba0bfc375cc3e1ec61fc099391fbe0594bb3f85aba0bd51bd1ac001c21340a08890bf2e01513ebd4a953fcc8995bf2269823f839c0e40c4f86abfe97f7a3f196804bf830d4ac0ae7b54bf001876e198518040"; 72 | //const char * hex_code1="4a8900c0619d38c089f2803fcf813840daa3a73f65f9713f58732c3fcf2a4a40ada30840a85d843ec746b4bff985a03f59285bbf98bce73f2a1803c0553c883fc7c7b4bf2b6e563f0f4693bf5e6156bf77c555bff110b9bf5bf786bf2e14c7bc5f881ebeedbfa83e098ca8bf49e0a0bf641ea03f5876bc3f9c3a523ea54600c0564aa4bde6952bbf364a653ece620fc0a1b9b1bfe38152bfe7c0213f16a558bf1dc0e53efb3915bf0c9816c053d3b5bdd9646fbf4e8f32bf4f1d17be804851bfc88e083ff8eb21bd7912babff9d31dbfcbe89e3fefc60f3f9b5b0b3fb8fa963f607f57bfca7a36bf1032753edf6689be989a0c3fa49ca83d302c50be65461bc0c476afbee56c043fa6ea49bef548363f6d7b06bfbd9b3bbe74f3553ec70edcbecc46e33fb1e76e3fd193cbbfb0c3b63f8661343f49823ebfb778353ec958393ee1b735be646d44bfbd0ec53eab4e5d3f0870f33d079477bf0a09c43ea33b78be515ffc3fb340b0bf88b304bd642c1abdfb17633eba1417bed455d7bf7dd1813f1550a6bf473c03bf9187663f2b68c2bf51138e3e09a128bf3b41bf3c01b58e3f81af81bef08ddebfe9f7ce3e2663183f854d48be4529debf2bfaf63e30d04cbfd410833f57f1473f3b6a083f6a93583f09f63fbfe49d1c3e7cce3ebc6353bcbec7bae53ee88b49bfbe56d6be3b03493f7ac1863ed380e1be785047bf91e7b63e47457cbfbf3eb83f457124bf1c978dbfd0ffd13dbe169bbfe2b981bff9a4593dea91a7befb1cbabfdd079d3e0fe5f13e347926bfc5c3293f554fb2bff73765bf65d5403f0966a63fa62d963fbb820240959cb33fb2b9963e6d9b0fbeda95a53f0dacda3f31af0d3ec4ca913fca7a86bfaeb6373eef29debd87f06b3f1329123fbf838a3f950b0c3f4f65823fdb3b5dbf8ed01bbfe08fdfbe2f6c24bfbef913be56de703f8d0f563f916df03fb9fe35be475dca3fea45a33f40d0f03fc1f95d3ff806803ecf09a93f8cc80f3cff3567bf95460dbf1b44923e61555b3e130fe8be9150193f2a89913fc4ed4b3eb2eb2fbf611205bedea9bbbf5a4fac3eb0c2edbf957dcfbee3b22ebfeebc70bf30f3383f622a103f5d2830be9f36f23e07332dbc1b8ca7be2a3755bee07695bf50f68d3f56f936bd5e472d3fd74419bd194386bfac25bcbea479bf3d00a4b8b5c7d76a40"; 73 | //const char * hex_code2="63923fc0837ecdc0c1d5293fd47a2c4000fa933f6ba60340e8910bbeaeac773fe3602e40069b493f16f541be26042e40e69f02c073d522c01258e3bf6048b6bf1086613e9ac1c4be12fbb1bff9f440bf7e7e10c0599bffbfda7abbbf18a7b93f3a45d0bfacfd343e5d453cbfdab5e6bc1f2ea93e8f31bd3f4badc1bedc1a26c002d1fb3f42c837bfb9f5db3f284b9d3f38662abff25c973e008dc0bd9dcaa9bf203db33e331f8e3f95110fc00c7f92bf6dc9533edc724cbf79d7c9bf0c53c6be4f2baabfeaf15fbf3d2fb83e10fa273f1c88e5bee559993f85eda9bfae39d33f5deaba3fa41310be4c5e5b3fc5e657bfa99e2fbfbb2bf63fb823f63f7d4e64bf54e9c13c086d04403f69cebf1256f33ec1aa8ebf1a6227c05e54093e1a0292baebc4f13fed3b7b3f363a97bf0e74123fdf27f9beceffd7bf917ae3bec449673ec16dde3f7c5bc23edcfb653f6787b63e3d55943e93c01b40b36e0cbe7d2df63d77dcdf3f0ba821bebc37823eab73523e7281073f2d439ebeb7e1e03ec6086c3f273900c061996b3fe2bc683ff35265bf93b4bdbea496b23a9bae1a3f145010403498903e01ca8fbfb74a9e3f43e4a63f5ab319bfe3e21fbf00351f3f97d5453f5bd199bedb397a3e92db5a3f082805402bf8a53ed75c6b3f30483b3f951c783f340776bf2e7889be67309bbfb3f4863eca8a5f3fc99e2b3ef90187befc5e37be380e62bfdb4e863fbd1033bf8e5cc8bf5a39213fe72a21bf96d15cbf3f2793bff6064c3e25c681bfeefa31bf6a9eb33f5c80c3bfb75ce6be4de589bf8487fc3d179c0bbfc17913bf5c0fd93e6bb940bf89a80a3fd9403f3f2ae12dbf304c0e3f03f968bfd3735ebd6e5e743db0e4b6be5169993edbfcd3bf4983253fb409043fe1ef00403b7db43f8c8013bf471422bf597997bf18db1d3f2f17c4bf8b600e3ff71d36bf4b5bbf3efcab3e3ff56710bfe1f51a4022b7a33e3accb73fe7bb2c3fec746dbf1b99703d3d6383bfc0bab53e97d860bf39812a3f2411933d73aa793f3d5b05bfc21f98beecf8c6be5bd2e93e45d71cbf9d6d693fbf398ebe997129c08e28d9be57ff08be80f893bfa68cdc3e730cd63f42e41ebfe60e73bfa230ab3efd65283f6fbebd3e277ccabe9af087be295229bf0ea6453fcc036d3f119b0dbf8f655dbf59961cbfe848944480f97240"; 74 | 75 | //try { 76 | // //cout << src1 << endl; 77 | // //auto bytes1 = hex_to_bytes(src1); 78 | // auto code1 =to_face_code(hex_code1); 79 | // auto code2 =to_face_code(hex_code2); 80 | // auto sim=feature::compare(code1,code2); 81 | // cout<<"sim="< 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include "gf_utility.h" 19 | using namespace std; 20 | namespace gdface { 21 | /* 计算比较函数的返回值类型 */ 22 | template 23 | struct compare_result{ 24 | using type=typename std::conditional ::value,decltype(declval()-declval()),int>::type; 25 | }; 26 | 27 | /* 计算比较器对象(Callable Object)的类型 */ 28 | template 29 | struct compartor{ 30 | using type=std::function::type(N,N)const noexcept>; 31 | }; 32 | /* 定义比较器类型,并根据N,CMP,DEC选择合适的比较器类型 33 | * dec为降序,inc为升序 34 | */ 35 | template ::type *CMP,bool DEC> 36 | struct comparator{ 37 | /* 缺省的算术比较器,只适用于arithmetic类型 */ 38 | struct arithmetic{ 39 | struct dec{ 40 | typename compare_result::type inline value(N n1,N n2) const noexcept{return n1-n2;}; 41 | }; 42 | struct inc{ 43 | typename compare_result::type inline value(N n1,N n2) const noexcept{return n2-n1;}; 44 | }; 45 | }; 46 | /* 使用CMP提供的自定义比较器 */ 47 | struct other{ 48 | struct dec{ 49 | static_assert(nullptr != CMP,"nullptr==CMP"); 50 | const typename compartor::type&_comparator = *CMP; 51 | typename compare_result::type inline value (N n1,N n2)const noexcept {return _comparator(n1-n2);}; 52 | }; 53 | struct inc{ 54 | static_assert(nullptr != CMP, "nullptr==CMP"); 55 | const typename compartor::type&_comparator = *CMP; 56 | typename compare_result::type inline value (N n1,N n2)const noexcept {return _comparator(n2-n1);}; 57 | }; 58 | }; 59 | using type=typename std::conditional< 60 | nullptr!=CMP, 61 | typename std::conditional::type, 62 | typename std::conditional::type 63 | >::type; 64 | }; 65 | template ::type *CMP=nullptr,bool DEC=true,typename Enable=void> 66 | class topk_base;//通例 67 | 68 | /* 69 | * topk排序(不稳定排序)类模板, 70 | * 对象内部有一个容量为m_capacity的排序缓冲区m_sort(初始化时生成), 71 | * 用insert方法每次加入一个要排序的对象T,当T的排序值N大于阀值时，将T按排序值插入m_sort(始终是一个有序数组) 72 | * T为要排序的对象类型, 73 | * N为排序依据的值类型, 74 | * CMP为比较器函数对象(Callable Object)指针 75 | * DEC为排序类型，true为降序, 76 | * N为数字/算术(arithmetic)类型时,CMP可以为nullptr,否则必须提供CMP, 77 | * 非算术类型的CMP函数返回值为int,函数签名int(N n1,N n2), 78 | * 如果希望n1要排在n2前面,则返回>0的整数,n1==n2,返回0,否则小0; 79 | * 特例: 禁止N为bool类型,比较值为bool类型无意义 80 | * + 操作符 将两个对象合并排序成新对象 81 | * += 操作符将参数对象合并到当前对象 82 | * 调用result()后会置为0，对象就不能再执行添加(insert)和合并(merge)操作 83 | */ 84 | template ::type *CMP,bool DEC> 85 | class topk_base::value>::type>{ 86 | //静态断言:N为非算术类型时,CMP不能为nullptr 87 | static_assert(std::is_arithmetic::value||nullptr!=CMP,"FUN must not be nullptr while N is not arithmetic"); 88 | public: 89 | using _Self=topk_base; 90 | using SIZE_TYPE =size_t; 91 | /* 排序节点 */ 92 | struct CompareNode{ 93 | const T* t; 94 | N n; 95 | }; 96 | protected: 97 | private: 98 | /* 初始阀值 */ 99 | const N m_threshold; 100 | /* 排序缓冲区容量 */ 101 | const SIZE_TYPE m_capacity; 102 | /* 排序缓冲区 */ 103 | unique_ptr m_sort; 104 | /* 缓冲区中已经排序元素个数,初始为零 */ 105 | SIZE_TYPE m_size=0; 106 | /* 当前阀值指针,初始值指向m_threshold */ 107 | const N *mp_threshold=std::addressof(m_threshold); 108 | public: 109 | using M_SIZE_TYPE=decltype(m_size); 110 | private: 111 | /* 计算比较器类型 */ 112 | using _CMP_TYPE=typename comparator::type; 113 | /* 比较器对象 _compatator.value(n1,n2)返回值>0则n1排在n2前面 */ 114 | _CMP_TYPE _compatator; 115 | /* 二分法获取新排序节点node在排序缓冲区的排序位置(循环) */ 116 | auto rank(const CompareNode &node, SIZE_TYPE from, SIZE_TYPE size)const noexcept->decltype(from){ 117 | auto start=from; 118 | auto len=size; 119 | decltype(len>>1) mid; 120 | typename compare_result::type cmp_res; 121 | do{ 122 | if(len>1){ 123 | mid=len >> 1; 124 | cmp_res=_compatator.value(node.n, m_sort[start+mid].n); 125 | if(cmp_res>0){ 126 | len-=mid;// 前半部搜索 127 | }else if(cmp_res<0){ 128 | start+=++mid,len-=mid; // 后半部搜索 129 | }else 130 | return start+mid; 131 | }else if(len==1){ 132 | return _compatator.value(node.n,m_sort[start].n)>=0?start:start+1; 133 | }else 134 | return start; 135 | }while(true); 136 | } 137 | /* 更新当前阀值指针 */ 138 | void inline update_threshold()noexcept{ 139 | if (m_capacity > 0 && m_size == m_capacity) { 140 | mp_threshold=&m_sort[m_size - 1].n;//当排序缓冲区满的时候,用最后一个节点的值更新当前阀值指针 141 | } 142 | } 143 | /* 断言两个对象类型相同 */ 144 | static void inline assert_same_type(const topk_base &t1,const topk_base &t2) { 145 | if(t1.m_capacity!=t2.m_capacity||t1.m_threshold!=t2.m_threshold) 146 | throw invalid_argument("t1 has different type with t2"); 147 | } 148 | public: 149 | /* 构造函数 */ 150 | topk_base(SIZE_TYPE maxRows,N threshold) noexcept: 151 | m_capacity(maxRows), 152 | m_threshold(threshold), 153 | m_sort(maxRows>0?make_unique(maxRows):unique_ptr()){ 154 | } 155 | /* 默认构造函数 */ 156 | topk_base():_Self((SIZE_TYPE)0,N()){} 157 | /* 复制构造函数 */ 158 | /*topk_base(const topk_base& v):_Self(v.m_capacity,v.m_threshold){ 159 | m_size=v.m_size; 160 | memcpy(m_sort.get(),v.m_sort.get(),sizeof(CompareNode)*v.m_size);//只复制有效数据 161 | update_threshold(); 162 | };*/ 163 | /* 移动构造函数 */ 164 | topk_base(topk_base&& rv)noexcept=default; 165 | /* 移动赋值操作符 */ 166 | topk_base& operator=(topk_base&& rv) { 167 | modify_const(m_capacity,rv.m_capacity); 168 | modify_const(m_threshold,rv.m_threshold); 169 | m_size=rv.m_size; 170 | m_sort=std::move(rv.m_sort); 171 | mp_threshold=std::addressof(m_threshold); 172 | update_threshold(); 173 | return *this; 174 | }; 175 | 176 | virtual ~topk_base()=default; 177 | 178 | /* 将obj加入topk,如果cmpValue小于当前阀值,则返回false */ 179 | bool insert(const T& obj,const N cmpValue)noexcept { 180 | auto c=_compatator.value(cmpValue,*mp_threshold); 181 | // 大于阀值才进一步执行比较排序，小于阀值则返回 182 | if(c<0||m_capacity==0||(c==0&&m_size==m_capacity)) 183 | return false; 184 | //将排序节node点加入排序缓冲区 185 | CompareNode node{std::addressof(obj),cmpValue}; 186 | auto offset = rank(node, 0, m_size); 187 | auto sort=m_sort.get(); 188 | // 在offset位置插入节点,offset(含)之后的数据向后移一位 189 | memmove(sort + offset + 1, sort + offset, 190 | sizeof(CompareNode) * (m_size < m_capacity ? m_size++ - offset : m_capacity - offset-1)); 191 | sort[offset] = node; 192 | update_threshold(); 193 | return true; 194 | } 195 | 196 | M_SIZE_TYPE size()const noexcept{return m_size;} 197 | /* 返回排序结果缓冲区的引用,执行本函数后，再执行insert/merge将无效 */ 198 | auto result() noexcept->typename std::add_lvalue_reference::type{ 199 | modify_const(m_capacity,(decltype(m_capacity))0);//将常量m_capacity置为0，阻止添加添加(insert)和合并(merge)操作 200 | return m_sort; 201 | } 202 | /* 排序缓冲区清零 */ 203 | void reset(){ 204 | m_size=0; 205 | mp_threshold=&m_threshold; 206 | } 207 | 208 | /* 将from中排序的数据合并到当前对象中 209 | * buf_in为排序用的临时对象,可以为nullptr,如果不为nullptr,类型必须与当前对象相同 210 | * 返回当前对象引用 211 | */ 212 | topk_base& merge(const topk_base &from,topk_base * buf_in=nullptr) noexcept{ 213 | if(0m_sort){ 215 | // buf_in为nullptr时只需要右值拷贝给this 216 | *this=merge_to(from,{this->m_capacity,this->m_threshold}); 217 | }else{ 218 | buf_in->reset(); 219 | merge_to(from,std::move(*buf_in)); 220 | // buf_in不为nullptr时将this与buf_in交换 221 | std::swap(*this,*buf_in); 222 | } 223 | } 224 | return *this; 225 | } 226 | 227 | /* 将当前对象和from合并排序到目标对象to 228 | * 返回to右值引用 229 | */ 230 | topk_base&& merge_to(const topk_base &from,topk_base &&to) { 231 | assert_same_type(from,*this); 232 | assert_same_type(from,to); 233 | auto buf = to.m_sort.get(); 234 | typename std::remove_const::type idx = 0; 235 | M_SIZE_TYPE idx_this = 0, idx_from = 0; 236 | decltype(rank(m_sort[0],0,0)) offset_this,offset_from; 237 | while ( idx < m_capacity&&idx_this < m_size&&idx_from < from.m_size) { 238 | offset_this=rank(from.m_sort[idx_from],idx_this,m_size-idx_this); 239 | if(offset_this==idx_this){ 240 | offset_from = from.rank(m_sort[idx_this], idx_from + 1, from.m_size - idx_from - 1); 241 | auto copy_size=min(this->m_capacity-idx,offset_from-idx_from); 242 | memcpy(buf+idx,from.m_sort.get()+idx_from,copy_size*sizeof(CompareNode)); 243 | idx+=copy_size; 244 | idx_from += copy_size; 245 | if (idx < m_capacity) { 246 | buf[idx++] = m_sort[idx_this++]; 247 | } 248 | }else{ 249 | auto copy_size=min(this->m_capacity-idx,offset_this-idx_this); 250 | memcpy(buf+idx,m_sort.get()+idx_this,copy_size*sizeof(CompareNode)); 251 | idx+=copy_size; 252 | idx_this += copy_size; 253 | if (idx < m_capacity) { 254 | buf[idx++] = from.m_sort[idx_from++]; 255 | } 256 | } 257 | } 258 | if (idx < m_capacity) { 259 | if (idx_from < from.m_size) 260 | memcpy(buf + idx, from.m_sort.get() + idx_from, sizeof(CompareNode) * min(m_capacity-idx, from.m_size - idx_from)); 261 | else if (idx_this < this->m_size) 262 | memcpy(buf + idx, m_sort.get() + idx_this, sizeof(CompareNode) * min(m_capacity-idx, this->m_size - idx_this)); 263 | } 264 | to.m_size = min(to.m_capacity, from.m_size + m_size); 265 | to.update_threshold(); 266 | return std::move(to); 267 | } 268 | 269 | /* += 操作符 将参数对象合并到当前对象 */ 270 | topk_base& operator+=(const topk_base &from) noexcept{ 271 | return merge(from); 272 | } 273 | /* + 操作符 将两个对象合并排序成新对象 */ 274 | topk_base operator+(const topk_base &from) noexcept{ 275 | return merge_to(from,{this->m_capacity,this->m_threshold}); 276 | } 277 | string debug()const { 278 | stringstream debugstr; 279 | for (decltype (m_size)i = 0; i < m_size; ++i) { 280 | debugstr << "["< 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include "gf_utility.h" 22 | #include "hash_fun.h" 23 | #include "hashtable_core.h" 24 | #include "intrinsic_wrapper.h" 25 | #include "OverTableManager.h" 26 | #include "RWLock.h" 27 | namespace gdface { 28 | /* 与HashtableCore 中所有整数类型变量的类型一致 */ 29 | using HASH_TABLE_SIZE_TYPE=decltype(HashtableCore::obj_size); 30 | 31 | /* 用于hashtable 的标准forward迭代器 32 | * position　first为table.capacity,second为nullptr时代表表结尾 33 | * */ 34 | template 35 | struct _HashTable_iterator:public std::iterator{ 36 | using _Self =_HashTable_iterator; 37 | using _Node =_HashNode; 38 | using reference =V&; 39 | using pointer =V*; 40 | using position =std::pair; 41 | const HashtableCore *m_table=nullptr; 42 | /* 当前节点 */ 43 | position m_cur; 44 | /* 当前节点的前一个节点 */ 45 | position m_prev; 46 | explicit 47 | _HashTable_iterator(const HashtableCore &table,position pos={0,nullptr}): 48 | m_table(&table), 49 | m_cur((pos.first>table.capacity)? position{table.capacity, nullptr}:pos){ 50 | if(nullptr==m_cur.second){ 51 | // 定位到第一个元素 52 | next(); 53 | } 54 | m_prev=m_cur; 55 | } 56 | _HashTable_iterator():m_table(),m_cur(),m_prev(){} 57 | _HashTable_iterator(const _Self &)=default; 58 | _Self& operator=(const _Self &)=default; 59 | /* 60 | * ++操作符,如果m_cur已经指向表结尾,则直接返回不修改对象数据 61 | * */ 62 | _Self& operator++()noexcept{ 63 | assert(nullptr!=m_cur.second&&nullptr!=m_table); 64 | // 如果已经到表尾部则跳过 65 | if(m_cur.firstcapacity||nullptr!=m_cur.second){ 66 | m_prev=m_cur; 67 | m_cur.second=m_cur.second->next; 68 | if(nullptr==m_cur.second){ 69 | ++m_cur.first; 70 | next(); 71 | } 72 | } 73 | return *this; 74 | } 75 | _Self operator++(int)noexcept{ 76 | _Self tmp=*this; 77 | this->operator++(); 78 | return tmp; 79 | } 80 | 81 | reference operator*() const noexcept { return *static_cast(m_cur.second->obj); } 82 | pointer operator->() const noexcept { return static_cast(m_cur.second->obj); } 83 | bool operator==(const _Self& v) const noexcept { assert(m_table == v.m_table); return m_cur == v.m_cur; } 84 | bool operator!=(const _Self& v) const noexcept { assert(m_table == v.m_table); return m_cur != v.m_cur; } 85 | private: 86 | /* 87 | * 跳到下一个有效的根节点 88 | * 如果后续没有元素，则m_cur.second==nullptr,m_cur.first==m_table.capacity 89 | * */ 90 | inline void next() noexcept{ 91 | // 定位到第一个元素,如果表为空或m_cur.first>=capacity，则m_cur.second为nullptr 92 | for(;m_cur.firstcapacity&&nullptr==(m_table->nodes+m_cur.first)->obj;++m_cur.first); 93 | if(m_cur.firstcapacity)m_cur.second=m_table->nodes+m_cur.first; 94 | } 95 | }; 96 | /* 用于hashtable 的标准forward迭代器(const) */ 97 | template 98 | struct _HashTable_const_iterator:public _HashTable_iterator{ 99 | using _Base =_HashTable_iterator; 100 | using _Self =_HashTable_const_iterator; 101 | using reference =const V&; 102 | using pointer =const V*; 103 | using _Base::_Base; // 继承基类构造函数 104 | _HashTable_const_iterator(const _Base &base)noexcept:_Base(base){} 105 | _Self& operator=(const _Self &)=default; 106 | _Self& operator++()noexcept{ 107 | return static_cast<_Self&>(_Base::operator ++()); 108 | } 109 | _Self operator++(int)noexcept { 110 | _Self tmp = *this; 111 | this->operator++(); 112 | return tmp; 113 | } 114 | 115 | reference operator*() const noexcept 116 | { return _Base::operator *();} 117 | pointer operator->() const noexcept 118 | { return _Base::operator ->();} 119 | 120 | }; 121 | 122 | /* 实现哈希表的抽象类, 123 | * 要求子类提供从V中可以获取K的虚函数 keyFrom 124 | * CONCURRENCY为true 支持并行 125 | * CONCURRENCY为true时begin(),end()为private 126 | * CONCURRENCY为false时begin(),end()为public 127 | */ 128 | template 129 | class HashTableAbstract { 130 | using value_type=V; 131 | using iterator=_HashTable_iterator; 132 | using const_iterator=_HashTable_const_iterator; 133 | using position=typename iterator::position; 134 | private: 135 | const bool m_isCopy; // 加入元素时是否将元素复制到新分配内存中 136 | const float m_loadFactor; // 装载因子 137 | HASH_TABLE_SIZE_TYPE m_threshold; //resize阀值,当tabe.size大于此阀值时重建哈希表 138 | HashtableCore m_table; 139 | OverTableManager m_overTableManager; 140 | unique_ptr mu_nodes;// 指向表的内存指针,析构时自动释放内存 141 | using lock_type = typename std::conditional::type; 142 | mutable lock_type m_lock{true};// 并发模式(CONCURRENCY=true)时的读写锁 143 | /* 并发模式(CONCURRENCY=true)时返回raii对象,编译根据CONCURRENCY决定 */ 144 | template 145 | typename std::enable_if::type read_guard()const noexcept{ 146 | return m_lock.read_guard(); 147 | } 148 | template 149 | typename std::enable_if::type read_guard()const noexcept{ 150 | return m_lock; 151 | } 152 | template 153 | typename std::enable_if::type write_guard()const noexcept{ 154 | return m_lock.write_guard(); 155 | } 156 | template 157 | typename std::enable_if::type write_guard()const noexcept{ 158 | return m_lock; 159 | } 160 | 161 | /* 162 | * 从hashcode指定偏移位置的链表节点查找指定的key 163 | * lastNode不为nullptr时,lastNode返回找到的节点上一个节点 164 | * 返回key所在的节点,找不到key则返回nullptr 165 | */ 166 | HASH_NODE_PTR findInBucket(const K &key, HASH_TABLE_SIZE_TYPE bucket, HASH_NODE_PTR *lastNode=nullptr)const noexcept { 167 | assert(bucket < m_table.capacity&&nullptr != m_table.nodes[bucket].obj); 168 | HASH_NODE_PTR pNode=std::addressof(m_table.nodes[bucket]),last = pNode; 169 | for (; nullptr != pNode && !equals(keyFrom(*static_cast(pNode->obj)), key); last = pNode, pNode = pNode->next); 170 | if (nullptr != lastNode) 171 | *lastNode = last; 172 | return pNode; 173 | } 174 | 175 | /* 计算obj的hashcode,并根据hashcode计算偏移量 */ 176 | inline auto bucketIndex(const K &key)const noexcept->decltype( m_table.capacity) { 177 | static hash_fun hash_impl;// hash函数实现 178 | return hash_impl.fun(key) % m_table.capacity; 179 | } 180 | 181 | /* 表长度加1 */ 182 | inline auto incTableSize() noexcept->decltype(m_table.size){ 183 | return ++m_table.size; 184 | } 185 | /* 表长度减1 */ 186 | inline auto decTableSize()noexcept->decltype(m_table.size) { 187 | return --m_table.size; 188 | } 189 | 190 | /* 191 | * 将initCapacity向上格式化为2的N次幂整数, 192 | * 最大不超过MAX_HASHTABLE_CAPACITY 193 | * 最小DEFAULT_HASHTABLE_INITCAPACITY 194 | */ 195 | static auto formatInitCapacity(HASH_TABLE_SIZE_TYPE initCapacity)noexcept->decltype(initCapacity) { 196 | if(initCapacity <= DEFAULT_HASHTABLE_INITCAPACITY) 197 | return DEFAULT_HASHTABLE_INITCAPACITY; 198 | else{ 199 | initCapacity= bitset(initCapacity).count() > 1 200 | ?1U << intrinsics::_bsr(initCapacity << 1) : initCapacity; 201 | // initCapacity为0时是左移溢出了 202 | return 0==initCapacity||initCapacity>MAX_HASHTABLE_CAPACITY?MAX_HASHTABLE_CAPACITY:initCapacity; 203 | } 204 | } 205 | /* 初始化表信息，根据initCapacity分配表空间 206 | * initCapacity为0或(table.capacity); 215 | table.nodes = uptr_nodes.get(); 216 | } 217 | /* 218 | * 右值引用版本 219 | * 向表中插入一个节点pNewNode(不能为nullptr)，返回插入后的新节点指针, 220 | * isOverNode为true时pNewNode是溢出表中的节点 221 | */ 222 | HashNode& _insertNode(HashNode&& newNode, bool isOverNode)noexcept { 223 | assert(nullptr!=newNode.obj); 224 | auto key=keyFrom(*static_cast(newNode.obj)); 225 | auto bkt=bucketIndex(key); 226 | auto pNode = m_table.nodes+bkt; 227 | if (nullptr == pNode->obj) { 228 | // hashcode不冲突，直接加到根节点 229 | // 添加第一个节点 230 | pNode->obj = newNode.obj; 231 | pNode->next = nullptr; 232 | // 如果是溢出表节点则从溢出表中删除该节点 233 | if (isOverNode) { 234 | auto b=m_overTableManager.deleteNode(newNode); 235 | assert(b); 236 | } 237 | incTableSize(); 238 | } else { 239 | //cout<<"conflict"<obj = newNode.obj; 251 | } 252 | // 将新节点插入到根节点之后 253 | found->next = pNode->next; 254 | pNode->next = found; 255 | incTableSize(); 256 | } 257 | pNode = found; 258 | } 259 | assert(nullptr != pNode->obj); 260 | return *pNode; 261 | } 262 | /* 根据newCapacity重建哈希表 */ 263 | void resize(HASH_TABLE_SIZE_TYPE newCapacity) { 264 | HashtableCore newTable; 265 | decltype(mu_nodes) new_nodes; 266 | newCapacity=formatInitCapacity(newCapacity); 267 | if (newCapacity == m_table.capacity) 268 | return; 269 | //cout << "resize to:" << newCapacity << endl; 270 | try { 271 | initTable(newTable, newCapacity,new_nodes); 272 | } catch (const bad_alloc&e) { 273 | // 内存分配失败则返回 274 | cerr << e.what() << endl; 275 | return; 276 | } 277 | // 将newTable数据更新到对象,原table数据交换到oldTable; 278 | auto oldTable = this->m_table; 279 | this->m_table = newTable; 280 | this->m_threshold = (HASH_TABLE_SIZE_TYPE)(this->m_loadFactor * this->m_table.capacity); // 更新阀值 281 | decltype(oldTable.nodes) pBucketNode; 282 | for (decltype(oldTable.capacity) bucket = 0; bucket obj) { 285 | _insertNode(std::move(*pBucketNode), false); 286 | for (auto pNode = pBucketNode->next; nullptr != pNode&&nullptr != pNode->obj;) { 287 | auto next = pNode->next; // 临时保存next，因为在_insertNode中可能会修改pNode的next值 288 | _insertNode(std::move(*pNode), true); 289 | pNode = next; 290 | } 291 | } 292 | } 293 | assert(oldTable.size==m_table.size); 294 | mu_nodes=std::move(new_nodes);//内存指针指向新分配的内存并删除之前的内存 295 | } 296 | 297 | /* (无锁)查找key所在的节点位置，找不到则返回nullptr */ 298 | position inline _find(const K & key)const noexcept{ 299 | auto bkt=bucketIndex(key); 300 | return {bkt,nullptr == m_table.nodes[bkt].obj ? nullptr : findInBucket(key, bkt)}; 301 | } 302 | /* (无锁)根据key向表中插入一个value，返回插入后所在节点指针 */ 303 | inline HashNode& _insert(const K &key, const V &value)noexcept { 304 | // 表大小超过重建阀值时则将表容量增加一倍重建哈希表 305 | if (m_table.size > m_threshold) { 306 | resize(m_table.capacity << 1); 307 | } 308 | //匿名对象使用右值引用版本的insertNode 309 | return _insertNode(HashNode{ m_isCopy?new V(value):std::addressof(const_cast(value)), nullptr }, false); 310 | } 311 | /* 删除bucket下的erased指定的节点 */ 312 | void _remove(HASH_TABLE_SIZE_TYPE bucket,HashNode& erased)noexcept { 313 | // 函数结束时释放内存 314 | raii var([=] { 315 | if (m_isCopy) { 316 | assert(nullptr != erased.obj); 317 | delete (static_cast(erased.obj)); 318 | } 319 | }); 320 | HashNode&bucketNode=m_table.nodes[bucket]; 321 | if (std::addressof(erased) == std::addressof(bucketNode)) { 322 | if (nullptr != bucketNode.next) { 323 | // 超过一个节点 将第二个节点数据复制到根节点，然后删除第二个节点 324 | auto next = bucketNode.next; 325 | bucketNode = *bucketNode.next; 326 | auto b = m_overTableManager.deleteNode(*next); 327 | assert(b); 328 | assert(nullptr == bucketNode.next||nullptr != bucketNode.next->obj); 329 | } else { 330 | // 只有一个节点 331 | bucketNode.obj = nullptr; 332 | } 333 | 334 | } else { 335 | HASH_NODE_PTR last; 336 | auto f = findInBucket(keyFrom(*static_cast(erased.obj)), bucket, &last);// 获取上一节点 337 | assert(nullptr != f&&nullptr != last); 338 | // 将上一节点与下一节点相连,然后删除当前节点 339 | last->next = erased.next; 340 | auto b=m_overTableManager.deleteNode(erased); 341 | assert(b); 342 | assert(nullptr == last->next || nullptr != last->next->obj); 343 | } 344 | 345 | decTableSize(); 346 | } 347 | 348 | /* (无锁)删除key指定的节点,如果key不存在则返回false */ 349 | bool _removeKey(const K &key) { 350 | auto pos=_find(key); 351 | if(nullptr!=pos.second){ 352 | _remove(pos.first,*pos.second); 353 | // 表大小小于重建阀值1/4时则将表容量减小一倍重建哈希表 354 | if (m_table.size <(m_threshold>>2)) { 355 | resize(m_table.capacity >> 1); 356 | } 357 | return true; 358 | } 359 | return false; 360 | } 361 | int _count_bucket(size_t bucket)const noexcept{ 362 | assert(bucketobj;p=p->next,++c); 365 | return c; 366 | } 367 | /* 生成哈希表的统计信息,如果内存分配失败则返回错误信息 */ 368 | string _stat_info()const { 369 | stringstream out; 370 | size_t b_empty=0,b_occupy=0,b_conflict=0; 371 | size_t c_sum=0; 372 | vectorv_stat; 373 | try{ 374 | v_stat=vector(m_table.capacity); 375 | }catch(const bad_alloc&e){ 376 | // 内存分配失败则返回 377 | cerr << e.what() << endl; 378 | return string("can't generate status information,caused by ")+string(e.what()); 379 | } 380 | for (auto i = m_table.capacity; i >0; --i) { 381 | auto c=_count_bucket(i-1); 382 | v_stat[i-1]=c; 383 | switch(c){ 384 | case 0: 385 | ++b_empty; 386 | break; 387 | case 1: 388 | ++b_occupy; 389 | break; 390 | default: 391 | ++b_conflict; 392 | c_sum+=c; 393 | } 394 | } 395 | double c_average= (0== b_conflict)? 0.0 : (double)c_sum/b_conflict; 396 | double c_sum_square=0; 397 | for (auto i = m_table.capacity; i >0; --i) { 398 | auto c=v_stat[i-1]; 399 | if(c>1) 400 | c_sum_square+=(c-c_average)*(c-c_average); 401 | } 402 | auto c_std_dev= (0 == b_conflict) ? 0.0 : sqrt(c_sum_square/b_conflict); 403 | out<<"hashtable:"<(); 420 | return _find(key).second; 421 | } 422 | /* 423 | * 根据key向表中插入一个value，返回插入后所在节点指针, 424 | */ 425 | HashNode& insert(const K &key, const V &value)noexcept{ 426 | auto guard = write_guard(); 427 | return _insert(key,value); 428 | } 429 | HashTableAbstract(HASH_TABLE_SIZE_TYPE initCapacity=0, bool isCopy=true, float loadFactor=0, size_t overBlockCapacity=0) : 430 | m_isCopy(isCopy), 431 | m_loadFactor(loadFactor > 0 && loadFactor < 1 ? loadFactor : DEFAULT_LOADFACTOR), 432 | m_overTableManager(overBlockCapacity){ 433 | initTable(m_table, initCapacity,mu_nodes); 434 | m_threshold = (HASH_TABLE_SIZE_TYPE)(this->m_loadFactor * m_table.capacity); 435 | } 436 | 437 | /* 判断k1,k2是否相等 */ 438 | /*bool equals(const K &k1, const K &k2)const { 439 | return 0 == default_equals(&k1, &k2, sizeof(K)); 440 | }*/ 441 | /* 判断k1,k2是否相等 442 | * 如果K支持==操作符则使用==比较版本，否则使用default_equals函数进行二进制比较 443 | */ 444 | template 445 | typename std::enable_if::value,bool>::type equals(const _K &k1, const _K &k2)const { 446 | return 0 == memcmp(&k1, &k2, sizeof(_K)); 447 | } 448 | template 449 | typename std::enable_if::value,bool>::type equals(const _K &k1, const _K &k2)const { 450 | return k1==k2; 451 | } 452 | 453 | /* 删除value指定的节点,如果value不存在则返回false */ 454 | bool removeValue(const V &value) { 455 | return removeKey(keyFrom(value)); 456 | } 457 | 458 | /* 删除key指定的节点,如果key不存在则返回false */ 459 | bool removeKey(const K &key) { 460 | auto guard=write_guard(); 461 | return _removeKey(key); 462 | } 463 | /* 从V中返回K,纯虚拟函数,子类必须实现 */ 464 | virtual const K& keyFrom(const V& value)const noexcept=0; 465 | 466 | private: 467 | template 468 | typename ::enable_if::type begin () noexcept 469 | { return iterator(this->m_table); } 470 | template 471 | typename ::enable_if::type end() noexcept 472 | { return iterator(this->m_table,position(this->m_table.capacity,nullptr)); } 473 | template 474 | typename ::enable_if::type begin ()const noexcept 475 | { return const_iterator(this->m_table); } 476 | template 477 | typename ::enable_if::type end()const noexcept 478 | { return const_iterator(this->m_table,position(this->m_table.capacity,nullptr)); } 479 | 480 | public: 481 | template 482 | typename ::enable_if::type begin () noexcept 483 | { return iterator(this->m_table); } 484 | template 485 | typename ::enable_if::type end() noexcept 486 | { return iterator(this->m_table,position(this->m_table.capacity,nullptr)); } 487 | template 488 | typename ::enable_if::type begin ()const noexcept 489 | { return const_iterator(this->m_table); } 490 | template 491 | typename ::enable_if::type end()const noexcept 492 | { return const_iterator(this->m_table,position(this->m_table.capacity,nullptr)); } 493 | 494 | public: 495 | bool has(const K & key)const noexcept{ 496 | return nullptr != findNode(key); 497 | } 498 | /* 返回表中元素个数 */ 499 | decltype(m_table.size) size()const noexcept{ 500 | return m_table.size; 501 | } 502 | bool empty()const noexcept{ 503 | return 0==size(); 504 | } 505 | decltype(m_table.capacity) capacity()const noexcept{ 506 | return m_table.capacity; 507 | } 508 | /* for_each 函数类型 */ 509 | using for_each_fun=std::function; 510 | using for_each_fun_const=std::function; 511 | /* 遍历所有节点执行指定的函数fun,当fun返回true时,中止遍历 */ 512 | inline void for_each_break_if(for_each_fun_const &&fun)const{ 513 | auto guard = read_guard(); 514 | for(auto itor=begin(),_end=end();itor!=_end;++itor){ 515 | if(fun(*itor)) 516 | break; 517 | } 518 | } 519 | /* 遍历所有节点执行指定的函数fun,当fun返回true时,中止遍历 */ 520 | inline void for_each_break_if(for_each_fun &&fun){ 521 | auto guard = write_guard(); 522 | for(auto itor=begin(),_end=end();itor!=_end;++itor){ 523 | if(fun(*itor)) 524 | break; 525 | } 526 | } 527 | /* 遍历表中所有节点, fun返回为true时删除该节点 */ 528 | inline HASH_TABLE_SIZE_TYPE erase_if(for_each_fun &&fun){ 529 | auto guard = write_guard(); 530 | HASH_TABLE_SIZE_TYPE count=0; 531 | for(auto itor=begin(),_end=end();itor!=_end;++itor){ 532 | if(fun(*itor)){ 533 | itor=erase(itor); 534 | ++count; 535 | } 536 | } 537 | return count; 538 | } 539 | /* 删除itor指定的节点，返回指向itor前一个的节点的迭代器(itorator) */ 540 | iterator erase(iterator &itor){ 541 | assert(itor.m_table == std::addressof(this->m_table)); 542 | _remove(itor.m_cur.first,*(itor.m_cur.second)); 543 | return iterator(m_table,itor.m_prev); 544 | } 545 | /* 删除itor指定的节点，返回指向itor前一个的节点的迭代器(const_iterator) */ 546 | const_iterator erase(const_iterator &itor){ 547 | return const_iterator(erase(static_cast(itor))); 548 | } 549 | /* 生成哈希表统计信息 */ 550 | string stat_info()const noexcept{ 551 | auto guard = read_guard(); 552 | return _stat_info(); 553 | } 554 | 555 | /* 允许移动构造 */ 556 | HashTableAbstract(HashTableAbstract&& )=default; 557 | virtual ~HashTableAbstract() { 558 | if (m_isCopy) { 559 | // 遍历每个节点,释放所有分配的空间 560 | for_each_break_if([](const V& v) { 561 | assert(nullptr != std::addressof(v)); 562 | delete (static_cast(std::addressof(v))); 563 | return false; 564 | }); 565 | } 566 | } 567 | }; 568 | 569 | } //end namespace gdface 570 | #endif /* FEATURE_SE_HASHTABLEABSTRACT_H_ */ 571 | --------------------------------------------------------------------------------