├── HardMargin-SVM ├── CMakeLists.txt ├── README.md ├── datasets │ ├── dataset.png │ └── toy │ │ ├── test │ │ ├── class1 │ │ │ ├── data1.txt │ │ │ ├── data2.txt │ │ │ ├── data3.txt │ │ │ ├── data4.txt │ │ │ ├── data5.txt │ │ │ └── data6.txt │ │ └── class2 │ │ │ ├── data1.txt │ │ │ ├── data2.txt │ │ │ ├── data3.txt │ │ │ ├── data4.txt │ │ │ ├── data5.txt │ │ │ └── data6.txt │ │ └── train │ │ ├── class1 │ │ ├── data1.txt │ │ ├── data2.txt │ │ ├── data3.txt │ │ ├── data4.txt │ │ └── data5.txt │ │ └── class2 │ │ ├── data1.txt │ │ ├── data2.txt │ │ ├── data3.txt │ │ ├── data4.txt │ │ └── data5.txt ├── scripts │ └── toy.sh └── src │ ├── main.cpp │ ├── svm.cpp │ └── svm.hpp ├── Kernel-SVM ├── CMakeLists.txt ├── README.md ├── datasets │ ├── dataset.png │ └── toy │ │ ├── test │ │ ├── class1 │ │ │ ├── data1.txt │ │ │ ├── data2.txt │ │ │ ├── data3.txt │ │ │ ├── data4.txt │ │ │ ├── data5.txt │ │ │ └── data6.txt │ │ └── class2 │ │ │ ├── data1.txt │ │ │ ├── data2.txt │ │ │ ├── data3.txt │ │ │ ├── data4.txt │ │ │ ├── data5.txt │ │ │ └── data6.txt │ │ └── train │ │ ├── class1 │ │ ├── data1.txt │ │ ├── data2.txt │ │ ├── data3.txt │ │ ├── data4.txt │ │ └── data5.txt │ │ └── class2 │ │ ├── data1.txt │ │ ├── data2.txt │ │ ├── data3.txt │ │ ├── data4.txt │ │ └── data5.txt ├── scripts │ └── toy.sh └── src │ ├── main.cpp │ ├── svm.cpp │ └── svm.hpp ├── LICENSE ├── OC-SVM ├── CMakeLists.txt ├── README.md ├── datasets │ ├── dataset.png │ └── toy │ │ ├── test │ │ ├── anomaly │ │ │ ├── data1.txt │ │ │ ├── data2.txt │ │ │ ├── data3.txt │ │ │ ├── data4.txt │ │ │ ├── data5.txt │ │ │ └── data6.txt │ │ └── normal │ │ │ ├── data1.txt │ │ │ ├── data2.txt │ │ │ ├── data3.txt │ │ │ ├── data4.txt │ │ │ ├── data5.txt │ │ │ └── data6.txt │ │ └── train │ │ ├── data1.txt │ │ ├── data2.txt │ │ ├── data3.txt │ │ ├── data4.txt │ │ └── data5.txt ├── scripts │ └── toy.sh └── src │ ├── main.cpp │ ├── svm.cpp │ └── svm.hpp ├── README.md ├── SVDD ├── CMakeLists.txt ├── README.md ├── datasets │ ├── dataset.png │ └── toy │ │ ├── test │ │ ├── anomaly │ │ │ ├── data1.txt │ │ │ ├── data2.txt │ │ │ ├── data3.txt │ │ │ ├── data4.txt │ │ │ ├── data5.txt │ │ │ └── data6.txt │ │ └── normal │ │ │ ├── data1.txt │ │ │ ├── data2.txt │ │ │ ├── data3.txt │ │ │ ├── data4.txt │ │ │ ├── data5.txt │ │ │ └── data6.txt │ │ └── train │ │ ├── data1.txt │ │ ├── data2.txt │ │ ├── data3.txt │ │ ├── data4.txt │ │ └── data5.txt ├── scripts │ └── toy.sh └── src │ ├── main.cpp │ ├── svdd.cpp │ └── svdd.hpp └── SoftMargin-SVM ├── CMakeLists.txt ├── README.md ├── datasets ├── dataset.png └── toy │ ├── test │ ├── class1 │ │ ├── data1.txt │ │ ├── data2.txt │ │ ├── data3.txt │ │ ├── data4.txt │ │ ├── data5.txt │ │ └── data6.txt │ └── class2 │ │ ├── data1.txt │ │ ├── data2.txt │ │ ├── data3.txt │ │ ├── data4.txt │ │ ├── data5.txt │ │ └── data6.txt │ └── train │ ├── class1 │ ├── data1.txt │ ├── data2.txt │ ├── data3.txt │ ├── data4.txt │ └── data5.txt │ └── class2 │ ├── data1.txt │ ├── data2.txt │ ├── data3.txt │ ├── data4.txt │ └── data5.txt ├── scripts └── toy.sh └── src ├── main.cpp ├── svm.cpp └── svm.hpp /HardMargin-SVM/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 2 | 3 | # Project Name 4 | project(HardMargin-SVM) 5 | 6 | # Directory Name 7 | set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) 8 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) 9 | 10 | # Set Compiler Options 11 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -Wall") 12 | if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8) 13 | set(CMAKE_CXX_COMPILER "g++-8") 14 | endif () 15 | 16 | # For OSX 17 | #set(CMAKE_OSX_ARCHITECTURES "x86_64") # Intel cpu 18 | set(CMAKE_OSX_ARCHITECTURES "arm64") # Apple cpu 19 | 20 | # Find Package 21 | find_package(Boost REQUIRED COMPONENTS program_options) 22 | 23 | # Set Include Directories 24 | set(INCLUDE_DIRS 25 | ${Boost_INCLUDE_DIRS} 26 | ${SRC_DIR} 27 | ) 28 | 29 | # Set CXX_LIBRARIES 30 | set(LIBRARIES 31 | ${Boost_LIBRARIES} 32 | ) 33 | 34 | # Set Source Code 35 | set(SRCS 36 | ${SRC_DIR}/main.cpp 37 | ${SRC_DIR}/svm.cpp 38 | ) 39 | 40 | # Link 41 | add_executable(${PROJECT_NAME} ${SRCS}) 42 | include_directories(${INCLUDE_DIRS}) 43 | target_link_libraries(${PROJECT_NAME} ${LIBRARIES}) 44 | set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 17) 45 | 46 | # Display Message 47 | message(STATUS "") 48 | message(STATUS "~~~ Boost Information ~~~") 49 | message(STATUS "${Boost_INCLUDE_DIRS};") 50 | message(STATUS "${Boost_LIBRARIES};") 51 | message(STATUS "") 52 | -------------------------------------------------------------------------------- /HardMargin-SVM/README.md: -------------------------------------------------------------------------------- 1 | # Hard Margin SVM 2 | 3 | This is the implementation of Hard Margin SVM (Hard Margin Support Vector Machine). 4 | 5 | - Class: 2 6 | - Problem: Linearly Separable 7 | - Decision Boundary: Hyperplane 8 | 9 | 10 | ## Usage 11 | 12 | ### 1. Build 13 | Please build the source file according to the procedure. 14 | ~~~ 15 | $ mkdir build 16 | $ cd build 17 | $ cmake .. 18 | $ make 19 | $ cd .. 20 | ~~~ 21 | 22 | ### 2. Dataset Setting 23 | 24 | The following hierarchical relationships are recommended. 25 | 26 | ![HardMargin-SVM_dataset](datasets/dataset.png) 27 | 28 | ### 3. Execution 29 | 30 | The following is an example for Toy Dataset. 31 | 32 | #### Setting 33 | Please set the shell for executable file. 34 | ~~~ 35 | $ vi scripts/toy.sh 36 | ~~~ 37 | If you want to view specific examples of command line arguments, please view "src/main.cpp" or add "--help" to the argument. 38 | ~~~ 39 | #!/bin/bash 40 | 41 | DATA='toy' 42 | 43 | ./HardMargin-SVM \ 44 | --dataset ${DATA} \ 45 | --nd 2 \ 46 | --lr 0.0001 47 | ~~~ 48 | 49 | #### Run 50 | Please execute the following to start the program. 51 | ~~~ 52 | $ sh scripts/toy.sh 53 | ~~~ 54 | 55 | ## Formula 56 | 57 | ![HardMargin-SVM_dual](https://user-images.githubusercontent.com/56967584/130267566-f6f7e656-2c39-4db2-8ba4-51fc3cf0354a.png) 58 | ![HardMargin-SVM_obj](https://user-images.githubusercontent.com/56967584/130267582-afff3278-3204-4d4b-aae1-1c3703822838.png) 59 | ![HardMargin-SVM_delta](https://user-images.githubusercontent.com/56967584/130267591-637f03d0-7ee1-4585-8ada-078b918e66e5.png) 60 | ![HardMargin-SVM_update](https://user-images.githubusercontent.com/56967584/130267594-8cff7a82-6645-4d4a-a4aa-1b87b966df2d.png) 61 | ![HardMargin-SVM_class](https://user-images.githubusercontent.com/56967584/130281675-ffc26b7c-bfd6-445b-8dd1-7cd7858b1843.png) 62 | 63 | 64 | ## Algorithm 65 | ![train](https://user-images.githubusercontent.com/56967584/130327291-57894837-dba8-4083-baaf-5854e0cd181c.png) 66 | ![test](https://user-images.githubusercontent.com/56967584/130327294-fa563cd0-5378-498f-817d-6dc13dfa38ab.png) 67 | 68 | 69 | -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/koba-jon/svm_cpp/5fa1b0951740b249a7b7bb170031b75470dbf090/HardMargin-SVM/datasets/dataset.png -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/test/class1/data1.txt: -------------------------------------------------------------------------------- 1 | 0.3 2 | 0.4 3 | -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/test/class1/data2.txt: -------------------------------------------------------------------------------- 1 | 0.2 2 | 0.49 3 | -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/test/class1/data3.txt: -------------------------------------------------------------------------------- 1 | 0.08 2 | 0.32 3 | -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/test/class1/data4.txt: -------------------------------------------------------------------------------- 1 | 0.49 2 | 0.49 3 | -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/test/class1/data5.txt: -------------------------------------------------------------------------------- 1 | 0.12 2 | 0.4 3 | -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/test/class1/data6.txt: -------------------------------------------------------------------------------- 1 | 0.4999999999999 2 | 0.4999999999999 -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/test/class2/data1.txt: -------------------------------------------------------------------------------- 1 | 0.9 2 | 0.8 3 | -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/test/class2/data2.txt: -------------------------------------------------------------------------------- 1 | 0.51 2 | 0.7 -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/test/class2/data3.txt: -------------------------------------------------------------------------------- 1 | 0.62 2 | 0.88 3 | -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/test/class2/data4.txt: -------------------------------------------------------------------------------- 1 | 0.7 2 | 0.55 -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/test/class2/data5.txt: -------------------------------------------------------------------------------- 1 | 0.51 2 | 0.51 3 | -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/test/class2/data6.txt: -------------------------------------------------------------------------------- 1 | 0.5000000000001 2 | 0.5000000000001 -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/train/class1/data1.txt: -------------------------------------------------------------------------------- 1 | 0.2 2 | 0.3 3 | -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/train/class1/data2.txt: -------------------------------------------------------------------------------- 1 | 0.1 2 | 0.4 3 | -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/train/class1/data3.txt: -------------------------------------------------------------------------------- 1 | 0.48 2 | 0.22 3 | -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/train/class1/data4.txt: -------------------------------------------------------------------------------- 1 | 0.4 2 | 0.05 3 | -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/train/class1/data5.txt: -------------------------------------------------------------------------------- 1 | 0.02 2 | 0.3 3 | -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/train/class2/data1.txt: -------------------------------------------------------------------------------- 1 | 0.8 2 | 0.7 3 | -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/train/class2/data2.txt: -------------------------------------------------------------------------------- 1 | 0.9 2 | 0.6 -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/train/class2/data3.txt: -------------------------------------------------------------------------------- 1 | 0.52 2 | 0.78 3 | -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/train/class2/data4.txt: -------------------------------------------------------------------------------- 1 | 0.6 2 | 0.95 -------------------------------------------------------------------------------- /HardMargin-SVM/datasets/toy/train/class2/data5.txt: -------------------------------------------------------------------------------- 1 | 0.98 2 | 0.7 3 | -------------------------------------------------------------------------------- /HardMargin-SVM/scripts/toy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DATA='toy' 4 | 5 | ./HardMargin-SVM \ 6 | --dataset ${DATA} \ 7 | --nd 2 \ 8 | --lr 0.0001 -------------------------------------------------------------------------------- /HardMargin-SVM/src/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | // 3rd-Party Libraries 9 | #include 10 | // Original 11 | #include "svm.hpp" 12 | 13 | // Define Namespace 14 | namespace fs = std::filesystem; 15 | namespace po = boost::program_options; 16 | 17 | // Function Prototype 18 | void Collect_Paths(const std::string root, const std::string sub, std::vector &paths); 19 | std::vector Get_Paths(const std::string root); 20 | std::vector> Get_Data(const std::vector paths, const size_t D); 21 | 22 | 23 | // ---------------------- 24 | // 0. Argument Function 25 | // ---------------------- 26 | po::options_description parse_arguments(){ 27 | 28 | po::options_description args("Options", 200, 30); 29 | 30 | args.add_options() 31 | 32 | // (1) Define for General Parameter 33 | ("help", "produce help message") 34 | ("dataset", po::value()->default_value("toy"), "dataset name") 35 | ("nd", po::value()->default_value(2), "number of dimensions") 36 | ("verbose", po::value()->default_value(true), "verbose") 37 | 38 | // (2) Define for Training 39 | ("train_class1_dir", po::value()->default_value("train/class1"), "training class 1 directory : ./datasets///") 40 | ("train_class2_dir", po::value()->default_value("train/class2"), "training class 2 directory : ./datasets///") 41 | ("lr", po::value()->default_value(0.0001), "learning rate of alpha") 42 | 43 | // (3) Define for Test 44 | ("test_class1_dir", po::value()->default_value("test/class1"), "test class 1 directory : ./datasets///") 45 | ("test_class2_dir", po::value()->default_value("test/class2"), "test class 2 directory : ./datasets///") 46 | 47 | ; 48 | 49 | return args; 50 | 51 | } 52 | 53 | 54 | // ------------------ 55 | // 1. Main Function 56 | // ------------------ 57 | int main(int argc, const char *argv[]){ 58 | 59 | // (1) Extract Arguments 60 | po::options_description args = parse_arguments(); 61 | po::variables_map vm{}; 62 | po::store(po::parse_command_line(argc, argv, args), vm); 63 | po::notify(vm); 64 | if (vm.empty() || vm.count("help")){ 65 | std::cout << args << std::endl; 66 | return 1; 67 | } 68 | 69 | // (2.1) Get Training Data for class 1 70 | std::string train_class1_dir; 71 | std::vector train_class1_paths; 72 | std::vector> train_class1_data; 73 | /*****************************************************/ 74 | train_class1_dir = "datasets/" + vm["dataset"].as() + "/" + vm["train_class1_dir"].as(); 75 | train_class1_paths = Get_Paths(train_class1_dir); 76 | train_class1_data = Get_Data(train_class1_paths, vm["nd"].as()); 77 | 78 | // (2.2) Get Training Data for class 2 79 | std::string train_class2_dir; 80 | std::vector train_class2_paths; 81 | std::vector> train_class2_data; 82 | /*****************************************************/ 83 | train_class2_dir = "datasets/" + vm["dataset"].as() + "/" + vm["train_class2_dir"].as(); 84 | train_class2_paths = Get_Paths(train_class2_dir); 85 | train_class2_data = Get_Data(train_class2_paths, vm["nd"].as()); 86 | 87 | // (2.3) Training for SVM 88 | HardMargin_SVM svm(vm["verbose"].as()); 89 | svm.train(train_class1_data, train_class2_data, vm["nd"].as(), vm["lr"].as()); 90 | 91 | // (3.1) Get Test Data for class 1 92 | std::string test_class1_dir; 93 | std::vector test_class1_paths; 94 | std::vector> test_class1_data; 95 | /*****************************************************/ 96 | test_class1_dir = "datasets/" + vm["dataset"].as() + "/" + vm["test_class1_dir"].as(); 97 | test_class1_paths = Get_Paths(test_class1_dir); 98 | test_class1_data = Get_Data(test_class1_paths, vm["nd"].as()); 99 | 100 | // (3.2) Get Test Data for class 2 101 | std::string test_class2_dir; 102 | std::vector test_class2_paths; 103 | std::vector> test_class2_data; 104 | /*****************************************************/ 105 | test_class2_dir = "datasets/" + vm["dataset"].as() + "/" + vm["test_class2_dir"].as(); 106 | test_class2_paths = Get_Paths(test_class2_dir); 107 | test_class2_data = Get_Data(test_class2_paths, vm["nd"].as()); 108 | 109 | // (3.3) Test for SVM 110 | svm.test(test_class1_data, test_class2_data); 111 | std::cout << "///////////////////////// Test /////////////////////////" << std::endl; 112 | std::cout << "accuracy-all: " << svm.accuracy << " (" << svm.correct_c1 + svm.correct_c2 << "/" << test_class1_data.size() + test_class2_data.size() << ")" << std::endl; 113 | std::cout << "accuracy-class1: " << svm.accuracy_c1 << " (" << svm.correct_c1 << "/" << test_class1_data.size() << ")" << std::endl; 114 | std::cout << "accuracy-class2: " << svm.accuracy_c2 << " (" << svm.correct_c2 << "/" << test_class2_data.size() << ")" << std::endl; 115 | std::cout << "////////////////////////////////////////////////////////" << std::endl; 116 | 117 | return 0; 118 | } 119 | 120 | 121 | // ------------------------------ 122 | // 2. Collecting Paths Function 123 | // ------------------------------ 124 | void Collect_Paths(const std::string root, const std::string sub, std::vector &paths){ 125 | 126 | fs::path ROOT(root); 127 | 128 | for (auto &p : fs::directory_iterator(ROOT)){ 129 | if (!fs::is_directory(p)){ 130 | std::stringstream rpath; 131 | rpath << p.path().string(); 132 | paths.push_back(rpath.str()); 133 | } 134 | else{ 135 | std::stringstream subsub; 136 | subsub << p.path().filename().string(); 137 | Collect_Paths(root + '/' + subsub.str(), sub + subsub.str() + '/', paths); 138 | } 139 | } 140 | 141 | return; 142 | } 143 | 144 | 145 | // --------------------------- 146 | // 3. Getting Paths Function 147 | // --------------------------- 148 | std::vector Get_Paths(const std::string root){ 149 | 150 | std::vector paths; 151 | 152 | Collect_Paths(root, "", paths); 153 | std::sort(paths.begin(), paths.end()); 154 | 155 | return paths; 156 | 157 | } 158 | 159 | 160 | // --------------------------- 161 | // 4. Getting Data Function 162 | // --------------------------- 163 | std::vector> Get_Data(const std::vector paths, const size_t D){ 164 | 165 | size_t i; 166 | double element; 167 | std::ifstream ifs; 168 | std::vector data_one; 169 | std::vector> data; 170 | 171 | for (std::string path : paths){ 172 | 173 | ifs.open(path); 174 | 175 | data_one = std::vector(D); 176 | for (i = 0; i < D; i++){ 177 | ifs >> element; 178 | data_one[i] = element; 179 | } 180 | data.push_back(data_one); 181 | 182 | ifs.close(); 183 | 184 | } 185 | 186 | return data; 187 | } -------------------------------------------------------------------------------- /HardMargin-SVM/src/svm.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | // Original 6 | #include "svm.hpp" 7 | 8 | 9 | // -------------------------------------- 10 | // class{HardMargin_SVM} -> constructor 11 | // -------------------------------------- 12 | HardMargin_SVM::HardMargin_SVM(const bool verbose_){ 13 | this->verbose = verbose_; 14 | } 15 | 16 | 17 | // ---------------------------------------- 18 | // class{HardMargin_SVM} -> function{dot} 19 | // ---------------------------------------- 20 | double HardMargin_SVM::dot(const std::vector x1, const std::vector x2){ 21 | 22 | size_t i; 23 | double ans; 24 | 25 | if (x1.size() != x2.size()){ 26 | std::cerr << "Error : Couldn't match the number of elements for inner product." << std::endl; 27 | std::exit(-1); 28 | } 29 | 30 | ans = 0.0; 31 | for (i = 0; i < x1.size(); i++){ 32 | ans += x1[i] * x2[i]; 33 | } 34 | 35 | return ans; 36 | 37 | } 38 | 39 | 40 | // ---------------------------------------- 41 | // class{HardMargin_SVM} -> function{log} 42 | // ---------------------------------------- 43 | void HardMargin_SVM::log(const std::string str){ 44 | if (this->verbose){ 45 | std::cout << str << std::flush; 46 | } 47 | return; 48 | } 49 | 50 | 51 | // ------------------------------------------ 52 | // class{HardMargin_SVM} -> function{train} 53 | // ------------------------------------------ 54 | void HardMargin_SVM::train(const std::vector> class1_data, const std::vector> class2_data, const size_t D, const double lr, const double limit){ 55 | 56 | constexpr double eps = 0.0000001; 57 | 58 | size_t i, j; 59 | size_t N, Ns; 60 | bool judge; 61 | double item1, item2, item3; 62 | double delta; 63 | double beta; 64 | double error; 65 | std::vector> x; 66 | std::vector y; 67 | std::vector alpha; 68 | 69 | // (1.1) Set class 1 data 70 | for (i = 0; i < class1_data.size(); i++){ 71 | x.push_back(class1_data[i]); 72 | y.push_back(1); 73 | } 74 | 75 | // (1.2) Set class 2 data 76 | for (i = 0; i < class2_data.size(); i++){ 77 | x.push_back(class2_data[i]); 78 | y.push_back(-1); 79 | } 80 | 81 | // (2) Set Lagrange Multiplier and Parameters 82 | N = x.size(); 83 | alpha = std::vector(N, 0.0); 84 | beta = 1.0; 85 | 86 | // (3) Training 87 | this->log("\n"); 88 | this->log("/////////////////////// Training ///////////////////////\n"); 89 | do { 90 | 91 | judge = false; 92 | error = 0.0; 93 | 94 | // (3.1) Update Alpha 95 | for (i = 0; i < N; i++){ 96 | 97 | // Set item 1 98 | item1 = 0.0; 99 | for (j = 0; j < N; j++){ 100 | item1 += alpha[j] * (double)y[i] * (double)y[j] * this->dot(x[i], x[j]); 101 | } 102 | 103 | // Set item 2 104 | item2 = 0.0; 105 | for (j = 0; j < N; j++){ 106 | item2 += alpha[j] * (double)y[i] * (double)y[j]; 107 | } 108 | 109 | // Set Delta 110 | delta = 1.0 - item1 - beta * item2; 111 | 112 | // Update 113 | alpha[i] += lr * delta; 114 | if (alpha[i] < 0.0){ 115 | alpha[i] = 0.0; 116 | } 117 | else if (std::abs(delta) > limit){ 118 | judge = true; 119 | error += std::abs(delta) - limit; 120 | } 121 | 122 | } 123 | 124 | // (3.2) Update Beta 125 | item3 = 0.0; 126 | for (i = 0; i < N; i++){ 127 | item3 += alpha[i] * (double)y[i]; 128 | } 129 | beta += item3 * item3 / 2.0; 130 | 131 | // (3.3) Output Residual Error 132 | this->log("\rerror: " + std::to_string(error)); 133 | 134 | }while (judge); 135 | this->log("\n"); 136 | this->log("////////////////////////////////////////////////////////\n"); 137 | 138 | // (4.1) Description for support vectors 139 | Ns = 0; 140 | this->xs = std::vector>(); 141 | this->ys = std::vector(); 142 | this->alpha_s = std::vector(); 143 | for (i = 0; i < N; i++){ 144 | if (alpha[i] > eps){ 145 | this->xs.push_back(x[i]); 146 | this->ys.push_back(y[i]); 147 | this->alpha_s.push_back(alpha[i]); 148 | Ns++; 149 | } 150 | } 151 | this->log("Ns (number of support vectors) = " + std::to_string(Ns) + "\n"); 152 | 153 | // (4.2) Description for w 154 | this->log("weight = [ "); 155 | this->w = std::vector(D, 0.0); 156 | for (j = 0; j < D; j++){ 157 | for (i = 0; i < Ns; i++){ 158 | this->w[j] += alpha_s[i] * (double)ys[i] * xs[i][j]; 159 | } 160 | this->log(std::to_string(this->w[j]) + " "); 161 | } 162 | this->log("]\n"); 163 | 164 | // (4.3) Description for b 165 | this->b = 0.0; 166 | for (i = 0; i < Ns; i++){ 167 | this->b += (double)this->ys[i] - this->dot(this->w, this->xs[i]); 168 | } 169 | this->b /= (double)Ns; 170 | this->log("bias = " + std::to_string(this->b) + "\n"); 171 | this->log("////////////////////////////////////////////////////////\n\n"); 172 | 173 | return; 174 | } 175 | 176 | 177 | // ----------------------------------------- 178 | // class{HardMargin_SVM} -> function{test} 179 | // ----------------------------------------- 180 | void HardMargin_SVM::test(const std::vector> class1_data, const std::vector> class2_data){ 181 | 182 | size_t i; 183 | 184 | this->correct_c1 = 0; 185 | for (i = 0; i < class1_data.size(); i++){ 186 | if (this->g(class1_data[i]) == 1){ 187 | this->correct_c1++; 188 | } 189 | } 190 | 191 | this->correct_c2 = 0; 192 | for (i = 0; i < class2_data.size(); i++){ 193 | if (this->g(class2_data[i]) == -1){ 194 | this->correct_c2++; 195 | } 196 | } 197 | 198 | this->accuracy = (double)(this->correct_c1 + this->correct_c2) / (double)(class1_data.size() + class2_data.size()); 199 | this->accuracy_c1 = (double)this->correct_c1 / (double)class1_data.size(); 200 | this->accuracy_c2 = (double)this->correct_c2 / (double)class2_data.size(); 201 | 202 | return; 203 | } 204 | 205 | 206 | // -------------------------------------- 207 | // class{HardMargin_SVM} -> function{f} 208 | // -------------------------------------- 209 | double HardMargin_SVM::f(const std::vector x){ 210 | return this->dot(this->w, x) + this->b; 211 | } 212 | 213 | 214 | // -------------------------------------- 215 | // class{HardMargin_SVM} -> function{g} 216 | // -------------------------------------- 217 | double HardMargin_SVM::g(const std::vector x){ 218 | 219 | double fx; 220 | int gx; 221 | 222 | fx = this->f(x); 223 | if (fx >= 0.0){ 224 | gx = 1; 225 | } 226 | else{ 227 | gx = -1; 228 | } 229 | 230 | return gx; 231 | } -------------------------------------------------------------------------------- /HardMargin-SVM/src/svm.hpp: -------------------------------------------------------------------------------- 1 | #ifndef SVM_HPP 2 | #define SVM_HPP 3 | 4 | #include 5 | #include 6 | 7 | 8 | // ---------------------- 9 | // class{HardMargin_SVM} 10 | // ---------------------- 11 | class HardMargin_SVM{ 12 | private: 13 | 14 | // member variable 15 | bool verbose; 16 | std::vector w; 17 | double b; 18 | std::vector> xs; 19 | std::vector ys; 20 | std::vector alpha_s; 21 | 22 | // fuction 23 | double dot(const std::vector x1, const std::vector x2); 24 | void log(const std::string str); 25 | 26 | public: 27 | 28 | // member variable 29 | double accuracy; 30 | double accuracy_c1, accuracy_c2; 31 | size_t correct_c1, correct_c2; 32 | 33 | // constructor 34 | HardMargin_SVM() = delete; 35 | HardMargin_SVM(const bool verbose_=true); 36 | 37 | // function 38 | void train(const std::vector> class1_data, const std::vector> class2_data, const size_t D, const double lr, const double limit=0.0001); 39 | void test(const std::vector> class1_data, const std::vector> class2_data); 40 | double f(const std::vector x); 41 | double g(const std::vector x); 42 | 43 | }; 44 | 45 | 46 | #endif -------------------------------------------------------------------------------- /Kernel-SVM/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 2 | 3 | # Project Name 4 | project(Kernel-SVM) 5 | 6 | # Directory Name 7 | set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) 8 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) 9 | 10 | # Set Compiler Options 11 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -Wall") 12 | if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8) 13 | set(CMAKE_CXX_COMPILER "g++-8") 14 | endif () 15 | 16 | # For OSX 17 | #set(CMAKE_OSX_ARCHITECTURES "x86_64") # Intel cpu 18 | set(CMAKE_OSX_ARCHITECTURES "arm64") # Apple cpu 19 | 20 | # Find Package 21 | find_package(Boost REQUIRED COMPONENTS program_options) 22 | 23 | # Set Include Directories 24 | set(INCLUDE_DIRS 25 | ${Boost_INCLUDE_DIRS} 26 | ${SRC_DIR} 27 | ) 28 | 29 | # Set CXX_LIBRARIES 30 | set(LIBRARIES 31 | ${Boost_LIBRARIES} 32 | ) 33 | 34 | # Set Source Code 35 | set(SRCS 36 | ${SRC_DIR}/main.cpp 37 | ${SRC_DIR}/svm.cpp 38 | ) 39 | 40 | # Link 41 | add_executable(${PROJECT_NAME} ${SRCS}) 42 | include_directories(${INCLUDE_DIRS}) 43 | target_link_libraries(${PROJECT_NAME} ${LIBRARIES}) 44 | set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 17) 45 | 46 | # Display Message 47 | message(STATUS "") 48 | message(STATUS "~~~ Boost Information ~~~") 49 | message(STATUS "${Boost_INCLUDE_DIRS};") 50 | message(STATUS "${Boost_LIBRARIES};") 51 | message(STATUS "") 52 | -------------------------------------------------------------------------------- /Kernel-SVM/README.md: -------------------------------------------------------------------------------- 1 | # Kernel SVM 2 | 3 | This is the implementation of Kernel SVM (Non-linear Support Vector Machine). 4 | 5 | - Class: 2 6 | - Problem: Linearly Non-separable 7 | - Decision Boundary: Hyperplane + Kernel 8 | 9 | ## Usage 10 | 11 | ### 1. Build 12 | Please build the source file according to the procedure. 13 | ~~~ 14 | $ mkdir build 15 | $ cd build 16 | $ cmake .. 17 | $ make 18 | $ cd .. 19 | ~~~ 20 | 21 | ### 2. Dataset Setting 22 | 23 | The following hierarchical relationships are recommended. 24 | 25 | ![Kernel-SVM_dataset](datasets/dataset.png) 26 | 27 | ### 3. Execution 28 | 29 | The following is an example for Toy Dataset. 30 | 31 | #### Setting 32 | Please set the shell for executable file. 33 | ~~~ 34 | $ vi scripts/toy.sh 35 | ~~~ 36 | If you want to view specific examples of command line arguments, please view "src/main.cpp" or add "--help" to the argument. 37 | ~~~ 38 | #!/bin/bash 39 | 40 | DATA='toy' 41 | 42 | ./Kernel-SVM \ 43 | --dataset ${DATA} \ 44 | --nd 2 \ 45 | --C 10.0 \ 46 | --lr 0.0001 \ 47 | --kernel "rbf" \ 48 | --gamma 5.0 49 | ~~~ 50 | 51 | #### Run 52 | Please execute the following to start the program. 53 | ~~~ 54 | $ sh scripts/toy.sh 55 | ~~~ 56 | 57 | 58 | ## Formula 59 | 60 | ![Kernel-SVM_dual](https://user-images.githubusercontent.com/56967584/130267966-98d98cb9-bfb0-4d85-aba6-606ba7b04568.png) 61 | ![Kernel-SVM_obj](https://user-images.githubusercontent.com/56967584/130268001-7c4fd249-c8a3-4b8c-bfbc-bdb9a1b9ec78.png) 62 | ![Kernel-SVM_delta](https://user-images.githubusercontent.com/56967584/130268014-4a2ac9fa-9491-407c-a925-5eecbb665039.png) 63 | ![Kernel-SVM_update](https://user-images.githubusercontent.com/56967584/130268022-7659c8ec-795d-465c-8e6d-b8ec923e7cfa.png) 64 | ![Kernel-SVM_class](https://user-images.githubusercontent.com/56967584/130281897-452729b3-c200-4fa7-8f57-20a2a1b6f296.png) 65 | 66 | 67 | ## Algorithm 68 | ![train](https://user-images.githubusercontent.com/56967584/130327320-5855fdfc-eb53-4407-a8a3-ad885805786e.png) 69 | ![test](https://user-images.githubusercontent.com/56967584/130327324-6c89897b-123f-4b0a-9360-375679932033.png) 70 | -------------------------------------------------------------------------------- /Kernel-SVM/datasets/dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/koba-jon/svm_cpp/5fa1b0951740b249a7b7bb170031b75470dbf090/Kernel-SVM/datasets/dataset.png -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/test/class1/data1.txt: -------------------------------------------------------------------------------- 1 | 0.5 2 | 0.5 3 | -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/test/class1/data2.txt: -------------------------------------------------------------------------------- 1 | 0.6 2 | 0.6 3 | -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/test/class1/data3.txt: -------------------------------------------------------------------------------- 1 | 0.4 2 | 0.4 3 | -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/test/class1/data4.txt: -------------------------------------------------------------------------------- 1 | 0.5 2 | 0.6 3 | -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/test/class1/data5.txt: -------------------------------------------------------------------------------- 1 | 0.5 2 | 0.45 3 | -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/test/class1/data6.txt: -------------------------------------------------------------------------------- 1 | 0.59 2 | 0.41 -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/test/class2/data1.txt: -------------------------------------------------------------------------------- 1 | 0.8 2 | 0.2 3 | -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/test/class2/data2.txt: -------------------------------------------------------------------------------- 1 | 0.8 2 | 0.25 -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/test/class2/data3.txt: -------------------------------------------------------------------------------- 1 | 0.19 2 | 0.19 3 | -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/test/class2/data4.txt: -------------------------------------------------------------------------------- 1 | 0.17 2 | 0.17 -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/test/class2/data5.txt: -------------------------------------------------------------------------------- 1 | 0.95 2 | 0.81 3 | -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/test/class2/data6.txt: -------------------------------------------------------------------------------- 1 | 0.19 2 | 0.81 -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/train/class1/data1.txt: -------------------------------------------------------------------------------- 1 | 0.5 2 | 0.5 3 | -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/train/class1/data2.txt: -------------------------------------------------------------------------------- 1 | 0.4 2 | 0.6 3 | -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/train/class1/data3.txt: -------------------------------------------------------------------------------- 1 | 0.6 2 | 0.65 3 | -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/train/class1/data4.txt: -------------------------------------------------------------------------------- 1 | 0.65 2 | 0.35 3 | -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/train/class1/data5.txt: -------------------------------------------------------------------------------- 1 | 0.4 2 | 0.6 3 | -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/train/class2/data1.txt: -------------------------------------------------------------------------------- 1 | 0.7 2 | 0.3 3 | -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/train/class2/data2.txt: -------------------------------------------------------------------------------- 1 | 0.7 2 | 0.8 -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/train/class2/data3.txt: -------------------------------------------------------------------------------- 1 | 0.9 2 | 0.1 3 | -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/train/class2/data4.txt: -------------------------------------------------------------------------------- 1 | 0.8 2 | 0.95 -------------------------------------------------------------------------------- /Kernel-SVM/datasets/toy/train/class2/data5.txt: -------------------------------------------------------------------------------- 1 | 0.2 2 | 0.3 3 | -------------------------------------------------------------------------------- /Kernel-SVM/scripts/toy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DATA='toy' 4 | 5 | ./Kernel-SVM \ 6 | --dataset ${DATA} \ 7 | --nd 2 \ 8 | --C 10.0 \ 9 | --lr 0.0001 \ 10 | --kernel "rbf" \ 11 | --gamma 5.0 -------------------------------------------------------------------------------- /Kernel-SVM/src/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | // 3rd-Party Libraries 9 | #include 10 | // Original 11 | #include "svm.hpp" 12 | 13 | // Define Namespace 14 | namespace fs = std::filesystem; 15 | namespace po = boost::program_options; 16 | 17 | // Function Prototype 18 | void Collect_Paths(const std::string root, const std::string sub, std::vector &paths); 19 | std::vector Get_Paths(const std::string root); 20 | std::vector> Get_Data(const std::vector paths, const size_t D); 21 | void Set_Kernel(po::variables_map &vm, KernelFunc &K, std::vector ¶ms); 22 | 23 | 24 | // ---------------------- 25 | // 0. Argument Function 26 | // ---------------------- 27 | po::options_description parse_arguments(){ 28 | 29 | po::options_description args("Options", 200, 30); 30 | 31 | args.add_options() 32 | 33 | // (1) Define for General Parameter 34 | ("help", "produce help message") 35 | ("dataset", po::value()->default_value("toy"), "dataset name") 36 | ("nd", po::value()->default_value(2), "number of dimensions") 37 | ("verbose", po::value()->default_value(true), "verbose") 38 | 39 | // (2) Define for Training 40 | ("train_class1_dir", po::value()->default_value("train/class1"), "training class 1 directory : ./datasets///") 41 | ("train_class2_dir", po::value()->default_value("train/class2"), "training class 2 directory : ./datasets///") 42 | ("C", po::value()->default_value(1.0), "regularization parameter (C > 0.0)") 43 | ("lr", po::value()->default_value(0.0001), "learning rate of alpha") 44 | 45 | // (3) Define for Test 46 | ("test_class1_dir", po::value()->default_value("test/class1"), "test class 1 directory : ./datasets///") 47 | ("test_class2_dir", po::value()->default_value("test/class2"), "test class 2 directory : ./datasets///") 48 | 49 | // (4) Define for Kernel 50 | ("kernel", po::value()->default_value("rbf"), "kernel : linear / polynomial / rbf") 51 | ("c", po::value()->default_value(1.0), "addition parameter for polynomial kernel (c > 0.0)") 52 | ("d", po::value()->default_value(2.0), "exponent parameter for polynomial kernel (natural number)") 53 | ("gamma", po::value()->default_value(1.0), "precision (inverse of variance) parameter for rbf kernel (gamma > 0.0)") 54 | 55 | ; 56 | 57 | return args; 58 | 59 | } 60 | 61 | 62 | // ------------------ 63 | // 1. Main Function 64 | // ------------------ 65 | int main(int argc, const char *argv[]){ 66 | 67 | // (1) Extract Arguments 68 | po::options_description args = parse_arguments(); 69 | po::variables_map vm{}; 70 | po::store(po::parse_command_line(argc, argv, args), vm); 71 | po::notify(vm); 72 | if (vm.empty() || vm.count("help")){ 73 | std::cout << args << std::endl; 74 | return 1; 75 | } 76 | 77 | // (2) Set Kernel 78 | KernelFunc K; 79 | std::vector params; 80 | Set_Kernel(vm, K, params); 81 | 82 | // (3.1) Get Training Data for class 1 83 | std::string train_class1_dir; 84 | std::vector train_class1_paths; 85 | std::vector> train_class1_data; 86 | /*****************************************************/ 87 | train_class1_dir = "datasets/" + vm["dataset"].as() + "/" + vm["train_class1_dir"].as(); 88 | train_class1_paths = Get_Paths(train_class1_dir); 89 | train_class1_data = Get_Data(train_class1_paths, vm["nd"].as()); 90 | 91 | // (3.2) Get Training Data for class 2 92 | std::string train_class2_dir; 93 | std::vector train_class2_paths; 94 | std::vector> train_class2_data; 95 | /*****************************************************/ 96 | train_class2_dir = "datasets/" + vm["dataset"].as() + "/" + vm["train_class2_dir"].as(); 97 | train_class2_paths = Get_Paths(train_class2_dir); 98 | train_class2_data = Get_Data(train_class2_paths, vm["nd"].as()); 99 | 100 | // (3.3) Training for SVM 101 | Kernel_SVM svm(K, params, vm["verbose"].as()); 102 | svm.train(train_class1_data, train_class2_data, vm["nd"].as(), vm["C"].as(), vm["lr"].as()); 103 | 104 | // (4.1) Get Test Data for class 1 105 | std::string test_class1_dir; 106 | std::vector test_class1_paths; 107 | std::vector> test_class1_data; 108 | /*****************************************************/ 109 | test_class1_dir = "datasets/" + vm["dataset"].as() + "/" + vm["test_class1_dir"].as(); 110 | test_class1_paths = Get_Paths(test_class1_dir); 111 | test_class1_data = Get_Data(test_class1_paths, vm["nd"].as()); 112 | 113 | // (4.2) Get Test Data for class 2 114 | std::string test_class2_dir; 115 | std::vector test_class2_paths; 116 | std::vector> test_class2_data; 117 | /*****************************************************/ 118 | test_class2_dir = "datasets/" + vm["dataset"].as() + "/" + vm["test_class2_dir"].as(); 119 | test_class2_paths = Get_Paths(test_class2_dir); 120 | test_class2_data = Get_Data(test_class2_paths, vm["nd"].as()); 121 | 122 | // (4.3) Test for SVM 123 | svm.test(test_class1_data, test_class2_data); 124 | std::cout << "///////////////////////// Test /////////////////////////" << std::endl; 125 | std::cout << "accuracy-all: " << svm.accuracy << " (" << svm.correct_c1 + svm.correct_c2 << "/" << test_class1_data.size() + test_class2_data.size() << ")" << std::endl; 126 | std::cout << "accuracy-class1: " << svm.accuracy_c1 << " (" << svm.correct_c1 << "/" << test_class1_data.size() << ")" << std::endl; 127 | std::cout << "accuracy-class2: " << svm.accuracy_c2 << " (" << svm.correct_c2 << "/" << test_class2_data.size() << ")" << std::endl; 128 | std::cout << "////////////////////////////////////////////////////////" << std::endl; 129 | 130 | return 0; 131 | } 132 | 133 | 134 | // ------------------------------ 135 | // 2. Collecting Paths Function 136 | // ------------------------------ 137 | void Collect_Paths(const std::string root, const std::string sub, std::vector &paths){ 138 | 139 | fs::path ROOT(root); 140 | 141 | for (auto &p : fs::directory_iterator(ROOT)){ 142 | if (!fs::is_directory(p)){ 143 | std::stringstream rpath; 144 | rpath << p.path().string(); 145 | paths.push_back(rpath.str()); 146 | } 147 | else{ 148 | std::stringstream subsub; 149 | subsub << p.path().filename().string(); 150 | Collect_Paths(root + '/' + subsub.str(), sub + subsub.str() + '/', paths); 151 | } 152 | } 153 | 154 | return; 155 | } 156 | 157 | 158 | // --------------------------- 159 | // 3. Getting Paths Function 160 | // --------------------------- 161 | std::vector Get_Paths(const std::string root){ 162 | 163 | std::vector paths; 164 | 165 | Collect_Paths(root, "", paths); 166 | std::sort(paths.begin(), paths.end()); 167 | 168 | return paths; 169 | 170 | } 171 | 172 | 173 | // --------------------------- 174 | // 4. Getting Data Function 175 | // --------------------------- 176 | std::vector> Get_Data(const std::vector paths, const size_t D){ 177 | 178 | size_t i; 179 | double element; 180 | std::ifstream ifs; 181 | std::vector data_one; 182 | std::vector> data; 183 | 184 | for (std::string path : paths){ 185 | 186 | ifs.open(path); 187 | 188 | data_one = std::vector(D); 189 | for (i = 0; i < D; i++){ 190 | ifs >> element; 191 | data_one[i] = element; 192 | } 193 | data.push_back(data_one); 194 | 195 | ifs.close(); 196 | 197 | } 198 | 199 | return data; 200 | } 201 | 202 | 203 | // ---------------------------- 204 | // 5. Setting Kernel Function 205 | // ---------------------------- 206 | void Set_Kernel(po::variables_map &vm, KernelFunc &K, std::vector ¶ms){ 207 | 208 | if (vm["kernel"].as() == "linear"){ 209 | K = kernel::linear; 210 | } 211 | else if (vm["kernel"].as() == "polynomial"){ 212 | K = kernel::polynomial; 213 | params = {vm["c"].as(), vm["d"].as()}; 214 | } 215 | else if (vm["kernel"].as() == "rbf"){ 216 | K = kernel::rbf; 217 | params = {vm["gamma"].as()}; 218 | } 219 | else{ 220 | std::cerr << "Error : Couldn't match the name of kernel." << std::endl; 221 | std::exit(-1); 222 | } 223 | 224 | return; 225 | 226 | } -------------------------------------------------------------------------------- /Kernel-SVM/src/svm.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | // Original 6 | #include "svm.hpp" 7 | 8 | 9 | // --------------------------------------- 10 | // namespace{kernel} -> function{linear} 11 | // --------------------------------------- 12 | double kernel::linear(const std::vector x1, const std::vector x2, const std::vector params){ 13 | 14 | size_t i; 15 | double ans; 16 | 17 | if (x1.size() != x2.size()){ 18 | std::cerr << "Error : Couldn't match the number of elements for inner product." << std::endl; 19 | std::exit(-1); 20 | } 21 | 22 | ans = 0.0; 23 | for (i = 0; i < x1.size(); i++){ 24 | ans += x1[i] * x2[i]; 25 | } 26 | 27 | return ans; 28 | 29 | } 30 | 31 | 32 | // ------------------------------------------- 33 | // namespace{kernel} -> function{polynomial} 34 | // ------------------------------------------- 35 | double kernel::polynomial(const std::vector x1, const std::vector x2, const std::vector params){ 36 | 37 | size_t i; 38 | double ans; 39 | 40 | if (x1.size() != x2.size()){ 41 | std::cerr << "Error : Couldn't match the number of elements for inner product." << std::endl; 42 | std::exit(-1); 43 | } 44 | else if (params.size() != 2){ 45 | std::cerr << "Error : Couldn't match the number of hyper-parameters." << std::endl; 46 | std::exit(-1); 47 | } 48 | 49 | ans = 0.0; 50 | for (i = 0; i < x1.size(); i++){ 51 | ans += x1[i] * x2[i]; 52 | } 53 | ans += params[0]; 54 | ans = std::pow(ans, params[1]); 55 | 56 | return ans; 57 | 58 | } 59 | 60 | 61 | // ------------------------------------ 62 | // namespace{kernel} -> function{rbf} 63 | // ------------------------------------ 64 | double kernel::rbf(const std::vector x1, const std::vector x2, const std::vector params){ 65 | 66 | size_t i; 67 | double ans; 68 | 69 | if (x1.size() != x2.size()){ 70 | std::cerr << "Error : Couldn't match the number of elements for inner product." << std::endl; 71 | std::exit(-1); 72 | } 73 | else if (params.size() != 1){ 74 | std::cerr << "Error : Couldn't match the number of hyper-parameters." << std::endl; 75 | std::exit(-1); 76 | } 77 | 78 | ans = 0.0; 79 | for (i = 0; i < x1.size(); i++){ 80 | ans += (x1[i] - x2[i]) * (x1[i] - x2[i]); 81 | } 82 | ans = std::exp(-params[0] * ans); 83 | 84 | return ans; 85 | 86 | } 87 | 88 | 89 | // ---------------------------------- 90 | // class{Kernel_SVM} -> constructor 91 | // ---------------------------------- 92 | Kernel_SVM::Kernel_SVM(const KernelFunc K_, const std::vector params_, const bool verbose_){ 93 | this->K = K_; 94 | this->params = params_; 95 | this->verbose = verbose_; 96 | } 97 | 98 | 99 | // ------------------------------------ 100 | // class{Kernel_SVM} -> function{log} 101 | // ------------------------------------ 102 | void Kernel_SVM::log(const std::string str){ 103 | if (this->verbose){ 104 | std::cout << str << std::flush; 105 | } 106 | return; 107 | } 108 | 109 | 110 | // -------------------------------------- 111 | // class{Kernel_SVM} -> function{train} 112 | // -------------------------------------- 113 | void Kernel_SVM::train(const std::vector> class1_data, const std::vector> class2_data, const size_t D, const double C, const double lr, const double limit){ 114 | 115 | constexpr double eps = 0.0000001; 116 | 117 | size_t i, j; 118 | size_t N, Ns, Ns_in; 119 | bool judge; 120 | double item1, item2, item3; 121 | double delta; 122 | double beta; 123 | double error; 124 | std::vector> x; 125 | std::vector y; 126 | std::vector alpha; 127 | 128 | // (1.1) Set class 1 data 129 | for (i = 0; i < class1_data.size(); i++){ 130 | x.push_back(class1_data[i]); 131 | y.push_back(1); 132 | } 133 | 134 | // (1.2) Set class 2 data 135 | for (i = 0; i < class2_data.size(); i++){ 136 | x.push_back(class2_data[i]); 137 | y.push_back(-1); 138 | } 139 | 140 | // (2) Set Lagrange Multiplier and Parameters 141 | N = x.size(); 142 | alpha = std::vector(N, 0.0); 143 | beta = 1.0; 144 | 145 | // (3) Training 146 | this->log("\n"); 147 | this->log("/////////////////////// Training ///////////////////////\n"); 148 | do { 149 | 150 | judge = false; 151 | error = 0.0; 152 | 153 | // (3.1) Update Alpha 154 | for (i = 0; i < N; i++){ 155 | 156 | // Set item 1 157 | item1 = 0.0; 158 | for (j = 0; j < N; j++){ 159 | item1 += alpha[j] * (double)y[i] * (double)y[j] * this->K(x[i], x[j], this->params); 160 | } 161 | 162 | // Set item 2 163 | item2 = 0.0; 164 | for (j = 0; j < N; j++){ 165 | item2 += alpha[j] * (double)y[i] * (double)y[j]; 166 | } 167 | 168 | // Set Delta 169 | delta = 1.0 - item1 - beta * item2; 170 | 171 | // Update 172 | alpha[i] += lr * delta; 173 | if (alpha[i] < 0.0){ 174 | alpha[i] = 0.0; 175 | } 176 | else if (alpha[i] > C){ 177 | alpha[i] = C; 178 | } 179 | else if (std::abs(delta) > limit){ 180 | judge = true; 181 | error += std::abs(delta) - limit; 182 | } 183 | 184 | } 185 | 186 | // (3.2) Update Beta 187 | item3 = 0.0; 188 | for (i = 0; i < N; i++){ 189 | item3 += alpha[i] * (double)y[i]; 190 | } 191 | beta += item3 * item3 / 2.0; 192 | 193 | // (3.3) Output Residual Error 194 | this->log("\rerror: " + std::to_string(error)); 195 | 196 | }while (judge); 197 | this->log("\n"); 198 | this->log("////////////////////////////////////////////////////////\n"); 199 | 200 | // (4.1) Description for support vectors 201 | Ns = 0; 202 | Ns_in = 0; 203 | this->xs = std::vector>(); 204 | this->ys = std::vector(); 205 | this->alpha_s = std::vector(); 206 | this->xs_in = std::vector>(); 207 | this->ys_in = std::vector(); 208 | this->alpha_s_in = std::vector(); 209 | for (i = 0; i < N; i++){ 210 | if ((eps < alpha[i]) && (alpha[i] < C - eps)){ 211 | this->xs.push_back(x[i]); 212 | this->ys.push_back(y[i]); 213 | this->alpha_s.push_back(alpha[i]); 214 | Ns++; 215 | } 216 | else if (alpha[i] >= C - eps){ 217 | this->xs_in.push_back(x[i]); 218 | this->ys_in.push_back(y[i]); 219 | this->alpha_s_in.push_back(alpha[i]); 220 | Ns_in++; 221 | } 222 | } 223 | this->log("Ns (number of support vectors on margin) = " + std::to_string(Ns) + "\n"); 224 | this->log("Ns_in (number of support vectors inside margin) = " + std::to_string(Ns_in) + "\n"); 225 | 226 | // (4.2) Description for b 227 | this->b = 0.0; 228 | for (i = 0; i < Ns; i++){ 229 | this->b += (double)this->ys[i]; 230 | for (j = 0; j < Ns; j++){ 231 | this->b -= this->alpha_s[j] * (double)this->ys[j] * this->K(this->xs[j], this->xs[i], this->params); 232 | } 233 | for (j = 0; j < Ns_in; j++){ 234 | this->b -= this->alpha_s_in[j] * (double)this->ys_in[j] * this->K(this->xs_in[j], this->xs[i], this->params); 235 | } 236 | } 237 | this->b /= (double)Ns; 238 | this->log("bias = " + std::to_string(this->b) + "\n"); 239 | this->log("////////////////////////////////////////////////////////\n\n"); 240 | 241 | return; 242 | } 243 | 244 | 245 | // ------------------------------------- 246 | // class{Kernel_SVM} -> function{test} 247 | // ------------------------------------- 248 | void Kernel_SVM::test(const std::vector> class1_data, const std::vector> class2_data){ 249 | 250 | size_t i; 251 | 252 | this->correct_c1 = 0; 253 | for (i = 0; i < class1_data.size(); i++){ 254 | if (this->g(class1_data[i]) == 1){ 255 | this->correct_c1++; 256 | } 257 | } 258 | 259 | this->correct_c2 = 0; 260 | for (i = 0; i < class2_data.size(); i++){ 261 | if (this->g(class2_data[i]) == -1){ 262 | this->correct_c2++; 263 | } 264 | } 265 | 266 | this->accuracy = (double)(this->correct_c1 + this->correct_c2) / (double)(class1_data.size() + class2_data.size()); 267 | this->accuracy_c1 = (double)this->correct_c1 / (double)class1_data.size(); 268 | this->accuracy_c2 = (double)this->correct_c2 / (double)class2_data.size(); 269 | 270 | return; 271 | } 272 | 273 | 274 | // ---------------------------------- 275 | // class{Kernel_SVM} -> function{f} 276 | // ---------------------------------- 277 | double Kernel_SVM::f(const std::vector x){ 278 | 279 | size_t i; 280 | double ans; 281 | 282 | ans = 0.0; 283 | for (i = 0; i < this->xs.size(); i++){ 284 | ans += this->alpha_s[i] * this->ys[i] * this->K(this->xs[i], x, this->params); 285 | } 286 | for (i = 0; i < this->xs_in.size(); i++){ 287 | ans += this->alpha_s_in[i] * this->ys_in[i] * this->K(this->xs_in[i], x, this->params); 288 | } 289 | ans += this->b; 290 | 291 | return ans; 292 | } 293 | 294 | 295 | // ---------------------------------- 296 | // class{Kernel_SVM} -> function{g} 297 | // ---------------------------------- 298 | double Kernel_SVM::g(const std::vector x){ 299 | 300 | double fx; 301 | int gx; 302 | 303 | fx = this->f(x); 304 | if (fx >= 0.0){ 305 | gx = 1; 306 | } 307 | else{ 308 | gx = -1; 309 | } 310 | 311 | return gx; 312 | } -------------------------------------------------------------------------------- /Kernel-SVM/src/svm.hpp: -------------------------------------------------------------------------------- 1 | #ifndef SVM_HPP 2 | #define SVM_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | 9 | // ------------------- 10 | // namespace{kernel} 11 | // ------------------- 12 | namespace kernel{ 13 | double linear(const std::vector x1, const std::vector x2, const std::vector params); 14 | double polynomial(const std::vector x1, const std::vector x2, const std::vector params); 15 | double rbf(const std::vector x1, const std::vector x2, const std::vector params); 16 | } 17 | typedef std::function, const std::vector, const std::vector)> KernelFunc; 18 | 19 | 20 | // ------------------- 21 | // class{Kernel_SVM} 22 | // ------------------- 23 | class Kernel_SVM{ 24 | private: 25 | 26 | // member variable 27 | bool verbose; 28 | double b; 29 | std::vector> xs; 30 | std::vector ys; 31 | std::vector alpha_s; 32 | std::vector> xs_in; 33 | std::vector ys_in; 34 | std::vector alpha_s_in; 35 | /***** kernel *****/ 36 | KernelFunc K; 37 | std::vector params; 38 | /***** kernel *****/ 39 | 40 | // fuction 41 | void log(const std::string str); 42 | 43 | public: 44 | 45 | // member variable 46 | double accuracy; 47 | double accuracy_c1, accuracy_c2; 48 | size_t correct_c1, correct_c2; 49 | 50 | // constructor 51 | Kernel_SVM() = delete; 52 | Kernel_SVM(const KernelFunc K_=kernel::rbf, const std::vector params_={1.0}, const bool verbose_=true); 53 | 54 | // function 55 | void train(const std::vector> class1_data, const std::vector> class2_data, const size_t D, const double C, const double lr, const double limit=0.0001); 56 | void test(const std::vector> class1_data, const std::vector> class2_data); 57 | double f(const std::vector x); 58 | double g(const std::vector x); 59 | 60 | }; 61 | 62 | 63 | #endif -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 koba-jon 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /OC-SVM/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 2 | 3 | # Project Name 4 | project(OC-SVM) 5 | 6 | # Directory Name 7 | set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) 8 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) 9 | 10 | # Set Compiler Options 11 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -Wall") 12 | if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8) 13 | set(CMAKE_CXX_COMPILER "g++-8") 14 | endif () 15 | 16 | # For OSX 17 | #set(CMAKE_OSX_ARCHITECTURES "x86_64") # Intel cpu 18 | set(CMAKE_OSX_ARCHITECTURES "arm64") # Apple cpu 19 | 20 | # Find Package 21 | find_package(Boost REQUIRED COMPONENTS program_options) 22 | 23 | # Set Include Directories 24 | set(INCLUDE_DIRS 25 | ${Boost_INCLUDE_DIRS} 26 | ${SRC_DIR} 27 | ) 28 | 29 | # Set CXX_LIBRARIES 30 | set(LIBRARIES 31 | ${Boost_LIBRARIES} 32 | ) 33 | 34 | # Set Source Code 35 | set(SRCS 36 | ${SRC_DIR}/main.cpp 37 | ${SRC_DIR}/svm.cpp 38 | ) 39 | 40 | # Link 41 | add_executable(${PROJECT_NAME} ${SRCS}) 42 | include_directories(${INCLUDE_DIRS}) 43 | target_link_libraries(${PROJECT_NAME} ${LIBRARIES}) 44 | set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 17) 45 | 46 | # Display Message 47 | message(STATUS "") 48 | message(STATUS "~~~ Boost Information ~~~") 49 | message(STATUS "${Boost_INCLUDE_DIRS};") 50 | message(STATUS "${Boost_LIBRARIES};") 51 | message(STATUS "") 52 | -------------------------------------------------------------------------------- /OC-SVM/README.md: -------------------------------------------------------------------------------- 1 | # OC-SVM 2 | 3 | This is the implementation of OC-SVM (One Class Support Vector Machine). 4 | 5 | - Class: 1 6 | - Problem: Linearly Non-separable 7 | - Decision Boundary: Hyperplane + Kernel 8 | 9 | ## Usage 10 | 11 | ### 1. Build 12 | Please build the source file according to the procedure. 13 | ~~~ 14 | $ mkdir build 15 | $ cd build 16 | $ cmake .. 17 | $ make 18 | $ cd .. 19 | ~~~ 20 | 21 | ### 2. Dataset Setting 22 | 23 | The following hierarchical relationships are recommended. 24 | 25 | ![OC-SVM_dataset](datasets/dataset.png) 26 | 27 | ### 3. Execution 28 | 29 | The following is an example for Toy Dataset. 30 | 31 | #### Setting 32 | Please set the shell for executable file. 33 | ~~~ 34 | $ vi scripts/toy.sh 35 | ~~~ 36 | If you want to view specific examples of command line arguments, please view "src/main.cpp" or add "--help" to the argument. 37 | ~~~ 38 | #!/bin/bash 39 | 40 | DATA='toy' 41 | 42 | ./OC-SVM \ 43 | --dataset ${DATA} \ 44 | --nd 2 \ 45 | --nu 0.003 \ 46 | --lr 0.0001 \ 47 | --kernel "rbf" \ 48 | --gamma 5.0 49 | ~~~ 50 | 51 | #### Run 52 | Please execute the following to start the program. 53 | ~~~ 54 | $ sh scripts/toy.sh 55 | ~~~ 56 | 57 | ## Formula 58 | 59 | ![OC-SVM_dual](https://user-images.githubusercontent.com/56967584/130268146-fd64d0e5-b781-4608-90a1-189ce9ed5173.png) 60 | ![OC-SVM_obj](https://user-images.githubusercontent.com/56967584/130268158-5ac93d71-f411-4aaf-9cb2-3a1dc7946fcd.png) 61 | ![OC-SVM_delta](https://user-images.githubusercontent.com/56967584/130329353-ee2b7cf0-0bad-4082-944d-44140ff4adf8.png) 62 | ![OC-SVM_update](https://user-images.githubusercontent.com/56967584/130329356-d196bad5-16eb-4eab-84dc-690092ca3db6.png) 63 | ![OC-SVM_class](https://user-images.githubusercontent.com/56967584/130329448-f5fc295a-8a82-499f-af50-d62dd133ba84.png) 64 | 65 | 66 | ## Algorithm 67 | ![train](https://user-images.githubusercontent.com/56967584/130329071-a78d24f5-aea1-4866-865d-6e03bdce15f6.png) 68 | ![test](https://user-images.githubusercontent.com/56967584/130328123-beac0ce3-50ed-4ca0-8a44-234c35ce5af9.png) 69 | -------------------------------------------------------------------------------- /OC-SVM/datasets/dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/koba-jon/svm_cpp/5fa1b0951740b249a7b7bb170031b75470dbf090/OC-SVM/datasets/dataset.png -------------------------------------------------------------------------------- /OC-SVM/datasets/toy/test/anomaly/data1.txt: -------------------------------------------------------------------------------- 1 | 0.8 2 | 0.2 3 | -------------------------------------------------------------------------------- /OC-SVM/datasets/toy/test/anomaly/data2.txt: -------------------------------------------------------------------------------- 1 | 0.8 2 | 0.25 -------------------------------------------------------------------------------- /OC-SVM/datasets/toy/test/anomaly/data3.txt: -------------------------------------------------------------------------------- 1 | 0.19 2 | 0.19 3 | -------------------------------------------------------------------------------- /OC-SVM/datasets/toy/test/anomaly/data4.txt: -------------------------------------------------------------------------------- 1 | 0.17 2 | 0.17 -------------------------------------------------------------------------------- /OC-SVM/datasets/toy/test/anomaly/data5.txt: -------------------------------------------------------------------------------- 1 | 0.95 2 | 0.81 3 | -------------------------------------------------------------------------------- /OC-SVM/datasets/toy/test/anomaly/data6.txt: -------------------------------------------------------------------------------- 1 | 0.19 2 | 0.81 -------------------------------------------------------------------------------- /OC-SVM/datasets/toy/test/normal/data1.txt: -------------------------------------------------------------------------------- 1 | 0.5 2 | 0.5 3 | -------------------------------------------------------------------------------- /OC-SVM/datasets/toy/test/normal/data2.txt: -------------------------------------------------------------------------------- 1 | 0.6 2 | 0.6 3 | -------------------------------------------------------------------------------- /OC-SVM/datasets/toy/test/normal/data3.txt: -------------------------------------------------------------------------------- 1 | 0.4 2 | 0.4 3 | -------------------------------------------------------------------------------- /OC-SVM/datasets/toy/test/normal/data4.txt: -------------------------------------------------------------------------------- 1 | 0.5 2 | 0.6 3 | -------------------------------------------------------------------------------- /OC-SVM/datasets/toy/test/normal/data5.txt: -------------------------------------------------------------------------------- 1 | 0.5 2 | 0.45 3 | -------------------------------------------------------------------------------- /OC-SVM/datasets/toy/test/normal/data6.txt: -------------------------------------------------------------------------------- 1 | 0.59 2 | 0.41 -------------------------------------------------------------------------------- /OC-SVM/datasets/toy/train/data1.txt: -------------------------------------------------------------------------------- 1 | 0.5 2 | 0.5 3 | -------------------------------------------------------------------------------- /OC-SVM/datasets/toy/train/data2.txt: -------------------------------------------------------------------------------- 1 | 0.4 2 | 0.6 3 | -------------------------------------------------------------------------------- /OC-SVM/datasets/toy/train/data3.txt: -------------------------------------------------------------------------------- 1 | 0.6 2 | 0.65 3 | -------------------------------------------------------------------------------- /OC-SVM/datasets/toy/train/data4.txt: -------------------------------------------------------------------------------- 1 | 0.65 2 | 0.35 3 | -------------------------------------------------------------------------------- /OC-SVM/datasets/toy/train/data5.txt: -------------------------------------------------------------------------------- 1 | 0.4 2 | 0.6 3 | -------------------------------------------------------------------------------- /OC-SVM/scripts/toy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DATA='toy' 4 | 5 | ./OC-SVM \ 6 | --dataset ${DATA} \ 7 | --nd 2 \ 8 | --nu 0.003 \ 9 | --lr 0.0001 \ 10 | --kernel "rbf" \ 11 | --gamma 5.0 -------------------------------------------------------------------------------- /OC-SVM/src/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | // 3rd-Party Libraries 9 | #include 10 | // Original 11 | #include "svm.hpp" 12 | 13 | // Define Namespace 14 | namespace fs = std::filesystem; 15 | namespace po = boost::program_options; 16 | 17 | // Function Prototype 18 | void Collect_Paths(const std::string root, const std::string sub, std::vector &paths); 19 | std::vector Get_Paths(const std::string root); 20 | std::vector> Get_Data(const std::vector paths, const size_t D); 21 | void Set_Kernel(po::variables_map &vm, KernelFunc &K, std::vector ¶ms); 22 | 23 | 24 | // ---------------------- 25 | // 0. Argument Function 26 | // ---------------------- 27 | po::options_description parse_arguments(){ 28 | 29 | po::options_description args("Options", 200, 30); 30 | 31 | args.add_options() 32 | 33 | // (1) Define for General Parameter 34 | ("help", "produce help message") 35 | ("dataset", po::value()->default_value("toy"), "dataset name") 36 | ("nd", po::value()->default_value(2), "number of dimensions") 37 | ("verbose", po::value()->default_value(true), "verbose") 38 | 39 | // (2) Define for Training 40 | ("train_dir", po::value()->default_value("train"), "training directory : ./datasets///") 41 | ("nu", po::value()->default_value(0.003), "regularization parameter (0.0 < nu <= 1.0)") 42 | ("lr", po::value()->default_value(0.0001), "learning rate of alpha") 43 | 44 | // (3) Define for Test 45 | ("test_normal_dir", po::value()->default_value("test/normal"), "test normal directory : ./datasets///") 46 | ("test_anomaly_dir", po::value()->default_value("test/anomaly"), "test anomaly directory : ./datasets///") 47 | 48 | // (4) Define for Kernel 49 | ("kernel", po::value()->default_value("rbf"), "kernel : linear / polynomial / rbf") 50 | ("c", po::value()->default_value(1.0), "addition parameter for polynomial kernel (c > 0.0)") 51 | ("d", po::value()->default_value(2.0), "exponent parameter for polynomial kernel (natural number)") 52 | ("gamma", po::value()->default_value(1.0), "precision (inverse of variance) parameter for rbf kernel (gamma > 0.0)") 53 | 54 | ; 55 | 56 | return args; 57 | 58 | } 59 | 60 | 61 | // ------------------ 62 | // 1. Main Function 63 | // ------------------ 64 | int main(int argc, const char *argv[]){ 65 | 66 | // (1) Extract Arguments 67 | po::options_description args = parse_arguments(); 68 | po::variables_map vm{}; 69 | po::store(po::parse_command_line(argc, argv, args), vm); 70 | po::notify(vm); 71 | if (vm.empty() || vm.count("help")){ 72 | std::cout << args << std::endl; 73 | return 1; 74 | } 75 | 76 | // (2) Set Kernel 77 | KernelFunc K; 78 | std::vector params; 79 | Set_Kernel(vm, K, params); 80 | 81 | // (3.1) Get Training Data 82 | std::string train_dir; 83 | std::vector train_paths; 84 | std::vector> train_data; 85 | /*****************************************************/ 86 | train_dir = "datasets/" + vm["dataset"].as() + "/" + vm["train_dir"].as(); 87 | train_paths = Get_Paths(train_dir); 88 | train_data = Get_Data(train_paths, vm["nd"].as()); 89 | 90 | // (3.2) Training for SVM 91 | OC_SVM svm(K, params, vm["verbose"].as()); 92 | svm.train(train_data, vm["nd"].as(), vm["nu"].as(), vm["lr"].as()); 93 | 94 | // (4.1) Get Test Normal Data 95 | std::string test_normal_dir; 96 | std::vector test_normal_paths; 97 | std::vector> test_normal_data; 98 | /*****************************************************/ 99 | test_normal_dir = "datasets/" + vm["dataset"].as() + "/" + vm["test_normal_dir"].as(); 100 | test_normal_paths = Get_Paths(test_normal_dir); 101 | test_normal_data = Get_Data(test_normal_paths, vm["nd"].as()); 102 | 103 | // (4.2) Get Test Anomaly Data 104 | std::string test_anomaly_dir; 105 | std::vector test_anomaly_paths; 106 | std::vector> test_anomaly_data; 107 | /*****************************************************/ 108 | test_anomaly_dir = "datasets/" + vm["dataset"].as() + "/" + vm["test_anomaly_dir"].as(); 109 | test_anomaly_paths = Get_Paths(test_anomaly_dir); 110 | test_anomaly_data = Get_Data(test_anomaly_paths, vm["nd"].as()); 111 | 112 | // (4.3) Test for SVM 113 | svm.test(test_normal_data, test_anomaly_data); 114 | std::cout << "///////////////////////// Test /////////////////////////" << std::endl; 115 | std::cout << "accuracy-all: " << svm.accuracy << " (" << svm.correct_n + svm.correct_a << "/" << test_normal_data.size() + test_anomaly_data.size() << ")" << std::endl; 116 | std::cout << "accuracy-normal: " << svm.accuracy_n << " (" << svm.correct_n << "/" << test_normal_data.size() << ")" << std::endl; 117 | std::cout << "accuracy-anomaly: " << svm.accuracy_a << " (" << svm.correct_a << "/" << test_anomaly_data.size() << ")" << std::endl; 118 | std::cout << "AUROC: " << svm.auroc << std::endl; 119 | std::cout << "////////////////////////////////////////////////////////" << std::endl; 120 | 121 | return 0; 122 | } 123 | 124 | 125 | // ------------------------------ 126 | // 2. Collecting Paths Function 127 | // ------------------------------ 128 | void Collect_Paths(const std::string root, const std::string sub, std::vector &paths){ 129 | 130 | fs::path ROOT(root); 131 | 132 | for (auto &p : fs::directory_iterator(ROOT)){ 133 | if (!fs::is_directory(p)){ 134 | std::stringstream rpath; 135 | rpath << p.path().string(); 136 | paths.push_back(rpath.str()); 137 | } 138 | else{ 139 | std::stringstream subsub; 140 | subsub << p.path().filename().string(); 141 | Collect_Paths(root + '/' + subsub.str(), sub + subsub.str() + '/', paths); 142 | } 143 | } 144 | 145 | return; 146 | } 147 | 148 | 149 | // --------------------------- 150 | // 3. Getting Paths Function 151 | // --------------------------- 152 | std::vector Get_Paths(const std::string root){ 153 | 154 | std::vector paths; 155 | 156 | Collect_Paths(root, "", paths); 157 | std::sort(paths.begin(), paths.end()); 158 | 159 | return paths; 160 | 161 | } 162 | 163 | 164 | // --------------------------- 165 | // 4. Getting Data Function 166 | // --------------------------- 167 | std::vector> Get_Data(const std::vector paths, const size_t D){ 168 | 169 | size_t i; 170 | double element; 171 | std::ifstream ifs; 172 | std::vector data_one; 173 | std::vector> data; 174 | 175 | for (std::string path : paths){ 176 | 177 | ifs.open(path); 178 | 179 | data_one = std::vector(D); 180 | for (i = 0; i < D; i++){ 181 | ifs >> element; 182 | data_one[i] = element; 183 | } 184 | data.push_back(data_one); 185 | 186 | ifs.close(); 187 | 188 | } 189 | 190 | return data; 191 | } 192 | 193 | 194 | // ---------------------------- 195 | // 5. Setting Kernel Function 196 | // ---------------------------- 197 | void Set_Kernel(po::variables_map &vm, KernelFunc &K, std::vector ¶ms){ 198 | 199 | if (vm["kernel"].as() == "linear"){ 200 | K = kernel::linear; 201 | } 202 | else if (vm["kernel"].as() == "polynomial"){ 203 | K = kernel::polynomial; 204 | params = {vm["c"].as(), vm["d"].as()}; 205 | } 206 | else if (vm["kernel"].as() == "rbf"){ 207 | K = kernel::rbf; 208 | params = {vm["gamma"].as()}; 209 | } 210 | else{ 211 | std::cerr << "Error : Couldn't match the name of kernel." << std::endl; 212 | std::exit(-1); 213 | } 214 | 215 | return; 216 | 217 | } -------------------------------------------------------------------------------- /OC-SVM/src/svm.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | // Original 7 | #include "svm.hpp" 8 | 9 | 10 | // --------------------------------------- 11 | // namespace{kernel} -> function{linear} 12 | // --------------------------------------- 13 | double kernel::linear(const std::vector x1, const std::vector x2, const std::vector params){ 14 | 15 | size_t i; 16 | double ans; 17 | 18 | if (x1.size() != x2.size()){ 19 | std::cerr << "Error : Couldn't match the number of elements for inner product." << std::endl; 20 | std::exit(-1); 21 | } 22 | 23 | ans = 0.0; 24 | for (i = 0; i < x1.size(); i++){ 25 | ans += x1[i] * x2[i]; 26 | } 27 | 28 | return ans; 29 | 30 | } 31 | 32 | 33 | // ------------------------------------------- 34 | // namespace{kernel} -> function{polynomial} 35 | // ------------------------------------------- 36 | double kernel::polynomial(const std::vector x1, const std::vector x2, const std::vector params){ 37 | 38 | size_t i; 39 | double ans; 40 | 41 | if (x1.size() != x2.size()){ 42 | std::cerr << "Error : Couldn't match the number of elements for inner product." << std::endl; 43 | std::exit(-1); 44 | } 45 | else if (params.size() != 2){ 46 | std::cerr << "Error : Couldn't match the number of hyper-parameters." << std::endl; 47 | std::exit(-1); 48 | } 49 | 50 | ans = 0.0; 51 | for (i = 0; i < x1.size(); i++){ 52 | ans += x1[i] * x2[i]; 53 | } 54 | ans += params[0]; 55 | ans = std::pow(ans, params[1]); 56 | 57 | return ans; 58 | 59 | } 60 | 61 | 62 | // ------------------------------------ 63 | // namespace{kernel} -> function{rbf} 64 | // ------------------------------------ 65 | double kernel::rbf(const std::vector x1, const std::vector x2, const std::vector params){ 66 | 67 | size_t i; 68 | double ans; 69 | 70 | if (x1.size() != x2.size()){ 71 | std::cerr << "Error : Couldn't match the number of elements for inner product." << std::endl; 72 | std::exit(-1); 73 | } 74 | else if (params.size() != 1){ 75 | std::cerr << "Error : Couldn't match the number of hyper-parameters." << std::endl; 76 | std::exit(-1); 77 | } 78 | 79 | ans = 0.0; 80 | for (i = 0; i < x1.size(); i++){ 81 | ans += (x1[i] - x2[i]) * (x1[i] - x2[i]); 82 | } 83 | ans = std::exp(-params[0] * ans); 84 | 85 | return ans; 86 | 87 | } 88 | 89 | 90 | // ------------------------------ 91 | // class{OC_SVM} -> constructor 92 | // ------------------------------ 93 | OC_SVM::OC_SVM(const KernelFunc K_, const std::vector params_, const bool verbose_){ 94 | this->K = K_; 95 | this->params = params_; 96 | this->verbose = verbose_; 97 | } 98 | 99 | 100 | // -------------------------------- 101 | // class{OC_SVM} -> function{log} 102 | // -------------------------------- 103 | void OC_SVM::log(const std::string str){ 104 | if (this->verbose){ 105 | std::cout << str << std::flush; 106 | } 107 | return; 108 | } 109 | 110 | 111 | // --------------------------------- 112 | // class{OC_SVM} -> function{sort} 113 | // --------------------------------- 114 | void OC_SVM::sort(std::vector> &data){ 115 | 116 | size_t i, j; 117 | std::pair tmp; 118 | 119 | for (i = 1; i < data.size(); i++){ 120 | j = i; 121 | while ((j > 0) && (data[j - 1].first > data[j].first)){ 122 | tmp = data[j - 1]; 123 | data[j - 1] = data[j]; 124 | data[j] = tmp; 125 | j--; 126 | } 127 | } 128 | 129 | return; 130 | } 131 | 132 | 133 | // ---------------------------------- 134 | // class{OC_SVM} -> function{train} 135 | // ---------------------------------- 136 | void OC_SVM::train(const std::vector> x, const size_t D, const double nu, const double lr, const double limit){ 137 | 138 | constexpr double eps = 0.0000001; 139 | 140 | size_t i, j; 141 | size_t N, Ns, Ns_in; 142 | bool judge; 143 | double C; 144 | double item1, item2, item3; 145 | double delta; 146 | double beta; 147 | double error; 148 | std::vector alpha; 149 | 150 | // (1) Set Lagrange Multiplier and Parameters 151 | N = x.size(); 152 | C = 1.0 / ((double)N * nu); 153 | alpha = std::vector(N, 0.0); 154 | beta = 1.0; 155 | 156 | // (2) Training 157 | this->log("\n"); 158 | this->log("/////////////////////// Training ///////////////////////\n"); 159 | do { 160 | 161 | judge = false; 162 | error = 0.0; 163 | 164 | // (2.1) Update Alpha 165 | for (i = 0; i < N; i++){ 166 | 167 | // Set item 1 168 | item1 = 0.0; 169 | for (j = 0; j < N; j++){ 170 | item1 += alpha[j] * this->K(x[i], x[j], this->params); 171 | } 172 | 173 | // Set item 2 174 | item2 = 0.0; 175 | for (j = 0; j < N; j++){ 176 | item2 += alpha[j]; 177 | } 178 | item2 -= 1.0; 179 | 180 | // Set Delta 181 | delta = item1 + beta * item2; 182 | 183 | // Update 184 | alpha[i] -= lr * delta; 185 | if (alpha[i] < 0.0){ 186 | alpha[i] = 0.0; 187 | } 188 | else if (alpha[i] > C){ 189 | alpha[i] = C; 190 | } 191 | else if (std::abs(delta) > limit){ 192 | judge = true; 193 | error += std::abs(delta) - limit; 194 | } 195 | 196 | } 197 | 198 | // (2.2) Update Beta 199 | item3 = 0.0; 200 | for (i = 0; i < N; i++){ 201 | item3 += alpha[i]; 202 | } 203 | item3 -= 1.0; 204 | beta += item3 * item3 / 2.0; 205 | 206 | // (2.3) Output Residual Error 207 | this->log("\rerror: " + std::to_string(error)); 208 | 209 | }while (judge); 210 | this->log("\n"); 211 | this->log("////////////////////////////////////////////////////////\n"); 212 | 213 | // (3.1) Description for support vectors 214 | Ns = 0; 215 | Ns_in = 0; 216 | this->xs = std::vector>(); 217 | this->alpha_s = std::vector(); 218 | this->xs_in = std::vector>(); 219 | this->alpha_s_in = std::vector(); 220 | for (i = 0; i < N; i++){ 221 | if ((eps < alpha[i]) && (alpha[i] < C - eps)){ 222 | this->xs.push_back(x[i]); 223 | this->alpha_s.push_back(alpha[i]); 224 | Ns++; 225 | } 226 | else if (alpha[i] >= C - eps){ 227 | this->xs_in.push_back(x[i]); 228 | this->alpha_s_in.push_back(alpha[i]); 229 | Ns_in++; 230 | } 231 | } 232 | this->log("Ns (number of support vectors on margin) = " + std::to_string(Ns) + "\n"); 233 | this->log("Ns_in (number of support vectors inside margin) = " + std::to_string(Ns_in) + "\n"); 234 | 235 | // (3.2) Description for b 236 | this->b = 0.0; 237 | for (i = 0; i < Ns; i++){ 238 | for (j = 0; j < Ns; j++){ 239 | this->b += this->alpha_s[j] * this->K(this->xs[j], this->xs[i], this->params); 240 | } 241 | for (j = 0; j < Ns_in; j++){ 242 | this->b += this->alpha_s_in[j] * this->K(this->xs_in[j], this->xs[i], this->params); 243 | } 244 | } 245 | this->b /= (double)Ns; 246 | this->log("bias = " + std::to_string(this->b) + "\n"); 247 | this->log("////////////////////////////////////////////////////////\n\n"); 248 | 249 | return; 250 | } 251 | 252 | 253 | // --------------------------------- 254 | // class{OC_SVM} -> function{test} 255 | // --------------------------------- 256 | void OC_SVM::test(const std::vector> normal_data, const std::vector> anomaly_data){ 257 | 258 | size_t i; 259 | std::vector> score; 260 | 261 | this->correct_n = 0; 262 | for (i = 0; i < normal_data.size(); i++){ 263 | score.push_back({this->f(normal_data[i]), 1}); 264 | if (this->g(normal_data[i]) == 1){ 265 | this->correct_n++; 266 | } 267 | } 268 | 269 | this->correct_a = 0; 270 | for (i = 0; i < anomaly_data.size(); i++){ 271 | score.push_back({this->f(anomaly_data[i]), -1}); 272 | if (this->g(anomaly_data[i]) == -1){ 273 | this->correct_a++; 274 | } 275 | } 276 | 277 | this->accuracy = (double)(this->correct_n + this->correct_a) / (double)(normal_data.size() + anomaly_data.size()); 278 | this->accuracy_n = (double)this->correct_n / (double)normal_data.size(); 279 | this->accuracy_a = (double)this->correct_a / (double)anomaly_data.size(); 280 | 281 | this->roc(score); 282 | 283 | return; 284 | } 285 | 286 | 287 | // -------------------------------- 288 | // class{OC_SVM} -> function{roc} 289 | // -------------------------------- 290 | void OC_SVM::roc(const std::vector> score){ 291 | 292 | size_t i; 293 | size_t Np, Nn; 294 | size_t TP, FP; 295 | double TP_rate, FP_rate; 296 | double pre_TP_rate, pre_FP_rate; 297 | std::vector> score_sorted; 298 | 299 | // (1) Sort (Ascending Order) 300 | score_sorted = score; 301 | this->sort(score_sorted); 302 | 303 | // (2) Set Parameters 304 | Np = 0; 305 | Nn = 0; 306 | for (i = 0; i < score_sorted.size(); i++){ 307 | if (score_sorted[i].second == -1){ 308 | Np++; 309 | } 310 | else{ 311 | Nn++; 312 | } 313 | } 314 | 315 | // (3) Set Parameters 316 | TP = 0; 317 | FP = 0; 318 | TP_rate = 0.0; 319 | FP_rate = 0.0; 320 | pre_TP_rate = 0.0; 321 | pre_FP_rate = 0.0; 322 | this->auroc = 0.0; 323 | 324 | // (4) Anomaly Detection 325 | for (i = 0; i < score_sorted.size() - 1; i++){ 326 | 327 | if (score_sorted[i].second == -1){ 328 | TP++; 329 | } 330 | else{ 331 | FP++; 332 | } 333 | 334 | if (score_sorted[i].first != score_sorted[i + 1].first){ 335 | TP_rate = (double)TP / (double)Np; 336 | FP_rate = (double)FP / (double)Nn; 337 | this->auroc += (TP_rate + pre_TP_rate) * (FP_rate - pre_FP_rate) * 0.5; 338 | pre_TP_rate = TP_rate; 339 | pre_FP_rate = FP_rate; 340 | } 341 | 342 | } 343 | this->auroc += (1.0 + pre_TP_rate) * (1.0 - pre_FP_rate) * 0.5; 344 | 345 | return; 346 | } 347 | 348 | 349 | // ------------------------------ 350 | // class{OC_SVM} -> function{f} 351 | // ------------------------------ 352 | double OC_SVM::f(const std::vector x){ 353 | 354 | size_t i; 355 | double ans; 356 | 357 | ans = 0.0; 358 | for (i = 0; i < this->xs.size(); i++){ 359 | ans += this->alpha_s[i] * this->K(this->xs[i], x, this->params); 360 | } 361 | for (i = 0; i < this->xs_in.size(); i++){ 362 | ans += this->alpha_s_in[i] * this->K(this->xs_in[i], x, this->params); 363 | } 364 | ans -= this->b; 365 | 366 | return ans; 367 | } 368 | 369 | 370 | // ------------------------------ 371 | // class{OC_SVM} -> function{g} 372 | // ------------------------------ 373 | double OC_SVM::g(const std::vector x){ 374 | 375 | double fx; 376 | int gx; 377 | 378 | fx = this->f(x); 379 | if (fx >= 0.0){ 380 | gx = 1; 381 | } 382 | else{ 383 | gx = -1; 384 | } 385 | 386 | return gx; 387 | } -------------------------------------------------------------------------------- /OC-SVM/src/svm.hpp: -------------------------------------------------------------------------------- 1 | #ifndef SVM_HPP 2 | #define SVM_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | 10 | // ------------------- 11 | // namespace{kernel} 12 | // ------------------- 13 | namespace kernel{ 14 | double linear(const std::vector x1, const std::vector x2, const std::vector params); 15 | double polynomial(const std::vector x1, const std::vector x2, const std::vector params); 16 | double rbf(const std::vector x1, const std::vector x2, const std::vector params); 17 | } 18 | typedef std::function, const std::vector, const std::vector)> KernelFunc; 19 | 20 | 21 | // --------------- 22 | // class{OC_SVM} 23 | // --------------- 24 | class OC_SVM{ 25 | private: 26 | 27 | // member variable 28 | bool verbose; 29 | double b; 30 | std::vector> xs; 31 | std::vector alpha_s; 32 | std::vector> xs_in; 33 | std::vector alpha_s_in; 34 | /***** kernel *****/ 35 | KernelFunc K; 36 | std::vector params; 37 | /***** kernel *****/ 38 | 39 | // fuction 40 | void log(const std::string str); 41 | void sort(std::vector> &data); 42 | 43 | public: 44 | 45 | // member variable 46 | double accuracy; 47 | double accuracy_n, accuracy_a; 48 | double auroc; 49 | size_t correct_n, correct_a; 50 | 51 | // constructor 52 | OC_SVM() = delete; 53 | OC_SVM(const KernelFunc K_=kernel::rbf, const std::vector params_={1.0}, const bool verbose_=true); 54 | 55 | // function 56 | void train(const std::vector> x, const size_t D, const double nu, const double lr, const double limit=0.0001); 57 | void test(const std::vector> normal_data, const std::vector> anomaly_data); 58 | void roc(const std::vector> score); 59 | double f(const std::vector x); 60 | double g(const std::vector x); 61 | 62 | }; 63 | 64 | 65 | #endif 66 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SVM C++ Samples 2 | These are sample programs of Support Vector Machines from scratch in C++. 3 | 4 | ## 1. Implementation 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 |
ModelClassProblemDecision BoundaryCode
Hard Margin SVM2Linearly SeparableHyperplaneHardMargin-SVM
Soft Margin SVM2Linearly Non-separableHyperplaneSoftMargin-SVM
Kernel SVM2Linearly Non-separableHyperplane + KernelKernel-SVM
OC-SVM1Linearly Non-separableHyperplane + KernelOC-SVM
SVDD1Linearly Non-separableHypersphere + KernelSVDD
50 | 51 | ## 2. Requirement 52 | 53 | ### Boost 54 | 55 | This is used for command line arguments, etc.
56 | ~~~ 57 | $ sudo apt install libboost-dev libboost-all-dev 58 | ~~~ 59 | 60 | ## 3. Preparation 61 | 62 | ### Git Clone 63 | ~~~ 64 | $ git clone https://github.com/koba-jon/svm_cpp.git 65 | $ cd svm_cpp 66 | ~~~ 67 | 68 | ## 4. Execution 69 | - [Hard Margin SVM (Hard Margin Support Vector Machine)](HardMargin-SVM) 70 | - [Soft Margin SVM (Soft Margin Support Vector Machine)](SoftMargin-SVM) 71 | - [Kernel SVM (Non-linear Support Vector Machine)](Kernel-SVM) 72 | - [OC-SVM (One Class Support Vector Machine)](OC-SVM) 73 | - [SVDD (Support Vector Data Description)](SVDD) 74 | 75 | ## 5. License 76 | 77 | This repository: [MIT License](LICENSE) 78 | 79 | ### 3rd-Party Libraries 80 | - Boost
81 | Official : https://www.boost.org/
82 | License : https://www.boost.org/users/license.html
83 | 84 | ## References 85 | - https://qiita.com/ta-ka/items/e6fd0b6fc46dbab4a651 (Japanese) 86 | - https://www.slideshare.net/ssuser186f56/svm-146231602 (Japanese) 87 | -------------------------------------------------------------------------------- /SVDD/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 2 | 3 | # Project Name 4 | project(SVDD) 5 | 6 | # Directory Name 7 | set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) 8 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) 9 | 10 | # Set Compiler Options 11 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -Wall") 12 | if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8) 13 | set(CMAKE_CXX_COMPILER "g++-8") 14 | endif () 15 | 16 | # For OSX 17 | #set(CMAKE_OSX_ARCHITECTURES "x86_64") # Intel cpu 18 | set(CMAKE_OSX_ARCHITECTURES "arm64") # Apple cpu 19 | 20 | # Find Package 21 | find_package(Boost REQUIRED COMPONENTS program_options) 22 | 23 | # Set Include Directories 24 | set(INCLUDE_DIRS 25 | ${Boost_INCLUDE_DIRS} 26 | ${SRC_DIR} 27 | ) 28 | 29 | # Set CXX_LIBRARIES 30 | set(LIBRARIES 31 | ${Boost_LIBRARIES} 32 | ) 33 | 34 | # Set Source Code 35 | set(SRCS 36 | ${SRC_DIR}/main.cpp 37 | ${SRC_DIR}/svdd.cpp 38 | ) 39 | 40 | # Link 41 | add_executable(${PROJECT_NAME} ${SRCS}) 42 | include_directories(${INCLUDE_DIRS}) 43 | target_link_libraries(${PROJECT_NAME} ${LIBRARIES}) 44 | set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 17) 45 | 46 | # Display Message 47 | message(STATUS "") 48 | message(STATUS "~~~ Boost Information ~~~") 49 | message(STATUS "${Boost_INCLUDE_DIRS};") 50 | message(STATUS "${Boost_LIBRARIES};") 51 | message(STATUS "") 52 | -------------------------------------------------------------------------------- /SVDD/README.md: -------------------------------------------------------------------------------- 1 | # SVDD 2 | 3 | This is the implementation of SVDD (Support Vector Data Description). 4 | 5 | - Class: 1 6 | - Problem: Linearly Non-separable 7 | - Decision Boundary: Hypersphere + Kernel 8 | 9 | ## Usage 10 | 11 | ### 1. Build 12 | Please build the source file according to the procedure. 13 | ~~~ 14 | $ mkdir build 15 | $ cd build 16 | $ cmake .. 17 | $ make 18 | $ cd .. 19 | ~~~ 20 | 21 | ### 2. Dataset Setting 22 | 23 | The following hierarchical relationships are recommended. 24 | 25 | ![SVDD_dataset](datasets/dataset.png) 26 | 27 | ### 3. Execution 28 | 29 | The following is an example for Toy Dataset. 30 | 31 | #### Setting 32 | Please set the shell for executable file. 33 | ~~~ 34 | $ vi scripts/toy.sh 35 | ~~~ 36 | If you want to view specific examples of command line arguments, please view "src/main.cpp" or add "--help" to the argument. 37 | ~~~ 38 | #!/bin/bash 39 | 40 | DATA='toy' 41 | 42 | ./SVDD \ 43 | --dataset ${DATA} \ 44 | --nd 2 \ 45 | --nu 0.003 \ 46 | --lr 0.0001 \ 47 | --kernel "rbf" \ 48 | --gamma 5.0 49 | ~~~ 50 | 51 | #### Run 52 | Please execute the following to start the program. 53 | ~~~ 54 | $ sh scripts/toy.sh 55 | ~~~ 56 | 57 | ## Formula 58 | ![SVDD_dual](https://user-images.githubusercontent.com/56967584/130436790-62bf1b4a-10d1-4419-b578-341e28392374.png) 59 | ![SVDD_obj](https://user-images.githubusercontent.com/56967584/130436797-a82d8950-2ecd-48a5-8cde-deb5a63eaaf5.png) 60 | ![SVDD_delta](https://user-images.githubusercontent.com/56967584/130436807-99953c32-8acc-498d-9182-08706ee5819b.png) 61 | ![SVDD_update](https://user-images.githubusercontent.com/56967584/130436813-3cabea09-0d5f-4065-ab0f-2f8d99964b86.png) 62 | ![SVDD_class](https://user-images.githubusercontent.com/56967584/130538309-9d5e691a-2223-4ef3-b2e8-f228de053e99.png) 63 | 64 | 65 | 66 | ## Algorithm 67 | ![train](https://user-images.githubusercontent.com/56967584/130538323-a9219a72-5895-4973-80f1-ed207a254085.png) 68 | ![test](https://user-images.githubusercontent.com/56967584/130538333-77f5cfd4-01b1-4cf2-9535-80c5d7be8796.png) 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /SVDD/datasets/dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/koba-jon/svm_cpp/5fa1b0951740b249a7b7bb170031b75470dbf090/SVDD/datasets/dataset.png -------------------------------------------------------------------------------- /SVDD/datasets/toy/test/anomaly/data1.txt: -------------------------------------------------------------------------------- 1 | 0.8 2 | 0.2 3 | -------------------------------------------------------------------------------- /SVDD/datasets/toy/test/anomaly/data2.txt: -------------------------------------------------------------------------------- 1 | 0.8 2 | 0.25 -------------------------------------------------------------------------------- /SVDD/datasets/toy/test/anomaly/data3.txt: -------------------------------------------------------------------------------- 1 | 0.19 2 | 0.19 3 | -------------------------------------------------------------------------------- /SVDD/datasets/toy/test/anomaly/data4.txt: -------------------------------------------------------------------------------- 1 | 0.17 2 | 0.17 -------------------------------------------------------------------------------- /SVDD/datasets/toy/test/anomaly/data5.txt: -------------------------------------------------------------------------------- 1 | 0.95 2 | 0.81 3 | -------------------------------------------------------------------------------- /SVDD/datasets/toy/test/anomaly/data6.txt: -------------------------------------------------------------------------------- 1 | 0.19 2 | 0.81 -------------------------------------------------------------------------------- /SVDD/datasets/toy/test/normal/data1.txt: -------------------------------------------------------------------------------- 1 | 0.5 2 | 0.5 3 | -------------------------------------------------------------------------------- /SVDD/datasets/toy/test/normal/data2.txt: -------------------------------------------------------------------------------- 1 | 0.6 2 | 0.6 3 | -------------------------------------------------------------------------------- /SVDD/datasets/toy/test/normal/data3.txt: -------------------------------------------------------------------------------- 1 | 0.4 2 | 0.4 3 | -------------------------------------------------------------------------------- /SVDD/datasets/toy/test/normal/data4.txt: -------------------------------------------------------------------------------- 1 | 0.5 2 | 0.6 3 | -------------------------------------------------------------------------------- /SVDD/datasets/toy/test/normal/data5.txt: -------------------------------------------------------------------------------- 1 | 0.5 2 | 0.45 3 | -------------------------------------------------------------------------------- /SVDD/datasets/toy/test/normal/data6.txt: -------------------------------------------------------------------------------- 1 | 0.59 2 | 0.41 -------------------------------------------------------------------------------- /SVDD/datasets/toy/train/data1.txt: -------------------------------------------------------------------------------- 1 | 0.5 2 | 0.5 3 | -------------------------------------------------------------------------------- /SVDD/datasets/toy/train/data2.txt: -------------------------------------------------------------------------------- 1 | 0.4 2 | 0.6 3 | -------------------------------------------------------------------------------- /SVDD/datasets/toy/train/data3.txt: -------------------------------------------------------------------------------- 1 | 0.6 2 | 0.65 3 | -------------------------------------------------------------------------------- /SVDD/datasets/toy/train/data4.txt: -------------------------------------------------------------------------------- 1 | 0.65 2 | 0.35 3 | -------------------------------------------------------------------------------- /SVDD/datasets/toy/train/data5.txt: -------------------------------------------------------------------------------- 1 | 0.4 2 | 0.6 3 | -------------------------------------------------------------------------------- /SVDD/scripts/toy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DATA='toy' 4 | 5 | ./SVDD \ 6 | --dataset ${DATA} \ 7 | --nd 2 \ 8 | --nu 0.003 \ 9 | --lr 0.0001 \ 10 | --kernel "rbf" \ 11 | --gamma 5.0 12 | -------------------------------------------------------------------------------- /SVDD/src/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | // 3rd-Party Libraries 9 | #include 10 | // Original 11 | #include "svdd.hpp" 12 | 13 | // Define Namespace 14 | namespace fs = std::filesystem; 15 | namespace po = boost::program_options; 16 | 17 | // Function Prototype 18 | void Collect_Paths(const std::string root, const std::string sub, std::vector &paths); 19 | std::vector Get_Paths(const std::string root); 20 | std::vector> Get_Data(const std::vector paths, const size_t D); 21 | void Set_Kernel(po::variables_map &vm, KernelFunc &K, std::vector ¶ms); 22 | 23 | 24 | // ---------------------- 25 | // 0. Argument Function 26 | // ---------------------- 27 | po::options_description parse_arguments(){ 28 | 29 | po::options_description args("Options", 200, 30); 30 | 31 | args.add_options() 32 | 33 | // (1) Define for General Parameter 34 | ("help", "produce help message") 35 | ("dataset", po::value()->default_value("toy"), "dataset name") 36 | ("nd", po::value()->default_value(2), "number of dimensions") 37 | ("verbose", po::value()->default_value(true), "verbose") 38 | 39 | // (2) Define for Training 40 | ("train_dir", po::value()->default_value("train"), "training directory : ./datasets///") 41 | ("nu", po::value()->default_value(0.003), "regularization parameter (0.0 < nu <= 1.0)") 42 | ("lr", po::value()->default_value(0.0001), "learning rate of alpha") 43 | 44 | // (3) Define for Test 45 | ("test_normal_dir", po::value()->default_value("test/normal"), "test normal directory : ./datasets///") 46 | ("test_anomaly_dir", po::value()->default_value("test/anomaly"), "test anomaly directory : ./datasets///") 47 | 48 | // (4) Define for Kernel 49 | ("kernel", po::value()->default_value("rbf"), "kernel : linear / polynomial / rbf") 50 | ("c", po::value()->default_value(1.0), "addition parameter for polynomial kernel (c > 0.0)") 51 | ("d", po::value()->default_value(2.0), "exponent parameter for polynomial kernel (natural number)") 52 | ("gamma", po::value()->default_value(1.0), "precision (inverse of variance) parameter for rbf kernel (gamma > 0.0)") 53 | 54 | ; 55 | 56 | return args; 57 | 58 | } 59 | 60 | 61 | // ------------------ 62 | // 1. Main Function 63 | // ------------------ 64 | int main(int argc, const char *argv[]){ 65 | 66 | // (1) Extract Arguments 67 | po::options_description args = parse_arguments(); 68 | po::variables_map vm{}; 69 | po::store(po::parse_command_line(argc, argv, args), vm); 70 | po::notify(vm); 71 | if (vm.empty() || vm.count("help")){ 72 | std::cout << args << std::endl; 73 | return 1; 74 | } 75 | 76 | // (2) Set Kernel 77 | KernelFunc K; 78 | std::vector params; 79 | Set_Kernel(vm, K, params); 80 | 81 | // (3.1) Get Training Data 82 | std::string train_dir; 83 | std::vector train_paths; 84 | std::vector> train_data; 85 | /*****************************************************/ 86 | train_dir = "datasets/" + vm["dataset"].as() + "/" + vm["train_dir"].as(); 87 | train_paths = Get_Paths(train_dir); 88 | train_data = Get_Data(train_paths, vm["nd"].as()); 89 | 90 | // (3.2) Training for SVDD 91 | SVDD svdd(K, params, vm["verbose"].as()); 92 | svdd.train(train_data, vm["nd"].as(), vm["nu"].as(), vm["lr"].as()); 93 | 94 | // (4.1) Get Test Normal Data 95 | std::string test_normal_dir; 96 | std::vector test_normal_paths; 97 | std::vector> test_normal_data; 98 | /*****************************************************/ 99 | test_normal_dir = "datasets/" + vm["dataset"].as() + "/" + vm["test_normal_dir"].as(); 100 | test_normal_paths = Get_Paths(test_normal_dir); 101 | test_normal_data = Get_Data(test_normal_paths, vm["nd"].as()); 102 | 103 | // (4.2) Get Test Anomaly Data 104 | std::string test_anomaly_dir; 105 | std::vector test_anomaly_paths; 106 | std::vector> test_anomaly_data; 107 | /*****************************************************/ 108 | test_anomaly_dir = "datasets/" + vm["dataset"].as() + "/" + vm["test_anomaly_dir"].as(); 109 | test_anomaly_paths = Get_Paths(test_anomaly_dir); 110 | test_anomaly_data = Get_Data(test_anomaly_paths, vm["nd"].as()); 111 | 112 | // (4.3) Test for SVDD 113 | svdd.test(test_normal_data, test_anomaly_data); 114 | std::cout << "//////////////////////////// Test ////////////////////////////" << std::endl; 115 | std::cout << "accuracy-all: " << svdd.accuracy << " (" << svdd.correct_n + svdd.correct_a << "/" << test_normal_data.size() + test_anomaly_data.size() << ")" << std::endl; 116 | std::cout << "accuracy-normal: " << svdd.accuracy_n << " (" << svdd.correct_n << "/" << test_normal_data.size() << ")" << std::endl; 117 | std::cout << "accuracy-anomaly: " << svdd.accuracy_a << " (" << svdd.correct_a << "/" << test_anomaly_data.size() << ")" << std::endl; 118 | std::cout << "AUROC: " << svdd.auroc << std::endl; 119 | std::cout << "//////////////////////////////////////////////////////////////" << std::endl; 120 | 121 | return 0; 122 | } 123 | 124 | 125 | // ------------------------------ 126 | // 2. Collecting Paths Function 127 | // ------------------------------ 128 | void Collect_Paths(const std::string root, const std::string sub, std::vector &paths){ 129 | 130 | fs::path ROOT(root); 131 | 132 | for (auto &p : fs::directory_iterator(ROOT)){ 133 | if (!fs::is_directory(p)){ 134 | std::stringstream rpath; 135 | rpath << p.path().string(); 136 | paths.push_back(rpath.str()); 137 | } 138 | else{ 139 | std::stringstream subsub; 140 | subsub << p.path().filename().string(); 141 | Collect_Paths(root + '/' + subsub.str(), sub + subsub.str() + '/', paths); 142 | } 143 | } 144 | 145 | return; 146 | } 147 | 148 | 149 | // --------------------------- 150 | // 3. Getting Paths Function 151 | // --------------------------- 152 | std::vector Get_Paths(const std::string root){ 153 | 154 | std::vector paths; 155 | 156 | Collect_Paths(root, "", paths); 157 | std::sort(paths.begin(), paths.end()); 158 | 159 | return paths; 160 | 161 | } 162 | 163 | 164 | // --------------------------- 165 | // 4. Getting Data Function 166 | // --------------------------- 167 | std::vector> Get_Data(const std::vector paths, const size_t D){ 168 | 169 | size_t i; 170 | double element; 171 | std::ifstream ifs; 172 | std::vector data_one; 173 | std::vector> data; 174 | 175 | for (std::string path : paths){ 176 | 177 | ifs.open(path); 178 | 179 | data_one = std::vector(D); 180 | for (i = 0; i < D; i++){ 181 | ifs >> element; 182 | data_one[i] = element; 183 | } 184 | data.push_back(data_one); 185 | 186 | ifs.close(); 187 | 188 | } 189 | 190 | return data; 191 | } 192 | 193 | 194 | // ---------------------------- 195 | // 5. Setting Kernel Function 196 | // ---------------------------- 197 | void Set_Kernel(po::variables_map &vm, KernelFunc &K, std::vector ¶ms){ 198 | 199 | if (vm["kernel"].as() == "linear"){ 200 | K = kernel::linear; 201 | } 202 | else if (vm["kernel"].as() == "polynomial"){ 203 | K = kernel::polynomial; 204 | params = {vm["c"].as(), vm["d"].as()}; 205 | } 206 | else if (vm["kernel"].as() == "rbf"){ 207 | K = kernel::rbf; 208 | params = {vm["gamma"].as()}; 209 | } 210 | else{ 211 | std::cerr << "Error : Couldn't match the name of kernel." << std::endl; 212 | std::exit(-1); 213 | } 214 | 215 | return; 216 | 217 | } -------------------------------------------------------------------------------- /SVDD/src/svdd.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | // Original 7 | #include "svdd.hpp" 8 | 9 | 10 | // --------------------------------------- 11 | // namespace{kernel} -> function{linear} 12 | // --------------------------------------- 13 | double kernel::linear(const std::vector x1, const std::vector x2, const std::vector params){ 14 | 15 | size_t i; 16 | double ans; 17 | 18 | if (x1.size() != x2.size()){ 19 | std::cerr << "Error : Couldn't match the number of elements for inner product." << std::endl; 20 | std::exit(-1); 21 | } 22 | 23 | ans = 0.0; 24 | for (i = 0; i < x1.size(); i++){ 25 | ans += x1[i] * x2[i]; 26 | } 27 | 28 | return ans; 29 | 30 | } 31 | 32 | 33 | // ------------------------------------------- 34 | // namespace{kernel} -> function{polynomial} 35 | // ------------------------------------------- 36 | double kernel::polynomial(const std::vector x1, const std::vector x2, const std::vector params){ 37 | 38 | size_t i; 39 | double ans; 40 | 41 | if (x1.size() != x2.size()){ 42 | std::cerr << "Error : Couldn't match the number of elements for inner product." << std::endl; 43 | std::exit(-1); 44 | } 45 | else if (params.size() != 2){ 46 | std::cerr << "Error : Couldn't match the number of hyper-parameters." << std::endl; 47 | std::exit(-1); 48 | } 49 | 50 | ans = 0.0; 51 | for (i = 0; i < x1.size(); i++){ 52 | ans += x1[i] * x2[i]; 53 | } 54 | ans += params[0]; 55 | ans = std::pow(ans, params[1]); 56 | 57 | return ans; 58 | 59 | } 60 | 61 | 62 | // ------------------------------------ 63 | // namespace{kernel} -> function{rbf} 64 | // ------------------------------------ 65 | double kernel::rbf(const std::vector x1, const std::vector x2, const std::vector params){ 66 | 67 | size_t i; 68 | double ans; 69 | 70 | if (x1.size() != x2.size()){ 71 | std::cerr << "Error : Couldn't match the number of elements for inner product." << std::endl; 72 | std::exit(-1); 73 | } 74 | else if (params.size() != 1){ 75 | std::cerr << "Error : Couldn't match the number of hyper-parameters." << std::endl; 76 | std::exit(-1); 77 | } 78 | 79 | ans = 0.0; 80 | for (i = 0; i < x1.size(); i++){ 81 | ans += (x1[i] - x2[i]) * (x1[i] - x2[i]); 82 | } 83 | ans = std::exp(-params[0] * ans); 84 | 85 | return ans; 86 | 87 | } 88 | 89 | 90 | // ---------------------------- 91 | // class{SVDD} -> constructor 92 | // ---------------------------- 93 | SVDD::SVDD(const KernelFunc K_, const std::vector params_, const bool verbose_){ 94 | this->K = K_; 95 | this->params = params_; 96 | this->verbose = verbose_; 97 | } 98 | 99 | 100 | // ------------------------------ 101 | // class{SVDD} -> function{log} 102 | // ------------------------------ 103 | void SVDD::log(const std::string str){ 104 | if (this->verbose){ 105 | std::cout << str << std::flush; 106 | } 107 | return; 108 | } 109 | 110 | 111 | // ------------------------------- 112 | // class{SVDD} -> function{sort} 113 | // ------------------------------- 114 | void SVDD::sort(std::vector> &data){ 115 | 116 | size_t i, j; 117 | std::pair tmp; 118 | 119 | for (i = 1; i < data.size(); i++){ 120 | j = i; 121 | while ((j > 0) && (data[j - 1].first > data[j].first)){ 122 | tmp = data[j - 1]; 123 | data[j - 1] = data[j]; 124 | data[j] = tmp; 125 | j--; 126 | } 127 | } 128 | 129 | return; 130 | } 131 | 132 | 133 | // -------------------------------- 134 | // class{SVDD} -> function{train} 135 | // -------------------------------- 136 | void SVDD::train(const std::vector> x, const size_t D, const double nu, const double lr, const double limit){ 137 | 138 | constexpr double eps = 0.0000001; 139 | 140 | size_t i, j, k; 141 | size_t N, Ns, Ns_out; 142 | bool judge; 143 | double C; 144 | double R2; 145 | double item1, item2, item3, item4; 146 | double delta; 147 | double beta; 148 | double error; 149 | std::vector alpha; 150 | 151 | // (1) Set Lagrange Multiplier and Parameters 152 | N = x.size(); 153 | C = 1.0 / ((double)N * nu); 154 | alpha = std::vector(N, 0.0); 155 | beta = 1.0; 156 | 157 | // (2) Training 158 | this->log("\n"); 159 | this->log("////////////////////////// Training //////////////////////////\n"); 160 | do { 161 | 162 | judge = false; 163 | error = 0.0; 164 | 165 | // (2.1) Update Alpha 166 | for (i = 0; i < N; i++){ 167 | 168 | // Set item 1 169 | item1 = this->K(x[i], x[i], this->params); 170 | 171 | // Set item 2 172 | item2 = 0.0; 173 | for (j = 0; j < N; j++){ 174 | item2 += alpha[j] * this->K(x[i], x[j], this->params); 175 | } 176 | 177 | // Set item 3 178 | item3 = 0.0; 179 | for (j = 0; j < N; j++){ 180 | item3 += alpha[j]; 181 | } 182 | item3 -= 1.0; 183 | 184 | // Set Delta 185 | delta = item1 - 2.0 * item2 - beta * item3; 186 | 187 | // Update 188 | alpha[i] += lr * delta; 189 | if (alpha[i] < 0.0){ 190 | alpha[i] = 0.0; 191 | } 192 | else if (alpha[i] > C){ 193 | alpha[i] = C; 194 | } 195 | else if (std::abs(delta) > limit){ 196 | judge = true; 197 | error += std::abs(delta) - limit; 198 | } 199 | 200 | } 201 | 202 | // (2.2) Update Beta 203 | item4 = 0.0; 204 | for (i = 0; i < N; i++){ 205 | item4 += alpha[i]; 206 | } 207 | item4 -= 1.0; 208 | beta += item4 * item4 / 2.0; 209 | 210 | // (2.3) Output Residual Error 211 | this->log("\rerror: " + std::to_string(error)); 212 | 213 | }while (judge); 214 | this->log("\n"); 215 | this->log("//////////////////////////////////////////////////////////////\n"); 216 | 217 | // (3.1) Description for support vectors 218 | Ns = 0; 219 | Ns_out = 0; 220 | this->xs = std::vector>(); 221 | this->alpha_s = std::vector(); 222 | this->xs_out = std::vector>(); 223 | this->alpha_s_out = std::vector(); 224 | for (i = 0; i < N; i++){ 225 | if ((eps < alpha[i]) && (alpha[i] < C - eps)){ 226 | this->xs.push_back(x[i]); 227 | this->alpha_s.push_back(alpha[i]); 228 | Ns++; 229 | } 230 | else if (alpha[i] >= C - eps){ 231 | this->xs_out.push_back(x[i]); 232 | this->alpha_s_out.push_back(alpha[i]); 233 | Ns_out++; 234 | } 235 | } 236 | this->log("Ns (number of support vectors on hypersphere) = " + std::to_string(Ns) + "\n"); 237 | this->log("Ns_out (number of support vectors outside hypersphere) = " + std::to_string(Ns_out) + "\n"); 238 | 239 | // (3.2) Description for b 240 | this->b = 0.0; 241 | for (i = 0; i < Ns; i++){ 242 | for (j = 0; j < Ns; j++){ 243 | this->b += 2.0 * this->alpha_s[j] * this->K(this->xs[j], this->xs[i], this->params); 244 | } 245 | for (j = 0; j < Ns_out; j++){ 246 | this->b += 2.0 * this->alpha_s_out[j] * this->K(this->xs_out[j], this->xs[i], this->params); 247 | } 248 | this->b -= this->K(this->xs[i], this->xs[i], this->params); 249 | } 250 | this->b /= (double)Ns; 251 | this->log("bias = " + std::to_string(this->b) + "\n"); 252 | 253 | // (3.3) Description for R 254 | this->R = 0.0; 255 | for (k = 0; k < Ns; k++){ 256 | 257 | // (3.3.1) 258 | R2 = this->K(this->xs[k], this->xs[k], this->params); 259 | 260 | // (3.3.2) 261 | for (i = 0; i < Ns; i++){ 262 | R2 -= 2.0 * this->alpha_s[i] * this->K(this->xs[i], this->xs[k], this->params); 263 | } 264 | for (i = 0; i < Ns_out; i++){ 265 | R2 -= 2.0 * this->alpha_s_out[i] * this->K(this->xs_out[i], this->xs[k], this->params); 266 | } 267 | 268 | // (3.3.3) 269 | for (j = 0; j < Ns; j++){ 270 | for (i = 0; i < Ns; i++){ 271 | R2 += this->alpha_s[i] * this->alpha_s[j] * this->K(this->xs[i], this->xs[j], this->params); 272 | } 273 | for (i = 0; i < Ns_out; i++){ 274 | R2 += this->alpha_s_out[i] * this->alpha_s[j] * this->K(this->xs_out[i], this->xs[j], this->params); 275 | } 276 | } 277 | for (j = 0; j < Ns_out; j++){ 278 | for (i = 0; i < Ns; i++){ 279 | R2 += this->alpha_s[i] * this->alpha_s_out[j] * this->K(this->xs[i], this->xs_out[j], this->params); 280 | } 281 | for (i = 0; i < Ns_out; i++){ 282 | R2 += this->alpha_s_out[i] * this->alpha_s_out[j] * this->K(this->xs_out[i], this->xs_out[j], this->params); 283 | } 284 | } 285 | 286 | // (3.3.4) 287 | this->R += std::sqrt(R2); 288 | 289 | } 290 | this->R /= (double)Ns; 291 | this->log("radius = " + std::to_string(this->R) + "\n"); 292 | this->log("//////////////////////////////////////////////////////////////\n\n"); 293 | 294 | return; 295 | } 296 | 297 | 298 | // ------------------------------- 299 | // class{SVDD} -> function{test} 300 | // ------------------------------- 301 | void SVDD::test(const std::vector> normal_data, const std::vector> anomaly_data){ 302 | 303 | size_t i; 304 | std::vector> score; 305 | 306 | this->correct_n = 0; 307 | for (i = 0; i < normal_data.size(); i++){ 308 | score.push_back({this->f(normal_data[i]), 1}); 309 | if (this->g(normal_data[i]) == 1){ 310 | this->correct_n++; 311 | } 312 | } 313 | 314 | this->correct_a = 0; 315 | for (i = 0; i < anomaly_data.size(); i++){ 316 | score.push_back({this->f(anomaly_data[i]), -1}); 317 | if (this->g(anomaly_data[i]) == -1){ 318 | this->correct_a++; 319 | } 320 | } 321 | 322 | this->accuracy = (double)(this->correct_n + this->correct_a) / (double)(normal_data.size() + anomaly_data.size()); 323 | this->accuracy_n = (double)this->correct_n / (double)normal_data.size(); 324 | this->accuracy_a = (double)this->correct_a / (double)anomaly_data.size(); 325 | 326 | this->roc(score); 327 | 328 | return; 329 | } 330 | 331 | 332 | // ------------------------------ 333 | // class{SVDD} -> function{roc} 334 | // ------------------------------ 335 | void SVDD::roc(const std::vector> score){ 336 | 337 | size_t i; 338 | size_t Np, Nn; 339 | size_t TP, FP; 340 | double TP_rate, FP_rate; 341 | double pre_TP_rate, pre_FP_rate; 342 | std::vector> score_sorted; 343 | 344 | // (1) Sort (Ascending Order) 345 | score_sorted = score; 346 | this->sort(score_sorted); 347 | 348 | // (2) Set Parameters 349 | Np = 0; 350 | Nn = 0; 351 | for (i = 0; i < score_sorted.size(); i++){ 352 | if (score_sorted[i].second == -1){ 353 | Np++; 354 | } 355 | else{ 356 | Nn++; 357 | } 358 | } 359 | 360 | // (3) Set Parameters 361 | TP = 0; 362 | FP = 0; 363 | TP_rate = 0.0; 364 | FP_rate = 0.0; 365 | pre_TP_rate = 0.0; 366 | pre_FP_rate = 0.0; 367 | this->auroc = 0.0; 368 | 369 | // (4) Anomaly Detection 370 | for (i = 0; i < score_sorted.size() - 1; i++){ 371 | 372 | if (score_sorted[i].second == -1){ 373 | TP++; 374 | } 375 | else{ 376 | FP++; 377 | } 378 | 379 | if (score_sorted[i].first != score_sorted[i + 1].first){ 380 | TP_rate = (double)TP / (double)Np; 381 | FP_rate = (double)FP / (double)Nn; 382 | this->auroc += (TP_rate + pre_TP_rate) * (FP_rate - pre_FP_rate) * 0.5; 383 | pre_TP_rate = TP_rate; 384 | pre_FP_rate = FP_rate; 385 | } 386 | 387 | } 388 | this->auroc += (1.0 + pre_TP_rate) * (1.0 - pre_FP_rate) * 0.5; 389 | 390 | return; 391 | } 392 | 393 | 394 | // ---------------------------- 395 | // class{SVDD} -> function{f} 396 | // ---------------------------- 397 | double SVDD::f(const std::vector x){ 398 | 399 | size_t i; 400 | double ans; 401 | 402 | ans = 0.0; 403 | for (i = 0; i < this->xs.size(); i++){ 404 | ans += this->alpha_s[i] * this->K(this->xs[i], x, this->params); 405 | } 406 | for (i = 0; i < this->xs_out.size(); i++){ 407 | ans += this->alpha_s_out[i] * this->K(this->xs_out[i], x, this->params); 408 | } 409 | ans *= 2.0; 410 | ans -= this->K(x, x, this->params); 411 | ans -= this->b; 412 | 413 | return ans; 414 | } 415 | 416 | 417 | // ---------------------------- 418 | // class{SVDD} -> function{g} 419 | // ---------------------------- 420 | double SVDD::g(const std::vector x){ 421 | 422 | double fx; 423 | int gx; 424 | 425 | fx = this->f(x); 426 | if (fx >= 0.0){ 427 | gx = 1; 428 | } 429 | else{ 430 | gx = -1; 431 | } 432 | 433 | return gx; 434 | } -------------------------------------------------------------------------------- /SVDD/src/svdd.hpp: -------------------------------------------------------------------------------- 1 | #ifndef SVDD_HPP 2 | #define SVDD_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | 10 | // ------------------- 11 | // namespace{kernel} 12 | // ------------------- 13 | namespace kernel{ 14 | double linear(const std::vector x1, const std::vector x2, const std::vector params); 15 | double polynomial(const std::vector x1, const std::vector x2, const std::vector params); 16 | double rbf(const std::vector x1, const std::vector x2, const std::vector params); 17 | } 18 | typedef std::function, const std::vector, const std::vector)> KernelFunc; 19 | 20 | 21 | // ------------- 22 | // class{SVDD} 23 | // ------------- 24 | class SVDD{ 25 | private: 26 | 27 | // member variable 28 | bool verbose; 29 | double b; 30 | double R; 31 | std::vector> xs; 32 | std::vector alpha_s; 33 | std::vector> xs_out; 34 | std::vector alpha_s_out; 35 | /***** kernel *****/ 36 | KernelFunc K; 37 | std::vector params; 38 | /***** kernel *****/ 39 | 40 | // fuction 41 | void log(const std::string str); 42 | void sort(std::vector> &data); 43 | 44 | public: 45 | 46 | // member variable 47 | double accuracy; 48 | double accuracy_n, accuracy_a; 49 | double auroc; 50 | size_t correct_n, correct_a; 51 | 52 | // constructor 53 | SVDD() = delete; 54 | SVDD(const KernelFunc K_=kernel::rbf, const std::vector params_={1.0}, const bool verbose_=true); 55 | 56 | // function 57 | void train(const std::vector> x, const size_t D, const double nu, const double lr, const double limit=0.0001); 58 | void test(const std::vector> normal_data, const std::vector> anomaly_data); 59 | void roc(const std::vector> score); 60 | double f(const std::vector x); 61 | double g(const std::vector x); 62 | 63 | }; 64 | 65 | 66 | #endif -------------------------------------------------------------------------------- /SoftMargin-SVM/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 2 | 3 | # Project Name 4 | project(SoftMargin-SVM) 5 | 6 | # Directory Name 7 | set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) 8 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) 9 | 10 | # Set Compiler Options 11 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -Wall") 12 | if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8) 13 | set(CMAKE_CXX_COMPILER "g++-8") 14 | endif () 15 | 16 | # For OSX 17 | #set(CMAKE_OSX_ARCHITECTURES "x86_64") # Intel cpu 18 | set(CMAKE_OSX_ARCHITECTURES "arm64") # Apple cpu 19 | 20 | # Find Package 21 | find_package(Boost REQUIRED COMPONENTS program_options) 22 | 23 | # Set Include Directories 24 | set(INCLUDE_DIRS 25 | ${Boost_INCLUDE_DIRS} 26 | ${SRC_DIR} 27 | ) 28 | 29 | # Set CXX_LIBRARIES 30 | set(LIBRARIES 31 | ${Boost_LIBRARIES} 32 | ) 33 | 34 | # Set Source Code 35 | set(SRCS 36 | ${SRC_DIR}/main.cpp 37 | ${SRC_DIR}/svm.cpp 38 | ) 39 | 40 | # Link 41 | add_executable(${PROJECT_NAME} ${SRCS}) 42 | include_directories(${INCLUDE_DIRS}) 43 | target_link_libraries(${PROJECT_NAME} ${LIBRARIES}) 44 | set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 17) 45 | 46 | # Display Message 47 | message(STATUS "") 48 | message(STATUS "~~~ Boost Information ~~~") 49 | message(STATUS "${Boost_INCLUDE_DIRS};") 50 | message(STATUS "${Boost_LIBRARIES};") 51 | message(STATUS "") 52 | -------------------------------------------------------------------------------- /SoftMargin-SVM/README.md: -------------------------------------------------------------------------------- 1 | # Soft Margin SVM 2 | 3 | This is the implementation of Soft Margin SVM (Soft Margin Support Vector Machine). 4 | 5 | - Class: 2 6 | - Problem: Linearly Non-separable 7 | - Decision Boundary: Hyperplane 8 | 9 | ## Usage 10 | 11 | ### 1. Build 12 | Please build the source file according to the procedure. 13 | ~~~ 14 | $ mkdir build 15 | $ cd build 16 | $ cmake .. 17 | $ make 18 | $ cd .. 19 | ~~~ 20 | 21 | ### 2. Dataset Setting 22 | 23 | The following hierarchical relationships are recommended. 24 | 25 | ![SoftMargin-SVM_dataset](datasets/dataset.png) 26 | 27 | ### 3. Execution 28 | 29 | The following is an example for Toy Dataset. 30 | 31 | #### Setting 32 | Please set the shell for executable file. 33 | ~~~ 34 | $ vi scripts/toy.sh 35 | ~~~ 36 | If you want to view specific examples of command line arguments, please view "src/main.cpp" or add "--help" to the argument. 37 | ~~~ 38 | #!/bin/bash 39 | 40 | DATA='toy' 41 | 42 | ./SoftMargin-SVM \ 43 | --dataset ${DATA} \ 44 | --nd 2 \ 45 | --C 10.0 \ 46 | --lr 0.0001 47 | ~~~ 48 | 49 | #### Run 50 | Please execute the following to start the program. 51 | ~~~ 52 | $ sh scripts/toy.sh 53 | ~~~ 54 | 55 | ## Formula 56 | 57 | ![SoftMargin-SVM_dual](https://user-images.githubusercontent.com/56967584/130267821-a46ce22f-1acd-4e37-9e14-0269e30b1e00.png) 58 | ![SoftMargin-SVM_obj](https://user-images.githubusercontent.com/56967584/130267828-c346f820-7c96-4b7d-afc9-127d7539b0d5.png) 59 | ![SoftMargin-SVM_delta](https://user-images.githubusercontent.com/56967584/130267833-48e2d2a8-e54f-407d-92e1-db31f85f13f1.png) 60 | ![SoftMargin-SVM_update](https://user-images.githubusercontent.com/56967584/130267837-854ac1fa-3f09-46a8-a030-d406dd96752c.png) 61 | ![SoftMargin-SVM_class](https://user-images.githubusercontent.com/56967584/130281866-5e8209af-89c0-4ff1-b686-47256a5461fe.png) 62 | 63 | 64 | ## Algorithm 65 | ![train](https://user-images.githubusercontent.com/56967584/130327310-f7a7d992-970e-4026-9ac0-6ac61129ca5c.png) 66 | ![test](https://user-images.githubusercontent.com/56967584/130327313-baf9e738-8918-49ea-857a-65f72a23f491.png) 67 | 68 | -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/koba-jon/svm_cpp/5fa1b0951740b249a7b7bb170031b75470dbf090/SoftMargin-SVM/datasets/dataset.png -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/test/class1/data1.txt: -------------------------------------------------------------------------------- 1 | 0.3 2 | 0.4 3 | -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/test/class1/data2.txt: -------------------------------------------------------------------------------- 1 | 0.2 2 | 0.49 3 | -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/test/class1/data3.txt: -------------------------------------------------------------------------------- 1 | 0.08 2 | 0.32 3 | -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/test/class1/data4.txt: -------------------------------------------------------------------------------- 1 | 0.49 2 | 0.49 3 | -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/test/class1/data5.txt: -------------------------------------------------------------------------------- 1 | 0.12 2 | 0.4 3 | -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/test/class1/data6.txt: -------------------------------------------------------------------------------- 1 | 0.4999999999999 2 | 0.4999999999999 -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/test/class2/data1.txt: -------------------------------------------------------------------------------- 1 | 0.9 2 | 0.8 3 | -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/test/class2/data2.txt: -------------------------------------------------------------------------------- 1 | 0.51 2 | 0.7 -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/test/class2/data3.txt: -------------------------------------------------------------------------------- 1 | 0.62 2 | 0.88 3 | -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/test/class2/data4.txt: -------------------------------------------------------------------------------- 1 | 0.7 2 | 0.55 -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/test/class2/data5.txt: -------------------------------------------------------------------------------- 1 | 0.51 2 | 0.51 3 | -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/test/class2/data6.txt: -------------------------------------------------------------------------------- 1 | 0.5000000000001 2 | 0.5000000000001 -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/train/class1/data1.txt: -------------------------------------------------------------------------------- 1 | 0.6 2 | 0.7 3 | -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/train/class1/data2.txt: -------------------------------------------------------------------------------- 1 | 0.1 2 | 0.4 3 | -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/train/class1/data3.txt: -------------------------------------------------------------------------------- 1 | 0.48 2 | 0.22 3 | -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/train/class1/data4.txt: -------------------------------------------------------------------------------- 1 | 0.4 2 | 0.05 3 | -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/train/class1/data5.txt: -------------------------------------------------------------------------------- 1 | 0.02 2 | 0.3 3 | -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/train/class2/data1.txt: -------------------------------------------------------------------------------- 1 | 0.4 2 | 0.3 3 | -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/train/class2/data2.txt: -------------------------------------------------------------------------------- 1 | 0.9 2 | 0.6 -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/train/class2/data3.txt: -------------------------------------------------------------------------------- 1 | 0.52 2 | 0.78 3 | -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/train/class2/data4.txt: -------------------------------------------------------------------------------- 1 | 0.6 2 | 0.95 -------------------------------------------------------------------------------- /SoftMargin-SVM/datasets/toy/train/class2/data5.txt: -------------------------------------------------------------------------------- 1 | 0.98 2 | 0.7 3 | -------------------------------------------------------------------------------- /SoftMargin-SVM/scripts/toy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DATA='toy' 4 | 5 | ./SoftMargin-SVM \ 6 | --dataset ${DATA} \ 7 | --nd 2 \ 8 | --C 10.0 \ 9 | --lr 0.0001 -------------------------------------------------------------------------------- /SoftMargin-SVM/src/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | // 3rd-Party Libraries 9 | #include 10 | // Original 11 | #include "svm.hpp" 12 | 13 | // Define Namespace 14 | namespace fs = std::filesystem; 15 | namespace po = boost::program_options; 16 | 17 | // Function Prototype 18 | void Collect_Paths(const std::string root, const std::string sub, std::vector &paths); 19 | std::vector Get_Paths(const std::string root); 20 | std::vector> Get_Data(const std::vector paths, const size_t D); 21 | 22 | 23 | // ---------------------- 24 | // 0. Argument Function 25 | // ---------------------- 26 | po::options_description parse_arguments(){ 27 | 28 | po::options_description args("Options", 200, 30); 29 | 30 | args.add_options() 31 | 32 | // (1) Define for General Parameter 33 | ("help", "produce help message") 34 | ("dataset", po::value()->default_value("toy"), "dataset name") 35 | ("nd", po::value()->default_value(2), "number of dimensions") 36 | ("verbose", po::value()->default_value(true), "verbose") 37 | 38 | // (2) Define for Training 39 | ("train_class1_dir", po::value()->default_value("train/class1"), "training class 1 directory : ./datasets///") 40 | ("train_class2_dir", po::value()->default_value("train/class2"), "training class 2 directory : ./datasets///") 41 | ("C", po::value()->default_value(1.0), "regularization parameter (C > 0.0)") 42 | ("lr", po::value()->default_value(0.0001), "learning rate of alpha") 43 | 44 | // (3) Define for Test 45 | ("test_class1_dir", po::value()->default_value("test/class1"), "test class 1 directory : ./datasets///") 46 | ("test_class2_dir", po::value()->default_value("test/class2"), "test class 2 directory : ./datasets///") 47 | 48 | ; 49 | 50 | return args; 51 | 52 | } 53 | 54 | 55 | // ------------------ 56 | // 1. Main Function 57 | // ------------------ 58 | int main(int argc, const char *argv[]){ 59 | 60 | // (1) Extract Arguments 61 | po::options_description args = parse_arguments(); 62 | po::variables_map vm{}; 63 | po::store(po::parse_command_line(argc, argv, args), vm); 64 | po::notify(vm); 65 | if (vm.empty() || vm.count("help")){ 66 | std::cout << args << std::endl; 67 | return 1; 68 | } 69 | 70 | // (2.1) Get Training Data for class 1 71 | std::string train_class1_dir; 72 | std::vector train_class1_paths; 73 | std::vector> train_class1_data; 74 | /*****************************************************/ 75 | train_class1_dir = "datasets/" + vm["dataset"].as() + "/" + vm["train_class1_dir"].as(); 76 | train_class1_paths = Get_Paths(train_class1_dir); 77 | train_class1_data = Get_Data(train_class1_paths, vm["nd"].as()); 78 | 79 | // (2.2) Get Training Data for class 2 80 | std::string train_class2_dir; 81 | std::vector train_class2_paths; 82 | std::vector> train_class2_data; 83 | /*****************************************************/ 84 | train_class2_dir = "datasets/" + vm["dataset"].as() + "/" + vm["train_class2_dir"].as(); 85 | train_class2_paths = Get_Paths(train_class2_dir); 86 | train_class2_data = Get_Data(train_class2_paths, vm["nd"].as()); 87 | 88 | // (2.3) Training for SVM 89 | SoftMargin_SVM svm(vm["verbose"].as()); 90 | svm.train(train_class1_data, train_class2_data, vm["nd"].as(), vm["C"].as(), vm["lr"].as()); 91 | 92 | // (3.1) Get Test Data for class 1 93 | std::string test_class1_dir; 94 | std::vector test_class1_paths; 95 | std::vector> test_class1_data; 96 | /*****************************************************/ 97 | test_class1_dir = "datasets/" + vm["dataset"].as() + "/" + vm["test_class1_dir"].as(); 98 | test_class1_paths = Get_Paths(test_class1_dir); 99 | test_class1_data = Get_Data(test_class1_paths, vm["nd"].as()); 100 | 101 | // (3.2) Get Test Data for class 2 102 | std::string test_class2_dir; 103 | std::vector test_class2_paths; 104 | std::vector> test_class2_data; 105 | /*****************************************************/ 106 | test_class2_dir = "datasets/" + vm["dataset"].as() + "/" + vm["test_class2_dir"].as(); 107 | test_class2_paths = Get_Paths(test_class2_dir); 108 | test_class2_data = Get_Data(test_class2_paths, vm["nd"].as()); 109 | 110 | // (3.3) Test for SVM 111 | svm.test(test_class1_data, test_class2_data); 112 | std::cout << "///////////////////////// Test /////////////////////////" << std::endl; 113 | std::cout << "accuracy-all: " << svm.accuracy << " (" << svm.correct_c1 + svm.correct_c2 << "/" << test_class1_data.size() + test_class2_data.size() << ")" << std::endl; 114 | std::cout << "accuracy-class1: " << svm.accuracy_c1 << " (" << svm.correct_c1 << "/" << test_class1_data.size() << ")" << std::endl; 115 | std::cout << "accuracy-class2: " << svm.accuracy_c2 << " (" << svm.correct_c2 << "/" << test_class2_data.size() << ")" << std::endl; 116 | std::cout << "////////////////////////////////////////////////////////" << std::endl; 117 | 118 | return 0; 119 | } 120 | 121 | 122 | // ------------------------------ 123 | // 2. Collecting Paths Function 124 | // ------------------------------ 125 | void Collect_Paths(const std::string root, const std::string sub, std::vector &paths){ 126 | 127 | fs::path ROOT(root); 128 | 129 | for (auto &p : fs::directory_iterator(ROOT)){ 130 | if (!fs::is_directory(p)){ 131 | std::stringstream rpath; 132 | rpath << p.path().string(); 133 | paths.push_back(rpath.str()); 134 | } 135 | else{ 136 | std::stringstream subsub; 137 | subsub << p.path().filename().string(); 138 | Collect_Paths(root + '/' + subsub.str(), sub + subsub.str() + '/', paths); 139 | } 140 | } 141 | 142 | return; 143 | } 144 | 145 | 146 | // --------------------------- 147 | // 3. Getting Paths Function 148 | // --------------------------- 149 | std::vector Get_Paths(const std::string root){ 150 | 151 | std::vector paths; 152 | 153 | Collect_Paths(root, "", paths); 154 | std::sort(paths.begin(), paths.end()); 155 | 156 | return paths; 157 | 158 | } 159 | 160 | 161 | // --------------------------- 162 | // 4. Getting Data Function 163 | // --------------------------- 164 | std::vector> Get_Data(const std::vector paths, const size_t D){ 165 | 166 | size_t i; 167 | double element; 168 | std::ifstream ifs; 169 | std::vector data_one; 170 | std::vector> data; 171 | 172 | for (std::string path : paths){ 173 | 174 | ifs.open(path); 175 | 176 | data_one = std::vector(D); 177 | for (i = 0; i < D; i++){ 178 | ifs >> element; 179 | data_one[i] = element; 180 | } 181 | data.push_back(data_one); 182 | 183 | ifs.close(); 184 | 185 | } 186 | 187 | return data; 188 | } -------------------------------------------------------------------------------- /SoftMargin-SVM/src/svm.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | // Original 6 | #include "svm.hpp" 7 | 8 | 9 | // -------------------------------------- 10 | // class{SoftMargin_SVM} -> constructor 11 | // -------------------------------------- 12 | SoftMargin_SVM::SoftMargin_SVM(const bool verbose_){ 13 | this->verbose = verbose_; 14 | } 15 | 16 | 17 | // ---------------------------------------- 18 | // class{SoftMargin_SVM} -> function{dot} 19 | // ---------------------------------------- 20 | double SoftMargin_SVM::dot(const std::vector x1, const std::vector x2){ 21 | 22 | size_t i; 23 | double ans; 24 | 25 | if (x1.size() != x2.size()){ 26 | std::cerr << "Error : Couldn't match the number of elements for inner product." << std::endl; 27 | std::exit(-1); 28 | } 29 | 30 | ans = 0.0; 31 | for (i = 0; i < x1.size(); i++){ 32 | ans += x1[i] * x2[i]; 33 | } 34 | 35 | return ans; 36 | 37 | } 38 | 39 | 40 | // ---------------------------------------- 41 | // class{SoftMargin_SVM} -> function{log} 42 | // ---------------------------------------- 43 | void SoftMargin_SVM::log(const std::string str){ 44 | if (this->verbose){ 45 | std::cout << str << std::flush; 46 | } 47 | return; 48 | } 49 | 50 | 51 | // ------------------------------------------ 52 | // class{SoftMargin_SVM} -> function{train} 53 | // ------------------------------------------ 54 | void SoftMargin_SVM::train(const std::vector> class1_data, const std::vector> class2_data, const size_t D, const double C, const double lr, const double limit){ 55 | 56 | constexpr double eps = 0.0000001; 57 | 58 | size_t i, j; 59 | size_t N, Ns, Ns_in; 60 | bool judge; 61 | double item1, item2, item3; 62 | double delta; 63 | double beta; 64 | double error; 65 | std::vector> x; 66 | std::vector y; 67 | std::vector alpha; 68 | 69 | // (1.1) Set class 1 data 70 | for (i = 0; i < class1_data.size(); i++){ 71 | x.push_back(class1_data[i]); 72 | y.push_back(1); 73 | } 74 | 75 | // (1.2) Set class 2 data 76 | for (i = 0; i < class2_data.size(); i++){ 77 | x.push_back(class2_data[i]); 78 | y.push_back(-1); 79 | } 80 | 81 | // (2) Set Lagrange Multiplier and Parameters 82 | N = x.size(); 83 | alpha = std::vector(N, 0.0); 84 | beta = 1.0; 85 | 86 | // (3) Training 87 | this->log("\n"); 88 | this->log("/////////////////////// Training ///////////////////////\n"); 89 | do { 90 | 91 | judge = false; 92 | error = 0.0; 93 | 94 | // (3.1) Update Alpha 95 | for (i = 0; i < N; i++){ 96 | 97 | // Set item 1 98 | item1 = 0.0; 99 | for (j = 0; j < N; j++){ 100 | item1 += alpha[j] * (double)y[i] * (double)y[j] * this->dot(x[i], x[j]); 101 | } 102 | 103 | // Set item 2 104 | item2 = 0.0; 105 | for (j = 0; j < N; j++){ 106 | item2 += alpha[j] * (double)y[i] * (double)y[j]; 107 | } 108 | 109 | // Set Delta 110 | delta = 1.0 - item1 - beta * item2; 111 | 112 | // Update 113 | alpha[i] += lr * delta; 114 | if (alpha[i] < 0.0){ 115 | alpha[i] = 0.0; 116 | } 117 | else if (alpha[i] > C){ 118 | alpha[i] = C; 119 | } 120 | else if (std::abs(delta) > limit){ 121 | judge = true; 122 | error += std::abs(delta) - limit; 123 | } 124 | 125 | } 126 | 127 | // (3.2) Update Beta 128 | item3 = 0.0; 129 | for (i = 0; i < N; i++){ 130 | item3 += alpha[i] * (double)y[i]; 131 | } 132 | beta += item3 * item3 / 2.0; 133 | 134 | // (3.3) Output Residual Error 135 | this->log("\rerror: " + std::to_string(error)); 136 | 137 | }while (judge); 138 | this->log("\n"); 139 | this->log("////////////////////////////////////////////////////////\n"); 140 | 141 | // (4.1) Description for support vectors 142 | Ns = 0; 143 | Ns_in = 0; 144 | this->xs = std::vector>(); 145 | this->ys = std::vector(); 146 | this->alpha_s = std::vector(); 147 | this->xs_in = std::vector>(); 148 | this->ys_in = std::vector(); 149 | this->alpha_s_in = std::vector(); 150 | for (i = 0; i < N; i++){ 151 | if ((eps < alpha[i]) && (alpha[i] < C - eps)){ 152 | this->xs.push_back(x[i]); 153 | this->ys.push_back(y[i]); 154 | this->alpha_s.push_back(alpha[i]); 155 | Ns++; 156 | } 157 | else if (alpha[i] >= C - eps){ 158 | this->xs_in.push_back(x[i]); 159 | this->ys_in.push_back(y[i]); 160 | this->alpha_s_in.push_back(alpha[i]); 161 | Ns_in++; 162 | } 163 | } 164 | this->log("Ns (number of support vectors on margin) = " + std::to_string(Ns) + "\n"); 165 | this->log("Ns_in (number of support vectors inside margin) = " + std::to_string(Ns_in) + "\n"); 166 | 167 | // (4.2) Description for w 168 | this->log("weight = [ "); 169 | this->w = std::vector(D, 0.0); 170 | for (j = 0; j < D; j++){ 171 | for (i = 0; i < Ns; i++){ 172 | this->w[j] += alpha_s[i] * (double)ys[i] * xs[i][j]; 173 | } 174 | for (i = 0; i < Ns_in; i++){ 175 | this->w[j] += alpha_s_in[i] * (double)ys_in[i] * xs_in[i][j]; 176 | } 177 | this->log(std::to_string(this->w[j]) + " "); 178 | } 179 | this->log("]\n"); 180 | 181 | // (4.3) Description for b 182 | this->b = 0.0; 183 | for (i = 0; i < Ns; i++){ 184 | this->b += (double)this->ys[i] - this->dot(this->w, this->xs[i]); 185 | } 186 | this->b /= (double)Ns; 187 | this->log("bias = " + std::to_string(this->b) + "\n"); 188 | this->log("////////////////////////////////////////////////////////\n\n"); 189 | 190 | return; 191 | } 192 | 193 | 194 | // ----------------------------------------- 195 | // class{SoftMargin_SVM} -> function{test} 196 | // ----------------------------------------- 197 | void SoftMargin_SVM::test(const std::vector> class1_data, const std::vector> class2_data){ 198 | 199 | size_t i; 200 | 201 | this->correct_c1 = 0; 202 | for (i = 0; i < class1_data.size(); i++){ 203 | if (this->g(class1_data[i]) == 1){ 204 | this->correct_c1++; 205 | } 206 | } 207 | 208 | this->correct_c2 = 0; 209 | for (i = 0; i < class2_data.size(); i++){ 210 | if (this->g(class2_data[i]) == -1){ 211 | this->correct_c2++; 212 | } 213 | } 214 | 215 | this->accuracy = (double)(this->correct_c1 + this->correct_c2) / (double)(class1_data.size() + class2_data.size()); 216 | this->accuracy_c1 = (double)this->correct_c1 / (double)class1_data.size(); 217 | this->accuracy_c2 = (double)this->correct_c2 / (double)class2_data.size(); 218 | 219 | return; 220 | } 221 | 222 | 223 | // -------------------------------------- 224 | // class{SoftMargin_SVM} -> function{f} 225 | // -------------------------------------- 226 | double SoftMargin_SVM::f(const std::vector x){ 227 | return this->dot(this->w, x) + this->b; 228 | } 229 | 230 | 231 | // -------------------------------------- 232 | // class{SoftMargin_SVM} -> function{g} 233 | // -------------------------------------- 234 | double SoftMargin_SVM::g(const std::vector x){ 235 | 236 | double fx; 237 | int gx; 238 | 239 | fx = this->f(x); 240 | if (fx >= 0.0){ 241 | gx = 1; 242 | } 243 | else{ 244 | gx = -1; 245 | } 246 | 247 | return gx; 248 | } -------------------------------------------------------------------------------- /SoftMargin-SVM/src/svm.hpp: -------------------------------------------------------------------------------- 1 | #ifndef SVM_HPP 2 | #define SVM_HPP 3 | 4 | #include 5 | #include 6 | 7 | 8 | // ---------------------- 9 | // class{SoftMargin_SVM} 10 | // ---------------------- 11 | class SoftMargin_SVM{ 12 | private: 13 | 14 | // member variable 15 | bool verbose; 16 | std::vector w; 17 | double b; 18 | std::vector> xs; 19 | std::vector ys; 20 | std::vector alpha_s; 21 | std::vector> xs_in; 22 | std::vector ys_in; 23 | std::vector alpha_s_in; 24 | 25 | // fuction 26 | double dot(const std::vector x1, const std::vector x2); 27 | void log(const std::string str); 28 | 29 | public: 30 | 31 | // member variable 32 | double accuracy; 33 | double accuracy_c1, accuracy_c2; 34 | size_t correct_c1, correct_c2; 35 | 36 | // constructor 37 | SoftMargin_SVM() = delete; 38 | SoftMargin_SVM(const bool verbose_=true); 39 | 40 | // function 41 | void train(const std::vector> class1_data, const std::vector> class2_data, const size_t D, const double C, const double lr, const double limit=0.0001); 42 | void test(const std::vector> class1_data, const std::vector> class2_data); 43 | double f(const std::vector x); 44 | double g(const std::vector x); 45 | 46 | }; 47 | 48 | 49 | #endif --------------------------------------------------------------------------------