├── .gitignore ├── FastSparse ├── .Rbuildignore ├── .gitignore ├── DESCRIPTION ├── LICENSE ├── NAMESPACE ├── R │ ├── FastSparse.R │ ├── RcppExports.R │ ├── coef.R │ ├── cvfit.R │ ├── fit.R │ ├── genhighcorr.R │ ├── gensynthetic.R │ ├── plot.R │ ├── predict.R │ └── print.R ├── Read-and-delete-me ├── cleanup ├── configure ├── configure.ac ├── man │ ├── FastSparse-package.Rd │ ├── FastSparse.cvfit.Rd │ ├── FastSparse.fit.Rd │ ├── GenSynthetic.Rd │ ├── GenSyntheticHighCorr.Rd │ ├── coef.FastSparse.Rd │ ├── plot.FastSparse.Rd │ ├── plot.FastSparseCV.Rd │ ├── predict.FastSparse.Rd │ └── print.FastSparse.Rd └── src │ ├── BetaVector.cpp │ ├── CDL012ExponentialSwaps.cpp │ ├── CDL012LogisticSwaps.cpp │ ├── CDL012SquaredHingeSwaps.cpp │ ├── CDL012Swaps.cpp │ ├── Grid.cpp │ ├── Grid1D.cpp │ ├── Grid2D.cpp │ ├── Interface.cpp │ ├── Makevars │ ├── Makevars.in │ ├── Makevars.win │ ├── Normalize.cpp │ ├── RcppExports.cpp │ ├── Test_Interface.cpp │ ├── include │ ├── BetaVector.h │ ├── CD.h │ ├── CDL0.h │ ├── CDL012.h │ ├── CDL012Exponential.h │ ├── CDL012ExponentialSwaps.h │ ├── CDL012Logistic.h │ ├── CDL012LogisticSwaps.h │ ├── CDL012SquaredHinge.h │ ├── CDL012SquaredHingeSwaps.h │ ├── CDL012Swaps.h │ ├── CDSwaps.h │ ├── FitResult.h │ ├── Grid.h │ ├── Grid1D.h │ ├── Grid2D.h │ ├── GridParams.h │ ├── Interface.h │ ├── MakeCD.h │ ├── Model.h │ ├── Normalize.h │ ├── Params.h │ ├── Test_Interface.h │ └── utils.h │ ├── profile.cpp │ └── utils.cpp ├── GOVERNANCE.md ├── LICENSE ├── MAINTAINERS.md ├── README.md ├── Unified_API └── README.md ├── application_and_usage_R_interface └── README.md ├── application_and_usage_python_interface └── README.md ├── environment.yml ├── experiments ├── README.md ├── comp_baselines.R ├── comp_time.R ├── run_baselines.R ├── run_time.R ├── simulation.R └── utils.R ├── installation └── README.md └── step_function_visualization ├── README.md ├── binarization_utils.py ├── binarize_continuousData_demo.ipynb ├── fico_bin_columnNames.csv ├── plot_stepFunction_demo.ipynb ├── plot_utils.py ├── results ├── fico.txt ├── fico.txt_fastsparse Exponential_2_5_0.001_coeff ├── fico.txt_fastsparse Exponential_2_5_0.001_index ├── fico.txt_fastsparse Logistic_2_5_0.001_coeff └── fico.txt_fastsparse Logistic_2_5_0.001_index ├── save_fico_columnNames.py └── spambase_binary.csv /.gitignore: -------------------------------------------------------------------------------- 1 | FastSparse_0.1.0.tar.gz 2 | step_function_visualization/fico_bin.csv 3 | step_function_visualization/spambase_continuous.csv 4 | usingRInsidePython.ipynb 5 | 6 | #################### notebook gitignore template 7 | # copied from https://github.com/jupyter/notebook/blob/main/.gitignore 8 | 9 | *.bundle.* 10 | lib/ 11 | node_modules/ 12 | *.egg-info/ 13 | .ipynb_checkpoints 14 | *.tsbuildinfo 15 | 16 | # Created by https://www.gitignore.io/api/python 17 | # Edit at https://www.gitignore.io/?templates=python 18 | 19 | ### Python ### 20 | # Byte-compiled / optimized / DLL files 21 | __pycache__/ 22 | *.py[cod] 23 | *$py.class 24 | 25 | # C extensions 26 | *.so 27 | 28 | # Distribution / packaging 29 | .Python 30 | build/ 31 | develop-eggs/ 32 | dist/ 33 | downloads/ 34 | eggs/ 35 | .eggs/ 36 | lib/ 37 | lib64/ 38 | parts/ 39 | sdist/ 40 | var/ 41 | wheels/ 42 | pip-wheel-metadata/ 43 | share/python-wheels/ 44 | .installed.cfg 45 | *.egg 46 | MANIFEST 47 | 48 | # PyInstaller 49 | # Usually these files are written by a python script from a template 50 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 51 | *.manifest 52 | *.spec 53 | 54 | # Installer logs 55 | pip-log.txt 56 | pip-delete-this-directory.txt 57 | 58 | # Unit test / coverage reports 59 | htmlcov/ 60 | .tox/ 61 | .nox/ 62 | .coverage 63 | .coverage.* 64 | .cache 65 | nosetests.xml 66 | coverage.xml 67 | *.cover 68 | .hypothesis/ 69 | .pytest_cache/ 70 | 71 | # Translations 72 | *.mo 73 | *.pot 74 | 75 | # Scrapy stuff: 76 | .scrapy 77 | 78 | # Sphinx documentation 79 | docs/_build/ 80 | 81 | # PyBuilder 82 | target/ 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # celery beat schedule file 88 | celerybeat-schedule 89 | 90 | # SageMath parsed files 91 | *.sage.py 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # Mr Developer 101 | .mr.developer.cfg 102 | .project 103 | .pydevproject 104 | 105 | # mkdocs documentation 106 | /site 107 | 108 | # mypy 109 | .mypy_cache/ 110 | .dmypy.json 111 | dmypy.json 112 | 113 | # Pyre type checker 114 | .pyre/ 115 | 116 | # OS X stuff 117 | *.DS_Store 118 | 119 | # End of https://www.gitignore.io/api/python 120 | 121 | _temp_extension 122 | junit.xml 123 | [uU]ntitled* 124 | notebook/static/* 125 | !notebook/static/favicons 126 | notebook/labextension 127 | notebook/schemas 128 | docs/source/changelog.md 129 | docs/source/contributing.md 130 | 131 | # playwright 132 | ui-tests/test-results 133 | ui-tests/playwright-report 134 | 135 | # VSCode 136 | .vscode 137 | -------------------------------------------------------------------------------- /FastSparse/.Rbuildignore: -------------------------------------------------------------------------------- 1 | ^.*\.Rproj$ 2 | ^\.Rproj\.user$ 3 | -------------------------------------------------------------------------------- /FastSparse/.gitignore: -------------------------------------------------------------------------------- 1 | ######################### my own .gitignore 2 | autom4te.cache/ 3 | config.* 4 | 5 | ######################### R 6 | # History files 7 | .Rhistory 8 | .Rapp.history 9 | 10 | # Session Data files 11 | .RData 12 | .RDataTmp 13 | 14 | # User-specific files 15 | .Ruserdata 16 | 17 | # Example code in package build process 18 | *-Ex.R 19 | 20 | # Output files from R CMD build 21 | /*.tar.gz 22 | 23 | # Output files from R CMD check 24 | /*.Rcheck/ 25 | 26 | # RStudio files 27 | .Rproj.user/ 28 | 29 | # produced vignettes 30 | vignettes/*.html 31 | vignettes/*.pdf 32 | 33 | # OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3 34 | .httr-oauth 35 | 36 | # knitr and R markdown default cache directories 37 | *_cache/ 38 | /cache/ 39 | 40 | # Temporary files created by R markdown 41 | *.utf8.md 42 | *.knit.md 43 | 44 | # R Environment Variables 45 | .Renviron 46 | 47 | # pkgdown site 48 | docs/ 49 | 50 | # translation temp files 51 | po/*~ 52 | 53 | # RStudio Connect folder 54 | rsconnect/ 55 | 56 | ######################### Rcpp 57 | inst/lib 58 | .Rproj.user 59 | src/*.o 60 | src/*.so 61 | src/*.dll 62 | src/symbols.rds 63 | .Rhistory 64 | .RData 65 | .DS_Store 66 | 67 | ## QtCreator 68 | Rcpp.pro 69 | Rcpp.pro.user 70 | *.autosave 71 | .#* 72 | 73 | *.tar.gz 74 | 75 | vignettes/*_cache 76 | 77 | ## GNU global 78 | GPATH 79 | GRTAGS 80 | GTAGS 81 | 82 | ## 83 | local/ 84 | 85 | ## docker helpers 86 | docker/*sh 87 | docker/*/*.sh 88 | 89 | ## Emacs 90 | *~ -------------------------------------------------------------------------------- /FastSparse/DESCRIPTION: -------------------------------------------------------------------------------- 1 | Package: FastSparse 2 | Type: Package 3 | Title: Fast Sparse Classification for Generalized Linear and Additive Models 4 | Version: 0.1.0 5 | Date: 2022-02-17 6 | Author: @R: c( 7 | person("Jiachang", "Liu", email = "jiachang.liu@duke.edu", role = c("aut", "cre")), 8 | person("Chudi", "Zhong", email = "chudi.zhong@duke.edu", role = c("aut")), 9 | person("Margo", "Seltzer", email = "mseltzer@cs.ubc.ca", role = "aut"), 10 | person("Cynthia", "Rudin", email = "cynthia@cs.duke.edu", role = "aut")) 11 | Maintainer: Jiachang Liu 12 | Description: We provide a toolkit for producing sparse and interpretable generalized linear and additive models for the binary classiciation task by solving the L0-regularized problems. 13 | The classiciation loss can be either the logistic loss or the exponential loss. 14 | The algorithms can produce high quality (swap 1-OPT) solutions and are generally 2 to 5 times faster than previous approaches. 15 | We propose several techniques to achieve this computational speedup. 16 | For fast sparse logistic regression, we propose to use linear and quadratic surrogate cuts that allow us to efficiently screen features for elimination, as well as use of a priority queue that favors a more uniform exploration of features. 17 | As an alterantive to the logistic loss, we propose the exponential loss, which permits an analytical solution to the line search at each iteration. 18 | For more details, check the paper Fast Sparse Classification for Generalized Linear and Additive Models by Liu, Zhong, Seltzer, and Rudin (AISTATS 2022). 19 | URL: https://github.com/jiachangliu/fastSparse 20 | License: MIT + file LICENSE 21 | Depends: R (>= 3.3.0) 22 | SystemRequirements: C++11 23 | Imports: Rcpp (>= 1.0.8), Matrix, methods, ggplot2, reshape2, MASS 24 | LinkingTo: Rcpp, RcppArmadillo 25 | NeedsCompilation: yes 26 | RoxygenNote: 7.1.2 27 | -------------------------------------------------------------------------------- /FastSparse/LICENSE: -------------------------------------------------------------------------------- 1 | YEAR: 2022 2 | COPYRIGHT HOLDER: Jiachang Liu -------------------------------------------------------------------------------- /FastSparse/NAMESPACE: -------------------------------------------------------------------------------- 1 | S3method(coef,FastSparse) 2 | S3method(coef,FastSparseCV) 3 | S3method(plot,FastSparse) 4 | S3method(plot,FastSparseCV) 5 | S3method(predict,FastSparse) 6 | S3method(predict,FastSparseCV) 7 | S3method(print,FastSparse) 8 | S3method(print,FastSparseCV) 9 | export(GenSynthetic) 10 | export(GenSyntheticHighCorr) 11 | export(FastSparse.cvfit) 12 | export(FastSparse.fit) 13 | import(Matrix) 14 | import(ggplot2) 15 | importFrom(MASS,mvrnorm) 16 | importFrom(Rcpp,cppFunction) 17 | importFrom(Rcpp,evalCpp) 18 | importFrom(methods,as) 19 | importFrom(methods,is) 20 | importFrom(reshape2,melt) 21 | importFrom(stats,rnorm) 22 | useDynLib(FastSparse) -------------------------------------------------------------------------------- /FastSparse/R/FastSparse.R: -------------------------------------------------------------------------------- 1 | #' @docType package 2 | #' @name FastSparse-package 3 | #' @title A package for L0-regularized learning 4 | #' 5 | #' @description FastSparse fits regularization paths for L0-regularized regression and classification problems. Specifically, 6 | #' it can solve either one of the following problems over a grid of \eqn{\lambda} and \eqn{\gamma} values: 7 | #' \deqn{\min_{\beta_0, \beta} \sum_{i=1}^{n} \ell(y_i, \beta_0+ \langle x_i, \beta \rangle) + \lambda ||\beta||_0 \quad \quad (L0)} 8 | #' \deqn{\min_{\beta_0, \beta} \sum_{i=1}^{n} \ell(y_i, \beta_0+ \langle x_i, \beta \rangle) + \lambda ||\beta||_0 + \gamma||\beta||_1 \quad (L0L1)} 9 | #' \deqn{\min_{\beta_0, \beta} \sum_{i=1}^{n} \ell(y_i, \beta_0+ \langle x_i, \beta \rangle) + \lambda ||\beta||_0 + \gamma||\beta||_2^2 \quad (L0L2)} 10 | #' where \eqn{\ell} is the loss function. We currently support regression using squared error loss and classification using either logistic loss or squared hinge loss. 11 | #' Pathwise optimization can be done using either cyclic coordinate descent (CD) or local combinatorial search. The core of the toolkit is implemented in C++ and employs 12 | #' many computational tricks and heuristics, leading to competitive running times. CD runs very fast and typically 13 | #' leads to relatively good solutions. Local combinatorial search can find higher-quality solutions (at the 14 | #' expense of increased running times). 15 | #' The toolkit has the following six main methods: 16 | #' \itemize{ 17 | #' \item{\code{\link{FastSparse.fit}}: }{Fits an L0-regularized model.} 18 | #' \item{\code{\link{FastSparse.cvfit}}: }{Performs k-fold cross-validation.} 19 | #' \item{\code{\link[=print.FastSparse]{print}}: }{Prints a summary of the path.} 20 | #' \item{\code{\link[=coef.FastSparse]{coef}}: }{Extracts solutions(s) from the path.} 21 | #' \item{\code{\link[=predict.FastSparse]{predict}}: }{Predicts response using a solution in the path.} 22 | #' \item{\code{\link[=plot.FastSparse]{plot}}: }{Plots the regularization path or cross-validation error.} 23 | #' } 24 | #' @references Hazimeh and Mazumder. Fast Best Subset Selection: Coordinate Descent and Local Combinatorial 25 | #' Optimization Algorithms. Operations Research (2020). \url{https://pubsonline.informs.org/doi/10.1287/opre.2019.1919}. 26 | NULL 27 | -------------------------------------------------------------------------------- /FastSparse/R/RcppExports.R: -------------------------------------------------------------------------------- 1 | # Generated by using Rcpp::compileAttributes() -> do not edit by hand 2 | # Generator token: 10BE3573-1514-4C36-9D1C-5A225CD40393 3 | 4 | FastSparseFit_sparse <- function(X, y, Loss, Penalty, Algorithm, NnzStopNum, G_ncols, G_nrows, Lambda2Max, Lambda2Min, PartialSort, MaxIters, rtol, atol, ActiveSet, ActiveSetNum, MaxNumSwaps, ScaleDownFactor, ScreenSize, LambdaU, Lambdas, ExcludeFirstK, Intercept, withBounds, Lows, Highs) { 5 | .Call('_FastSparse_FastSparseFit_sparse', PACKAGE = 'FastSparse', X, y, Loss, Penalty, Algorithm, NnzStopNum, G_ncols, G_nrows, Lambda2Max, Lambda2Min, PartialSort, MaxIters, rtol, atol, ActiveSet, ActiveSetNum, MaxNumSwaps, ScaleDownFactor, ScreenSize, LambdaU, Lambdas, ExcludeFirstK, Intercept, withBounds, Lows, Highs) 6 | } 7 | 8 | FastSparseFit_dense <- function(X, y, Loss, Penalty, Algorithm, NnzStopNum, G_ncols, G_nrows, Lambda2Max, Lambda2Min, PartialSort, MaxIters, rtol, atol, ActiveSet, ActiveSetNum, MaxNumSwaps, ScaleDownFactor, ScreenSize, LambdaU, Lambdas, ExcludeFirstK, Intercept, withBounds, Lows, Highs) { 9 | .Call('_FastSparse_FastSparseFit_dense', PACKAGE = 'FastSparse', X, y, Loss, Penalty, Algorithm, NnzStopNum, G_ncols, G_nrows, Lambda2Max, Lambda2Min, PartialSort, MaxIters, rtol, atol, ActiveSet, ActiveSetNum, MaxNumSwaps, ScaleDownFactor, ScreenSize, LambdaU, Lambdas, ExcludeFirstK, Intercept, withBounds, Lows, Highs) 10 | } 11 | 12 | FastSparseCV_sparse <- function(X, y, Loss, Penalty, Algorithm, NnzStopNum, G_ncols, G_nrows, Lambda2Max, Lambda2Min, PartialSort, MaxIters, rtol, atol, ActiveSet, ActiveSetNum, MaxNumSwaps, ScaleDownFactor, ScreenSize, LambdaU, Lambdas, nfolds, seed, ExcludeFirstK, Intercept, withBounds, Lows, Highs) { 13 | .Call('_FastSparse_FastSparseCV_sparse', PACKAGE = 'FastSparse', X, y, Loss, Penalty, Algorithm, NnzStopNum, G_ncols, G_nrows, Lambda2Max, Lambda2Min, PartialSort, MaxIters, rtol, atol, ActiveSet, ActiveSetNum, MaxNumSwaps, ScaleDownFactor, ScreenSize, LambdaU, Lambdas, nfolds, seed, ExcludeFirstK, Intercept, withBounds, Lows, Highs) 14 | } 15 | 16 | FastSparseCV_dense <- function(X, y, Loss, Penalty, Algorithm, NnzStopNum, G_ncols, G_nrows, Lambda2Max, Lambda2Min, PartialSort, MaxIters, rtol, atol, ActiveSet, ActiveSetNum, MaxNumSwaps, ScaleDownFactor, ScreenSize, LambdaU, Lambdas, nfolds, seed, ExcludeFirstK, Intercept, withBounds, Lows, Highs) { 17 | .Call('_FastSparse_FastSparseCV_dense', PACKAGE = 'FastSparse', X, y, Loss, Penalty, Algorithm, NnzStopNum, G_ncols, G_nrows, Lambda2Max, Lambda2Min, PartialSort, MaxIters, rtol, atol, ActiveSet, ActiveSetNum, MaxNumSwaps, ScaleDownFactor, ScreenSize, LambdaU, Lambdas, nfolds, seed, ExcludeFirstK, Intercept, withBounds, Lows, Highs) 18 | } 19 | 20 | cor_matrix <- function(p, base_cor) { 21 | .Call('_FastSparse_cor_matrix', PACKAGE = 'FastSparse', p, base_cor) 22 | } 23 | 24 | R_matrix_column_get_dense <- function(mat, col) { 25 | .Call('_FastSparse_R_matrix_column_get_dense', PACKAGE = 'FastSparse', mat, col) 26 | } 27 | 28 | R_matrix_column_get_sparse <- function(mat, col) { 29 | .Call('_FastSparse_R_matrix_column_get_sparse', PACKAGE = 'FastSparse', mat, col) 30 | } 31 | 32 | R_matrix_rows_get_dense <- function(mat, rows) { 33 | .Call('_FastSparse_R_matrix_rows_get_dense', PACKAGE = 'FastSparse', mat, rows) 34 | } 35 | 36 | R_matrix_rows_get_sparse <- function(mat, rows) { 37 | .Call('_FastSparse_R_matrix_rows_get_sparse', PACKAGE = 'FastSparse', mat, rows) 38 | } 39 | 40 | R_matrix_vector_schur_product_dense <- function(mat, u) { 41 | .Call('_FastSparse_R_matrix_vector_schur_product_dense', PACKAGE = 'FastSparse', mat, u) 42 | } 43 | 44 | R_matrix_vector_schur_product_sparse <- function(mat, u) { 45 | .Call('_FastSparse_R_matrix_vector_schur_product_sparse', PACKAGE = 'FastSparse', mat, u) 46 | } 47 | 48 | R_matrix_vector_divide_dense <- function(mat, u) { 49 | .Call('_FastSparse_R_matrix_vector_divide_dense', PACKAGE = 'FastSparse', mat, u) 50 | } 51 | 52 | R_matrix_vector_divide_sparse <- function(mat, u) { 53 | .Call('_FastSparse_R_matrix_vector_divide_sparse', PACKAGE = 'FastSparse', mat, u) 54 | } 55 | 56 | R_matrix_column_sums_dense <- function(mat) { 57 | .Call('_FastSparse_R_matrix_column_sums_dense', PACKAGE = 'FastSparse', mat) 58 | } 59 | 60 | R_matrix_column_sums_sparse <- function(mat) { 61 | .Call('_FastSparse_R_matrix_column_sums_sparse', PACKAGE = 'FastSparse', mat) 62 | } 63 | 64 | R_matrix_column_dot_dense <- function(mat, col, u) { 65 | .Call('_FastSparse_R_matrix_column_dot_dense', PACKAGE = 'FastSparse', mat, col, u) 66 | } 67 | 68 | R_matrix_column_dot_sparse <- function(mat, col, u) { 69 | .Call('_FastSparse_R_matrix_column_dot_sparse', PACKAGE = 'FastSparse', mat, col, u) 70 | } 71 | 72 | R_matrix_column_mult_dense <- function(mat, col, u) { 73 | .Call('_FastSparse_R_matrix_column_mult_dense', PACKAGE = 'FastSparse', mat, col, u) 74 | } 75 | 76 | R_matrix_column_mult_sparse <- function(mat, col, u) { 77 | .Call('_FastSparse_R_matrix_column_mult_sparse', PACKAGE = 'FastSparse', mat, col, u) 78 | } 79 | 80 | R_matrix_normalize_dense <- function(mat_norm) { 81 | .Call('_FastSparse_R_matrix_normalize_dense', PACKAGE = 'FastSparse', mat_norm) 82 | } 83 | 84 | R_matrix_normalize_sparse <- function(mat_norm) { 85 | .Call('_FastSparse_R_matrix_normalize_sparse', PACKAGE = 'FastSparse', mat_norm) 86 | } 87 | 88 | R_matrix_center_dense <- function(mat, X_normalized, intercept) { 89 | .Call('_FastSparse_R_matrix_center_dense', PACKAGE = 'FastSparse', mat, X_normalized, intercept) 90 | } 91 | 92 | R_matrix_center_sparse <- function(mat, X_normalized, intercept) { 93 | .Call('_FastSparse_R_matrix_center_sparse', PACKAGE = 'FastSparse', mat, X_normalized, intercept) 94 | } 95 | 96 | start_profiler <- function(str) { 97 | .Call('_FastSparse_start_profiler', PACKAGE = 'FastSparse', str) 98 | } 99 | 100 | stop_profiler <- function() { 101 | .Call('_FastSparse_stop_profiler', PACKAGE = 'FastSparse') 102 | } 103 | 104 | -------------------------------------------------------------------------------- /FastSparse/R/coef.R: -------------------------------------------------------------------------------- 1 | #' @title Extract Solutions 2 | #' 3 | #' @description Extracts a specific solution in the regularization path. 4 | #' @param object The output of FastSparse.fit or FastSparse.cvfit 5 | #' @param ... ignore 6 | #' @param lambda The value of lambda at which to extract the solution. 7 | #' @param gamma The value of gamma at which to extract the solution. 8 | #' @method coef FastSparse 9 | #' @details 10 | #' If both lambda and gamma are not supplied, then a matrix of coefficients 11 | #' for all the solutions in the regularization path is returned. If lambda is 12 | #' supplied but gamma is not, the smallest value of gamma is used. 13 | #' @examples 14 | #' # Generate synthetic data for this example 15 | #' data <- GenSynthetic(n=500,p=1000,k=10,seed=1) 16 | #' X = data$X 17 | #' y = data$y 18 | #' 19 | #' # Fit an L0L2 Model with 10 values of Gamma ranging from 0.0001 to 10, using coordinate descent 20 | #' fit <- FastSparse.fit(X, y, penalty="L0L2", maxSuppSize=50, nGamma=10, gammaMin=0.0001, gammaMax = 10) 21 | #' print(fit) 22 | #' # Extract the coefficients of the solution at lambda = 0.0361829 and gamma = 0.0001 23 | #' coef(fit, lambda=0.0361829, gamma=0.0001) 24 | #' # Extract the coefficients of all the solutions in the path 25 | #' coef(fit) 26 | #' 27 | #' @export 28 | coef.FastSparse <- function(object,lambda=NULL,gamma=NULL, ...){ 29 | if (is.null(lambda) && is.null(gamma)){ 30 | t = do.call(cbind,object$beta) 31 | if (object$settings$intercept){ 32 | intercepts = unlist(object$a0) 33 | t = rbind(intercepts, t) 34 | } 35 | } 36 | else{ 37 | if (is.null(gamma)){ # if lambda is present but gamma is not, use smallest value of gamma 38 | gamma = object$gamma[1] 39 | } 40 | diffGamma = abs(object$gamma-gamma) 41 | gammaindex = which(diffGamma==min(diffGamma)) 42 | diffLambda = abs(lambda - object$lambda[[gammaindex]]) 43 | indices = which(diffLambda == min(diffLambda)) 44 | #indices = match(lambda,object$lambda[[gammaindex]]) 45 | if (object$settings$intercept){ 46 | t = rbind(object$a0[[gammaindex]][indices],object$beta[[gammaindex]][,indices,drop=FALSE]) 47 | rownames(t) = c("Intercept",paste(rep("V",object$p),1:object$p,sep="")) 48 | } 49 | else{ 50 | t = object$beta[[gammaindex]][,indices,drop=FALSE] 51 | rownames(t) = paste(rep("V",object$p),1:object$p,sep="") 52 | } 53 | } 54 | t 55 | } 56 | 57 | #' @rdname coef.FastSparse 58 | #' @method coef FastSparseCV 59 | #' @export 60 | coef.FastSparseCV <- function(object,lambda=NULL,gamma=NULL, ...){ 61 | coef.FastSparse(object$fit,lambda,gamma, ...) 62 | } 63 | -------------------------------------------------------------------------------- /FastSparse/R/genhighcorr.R: -------------------------------------------------------------------------------- 1 | #' @importFrom stats rnorm 2 | #' @importFrom MASS mvrnorm 3 | #' @importFrom Rcpp cppFunction 4 | #' @title Generate Expoententiall Correlated Synthetic Data 5 | #' 6 | #' @description Generates a synthetic dataset as follows: 1) Generate a correlation matrix, SIG, where item [i, j] = A^|i-j|. 7 | #' 2) Draw from a Multivariate Normal Distribution using (mu and SIG) to generate X. 3) Generate a vector B with every ~p/k entry set to 1 and the rest are zeros. 8 | #' 4) Sample every element in the noise vector e from N(0,1). 4) Set y = XB + b0 + e. 9 | #' @param n Number of samples 10 | #' @param p Number of features 11 | #' @param k Number of non-zeros in true vector of coefficients 12 | #' @param seed The seed used for randomly generating the data 13 | #' @param rho The threshold for setting values to 0. if |X(i, j)| > rho => X(i, j) <- 0 14 | #' @param b0 intercept value to scale y by. 15 | #' @param noise_ratio The multiplier of noise to apply when calculating e. e[i] = noise_ratio*N(0, 1). 16 | #' @param mu The mean for drawing from the Multivariate Normal Distribution. A scalar of vector of length p. 17 | #' @param base_cor The base correlation, A in [i, j] = A^|i-j|. 18 | #' @return A list containing: 19 | #' the data matrix X, 20 | #' the response vector y, 21 | #' the coefficients B, 22 | #' the error vector e, 23 | #' the intercept term b0. 24 | #' @examples 25 | #' data <- GenSyntheticHighCorr(n=500,p=1000,k=10,seed=1) 26 | #' X = data$X 27 | #' y = data$y 28 | #' @export 29 | GenSyntheticHighCorr <- function(n, p, k, seed, rho=0, b0=0, noise_ratio=1, mu=0, base_cor=.9) 30 | { 31 | set.seed(seed) # fix the seed to get a reproducible result 32 | cor <- .Call("_FastSparse_cor_matrix", p, base_cor) 33 | 34 | if (length(mu) == 1){ 35 | mu = rep(mu, p) 36 | } 37 | 38 | X <- mvrnorm(n, mu, forceSymmetric(cor)) 39 | 40 | X[abs(X) < rho] <- 0. 41 | 42 | B_indices = seq(from=1, to=p, by=as.integer(p/k)) 43 | 44 | B = rep(0, p) 45 | 46 | for (i in B_indices){ 47 | B[i] = 1 48 | } 49 | 50 | e = noise_ratio*rnorm(n) 51 | y = X%*%B + e + b0 52 | list(X=X, y = y, B=B, e=e, b0=b0) 53 | } 54 | 55 | -------------------------------------------------------------------------------- /FastSparse/R/gensynthetic.R: -------------------------------------------------------------------------------- 1 | #' @importFrom stats rnorm 2 | #' @title Generate Synthetic Data 3 | #' 4 | #' @description Generates a synthetic dataset as follows: 1) Sample every element in data matrix X from N(0,1). 5 | #' 2) Generate a vector B with the first k entries set to 1 and the rest are zeros. 3) Sample every element in the noise 6 | #' vector e from N(0,1). 4) Set y = XB + b0 + e. 7 | #' @param n Number of samples 8 | #' @param p Number of features 9 | #' @param k Number of non-zeros in true vector of coefficients 10 | #' @param seed The seed used for randomly generating the data 11 | #' @param rho The threshold for setting values to 0. if |X(i, j)| > rho => X(i, j) <- 0 12 | #' @param b0 intercept value to translate y by. 13 | #' @param error_ratio multiplier for the magnitude of the error term 'e'. 14 | #' @return A list containing: 15 | #' the data matrix X, 16 | #' the response vector y, 17 | #' the coefficients B, 18 | #' the error vector e, 19 | #' the intercept term b0. 20 | #' @examples 21 | #' data <- GenSynthetic(n=500,p=1000,k=10,seed=1) 22 | #' X = data$X 23 | #' y = data$y 24 | #' @export 25 | GenSynthetic <- function(n, p, k, seed, rho=0, b0=0, error_ratio=1) 26 | { 27 | set.seed(seed) # fix the seed to get a reproducible result 28 | X = matrix(rnorm(n*p),nrow=n,ncol=p) 29 | X[abs(X) < rho] <- 0. 30 | B = c(rep(1,k),rep(0,p-k)) 31 | e = rnorm(n)*error_ratio 32 | y = X%*%B + e + b0 33 | list(X=X, y=y, B=B, e=e, b0=b0) 34 | } 35 | -------------------------------------------------------------------------------- /FastSparse/R/plot.R: -------------------------------------------------------------------------------- 1 | #' @title Plot Regularization Path 2 | #' 3 | #' @description Plots the regularization path for a given gamma. 4 | #' @param gamma The value of gamma at which to plot. 5 | #' @param x The output of FastSparse.fit 6 | #' @param showLines If TRUE, the lines connecting the points in the plot are shown. 7 | #' @param ... ignore 8 | #' 9 | #' @examples 10 | #' # Generate synthetic data for this example 11 | #' data <- GenSynthetic(n=500,p=1000,k=10,seed=1) 12 | #' X = data$X 13 | #' y = data$y 14 | #' # Fit an L0 Model with a maximum of 50 non-zeros 15 | #' fit <- FastSparse.fit(X, y, penalty="L0", maxSuppSize=50) 16 | #' plot(fit, gamma=0) 17 | #' 18 | #' @import ggplot2 19 | #' @importFrom reshape2 melt 20 | #' @method plot FastSparse 21 | #' @export 22 | plot.FastSparse <- function(x, gamma=0, showLines=FALSE, ...) 23 | { 24 | j = which(abs(x$gamma-gamma)==min(abs(x$gamma-gamma))) 25 | p = x$p 26 | allin = c() # contains all the non-zero variables in the path 27 | for (i in 1:length(x$lambda[[j]])){ 28 | BetaTemp = x$beta[[j]][,i] 29 | supp = which(as.matrix(BetaTemp != 0)) 30 | allin = c(allin, supp) 31 | } 32 | allin = unique(allin) 33 | 34 | #ggplot needs a dataframe 35 | yy = t(as.matrix(x$beta[[j]][allin,])) # length(lambda) x length(allin) matrix 36 | data <- as.data.frame(yy) 37 | 38 | colnames(data) = x$varnames[allin] 39 | 40 | #id variable for position in matrix 41 | data$id <- x$suppSize[[j]] 42 | 43 | #reshape to long format 44 | plot_data <- melt(data,id.var="id") 45 | 46 | #breaks = x$suppSize[[j]] 47 | 48 | #plot 49 | plotObject = ggplot(plot_data, aes_string(x="id",y="value",group="variable",colour="variable")) + geom_point(size=2.5) + 50 | labs(x = "Support Size", y = "Coefficient") + theme(axis.title=element_text(size=14)) # + scale_x_continuous(breaks = breaks) + theme(axis.text = element_text(size = 12)) 51 | 52 | if (showLines == TRUE){ 53 | plotObject = plotObject + geom_line(aes_string(lty="variable"),alpha=0.3) 54 | } 55 | plotObject 56 | } 57 | 58 | #' @title Plot Cross-validation Errors 59 | #' 60 | #' @description Plots cross-validation errors for a given gamma. 61 | #' @param x The output of FastSparse.cvfit 62 | #' @inheritParams plot.FastSparse 63 | #' @examples 64 | #' # Generate synthetic data for this example 65 | #' data <- GenSynthetic(n=500,p=1000,k=10,seed=1) 66 | #' X = data$X 67 | #' y = data$y 68 | #' 69 | #' # Perform 5-fold cross-validation on an L0L2 Model with 5 values of 70 | #' # Gamma ranging from 0.0001 to 10 71 | #' fit <- FastSparse.cvfit(X, y, nFolds=5, seed=1, penalty="L0L2", 72 | #' maxSuppSize=20, nGamma=5, gammaMin=0.0001, gammaMax = 10) 73 | #' # Plot the graph of cross-validation error versus lambda for gamma = 0.0001 74 | #' plot(fit, gamma=0.0001) 75 | #' 76 | #' @method plot FastSparseCV 77 | #' @export 78 | plot.FastSparseCV <- function(x, gamma=0, ...) 79 | { 80 | j = which(abs(x$fit$gamma-gamma)==min(abs(x$fit$gamma-gamma))) 81 | data = data.frame(x=x$fit$suppSize[[j]], y=x$cvMeans[[j]], sd=x$cvSDs[[j]]) 82 | ggplot(data, aes_string(x="x",y="y")) + geom_point() + geom_errorbar(aes_string(ymin="y-sd", ymax="y+sd"))+ 83 | labs(x = "Support Size", y = "Cross-validation Error") + theme(axis.title=element_text(size=14)) + theme(axis.text = element_text(size = 12)) 84 | } 85 | -------------------------------------------------------------------------------- /FastSparse/R/predict.R: -------------------------------------------------------------------------------- 1 | #' @title Predict Response 2 | #' 3 | #' @description Predicts the response for a given sample. 4 | #' @param object The output of FastSparse.fit or FastSparse.cvfit 5 | #' @param ... ignore 6 | #' @param newx A matrix on which predictions are made. The matrix should have p columns. 7 | #' @param lambda The value of lambda to use for prediction. A summary of the lambdas in the regularization 8 | #' path can be obtained using \code{print(fit)}. 9 | #' @param gamma The value of gamma to use for prediction. A summary of the gammas in the regularization 10 | #' path can be obtained using \code{print(fit)}. 11 | #' @method predict FastSparse 12 | #' @details 13 | #' If both lambda and gamma are not supplied, then a matrix of predictions 14 | #' for all the solutions in the regularization path is returned. If lambda is 15 | #' supplied but gamma is not, the smallest value of gamma is used. In case of 16 | #' of logistic regression, probability values are returned. 17 | #' @examples 18 | #' # Generate synthetic data for this example 19 | #' data <- GenSynthetic(n=500,p=1000,k=10,seed=1) 20 | #' X = data$X 21 | #' y = data$y 22 | #' 23 | #' # Fit an L0L2 Model with 10 values of Gamma ranging from 0.0001 to 10, using coordinate descent 24 | #' fit <- FastSparse.fit(X,y, penalty="L0L2", maxSuppSize=50, nGamma=10, gammaMin=0.0001, gammaMax = 10) 25 | #' print(fit) 26 | #' # Apply the fitted model with lambda=0.0361829 and gamma=0.0001 on X to predict the response 27 | #' predict(fit, newx = X, lambda=0.0361829, gamma=0.0001) 28 | #' # Apply the fitted model on X to predict the response for all the solutions in the path 29 | #' predict(fit, newx = X) 30 | #' 31 | #' @export 32 | predict.FastSparse <- function(object,newx,lambda=NULL,gamma=NULL, ...) 33 | { 34 | beta = coef.FastSparse(object, lambda, gamma) 35 | if (object$settings$intercept){ 36 | # add a column of ones for the intercept 37 | x = cbind(1,newx) 38 | } 39 | else{ 40 | x = newx 41 | } 42 | prediction = x%*%beta 43 | #if (object$loss == "Logistic" || object$loss == "SquaredHinge"){ 44 | # prediction = sign(prediction) 45 | #} 46 | if (object$loss == "Logistic"){ 47 | prediction = 1/(1+exp(-prediction)) 48 | } else if (object$loss == "Exponential") { 49 | prediction = 1/(1+exp(-2*prediction)) 50 | } 51 | prediction 52 | } 53 | 54 | #' @rdname predict.FastSparse 55 | #' @method predict FastSparseCV 56 | #' @export 57 | predict.FastSparseCV <- function(object,newx,lambda=NULL,gamma=NULL, ...) 58 | { 59 | predict.FastSparse(object$fit,newx,lambda,gamma, ...) 60 | } 61 | -------------------------------------------------------------------------------- /FastSparse/R/print.R: -------------------------------------------------------------------------------- 1 | #' @title Print FastSparse.fit object 2 | #' 3 | #' @description Prints a summary of FastSparse.fit 4 | #' @param x The output of FastSparse.fit or FastSparse.cvfit 5 | #' @param ... ignore 6 | #' @method print FastSparse 7 | #' @export 8 | print.FastSparse <- function(x, ...) 9 | { 10 | gammas = rep(x$gamma, times=lapply(x$lambda, length) ) 11 | data.frame(lambda = unlist(x["lambda"]), gamma = gammas, suppSize = unlist(x["suppSize"]), row.names = NULL) 12 | } 13 | 14 | #' @rdname print.FastSparse 15 | #' @method print FastSparseCV 16 | #' @export 17 | print.FastSparseCV <- function(x, ...) 18 | { 19 | print.FastSparse(x$fit) 20 | } 21 | -------------------------------------------------------------------------------- /FastSparse/Read-and-delete-me: -------------------------------------------------------------------------------- 1 | * Edit the help file skeletons in 'man', possibly combining help files for multiple functions. 2 | * Edit the exports in 'NAMESPACE', and add necessary imports. 3 | * Put any C/C++/Fortran code in 'src'. 4 | * If you have compiled code, add a useDynLib() directive to 'NAMESPACE'. 5 | * Run R CMD build to build the package tarball. 6 | * Run R CMD check to check the package tarball. 7 | 8 | Read "Writing R Extensions" for more information. 9 | -------------------------------------------------------------------------------- /FastSparse/cleanup: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | rm -f config.* src/Makevars src/*.o 3 | -------------------------------------------------------------------------------- /FastSparse/configure.ac: -------------------------------------------------------------------------------- 1 | ## -*- mode: autoconf; autoconf-indentation: 4; -*- 2 | ## 3 | ## FastSparse configure.ac 4 | ## Author: Jiachang Liu 5 | ## Note: This file has been adapted from RcppArmadillo's 6 | ## configure.ac, which is written by Dirk Eddelbuettel. 7 | 8 | ## require at least autoconf 2.61 9 | AC_PREREQ(2.61) 10 | 11 | ## Process this file with autoconf to produce a configure script. 12 | AC_INIT([FastSparse], 0.1.0) 13 | 14 | ## Set R_HOME, respecting an environment variable if one is set 15 | : ${R_HOME=$(R RHOME)} 16 | if test -z "${R_HOME}"; then 17 | AC_MSG_ERROR([Could not determine R_HOME.]) 18 | fi 19 | ## Use R to set CXX and CXXFLAGS 20 | CXX=$(${R_HOME}/bin/R CMD config CXX) 21 | CXXFLAGS=$("${R_HOME}/bin/R" CMD config CXXFLAGS) 22 | 23 | ## We are using C++ 24 | AC_LANG(C++) 25 | AC_REQUIRE_CPP 26 | 27 | ## variant available if and only if C++11 is used with g++ 5.4 or newer 28 | can_use_openmp="" 29 | 30 | ## Check the C++ compiler using the CXX value set 31 | AC_PROG_CXX 32 | ## If it is g++, we have GXX set so let's examine it 33 | if test "${GXX}" = yes; then 34 | AC_MSG_CHECKING([whether g++ version is sufficient]) 35 | gxx_version=$(${CXX} -v 2>&1 | awk '/^.*g.. version/ {print $3}') 36 | case ${gxx_version} in 37 | 1.*|2.*|3.*|4.0.*|4.1.*|4.2.*|4.3.*|4.4.*|4.5.*|4.6.*|4.7.0|4.7.1) 38 | AC_MSG_RESULT([no]) 39 | AC_MSG_WARN([Only g++ version 4.7.2 or greater can be used with FastSparse.]) 40 | AC_MSG_ERROR([Please use a different compiler.]) 41 | ;; 42 | 4.7.*|4.8.*|4.9.*|5.0*|5.1*|5.2*|5.3*) 43 | AC_MSG_RESULT([yes, but without OpenMP as version ${gxx_version}]) 44 | ## we know this one is bad 45 | can_use_openmp="no" 46 | ;; 47 | 5.4*|5.5*|5.6*|5.7*|5.8*|5.9*|6.*|7.*|8.*|9.*|10.*|11.*|12.*) 48 | AC_MSG_RESULT([yes, with OpenMP as version ${gxx_version}]) 49 | ## we know this one is good, yay 50 | can_use_openmp="yes" 51 | ;; 52 | *) 53 | AC_MSG_RESULT([almost]) 54 | AC_MSG_WARN([Compiler self-identifies as being compliant with GNUC extensions but is not g++.]) 55 | ## we know nothing, so no 56 | can_use_openmp="no" 57 | ;; 58 | esac 59 | fi 60 | 61 | ## Check for Apple LLVM 62 | 63 | AC_MSG_CHECKING([for macOS]) 64 | RSysinfoName=$("${R_HOME}/bin/Rscript" --vanilla -e 'cat(Sys.info()[["sysname"]])') 65 | 66 | if test x"${RSysinfoName}" == x"Darwin"; then 67 | AC_MSG_RESULT([found]) 68 | AC_MSG_CHECKING([for macOS Apple compiler]) 69 | 70 | apple_compiler=$($CXX --version 2>&1 | grep -i -c -e 'apple llvm') 71 | 72 | if test x"${apple_compiler}" == x"1"; then 73 | AC_MSG_RESULT([found]) 74 | AC_MSG_WARN([OpenMP unavailable and turned off.]) 75 | can_use_openmp="no" 76 | else 77 | AC_MSG_RESULT([not found]) 78 | AC_MSG_CHECKING([for clang compiler]) 79 | clang_compiler=$($CXX --version 2>&1 | grep -i -c -e 'clang ') 80 | 81 | if test x"${clang_compiler}" == x"1"; then 82 | AC_MSG_RESULT([found]) 83 | AC_MSG_CHECKING([for OpenMP compatible version of clang]) 84 | clang_version=$(${CXX} -v 2>&1 | awk '/^.*clang version/ {print $3}') 85 | 86 | case ${clang_version} in 87 | 4.*|5.*|6.*|7.*|8.*|9.*|10.*|11.*) 88 | AC_MSG_RESULT([found and suitable]) 89 | can_use_openmp="yes" 90 | ;; 91 | *) 92 | AC_MSG_RESULT([not found]) 93 | AC_MSG_WARN([OpenMP unavailable and turned off.]) 94 | can_use_openmp="no" 95 | ;; 96 | esac 97 | else 98 | AC_MSG_RESULT([not found]) 99 | AC_MSG_WARN([unsupported macOS build detected; if anything breaks, you keep the pieces.]) 100 | fi 101 | fi 102 | fi 103 | 104 | ## Default the OpenMP flag to the empty string. 105 | ## If and only if OpenMP is found, expand to $(SHLIB_OPENMP_CXXFLAGS) 106 | openmp_flag="" 107 | 108 | if test x"${can_use_openmp}" == x"yes"; then 109 | AC_MSG_CHECKING([for OpenMP]) 110 | ## if R has -fopenmp we should be good 111 | allldflags=$(${R_HOME}/bin/R CMD config --ldflags) 112 | hasOpenMP=$(echo ${allldflags} | grep -- -fopenmp) 113 | if test x"${hasOpenMP}" == x""; then 114 | AC_MSG_RESULT([missing]) 115 | arma_have_openmp="#define ARMA_DONT_USE_OPENMP 1" 116 | else 117 | AC_MSG_RESULT([found and suitable]) 118 | arma_have_openmp="#define ARMA_USE_OPENMP 1" 119 | openmp_flag='$(SHLIB_OPENMP_CXXFLAGS)' 120 | fi 121 | fi 122 | 123 | 124 | ## Make AC_OUTPUT create each file by copying an input file (by default file.in), substituting the output variable values. 125 | AC_SUBST([OPENMP_FLAG], ["${openmp_flag}"]) 126 | AC_CONFIG_FILES([src/Makevars]) 127 | AC_OUTPUT 128 | -------------------------------------------------------------------------------- /FastSparse/man/FastSparse-package.Rd: -------------------------------------------------------------------------------- 1 | \name{FastSparse-package} 2 | \alias{FastSparse-package} 3 | \alias{FastSparse} 4 | \docType{package} 5 | \title{ 6 | \packageTitle{FastSparse} 7 | } 8 | \description{ 9 | \packageDescription{FastSparse} 10 | } 11 | \details{ 12 | 13 | The DESCRIPTION file: 14 | \packageDESCRIPTION{FastSparse} 15 | \packageIndices{FastSparse} 16 | ~~ An overview of how to use the package, including the most important functions ~~ 17 | } 18 | \author{ 19 | \packageAuthor{FastSparse} 20 | 21 | Maintainer: \packageMaintainer{FastSparse} 22 | } 23 | \references{ 24 | ~~ Literature or other references for background information ~~ 25 | } 26 | ~~ Optionally other standard keywords, one per line, from file KEYWORDS in the R documentation directory ~~ 27 | \keyword{ package } 28 | \seealso{ 29 | ~~ Optional links to other man pages, e.g. ~~ 30 | ~~ \code{\link[:-package]{}} ~~ 31 | } 32 | \examples{ 33 | ~~ simple examples of the most important functions ~~ 34 | } 35 | -------------------------------------------------------------------------------- /FastSparse/man/FastSparse.cvfit.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/cvfit.R 3 | \name{FastSparse.cvfit} 4 | \alias{FastSparse.cvfit} 5 | \title{Cross Validation} 6 | \usage{ 7 | FastSparse.cvfit( 8 | x, 9 | y, 10 | loss = "SquaredError", 11 | penalty = "L0", 12 | algorithm = "CD", 13 | maxSuppSize = 100, 14 | nLambda = 100, 15 | nGamma = 10, 16 | gammaMax = 10, 17 | gammaMin = 1e-04, 18 | partialSort = TRUE, 19 | maxIters = 200, 20 | rtol = 1e-06, 21 | atol = 1e-09, 22 | activeSet = TRUE, 23 | activeSetNum = 3, 24 | maxSwaps = 100, 25 | scaleDownFactor = 0.8, 26 | screenSize = 1000, 27 | autoLambda = NULL, 28 | lambdaGrid = list(), 29 | nFolds = 10, 30 | seed = 1, 31 | excludeFirstK = 0, 32 | intercept = TRUE, 33 | lows = -Inf, 34 | highs = Inf 35 | ) 36 | } 37 | \arguments{ 38 | \item{x}{The data matrix.} 39 | 40 | \item{y}{The response vector. For classification, we only support binary vectors.} 41 | 42 | \item{loss}{The loss function. Currently we support the choices "SquaredError" (for regression), "Logistic" (for logistic regression), and "SquaredHinge" (for smooth SVM).} 43 | 44 | \item{penalty}{The type of regularization. This can take either one of the following choices: 45 | "L0", "L0L2", and "L0L1".} 46 | 47 | \item{algorithm}{The type of algorithm used to minimize the objective function. Currently "CD" and "CDPSI" are 48 | are supported. "CD" is a variant of cyclic coordinate descent and runs very fast. "CDPSI" performs 49 | local combinatorial search on top of CD and typically achieves higher quality solutions (at the expense 50 | of increased running time).} 51 | 52 | \item{maxSuppSize}{The maximum support size at which to terminate the regularization path. We recommend setting 53 | this to a small fraction of min(n,p) (e.g. 0.05 * min(n,p)) as L0 regularization typically selects a small 54 | portion of non-zeros.} 55 | 56 | \item{nLambda}{The number of Lambda values to select (recall that Lambda is the regularization parameter 57 | corresponding to the L0 norm). This value is ignored if 'lambdaGrid' is supplied.} 58 | 59 | \item{nGamma}{The number of Gamma values to select (recall that Gamma is the regularization parameter 60 | corresponding to L1 or L2, depending on the chosen penalty). This value is ignored if 'lambdaGrid' is supplied 61 | and will be set to length(lambdaGrid)} 62 | 63 | \item{gammaMax}{The maximum value of Gamma when using the L0L2 penalty. For the L0L1 penalty this is 64 | automatically selected.} 65 | 66 | \item{gammaMin}{The minimum value of Gamma when using the L0L2 penalty. For the L0L1 penalty, the minimum 67 | value of gamma in the grid is set to gammaMin * gammaMax. Note that this should be a strictly positive quantity.} 68 | 69 | \item{partialSort}{If TRUE partial sorting will be used for sorting the coordinates to do greedy cycling (see our paper for 70 | for details). Otherwise, full sorting is used.} 71 | 72 | \item{maxIters}{The maximum number of iterations (full cycles) for CD per grid point.} 73 | 74 | \item{rtol}{The relative tolerance which decides when to terminate optimization (based on the relative change in the objective between iterations).} 75 | 76 | \item{atol}{The absolute tolerance which decides when to terminate optimization (based on the absolute L2 norm of the residuals).} 77 | 78 | \item{activeSet}{If TRUE, performs active set updates.} 79 | 80 | \item{activeSetNum}{The number of consecutive times a support should appear before declaring support stabilization.} 81 | 82 | \item{maxSwaps}{The maximum number of swaps used by CDPSI for each grid point.} 83 | 84 | \item{scaleDownFactor}{This parameter decides how close the selected Lambda values are. The choice should be 85 | strictly between 0 and 1 (i.e., 0 and 1 are not allowed). Larger values lead to closer lambdas and typically to smaller 86 | gaps between the support sizes. For details, see our paper - Section 5 on Adaptive Selection of Tuning Parameters).} 87 | 88 | \item{screenSize}{The number of coordinates to cycle over when performing initial correlation screening.} 89 | 90 | \item{autoLambda}{Ignored parameter. Kept for backwards compatibility.} 91 | 92 | \item{lambdaGrid}{A grid of Lambda values to use in computing the regularization path. This is by default an empty list and is ignored. 93 | When specified, LambdaGrid should be a list of length 'nGamma', where the ith element (corresponding to the ith gamma) should be a decreasing sequence of lambda values 94 | which are used by the algorithm when fitting for the ith value of gamma (see the vignette for details).} 95 | 96 | \item{nFolds}{The number of folds for cross-validation.} 97 | 98 | \item{seed}{The seed used in randomly shuffling the data for cross-validation.} 99 | 100 | \item{excludeFirstK}{This parameter takes non-negative integers. The first excludeFirstK features in x will be excluded from variable selection, 101 | i.e., the first excludeFirstK variables will not be included in the L0-norm penalty (they will still be included in the L1 or L2 norm penalties.).} 102 | 103 | \item{intercept}{If FALSE, no intercept term is included in the model.} 104 | 105 | \item{lows}{Lower bounds for coefficients. Either a scalar for all coefficients to have the same bound or a vector of size p (number of columns of X) where lows[i] is the lower bound for coefficient i.} 106 | 107 | \item{highs}{Upper bounds for coefficients. Either a scalar for all coefficients to have the same bound or a vector of size p (number of columns of X) where highs[i] is the upper bound for coefficient i.} 108 | } 109 | \value{ 110 | An S3 object of type "FastSparseCV" describing the regularization path. The object has the following members. 111 | \item{cvMeans}{This is a list, where the ith element is the sequence of cross-validation errors corresponding to the ith gamma value, i.e., the sequence 112 | cvMeans[[i]] corresponds to fit$gamma[i]} 113 | \item{cvSDs}{This a list, where the ith element is a sequence of standard deviations for the cross-validation errors: cvSDs[[i]] corresponds to cvMeans[[i]].} 114 | \item{fit}{The fitted model with type "FastSparse", i.e., this is the same object returned by \code{\link{FastSparse.fit}}.} 115 | } 116 | \description{ 117 | Computes a regularization path and performs K-fold cross-validation. 118 | } 119 | \examples{ 120 | # Generate synthetic data for this example 121 | data <- GenSynthetic(n=500,p=1000,k=10,seed=1) 122 | X = data$X 123 | y = data$y 124 | 125 | # Perform 5-fold cross-validation on an L0L2 regression model with 5 values of 126 | # Gamma ranging from 0.0001 to 10 127 | fit <- FastSparse.cvfit(X, y, nFolds=5, seed=1, penalty="L0L2", maxSuppSize=20, nGamma=5, 128 | gammaMin=0.0001, gammaMax = 10) 129 | print(fit) 130 | # Plot the graph of cross-validation error versus lambda for gamma = 0.0001 131 | plot(fit, gamma=0.0001) 132 | # Extract the coefficients at lambda = 0.0361829 and gamma = 0.0001 133 | coef(fit, lambda=0.0361829, gamma=0.0001) 134 | # Apply the fitted model on X to predict the response 135 | predict(fit, newx = X, lambda=0.0361829, gamma=0.0001) 136 | 137 | } 138 | -------------------------------------------------------------------------------- /FastSparse/man/FastSparse.fit.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/fit.R 3 | \name{FastSparse.fit} 4 | \alias{FastSparse.fit} 5 | \title{Fit an L0-regularized model} 6 | \usage{ 7 | FastSparse.fit( 8 | x, 9 | y, 10 | loss = "SquaredError", 11 | penalty = "L0", 12 | algorithm = "CD", 13 | maxSuppSize = 100, 14 | nLambda = 100, 15 | nGamma = 10, 16 | gammaMax = 10, 17 | gammaMin = 1e-04, 18 | partialSort = TRUE, 19 | maxIters = 200, 20 | rtol = 1e-06, 21 | atol = 1e-09, 22 | activeSet = TRUE, 23 | activeSetNum = 3, 24 | maxSwaps = 100, 25 | scaleDownFactor = 0.8, 26 | screenSize = 1000, 27 | autoLambda = NULL, 28 | lambdaGrid = list(), 29 | excludeFirstK = 0, 30 | intercept = TRUE, 31 | lows = -Inf, 32 | highs = Inf 33 | ) 34 | } 35 | \arguments{ 36 | \item{x}{The data matrix.} 37 | 38 | \item{y}{The response vector. For classification, we only support binary vectors.} 39 | 40 | \item{loss}{The loss function. Currently we support the choices "SquaredError" (for regression), "Logistic" (for logistic regression), and "SquaredHinge" (for smooth SVM).} 41 | 42 | \item{penalty}{The type of regularization. This can take either one of the following choices: 43 | "L0", "L0L2", and "L0L1".} 44 | 45 | \item{algorithm}{The type of algorithm used to minimize the objective function. Currently "CD" and "CDPSI" are 46 | are supported. "CD" is a variant of cyclic coordinate descent and runs very fast. "CDPSI" performs 47 | local combinatorial search on top of CD and typically achieves higher quality solutions (at the expense 48 | of increased running time).} 49 | 50 | \item{maxSuppSize}{The maximum support size at which to terminate the regularization path. We recommend setting 51 | this to a small fraction of min(n,p) (e.g. 0.05 * min(n,p)) as L0 regularization typically selects a small 52 | portion of non-zeros.} 53 | 54 | \item{nLambda}{The number of Lambda values to select (recall that Lambda is the regularization parameter 55 | corresponding to the L0 norm). This value is ignored if 'lambdaGrid' is supplied.} 56 | 57 | \item{nGamma}{The number of Gamma values to select (recall that Gamma is the regularization parameter 58 | corresponding to L1 or L2, depending on the chosen penalty). This value is ignored if 'lambdaGrid' is supplied 59 | and will be set to length(lambdaGrid)} 60 | 61 | \item{gammaMax}{The maximum value of Gamma when using the L0L2 penalty. For the L0L1 penalty this is 62 | automatically selected.} 63 | 64 | \item{gammaMin}{The minimum value of Gamma when using the L0L2 penalty. For the L0L1 penalty, the minimum 65 | value of gamma in the grid is set to gammaMin * gammaMax. Note that this should be a strictly positive quantity.} 66 | 67 | \item{partialSort}{If TRUE partial sorting will be used for sorting the coordinates to do greedy cycling (see our paper for 68 | for details). Otherwise, full sorting is used.} 69 | 70 | \item{maxIters}{The maximum number of iterations (full cycles) for CD per grid point.} 71 | 72 | \item{rtol}{The relative tolerance which decides when to terminate optimization (based on the relative change in the objective between iterations).} 73 | 74 | \item{atol}{The absolute tolerance which decides when to terminate optimization (based on the absolute L2 norm of the residuals).} 75 | 76 | \item{activeSet}{If TRUE, performs active set updates.} 77 | 78 | \item{activeSetNum}{The number of consecutive times a support should appear before declaring support stabilization.} 79 | 80 | \item{maxSwaps}{The maximum number of swaps used by CDPSI for each grid point.} 81 | 82 | \item{scaleDownFactor}{This parameter decides how close the selected Lambda values are. The choice should be 83 | strictly between 0 and 1 (i.e., 0 and 1 are not allowed). Larger values lead to closer lambdas and typically to smaller 84 | gaps between the support sizes. For details, see our paper - Section 5 on Adaptive Selection of Tuning Parameters).} 85 | 86 | \item{screenSize}{The number of coordinates to cycle over when performing initial correlation screening.} 87 | 88 | \item{autoLambda}{Ignored parameter. Kept for backwards compatibility.} 89 | 90 | \item{lambdaGrid}{A grid of Lambda values to use in computing the regularization path. This is by default an empty list and is ignored. 91 | When specified, LambdaGrid should be a list of length 'nGamma', where the ith element (corresponding to the ith gamma) should be a decreasing sequence of lambda values 92 | which are used by the algorithm when fitting for the ith value of gamma (see the vignette for details).} 93 | 94 | \item{excludeFirstK}{This parameter takes non-negative integers. The first excludeFirstK features in x will be excluded from variable selection, 95 | i.e., the first excludeFirstK variables will not be included in the L0-norm penalty (they will still be included in the L1 or L2 norm penalties.).} 96 | 97 | \item{intercept}{If FALSE, no intercept term is included in the model.} 98 | 99 | \item{lows}{Lower bounds for coefficients. Either a scalar for all coefficients to have the same bound or a vector of size p (number of columns of X) where lows[i] is the lower bound for coefficient i.} 100 | 101 | \item{highs}{Upper bounds for coefficients. Either a scalar for all coefficients to have the same bound or a vector of size p (number of columns of X) where highs[i] is the upper bound for coefficient i.} 102 | } 103 | \value{ 104 | An S3 object of type "FastSparse" describing the regularization path. The object has the following members. 105 | \item{a0}{a0 is a list of intercept sequences. The ith element of the list (i.e., a0[[i]]) is the sequence of intercepts corresponding to the ith gamma value (i.e., gamma[i]).} 106 | \item{beta}{This is a list of coefficient matrices. The ith element of the list is a p x \code{length(lambda)} matrix which 107 | corresponds to the ith gamma value. The jth column in each coefficient matrix is the vector of coefficients for the jth lambda value.} 108 | \item{lambda}{This is the list of lambda sequences used in fitting the model. The ith element of lambda (i.e., lambda[[i]]) is the sequence 109 | of Lambda values corresponding to the ith gamma value.} 110 | \item{gamma}{This is the sequence of gamma values used in fitting the model.} 111 | \item{suppSize}{This is a list of support size sequences. The ith element of the list is a sequence of support sizes (i.e., number of non-zero coefficients) 112 | corresponding to the ith gamma value.} 113 | \item{converged}{This is a list of sequences for checking whether the algorithm has converged at every grid point. The ith element of the list is a sequence 114 | corresponding to the ith value of gamma, where the jth element in each sequence indicates whether the algorithm has converged at the jth value of lambda.} 115 | } 116 | \description{ 117 | Computes the regularization path for the specified loss function and 118 | penalty function (which can be a combination of the L0, L1, and L2 norms). 119 | } 120 | \examples{ 121 | # Generate synthetic data for this example 122 | data <- GenSynthetic(n=500,p=1000,k=10,seed=1) 123 | X = data$X 124 | y = data$y 125 | 126 | # Fit an L0 regression model with a maximum of 50 non-zeros using coordinate descent (CD) 127 | fit1 <- FastSparse.fit(X, y, penalty="L0", maxSuppSize=50) 128 | print(fit1) 129 | # Extract the coefficients at lambda = 0.0325142 130 | coef(fit1, lambda=0.0325142) 131 | # Apply the fitted model on X to predict the response 132 | predict(fit1, newx = X, lambda=0.0325142) 133 | 134 | # Fit an L0 regression model with a maximum of 50 non-zeros using CD and local search 135 | fit2 <- FastSparse.fit(X, y, penalty="L0", algorithm="CDPSI", maxSuppSize=50) 136 | print(fit2) 137 | 138 | # Fit an L0L2 regression model with 10 values of Gamma ranging from 0.0001 to 10, using CD 139 | fit3 <- FastSparse.fit(X, y, penalty="L0L2", maxSuppSize=50, nGamma=10, gammaMin=0.0001, gammaMax = 10) 140 | print(fit3) 141 | # Extract the coefficients at lambda = 0.0361829 and gamma = 0.0001 142 | coef(fit3, lambda=0.0361829, gamma=0.0001) 143 | # Apply the fitted model on X to predict the response 144 | predict(fit3, newx = X, lambda=0.0361829, gamma=0.0001) 145 | 146 | # Fit an L0 logistic regression model 147 | # First, convert the response to binary 148 | y = sign(y) 149 | fit4 <- FastSparse.fit(X, y, loss="Logistic", maxSuppSize=20) 150 | print(fit4) 151 | 152 | } 153 | -------------------------------------------------------------------------------- /FastSparse/man/GenSynthetic.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/gensynthetic.R 3 | \name{GenSynthetic} 4 | \alias{GenSynthetic} 5 | \title{Generate Synthetic Data} 6 | \usage{ 7 | GenSynthetic(n, p, k, seed, rho = 0, b0 = 0, error_ratio = 1) 8 | } 9 | \arguments{ 10 | \item{n}{Number of samples} 11 | 12 | \item{p}{Number of features} 13 | 14 | \item{k}{Number of non-zeros in true vector of coefficients} 15 | 16 | \item{seed}{The seed used for randomly generating the data} 17 | 18 | \item{rho}{The threshold for setting values to 0. if |X(i, j)| > rho => X(i, j) <- 0} 19 | 20 | \item{b0}{intercept value to translate y by.} 21 | 22 | \item{error_ratio}{multiplier for the magnitude of the error term 'e'.} 23 | } 24 | \value{ 25 | A list containing: 26 | the data matrix X, 27 | the response vector y, 28 | the coefficients B, 29 | the error vector e, 30 | the intercept term b0. 31 | } 32 | \description{ 33 | Generates a synthetic dataset as follows: 1) Sample every element in data matrix X from N(0,1). 34 | 2) Generate a vector B with the first k entries set to 1 and the rest are zeros. 3) Sample every element in the noise 35 | vector e from N(0,1). 4) Set y = XB + b0 + e. 36 | } 37 | \examples{ 38 | data <- GenSynthetic(n=500,p=1000,k=10,seed=1) 39 | X = data$X 40 | y = data$y 41 | } 42 | -------------------------------------------------------------------------------- /FastSparse/man/GenSyntheticHighCorr.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/genhighcorr.R 3 | \name{GenSyntheticHighCorr} 4 | \alias{GenSyntheticHighCorr} 5 | \title{Generate Expoententiall Correlated Synthetic Data} 6 | \usage{ 7 | GenSyntheticHighCorr( 8 | n, 9 | p, 10 | k, 11 | seed, 12 | rho = 0, 13 | b0 = 0, 14 | noise_ratio = 1, 15 | mu = 0, 16 | base_cor = 0.9 17 | ) 18 | } 19 | \arguments{ 20 | \item{n}{Number of samples} 21 | 22 | \item{p}{Number of features} 23 | 24 | \item{k}{Number of non-zeros in true vector of coefficients} 25 | 26 | \item{seed}{The seed used for randomly generating the data} 27 | 28 | \item{rho}{The threshold for setting values to 0. if |X(i, j)| > rho => X(i, j) <- 0} 29 | 30 | \item{b0}{intercept value to scale y by.} 31 | 32 | \item{noise_ratio}{The multiplier of noise to apply when calculating e. e[i] = noise_ratio*N(0, 1).} 33 | 34 | \item{mu}{The mean for drawing from the Multivariate Normal Distribution. A scalar of vector of length p.} 35 | 36 | \item{base_cor}{The base correlation, A in [i, j] = A^|i-j|.} 37 | } 38 | \value{ 39 | A list containing: 40 | the data matrix X, 41 | the response vector y, 42 | the coefficients B, 43 | the error vector e, 44 | the intercept term b0. 45 | } 46 | \description{ 47 | Generates a synthetic dataset as follows: 1) Generate a correlation matrix, SIG, where item [i, j] = A^|i-j|. 48 | 2) Draw from a Multivariate Normal Distribution using (mu and SIG) to generate X. 3) Generate a vector B with every ~p/k entry set to 1 and the rest are zeros. 49 | 4) Sample every element in the noise vector e from N(0,1). 4) Set y = XB + b0 + e. 50 | } 51 | \examples{ 52 | data <- GenSyntheticHighCorr(n=500,p=1000,k=10,seed=1) 53 | X = data$X 54 | y = data$y 55 | } 56 | -------------------------------------------------------------------------------- /FastSparse/man/coef.FastSparse.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/coef.R 3 | \name{coef.FastSparse} 4 | \alias{coef.FastSparse} 5 | \alias{coef.FastSparseCV} 6 | \title{Extract Solutions} 7 | \usage{ 8 | \method{coef}{FastSparse}(object, lambda = NULL, gamma = NULL, ...) 9 | 10 | \method{coef}{FastSparseCV}(object, lambda = NULL, gamma = NULL, ...) 11 | } 12 | \arguments{ 13 | \item{object}{The output of FastSparse.fit or FastSparse.cvfit} 14 | 15 | \item{lambda}{The value of lambda at which to extract the solution.} 16 | 17 | \item{gamma}{The value of gamma at which to extract the solution.} 18 | 19 | \item{...}{ignore} 20 | } 21 | \description{ 22 | Extracts a specific solution in the regularization path. 23 | } 24 | \details{ 25 | If both lambda and gamma are not supplied, then a matrix of coefficients 26 | for all the solutions in the regularization path is returned. If lambda is 27 | supplied but gamma is not, the smallest value of gamma is used. 28 | } 29 | \examples{ 30 | # Generate synthetic data for this example 31 | data <- GenSynthetic(n=500,p=1000,k=10,seed=1) 32 | X = data$X 33 | y = data$y 34 | 35 | # Fit an L0L2 Model with 10 values of Gamma ranging from 0.0001 to 10, using coordinate descent 36 | fit <- FastSparse.fit(X, y, penalty="L0L2", maxSuppSize=50, nGamma=10, gammaMin=0.0001, gammaMax = 10) 37 | print(fit) 38 | # Extract the coefficients of the solution at lambda = 0.0361829 and gamma = 0.0001 39 | coef(fit, lambda=0.0361829, gamma=0.0001) 40 | # Extract the coefficients of all the solutions in the path 41 | coef(fit) 42 | 43 | } 44 | -------------------------------------------------------------------------------- /FastSparse/man/plot.FastSparse.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/plot.R 3 | \name{plot.FastSparse} 4 | \alias{plot.FastSparse} 5 | \title{Plot Regularization Path} 6 | \usage{ 7 | \method{plot}{FastSparse}(x, gamma = 0, showLines = FALSE, ...) 8 | } 9 | \arguments{ 10 | \item{x}{The output of FastSparse.fit} 11 | 12 | \item{gamma}{The value of gamma at which to plot.} 13 | 14 | \item{showLines}{If TRUE, the lines connecting the points in the plot are shown.} 15 | 16 | \item{...}{ignore} 17 | } 18 | \description{ 19 | Plots the regularization path for a given gamma. 20 | } 21 | \examples{ 22 | # Generate synthetic data for this example 23 | data <- GenSynthetic(n=500,p=1000,k=10,seed=1) 24 | X = data$X 25 | y = data$y 26 | # Fit an L0 Model with a maximum of 50 non-zeros 27 | fit <- FastSparse.fit(X, y, penalty="L0", maxSuppSize=50) 28 | plot(fit, gamma=0) 29 | 30 | } 31 | -------------------------------------------------------------------------------- /FastSparse/man/plot.FastSparseCV.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/plot.R 3 | \name{plot.FastSparseCV} 4 | \alias{plot.FastSparseCV} 5 | \title{Plot Cross-validation Errors} 6 | \usage{ 7 | \method{plot}{FastSparseCV}(x, gamma = 0, ...) 8 | } 9 | \arguments{ 10 | \item{x}{The output of FastSparse.cvfit} 11 | 12 | \item{gamma}{The value of gamma at which to plot.} 13 | 14 | \item{...}{ignore} 15 | } 16 | \description{ 17 | Plots cross-validation errors for a given gamma. 18 | } 19 | \examples{ 20 | # Generate synthetic data for this example 21 | data <- GenSynthetic(n=500,p=1000,k=10,seed=1) 22 | X = data$X 23 | y = data$y 24 | 25 | # Perform 5-fold cross-validation on an L0L2 Model with 5 values of 26 | # Gamma ranging from 0.0001 to 10 27 | fit <- FastSparse.cvfit(X, y, nFolds=5, seed=1, penalty="L0L2", 28 | maxSuppSize=20, nGamma=5, gammaMin=0.0001, gammaMax = 10) 29 | # Plot the graph of cross-validation error versus lambda for gamma = 0.0001 30 | plot(fit, gamma=0.0001) 31 | 32 | } 33 | -------------------------------------------------------------------------------- /FastSparse/man/predict.FastSparse.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/predict.R 3 | \name{predict.FastSparse} 4 | \alias{predict.FastSparse} 5 | \alias{predict.FastSparseCV} 6 | \title{Predict Response} 7 | \usage{ 8 | \method{predict}{FastSparse}(object, newx, lambda = NULL, gamma = NULL, ...) 9 | 10 | \method{predict}{FastSparseCV}(object, newx, lambda = NULL, gamma = NULL, ...) 11 | } 12 | \arguments{ 13 | \item{object}{The output of FastSparse.fit or FastSparse.cvfit} 14 | 15 | \item{newx}{A matrix on which predictions are made. The matrix should have p columns.} 16 | 17 | \item{lambda}{The value of lambda to use for prediction. A summary of the lambdas in the regularization 18 | path can be obtained using \code{print(fit)}.} 19 | 20 | \item{gamma}{The value of gamma to use for prediction. A summary of the gammas in the regularization 21 | path can be obtained using \code{print(fit)}.} 22 | 23 | \item{...}{ignore} 24 | } 25 | \description{ 26 | Predicts the response for a given sample. 27 | } 28 | \details{ 29 | If both lambda and gamma are not supplied, then a matrix of predictions 30 | for all the solutions in the regularization path is returned. If lambda is 31 | supplied but gamma is not, the smallest value of gamma is used. In case of 32 | of logistic regression, probability values are returned. 33 | } 34 | \examples{ 35 | # Generate synthetic data for this example 36 | data <- GenSynthetic(n=500,p=1000,k=10,seed=1) 37 | X = data$X 38 | y = data$y 39 | 40 | # Fit an L0L2 Model with 10 values of Gamma ranging from 0.0001 to 10, using coordinate descent 41 | fit <- FastSparse.fit(X,y, penalty="L0L2", maxSuppSize=50, nGamma=10, gammaMin=0.0001, gammaMax = 10) 42 | print(fit) 43 | # Apply the fitted model with lambda=0.0361829 and gamma=0.0001 on X to predict the response 44 | predict(fit, newx = X, lambda=0.0361829, gamma=0.0001) 45 | # Apply the fitted model on X to predict the response for all the solutions in the path 46 | predict(fit, newx = X) 47 | 48 | } 49 | -------------------------------------------------------------------------------- /FastSparse/man/print.FastSparse.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/print.R 3 | \name{print.FastSparse} 4 | \alias{print.FastSparse} 5 | \alias{print.FastSparseCV} 6 | \title{Print FastSparse.fit object} 7 | \usage{ 8 | \method{print}{FastSparse}(x, ...) 9 | 10 | \method{print}{FastSparseCV}(x, ...) 11 | } 12 | \arguments{ 13 | \item{x}{The output of FastSparse.fit or FastSparse.cvfit} 14 | 15 | \item{...}{ignore} 16 | } 17 | \description{ 18 | Prints a summary of FastSparse.fit 19 | } 20 | -------------------------------------------------------------------------------- /FastSparse/src/BetaVector.cpp: -------------------------------------------------------------------------------- 1 | #include "BetaVector.h" 2 | 3 | /* 4 | * arma::vec implementation 5 | */ 6 | 7 | std::vector nnzIndicies(const arma::vec& B){ 8 | // Returns a vector of the Non Zero Indicies of B 9 | const arma::ucolvec nnzs_indicies = arma::find(B); 10 | return arma::conv_to>::from(nnzs_indicies); 11 | } 12 | 13 | std::vector nnzIndicies(const arma::sp_mat& B){ 14 | // Returns a vector of the Non Zero Indicies of B 15 | std::vector S; 16 | arma::sp_mat::const_iterator it; 17 | const arma::sp_mat::const_iterator it_end = B.end(); 18 | for(it = B.begin(); it != it_end; ++it) 19 | { 20 | S.push_back(it.row()); 21 | } 22 | return S; 23 | } 24 | 25 | std::vector nnzIndicies(const arma::vec& B, const std::size_t low){ 26 | // Returns a vector of the Non Zero Indicies of a slice of B starting at low 27 | // This is for NoSelectK situations 28 | const arma::vec B_slice = B.subvec(low, B.n_rows-1); 29 | const arma::ucolvec nnzs_indicies = arma::find(B_slice); 30 | return arma::conv_to>::from(nnzs_indicies); 31 | } 32 | 33 | std::vector nnzIndicies(const arma::sp_mat& B, const std::size_t low){ 34 | // Returns a vector of the Non Zero Indicies of B 35 | std::vector S; 36 | 37 | 38 | arma::sp_mat::const_iterator it; 39 | const arma::sp_mat::const_iterator it_end = B.end(); 40 | 41 | 42 | for(it = B.begin(); it != it_end; ++it) 43 | { 44 | if (it.row() >= low){ 45 | S.push_back(it.row()); 46 | } 47 | } 48 | return S; 49 | } 50 | 51 | 52 | std::size_t n_nonzero(const arma::vec& B){ 53 | const arma::vec nnzs = arma::nonzeros(B); 54 | return nnzs.n_rows; 55 | 56 | } 57 | 58 | std::size_t n_nonzero(const arma::sp_mat& B){ 59 | return B.n_nonzero; 60 | 61 | } 62 | 63 | bool has_same_support(const arma::vec& B1, const arma::vec& B2){ 64 | if (B1.size() != B2.size()){ 65 | return false; 66 | } 67 | std::size_t n = B1.n_rows; 68 | 69 | bool same_support = true; 70 | for (std::size_t i = 0; i < n; i++){ 71 | same_support = same_support && ((B1.at(i) != 0) == (B2.at(i) != 0)); 72 | } 73 | return same_support; 74 | } 75 | 76 | bool has_same_support(const arma::sp_mat& B1, const arma::sp_mat& B2){ 77 | 78 | if (B1.n_nonzero != B2.n_nonzero) { 79 | return false; 80 | } else { // same number of nnz and Supp is sorted 81 | arma::sp_mat::const_iterator i1, i2; 82 | const arma::sp_mat::const_iterator i1_end = B1.end(); 83 | 84 | 85 | for(i1 = B1.begin(), i2 = B2.begin(); i1 != i1_end; ++i1, ++i2) 86 | { 87 | if(i1.row() != i2.row()) 88 | { 89 | return false; 90 | } 91 | } 92 | return true; 93 | } 94 | } 95 | 96 | -------------------------------------------------------------------------------- /FastSparse/src/CDL012ExponentialSwaps.cpp: -------------------------------------------------------------------------------- 1 | #include "CDL012ExponentialSwaps.h" 2 | 3 | template 4 | CDL012ExponentialSwaps::CDL012ExponentialSwaps(const T& Xi, const arma::vec& yi, const Params& Pi) : CDSwaps(Xi, yi, Pi) { 5 | twolambda2 = 2 * this->lambda2; 6 | qp2lamda2 = (LipschitzConst + twolambda2); // this is the univariate lipschitz const of the differentiable objective 7 | this->thr2 = (2 * this->lambda0) / qp2lamda2; 8 | this->thr = std::sqrt(this->thr2); 9 | stl0Lc = std::sqrt((2 * this->lambda0) * qp2lamda2); 10 | lambda1ol = this->lambda1 / qp2lamda2; 11 | Xy = Pi.Xy; 12 | 13 | this->Xy_neg_indices = Pi.Xy_neg_indices; 14 | } 15 | 16 | template 17 | FitResult CDL012ExponentialSwaps::_FitWithBounds() { 18 | throw "This Error should not happen. Please report it as an issue to https://github.com/jiachangliu/fastSparse "; 19 | } 20 | 21 | template 22 | FitResult CDL012ExponentialSwaps::_Fit() { 23 | // std::cout << "CDL012LogisticSwaps.cpp i'm in line 22\n"; 24 | 25 | auto result = CDL012Exponential(*(this->X), *(this->y), this->P).Fit(); // result will be maintained till the end 26 | this->b0 = result.b0; // Initialize from previous later....! 27 | this->B = result.B; 28 | arma::vec inverse_ExpyXB = result.inverse_ExpyXB; // Maintained throughout the algorithm 29 | 30 | double Fmin = arma::sum(inverse_ExpyXB); 31 | std::size_t maxindex; 32 | double Bmaxindex; 33 | 34 | this->P.Init = 'u'; 35 | 36 | bool foundbetter = false; 37 | bool foundbetter_i = false; 38 | 39 | int start_NnzIndices_value = 0; 40 | arma::rowvec frequency_count = arma::zeros(this->p); 41 | 42 | std::size_t ll_max = std::min(50, (int) this->p); // consider set 50 to be a parameter 43 | arma::uvec support_indices = arma::regspace(0, this->p - 1); 44 | for (std::size_t t = 0; t < this->MaxNumSwaps; ++t) { 45 | // std::cout << "Exponential Swaps is fitting!\n"; 46 | ///////////////////////////////////////////////////// Sequential order 47 | // std::vector NnzIndices = nnzIndicies(this->B, this->NoSelectK); 48 | 49 | ///////////////////////////////////////////////////// Priority Queue order 50 | std::vector NnzIndices = nnzIndicies(this->B, this->NoSelectK); 51 | std::sort(NnzIndices.begin(), NnzIndices.end(), [&](std::size_t i, std::size_t j){ 52 | if (frequency_count(i) == frequency_count(j)){ 53 | return i < j; 54 | } 55 | return frequency_count(i) < frequency_count(j); 56 | }); 57 | ///////////////////////////////////////////////////// 58 | 59 | 60 | foundbetter = false; 61 | 62 | // TODO: Check if this should be Templated Operation 63 | arma::mat inverse_ExpyXBnojs = arma::zeros(this->n, NnzIndices.size()); 64 | 65 | int j_index = -1; 66 | for (auto& j : NnzIndices) 67 | { 68 | // Remove NnzIndices[j] 69 | ++j_index; 70 | inverse_ExpyXBnojs.col(j_index) = inverse_ExpyXB % arma::exp( this->B.at(j) * matrix_column_get(*(this->Xy), j)); 71 | 72 | } 73 | arma::mat gradients = - inverse_ExpyXBnojs.t() * *Xy; 74 | arma::mat abs_gradients = arma::abs(gradients); 75 | 76 | j_index = -1; 77 | for (auto& j : NnzIndices) { 78 | // std::cout << "trying to find alternative to index " + std::to_string(j) + "\n"; 79 | // Set B[j] = 0 80 | ++j_index; 81 | arma::vec inverse_ExpyXBnoj = inverse_ExpyXBnojs.col(j_index); 82 | arma::rowvec gradient = gradients.row(j_index); 83 | arma::rowvec abs_gradient = abs_gradients.row(j_index); 84 | 85 | // arma::uvec indices = arma::sort_index(arma::abs(gradient), "descend"); 86 | std::partial_sort(support_indices.begin(), support_indices.begin()+ll_max, support_indices.end(), [&](std::size_t ii, std::size_t jj){ 87 | return abs_gradient(ii) > abs_gradient(jj); 88 | }); // partial sort 89 | foundbetter_i = false; 90 | 91 | double loss_noj = arma::sum(inverse_ExpyXBnoj); 92 | // TODO: make sure this scans at least 100 coordinates from outside supp (now it does not) 93 | for(std::size_t ll = 0; ll < ll_max; ++ll) { 94 | std::size_t i = support_indices(ll); 95 | 96 | if(this->B[i] == 0 && i >= this->NoSelectK) { 97 | // Do not swap B[i] if i between 0 and NoSelectK; 98 | 99 | indices = (*(this->Xy_neg_indices))[i]; 100 | double d_minus_tmp = arma::sum(inverse_ExpyXBnoj.elem(indices))/loss_noj; 101 | 102 | double ObjTemp = loss_noj * 2 * std::sqrt(d_minus_tmp*(1-d_minus_tmp)); 103 | 104 | // double Biold = 0; 105 | // double partial_i = -0.5*std::log((1.0-d_minus_tmp)/d_minus_tmp); 106 | // double Binew = Biold - partial_i; 107 | double Binew = 0.5*std::log((1.0-d_minus_tmp)/d_minus_tmp); 108 | 109 | 110 | ////////////////////////////////////////////////////////////////////////////////// 111 | 112 | if (ObjTemp < Fmin) { 113 | // std::cout << "because loss terms are " + std::to_string(ObjTemp) + " vs " + std::to_string(Fmin) + "\n"; 114 | Fmin = ObjTemp; 115 | maxindex = i; 116 | Bmaxindex = Binew; 117 | foundbetter_i = true; 118 | // std::cout << "successfully swapping index " + std::to_string(j) + " with index " + std::to_string(i) + " at iteration " + std::to_string(ll) + "\n"; 119 | } else { 120 | // std::cout << "unsuccessfully swapping with index " + std::to_string(i) + "\n"; 121 | frequency_count(j) += 1; 122 | } 123 | } 124 | 125 | if (foundbetter_i) { 126 | this->B[j] = 0; 127 | this->B[maxindex] = Bmaxindex; 128 | this->P.InitialSol = &(this->B); 129 | 130 | // TODO: Check if this line is necessary. P should already have b0. 131 | this->P.b0 = this->b0; 132 | 133 | result = CDL012Exponential(*(this->X), *(this->y), this->P).Fit(); 134 | 135 | inverse_ExpyXB = result.inverse_ExpyXB; 136 | this->B = result.B; 137 | this->b0 = result.b0; 138 | 139 | // // early stopping 140 | // const arma::ucolvec wronglyClassifiedIndicies = arma::find(inverse_ExpyXB > 1); 141 | // if (wronglyClassifiedIndicies.size() == 0) { 142 | // // std::cout << "Early stopping because classification accuracy is 100%\n"; 143 | // return result; 144 | // } 145 | // std::cout << "there are " + std::to_string(wronglyClassifiedIndicies.size()) + " wrongly classified points\n"; 146 | 147 | Fmin = arma::sum(inverse_ExpyXB); 148 | foundbetter = true; 149 | break; 150 | } 151 | } 152 | 153 | //auto end2 = std::chrono::high_resolution_clock::now(); 154 | //std::cout<<"restricted: "<(end2-start2).count() << " ms " << std::endl; 155 | 156 | if (foundbetter){ 157 | break; 158 | } 159 | 160 | } 161 | 162 | if(!foundbetter) { 163 | // Early exit to prevent looping 164 | return result; 165 | } 166 | } 167 | 168 | //result.Model = this; 169 | return result; 170 | } 171 | 172 | template class CDL012ExponentialSwaps; 173 | template class CDL012ExponentialSwaps; 174 | -------------------------------------------------------------------------------- /FastSparse/src/CDL012SquaredHingeSwaps.cpp: -------------------------------------------------------------------------------- 1 | #include "CDL012SquaredHingeSwaps.h" 2 | 3 | template 4 | CDL012SquaredHingeSwaps::CDL012SquaredHingeSwaps(const T& Xi, const arma::vec& yi, const Params& Pi) : CDSwaps(Xi, yi, Pi) { 5 | twolambda2 = 2 * this->lambda2; 6 | qp2lamda2 = (LipschitzConst + twolambda2); // this is the univariate lipschitz constant of the differentiable objective 7 | this->thr2 = (2 * this->lambda0) / qp2lamda2; 8 | this->thr = std::sqrt(this->thr2); 9 | stl0Lc = std::sqrt((2 * this->lambda0) * qp2lamda2); 10 | lambda1ol = this->lambda1 / qp2lamda2; 11 | } 12 | 13 | template 14 | FitResult CDL012SquaredHingeSwaps::_FitWithBounds() { 15 | throw "This Error should not happen. Please report it as an issue to https://github.com/jiachangliu/fastSparse"; 16 | } 17 | 18 | template 19 | FitResult CDL012SquaredHingeSwaps::_Fit() { 20 | auto result = CDL012SquaredHinge(*(this->X), *(this->y), this->P).Fit(); // result will be maintained till the end 21 | this->b0 = result.b0; // Initialize from previous later....! 22 | this->B = result.B; 23 | 24 | arma::vec onemyxb = result.onemyxb; 25 | 26 | this->objective = result.Objective; 27 | double Fmin = this->objective; 28 | std::size_t maxindex; 29 | double Bmaxindex; 30 | 31 | this->P.Init = 'u'; 32 | 33 | bool foundbetter = false; 34 | 35 | for (std::size_t t = 0; t < this->MaxNumSwaps; ++t) { 36 | // Rcpp::Rcout << "Swap Number: " << t << "|mean(onemyxb): " << arma::mean(onemyxb) << "\n"; 37 | 38 | std::vector NnzIndices = nnzIndicies(this->B, this->NoSelectK); 39 | 40 | // TODO: Implement shuffle of NnzIndices Indicies 41 | 42 | foundbetter = false; 43 | 44 | for (auto& j : NnzIndices) { 45 | 46 | arma::vec onemyxbnoj = onemyxb + this->B[j] * *(this->y) % matrix_column_get(*(this->X), j); 47 | arma::uvec indices = arma::find(onemyxbnoj > 0); 48 | 49 | 50 | for(std::size_t i = 0; i < this->p; ++i) { 51 | if(this->B[i] == 0 && i>=this->NoSelectK) { 52 | 53 | double Biold = 0; 54 | double Binew; 55 | 56 | 57 | double partial_i = arma::sum(2 * onemyxbnoj.elem(indices) % (- (this->y)->elem(indices) % matrix_column_get(*(this->X), i).elem(indices))); 58 | 59 | bool converged = false; 60 | if (std::abs(partial_i) >= this->lambda1 + stl0Lc){ 61 | 62 | //std::cout<<"Adding: "<B; 67 | Btemp[j] = 0; 68 | //double ObjTemp = Objective(onemyxbnoj,Btemp); 69 | //double Biolddescent = 0; 70 | while(!converged) { 71 | 72 | double x = Biold - partial_i / qp2lamda2; 73 | double z = std::abs(x) - lambda1ol; 74 | Binew = std::copysign(z, x); 75 | 76 | // Binew = clamp(std::copysign(z, x), this->Lows[i], this->Highs[i]); // no need to check if >= sqrt(2lambda_0/Lc) 77 | onemyxbnoji += (Biold - Binew) * *(this->y) % matrix_column_get(*(this->X), i); 78 | 79 | arma::uvec indicesi = arma::find(onemyxbnoji > 0); 80 | partial_i = arma::sum(2 * onemyxbnoji.elem(indicesi) % (- (this->y)->elem(indicesi) % matrix_column_get(*(this->X), i).elem(indicesi))); 81 | 82 | if (std::abs((Binew - Biold) / Biold) < 0.0001){ 83 | converged = true; 84 | } 85 | 86 | Biold = Binew; 87 | l += 1; 88 | 89 | } 90 | 91 | Btemp[i] = Binew; 92 | double Fnew = Objective(onemyxbnoji, Btemp); 93 | 94 | if (Fnew < Fmin) { 95 | Fmin = Fnew; 96 | maxindex = i; 97 | Bmaxindex = Binew; 98 | } 99 | } 100 | } 101 | } 102 | 103 | if (Fmin < this->objective) { 104 | this->B[j] = 0; 105 | this->B[maxindex] = Bmaxindex; 106 | 107 | this->P.InitialSol = &(this->B); 108 | 109 | // TODO: Check if this line is needed. P should already have b0. 110 | this->P.b0 = this->b0; 111 | 112 | result = CDL012SquaredHinge(*(this->X), *(this->y), this->P).Fit(); 113 | 114 | this->B = result.B; 115 | this->b0 = result.b0; 116 | 117 | onemyxb = result.onemyxb; 118 | this->objective = result.Objective; 119 | Fmin = this->objective; 120 | foundbetter = true; 121 | break; 122 | } 123 | if (foundbetter){break;} 124 | } 125 | 126 | if(!foundbetter) { 127 | return result; 128 | } 129 | } 130 | 131 | return result; 132 | } 133 | 134 | template class CDL012SquaredHingeSwaps; 135 | template class CDL012SquaredHingeSwaps; 136 | -------------------------------------------------------------------------------- /FastSparse/src/CDL012Swaps.cpp: -------------------------------------------------------------------------------- 1 | #include "CDL012Swaps.h" 2 | 3 | template 4 | CDL012Swaps::CDL012Swaps(const T& Xi, const arma::vec& yi, const Params& Pi) : CDSwaps(Xi, yi, Pi) {} 5 | 6 | 7 | template 8 | FitResult CDL012Swaps::_FitWithBounds() { 9 | throw "This Error should not happen. Please report it as an issue to https://github.com/jiachangliu/fastSparse"; 10 | } 11 | 12 | template 13 | FitResult CDL012Swaps::_Fit() { 14 | auto result = CDL012(*(this->X), *(this->y), this->P).Fit(); // result will be maintained till the end 15 | this->B = result.B; 16 | this->b0 = result.b0; 17 | double objective = result.Objective; 18 | this->P.Init = 'u'; 19 | 20 | bool foundbetter = false; 21 | 22 | // std::cout<<"Hello, this is after CD.\n"; 23 | arma::rowvec frequency_count = arma::zeros(this->p); 24 | 25 | for (std::size_t t = 0; t < this->MaxNumSwaps; ++t) { 26 | 27 | ///////////////////////////////////////////////////// Method 1 28 | // std::vector NnzIndices = nnzIndicies(this->B, this->NoSelectK); 29 | ///////////////////////////////////////////////////// Priority Queue order, Method II 30 | std::vector NnzIndices = nnzIndicies(this->B, this->NoSelectK); 31 | std::sort(NnzIndices.begin(), NnzIndices.end(), [&](std::size_t i, std::size_t j){ 32 | if (frequency_count(i) == frequency_count(j)){ 33 | return i < j; 34 | } 35 | return frequency_count(i) < frequency_count(j); 36 | }); 37 | ///////////////////////////////////////////////////// 38 | 39 | foundbetter = false; 40 | 41 | // TODO: shuffle NNz Indices to prevent bias. 42 | //std::shuffle(std::begin(Order), std::end(Order), engine); 43 | 44 | // TODO: This calculation is already preformed in a previous step 45 | // Can be pulled/stored 46 | arma::vec r = *(this->y) - *(this->X) * this->B - this->b0; 47 | 48 | for (auto& i : NnzIndices) { 49 | arma::rowvec riX = (r + this->B[i] * matrix_column_get(*(this->X), i)).t() * *(this->X); 50 | 51 | double maxcorr = -1; 52 | std::size_t maxindex = -1; 53 | 54 | for(std::size_t j = this->NoSelectK; j < this->p; ++j) { 55 | // TODO: Account for bounds when determining best swap 56 | // Loops through each column and finds the column with the highest correlation to residuals 57 | // In non-constrained cases, the highest correlation will always be the best option 58 | // However, if bounds restrict the value of B[j], it is possible that swapping column 'i' 59 | // and column 'j' might be rejected as B[j], when constrained, is not able to take a value 60 | // with sufficient magnitude to utilize the correlation. 61 | // Therefore, we must ensure that 'j' was not already rejected. 62 | if (std::fabs(riX[j]) > maxcorr && this->B[j] == 0) { 63 | maxcorr = std::fabs(riX[j]); 64 | maxindex = j; 65 | } 66 | } 67 | 68 | // Check if the correlation is sufficiently large to make up for regularization 69 | if(maxcorr > (1 + 2 * this->ModelParams[2])*std::fabs(this->B[i]) + this->ModelParams[1]) { 70 | // Rcpp::Rcout << t << ": Proposing Swap " << i << " => NNZ and " << maxindex << " => 0 \n"; 71 | // Proposed new Swap 72 | // Value (without considering bounds are solvable in closed form) 73 | // Must be clamped to bounds 74 | // std::cout<<"I'm doing swapping!!!\n"; 75 | this->B[i] = 0; 76 | 77 | // Bi with No Bounds (nb); 78 | double Bi_nb = (riX[maxindex] - std::copysign(this->ModelParams[1],riX[maxindex])) / (1 + 2 * this->ModelParams[2]); 79 | //double Bi_wb = clamp(Bi_nb, this->Lows[maxindex], this->Highs[maxindex]); // Bi With Bounds (wb) 80 | this->B[maxindex] = Bi_nb; 81 | 82 | // Change initial solution to Swapped value to seed standard CD algorithm. 83 | this->P.InitialSol = &(this->B); 84 | *this->P.r = *(this->y) - *(this->X) * (this->B) - this->b0; 85 | // this->P already has access to b0. 86 | 87 | // proposed_result object. 88 | // Keep tack of previous_best result object 89 | // Only override previous_best if proposed_result has a better objective. 90 | result = CDL012(*(this->X), *(this->y), this->P).Fit(); 91 | 92 | // Rcpp::Rcout << "Swap Objective " << result.Objective << " \n"; 93 | // Rcpp::Rcout << "Old Objective " << objective << " \n"; 94 | this->B = result.B; 95 | objective = result.Objective; 96 | foundbetter = true; 97 | break; 98 | } 99 | else { 100 | frequency_count(i) += 1; 101 | } 102 | } 103 | 104 | if(!foundbetter) { 105 | // Early exit to prevent looping 106 | return result; 107 | } 108 | } 109 | 110 | return result; 111 | } 112 | 113 | template class CDL012Swaps; 114 | template class CDL012Swaps; 115 | 116 | -------------------------------------------------------------------------------- /FastSparse/src/Grid.cpp: -------------------------------------------------------------------------------- 1 | #include "Grid.h" 2 | 3 | // Assumes PG.P.Specs have been already set 4 | template 5 | Grid::Grid(const T& X, const arma::vec& y, const GridParams& PGi) { 6 | PG = PGi; 7 | // std::cout << "Grid.cpp i'm in line 7\n"; 8 | 9 | if (!PG.P.Specs.Exponential) { // if the loss is not exponential 10 | std::tie(BetaMultiplier, meanX, meany, scaley) = Normalize(X, 11 | y, Xscaled, yscaled, !PG.P.Specs.Classification, PG.intercept); 12 | } else { // if the loss is exponential 13 | Xscaled = 2*X-1; 14 | meanX.set_size(X.n_cols); 15 | meanX.fill(0.5); 16 | BetaMultiplier.set_size(X.n_cols); 17 | BetaMultiplier.fill(2.0); 18 | meany = 0; 19 | scaley = 1; 20 | yscaled = y; 21 | } 22 | // Must rescale bounds by BetaMultiplier in order for final result to conform to bounds 23 | if (PG.P.withBounds){ 24 | PG.P.Lows /= BetaMultiplier; 25 | PG.P.Highs /= BetaMultiplier; 26 | } 27 | } 28 | 29 | template 30 | void Grid::Fit() { 31 | 32 | std::vector>>> G; 33 | 34 | if (PG.P.Specs.L0) { 35 | // L0 is only used for linear regression 36 | // for other losses, a small L2 is added in fit.R so that the code runs the else statement below 37 | // std::cout << "Grid.cpp i'm in line 37\n"; 38 | G.push_back(std::move(Grid1D(Xscaled, yscaled, PG).Fit())); 39 | Lambda12.push_back(0); 40 | } else { 41 | // std::cout << "Grid.cpp i'm in line 41\n"; 42 | G = std::move(Grid2D(Xscaled, yscaled, PG).Fit()); 43 | } 44 | 45 | // std::cout << "Grid.cpp i'm in line 45\n"; 46 | 47 | Lambda0 = std::vector< std::vector >(G.size()); 48 | NnzCount = std::vector< std::vector >(G.size()); 49 | Solutions = std::vector< std::vector >(G.size()); 50 | Intercepts = std::vector< std::vector >(G.size()); 51 | Converged = std::vector< std::vector >(G.size()); 52 | Objectives = std::vector< std::vector >(G.size()); 53 | 54 | for (std::size_t i=0; iModelParams[1]); 57 | // std::cout << "Grid.cpp i'm in line 41\n"; 58 | } else if (PG.P.Specs.L0L2) { 59 | Lambda12.push_back(G[i][0]->ModelParams[2]); 60 | // std::cout << "Grid.cpp i'm in line 44\n"; 61 | } 62 | 63 | for (auto &g : G[i]) { 64 | Lambda0[i].push_back(g->ModelParams[0]); 65 | 66 | NnzCount[i].push_back(n_nonzero(g->B)); 67 | 68 | if (g->IterNum != PG.P.MaxIters){ 69 | Converged[i].push_back(true); 70 | } else { 71 | Converged[i].push_back(false); 72 | } 73 | 74 | Objectives[i].push_back(g->Objective); 75 | 76 | beta_vector B_unscaled; 77 | double b0; 78 | 79 | std::tie(B_unscaled, b0) = DeNormalize(g->B, BetaMultiplier, meanX, meany); 80 | Solutions[i].push_back(arma::sp_mat(B_unscaled)); 81 | /* scaley is 1 for classification problems. 82 | * g->intercept is 0 unless specifically optimized for in: 83 | * classification 84 | * sparse regression and intercept = true 85 | */ 86 | Intercepts[i].push_back(scaley*g->b0 + b0); 87 | } 88 | } 89 | } 90 | 91 | template class Grid; 92 | template class Grid; 93 | -------------------------------------------------------------------------------- /FastSparse/src/Grid1D.cpp: -------------------------------------------------------------------------------- 1 | #include "Grid1D.h" 2 | 3 | template 4 | Grid1D::Grid1D(const T& Xi, const arma::vec& yi, const GridParams& PG) { 5 | // automatically selects lambda_0 (but assumes other lambdas are given in PG.P.ModelParams) 6 | 7 | X = Ξ 8 | y = &yi; 9 | p = Xi.n_cols; 10 | LambdaMinFactor = PG.LambdaMinFactor; 11 | ScaleDownFactor = PG.ScaleDownFactor; 12 | P = PG.P; 13 | P.Xtr = new std::vector(X->n_cols); // needed! careful 14 | P.ytX = new arma::rowvec(X->n_cols); 15 | P.D = new std::map(); 16 | P.r = new arma::vec(Xi.n_rows); 17 | Xtr = P.Xtr; 18 | ytX = P.ytX; 19 | NoSelectK = P.NoSelectK; 20 | 21 | LambdaU = PG.LambdaU; 22 | 23 | if (!LambdaU) { 24 | G_ncols = PG.G_ncols; 25 | } else { 26 | G_ncols = PG.Lambdas.n_rows; // override the user's ncols if LambdaU = 1 27 | } 28 | 29 | G.reserve(G_ncols); 30 | if (LambdaU) { 31 | Lambdas = PG.Lambdas; 32 | } // user-defined lambda0 grid 33 | /* 34 | else { 35 | Lambdas.reserve(G_ncols); 36 | Lambdas.push_back((0.5*arma::square(y->t() * *X)).max()); 37 | } 38 | */ 39 | NnzStopNum = PG.NnzStopNum; 40 | PartialSort = PG.PartialSort; 41 | XtrAvailable = PG.XtrAvailable; 42 | if (XtrAvailable) { 43 | ytXmax2d = PG.ytXmax; 44 | Xtr = PG.Xtr; 45 | } 46 | } 47 | 48 | template 49 | Grid1D::~Grid1D() { 50 | // delete all dynamically allocated memory 51 | delete P.Xtr; 52 | delete P.ytX; 53 | delete P.D; 54 | delete P.r; 55 | } 56 | 57 | 58 | template 59 | std::vector>> Grid1D::Fit() { 60 | 61 | if (P.Specs.L0 || P.Specs.L0L2 || P.Specs.L0L1) { 62 | // std::cout << "Grid1D.cpp i'm in line 62\n"; 63 | bool scaledown = false; 64 | 65 | double Lipconst; 66 | arma::vec Xtrarma; 67 | if (P.Specs.Logistic) { 68 | if (!XtrAvailable) { 69 | Xtrarma = 0.5 * arma::abs(y->t() * *X).t(); 70 | } // = gradient of logistic loss at zero} 71 | Lipconst = 0.25 + 2 * P.ModelParams[2]; 72 | } else if (P.Specs.Exponential) { 73 | if (!XtrAvailable) { 74 | Xtrarma = 0.5 * arma::abs(y->t() * *X).t(); 75 | } // = gradient of logistic loss at zero} 76 | Lipconst = 0.25 + 2 * P.ModelParams[2]; 77 | } else if (P.Specs.SquaredHinge) { 78 | if (!XtrAvailable) { 79 | // gradient of loss function at zero 80 | Xtrarma = 2 * arma::abs(y->t() * *X).t(); 81 | } 82 | Lipconst = 2 + 2 * P.ModelParams[2]; 83 | } else { 84 | if (!XtrAvailable) { 85 | *ytX = y->t() * *X; 86 | Xtrarma = arma::abs(*ytX).t(); // Least squares 87 | } 88 | Lipconst = 1 + 2 * P.ModelParams[2]; 89 | *P.r = *y - P.b0; // B = 0 initially 90 | } 91 | 92 | double ytXmax; 93 | if (!XtrAvailable) { 94 | *Xtr = arma::conv_to< std::vector >::from(Xtrarma); 95 | ytXmax = arma::max(Xtrarma); 96 | } else { 97 | ytXmax = ytXmax2d; 98 | } 99 | 100 | double lambdamax = ((ytXmax - P.ModelParams[1]) * (ytXmax - P.ModelParams[1])) / (2 * (Lipconst)); 101 | 102 | // Rcpp::Rcout << "lambdamax: " << lambdamax << "\n"; 103 | 104 | if (!LambdaU) { 105 | P.ModelParams[0] = lambdamax; 106 | } else { 107 | P.ModelParams[0] = Lambdas[0]; 108 | } 109 | 110 | // Rcpp::Rcout << "P ModelParams: {" << P.ModelParams[0] << ", " << P.ModelParams[1] << ", " << P.ModelParams[2] << ", " << P.ModelParams[3] << "}\n"; 111 | 112 | P.Init = 'z'; 113 | 114 | 115 | //std::cout<< "Lambda max: "<< lambdamax << std::endl; 116 | //double lambdamin = lambdamax*LambdaMinFactor; 117 | //Lambdas = arma::logspace(std::log10(lambdamin), std::log10(lambdamax), G_ncols); 118 | //Lambdas = arma::flipud(Lambdas); 119 | 120 | 121 | //std::size_t StopNum = (X->n_rows < NnzStopNum) ? X->n_rows : NnzStopNum; 122 | std::size_t StopNum = NnzStopNum; 123 | //std::vector* Xtr = P.Xtr; 124 | std::vector idx(p); 125 | double Xrmax; 126 | bool prevskip = false; //previous grid point was skipped 127 | bool currentskip = false; // current grid point should be skipped 128 | 129 | for (std::size_t i = 0; i < G_ncols; ++i) { 130 | Rcpp::checkUserInterrupt(); 131 | // Rcpp::Rcout << "Grid1D: " << i << "\n"; 132 | FitResult * prevresult = new FitResult; // prevresult is ptr to the prev result object 133 | //std::unique_ptr prevresult; 134 | if (i > 0) { 135 | //prevresult = std::move(G.back()); 136 | *prevresult = *(G.back()); 137 | 138 | } 139 | 140 | currentskip = false; 141 | 142 | if (!prevskip) { 143 | 144 | std::iota(idx.begin(), idx.end(), 0); // make global class var later 145 | // Exclude the first NoSelectK features from sorting. 146 | if (PartialSort && p > 5000 + NoSelectK) 147 | std::partial_sort(idx.begin() + NoSelectK, idx.begin() + 5000 + NoSelectK, idx.end(), [this](std::size_t i1, std::size_t i2) {return (*Xtr)[i1] > (*Xtr)[i2] ;}); 148 | else 149 | std::sort(idx.begin() + NoSelectK, idx.end(), [this](std::size_t i1, std::size_t i2) {return (*Xtr)[i1] > (*Xtr)[i2] ;}); 150 | P.CyclingOrder = 'u'; 151 | P.Uorder = idx; // can be made faster 152 | 153 | // 154 | Xrmax = (*Xtr)[idx[NoSelectK]]; 155 | 156 | if (i > 0) { 157 | std::vector Sp = nnzIndicies(prevresult->B); 158 | 159 | for(std::size_t l = NoSelectK; l < p; ++l) { 160 | if ( std::binary_search(Sp.begin(), Sp.end(), idx[l]) == false ) { 161 | Xrmax = (*Xtr)[idx[l]]; 162 | //std::cout<<"Grid Iteration: "<> result(new FitResult); 190 | // std::cout << "Grid1D.cpp i'm in line 185\n"; 191 | *result = Model->Fit(); 192 | 193 | delete Model; 194 | 195 | scaledown = false; 196 | if (i >= 1) { 197 | std::vector Spold = nnzIndicies(prevresult->B); 198 | 199 | std::vector Spnew = nnzIndicies(result->B); 200 | 201 | bool samesupp = false; 202 | 203 | if (Spold == Spnew) { 204 | samesupp = true; 205 | scaledown = true; 206 | } 207 | 208 | // // 209 | // 210 | // if (samesupp) { 211 | // scaledown = true; 212 | // } // got same solution 213 | } 214 | 215 | //else {scaledown = false;} 216 | G.push_back(std::move(result)); 217 | 218 | 219 | if(n_nonzero(G.back()->B) >= StopNum) { 220 | break; 221 | } 222 | //result->B.t().print(); 223 | P.InitialSol = &(G.back()->B); 224 | P.b0 = G.back()->b0; 225 | // Udate: After 1.1.0, P.r is automatically updated by the previous call to CD 226 | //*P.r = G.back()->r; 227 | 228 | } 229 | 230 | delete prevresult; 231 | 232 | 233 | P.Init = 'u'; 234 | P.Iter += 1; 235 | prevskip = currentskip; 236 | } 237 | } 238 | 239 | return std::move(G); 240 | } 241 | 242 | 243 | template class Grid1D; 244 | template class Grid1D; 245 | -------------------------------------------------------------------------------- /FastSparse/src/Grid2D.cpp: -------------------------------------------------------------------------------- 1 | #include "Grid2D.h" 2 | 3 | template 4 | Grid2D::Grid2D(const T& Xi, const arma::vec& yi, const GridParams& PGi) 5 | { 6 | // automatically selects lambda_0 (but assumes other lambdas are given in PG.P.ModelParams) 7 | X = Ξ 8 | y = &yi; 9 | p = Xi.n_cols; 10 | PG = PGi; 11 | G_nrows = PG.G_nrows; 12 | G_ncols = PG.G_ncols; 13 | G.reserve(G_nrows); 14 | Lambda2Max = PG.Lambda2Max; 15 | Lambda2Min = PG.Lambda2Min; 16 | LambdaMinFactor = PG.LambdaMinFactor; 17 | 18 | P = PG.P; 19 | } 20 | 21 | template 22 | Grid2D::~Grid2D(){ 23 | delete Xtr; 24 | if (PG.P.Specs.Logistic) { 25 | delete PG.P.Xy; 26 | } 27 | if (PG.P.Specs.SquaredHinge) { 28 | delete PG.P.Xy; 29 | } 30 | if (PG.P.Specs.Exponential) { 31 | delete PG.P.Xy; 32 | delete PG.P.Xy_neg_indices; 33 | } 34 | } 35 | 36 | template 37 | std::vector< std::vector> > > Grid2D::Fit() { 38 | arma::vec Xtrarma; 39 | 40 | if (PG.P.Specs.Logistic) { 41 | // std::cout << "Grid2D.cpp i'm in line 35\n"; 42 | auto n = X->n_rows; 43 | double b0 = 0; 44 | arma::vec ExpyXB = arma::ones(n); 45 | if (PG.intercept) { 46 | for (std::size_t t = 0; t < 50; ++t) { 47 | double partial_b0 = - arma::sum( *y / (1 + ExpyXB) ); 48 | b0 -= partial_b0 / (n * 0.25); // intercept is not regularized 49 | ExpyXB = arma::exp(b0 * *y); 50 | } 51 | } 52 | PG.P.b0 = b0; 53 | Xtrarma = arma::abs(- arma::trans(*y /(1+ExpyXB)) * *X).t(); // = gradient of logistic loss at zero 54 | //Xtrarma = 0.5 * arma::abs(y->t() * *X).t(); // = gradient of logistic loss at zero 55 | 56 | T Xy = matrix_vector_schur_product(*X, y); // X->each_col() % *y; 57 | 58 | PG.P.Xy = new T; 59 | *PG.P.Xy = Xy; 60 | } else if (PG.P.Specs.Exponential) { 61 | // std::cout << "Grid2D.cpp i'm in line 61\n"; 62 | auto n = X->n_rows; 63 | double b0 = 0; 64 | // arma::vec ExpyXB = arma::ones(n); 65 | // if (PG.intercept) { 66 | // for (std::size_t t = 0; t < 50; ++t) { 67 | // double partial_b0 = - arma::sum( *y / (1 + ExpyXB) ); 68 | // b0 -= partial_b0 / (n * 0.25); // intercept is not regularized 69 | // ExpyXB = arma::exp(b0 * *y); 70 | // } 71 | // } 72 | 73 | 74 | // PG.P.b0 = b0; 75 | // Xtrarma = arma::abs(- arma::trans(*y /(1+ExpyXB)) * *X).t(); // = gradient of logistic loss at zero 76 | // //Xtrarma = 0.5 * arma::abs(y->t() * *X).t(); // = gradient of logistic loss at zero 77 | 78 | T Xy = matrix_vector_schur_product(*X, y); // X->each_col() % *y; 79 | 80 | PG.P.Xy = new T; 81 | *PG.P.Xy = Xy; 82 | 83 | std::unordered_map Xy_neg_indices; 84 | for (size_t tmp = 0; tmp < Xy.n_cols; ++tmp){ 85 | Xy_neg_indices.insert(std::make_pair(tmp, arma::find(matrix_column_get(*(PG.P.Xy), tmp) < 0))); 86 | } 87 | Xy_neg_indices.insert(std::make_pair(-1, arma::find(*y < 0))); 88 | PG.P.Xy_neg_indices = new std::unordered_map; 89 | *PG.P.Xy_neg_indices = Xy_neg_indices; 90 | 91 | // indices = (*(this->Xy_neg_indices))[-1]; 92 | // // this->d_minus = arma::sum(this->inverse_ExpyXB.elem(indices)) / arma::sum(this->inverse_ExpyXB); 93 | // this->d_minus = arma::sum(this->inverse_ExpyXB.elem(indices)) / this->current_expo_loss; 94 | // const double partial_b0 = -0.5*std::log((1-this->d_minus)/this->d_minus); 95 | // this->b0 -= partial_b0; 96 | 97 | arma::vec inverse_ExpyXB = arma::ones(n); 98 | // calcualte the exponential intercept when all coordinates are zero 99 | b0 = 0.0; 100 | if (PG.intercept) { 101 | arma::uvec indices = Xy_neg_indices[-1]; 102 | double d_minus = (double)indices.n_elem / (double)n; 103 | double partial_b0 = -0.5*std::log((1-d_minus)/d_minus); 104 | b0 -= partial_b0; 105 | inverse_ExpyXB %= arma::exp( partial_b0 * *y); 106 | } 107 | PG.P.b0 = b0; 108 | Xtrarma = arma::abs(- arma::trans(*y % inverse_ExpyXB) * *X).t(); // = gradient of logistic loss at zero 109 | } else if (PG.P.Specs.SquaredHinge) { 110 | auto n = X->n_rows; 111 | double b0 = 0; 112 | arma::vec onemyxb = arma::ones(n); 113 | arma::uvec indices = arma::find(onemyxb > 0); 114 | if (PG.intercept){ 115 | for (std::size_t t = 0; t < 50; ++t){ 116 | double partial_b0 = arma::sum(2 * onemyxb.elem(indices) % (- y->elem(indices) ) ); 117 | b0 -= partial_b0 / (n * 2); // intercept is not regularized 118 | onemyxb = 1 - (*y * b0); 119 | indices = arma::find(onemyxb > 0); 120 | } 121 | } 122 | PG.P.b0 = b0; 123 | T indices_rows = matrix_rows_get(*X, indices); 124 | Xtrarma = 2 * arma::abs(arma::trans(y->elem(indices) % onemyxb.elem(indices))* indices_rows).t(); // = gradient of loss function at zero 125 | //Xtrarma = 2 * arma::abs(y->t() * *X).t(); // = gradient of loss function at zero 126 | T Xy = matrix_vector_schur_product(*X, y); // X->each_col() % *y; 127 | PG.P.Xy = new T; 128 | *PG.P.Xy = Xy; 129 | } else { 130 | Xtrarma = arma::abs(y->t() * *X).t(); 131 | } 132 | 133 | 134 | double ytXmax = arma::max(Xtrarma); 135 | 136 | std::size_t index; 137 | if (PG.P.Specs.L0L1) { 138 | index = 1; 139 | if(G_nrows != 1) { 140 | Lambda2Max = ytXmax; 141 | Lambda2Min = Lambda2Max * LambdaMinFactor; 142 | } 143 | } else if (PG.P.Specs.L0L2) { 144 | index = 2; 145 | } 146 | 147 | arma::vec Lambdas2 = arma::logspace(std::log10(Lambda2Min), std::log10(Lambda2Max), G_nrows); 148 | Lambdas2 = arma::flipud(Lambdas2); 149 | 150 | std::vector Xtrvec = arma::conv_to< std::vector >::from(Xtrarma); 151 | 152 | Xtr = new std::vector(X->n_cols); // needed! careful 153 | 154 | 155 | PG.XtrAvailable = true; 156 | // Rcpp::Rcout << "Grid2D Start\n"; 157 | for(std::size_t i=0; i> Gl(); 170 | //auto Gl = Grid1D(*X, *y, PG).Fit(); 171 | // Rcpp::Rcout << "Grid1D Start: " << i << "\n"; 172 | G.push_back(std::move(Grid1D(*X, *y, PG).Fit())); 173 | } 174 | 175 | return std::move(G); 176 | 177 | } 178 | 179 | template class Grid2D; 180 | template class Grid2D; 181 | -------------------------------------------------------------------------------- /FastSparse/src/Makevars: -------------------------------------------------------------------------------- 1 | CXX_STD = CXX11 2 | PKG_CXXFLAGS = "-Iinclude" $(SHLIB_OPENMP_CXXFLAGS) 3 | PKG_LIBS= $(SHLIB_OPENMP_CXXFLAGS) $(LAPACK_LIBS) $(BLAS_LIBS) $(FLIBS) -lprofiler 4 | -------------------------------------------------------------------------------- /FastSparse/src/Makevars.in: -------------------------------------------------------------------------------- 1 | CXX_STD = CXX11 2 | PKG_CXXFLAGS = "-Iinclude" @OPENMP_FLAG@ 3 | PKG_LIBS= @OPENMP_FLAG@ $(LAPACK_LIBS) $(BLAS_LIBS) $(FLIBS) -lprofiler 4 | -------------------------------------------------------------------------------- /FastSparse/src/Makevars.win: -------------------------------------------------------------------------------- 1 | 2 | ## With R 3.1.0 or later, you can uncomment the following line to tell R to 3 | ## enable compilation with C++11 (where available) 4 | ## 5 | ## Also, OpenMP support in Armadillo prefers C++11 support. However, for wider 6 | ## availability of the package we do not yet enforce this here. It is however 7 | ## recommended for client packages to set it. 8 | ## 9 | ## And with R 3.4.0, and RcppArmadillo 0.7.960.*, we turn C++11 on as OpenMP 10 | ## support within Armadillo prefers / requires it 11 | 12 | CXX_STD = CXX11 13 | PKG_CXXFLAGS = "-Iinclude" $(SHLIB_OPENMP_CXXFLAGS) 14 | PKG_LIBS = $(SHLIB_OPENMP_CXXFLAGS) $(LAPACK_LIBS) $(BLAS_LIBS) $(FLIBS) -lprofiler 15 | -------------------------------------------------------------------------------- /FastSparse/src/Normalize.cpp: -------------------------------------------------------------------------------- 1 | #include "Normalize.h" 2 | 3 | std::tuple DeNormalize(beta_vector & B_scaled, 4 | arma::vec & BetaMultiplier, 5 | arma::vec & meanX, double meany) { 6 | beta_vector B_unscaled = B_scaled % BetaMultiplier; 7 | double intercept = meany - arma::dot(B_unscaled, meanX); 8 | // Matrix Type, Intercept 9 | // Dense, True -> meanX = colMeans(X) 10 | // Dense, False -> meanX = 0 Vector (meany = 0) 11 | // Sparse, True -> meanX = 0 Vector 12 | // Sparse, False -> meanX = 0 Vector 13 | return std::make_tuple(B_unscaled, intercept); 14 | } 15 | -------------------------------------------------------------------------------- /FastSparse/src/Test_Interface.cpp: -------------------------------------------------------------------------------- 1 | #include "Test_Interface.h" 2 | // [[Rcpp::depends(RcppArmadillo)]] 3 | 4 | 5 | // [[Rcpp::export]] 6 | arma::vec R_matrix_column_get_dense(const arma::mat &mat, int col) { 7 | return matrix_column_get(mat, col); 8 | } 9 | 10 | // [[Rcpp::export]] 11 | arma::vec R_matrix_column_get_sparse(const arma::sp_mat &mat, int col) { 12 | return matrix_column_get(mat, col); 13 | } 14 | 15 | // [[Rcpp::export]] 16 | arma::mat R_matrix_rows_get_dense(const arma::mat &mat, const arma::ucolvec rows){ 17 | return matrix_rows_get(mat, rows); 18 | } 19 | 20 | // [[Rcpp::export]] 21 | arma::sp_mat R_matrix_rows_get_sparse(const arma::sp_mat &mat, const arma::ucolvec rows){ 22 | return matrix_rows_get(mat, rows); 23 | } 24 | 25 | // [[Rcpp::export]] 26 | arma::mat R_matrix_vector_schur_product_dense(const arma::mat &mat, const arma::vec &u){ 27 | return matrix_vector_schur_product(mat, &u); 28 | } 29 | 30 | // [[Rcpp::export]] 31 | arma::sp_mat R_matrix_vector_schur_product_sparse(const arma::sp_mat &mat, const arma::vec &u){ 32 | return matrix_vector_schur_product(mat, &u); 33 | } 34 | 35 | 36 | // [[Rcpp::export]] 37 | arma::mat R_matrix_vector_divide_dense(const arma::mat &mat, const arma::vec &u){ 38 | return matrix_vector_divide(mat, u); 39 | } 40 | 41 | // [[Rcpp::export]] 42 | arma::sp_mat R_matrix_vector_divide_sparse(const arma::sp_mat &mat, const arma::vec &u){ 43 | return matrix_vector_divide(mat, u); 44 | } 45 | 46 | // [[Rcpp::export]] 47 | arma::rowvec R_matrix_column_sums_dense(const arma::mat &mat){ 48 | return matrix_column_sums(mat); 49 | } 50 | 51 | // [[Rcpp::export]] 52 | arma::rowvec R_matrix_column_sums_sparse(const arma::sp_mat &mat){ 53 | return matrix_column_sums(mat); 54 | } 55 | 56 | 57 | // [[Rcpp::export]] 58 | double R_matrix_column_dot_dense(const arma::mat &mat, int col, const arma::vec u){ 59 | return matrix_column_dot(mat, col, u); 60 | } 61 | 62 | // [[Rcpp::export]] 63 | double R_matrix_column_dot_sparse(const arma::sp_mat &mat, int col, const arma::vec u){ 64 | return matrix_column_dot(mat, col, u); 65 | } 66 | 67 | // [[Rcpp::export]] 68 | arma::vec R_matrix_column_mult_dense(const arma::mat &mat, int col, double u){ 69 | return matrix_column_mult(mat, col, u); 70 | } 71 | 72 | // [[Rcpp::export]] 73 | arma::vec R_matrix_column_mult_sparse(const arma::sp_mat &mat, int col, double u){ 74 | return matrix_column_mult(mat, col, u); 75 | } 76 | 77 | // [[Rcpp::export]] 78 | Rcpp::List R_matrix_normalize_dense(arma::mat mat_norm){ 79 | arma::rowvec ScaleX = matrix_normalize(mat_norm); 80 | return Rcpp::List::create(Rcpp::Named("mat_norm") = mat_norm, 81 | Rcpp::Named("ScaleX") = ScaleX); 82 | }; 83 | 84 | // [[Rcpp::export]] 85 | Rcpp::List R_matrix_normalize_sparse(arma::sp_mat mat_norm){ 86 | arma::rowvec ScaleX = matrix_normalize(mat_norm); 87 | return Rcpp::List::create(Rcpp::Named("mat_norm") = mat_norm, 88 | Rcpp::Named("ScaleX") = ScaleX); 89 | }; 90 | 91 | // [[Rcpp::export]] 92 | Rcpp::List R_matrix_center_dense(const arma::mat mat, arma::mat X_normalized, bool intercept){ 93 | arma::rowvec meanX = matrix_center(mat, X_normalized, intercept); 94 | return Rcpp::List::create(Rcpp::Named("mat_norm") = X_normalized, 95 | Rcpp::Named("MeanX") = meanX); 96 | }; 97 | 98 | // [[Rcpp::export]] 99 | Rcpp::List R_matrix_center_sparse(const arma::sp_mat mat, arma::sp_mat X_normalized,bool intercept){ 100 | arma::rowvec meanX = matrix_center(mat, X_normalized, intercept); 101 | return Rcpp::List::create(Rcpp::Named("mat_norm") = X_normalized, 102 | Rcpp::Named("MeanX") = meanX); 103 | }; 104 | -------------------------------------------------------------------------------- /FastSparse/src/include/BetaVector.h: -------------------------------------------------------------------------------- 1 | #ifndef BETA_VECTOR_H 2 | #define BETA_VECTOR_H 3 | #include 4 | #include "RcppArmadillo.h" 5 | 6 | /* 7 | * arma::vec implementation 8 | */ 9 | 10 | 11 | using beta_vector = arma::vec; 12 | //using beta_vector = arma::sp_mat; 13 | 14 | std::vector nnzIndicies(const arma::vec& B); 15 | 16 | std::vector nnzIndicies(const arma::sp_mat& B); 17 | 18 | std::vector nnzIndicies(const arma::vec& B, const std::size_t low); 19 | 20 | std::vector nnzIndicies(const arma::sp_mat& B, const std::size_t low); 21 | 22 | std::size_t n_nonzero(const arma::vec& B); 23 | 24 | std::size_t n_nonzero(const arma::sp_mat& B); 25 | 26 | bool has_same_support(const arma::vec& B1, const arma::vec& B2); 27 | 28 | bool has_same_support(const arma::sp_mat& B1, const arma::sp_mat& B2); 29 | 30 | 31 | #endif // BETA_VECTOR_H 32 | -------------------------------------------------------------------------------- /FastSparse/src/include/CDL0.h: -------------------------------------------------------------------------------- 1 | #ifndef CDL0_H 2 | #define CDL0_H 3 | #include 4 | #include "RcppArmadillo.h" 5 | #include "CD.h" 6 | #include "Params.h" 7 | #include "FitResult.h" 8 | #include "utils.h" 9 | #include "BetaVector.h" 10 | 11 | 12 | template 13 | class CDL0: public CD>{ 14 | private: 15 | 16 | arma::vec r; //vector of residuals 17 | 18 | public: 19 | CDL0(const T& Xi, const arma::vec& yi, const Params& P); 20 | //~CDL0(){} 21 | 22 | FitResult _FitWithBounds() final; 23 | 24 | FitResult _Fit() final; 25 | 26 | inline double Objective(const arma::vec &, const beta_vector &) final; 27 | 28 | inline double Objective() final; 29 | 30 | inline double GetBiGrad(const std::size_t i); 31 | 32 | inline double GetBiValue(const double old_Bi, const double grd_Bi); 33 | 34 | inline double GetBiReg(const double nrb_Bi); 35 | 36 | inline void ApplyNewBi(const std::size_t i, const double old_Bi, const double new_Bi); 37 | 38 | inline void ApplyNewBiCWMinCheck(const std::size_t i, const double old_Bi, const double new_Bi); 39 | }; 40 | 41 | template 42 | inline double CDL0::GetBiGrad(const std::size_t i){ 43 | return matrix_column_dot(*(this->X), i, this->r); 44 | } 45 | 46 | template 47 | inline double CDL0::GetBiValue(const double old_Bi, const double grd_Bi){ 48 | return grd_Bi + old_Bi; 49 | } 50 | 51 | template 52 | inline double CDL0::GetBiReg(const double nrb_Bi){ 53 | return std::abs(nrb_Bi); 54 | } 55 | 56 | template 57 | inline void CDL0::ApplyNewBi(const std::size_t i, const double old_Bi, const double new_Bi){ 58 | this->r += matrix_column_mult(*(this->X), i, old_Bi - new_Bi); 59 | this->B[i] = new_Bi; 60 | } 61 | 62 | template 63 | inline void CDL0::ApplyNewBiCWMinCheck(const std::size_t i, const double old_Bi, const double new_Bi){ 64 | this->r += matrix_column_mult(*(this->X), i, old_Bi - new_Bi); 65 | this->B[i] = new_Bi; 66 | this->Order.push_back(i); 67 | } 68 | 69 | template 70 | inline double CDL0::Objective(const arma::vec & r, const beta_vector & B) { 71 | return 0.5 * arma::dot(r, r) + this->lambda0 * n_nonzero(B); 72 | } 73 | 74 | template 75 | inline double CDL0::Objective() { 76 | return 0.5 * arma::dot(this->r, this->r) + this->lambda0 * n_nonzero(this->B); 77 | } 78 | 79 | template 80 | CDL0::CDL0(const T& Xi, const arma::vec& yi, const Params& P) : CD>(Xi, yi, P){ 81 | this->thr2 = 2 * this->lambda0; 82 | this->thr = sqrt(this->thr2); 83 | this->r = *P.r; 84 | this->result.r = P.r; 85 | } 86 | 87 | template 88 | FitResult CDL0::_Fit() { 89 | // Rcpp::Rcout << "CDL0 Fit:"; 90 | this->objective = Objective(this->r, this->B); 91 | 92 | std::vector FullOrder = this->Order; 93 | 94 | if (this->ActiveSet) { 95 | this->Order.resize(std::min((int) (n_nonzero(this->B) + this->ScreenSize + this->NoSelectK), (int)(this->p))); // std::min(1000,Order.size()) 96 | } 97 | 98 | for (std::size_t t = 0; t < this->MaxIters; ++t) { 99 | this->Bprev = this->B; 100 | 101 | if (this->isSparse && this->intercept){ 102 | this->UpdateSparse_b0(this->r); 103 | } 104 | 105 | //Rcpp::Rcout << "{" << this->Order.size() << "}"; 106 | for (auto& i : this->Order) { 107 | this->UpdateBi(i); 108 | } 109 | 110 | this->RestrictSupport(); 111 | 112 | if (this->isConverged() && this->CWMinCheck()) { 113 | //Rcpp::Rcout << " |Converged on iter:" << t << "CWMinCheck \n"; 114 | break; 115 | } 116 | } 117 | 118 | 119 | // Re-optimize b0 after convergence. 120 | if (this->isSparse && this->intercept){ 121 | this->UpdateSparse_b0(this->r); 122 | } 123 | 124 | this->result.Objective = this->objective; 125 | this->result.B = this->B; 126 | *(this->result.r) = this->r; // change to pointer later 127 | this->result.IterNum = this->CurrentIters; 128 | this->result.b0 = this->b0; 129 | 130 | return this->result; 131 | } 132 | 133 | template 134 | FitResult CDL0::_FitWithBounds() { 135 | // Rcpp::Rcout << "CDL0 Fit: "; 136 | clamp_by_vector(this->B, this->Lows, this->Highs); 137 | 138 | this->objective = Objective(this->r, this->B); 139 | 140 | std::vector FullOrder = this->Order; 141 | 142 | if (this->ActiveSet) { 143 | this->Order.resize(std::min((int) (n_nonzero(this->B) + this->ScreenSize + this->NoSelectK), (int)(this->p))); // std::min(1000,Order.size()) 144 | } 145 | 146 | for (std::size_t t = 0; t < this->MaxIters; ++t) { 147 | this->Bprev = this->B; 148 | 149 | if (this->isSparse && this->intercept){ 150 | this->UpdateSparse_b0(this->r); 151 | } 152 | 153 | for (auto& i : this->Order) { 154 | this->UpdateBiWithBounds(i); 155 | } 156 | 157 | this->RestrictSupport(); 158 | 159 | if (this->isConverged() && this->CWMinCheckWithBounds()) { 160 | // Rcpp::Rcout << "Converged on iter:" << t << "\n"; 161 | break; 162 | } 163 | } 164 | 165 | // Re-optimize b0 after convergence. 166 | if (this->isSparse && this->intercept){ 167 | this->UpdateSparse_b0(this->r); 168 | } 169 | 170 | this->result.Objective = this->objective; 171 | this->result.B = this->B; 172 | *(this->result.r) = this->r; // change to pointer later 173 | this->result.IterNum = this->CurrentIters; 174 | this->result.b0 = this->b0; 175 | 176 | return this->result; 177 | } 178 | 179 | template class CDL0; 180 | template class CDL0; 181 | 182 | #endif 183 | -------------------------------------------------------------------------------- /FastSparse/src/include/CDL012.h: -------------------------------------------------------------------------------- 1 | #ifndef CDL012_H 2 | #define CDL012_H 3 | #include "RcppArmadillo.h" 4 | #include "CD.h" 5 | #include "FitResult.h" 6 | #include "utils.h" 7 | #include "BetaVector.h" 8 | 9 | template 10 | class CDL012 : public CD>{ 11 | private: 12 | double Onep2lamda2; 13 | arma::vec r; //vector of residuals 14 | 15 | public: 16 | CDL012(const T& Xi, const arma::vec& yi, const Params& P); 17 | //~CDL012(){} 18 | 19 | FitResult _FitWithBounds() final; 20 | 21 | FitResult _Fit() final; 22 | 23 | inline double Objective(const arma::vec & r, const beta_vector & B) final; 24 | 25 | inline double Objective() final; 26 | 27 | inline double GetBiGrad(const std::size_t i); 28 | 29 | inline double GetBiValue(const double old_Bi, const double grd_Bi); 30 | 31 | inline double GetBiReg(const double nrb_Bi); 32 | 33 | inline void ApplyNewBi(const std::size_t i, const double old_Bi, const double new_Bi); 34 | 35 | inline void ApplyNewBiCWMinCheck(const std::size_t i, const double old_Bi, const double new_Bi); 36 | 37 | }; 38 | 39 | 40 | template 41 | inline double CDL012::GetBiGrad(const std::size_t i){ 42 | return matrix_column_dot(*(this->X), i, this->r); 43 | } 44 | 45 | template 46 | inline double CDL012::GetBiValue(const double old_Bi, const double grd_Bi){ 47 | return grd_Bi + old_Bi; 48 | } 49 | 50 | template 51 | inline double CDL012::GetBiReg(const double nrb_Bi){ 52 | // sign(nrb_Bi)*(|nrb_Bi| - lambda1)/(1 + 2*lambda2) 53 | return (std::abs(nrb_Bi) - this->lambda1) / Onep2lamda2; 54 | } 55 | 56 | template 57 | inline void CDL012::ApplyNewBi(const std::size_t i, const double Bi_old, const double Bi_new){ 58 | this->r += matrix_column_mult(*(this->X), i, Bi_old - Bi_new); 59 | this->B[i] = Bi_new; 60 | } 61 | 62 | template 63 | inline void CDL012::ApplyNewBiCWMinCheck(const std::size_t i, const double Bi_old, const double Bi_new){ 64 | this->r += matrix_column_mult(*(this->X), i, Bi_old - Bi_new); 65 | this->B[i] = Bi_new; 66 | this->Order.push_back(i); 67 | } 68 | 69 | template 70 | inline double CDL012::Objective(const arma::vec & r, const beta_vector & B) { 71 | auto l2norm = arma::norm(B, 2); 72 | return 0.5 * arma::dot(r, r) + this->lambda0 * n_nonzero(this->B) + this->lambda1 * arma::norm(B, 1) + this->lambda2 * l2norm * l2norm; 73 | } 74 | 75 | template 76 | inline double CDL012::Objective() { 77 | auto l2norm = arma::norm(this->B, 2); 78 | return 0.5 * arma::dot(this->r, this->r) + this->lambda0 * n_nonzero(this->B) + this->lambda1 * arma::norm(this->B, 1) + this->lambda2 * l2norm * l2norm; 79 | } 80 | 81 | template 82 | CDL012::CDL012(const T& Xi, const arma::vec& yi, const Params& P) : CD>(Xi, yi, P) { 83 | Onep2lamda2 = 1 + 2 * this->lambda2; 84 | 85 | this->thr2 = 2 * this->lambda0 / Onep2lamda2; 86 | this->thr = std::sqrt(this->thr2); 87 | this->r = *P.r; 88 | this->result.r = P.r; 89 | } 90 | 91 | template 92 | FitResult CDL012::_Fit() { 93 | 94 | this->objective = Objective(this->r, this->B); 95 | 96 | std::vector FullOrder = this->Order; 97 | 98 | if (this->ActiveSet) { 99 | this->Order.resize(std::min((int) (n_nonzero(this->B) + this->ScreenSize + this->NoSelectK), (int)(this->p))); 100 | } 101 | 102 | for (std::size_t t = 0; t < this->MaxIters; ++t) { 103 | this->Bprev = this->B; 104 | 105 | if (this->isSparse && this->intercept){ 106 | this->UpdateSparse_b0(this->r); 107 | } 108 | 109 | for (auto& i : this->Order) { 110 | this->UpdateBi(i); 111 | } 112 | 113 | this->RestrictSupport(); 114 | 115 | if (this->isConverged() && this->CWMinCheck()) { 116 | break; 117 | } 118 | } 119 | 120 | if (this->isSparse && this->intercept){ 121 | this->UpdateSparse_b0(this->r); 122 | } 123 | 124 | this->result.Objective = this->objective; 125 | this->result.B = this->B; 126 | *(this->result.r) = this->r; // change to pointer later 127 | this->result.IterNum = this->CurrentIters; 128 | this->result.b0 = this->b0; 129 | return this->result; 130 | } 131 | 132 | template 133 | FitResult CDL012::_FitWithBounds() { 134 | 135 | clamp_by_vector(this->B, this->Lows, this->Highs); 136 | 137 | this->objective = Objective(this->r, this->B); 138 | 139 | std::vector FullOrder = this->Order; 140 | 141 | if (this->ActiveSet) { 142 | this->Order.resize(std::min((int) (n_nonzero(this->B) + this->ScreenSize + this->NoSelectK), (int)(this->p))); 143 | } 144 | 145 | for (std::size_t t = 0; t < this->MaxIters; ++t) { 146 | this->Bprev = this->B; 147 | 148 | if (this->isSparse && this->intercept){ 149 | this->UpdateSparse_b0(this->r); 150 | } 151 | 152 | for (auto& i : this->Order) { 153 | this->UpdateBiWithBounds(i); 154 | } 155 | 156 | this->RestrictSupport(); 157 | 158 | //B.print(); 159 | if (this->isConverged() && this->CWMinCheckWithBounds()) { 160 | break; 161 | } 162 | 163 | } 164 | 165 | if (this->isSparse && this->intercept){ 166 | this->UpdateSparse_b0(this->r); 167 | } 168 | 169 | this->result.Objective = this->objective; 170 | this->result.B = this->B; 171 | *(this->result.r) = this->r; // change to pointer later 172 | this->result.IterNum = this->CurrentIters; 173 | this->result.b0 = this->b0; 174 | return this->result; 175 | } 176 | 177 | template class CDL012; 178 | template class CDL012; 179 | 180 | #endif 181 | -------------------------------------------------------------------------------- /FastSparse/src/include/CDL012ExponentialSwaps.h: -------------------------------------------------------------------------------- 1 | #ifndef CDL012ExponentialSwaps_H 2 | #define CDL012ExponentialSwaps_H 3 | #include "RcppArmadillo.h" 4 | #include "CD.h" 5 | #include "CDSwaps.h" 6 | #include "CDL012Exponential.h" 7 | #include "FitResult.h" 8 | #include "Params.h" 9 | #include "utils.h" 10 | #include "BetaVector.h" 11 | 12 | template 13 | class CDL012ExponentialSwaps : public CDSwaps { 14 | private: 15 | const double LipschitzConst = 0.25; 16 | double twolambda2; 17 | double qp2lamda2; 18 | double lambda1ol; 19 | double stl0Lc; 20 | arma::vec ExpyXB; 21 | // std::vector * Xtr; 22 | T * Xy; 23 | 24 | arma::uvec indices; 25 | 26 | //////// new variables for expo loss 27 | arma::vec inverse_ExpyXB; 28 | double d_minus; 29 | double d_plus; 30 | double current_expo_loss; 31 | 32 | std::unordered_map * Xy_neg_indices; 33 | 34 | public: 35 | CDL012ExponentialSwaps(const T& Xi, const arma::vec& yi, const Params& P); 36 | 37 | FitResult _FitWithBounds() final; 38 | 39 | FitResult _Fit() final; 40 | 41 | inline double Objective(const arma::vec & r, const beta_vector & B) final; 42 | 43 | inline double Objective() final; 44 | 45 | }; 46 | 47 | 48 | template 49 | inline double CDL012ExponentialSwaps::Objective(const arma::vec & r, const beta_vector & B) { 50 | // auto l2norm = arma::norm(B, 2); 51 | return arma::sum(r) + this->lambda0 * n_nonzero(B); //+ this->lambda1 * arma::norm(B, 1) + this->lambda2 * l2norm * l2norm; 52 | } 53 | 54 | template 55 | inline double CDL012ExponentialSwaps::Objective() { 56 | // auto l2norm = arma::norm(this->B, 2); 57 | return arma::sum(this->inverse_ExpyXB) + this->lambda0 * n_nonzero(this->B); //+ this->lambda1 * arma::norm(this->B, 1) + this->lambda2 * l2norm * l2norm; 58 | } 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /FastSparse/src/include/CDL012Logistic.h: -------------------------------------------------------------------------------- 1 | #ifndef CDL012Logistic_H 2 | #define CDL012Logistic_H 3 | #include "RcppArmadillo.h" 4 | #include "CD.h" 5 | #include "FitResult.h" 6 | #include "Params.h" 7 | #include "utils.h" 8 | #include "BetaVector.h" 9 | 10 | template 11 | class CDL012Logistic : public CD> { 12 | private: 13 | const double LipschitzConst = 0.25; 14 | double twolambda2; 15 | double qp2lamda2; 16 | double lambda1ol; 17 | arma::vec ExpyXB; 18 | // std::vector * Xtr; 19 | T * Xy; 20 | 21 | public: 22 | CDL012Logistic(const T& Xi, const arma::vec& yi, const Params& P); 23 | //~CDL012Logistic(){} 24 | 25 | FitResult _FitWithBounds() final; 26 | 27 | FitResult _Fit() final; 28 | 29 | inline double Objective(const arma::vec & r, const beta_vector & B) final; 30 | 31 | inline double Objective() final; 32 | 33 | inline double GetBiGrad(const std::size_t i); 34 | 35 | inline double GetBiValue(const double old_Bi, const double grd_Bi); 36 | 37 | inline double GetBiReg(const double Bi_step); 38 | 39 | inline void ApplyNewBi(const std::size_t i, const double Bi_old, const double Bi_new); 40 | 41 | inline void ApplyNewBiCWMinCheck(const std::size_t i, const double old_Bi, const double new_Bi); 42 | 43 | }; 44 | 45 | template 46 | inline double CDL012Logistic::GetBiGrad(const std::size_t i){ 47 | /* 48 | * Notes: 49 | * When called in CWMinCheck, we know that this->B[i] is 0. 50 | */ 51 | return -arma::dot(matrix_column_get(*(this->Xy), i), 1 / (1 + ExpyXB) ) + twolambda2 * this->B[i]; 52 | //return -arma::sum( matrix_column_get(*(this->Xy), i) / (1 + ExpyXB) ) + twolambda2 * this->B[i]; 53 | } 54 | 55 | template 56 | inline double CDL012Logistic::GetBiValue(const double old_Bi, const double grd_Bi){ 57 | return old_Bi - grd_Bi/qp2lamda2; 58 | } 59 | 60 | template 61 | inline double CDL012Logistic::GetBiReg(const double Bi_step){ 62 | return std::abs(Bi_step) - lambda1ol; 63 | } 64 | 65 | template 66 | inline void CDL012Logistic::ApplyNewBi(const std::size_t i, const double old_Bi, const double new_Bi){ 67 | ExpyXB %= arma::exp( (new_Bi - old_Bi) * matrix_column_get(*(this->Xy), i)); 68 | this->B[i] = new_Bi; 69 | } 70 | 71 | template 72 | inline void CDL012Logistic::ApplyNewBiCWMinCheck(const std::size_t i, 73 | const double old_Bi, 74 | const double new_Bi){ 75 | ExpyXB %= arma::exp( (new_Bi - old_Bi) * matrix_column_get(*(this->Xy), i)); 76 | this->B[i] = new_Bi; 77 | this->Order.push_back(i); 78 | } 79 | 80 | template 81 | inline double CDL012Logistic::Objective(const arma::vec & expyXB, const beta_vector & B) { // hint inline 82 | const auto l2norm = arma::norm(B, 2); 83 | // arma::sum(arma::log(1 + 1 / expyXB)) is the negative log-likelihood 84 | return arma::sum(arma::log(1 + 1 / expyXB)) + this->lambda0 * n_nonzero(B) + this->lambda1 * arma::norm(B, 1) + this->lambda2 * l2norm * l2norm; 85 | } 86 | 87 | template 88 | inline double CDL012Logistic::Objective() { // hint inline 89 | const auto l2norm = arma::norm(this->B, 2); 90 | // arma::sum(arma::log(1 + 1 / ExpyXB)) is the negative log-likelihood 91 | return arma::sum(arma::log(1 + 1 / ExpyXB)) + this->lambda0 * n_nonzero(this->B) + this->lambda1 * arma::norm(this->B, 1) + this->lambda2 * l2norm * l2norm; 92 | } 93 | 94 | template 95 | CDL012Logistic::CDL012Logistic(const T& Xi, const arma::vec& yi, const Params& P) : CD>(Xi, yi, P) { 96 | twolambda2 = 2 * this->lambda2; 97 | qp2lamda2 = (LipschitzConst + twolambda2); // this is the univariate lipschitz const of the differentiable objective 98 | this->thr2 = (2 * this->lambda0) / qp2lamda2; 99 | this->thr = std::sqrt(this->thr2); 100 | lambda1ol = this->lambda1 / qp2lamda2; 101 | 102 | ExpyXB = arma::exp(*this->y % (*(this->X) * this->B + this->b0)); // Maintained throughout the algorithm 103 | Xy = P.Xy; 104 | } 105 | 106 | template 107 | FitResult CDL012Logistic::_Fit() { 108 | this->objective = Objective(); // Implicitly used ExpyXB 109 | 110 | std::vector FullOrder = this->Order; // never used in LR 111 | this->Order.resize(std::min((int) (n_nonzero(this->B) + this->ScreenSize + this->NoSelectK), (int)(this->p))); 112 | 113 | for (std::size_t t = 0; t < this->MaxIters; ++t) { 114 | this->Bprev = this->B; 115 | 116 | // Update the intercept 117 | if (this->intercept){ 118 | const double b0old = this->b0; 119 | // const double partial_b0 = - arma::sum( *(this->y) / (1 + ExpyXB) ); 120 | const double partial_b0 = - arma::dot( *(this->y) , 1/(1 + ExpyXB) ); 121 | this->b0 -= partial_b0 / (this->n * LipschitzConst); // intercept is not regularized 122 | ExpyXB %= arma::exp( (this->b0 - b0old) * *(this->y)); 123 | } 124 | 125 | for (auto& i : this->Order) { 126 | this->UpdateBi(i); 127 | } 128 | 129 | this->RestrictSupport(); 130 | 131 | // only way to terminate is by (i) converging on active set and (ii) CWMinCheck 132 | if (this->isConverged() && this->CWMinCheck()) { 133 | break; 134 | } 135 | } 136 | 137 | this->result.Objective = this->objective; 138 | this->result.B = this->B; 139 | this->result.Model = this; 140 | this->result.b0 = this->b0; 141 | this->result.ExpyXB = ExpyXB; 142 | this->result.IterNum = this->CurrentIters; 143 | 144 | return this->result; 145 | } 146 | 147 | template 148 | FitResult CDL012Logistic::_FitWithBounds() { // always uses active sets 149 | 150 | //arma::sp_mat B2 = this->B; 151 | clamp_by_vector(this->B, this->Lows, this->Highs); 152 | 153 | this->objective = Objective(); // Implicitly used ExpyXB 154 | 155 | std::vector FullOrder = this->Order; // never used in LR 156 | this->Order.resize(std::min((int) (n_nonzero(this->B) + this->ScreenSize + this->NoSelectK), (int)(this->p))); 157 | 158 | for (std::size_t t = 0; t < this->MaxIters; ++t) { 159 | this->Bprev = this->B; 160 | 161 | // Update the intercept 162 | if (this->intercept){ 163 | const double b0old = this->b0; 164 | // const double partial_b0 = - arma::sum( *(this->y) / (1 + ExpyXB) ); 165 | const double partial_b0 = - arma::dot( *(this->y) , 1/(1 + ExpyXB) ); 166 | this->b0 -= partial_b0 / (this->n * LipschitzConst); // intercept is not regularized 167 | ExpyXB %= arma::exp( (this->b0 - b0old) * *(this->y)); 168 | } 169 | 170 | for (auto& i : this->Order) { 171 | this->UpdateBiWithBounds(i); 172 | } 173 | 174 | this->RestrictSupport(); 175 | 176 | // only way to terminate is by (i) converging on active set and (ii) CWMinCheck 177 | if (this->isConverged() && this->CWMinCheckWithBounds()) { 178 | break; 179 | } 180 | } 181 | 182 | this->result.Objective = this->objective; 183 | this->result.B = this->B; 184 | this->result.Model = this; 185 | this->result.b0 = this->b0; 186 | this->result.ExpyXB = ExpyXB; 187 | this->result.IterNum = this->CurrentIters; 188 | 189 | return this->result; 190 | } 191 | 192 | template class CDL012Logistic; 193 | template class CDL012Logistic; 194 | 195 | 196 | #endif 197 | -------------------------------------------------------------------------------- /FastSparse/src/include/CDL012LogisticSwaps.h: -------------------------------------------------------------------------------- 1 | #ifndef CDL012LogisticSwaps_H 2 | #define CDL012LogisticSwaps_H 3 | #include "RcppArmadillo.h" 4 | #include "CD.h" 5 | #include "CDSwaps.h" 6 | #include "CDL012Logistic.h" 7 | #include "FitResult.h" 8 | #include "Params.h" 9 | #include "utils.h" 10 | #include "BetaVector.h" 11 | 12 | template 13 | class CDL012LogisticSwaps : public CDSwaps { 14 | private: 15 | const double LipschitzConst = 0.25; 16 | double twolambda2; 17 | double qp2lamda2; 18 | double lambda1ol; 19 | double stl0Lc; 20 | arma::vec ExpyXB; 21 | // std::vector * Xtr; 22 | T * Xy; 23 | 24 | double Fmin; 25 | 26 | // my new attributes 27 | arma::rowvec frequency_count; 28 | beta_vector Btemp; 29 | 30 | arma::vec ExpyXBnoji_mid; 31 | double partial_i_mid; 32 | arma::vec ExpyXBnoji_double; 33 | double partial_i_double; 34 | arma::vec ExpyXBnoji_triple; 35 | double partial_i_triple; 36 | 37 | arma::vec ExpyXBnoji; 38 | double Biold; 39 | double partial_i; 40 | double Binew; 41 | 42 | public: 43 | CDL012LogisticSwaps(const T& Xi, const arma::vec& yi, const Params& P); 44 | 45 | FitResult _FitWithBounds() final; 46 | 47 | FitResult _Fit() final; 48 | 49 | inline double Objective(const arma::vec & r, const beta_vector & B) final; 50 | 51 | inline double Objective() final; 52 | 53 | FitResult finetune(); 54 | FitResult replace_indexJ_with_indexI_and_finetune(std::size_t j, std::size_t i, double coef_i); 55 | bool evaluate_pruning_by_quadratic_cut_1point(double f1, double df1, double bestf); 56 | bool evaluate_pruning_by_quadratic_cut_2points(double f1, double x1, double df1, double f2, double x2, double df2, double bestf); 57 | bool evaluate_pruning_by_linear_cut_2points(double f1, double x1, double df1, double f2, double x2, double df2, double bestf); 58 | double one_gradientDescent_step(double Biold, double partial_i); 59 | 60 | void update_ExpyXB_and_partial(arma::vec & oldExpyXB, double BiOld, double BiNew, std::size_t i, arma::vec & newExpyXB, double & partial_i_new); 61 | void update_Biold_and_Binew(double & Biold, double & Binew, double partial_i); 62 | inline double Objective(const arma::vec & r, beta_vector & B, std::size_t i, double Binew); 63 | 64 | bool evaluate_early_break(std::size_t i); 65 | 66 | }; 67 | 68 | template 69 | inline double CDL012LogisticSwaps::Objective(const arma::vec & r, beta_vector & B, std::size_t i, double Binew) { 70 | B[i] = Binew; 71 | auto l2norm = arma::norm(B, 2); 72 | return arma::sum(arma::log(1 + 1 / r)) + this->lambda0 * n_nonzero(B) + this->lambda1 * arma::norm(B, 1) + this->lambda2 * l2norm * l2norm; 73 | } 74 | 75 | template 76 | inline double CDL012LogisticSwaps::Objective(const arma::vec & r, const beta_vector & B) { 77 | auto l2norm = arma::norm(B, 2); 78 | return arma::sum(arma::log(1 + 1 / r)) + this->lambda0 * n_nonzero(B) + this->lambda1 * arma::norm(B, 1) + this->lambda2 * l2norm * l2norm; 79 | } 80 | 81 | template 82 | inline double CDL012LogisticSwaps::Objective() { 83 | auto l2norm = arma::norm(this->B, 2); 84 | return arma::sum(arma::log(1 + 1 / ExpyXB)) + this->lambda0 * n_nonzero(this->B) + this->lambda1 * arma::norm(this->B, 1) + this->lambda2 * l2norm * l2norm; 85 | } 86 | 87 | #endif 88 | -------------------------------------------------------------------------------- /FastSparse/src/include/CDL012SquaredHinge.h: -------------------------------------------------------------------------------- 1 | #ifndef CDL012SquaredHinge_H 2 | #define CDL012SquaredHinge_H 3 | #include "RcppArmadillo.h" 4 | #include "CD.h" 5 | #include "FitResult.h" 6 | #include "Params.h" 7 | #include "utils.h" 8 | #include "BetaVector.h" 9 | 10 | template 11 | class CDL012SquaredHinge : public CD> { 12 | private: 13 | const double LipschitzConst = 2; // for f (without regularization) 14 | double twolambda2; 15 | double qp2lamda2; 16 | double lambda1ol; 17 | // std::vector * Xtr; 18 | arma::vec onemyxb; 19 | arma::uvec indices; 20 | T * Xy; 21 | 22 | 23 | public: 24 | CDL012SquaredHinge(const T& Xi, const arma::vec& yi, const Params& P); 25 | 26 | //~CDL012SquaredHinge(){} 27 | 28 | FitResult _FitWithBounds() final; 29 | 30 | FitResult _Fit() final; 31 | 32 | inline double Objective(const arma::vec & r, const beta_vector & B) final; 33 | 34 | inline double Objective() final; 35 | 36 | inline double GetBiGrad(const std::size_t i); 37 | 38 | inline double GetBiValue(const double old_Bi, const double grd_Bi); 39 | 40 | inline double GetBiReg(const double Bi_step); 41 | 42 | inline void ApplyNewBi(const std::size_t i, const double Bi_old, const double Bi_new); 43 | 44 | inline void ApplyNewBiCWMinCheck(const std::size_t i, const double old_Bi, const double new_Bi); 45 | 46 | }; 47 | 48 | template 49 | inline double CDL012SquaredHinge::GetBiGrad(const std::size_t i){ 50 | // Rcpp::Rcout << "Grad stuff: " << arma::sum(2 * onemyxb.elem(indices) % (- matrix_column_get(*Xy, i).elem(indices)) ) << "\n"; 51 | return arma::sum(2 * onemyxb.elem(indices) % (- matrix_column_get(*Xy, i).elem(indices)) ) + twolambda2 * this->B[i]; 52 | } 53 | 54 | template 55 | inline double CDL012SquaredHinge::GetBiValue(const double old_Bi, const double grd_Bi){ 56 | return old_Bi - grd_Bi / qp2lamda2; 57 | } 58 | 59 | template 60 | inline double CDL012SquaredHinge::GetBiReg(const double Bi_step){ 61 | return std::abs(Bi_step) - lambda1ol; 62 | } 63 | 64 | template 65 | inline void CDL012SquaredHinge::ApplyNewBi(const std::size_t i, const double Bi_old, const double Bi_new){ 66 | onemyxb += (Bi_old - Bi_new) * matrix_column_get(*(this->Xy), i); 67 | this->B[i] = Bi_new; 68 | indices = arma::find(onemyxb > 0); 69 | } 70 | 71 | template 72 | inline void CDL012SquaredHinge::ApplyNewBiCWMinCheck(const std::size_t i, const double Bi_old, const double Bi_new){ 73 | onemyxb += (Bi_old - Bi_new) * matrix_column_get(*(this->Xy), i); 74 | this->B[i] = Bi_new; 75 | indices = arma::find(onemyxb > 0); 76 | this->Order.push_back(i); 77 | } 78 | 79 | template 80 | inline double CDL012SquaredHinge::Objective(const arma::vec & onemyxb, const beta_vector & B) { 81 | 82 | auto l2norm = arma::norm(B, 2); 83 | arma::uvec indices = arma::find(onemyxb > 0); 84 | 85 | return arma::sum(onemyxb.elem(indices) % onemyxb.elem(indices)) + this->lambda0 * n_nonzero(B) + this->lambda1 * arma::norm(B, 1) + this->lambda2 * l2norm * l2norm; 86 | } 87 | 88 | 89 | template 90 | inline double CDL012SquaredHinge::Objective() { 91 | 92 | auto l2norm = arma::norm(this->B, 2); 93 | return arma::sum(onemyxb.elem(indices) % onemyxb.elem(indices)) + this->lambda0 * n_nonzero(this->B) + this->lambda1 * arma::norm(this->B, 1) + this->lambda2 * l2norm * l2norm; 94 | } 95 | 96 | template 97 | CDL012SquaredHinge::CDL012SquaredHinge(const T& Xi, const arma::vec& yi, const Params& P) : CD>(Xi, yi, P) { 98 | twolambda2 = 2 * this->lambda2; 99 | qp2lamda2 = (LipschitzConst + twolambda2); // this is the univariate lipschitz const of the differentiable objective 100 | this->thr2 = (2 * this->lambda0) / qp2lamda2; 101 | this->thr = std::sqrt(this->thr2); 102 | lambda1ol = this->lambda1 / qp2lamda2; 103 | 104 | // TODO: Review this line 105 | // TODO: Pass work from previous solution. 106 | onemyxb = 1 - *(this->y) % (*(this->X) * this->B + this->b0); 107 | 108 | // TODO: Add comment for purpose of 'indices' 109 | indices = arma::find(onemyxb > 0); 110 | Xy = P.Xy; 111 | } 112 | 113 | template 114 | FitResult CDL012SquaredHinge::_Fit() { 115 | 116 | this->objective = Objective(); // Implicitly uses onemyx 117 | 118 | 119 | std::vector FullOrder = this->Order; // never used in LR 120 | this->Order.resize(std::min((int) (n_nonzero(this->B) + this->ScreenSize + this->NoSelectK), (int)(this->p))); 121 | 122 | 123 | for (auto t = 0; t < this->MaxIters; ++t) { 124 | 125 | this->Bprev = this->B; 126 | 127 | // Update the intercept 128 | if (this->intercept) { 129 | const double b0old = this->b0; 130 | const double partial_b0 = arma::sum(2 * onemyxb.elem(indices) % (- (this->y)->elem(indices) ) ); 131 | this->b0 -= partial_b0 / (this->n * LipschitzConst); // intercept is not regularized 132 | onemyxb += *(this->y) * (b0old - this->b0); 133 | indices = arma::find(onemyxb > 0); 134 | } 135 | 136 | for (auto& i : this->Order) { 137 | this->UpdateBi(i); 138 | } 139 | 140 | this->RestrictSupport(); 141 | 142 | // only way to terminate is by (i) converging on active set and (ii) CWMinCheck 143 | if ((this->isConverged()) && this->CWMinCheck()) { 144 | break; 145 | } 146 | } 147 | 148 | this->result.Objective = this->objective; 149 | this->result.B = this->B; 150 | this->result.Model = this; 151 | this->result.b0 = this->b0; 152 | this->result.IterNum = this->CurrentIters; 153 | this->result.onemyxb = this->onemyxb; 154 | return this->result; 155 | } 156 | 157 | 158 | template 159 | FitResult CDL012SquaredHinge::_FitWithBounds() { 160 | 161 | clamp_by_vector(this->B, this->Lows, this->Highs); 162 | 163 | this->objective = Objective(); // Implicitly uses onemyx 164 | 165 | std::vector FullOrder = this->Order; // never used in LR 166 | this->Order.resize(std::min((int) (n_nonzero(this->B) + this->ScreenSize + this->NoSelectK), (int)(this->p))); 167 | 168 | 169 | for (auto t = 0; t < this->MaxIters; ++t) { 170 | 171 | this->Bprev = this->B; 172 | 173 | // Update the intercept 174 | if (this->intercept) { 175 | const double b0old = this->b0; 176 | const double partial_b0 = arma::sum(2 * onemyxb.elem(indices) % (- (this->y)->elem(indices) ) ); 177 | this->b0 -= partial_b0 / (this->n * LipschitzConst); // intercept is not regularized 178 | onemyxb += *(this->y) * (b0old - this->b0); 179 | indices = arma::find(onemyxb > 0); 180 | } 181 | 182 | for (auto& i : this->Order) { 183 | this->UpdateBiWithBounds(i); 184 | } 185 | 186 | this->RestrictSupport(); 187 | 188 | // only way to terminate is by (i) converging on active set and (ii) CWMinCheck 189 | if (this->isConverged()) { 190 | if (this->CWMinCheckWithBounds()) { 191 | break; 192 | } 193 | } 194 | } 195 | 196 | this->result.Objective = this->objective; 197 | this->result.B = this->B; 198 | this->result.Model = this; 199 | this->result.b0 = this->b0; 200 | this->result.IterNum = this->CurrentIters; 201 | this->result.onemyxb = this->onemyxb; 202 | return this->result; 203 | } 204 | 205 | template class CDL012SquaredHinge; 206 | template class CDL012SquaredHinge; 207 | 208 | #endif 209 | -------------------------------------------------------------------------------- /FastSparse/src/include/CDL012SquaredHingeSwaps.h: -------------------------------------------------------------------------------- 1 | #ifndef CDL012SquredHingeSwaps_H 2 | #define CDL012SquredHingeSwaps_H 3 | #include "RcppArmadillo.h" 4 | #include "CD.h" 5 | #include "CDSwaps.h" 6 | #include "CDL012SquaredHinge.h" 7 | #include "Params.h" 8 | #include "FitResult.h" 9 | #include "utils.h" 10 | #include "BetaVector.h" 11 | 12 | template 13 | class CDL012SquaredHingeSwaps : public CDSwaps { 14 | private: 15 | const double LipschitzConst = 2; 16 | double twolambda2; 17 | double qp2lamda2; 18 | double lambda1ol; 19 | double stl0Lc; 20 | // std::vector * Xtr; 21 | 22 | public: 23 | CDL012SquaredHingeSwaps(const T& Xi, const arma::vec& yi, const Params& P); 24 | 25 | FitResult _FitWithBounds() final; 26 | 27 | FitResult _Fit() final; 28 | 29 | inline double Objective(const arma::vec & r, const beta_vector & B) final; 30 | 31 | inline double Objective() final; 32 | 33 | 34 | }; 35 | 36 | template 37 | inline double CDL012SquaredHingeSwaps::Objective(const arma::vec & onemyxb, const beta_vector & B) { 38 | auto l2norm = arma::norm(B, 2); 39 | arma::uvec indices = arma::find(onemyxb > 0); 40 | return arma::sum(onemyxb.elem(indices) % onemyxb.elem(indices)) + this->lambda0 * n_nonzero(B) + this->lambda1 * arma::norm(B, 1) + this->lambda2 * l2norm * l2norm; 41 | } 42 | 43 | template 44 | inline double CDL012SquaredHingeSwaps::Objective() { 45 | throw std::runtime_error("CDL012SquaredHingeSwaps does not have this->onemyxb"); 46 | } 47 | 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /FastSparse/src/include/CDL012Swaps.h: -------------------------------------------------------------------------------- 1 | #ifndef CDL012SWAPS_H 2 | #define CDL012SWAPS_H 3 | #include 4 | #include "RcppArmadillo.h" 5 | #include "CD.h" 6 | #include "CDSwaps.h" 7 | #include "CDL012.h" 8 | #include "FitResult.h" 9 | #include "Params.h" 10 | #include "utils.h" 11 | #include "BetaVector.h" 12 | 13 | template 14 | class CDL012Swaps : public CDSwaps { 15 | public: 16 | CDL012Swaps(const T& Xi, const arma::vec& yi, const Params& Pi); 17 | 18 | FitResult _FitWithBounds() final; 19 | 20 | FitResult _Fit() final; 21 | 22 | double Objective(const arma::vec & r, const beta_vector & B) final; 23 | 24 | double Objective() final; 25 | 26 | }; 27 | 28 | template 29 | inline double CDL012Swaps::Objective(const arma::vec & r, const beta_vector & B) { 30 | auto l2norm = arma::norm(B, 2); 31 | return 0.5 * arma::dot(r, r) + this->lambda0 * n_nonzero(B) + this->lambda1 * arma::norm(B, 1) + this->lambda2 * l2norm * l2norm; 32 | } 33 | 34 | template 35 | inline double CDL012Swaps::Objective() { 36 | throw std::runtime_error("CDL012Swaps does not have this->r."); 37 | } 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /FastSparse/src/include/CDSwaps.h: -------------------------------------------------------------------------------- 1 | #ifndef CDSWAPS_H 2 | #define CDSWAPS_H 3 | #include "CD.h" 4 | 5 | #endif 6 | -------------------------------------------------------------------------------- /FastSparse/src/include/FitResult.h: -------------------------------------------------------------------------------- 1 | #ifndef FITRESULT_H 2 | #define FITRESULT_H 3 | #include "RcppArmadillo.h" 4 | #include "BetaVector.h" 5 | 6 | template // Forward Reference to prevent circular dependencies 7 | class CDBase; 8 | 9 | template 10 | struct FitResult { 11 | double Objective; 12 | beta_vector B; 13 | CDBase * Model; 14 | std::size_t IterNum; 15 | arma::vec * r; 16 | std::vector ModelParams; 17 | double b0 = 0; // used by classification models and sparse regression models 18 | arma::vec ExpyXB; // Used by Logistic regression, [p] <- ExpyXB[i] = exp(y_i * (x_i^T B + b0)) 19 | arma::vec onemyxb; // Used by SquaredHinge regression 20 | 21 | arma::vec inverse_ExpyXB; // used by exponential loss for binary classification with binary features 22 | }; 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /FastSparse/src/include/Grid.h: -------------------------------------------------------------------------------- 1 | #ifndef GRID_H 2 | #define GRID_H 3 | #include 4 | #include 5 | #include 6 | #include "RcppArmadillo.h" 7 | #include "GridParams.h" 8 | #include "FitResult.h" 9 | #include "Grid1D.h" 10 | #include "Grid2D.h" 11 | #include "Normalize.h" 12 | 13 | template 14 | class Grid { 15 | private: 16 | T Xscaled; 17 | arma::vec yscaled; 18 | arma::vec BetaMultiplier; 19 | arma::vec meanX; 20 | double meany; 21 | double scaley; 22 | 23 | public: 24 | 25 | GridParams PG; 26 | 27 | std::vector< std::vector > Lambda0; 28 | std::vector Lambda12; 29 | std::vector< std::vector > NnzCount; 30 | std::vector< std::vector > Solutions; 31 | std::vector< std::vector >Intercepts; 32 | std::vector< std::vector > Converged; 33 | std::vector< std::vector > Objectives; 34 | 35 | Grid(const T& X, const arma::vec& y, const GridParams& PG); 36 | //~Grid(); 37 | 38 | void Fit(); 39 | 40 | }; 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /FastSparse/src/include/Grid1D.h: -------------------------------------------------------------------------------- 1 | #ifndef GRID1D_H 2 | #define GRID1D_H 3 | #include 4 | #include 5 | #include 6 | #include "RcppArmadillo.h" 7 | #include "Params.h" 8 | #include "GridParams.h" 9 | #include "FitResult.h" 10 | #include "MakeCD.h" 11 | 12 | template 13 | class Grid1D { 14 | private: 15 | std::size_t G_ncols; 16 | Params P; 17 | const T * X; 18 | const arma::vec * y; 19 | std::size_t p; 20 | std::vector>> G; 21 | arma::vec Lambdas; 22 | bool LambdaU; 23 | std::size_t NnzStopNum; 24 | std::vector * Xtr; 25 | arma::rowvec * ytX; 26 | double LambdaMinFactor; 27 | bool PartialSort; 28 | bool XtrAvailable; 29 | double ytXmax2d; 30 | double ScaleDownFactor; 31 | std::size_t NoSelectK; 32 | 33 | public: 34 | Grid1D(const T& Xi, const arma::vec& yi, const GridParams& PG); 35 | ~Grid1D(); 36 | std::vector>> Fit(); 37 | }; 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /FastSparse/src/include/Grid2D.h: -------------------------------------------------------------------------------- 1 | #ifndef GRID2D_H 2 | #define GRID2D_H 3 | #include 4 | #include "RcppArmadillo.h" 5 | #include "GridParams.h" 6 | #include "Params.h" 7 | #include "FitResult.h" 8 | #include "Grid1D.h" 9 | #include "utils.h" 10 | 11 | template 12 | class Grid2D 13 | { 14 | private: 15 | std::size_t G_nrows; 16 | std::size_t G_ncols; 17 | GridParams PG; 18 | const T * X; 19 | const arma::vec * y; 20 | std::size_t p; 21 | std::vector>>> G; 22 | // each inner vector corresponds to a single lambda_1/lambda_2 23 | 24 | double Lambda2Max; 25 | double Lambda2Min; 26 | double LambdaMinFactor; 27 | std::vector * Xtr; 28 | Params P; 29 | 30 | 31 | public: 32 | Grid2D(const T& Xi, const arma::vec& yi, const GridParams& PGi); 33 | ~Grid2D(); 34 | std::vector>>> Fit(); 35 | 36 | }; 37 | 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /FastSparse/src/include/GridParams.h: -------------------------------------------------------------------------------- 1 | #ifndef GRIDPARAMS_H 2 | #define GRIDPARAMS_H 3 | #include "RcppArmadillo.h" 4 | #include "Params.h" 5 | 6 | template 7 | struct GridParams 8 | { 9 | Params P; 10 | std::size_t G_ncols = 100; 11 | std::size_t G_nrows = 10; 12 | bool LambdaU = false; 13 | std::size_t NnzStopNum = 200; 14 | double LambdaMinFactor = 0.01; 15 | arma::vec Lambdas; 16 | std::vector< std::vector > LambdasGrid; 17 | double Lambda2Max = 0.1; 18 | double Lambda2Min = 0.001; 19 | std::string Type = "L0"; 20 | bool PartialSort = true; 21 | bool XtrAvailable = false; 22 | double ytXmax; 23 | std::vector * Xtr; 24 | double ScaleDownFactor = 0.8; 25 | bool intercept; 26 | }; 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /FastSparse/src/include/Interface.h: -------------------------------------------------------------------------------- 1 | #ifndef RINTERFACE_H 2 | #define RINTERFACE_H 3 | 4 | #include 5 | #include 6 | #include "RcppArmadillo.h" 7 | #include "Grid.h" 8 | #include "GridParams.h" 9 | #include "FitResult.h" 10 | 11 | 12 | inline void to_arma_error() { 13 | Rcpp::stop("fastSparse.fit only supports sparse matricies (dgCMatrix), 2D arrays (Dense Matricies)"); 14 | } 15 | 16 | #endif // RINTERFACE_H 17 | -------------------------------------------------------------------------------- /FastSparse/src/include/MakeCD.h: -------------------------------------------------------------------------------- 1 | #ifndef MAKECD_H 2 | #define MAKECD_H 3 | #include "RcppArmadillo.h" 4 | #include "Params.h" 5 | #include "CD.h" 6 | #include "CDL0.h" 7 | #include "CDL012.h" 8 | #include "CDL012Swaps.h" 9 | #include "CDL012Logistic.h" 10 | #include "CDL012SquaredHinge.h" 11 | #include "CDL012Exponential.h" 12 | #include "CDL012LogisticSwaps.h" 13 | #include "CDL012SquaredHingeSwaps.h" 14 | #include "CDL012ExponentialSwaps.h" 15 | 16 | 17 | template 18 | CDBase * make_CD(const T& Xi, const arma::vec& yi, const Params& P) { 19 | if (P.Specs.SquaredError) { 20 | if (P.Specs.CD) { 21 | if (P.Specs.L0) { 22 | return new CDL0(Xi, yi, P); 23 | } else { 24 | return new CDL012(Xi, yi, P); 25 | } 26 | } else if (P.Specs.PSI) { 27 | return new CDL012Swaps(Xi, yi, P); 28 | } 29 | } else if (P.Specs.Logistic) { 30 | if (P.Specs.CD) { 31 | return new CDL012Logistic(Xi, yi, P); 32 | } else if (P.Specs.PSI) { 33 | return new CDL012LogisticSwaps(Xi, yi, P); 34 | } 35 | } else if (P.Specs.Exponential) { 36 | if (P.Specs.CD) { 37 | return new CDL012Exponential(Xi, yi, P); 38 | } else if (P.Specs.PSI) { 39 | return new CDL012ExponentialSwaps(Xi, yi, P); 40 | } 41 | }else if (P.Specs.SquaredHinge) { 42 | if (P.Specs.CD) { 43 | return new CDL012SquaredHinge(Xi, yi, P); 44 | } else if (P.Specs.PSI) { 45 | return new CDL012SquaredHingeSwaps(Xi, yi, P); 46 | } 47 | } 48 | return new CDL0(Xi, yi, P); // handle later 49 | } 50 | 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /FastSparse/src/include/Model.h: -------------------------------------------------------------------------------- 1 | #ifndef MODEL_H 2 | #define MODEL_H 3 | 4 | struct Model { 5 | 6 | bool SquaredError = false; 7 | bool Logistic = false; 8 | bool SquaredHinge = false; 9 | bool Exponential = false; 10 | bool Classification = false; 11 | 12 | bool CD = false; 13 | bool PSI = false; 14 | 15 | bool L0 = false; 16 | bool L0L1 = false; 17 | bool L0L2 = false; 18 | bool L1 = false; 19 | bool L1Relaxed = false; 20 | 21 | }; 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /FastSparse/src/include/Normalize.h: -------------------------------------------------------------------------------- 1 | #ifndef NORMALIZE_H 2 | #define NORMALIZE_H 3 | 4 | #include 5 | #include "RcppArmadillo.h" 6 | #include "utils.h" 7 | #include "BetaVector.h" 8 | 9 | std::tuple DeNormalize(beta_vector & B_scaled, 10 | arma::vec & BetaMultiplier, 11 | arma::vec & meanX, double meany); 12 | 13 | template 14 | std::tuple Normalize(const T& X, 15 | const arma::vec& y, 16 | T& X_normalized, 17 | arma::vec & y_normalized, 18 | bool Normalizey, 19 | bool intercept) { 20 | 21 | arma::rowvec meanX = matrix_center(X, X_normalized, intercept); 22 | arma::rowvec scaleX = matrix_normalize(X_normalized); 23 | 24 | arma::vec BetaMultiplier; 25 | double meany = 0; 26 | double scaley; 27 | if (Normalizey) { 28 | if (intercept){ 29 | meany = arma::mean(y); 30 | } 31 | y_normalized = y - meany; 32 | 33 | // TODO: Use l2_norm 34 | scaley = arma::norm(y_normalized, 2); 35 | 36 | // properly handle cases where y is constant 37 | if (scaley == 0){ 38 | scaley = 1; 39 | } 40 | 41 | y_normalized = y_normalized / scaley; 42 | BetaMultiplier = scaley / (scaleX.t()); // transpose scale to get a col vec 43 | // Multiplying the learned Beta by BetaMultiplier gives the optimal Beta on the original scale 44 | } else { 45 | y_normalized = y; 46 | BetaMultiplier = 1 / (scaleX.t()); // transpose scale to get a col vec 47 | scaley = 1; 48 | } 49 | return std::make_tuple(BetaMultiplier, meanX.t(), meany, scaley); 50 | } 51 | 52 | #endif // NORMALIZE_H 53 | -------------------------------------------------------------------------------- /FastSparse/src/include/Params.h: -------------------------------------------------------------------------------- 1 | #ifndef PARAMS_H 2 | #define PARAMS_H 3 | #include 4 | #include "RcppArmadillo.h" 5 | #include "Model.h" 6 | #include "BetaVector.h" 7 | 8 | template 9 | struct Params { 10 | 11 | Model Specs; 12 | std::vector ModelParams {0, 0, 0, 2}; 13 | std::size_t MaxIters = 500; 14 | double rtol = 1e-8; 15 | double atol = 1e-12; 16 | char Init = 'z'; // 'z' => zeros 17 | std::size_t RandomStartSize = 10; 18 | beta_vector * InitialSol; 19 | double b0 = 0; // intercept 20 | char CyclingOrder = 'c'; 21 | std::vector Uorder; 22 | bool ActiveSet = true; 23 | std::size_t ActiveSetNum = 6; 24 | std::size_t MaxNumSwaps = 200; // Used by CDSwaps 25 | std::vector * Xtr; 26 | arma::rowvec * ytX; 27 | std::map * D; 28 | std::size_t Iter = 0; // Current iteration number in the grid 29 | std::size_t ScreenSize = 1000; 30 | arma::vec * r; 31 | T * Xy; // used for classification. 32 | std::size_t NoSelectK = 0; 33 | bool intercept = false; 34 | bool withBounds = false; 35 | arma::vec Lows; 36 | arma::vec Highs; 37 | 38 | std::unordered_map * Xy_neg_indices; // my new stuff, row indices for entries == -1 for each feature column 39 | 40 | }; 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /FastSparse/src/include/Test_Interface.h: -------------------------------------------------------------------------------- 1 | #ifndef R_TEST_INTERFACE_H 2 | #define R_TEST_INTERFACE_H 3 | 4 | #include 5 | #include "utils.h" 6 | #include "BetaVector.h" 7 | 8 | #endif //R_TEST_INTERFACE_H 9 | -------------------------------------------------------------------------------- /FastSparse/src/include/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef FASTSPARSE_UTILS_H 2 | #define FASTSPARSE_UTILS_H 3 | #include 4 | #include "RcppArmadillo.h" 5 | #include "BetaVector.h" 6 | 7 | 8 | template 9 | inline T clamp(T x, T low, T high) { 10 | // -O3 Compiler should remove branches 11 | if (x < low) 12 | x = low; 13 | if (x > high) 14 | x = high; 15 | return x; 16 | } 17 | 18 | void clamp_by_vector(arma::vec &B, const arma::vec& lows, const arma::vec& highs); 19 | 20 | void clamp_by_vector(arma::sp_mat &B, const arma::vec& lows, const arma::vec& highs); 21 | 22 | 23 | template 24 | arma::vec inline matrix_column_get(const arma::mat &mat, T1 col){ 25 | return mat.unsafe_col(col); 26 | } 27 | 28 | template 29 | arma::vec inline matrix_column_get(const arma::sp_mat &mat, T1 col){ 30 | return arma::vec(mat.col(col)); 31 | } 32 | 33 | template 34 | arma::mat inline matrix_rows_get(const arma::mat &mat, const T1 vector_of_row_indices){ 35 | return mat.rows(vector_of_row_indices); 36 | } 37 | 38 | template 39 | arma::sp_mat inline matrix_rows_get(const arma::sp_mat &mat, const T1 vector_of_row_indices){ 40 | // Option for CV for random splitting or contiguous splitting. 41 | // 1 - N without permutations splitting at floor(N/n_folds) 42 | arma::sp_mat row_mat = arma::sp_mat(vector_of_row_indices.n_elem, mat.n_cols); 43 | 44 | for (auto i = 0; i < vector_of_row_indices.n_elem; i++){ 45 | auto row_index = vector_of_row_indices(i); 46 | arma::sp_mat::const_row_iterator begin = mat.begin_row(row_index); 47 | arma::sp_mat::const_row_iterator end = mat.end_row(row_index); 48 | 49 | for (; begin != end; ++begin){ 50 | row_mat(i, begin.col()) = *begin; 51 | } 52 | } 53 | return row_mat; 54 | } 55 | 56 | template 57 | arma::mat inline matrix_vector_schur_product(const arma::mat &mat, const T1 &y){ 58 | // return mat[i, j] * y[i] for each j 59 | return mat.each_col() % *y; 60 | } 61 | 62 | template 63 | arma::sp_mat inline matrix_vector_schur_product(const arma::sp_mat &mat, const T1 &y){ 64 | 65 | arma::sp_mat Xy = arma::sp_mat(mat); 66 | arma::sp_mat::iterator begin = Xy.begin(); 67 | arma::sp_mat::iterator end = Xy.end(); 68 | 69 | auto yp = (*y); 70 | for (; begin != end; ++begin){ 71 | auto row = begin.row(); 72 | *begin = (*begin) * yp(row); 73 | } 74 | return Xy; 75 | } 76 | 77 | template 78 | arma::sp_mat inline matrix_vector_divide(const arma::sp_mat& mat, const T1 &u){ 79 | arma::sp_mat divided_mat = arma::sp_mat(mat); 80 | 81 | //auto up = (*u); 82 | arma::sp_mat::iterator begin = divided_mat.begin(); 83 | arma::sp_mat::iterator end = divided_mat.end(); 84 | for ( ; begin != end; ++begin){ 85 | *begin = (*begin) / u(begin.row()); 86 | } 87 | return divided_mat; 88 | } 89 | 90 | template 91 | arma::mat inline matrix_vector_divide(const arma::mat& mat, const T1 &u){ 92 | return mat.each_col() / u; 93 | } 94 | 95 | arma::rowvec inline matrix_column_sums(const arma::mat& mat){ 96 | return arma::sum(mat, 0); 97 | } 98 | 99 | arma::rowvec inline matrix_column_sums(const arma::sp_mat& mat){ 100 | return arma::rowvec(arma::sum(mat, 0)); 101 | } 102 | 103 | template 104 | double inline matrix_column_dot(const arma::mat &mat, T1 col, const T2 &u){ 105 | return arma::dot(matrix_column_get(mat, col), u); 106 | } 107 | 108 | template 109 | double inline matrix_column_dot(const arma::sp_mat &mat, T1 col, const T2 &u){ 110 | return arma::dot(matrix_column_get(mat, col), u); 111 | } 112 | 113 | template 114 | arma::vec inline matrix_column_mult(const arma::mat &mat, T1 col, const T2 &u){ 115 | return matrix_column_get(mat, col)*u; 116 | } 117 | 118 | template 119 | arma::vec inline matrix_column_mult(const arma::sp_mat &mat, T1 col, const T2 &u){ 120 | return matrix_column_get(mat, col)*u; 121 | } 122 | 123 | arma::rowvec matrix_normalize(arma::sp_mat &mat_norm); 124 | 125 | arma::rowvec matrix_normalize(arma::mat &mat_norm); 126 | 127 | arma::rowvec matrix_center(const arma::mat& X, arma::mat& X_normalized, bool intercept); 128 | 129 | arma::rowvec matrix_center(const arma::sp_mat& X, arma::sp_mat& X_normalized, bool intercept); 130 | 131 | #endif //FASTSPARSE_UTILS_H 132 | -------------------------------------------------------------------------------- /FastSparse/src/profile.cpp: -------------------------------------------------------------------------------- 1 | #include "RcppArmadillo.h" 2 | #include "gperftools/profiler.h" 3 | 4 | using namespace Rcpp; 5 | 6 | // [[Rcpp::export]] 7 | SEXP start_profiler(SEXP str) { 8 | ProfilerStart(as(str)); 9 | return R_NilValue; 10 | } 11 | 12 | // [[Rcpp::export]] 13 | SEXP stop_profiler() { 14 | ProfilerStop(); 15 | return R_NilValue; 16 | } 17 | -------------------------------------------------------------------------------- /FastSparse/src/utils.cpp: -------------------------------------------------------------------------------- 1 | #include "utils.h" 2 | 3 | void clamp_by_vector(arma::vec &B, const arma::vec& lows, const arma::vec& highs){ 4 | const std::size_t n = B.n_rows; 5 | for (std::size_t i = 0; i < n; i++){ 6 | B.at(i) = clamp(B.at(i), lows.at(i), highs.at(i)); 7 | } 8 | } 9 | 10 | void clamp_by_vector(arma::sp_mat &B, const arma::vec& lows, const arma::vec& highs){ 11 | // See above implementation without filter for error. 12 | auto begin = B.begin(); 13 | auto end = B.end(); 14 | 15 | std::vector inds; 16 | for (; begin != end; ++begin) 17 | inds.push_back(begin.row()); 18 | 19 | auto n = B.size(); 20 | inds.erase(std::remove_if(inds.begin(), 21 | inds.end(), 22 | [n](size_t x){return (x > n) && (x < 0);}), 23 | inds.end()); 24 | for (auto& it : inds) { 25 | double B_item = B(it, 0); 26 | const double low = lows(it); 27 | const double high = highs(it); 28 | B(it, 0) = clamp(B_item, low, high); 29 | } 30 | } 31 | 32 | arma::rowvec matrix_normalize(arma::sp_mat &mat_norm){ 33 | auto p = mat_norm.n_cols; 34 | arma::rowvec scaleX = arma::zeros(p); // will contain the l2norm of every col 35 | 36 | for (auto col = 0; col < p; col++){ 37 | double l2norm = arma::norm(matrix_column_get(mat_norm, col), 2); 38 | scaleX(col) = l2norm; 39 | } 40 | 41 | scaleX.replace(0, -1); 42 | 43 | for (auto col = 0; col < p; col++){ 44 | arma::sp_mat::col_iterator begin = mat_norm.begin_col(col); 45 | arma::sp_mat::col_iterator end = mat_norm.end_col(col); 46 | for (; begin != end; ++begin) 47 | (*begin) = (*begin)/scaleX(col); 48 | } 49 | 50 | if (mat_norm.has_nan()) 51 | mat_norm.replace(arma::datum::nan, 0); // can handle numerical instabilities. 52 | 53 | return scaleX; 54 | } 55 | 56 | arma::rowvec matrix_normalize(arma::mat& mat_norm){ 57 | 58 | auto p = mat_norm.n_cols; 59 | arma::rowvec scaleX = arma::zeros(p); // will contain the l2norm of every col 60 | 61 | for (auto col = 0; col < p; col++) { 62 | double l2norm = arma::norm(matrix_column_get(mat_norm, col), 2); 63 | scaleX(col) = l2norm; 64 | } 65 | 66 | scaleX.replace(0, -1); 67 | mat_norm.each_row() /= scaleX; 68 | 69 | if (mat_norm.has_nan()){ 70 | mat_norm.replace(arma::datum::nan, 0); // can handle numerical instabilities. 71 | } 72 | 73 | return scaleX; 74 | } 75 | 76 | arma::rowvec matrix_center(const arma::mat& X, arma::mat& X_normalized, 77 | bool intercept){ 78 | auto p = X.n_cols; 79 | arma::rowvec meanX; 80 | 81 | if (intercept){ 82 | meanX = arma::mean(X, 0); 83 | X_normalized = X.each_row() - meanX; 84 | } else { 85 | meanX = arma::zeros(p); 86 | X_normalized = arma::mat(X); 87 | } 88 | 89 | return meanX; 90 | } 91 | 92 | arma::rowvec matrix_center(const arma::sp_mat& X, arma::sp_mat& X_normalized, 93 | bool intercept){ 94 | auto p = X.n_cols; 95 | arma::rowvec meanX = arma::zeros(p); 96 | X_normalized = arma::sp_mat(X); 97 | return meanX; 98 | } 99 | -------------------------------------------------------------------------------- /GOVERNANCE.md: -------------------------------------------------------------------------------- 1 | # Governance Policy 2 | 3 | This document provides the minimum governance policy for Projects in the Organization. Maintainers agree to this policy and to abide by all of the Organization's polices, including the code of conduct, trademark policy, and antitrust policy by adding their name to the Maintainer's file. 4 | 5 | ## 1. Roles. 6 | 7 | Each Project may include the following roles. Additional roles may be adopted and documented by the Project. 8 | 9 | **1.1. Maintainer**. “Maintainers” are responsible for organizing activities around developing, maintaining, and updating the Project. Maintainers are also responsible for determining consensus. Each Project will designate one or more Maintainer. A Project may add or remove Maintainers with the approval of the current Maintainers (absent the maintainer being removed) or oversight of the Organization's Technical Steering Committee ("TSC"). 10 | 11 | **1.2. Contributors**. “Contributors” are those that have made Contributions to the Project. 12 | 13 | ## 2. Decisions. 14 | 15 | **2.1. Consensus-Based Decision Making**. Projects make decisions through consensus of the Maintainers. While explicit agreement of all Maintainers is preferred, it is not required for consensus. Rather, the Maintainers will determine consensus based on their good faith consideration of a number of factors, including the dominant view of the Contributors and nature of support and objections. The Maintainers will document evidence of consensus in accordance with these requirements. 16 | 17 | **2.2. Appeal Process**. Decisions may be appealed by opening an issue and that appeal will be considered by the maintainers in good faith, who will respond in writing within a reasonable time. If the maintainers deny the appeal, the appeal my be brought before the TSC, who will also respond in writing in a reasonable time. 18 | 19 | ## 3. How We Work. 20 | 21 | **3.1. Openness**. Participation shall be open to all persons who are directly and materially affected by the activity in question. There shall be no undue financial barriers to participation. 22 | 23 | **3.2. Balance.** The development process should have a balance of interests. Contributors from diverse interest categories shall be sought with the objective of achieving balance. 24 | 25 | **3.3. Coordination and Harmonization.** Good faith efforts shall be made to resolve potential conflicts or incompatibility between releases in this Project. 26 | 27 | **3.4. Consideration of Views and Objections.** Prompt consideration shall be given to the written views and objections of all Contributors. 28 | 29 | **3.5. Written procedures.** This governance document and other materials documenting this project's development process shall be available to any interested person. 30 | 31 | ## 4. No Confidentiality. 32 | 33 | Information disclosed in connection with any Project activity, including but not limited to meetings, Contributions, and submissions, is not confidential, regardless of any markings or statements to the contrary. 34 | 35 | ## 5. Trademarks. 36 | 37 | Any names, trademarks, logos, or goodwill arising out of the Project - however owned - may be only used in accordance with the Organization's Trademark Policy. Maintainer's obligations under this section survive their affiliation with the Project. 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Hussein Hazimeh 4 | Copyright (c) 2022, 2023 Jiachang Liu 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | # Maintainers 2 | 3 | This document lists the Maintainers of Project InterpretML. Maintainers may be added once approved by consensus of the existing maintainers as described in the Governance document. By adding your name to this list you are agreeing to and to abide by the Project governance documents and to abide by all of the Organization's polices, including the code of conduct, trademark policy, and antitrust policy. If you are participating on behalf another organization (designated below), you represent that you have permission to bind that organization to these policies. 4 | 5 | | **NAME** | **Organization** | 6 | | --- | --- | 7 | | Jiachang Liu | | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fastSparse 2 | 3 | This repository contains source code to our AISTATS 2022 paper: 4 | 5 | * [Fast Sparse Classification for Generalized Linear and Additive Models](https://arxiv.org/abs/2202.11389) 6 | 7 | --- 8 | ## 1. Installation ## 9 | 10 | To install this R package, please go to the [installation folder](./installation) and follow the installation instructions. 11 | 12 | --- 13 | ## 2. Application and Usage 14 | We provide a toolkit for producing sparse and interpretable generalized linear and additive models for the binary classiciation task by solving the L0-regularized problems. The classiciation loss can be either the logistic loss or the exponential loss. The algorithms can produce high quality (swap 1-OPT) solutions and are generally 2 to 5 times faster than previous approaches. 15 | 16 | ### 2.1 R Interface 17 | To understand how to use this R package, please go to the [application_and_usage_R_interface_folder](./application_and_usage_R_interface). 18 | 19 | ### 2.2 Python Interface 20 | To understand how to use this package in a python environment, we provide a python wrapper to acheive this. Please go to the [application_and_usage_python_interface_folder](./application_and_usage_python_interface). 21 | 22 | 23 | ## 3. Experiment Replication 24 | To replicate our experimental results shown in the paper, please go to the [experiments folder](./experiments). 25 | 26 | ## 4. Step Function Visualization 27 | To reproduce the step function plots shown in the paper, please go to the [step_function_visualization folder](./step_function_visualization). 28 | 29 | ## 5. Citing Our Work ## 30 | If you find our work useful in your research, please consider citing the following paper: 31 | 32 | ```bibTex 33 | @inproceedings{liu2022fast, 34 | title={Fast Sparse Classification for Generalized Linear and Additive Models}, 35 | author={Liu, Jiachang and Zhong, Chudi and Seltzer, Margo and Rudin, Cynthia}, 36 | booktitle={International Conference on Artificial Intelligence and Statistics}, 37 | pages={9304--9333}, 38 | year={2022}, 39 | organization={PMLR} 40 | } 41 | ``` 42 | 43 | Acknowledgement: For this AISTATS camera ready code repository, we build our method based on [L0Learn](https://github.com/hazimehh/L0Learn#l0learn-fast-best-subset-selection)’s codebase, so that we could use its preprocessing steps, and the pipeline for running the full regularization path of $λ_0$ values. Therefore, the computational speedup shown in our paper solely comes from our new proposed algorithms. 44 | 45 | We plan to build our own pipeline and further extend this work before pushing the project to CRAN. Right now you can install the project from GitHub directly. Our repository will be actively maintained, and the most updated version can be found at this current GitHub page. 46 | 47 | 48 | ## Package Development ToDo List 49 | - [ ] Fix windows installation issues. 50 | - [x] Add language specification to codeblock in README. 51 | - [x] Add binarization preprocessing function and provide usage in jupyter notebook. 52 | -------------------------------------------------------------------------------- /Unified_API/README.md: -------------------------------------------------------------------------------- 1 | # Table of Content 2 | - [Overview](#overview) 3 | - [Comparison between FastSparse and FasterRisk](#comparison-between-fastsparse-and-fasterrisk) 4 | - [Loss functions for different models:](#loss-functions-for-different-models) 5 | - [API - Sparse Additive Models](#api---sparse-additive-models) 6 | - [FastSparse:](#fastsparse) 7 | - [1. Fit on Data](#1-fit-on-data) 8 | - [1a. Fit with No Monotonicity Constraint](#1a-fit-with-no-monotonicity-constraint) 9 | - [1b. Fit with Monotonicity Constraint](#1b-fit-with-monotonicity-constraint) 10 | - [2. Get Parameters after Fitting](#2-get-parameters-after-fitting) 11 | - [3. Predict](#3-predict) 12 | - [FasterRisk](#fasterrisk) 13 | - [1. Fit on Data](#1-fit-on-data-1) 14 | - [2. Get Parameters after Fitting](#2-get-parameters-after-fitting-1) 15 | - [3. Predict](#3-predict-1) 16 | 17 | # Overview 18 | The end goal is to provide a unified API so that people can use both FastSparse and FasterRisk from the same library. To do this, the easiest way is to write another python library wrapper on top of the FastSparse and FasterRisk python libraries. We call this library **Sparse Additive Models**. 19 | 20 | When we use pip to install sparseadditivemodels, pip will first install fastsparse and fasterrisk. When we use sparseadditivemodels's functions, the function will call the corresponding function from FastSparse or FasterRisk. 21 | 22 | FasterRisk is already in python implementation and published on PyPI. The main enigieering difficulty is to **build a python library** for FastSparse (with C++ Armadillo implementation) without calling R. 23 | 24 | ## Comparison between FastSparse and FasterRisk 25 | Summary of differences between FastSparse and FasterRisk: 26 | | Key Differences | FastSparse | FasterRisk | 27 | | :--- | :---- | :--- | 28 | | sparsity is controlled | implicitly by $\ell_0$ regularization | explicitly by support size $k$ | 29 | |objective function| $\mathcal{L}(\beta) = \sum \text{}_{i=1}^{n} l(y_i, x_i, \beta) + \lambda_2 \Vert \beta \Vert _2^2 + \lambda_0 \Vert \beta \Vert _0$ |

$\mathcal{L}(\beta) = \sum \text{}_{i=1}^{n} l(y_i, x_i, \beta)$

subject to $\Vert \beta \Vert _0 \leq k$ | 30 | | model options |

Linear Regression

Logistic Regression

Adaboost (Note: $\lambda_2$ must be $0$) | Logistic Regression | 31 | | continuous coefficient | ✓ | ✗ (not yet; can be done easily) | 32 | | integer coefficient | ✗ | ✓ | 33 | | bound on coefficient| ✗ (not yet; can be done with effort) | ✓ | 34 | 35 | ## Loss functions for different models: 36 | 37 | | model | loss name | loss function $l(\cdot)$ | 38 | | :-- | :-- | :-- | 39 | |Linear Regression | square loss | $l(y_i, x_i, \beta) := (y_i - x_i^T \beta)^2$| 40 | |Logistic Regression | logistic loss| $l(y_i, x_i, \beta) := \log(1+\exp(-y_i x_i^T \beta))$| 41 | |Adaboost | exponential loss |$l(y_i, x_i, \beta) := \exp(-y_i x_i^T \beta)$| 42 | 43 | 44 | # API - Sparse Additive Models 45 | We call our unified API "SparseAdditiveModels", which can be used to call either FastSparse or FasterRisk. 46 | 47 | ```python 48 | import SparseAdditiveModels as SAM 49 | ``` 50 | 51 | ## FastSparse: 52 | ### 1. Fit on Data 53 | #### 1a. Fit with No Monotonicity Constraint 54 | Fit with a specified $(\lambda_2, \lambda_0)$ pair. 55 | ```python 56 | # X_trian.shape = (n, p), y_train.shape = (n, ), lambda2 and lambda0 are scalars 57 | sam = SAM.FastSparse_fit(X=X_train, y=y_train, loss="Square", lambda2=lambda2, lambda0=lambda0) # loss can also be "Logistic" or "Exponential" 58 | ``` 59 | 60 | Fit with a specified $\lambda_2$ value and maximum support size $k$. The algorithm will fit a regularization path with different $\lambda_0$ values (chosen automatically) from large to small until the support size exceeds $k$. 61 | ```python 62 | # X_trian.shape = (n, p), y_train.shape = (n, ), lambda2 and k are scalars 63 | sam = SAM.FastSparse_fit_path(X=X_train, y=y_train, loss="Square", lambda2=lambda2, maxSupp=k) # loss can also be "Logistic" or "Exponential" 64 | ``` 65 | 66 | #### 1b. Fit with Monotonicity Constraint 67 | To impose monotonicity constraint, we can achieve this by adding lower and upper bounds on the coefficients. For example, to constrain a coefficient to be positive, we set the lower bound to be 0, and the upper bound to be a very large number like $1 \times 10^6$. 68 | 69 | Fit with a specified $(\lambda_2, \lambda_0)$ pair and lower and upper bounds on each coefficient. 70 | ```python 71 | # X_trian.shape = (n, p), y_train.shape = (n, ), lambda2 and lambda0 are scalars, coeff_low.shape = (p, ), coeff_high.shape = (p, ) 72 | sam = SAM.FastSparse_fit(X=X_train, y=y_train, loss="Square", lambda2=lambda2, lambda0=lambda0, coeffs_low=coeff_low, coeff_high=coeff_high) 73 | ``` 74 | 75 | Fit with a specified $\lambda_2$ value, maximum support size $k$, and lower and upper bounds on each coefficient. The algorithm will fit a regularization path with different $\lambda_0$ values (chosen automatically) from large to small until the support size exceeds $k$. 76 | ```python 77 | # X_trian.shape = (n, p), y_train.shape = (n, ), lambda2 and k are scalars, coeff_low.shape = (p, ), coeff_high.shape = (p, ) 78 | sam = SAM.FastSparse_fit_path(X=X_train, y=y_train, loss="Square", lambda2=lambda2, maxSupp=k, coeff_low=coeff_low, coeff_high=coeff_high) # loss can also be "Logistic" or "Exponential" 79 | ``` 80 | 81 | ### 2. Get Parameters after Fitting 82 | ```python 83 | print("training time is {}".format(sam.train_duration)) 84 | for model in sam.models : 85 | # coeff.shape = (p, ), intercept, lambda2 and lambda0 are scalars 86 | coeff, intercept, lambda2, lambda0 = model.coeff_ model.intercept, lambda2, lambda0 87 | print("The coefficients are {}; the intercept is {}; lambda2 is {}; lambda0 is {}".format(coeff, intercept, lambda2, lambda0)) 88 | ``` 89 | 90 | ### 3. Predict 91 | ```python 92 | for model in sam.models: 93 | # X_test.shape = (n_test, p), y_test.shape = (n_test, ) 94 | y_test = model.predict(X=X_test) 95 | ``` 96 | 97 | ## FasterRisk 98 | ### 1. Fit on Data 99 | Fit with a specified support size $k$, a coefficient lower bound coeff$_{low}$, and a coefficient upper bound coeff$_{high}$ with **continuous** coefficients 100 | ```python 101 | # X_train.shape = (n, p), y_train.shape = (n, ), k is a scalar, coeff_low.shape = (p, ), coeff_high.shape = (p, ) 102 | sam = SAM.FastSparse_fit(X=X_train, y=y_train, suppSize=k, coeff_low=coeff_low, coeff_high=coeff_high, coefficient_type="continuous") 103 | ``` 104 | 105 | Fit with a specified support size $k$, a coefficient lower bound coeff$_{low}$, and a coefficient upper bound coeff$_{high}$ with **integer** coefficients 106 | ```python 107 | # X_train.shape = (n, p), y_train.shape = (n, ), k is a scalar, coeff_low.shape = (p, ), coeff_high.shape = (p, ) 108 | sam = SAM.FastSparse_fit(X=X_train, y=y_train, suppSize=k, coeff_low=coeff_low, coeff_high=coeff_high, coefficient_type="integer") 109 | ``` 110 | 111 | 112 | 113 | ### 2. Get Parameters after Fitting 114 | ```python 115 | print("training time is {}".format(sam.train_duration)) 116 | for model in sam.models : 117 | # coeff.shape = (p, ), intercept and multiplier are scalars 118 | coeff, intercept, multiplier = model.coeff_ model.intercept, lambda2, lambda0 119 | print("The coefficients are {}; the intercept is {}; the multiplier is {}".format(coeff, intercept, multiplier)) 120 | ``` 121 | 122 | ### 3. Predict 123 | ```python 124 | for model in sam.models: 125 | # X_test.shape = (n_test, p), y_test.shape = (n_test, ) 126 | y_test = model.predict(X=X_test) 127 | ``` -------------------------------------------------------------------------------- /application_and_usage_R_interface/README.md: -------------------------------------------------------------------------------- 1 | # fastSparse 2 | 3 | This repository contains source code to our AISTATS 2022 paper: 4 | 5 | * [Fast Sparse Classification for Generalized Linear and Additive Models](https://arxiv.org/abs/2202.11389) 6 | 7 | --- 8 | ## 2. Application and Usage - R Interface 9 | We provide a toolkit for producing sparse and interpretable generalized linear and additive models for the binary classiciation task by solving the L0-regularized problems. The classiciation loss can be either the logistic loss or the exponential loss. The algorithms can produce high quality (swap 1-OPT) solutions and are generally 2 to 5 times faster than previous approaches. 10 | 11 | **To use fastSparse directly in a python environment, please go to the folder [application_and_usage_python_interface](../application_and_usage_python_interface).** 12 | 13 | ### 2.1 Logistic Regression 14 | For fast sparse logistic regression, we propose to use linear/quadratic surrogate cuts that allow us to efficiently screen features for elimination, as well as use of a priority queue that favors a more uniform exploration of features. 15 | 16 | If you go inside FastSparse_0.1.0.tar.gz, the proposed linear/quadratic surrogate cuts and priority queue techniques can be found in "src/CDL012LogisticSwaps.cpp". 17 | 18 | To fit a single pair (λ0=3.0, λ2=0.001) regularization and extract the coefficients, you can use the following code in your Rscript: 19 | ```r 20 | library(FastSparse) 21 | fit <- FastSparse.fit(X_train, y_train, loss="Logistic", algorithm="CDPSI", penalty="L0L2", autoLambda=FALSE, lambdaGrid=list(3.0), nGamma=1, gammaMin=0.001, gammaMax=0.001) 22 | beta = as.vector(coef(fit, lambda=3.0, gamma=0.001)) # first element is intercept 23 | ``` 24 | 25 | To fit a full regularization path with just a single (λ2=0.001) regularization (the algorithm will automatically pick appropriate λ0 values) and extract all coefficients along this regularization path, you can use the following code in your Rscript: 26 | ```r 27 | library(FastSparse) 28 | fit <- FastSparse.fit(X_train, y_train, loss="Logistic", algorithm="CDPSI", penalty="L0L2", nGamma=1, gammaMin=0.001, gammaMax=0.001) 29 | for (i in 1:lengths(fit$lambda)){ 30 | lamb = fit$lambda[[1]][i] 31 | beta = as.vector(coef(fit, lambda=lamb, gamma=0.001)) # first element is intercept 32 | ``` 33 | 34 | 35 | ### 2.2 Exponential Loss 36 | As an alterantive to the logistic loss, we propose the exponential loss, which permits an analytical solution to the line search at each iteration. 37 | 38 | One caveat of using the exponential loss is that make sure your X_train feature matrix are binary with each entry equal only to 0 or 1. Please refer to Appendix D.4 and Figure 10-13 in our paper to see why it is necessary for the feature matrix to be binary (0 and 1) to produce visually interpretable additive models. 39 | 40 | If you inside FastSparse_0.1.0.tar.gz, the proposed exponential loss implementations can be found in "src/include/CDL012Exponential.h", "src/include/CDL012ExponentialSwaps.h", and "src/CDL012ExponentialSwaps.cpp". 41 | 42 | Like the logistic loss shown above, to fit a single (λ0=3.0) regularization and extract the coefficients, you can use the following code in your Rscript: 43 | ```r 44 | library(FastSparse) 45 | fit <- FastSparse.fit(X_train, y_train, loss="Exponential", algorithm="CDPSI", penalty="L0L2", autoLambda=FALSE, lambdaGrid=list(3.0), nGamma=1, gammaMin=0.001, gammaMax=0.001) 46 | beta = as.vector(coef(fit, lambda=3.0, gamma=0.001)) # first element is intercept 47 | ``` 48 | 49 | To fit a full regularization path (the algorithm will automatically pick appropriate λ0 values) and extract all coefficients along this regularization path, you can use the following code in your Rscript: 50 | ```r 51 | library(FastSparse) 52 | fit <- FastSparse.fit(X_train, y_train, loss="Exponential", algorithm="CDPSI", penalty="L0L2", nGamma=1, gammaMin=0.00001, gammaMax=0.001) 53 | for (i in 1:lengths(fit$lambda)){ 54 | lamb = fit$lambda[[1]][i] 55 | beta = as.vector(coef(fit, lambda=lamb, gamma=0.00001)) # first element is intercept 56 | ``` 57 | 58 | Note that for the above two examples, the internal code actually does not impose λ2 regularization for the exponential loss (please refer to Section 4 in our paper for the detailed reason). The "gamma=0.00001" only serves as a placeholder so that we can extract the coefficient correctly. 59 | 60 | ### 2.3 Linear Regression 61 | Although our method is designed for classification problems, our proposed dynamic ordering technique can also speed up the local swap process for linear regression. 62 | 63 | If you inside FastSparse_0.1.0.tar.gz, the proposed priority queue technique is implemented in "src/include/CDL012Swaps". 64 | 65 | To fit a full regularization path with just a single (λ2=0.001) regularization (the algorithm will automatically pick appropriate λ0 values) and extract all coefficients along this regularization path, you can use the following code in your Rscript: 66 | ```r 67 | fit <- FastSparse.fit(X_train, y_train, penalty="L0L2", algorithm="CDPSI", maxSuppSize = 300, autoLambda=False, nGamma = 1, gammaMin = 0.001, gammaMax = 0.001) 68 | for (i in 1:lengths(fit$lambda)){ 69 | lamb = fit$lambda[[1]][i] 70 | beta = as.vector(coef(fit, lambda=lamb, gamma=0.001)) # first element is intercept 71 | ``` -------------------------------------------------------------------------------- /application_and_usage_python_interface/README.md: -------------------------------------------------------------------------------- 1 | # fastSparse 2 | 3 | This repository contains source code to our AISTATS 2022 paper: 4 | 5 | * [Fast Sparse Classification for Generalized Linear and Additive Models](https://arxiv.org/abs/2202.11389) 6 | 7 | Update (09/17/23): We have created a new python package called [FastSparseGAMS](https://github.com/ubc-systopia/L0Learn/tree/master/python). Instead of using a python wrapper as stated below, you can now install FastSparse via pip directly through the following commands: 8 | 9 | ```bash 10 | pip install fastsparsegams 11 | ``` 12 | 13 | Please go to FastSparseGAM's [tutorial page](https://github.com/ubc-systopia/L0Learn/blob/master/python/tutorial_example/example.py) for examples on how to use the new interface. 14 | 15 | The instructions (Section 2) below still work, but we recommand using the new python package first. 16 | 17 | ## 2. Application and Usage - Python Interface 18 | **We provide a wrapper to use fastSparse in a python environment** 19 | 20 | **To use fastSparse directly in an R environment, please go to the folder [application_and_usage_R_interface](../application_and_usage_R_interface).** 21 | 22 | We provide a toolkit for producing sparse and interpretable generalized linear and additive models for the binary classiciation task by solving the L0-regularized problems. The classiciation loss can be either the logistic loss or the exponential loss. The algorithms can produce high quality (swap 1-OPT) solutions and are generally 2 to 5 times faster than previous approaches. 23 | 24 | ### 2.0 Import the python wrapper 25 | We need to use a python wrapper to interact with the R code. To do this, make sure to copy and paste the following code at the top of your python file 26 | 27 | ```python 28 | from rpy2.robjects.packages import importr 29 | import rpy2.robjects.numpy2ri 30 | rpy2.robjects.numpy2ri.activate() 31 | base = importr('base') 32 | d = {'package.dependencies': 'package_dot_dependencies'} 33 | FastSparse = importr('FastSparse', robject_translations = d) 34 | ``` 35 | 36 | ### 2.1 Logistic Regression 37 | For fast sparse logistic regression, we propose to use linear/quadratic surrogate cuts that allow us to efficiently screen features for elimination, as well as use of a priority queue that favors a more uniform exploration of features. 38 | 39 | If you go inside FastSparse_0.1.0.tar.gz, the proposed linear/quadratic surrogate cuts and priority queue techniques can be found in "src/CDL012LogisticSwaps.cpp". 40 | 41 | To fit a single pair (λ0=3.0, λ2=0.001) regularization and extract the coefficients, you can use the following code in your Rscript: 42 | ```python 43 | fit = FastSparse.FastSparse_fit(X_train, y_train, loss="Logistic", algorithm="CDPSI", penalty="L0L2", autoLambda=FALSE, lambdaGrid=[3.0], nGamma=1, gammaMin=0.001, gammaMax=0.001) 44 | beta = np.asarray(base.as_matrix(FastSparse.coef_FastSparse(fit))) # first element is intercept 45 | ``` 46 | 47 | To fit a full regularization path with just a single (λ2=0.001) regularization (the algorithm will automatically pick appropriate λ0 values) and extract all coefficients along this regularization path, you can use the following code in your Rscript: 48 | ```python 49 | fit = FastSparse.FastSparse_fit(X_train, y_train, loss="Logistic", algorithm="CDPSI", penalty="L0L2", nGamma=1, gammaMin=0.001, gammaMax=0.001) 50 | 51 | lambda0s = np.asarray(base.as_matrix(fit.rx2('lambda')))[0] 52 | betas = np.asarray(base.as_matrix(FastSparse.coef_FastSparse(fit))) 53 | 54 | # examine pairs of lambda0, beta 55 | for i in range(len(lambda0s)): 56 | lambda0 = lambda0s[i] 57 | beta = betas[:, i] # first element is intercept 58 | ``` 59 | 60 | 61 | ### 2.2 Exponential Loss 62 | As an alterantive to the logistic loss, we propose the exponential loss, which permits an analytical solution to the line search at each iteration. 63 | 64 | One caveat of using the exponential loss is that make sure your X_train feature matrix are binary with each entry equal only to 0 or 1. Please refer to Appendix D.4 and Figure 10-13 in our paper to see why it is necessary for the feature matrix to be binary (0 and 1) to produce visually interpretable additive models. 65 | 66 | If you inside FastSparse_0.1.0.tar.gz, the proposed exponential loss implementations can be found in "src/include/CDL012Exponential.h", "src/include/CDL012ExponentialSwaps.h", and "src/CDL012ExponentialSwaps.cpp". 67 | 68 | Like the logistic loss shown above, to fit a single (λ0=3.0) regularization and extract the coefficients, you can use the following code in your Rscript: 69 | ```python 70 | fit = FastSparse.FastSparse_fit(X_train, y_train, loss="Exponential", algorithm="CDPSI", penalty="L0L2", autoLambda=FALSE, lambdaGrid=[3.0], nGamma=1, gammaMin=0.001, gammaMax=0.001) 71 | beta = np.asarray(base.as_matrix(FastSparse.coef_FastSparse(fit))) # first element is intercept 72 | ``` 73 | 74 | To fit a full regularization path (the algorithm will automatically pick appropriate λ0 values) and extract all coefficients along this regularization path, you can use the following code in your Rscript: 75 | ```python 76 | fit = FastSparse.FastSparse_fit(X_train, y_train, loss="Exponential", algorithm="CDPSI", penalty="L0L2", nGamma=1, gammaMin=0.00001, gammaMax=0.00001) 77 | 78 | lambda0s = np.asarray(base.as_matrix(fit.rx2('lambda')))[0] 79 | betas = np.asarray(base.as_matrix(FastSparse.coef_FastSparse(fit))) 80 | 81 | # examine pairs of lambda0, beta 82 | for i in range(len(lambda0s)): 83 | lambda0 = lambda0s[i] 84 | beta = betas[:, i] # first element is intercept 85 | ``` 86 | 87 | Note that for the above two examples, the internal code actually does not impose λ2 regularization for the exponential loss (please refer to Section 4 in our paper for the detailed reason). The "gamma=0.00001" only serves as a placeholder so that we can extract the coefficient correctly. 88 | 89 | ### 2.3 Linear Regression 90 | Although our method is designed for classification problems, our proposed dynamic ordering technique can also speed up the local swap process for linear regression. 91 | 92 | If you inside FastSparse_0.1.0.tar.gz, the proposed priority queue technique is implemented in "src/include/CDL012Swaps". 93 | 94 | To fit a full regularization path with just a single (λ2=0.001) regularization (the algorithm will automatically pick appropriate λ0 values) and extract all coefficients along this regularization path, you can use the following code in your Rscript: 95 | ```python 96 | fit <- FastSparse.fit(X_train, y_train, penalty="L0L2", algorithm="CDPSI", maxSuppSize = 300, autoLambda=False, nGamma = 1, gammaMin = 0.001, gammaMax = 0.001) 97 | for (i in 1:lengths(fit$lambda)){ 98 | lamb = fit$lambda[[1]][i] 99 | beta = as.vector(coef(fit, lambda=lamb, gamma=0.001)) # first element is intercept 100 | 101 | fit = FastSparse.FastSparse_fit(X_train, y_train, algorithm="CDPSI", penalty="L0L2", nGamma=1, gammaMin=0.001, gammaMax=0.001) 102 | 103 | lambda0s = np.asarray(base.as_matrix(fit.rx2('lambda')))[0] 104 | betas = np.asarray(base.as_matrix(FastSparse.coef_FastSparse(fit))) 105 | 106 | # examine pairs of lambda0, beta 107 | for i in range(len(lambda0s)): 108 | lambda0 = lambda0s[i] 109 | beta = betas[:, i] # first element is intercept 110 | ``` 111 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: fastSparse_environment 2 | channels: 3 | - conda-forge 4 | - r 5 | dependencies: 6 | - python=3.8 7 | - numpy 8 | - matplotlib 9 | - pandas 10 | - notebook 11 | - r-base 12 | - r-essentials 13 | - tqdm 14 | - scipy 15 | - rclone 16 | - scikit-learn 17 | - seaborn 18 | - jupyterlab 19 | - rpy2 20 | - seaborn 21 | - numba 22 | - htop 23 | - gperftools 24 | - autoconf 25 | - ipympl 26 | -------------------------------------------------------------------------------- /experiments/README.md: -------------------------------------------------------------------------------- 1 | # fastSparse 2 | 3 | This page contains the source code to reproduce results in our AISTATS 2022 paper: 4 | 5 | * [Fast Sparse Classification for Generalized Linear and Additive Models](https://arxiv.org/abs/2202.11389) 6 | 7 | The FICO dataset is publically available. You can request the data from this [link](https://community.fico.com/s/explainable-machine-learning-challenge). 8 | 9 | To convert the original FICO dataset with continuous features into binary features, please refer to our header in this [file](../visualization/fico_bin_first_5_rows.csv). 10 | 11 | ## 1. Time Comparison Experiment 12 | For the time comparison experiments, please run the following line in your terminal 13 | 14 | ``` 15 | Rscript run_time.R 16 | ``` 17 | 18 | ## 2. Solution Quality Experiment of the Entire Regularization Path 19 | For the solution quality experiment, please run the following line in your terminal 20 | 21 | ``` 22 | Rscript run_baseline.R 23 | ``` 24 | The above script contains code to run experiments on both the real datasets and the synthetically generated data. -------------------------------------------------------------------------------- /experiments/comp_time.R: -------------------------------------------------------------------------------- 1 | source("utils.R") 2 | library(L0Learn) 3 | library(FastSparse) 4 | library(dplyr) 5 | library(reticulate) 6 | 7 | fs = function(dataset, data_type, penalty_type, lambs, gammas, X_train, y_train, 8 | X_test, y_test, LogFile, B, sim_seed, binary, fold, ell){ 9 | print(paste("train model", penalty_type, sep=" ")) 10 | n = dim(X_train)[1] 11 | p = dim(X_train)[2] 12 | if (penalty_type != "L0") { 13 | gammas = gammas 14 | } else { 15 | gammas = c(0) 16 | } 17 | for (g in gammas){ 18 | for (l in lambs){ 19 | start_time <- Sys.time() 20 | fit <- FastSparse.fit(X_train, y_train, loss=ell, algorithm="CDPSI", 21 | penalty=penalty_type, autoLambda=FALSE, lambdaGrid=list(l), nGamma=1, gammaMin=g, gammaMax=g) 22 | end_time = Sys.time() 23 | print("train finished") 24 | train_duration = difftime(end_time, start_time, units="secs") 25 | print(train_duration) 26 | for (i in 1:lengths(fit$lambda)){ 27 | lamb = fit$lambda[[1]][i] 28 | beta = as.vector(coef(fit, lambda=lamb, gamma=g)) # first element is intercept 29 | if (length(beta) != p+1){ 30 | if (data_type == "real"){ 31 | out= c(dataset, data_type, binary, fold, n, p, paste("fastsparse", ell), penalty_type, lamb, g, NA, 32 | fit$suppSize[[1]][i], NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, train_duration, "\n") 33 | } else{ 34 | out= c(dataset, data_type, binary, fold, n, p, paste("fastsparse", ell), penalty_type, lamb, g, sim_seed, 35 | fit$suppSize[[1]][i], NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, train_duration, "\n") 36 | } 37 | write_log(LogFile, out) 38 | next 39 | } else { 40 | if (ell=="Exponential"){ 41 | intercept = beta[1] 42 | b = beta[2:length(beta)] 43 | 44 | pred_train = rep(1, dim(X_train)[1]) 45 | f_train = X_train %*% b + intercept 46 | pred_train[f_train < 0] = -1 47 | prob_train = exp(2*f_train) / (1+exp(2*f_train)) 48 | 49 | 50 | pred_test = rep(1, dim(X_test)[1]) 51 | f_test = X_test %*% b + intercept 52 | pred_test[f_test < 0] = -1 53 | prob_test = exp(2*f_test) / (1+exp(2*f_test)) 54 | 55 | acc_train = 1 - (sum(y_train != pred_train)/length(y_train)) 56 | acc_test = 1 - (sum(y_test != pred_test)/length(y_test)) 57 | 58 | exp_loss_train = sum(exp(-y_train * f_train)) 59 | exp_loss_test = sum(exp(-y_test * f_test)) 60 | 61 | log_loss_train = get_logistic_loss(intercept, b, X_train, y_train) 62 | log_loss_test = get_logistic_loss(intercept, b, X_test, y_test) 63 | if (fit$penalty == "L0L2"){ 64 | penalty_term = lamb * fit$suppSize[[1]][i] + g * sum(beta[beta != 0]^2) 65 | } else if (fit$penalty == "L0L1") { 66 | penalty_term = lamb * fit$suppSize[[1]][i] + g * sum(abs(beta[beta != 0])) 67 | } else { 68 | penalty_term = lamb * fit$suppSize[[1]][i] 69 | } 70 | 71 | obj_train = log_loss_train + penalty_term 72 | obj_test = log_loss_test + penalty_term 73 | if (data_type == "real"){ 74 | out= c(dataset, data_type, binary, fold, n, p, paste("fastsparse", ell), penalty_type, lamb, g, NA, 75 | fit$suppSize[[1]][i], get_auc(y_train, prob_train), get_auc(y_test, prob_test), 76 | acc_train, acc_test, log_loss_train, log_loss_test, obj_train, obj_test, NA, NA, NA, 77 | exp_loss_train, exp_loss_test, train_duration, "\n") 78 | } else{ 79 | print("check dataset!") 80 | } 81 | write_log(LogFile, out) 82 | # cat(c(intercept, b[b!= 0]), file=paste(LogFile, paste("fastsparse", ell), fold, lamb, g, "coeff", sep="_"), append=TRUE, sep=";") 83 | # cat(which(b!=0), file=paste(LogFile, paste("fastsparse", ell), fold, lamb, g, "index", sep="_"), append=TRUE, sep=";") 84 | } else{ 85 | intercept = beta[1] 86 | b = beta[2:length(beta)] 87 | pred_train = predict(fit, newx=X_train, lambda=lamb, gamma=g) 88 | pred_test = predict(fit, newx=X_test, lambda=lamb, gamma=g) 89 | 90 | normCenteredX = get_norm_from_centeredX(X_train) 91 | 92 | if (fit$penalty == "L0L2"){ 93 | penalty_term = lamb * fit$suppSize[[1]][i] + g * sum((b[b != 0]*normCenteredX[b != 0])^2) 94 | } else if (fit$penalty == "L0L1") { 95 | penalty_term = lamb * fit$suppSize[[1]][i] + g * sum(abs(b[b != 0]*normCenteredX[b != 0])) 96 | } else { 97 | penalty_term = lamb * fit$suppSize[[1]][i] 98 | } 99 | 100 | results = get_results(beta, pred_train, pred_test, X_train, y_train, X_test, y_test, 101 | data_type, B, penalty_term, train_duration) 102 | 103 | param = c(dataset, data_type, binary, fold, n, p, paste("fastsparse", ell), penalty_type, lamb, g) 104 | out = append(param, results) 105 | write_log(LogFile, out) 106 | # cat(c(intercept, b[b!= 0]), file=paste(LogFile, paste("fastsparse", ell), fold, lamb, g, "coeff", sep="_"), append=TRUE, sep=";") 107 | # cat(which(b!=0), file=paste(LogFile, paste("fastsparse", ell), fold, lamb, g, "index", sep="_"), append=TRUE, sep=";") 108 | } 109 | 110 | } 111 | 112 | } 113 | } 114 | } 115 | } 116 | 117 | 118 | 119 | l0learn = function(dataset, data_type, penalty_type, lambs, gammas, X_train, y_train, 120 | X_test, y_test, LogFile, B, sim_seed, binary, fold){ 121 | print(paste("train model", penalty_type, sep=" ")) 122 | n = dim(X_train)[1] 123 | p = dim(X_train)[2] 124 | if (penalty_type != "L0") { 125 | gammas = gammas 126 | } else { 127 | gammas = c(0) 128 | } 129 | for (g in gammas){ 130 | for (l in lambs){ 131 | start_time <- Sys.time() 132 | fit <- L0Learn.fit(X_train, y_train, loss="Logistic", algorithm="CDPSI", 133 | penalty=penalty_type, autoLambda=FALSE, lambdaGrid=list(l), nGamma=1, gammaMin=g, gammaMax=g) 134 | end_time = Sys.time() 135 | 136 | print("train finished") 137 | train_duration = difftime(end_time, start_time, units="secs") 138 | print(train_duration) 139 | for (i in 1:lengths(fit$lambda)){ 140 | lamb = fit$lambda[[1]][i] 141 | beta = as.vector(coef(fit, lambda=lamb, gamma=g)) # first element is intercept 142 | if (length(beta) != p+1){ 143 | if (data_type == "real"){ 144 | out= c(dataset, data_type, binary, fold, n, p, "l0learn", penalty_type, lamb, g, NA, 145 | fit$suppSize[[1]][i], NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, train_duration, "\n") 146 | } else{ 147 | out= c(dataset, data_type, binary, fold, n, p, "l0learn", penalty_type, lamb, g, sim_seed, 148 | fit$suppSize[[1]][i], NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, train_duration, "\n") 149 | } 150 | write_log(LogFile, out) 151 | next 152 | } else { 153 | intercept = beta[1] 154 | b = beta[2:length(beta)] 155 | pred_train = predict(fit, newx=X_train, lambda=lamb, gamma=g) 156 | pred_test = predict(fit, newx=X_test, lambda=lamb, gamma=g) 157 | 158 | normCenteredX = get_norm_from_centeredX(X_train) 159 | 160 | if (fit$penalty == "L0L2"){ 161 | penalty_term = lamb * fit$suppSize[[1]][i] + g * sum((b[b != 0]*normCenteredX[b != 0])^2) 162 | } else if (fit$penalty == "L0L1") { 163 | penalty_term = lamb * fit$suppSize[[1]][i] + g * sum(abs(b[b != 0]*normCenteredX[b != 0])) 164 | } else { 165 | penalty_term = lamb * fit$suppSize[[1]][i] 166 | } 167 | 168 | results = get_results(beta, pred_train, pred_test, X_train, y_train, X_test, y_test, 169 | data_type, B, penalty_term, train_duration) 170 | 171 | param = c(dataset, data_type, binary, fold, n, p, "l0learn", penalty_type, lamb, g) 172 | out = append(param, results) 173 | write_log(LogFile, out) 174 | # cat(c(intercept, b[b!= 0]), file=paste(LogFile, "l0learn", fold, lamb, g, "coeff", sep="_"), append=TRUE, sep=";") 175 | # cat(which(b!=0), file=paste(LogFile, "l0learn", fold, lamb, g, "index", sep="_"), append=TRUE, sep=";") 176 | } 177 | 178 | } 179 | } 180 | 181 | } 182 | } 183 | 184 | 185 | 186 | comp_time = function(dataset, binary, data_type, LogFile, algorithm, ell=NULL, fold=NULL, 187 | penalty_type=NULL, lambs=NULL, gammas=NULL, 188 | sim_seed=NULL, sim_n=NULL, sim_p=NULL, sim_k=NULL, g=NULL){ 189 | 190 | data_info = get_dataset(dataset, binary, data_type, fold, 191 | sim_seed, sim_n, sim_p, sim_k) 192 | X_train = data_info$X_train 193 | y_train = data_info$y_train 194 | X_test = data_info$X_test 195 | y_test = data_info$y_test 196 | B = data_info$B 197 | 198 | if (algorithm == "fastsparse"){ 199 | fs(dataset, data_type, penalty_type, lambs, gammas, 200 | X_train, y_train, X_test, y_test, LogFile, B, sim_seed, binary, fold, ell) 201 | } else if (algorithm == "l0learn"){ 202 | l0learn(dataset, data_type, penalty_type, lambs, gammas, 203 | X_train, y_train, X_test, y_test, LogFile, B, sim_seed, binary, fold) 204 | } else { 205 | print("check algorithm") 206 | } 207 | } 208 | 209 | 210 | 211 | -------------------------------------------------------------------------------- /experiments/run_baselines.R: -------------------------------------------------------------------------------- 1 | source("comp_baselines.R") 2 | 3 | # Figure 5 4 | # run real dataset 5 | dataset = "fico" #"compas" 6 | data_type = "real" 7 | binary = TRUE 8 | folds = c(0, 1, 2, 3, 4) 9 | 10 | algorithms = c("fastsparse", "l0learn", "lasso", "MCP") 11 | LogFile = paste("results/baselines/", dataset, ".txt", sep="") 12 | for (fold in folds){ 13 | for (algorithm in algorithms){ 14 | print(algorithm) 15 | gammas = c(0.00001, 0.0001, 0.001, 0.01, 0.1, 1, 10) 16 | penalty_type = "L0L2" 17 | if (algorithm == "fastsparse"){ 18 | for (ell in c("Logistic", "Exponential")){ 19 | comp_mb_perf(dataset, binary, data_type, LogFile, algorithm, ell, fold, penalty_type, gammas) 20 | } 21 | } else { 22 | ell = NULL 23 | comp_mb_perf(dataset, binary, data_type, LogFile, algorithm, ell, fold, penalty_type, gammas) 24 | } 25 | } 26 | } 27 | 28 | 29 | # Figure 4 30 | # run simulated dataset 31 | dataset = "high_corr" 32 | data_type = "sim" 33 | 34 | binary = FALSE 35 | fold = NULL 36 | seeds = c(1,2,3,4,5) 37 | n = 800 38 | p = 1000 39 | k = 25 40 | 41 | algorithms = c("fastsparse", "l0learn", "lasso", "MCP") 42 | LogFile = paste("results/baselines/", dataset, ".txt", sep="") 43 | for (seed in seeds){ 44 | for (algorithm in algorithms){ 45 | print(algorithm) 46 | gammas = c(1e-9, 1e-7, 0.00001, 0.0001, 0.001, 0.01, 0.1, 1, 10) 47 | penalty_type = "L0L2" 48 | if (algorithm == "fastsparse"){ 49 | ell = "Logistic" 50 | } else { 51 | ell = NULL 52 | } 53 | comp_mb_perf(dataset, binary, data_type, LogFile, algorithm, ell, fold, penalty_type, gammas, 54 | sim_seed = seed, sim_n=n, sim_p=p, sim_k=k) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /experiments/run_time.R: -------------------------------------------------------------------------------- 1 | source("comp_time.R") 2 | 3 | # Figure 3 4 | dataset = "fico" # "compas" 5 | data_type = "real" 6 | binary = TRUE 7 | folds = c(0, 1, 2, 3, 4) 8 | algorithms = c("fastsparse", "l0learn") 9 | LogFile = paste("results/time/", dataset, ".txt", sep="") 10 | for (fold in folds){ 11 | for (algorithm in algorithms){ 12 | print(algorithm) 13 | lambs = c(0.8, 1, 2, 3, 4, 5, 6, 7) 14 | gammas = c(0.00001, 0.001) 15 | penalty_type = "L0L2" 16 | ell = "Logistic" 17 | comp_time(dataset, binary, data_type, LogFile, algorithm, ell, fold, penalty_type, lambs, gammas) 18 | if (algorithm == "fastsparse"){ 19 | ell = "Exponential" 20 | comp_time(dataset, binary, data_type, LogFile, algorithm, ell, fold, penalty_type, lambs, gammas) 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /experiments/simulation.R: -------------------------------------------------------------------------------- 1 | library(reticulate) 2 | library(MASS) 3 | 4 | dataset = "high_corr" 5 | sim_seeds = c(1,2,3,4,5) 6 | sim_p = 1000 7 | sim_n = 800 8 | sim_k = 25 9 | np = import("numpy") 10 | 11 | for (sim_seed in sim_seeds){ 12 | set.seed(sim_seed) 13 | mat = matrix(rep(c(0:(sim_p-1)), sim_p), nrow=sim_p, ncol=sim_p, byrow=TRUE) 14 | Sigma = .9 ** abs(mat - t(mat)) 15 | 16 | X = mvrnorm(n=1.2*sim_n, rep(0, sim_p), Sigma) 17 | B = c(rep(0, sim_p)) 18 | step = floor(sim_p/sim_k) 19 | for (l in 1:sim_k){ 20 | B[l*step] = 1 21 | } 22 | prob = 1/(1+exp(-1*(X %*% B))) # check prob 23 | y = rep(1, sim_n*1.2) 24 | y[prob < 0.5] = -1 25 | 26 | X_test = X[(sim_n+1):(sim_n*1.2), 1:sim_p] 27 | y_test = y[(sim_n+1):(sim_n*1.2)] 28 | test = cbind(X_test, y_test) 29 | 30 | X_train = X[1:sim_n, 1:sim_p] 31 | y_train = y[1:sim_n] 32 | train = cbind(X_train, y_train) 33 | 34 | trainfile = paste("datasets/high_corr/high_corr_train", sim_n, sim_p, sim_k, sim_seed, sep="_") 35 | trainfile = paste(trainfile, ".npy", sep="") 36 | np$save(trainfile, train) 37 | 38 | Bfile = paste("datasets/high_corr/high_corr_coef", sim_n, sim_p, sim_k, sim_seed, sep="_") 39 | np$save(paste(Bfile, ".npy", sep=""), B) 40 | 41 | testfile = paste("datasets/high_corr/high_corr_test", sim_n, sim_p, sim_k, sim_seed, sep="_") 42 | testfile = paste(testfile, ".npy", sep="") 43 | np$save(testfile, test) 44 | } -------------------------------------------------------------------------------- /experiments/utils.R: -------------------------------------------------------------------------------- 1 | library(dplyr) 2 | library(pROC) 3 | library(reticulate) 4 | library(MASS) 5 | 6 | get_auc = function(y_true, y_pred){ 7 | y_true = factor(y_true) 8 | y_pred = as.vector(y_pred) 9 | if (sum(is.nan(y_pred)) > 0) { 10 | return(NA) 11 | } else{ 12 | roccurve = roc(y_true ~ y_pred) 13 | return(as.numeric(roccurve$auc)) 14 | } 15 | } 16 | 17 | get_acc = function(y_true, y_pred){ 18 | y_pred = as.vector(y_pred) 19 | if (sum(is.nan(y_pred))>0){ 20 | return(NA) 21 | } else { 22 | y_pred[y_pred>=0.5]=1 23 | y_pred[y_pred<0.5]= -1 24 | loss = sum(y_true != y_pred)/length(y_true) 25 | return(1-loss) 26 | } 27 | } 28 | 29 | get_logistic_loss = function(intercept, b, X, y){ 30 | if (sum(is.nan(b)) > 0){ 31 | return(NA) 32 | } else { 33 | wx = X %*% b + intercept 34 | loss = sum(log(exp(wx*y * -1) + 1)) 35 | return(loss) 36 | } 37 | } 38 | 39 | get_recover_f1 = function(beta_true, beta_fit){ 40 | # beta_true and beta_fit should be vectors 41 | if (sum(is.nan(beta_fit)) > 0) { 42 | returnlist = listlist("precision" = NA, "recall"=NA, "f1"=NA) 43 | return(returnlist) 44 | } else { 45 | count_intersect = sum((beta_true != 0) & (beta_fit != 0)) 46 | p = count_intersect/sum(beta_fit != 0) 47 | r = count_intersect/sum(beta_true != 0) 48 | returnlist = list("precision" = p, "recall"=r, "f1"=2*p*r/(p+r)) 49 | return(returnlist) 50 | } 51 | } 52 | 53 | 54 | 55 | get_dataset = function(train_path, test_path){ 56 | # dataset is a string 57 | np = import("numpy") 58 | mat_train <- np$load(train_path) 59 | mat_test = np$load(test_path) 60 | 61 | p = dim(mat_train)[2] 62 | 63 | X_train = mat_train[,1:p-1] 64 | y_train = mat_train[,p] 65 | 66 | X_test = mat_test[,1:p-1] 67 | y_test = mat_test[,p] 68 | 69 | returnlist = list("X_train" = X_train, "y_train" = y_train, 70 | "X_test" = X_test, "y_test" = y_test) 71 | 72 | return(returnlist) 73 | } 74 | 75 | 76 | get_dataset = function(dataset, binary, data_type, fold=NULL, 77 | sim_seed=NULL, sim_n=NULL, sim_p=NULL, sim_k=NULL){ 78 | # dataset is a string 79 | np = import("numpy") 80 | if (data_type=="real"){ 81 | if (binary){ 82 | train_path = paste("datasets/",dataset,"/", dataset, "_bin_train", fold,".npy", sep="") 83 | test_path = paste("datasets/",dataset,"/", dataset, "_bin_test", fold,".npy", sep="") 84 | } else{ 85 | train_path = paste("datasets/", dataset, "/", dataset, "_train", fold,".npy", sep="") 86 | test_path = paste("datasets/", dataset, "/", dataset, "_test", fold,".npy", sep="") 87 | } 88 | B = NULL 89 | } else{ 90 | train_path = paste(dataset, "train", sim_n, sim_p, sim_k, sim_seed, sep="_") 91 | train_path = paste("datasets/", dataset, "/", train_path, ".npy", sep="") 92 | test_path = paste(dataset, "test", sim_n, sim_p, sim_k, sim_seed, sep="_") 93 | test_path = paste("datasets/", dataset, "/", test_path, ".npy", sep="") 94 | B = c(rep(0, sim_p)) 95 | step = floor(sim_p/sim_k) 96 | for (l in 1:sim_k){ 97 | B[l*step] = 1 98 | } 99 | } 100 | 101 | 102 | mat_train <- np$load(train_path) 103 | mat_test = np$load(test_path) 104 | 105 | p = dim(mat_train)[2] 106 | 107 | X_train = mat_train[,1:p-1] 108 | y_train = mat_train[,p] 109 | 110 | X_test = mat_test[,1:p-1] 111 | y_test = mat_test[,p] 112 | 113 | returnlist = list("X_train" = X_train, "y_train" = y_train, 114 | "X_test" = X_test, "y_test" = y_test, "B" = B) 115 | 116 | return(returnlist) 117 | } 118 | 119 | 120 | get_norm_from_centeredX = function(X){ # l0study X transform 121 | Xmean = colMeans(X) 122 | Xcentered = sweep(X, 2, Xmean) 123 | Xcentered_squared = Xcentered * Xcentered 124 | Xnorm = sqrt(colSums(Xcentered_squared)) 125 | return(Xnorm) 126 | } 127 | 128 | 129 | write_log = function(LogFile, out){ 130 | if (!file.exists(LogFile)){ 131 | colnames = c("dataset", "data_type", "binary_feature", "fold", "n", "p", 132 | "algorithm", "penalty_type", "lamb", "g", "simulation_seed", "support_size", 133 | "train_auc", "test_auc", 134 | "train_acc", "test_acc", "train_log_loss", "test_log_loss", "train_obj", "test_obj", 135 | "train_precision", "train_recall", "train_f1_score", 136 | "train_exp_loss", "test_exp_loss","train_duration", "\n") 137 | cat(colnames, file=LogFile, append=TRUE, sep=";") 138 | } 139 | cat(out, file=LogFile, append=TRUE, sep=";") 140 | } 141 | 142 | 143 | get_results = function(beta, pred_train, pred_test, X_train, y_train, X_test, y_test, 144 | data_type, B, penalty_term, train_duration, sim_seed){ 145 | intercept = beta[1] 146 | b = beta[2:length(beta)] 147 | support = sum(b!=0) 148 | train_log_loss = get_logistic_loss(intercept, b, X_train, y_train) 149 | test_log_loss = get_logistic_loss(intercept, b, X_test, y_test) 150 | train_obj = train_log_loss + penalty_term 151 | test_obj = test_log_loss + penalty_term 152 | exp_loss_train = sum(exp(-y_train * (X_train %*% b + intercept))) 153 | exp_loss_test = sum(exp(-y_test * (X_test %*% b + intercept))) 154 | 155 | results = c(support, get_auc(y_train, pred_train), get_auc(y_test, pred_test), 156 | get_acc(y_train, pred_train), get_acc(y_test, pred_test), 157 | train_log_loss, test_log_loss, train_obj, test_obj) 158 | 159 | if (data_type == "real"){ 160 | seed = NA 161 | more_results = c(NA, NA, NA, exp_loss_train, exp_loss_test, train_duration, "\n") 162 | } else{ 163 | seed = sim_seed 164 | f1return = get_recover_f1(B, b) 165 | more_results = c(f1return$precision, f1return$recall, f1return$f1, exp_loss_train, exp_loss_test, train_duration, "\n") 166 | } 167 | 168 | out = append(seed, results) 169 | out = append(out, more_results) 170 | 171 | return(out) 172 | } 173 | -------------------------------------------------------------------------------- /installation/README.md: -------------------------------------------------------------------------------- 1 | # fastSparse 2 | 3 | This repository contains source code to our AISTATS 2022 paper: 4 | 5 | * [Fast Sparse Classification for Generalized Linear and Additive Models](https://arxiv.org/abs/2202.11389) 6 | 7 | --- 8 | ## 1. Installation ## 9 | 10 | 1.1 New environment setup. We recommend using Anaconda (preferrably in a Linux system) to set up a new R environment using the the "environment.yml" file we provide. First make sure you git clone our GitHub repo and go to the fastSparse folder. Then, in your terminal, create the new environment and activate this environment: 11 | ``` 12 | git clone https://github.com/jiachangliu/fastSparse 13 | cd fastSparse 14 | conda env create -f environment.yml 15 | conda activate fastSparse_environment 16 | ``` 17 | 18 | 1.2 Relevant RcppArmadillo library installation. In terminal, type R. Then, inside R, do the following commands "install.packages("RcppArmadillo")". Quit R by typing "quit()". The three commands together in the terminal are 19 | ``` 20 | R 21 | install.packages("RcppArmadillo") 22 | quit() 23 | ``` 24 | 25 | 1.3 Install our FastSparse library by typing in the terminal: 26 | 29 | 30 | ``` 31 | R CMD build --no-build-vignettes FastSparse 32 | R CMD INSTALL FastSparse_0.1.0.tar.gz 33 | ``` 34 | 35 | 1.4 Now you can open the rstudio under this conda environment to start exploring our library by typing in the terminal 36 | ``` 37 | rstudio 38 | ``` 39 | 40 | -------------------------------------------------------------------------------- /step_function_visualization/README.md: -------------------------------------------------------------------------------- 1 | # fastSparse 2 | 3 | This page contains the demo code to plot the step functions shown in our AISTATS 2022 paper: 4 | 5 | * [Fast Sparse Classification for Generalized Linear and Additive Models](https://arxiv.org/abs/2202.11389) 6 | 7 | To plot sparse step functions, we need to do the following two steps: 8 | 9 | 10 | ## 1. Binarization Preprocessing 11 | Binarize all continuous features into {0, 1} binary features by creating thresholds. This is a preprocessing step. The binarization helper function is in [binarization_utils.py](./binarization_utils.py) 12 | 13 | An example is given in the [binarize_continuousData_demo notebook](./binarize_continuousData_demo.ipynb). 14 | 15 | ## 2. Step Function Plotting 16 | Apply fastSparse on the preprocessed binary features to produce sparse coefficients and apply the plot functions in [plot_utils.py](./plot_utils.py). 17 | 18 | An example is given in the [plot_stepFunction_demo notebook](./plot_stepFunction_demo.ipynb), which reproduces the FICO step functions shown in the paper. Or you can modify the notebook to tailor to plot step functions on your own data. 19 | 20 | -------------------------------------------------------------------------------- /step_function_visualization/binarization_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | 4 | def convert_continuous_df_to_binary_df(df): 5 | colnames = df.columns 6 | n = len(df) 7 | print("Make sure your first column corresponds to the y label") 8 | print("Converting continuous features to binary features in the dataframe......") 9 | 10 | percentile_ticks = range(1, 101) 11 | 12 | binarized_dict = {} 13 | 14 | for i in range(0, len(colnames)): 15 | uni = df[colnames[i]].unique() 16 | if len(uni) == 2: 17 | binarized_dict[colnames[i]] = np.asarray(df[colnames[i]], dtype=int) 18 | continue 19 | 20 | uni.sort() 21 | if len(uni) >= 100: 22 | uni = np.percentile(uni, percentile_ticks) 23 | for j in range(len(uni)-1): 24 | tmp_feature = np.ones(n, dtype=int) 25 | tmp_name = colnames[i] + "<=" + str(uni[j]) 26 | 27 | zero_indices = df[colnames[i]] > uni[j] 28 | tmp_feature[zero_indices] = 0 29 | 30 | binarized_dict[tmp_name] = tmp_feature 31 | 32 | 33 | binarized_df = pd.DataFrame(binarized_dict) 34 | print("Finish converting continuous features to binary features......") 35 | return binarized_df -------------------------------------------------------------------------------- /step_function_visualization/results/fico.txt_fastsparse Exponential_2_5_0.001_coeff: -------------------------------------------------------------------------------- 1 | -0.2584626;0.1825955;0.1387806;0.2286364;0.2569742;0.1840013;0.172138;0.2015039;0.1923697;0.2654667;0.2320259;0.1009372;-0.2311165;-0.7723769;0.3636577;-0.2762694;-0.1897788;-0.2742168;-0.1038025;-0.1938047 -------------------------------------------------------------------------------- /step_function_visualization/results/fico.txt_fastsparse Exponential_2_5_0.001_index: -------------------------------------------------------------------------------- 1 | 30;37;41;50;745;769;947;965;1107;1147;1222;1364;1414;1416;1446;1531;1567;1764;1766 -------------------------------------------------------------------------------- /step_function_visualization/results/fico.txt_fastsparse Logistic_2_5_0.001_coeff: -------------------------------------------------------------------------------- 1 | 2.805021;0.4071199;0.310368;0.4604512;0.5471219;0.408959;0.3283239;0.4225237;0.3396898;0.525166;0.4427697;-0.4317725;-1.576435;0.5045199;0.2874494;-3.97116;-0.3657186;-0.5681891;-0.4969551 -------------------------------------------------------------------------------- /step_function_visualization/results/fico.txt_fastsparse Logistic_2_5_0.001_index: -------------------------------------------------------------------------------- 1 | 30;37;41;50;745;769;947;965;1107;1147;1364;1414;1416;1417;1452;1531;1567;1766 -------------------------------------------------------------------------------- /step_function_visualization/save_fico_columnNames.py: -------------------------------------------------------------------------------- 1 | import csv 2 | 3 | # Define the input and output file paths 4 | input_csv_file = 'fico_bin_first_5_rows.csv' # Replace with your input CSV file path 5 | output_csv_file = 'fico_bin_columnNames.csv' # Replace with your desired output CSV file path 6 | 7 | # Read the input CSV file 8 | with open(input_csv_file, 'r', newline='') as csvfile: 9 | reader = csv.reader(csvfile) 10 | 11 | # Read the first row to get the column names 12 | column_names = next(reader) 13 | 14 | # Write the column names to the output CSV file 15 | with open(output_csv_file, 'w', newline='') as csvfile: 16 | writer = csv.writer(csvfile) 17 | 18 | # Write the column names as a single row 19 | writer.writerow(column_names) 20 | 21 | print(f'Column names from {input_csv_file} have been exported to {output_csv_file}.') 22 | --------------------------------------------------------------------------------