├── src ├── Makevars ├── helpers.h ├── Makevars.win ├── unitFunction.h ├── helpers.cpp ├── softplusUnit.cpp ├── sigmoidUnitFunction.cpp ├── normalizeWeights.cpp ├── dither.cpp ├── applyDropoutMask.cpp ├── rectifiedLinearUnit.cpp ├── exponentialLinearUnit.cpp ├── softmaxUnitFunction.cpp └── rpropagation.cpp ├── .gitignore ├── .Rbuildignore ├── darch.Rproj ├── examples ├── example.autoencoder.R ├── example.regression.R ├── examples.R ├── example.imputation.R ├── example.cg.R ├── example.caret.R ├── example.maxout.R ├── example.xor_nominal.R ├── example.iris.R ├── example.dropconnect.R ├── example.xor.R ├── example.paramsList.R └── example.mnist.R ├── man ├── getMomentum.Rd ├── contr.ltfr.Rd ├── show-DArch-method.Rd ├── resetRBM.Rd ├── addLayerField.Rd ├── addLayerField-DArch-method.Rd ├── generateDropoutMask.Rd ├── setDarchParams.Rd ├── validateDataSet-DataSet-method.Rd ├── newDArch.Rd ├── generateRBMs-DArch-method.Rd ├── getDropoutMask.Rd ├── addLayer-DArch-method.Rd ├── addLayer.Rd ├── setLogLevel.Rd ├── Net.Rd ├── validateDataSet.Rd ├── saveDArch.Rd ├── setDropoutMask-set.Rd ├── makeStartEndPoints.Rd ├── rbmUpdate.Rd ├── DataSet.Rd ├── print.DArch.Rd ├── createDataSet.Rd ├── provideMNIST.Rd ├── createDataSet-ANY-ANY-missing-missing-method.Rd ├── readMNIST.Rd ├── loadDArch.Rd ├── tanhUnit.Rd ├── createDataSet-ANY-ANY-missing-DataSet-method.Rd ├── mseError.Rd ├── rmseError.Rd ├── linearUnit.Rd ├── sigmoidUnit.Rd ├── runDArch.Rd ├── preTrainDArch-DArch-method.Rd ├── softmaxUnit.Rd ├── createDataSet-ANY-missing-formula-missing-method.Rd ├── weightDecayWeightUpdate.Rd ├── generateWeightsUniform.Rd ├── crossEntropyError.Rd ├── sigmoidUnitRbm.Rd ├── linearUnitRbm.Rd ├── runDArchDropout.Rd ├── tanhUnitRbm.Rd ├── generateWeightsNormal.Rd ├── generateWeightsGlorotUniform.Rd ├── plot.DArch.Rd ├── generateWeightsHeUniform.Rd ├── softplusUnit.Rd ├── darchTest.Rd ├── exponentialLinearUnit.Rd ├── generateWeightsGlorotNormal.Rd ├── generateWeightsHeNormal.Rd ├── preTrainDArch.Rd ├── rectifiedLinearUnit.Rd ├── DArch-class.Rd ├── RBM.Rd ├── fineTuneDArch-DArch-method.Rd ├── trainRBM.Rd ├── maxoutWeightUpdate.Rd ├── darchModelInfo.Rd ├── predict.DArch.Rd ├── minimizeAutoencoder.Rd ├── fineTuneDArch.Rd ├── maxoutUnit.Rd ├── darchBench.Rd ├── backpropagation.Rd ├── minimizeClassifier.Rd └── minimize.Rd ├── inst └── CITATION ├── R ├── autosave.R ├── momentum.R ├── net.Class.R ├── net.Getter.R ├── darch.Getter.R ├── config.R ├── saveDArch.R ├── loadDArch.R ├── darch.Setter.R ├── makeStartEndPoints.R ├── bootstrap.R ├── dropout.R ├── newDArch.R ├── rbm.Reset.R ├── RcppExports.R ├── compat.R ├── generateRBMs.R ├── log.R ├── darch.Class.R ├── rbm.Class.R ├── darch.Add.R ├── util.R ├── rbmUpdate.R ├── test.R ├── errorFunctions.R ├── runDArch.R └── caret.R ├── NAMESPACE ├── README ├── DESCRIPTION └── README.md /src/Makevars: -------------------------------------------------------------------------------- 1 | CXX_STD = CXX11 2 | PKG_LIBS += $(shell ${R_HOME}/bin/Rscript -e "RcppParallel::RcppParallelLibs()") -------------------------------------------------------------------------------- /src/helpers.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | double cppSD(RcppParallel::RMatrix::Column column); 4 | double normalizeWeightsSum(RcppParallel::RMatrix::Column column); -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .Rproj.user 2 | .Rhistory 3 | .Renviron 4 | .RData 5 | .gitignore~ 6 | *.Rout 7 | *.o 8 | *.so 9 | data/ 10 | helper/ 11 | benchmarks/ 12 | logs/ 13 | local/ 14 | tests/ 15 | 16 | -------------------------------------------------------------------------------- /src/Makevars.win: -------------------------------------------------------------------------------- 1 | CXX_STD = CXX11 2 | PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=1 3 | 4 | PKG_LIBS += $(shell "${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" \ 5 | -e "RcppParallel::RcppParallelLibs()") -------------------------------------------------------------------------------- /.Rbuildignore: -------------------------------------------------------------------------------- 1 | ^.*\.Rproj$ 2 | ^\.Rproj\.user$ 3 | ^data$ 4 | ^local$ 5 | ^benchmarks$ 6 | ^examples$ 7 | ^\.Rhistory$ 8 | ^\.Rprofile$ 9 | ^\.gitignore$ 10 | ^\.git$ 11 | ^DARCH_CANCEL_?$ 12 | README\.md 13 | ^tests$ 14 | -------------------------------------------------------------------------------- /darch.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | 3 | RestoreWorkspace: Default 4 | SaveWorkspace: Default 5 | AlwaysSaveHistory: Default 6 | 7 | EnableCodeIndexing: Yes 8 | UseSpacesForTab: Yes 9 | NumSpacesForTab: 2 10 | Encoding: UTF-8 11 | 12 | RnwWeave: knitr 13 | LaTeX: pdfLaTeX 14 | 15 | BuildType: Package 16 | PackageInstallArgs: --no-multiarch --with-keep.source 17 | PackageBuildBinaryArgs: --html 18 | PackageCheckArgs: --as-cran 19 | PackageRoxygenize: rd,collate,namespace 20 | -------------------------------------------------------------------------------- /examples/example.autoencoder.R: -------------------------------------------------------------------------------- 1 | example.autoencoder <- function(...) 2 | { 3 | data(iris) 4 | darch <- darch(iris[,1:4], iris[,1:4], c(4,100,2,100,4), darch.isClass = F, 5 | preProc.params = list(method = c("center", "scale")), preProc.targets = T, 6 | darch.numEpochs = 200, darch.batchSize = 3, 7 | darch.unitFunction = softplusUnit, bp.learnRate = .1, 8 | darch.fineTuneFunction = "backpropagation") 9 | 10 | predict(darch, newdata = iris[,1:4]) 11 | } -------------------------------------------------------------------------------- /man/getMomentum.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/net.Getter.R 3 | \name{getMomentum} 4 | \alias{getMomentum} 5 | \title{Returns the current momentum of the \code{Net}.} 6 | \usage{ 7 | getMomentum(net) 8 | } 9 | \arguments{ 10 | \item{net}{A instance of the class \code{Net}.} 11 | } 12 | \description{ 13 | Returns the current momentum of the \code{Net}. 14 | } 15 | \seealso{ 16 | \code{\linkS4class{Net}} 17 | } 18 | \keyword{internal} 19 | 20 | -------------------------------------------------------------------------------- /man/contr.ltfr.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/caret.R 3 | \name{contr.ltfr} 4 | \alias{contr.ltfr} 5 | \title{Wrapper for \code{\link[caret]{contr.ltfr}}} 6 | \usage{ 7 | contr.ltfr(...) 8 | } 9 | \arguments{ 10 | \item{...}{\code{\link[caret]{contr.ltfr}} parameters.} 11 | } 12 | \description{ 13 | Simply redirects the call to \code{\link[caret]{contr.ltfr}}, this is done 14 | to avoid errors when the caret namespace is not attached. 15 | } 16 | \keyword{internal} 17 | 18 | -------------------------------------------------------------------------------- /inst/CITATION: -------------------------------------------------------------------------------- 1 | citHeader("To cite darch in publications use:") 2 | 3 | citEntry(entry = "MastersThesis", 4 | title = "Implementierung und Analyse von tiefen Architekturen in R", 5 | author = personList(as.person("Martin Drees")), 6 | year = "2013", 7 | school = "Fachhochschule Dortmund", 8 | 9 | textVersion = 10 | paste("Drees, Martin (2013). Implementierung und", 11 | "Analyse von tiefen Architekturen in R. German.", 12 | "Master's thesis. Fachhochschule Dortmund.") 13 | ) 14 | -------------------------------------------------------------------------------- /man/show-DArch-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/print.R 3 | \docType{methods} 4 | \name{show,DArch-method} 5 | \alias{show,DArch-method} 6 | \title{Print \code{\linkS4class{DArch}} details.} 7 | \usage{ 8 | \S4method{show}{DArch}(object) 9 | } 10 | \arguments{ 11 | \item{object}{\code{\linkS4class{DArch}} instance} 12 | 13 | \item{...}{Further parameters to \code{\link{print.DArch}}} 14 | } 15 | \description{ 16 | Simple redirection to \code{\link{print.DArch}}. 17 | } 18 | \keyword{internal} 19 | 20 | -------------------------------------------------------------------------------- /man/resetRBM.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/rbm.Reset.R 3 | \name{resetRBM} 4 | \alias{resetRBM} 5 | \title{Resets the weights and biases of the \code{RBM} object} 6 | \usage{ 7 | resetRBM(rbm, ...) 8 | } 9 | \arguments{ 10 | \item{rbm}{An instance of class \code{\linkS4class{RBM}}.} 11 | 12 | \item{...}{Additional arguments.} 13 | } 14 | \description{ 15 | This function resets the weights and biases of the \code{\linkS4class{RBM}} 16 | object. 17 | } 18 | \seealso{ 19 | \code{\linkS4class{RBM}} 20 | } 21 | \keyword{internal} 22 | 23 | -------------------------------------------------------------------------------- /man/addLayerField.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darch.Add.R 3 | \name{addLayerField} 4 | \alias{addLayerField} 5 | \title{Adds a field to a layer} 6 | \usage{ 7 | addLayerField(darch, index, field) 8 | } 9 | \arguments{ 10 | \item{darch}{An instance of the class \code{\linkS4class{DArch}}.} 11 | 12 | \item{index}{The position of the layer.} 13 | 14 | \item{field}{The new field for the layer.} 15 | } 16 | \description{ 17 | Adds a field to the layer given by the index. 18 | } 19 | \seealso{ 20 | \code{\linkS4class{DArch}} 21 | } 22 | \keyword{internal} 23 | 24 | -------------------------------------------------------------------------------- /examples/example.regression.R: -------------------------------------------------------------------------------- 1 | example.regression <- function(...) 2 | { 3 | library("MASS") 4 | library(caret) 5 | data(cats) 6 | pp <- preProcess(cats, method=c("scale")) 7 | catsScaled <- predict(pp, newdata = cats) 8 | 9 | darch <- darch(Hwt ~ Bwt, 10 | cats, 11 | preProc.params = list(method = c("center", "scale")), 12 | preProc.targets = T, 13 | layers = c(1,20,50,20,1), 14 | darch.batchSize = 10, 15 | bp.learnRate = .01, 16 | darch.isClass = F, 17 | darch.numEpochs = 100, 18 | darch.unitFunction = linearUnit, 19 | ...) 20 | 21 | print(darchTest(darch, newdata = cats)) 22 | 23 | darch 24 | } 25 | -------------------------------------------------------------------------------- /man/addLayerField-DArch-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darch.Add.R 3 | \docType{methods} 4 | \name{addLayerField,DArch-method} 5 | \alias{addLayerField,DArch-method} 6 | \title{Adds a field to a layer} 7 | \usage{ 8 | \S4method{addLayerField}{DArch}(darch, index, field) 9 | } 10 | \arguments{ 11 | \item{darch}{An instance of the class \code{\linkS4class{DArch}}.} 12 | 13 | \item{index}{The position of the layer.} 14 | 15 | \item{field}{The new field for the layer.} 16 | } 17 | \description{ 18 | Adds a field to a layer 19 | } 20 | \seealso{ 21 | \code{\link{addLayerField}} 22 | } 23 | \keyword{internal} 24 | 25 | -------------------------------------------------------------------------------- /man/generateDropoutMask.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/dropout.R 3 | \name{generateDropoutMask} 4 | \alias{generateDropoutMask} 5 | \title{Dropout mask generator function.} 6 | \usage{ 7 | generateDropoutMask(length, dropoutRate) 8 | } 9 | \arguments{ 10 | \item{length}{length of the dropout mask} 11 | 12 | \item{dropoutRate}{The dropout rate (i.e. percentage of zeros in the mask)} 13 | } 14 | \value{ 15 | Matrix containing the dropout mask 16 | } 17 | \description{ 18 | This function generates droput masks as a vector of given length, according 19 | to the given dropout rate. 20 | } 21 | \keyword{internal} 22 | 23 | -------------------------------------------------------------------------------- /man/setDarchParams.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/params.R 3 | \name{setDarchParams} 4 | \alias{setDarchParams} 5 | \title{Set \code{\linkS4class{DArch}} parameters} 6 | \usage{ 7 | setDarchParams(darch, ...) 8 | } 9 | \arguments{ 10 | \item{darch}{\code{\linkS4class{DArch}} instance.} 11 | 12 | \item{...}{Parameters to be set, see \code{\link{darch.default}}.} 13 | } 14 | \description{ 15 | Allows setting \code{\linkS4class{DArch}} parameters normally passed to the 16 | \code{\link{darch}} interface function when not using said interface. 17 | These parameters can also be passed to \code{\link{newDArch}}. 18 | } 19 | \keyword{internal} 20 | 21 | -------------------------------------------------------------------------------- /man/validateDataSet-DataSet-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/dataset.R 3 | \docType{methods} 4 | \name{validateDataSet,DataSet-method} 5 | \alias{validateDataSet,DataSet-method} 6 | \title{Validate \code{\linkS4class{DataSet}}} 7 | \usage{ 8 | \S4method{validateDataSet}{DataSet}(dataSet, darch) 9 | } 10 | \arguments{ 11 | \item{dataSet}{\code{\linkS4class{DataSet}} to validate} 12 | 13 | \item{darch}{\code{\linkS4class{DArch}} object to validate this 14 | \code{\linkS4class{DataSet}} against.} 15 | } 16 | \description{ 17 | Validate \code{\linkS4class{DataSet}} 18 | } 19 | \seealso{ 20 | \link{validateDataSet} 21 | } 22 | \keyword{internal} 23 | 24 | -------------------------------------------------------------------------------- /man/newDArch.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/newDArch.R 3 | \name{newDArch} 4 | \alias{newDArch} 5 | \title{Constructor function for \code{\linkS4class{DArch}} objects.} 6 | \usage{ 7 | newDArch(params) 8 | } 9 | \arguments{ 10 | \item{params}{Additional parameters as a list, also stored in 11 | \code{darch@parameters}.} 12 | } 13 | \value{ 14 | The new DArch object 15 | } 16 | \description{ 17 | Generate a new \code{\linkS4class{DArch}} object with the given parameters. 18 | } 19 | \details{ 20 | This function is used internally only, please call \code{\link{darch}} 21 | to create a new \code{\linkS4class{DArch}} instance. 22 | } 23 | \keyword{internal} 24 | 25 | -------------------------------------------------------------------------------- /src/unitFunction.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | using namespace RcppParallel; 5 | using namespace Rcpp; 6 | 7 | #ifndef UNIT_FUNCTION_H 8 | #define UNIT_FUNCTION_H 9 | 10 | struct UnitFunction : public Worker 11 | { 12 | const RMatrix input; 13 | 14 | int ncolInput; 15 | int nrowInput; 16 | 17 | RMatrix activations; 18 | RMatrix derivatives; 19 | 20 | UnitFunction(const NumericMatrix input, NumericMatrix activations, 21 | NumericMatrix derivatives) : input(input), activations(activations), 22 | derivatives(derivatives) 23 | { 24 | nrowInput = input.nrow(); 25 | ncolInput = input.ncol(); 26 | } 27 | }; 28 | 29 | #endif -------------------------------------------------------------------------------- /examples/examples.R: -------------------------------------------------------------------------------- 1 | \dontrun{ 2 | data(iris) 3 | model <- darch(Species ~ ., iris) 4 | print(model) 5 | predictions <- predict(model, newdata = iris, type = "class") 6 | cat(paste("Incorrect classifications:", sum(predictions != iris[,5]))) 7 | 8 | trainData <- matrix(c(0,0,0,1,1,0,1,1), ncol = 2, byrow = TRUE) 9 | trainTargets <- matrix(c(0,1,1,0), nrow = 4) 10 | model2 <- darch(trainData, trainTargets, layers = c(2, 10, 1), 11 | darch.numEpochs = 500, darch.stopClassErr = 0, retainData = T) 12 | e <- darchTest(model2) 13 | cat(paste0("Incorrect classifications on all examples: ", e[3], " (", 14 | e[2], "%)\n")) 15 | 16 | plot(model2) 17 | } 18 | 19 | # 20 | # More examples can be found at 21 | # https://github.com/maddin79/darch/tree/v0.12.0/examples 22 | -------------------------------------------------------------------------------- /examples/example.imputation.R: -------------------------------------------------------------------------------- 1 | # PIMA2 example using caret imputation, requires packages MASS and RANN 2 | example.imputation <- function(...) 3 | { 4 | library(mlbench) 5 | library(RANN) 6 | data("PimaIndiansDiabetes2") 7 | 8 | darch <- darch(diabetes ~ ., PimaIndiansDiabetes2, 9 | preProc.params = list("method" = c("knnImpute")), 10 | layers = c(0,100,50,0), 11 | darch.batchSize = 1, 12 | darch.returnBestModel.validationErrorFactor = 1, 13 | darch.fineTuneFunction = "backpropagation", 14 | darch.unitFunction = c("tanhUnit", "tanhUnit", "softmaxUnit"), 15 | darch.numEpochs = 10, 16 | bootstrap = T, 17 | bootstrap.num = 507, 18 | ... 19 | ) 20 | 21 | darchTest(darch, newdata = PimaIndiansDiabetes2) 22 | 23 | darch 24 | } 25 | -------------------------------------------------------------------------------- /man/generateRBMs-DArch-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/generateRBMs.R 3 | \docType{methods} 4 | \name{generateRBMs,DArch-method} 5 | \alias{generateRBMs,DArch-method} 6 | \title{Generates the RBMs for the pre-training.} 7 | \usage{ 8 | \S4method{generateRBMs}{DArch}(darch) 9 | } 10 | \arguments{ 11 | \item{darch}{An instance of the class \code{DArch} for which RBMs are 12 | to be generated.} 13 | } 14 | \value{ 15 | The DArch object with the generated RBMs 16 | } 17 | \description{ 18 | Used the layer sizes from the \code{\linkS4class{DArch}} object to create the 19 | RBM objects for the pre-training. 20 | } 21 | \seealso{ 22 | \code{\linkS4class{DArch}} 23 | \code{\linkS4class{RBM}} 24 | } 25 | \keyword{internal} 26 | 27 | -------------------------------------------------------------------------------- /man/getDropoutMask.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darch.Getter.R 3 | \name{getDropoutMask} 4 | \alias{getDropoutMask} 5 | \title{Returns the dropout mask for the given layer} 6 | \usage{ 7 | getDropoutMask(darch, i) 8 | } 9 | \arguments{ 10 | \item{darch}{A instance of the class \code{\linkS4class{DArch}}.} 11 | 12 | \item{i}{Layer index or 0 for input dropout mask.} 13 | } 14 | \value{ 15 | Dropout mask for the given layer. 16 | } 17 | \description{ 18 | The dropout mask is applied to the weights between layer i and i+1, for 0 < i 19 | < numLayers. For i = 0, the dropout mask for the input layer is returned, 20 | which will be applied to the initial input data. 21 | } 22 | \seealso{ 23 | \code{\linkS4class{DArch}} 24 | } 25 | \keyword{internal} 26 | 27 | -------------------------------------------------------------------------------- /man/addLayer-DArch-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darch.Add.R 3 | \docType{methods} 4 | \name{addLayer,DArch-method} 5 | \alias{addLayer,DArch-method} 6 | \title{Adds a layer to the \code{\linkS4class{DArch}} object} 7 | \usage{ 8 | \S4method{addLayer}{DArch}(darch, weights, biases, unitFunction, 9 | weightUpdateFunction) 10 | } 11 | \arguments{ 12 | \item{darch}{An instance of the class \code{\linkS4class{DArch}}.} 13 | 14 | \item{weights}{The weights for the layer.} 15 | 16 | \item{biases}{The biases for the layer.} 17 | 18 | \item{unitFunction}{The functions of the units in the layer.} 19 | } 20 | \description{ 21 | Adds a layer to the \code{\linkS4class{DArch}} object 22 | } 23 | \seealso{ 24 | \code{\link{addLayer}} 25 | } 26 | \keyword{internal} 27 | 28 | -------------------------------------------------------------------------------- /man/addLayer.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darch.Add.R 3 | \name{addLayer} 4 | \alias{addLayer} 5 | \title{Adds a layer to the \code{\linkS4class{DArch}} object} 6 | \usage{ 7 | addLayer(darch, weights, biases, unitFunction, weightUpdateFunction) 8 | } 9 | \arguments{ 10 | \item{darch}{An instance of the class \code{\linkS4class{DArch}}.} 11 | 12 | \item{weights}{The weights for the layer.} 13 | 14 | \item{biases}{The biases for the layer.} 15 | 16 | \item{unitFunction}{The functions of the units in the layer.} 17 | } 18 | \description{ 19 | Adds a layer to the given \code{\linkS4class{DArch}} object. The parameter 20 | weights and biases will be put together in one matrix. 21 | } 22 | \seealso{ 23 | \code{\linkS4class{DArch}} 24 | } 25 | \keyword{internal} 26 | 27 | -------------------------------------------------------------------------------- /man/setLogLevel.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/log.R 3 | \name{setLogLevel} 4 | \alias{setLogLevel} 5 | \title{Set the log level.} 6 | \usage{ 7 | setLogLevel(value) 8 | } 9 | \arguments{ 10 | \item{value}{Log level, must be one of the \code{futile.logger} constants.} 11 | } 12 | \description{ 13 | Convenience wrapper for \code{\link[futile.logger]{flog.threshold}} with 14 | sanity checking and current level output. 15 | } 16 | \details{ 17 | The log levels are defined by the \code{\link{futile.logger}} package. 18 | The following levels a available: 19 | \tabular{ll}{ 20 | futile.logger::TRACE\cr 21 | futile.logger::DEBUG\cr 22 | futile.logger::INFO\cr 23 | futile.logger::WARN\cr 24 | futile.logger::ERROR\cr 25 | futile.logger::FATAL 26 | } 27 | } 28 | \keyword{internal} 29 | 30 | -------------------------------------------------------------------------------- /man/Net.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/net.Class.R 3 | \docType{class} 4 | \name{Net-class} 5 | \alias{Net-class} 6 | \title{Abstract class for neural networks.} 7 | \description{ 8 | This is an abstract class for neural networks. It provides some 9 | functionalities used in more than one network type. 10 | } 11 | \section{Slots}{ 12 | 13 | \describe{ 14 | \item{\code{epochs}}{Number of epochs the network has been trained for.} 15 | 16 | \item{\code{stats}}{Training statistics.} 17 | 18 | \item{\code{parameters}}{List of parameters which do not change throughout training.} 19 | }} 20 | \author{ 21 | Martin Drees 22 | } 23 | \seealso{ 24 | Other darch classes: \code{\link{DArch-class}}, 25 | \code{\link{DataSet-class}}, \code{\link{RBM-class}} 26 | } 27 | \keyword{internal} 28 | 29 | -------------------------------------------------------------------------------- /man/validateDataSet.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/dataset.R 3 | \name{validateDataSet} 4 | \alias{validateDataSet} 5 | \title{Validate \code{\linkS4class{DataSet}}} 6 | \usage{ 7 | validateDataSet(dataSet, darch) 8 | } 9 | \arguments{ 10 | \item{dataSet}{\code{\linkS4class{DataSet}} to validate} 11 | 12 | \item{darch}{\code{\linkS4class{DArch}} object to validate this 13 | \code{\linkS4class{DataSet}} against.} 14 | } 15 | \value{ 16 | Logical indicating whether the \code{\linkS4class{DataSet}} is valid. 17 | } 18 | \description{ 19 | Validates the \code{\linkS4class{DataSet}} for the given 20 | \code{\linkS4class{DArch}} object. 21 | } 22 | \details{ 23 | Validates the data dimensions and data types of the 24 | \code{\linkS4class{DataSet}}. 25 | } 26 | \keyword{internal} 27 | 28 | -------------------------------------------------------------------------------- /man/saveDArch.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/saveDArch.R 3 | \name{saveDArch} 4 | \alias{saveDArch} 5 | \title{Saves a DArch network} 6 | \usage{ 7 | saveDArch(darch, name = "darch", trim = F) 8 | } 9 | \arguments{ 10 | \item{darch}{An instance of the class \code{\linkS4class{DArch}}.} 11 | 12 | \item{name}{The name for the file. Default value is "darch".} 13 | 14 | \item{trim}{Whether to trim the model before saving.} 15 | } 16 | \description{ 17 | Saves the DArch object to the filename given through the parameter \code{name} 18 | plus the ending ".net". 19 | } 20 | \details{ 21 | If the parameter \code{saveRBM} is \code{FALSE} the field 22 | \code{rbmList} of the DArch object is overwritten by an empty list. 23 | } 24 | \seealso{ 25 | \code{\link{loadDArch}} 26 | } 27 | \keyword{internal} 28 | 29 | -------------------------------------------------------------------------------- /man/setDropoutMask-set.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darch.Setter.R 3 | \name{setDropoutMask<-} 4 | \alias{setDropoutMask<-} 5 | \title{Set the dropout mask for the given layer.} 6 | \usage{ 7 | setDropoutMask(darch, i) <- value 8 | } 9 | \arguments{ 10 | \item{darch}{A instance of the class \code{\linkS4class{DArch}}.} 11 | 12 | \item{i}{Layer index or 0 for input layer.} 13 | 14 | \item{value}{Dropout mask for the given layer.} 15 | } 16 | \value{ 17 | The darch with the updated dropout mask 18 | } 19 | \description{ 20 | Sets the dropout mask to be applied to the weights between layer i and i+1, 21 | for 0 < i < numLayers. For i = 0, sets the dropout mask for the input layer, 22 | which will be applied to the initiali input data. 23 | } 24 | \seealso{ 25 | \code{\linkS4class{DArch}} 26 | } 27 | \keyword{internal} 28 | 29 | -------------------------------------------------------------------------------- /man/makeStartEndPoints.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/makeStartEndPoints.R 3 | \name{makeStartEndPoints} 4 | \alias{makeStartEndPoints} 5 | \title{Makes start- and end-points for the batches.} 6 | \usage{ 7 | makeStartEndPoints(batchSize, numRows) 8 | } 9 | \arguments{ 10 | \item{batchSize}{Desired batch size} 11 | 12 | \item{numRows}{Number of rows of the data} 13 | } 14 | \description{ 15 | The start- and end-points are used for dividing the data into batches. 16 | } 17 | \details{ 18 | If the data is not divisible by the \code{batchSize} the last batch 19 | will contain the rest of the data. 20 | The function returns a list with in which the first entry is a list with the 21 | values for the start and end points for reading the data matrix. The second 22 | entry is the number of batches. 23 | } 24 | \keyword{internal} 25 | 26 | -------------------------------------------------------------------------------- /man/rbmUpdate.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/rbmUpdate.R 3 | \name{rbmUpdate} 4 | \alias{rbmUpdate} 5 | \title{Function for updating the weights and biases of an \code{RBM}} 6 | \usage{ 7 | rbmUpdate(rbm, matMult = getParameter(".matMult", net = rbm)) 8 | } 9 | \arguments{ 10 | \item{rbm}{A instance of the class \code{\linkS4class{RBM}}.} 11 | 12 | \item{matMult}{Matrix multiplication function.} 13 | 14 | \item{...}{Additional arguments.} 15 | } 16 | \value{ 17 | The updated \code{\linkS4class{RBM}}. 18 | } 19 | \description{ 20 | This function updates the weights and biases for an \code{\linkS4class{RBM}} 21 | network. It is saved in the attribute \code{updateFunction} of the 22 | \code{RBM} object and called from the training function 23 | \code{\link{trainRBM}}. 24 | } 25 | \seealso{ 26 | \code{\linkS4class{RBM}} 27 | } 28 | \keyword{internal} 29 | 30 | -------------------------------------------------------------------------------- /man/DataSet.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/dataset.R 3 | \docType{class} 4 | \name{DataSet-class} 5 | \alias{DataSet-class} 6 | \title{Class for specifying datasets.} 7 | \description{ 8 | Class for specifying datasets. 9 | } 10 | \details{ 11 | This class deals with data (input) and targets (output) for neural 12 | networks, including conversion of ordinal and nominal data, validation of 13 | data, and application of model formulae to the data. 14 | } 15 | \section{Slots}{ 16 | 17 | \describe{ 18 | \item{\code{data}}{Input data.} 19 | 20 | \item{\code{targets}}{Target output.} 21 | 22 | \item{\code{formula}}{\code{\link{formula}} for the data.} 23 | 24 | \item{\code{parameters}}{Fit parameters.} 25 | }} 26 | \seealso{ 27 | Other darch classes: \code{\link{DArch-class}}, 28 | \code{\link{Net-class}}, \code{\link{RBM-class}} 29 | } 30 | \keyword{internal} 31 | 32 | -------------------------------------------------------------------------------- /man/print.DArch.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/print.R 3 | \name{print.DArch} 4 | \alias{print.DArch} 5 | \title{Print \code{\linkS4class{DArch}} details.} 6 | \usage{ 7 | \method{print}{DArch}(x, ...) 8 | } 9 | \arguments{ 10 | \item{x}{\code{\linkS4class{DArch}} instance} 11 | 12 | \item{...}{Further parameters, not used.} 13 | } 14 | \description{ 15 | Print verbose information about a \linkS4class{DArch} instance. 16 | } 17 | \details{ 18 | Information printed include \code{\link{darch}} parameters and a summary of 19 | training statistics. 20 | } 21 | \examples{ 22 | \dontrun{ 23 | data(iris) 24 | model <- darch(Species ~ ., iris) 25 | print(model) 26 | } 27 | } 28 | \seealso{ 29 | Other darch interface functions: \code{\link{darchBench}}, 30 | \code{\link{darchTest}}, \code{\link{darch}}, 31 | \code{\link{plot.DArch}}, \code{\link{predict.DArch}} 32 | } 33 | 34 | -------------------------------------------------------------------------------- /examples/example.cg.R: -------------------------------------------------------------------------------- 1 | # Example using conjugate gradients (minimizeClassifier) 2 | example.cg <- function(...) 3 | { 4 | data(iris) 5 | 6 | darch <- darch(Species ~ ., iris, 7 | layers = c(0,20,0), 8 | preProc.params = list("method" = c("scale", "center")), 9 | darch.batchSize = 6, 10 | darch.fineTuneFunction = minimizeClassifier, 11 | darch.unitFunction = c(linearUnit, softmaxUnit), 12 | darch.weightDecay = .001, 13 | darch.numEpochs = 20, 14 | cg.length = 3, 15 | cg.switchLayers = 2, 16 | generateWeightsFunction = generateWeightsHeNormal, 17 | retainData = T, 18 | ... 19 | ) 20 | 21 | # Since retainData is TRUE, when no new data is passed, predict() and 22 | # darchTest() will use the data stored in the DArch instance 23 | e <- darchTest(darch) 24 | cat(paste0("Incorrect classifications on all examples: ", e[3], " (", 25 | e[2], "%)\n")) 26 | 27 | darch 28 | } -------------------------------------------------------------------------------- /man/createDataSet.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/dataset.R 3 | \name{createDataSet} 4 | \alias{createDataSet} 5 | \title{Create data set using data, targets, a formula, and possibly an existing data 6 | set.} 7 | \usage{ 8 | createDataSet(data, targets, formula, dataSet, ...) 9 | } 10 | \arguments{ 11 | \item{data}{Input data, possibly also target data if a formula is used.} 12 | 13 | \item{targets}{Target data.} 14 | 15 | \item{formula}{Model \code{\link{formula}}.} 16 | 17 | \item{dataSet}{\code{\linkS4class{DataSet}} to be used as the basis for the 18 | new one} 19 | 20 | \item{...}{Further parameters.} 21 | } 22 | \value{ 23 | New \code{\linkS4class{DataSet}} 24 | } 25 | \description{ 26 | Based partly on code from nnet. 27 | } 28 | \seealso{ 29 | \link{createDataSet.default}, \link{createDataSet.formula}, 30 | \link{createDataSet.DataSet} 31 | } 32 | \keyword{internal} 33 | 34 | -------------------------------------------------------------------------------- /man/provideMNIST.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/mnist.R 3 | \name{provideMNIST} 4 | \alias{provideMNIST} 5 | \title{Provides MNIST data set in the given folder.} 6 | \usage{ 7 | provideMNIST(folder = "data/", download = F) 8 | } 9 | \arguments{ 10 | \item{folder}{Folder name, including a trailing slash.} 11 | 12 | \item{download}{Logical indicating whether download is allowed.} 13 | } 14 | \value{ 15 | Boolean value indicating success or failure. 16 | } 17 | \description{ 18 | This function will, if necessary and allowed, download the compressed MNIST 19 | data set and save it to .RData files using \code{\link{readMNIST}}. If the 20 | compressed MNIST archives are available, it will convert them into RData 21 | files loadable from within R. If the RData files are already available, 22 | nothing will be done. 23 | } 24 | \examples{ 25 | \dontrun{ 26 | provideMNIST("mnist/", download = T) 27 | } 28 | } 29 | 30 | -------------------------------------------------------------------------------- /man/createDataSet-ANY-ANY-missing-missing-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/dataset.R 3 | \docType{methods} 4 | \name{createDataSet,ANY,ANY,missing,missing-method} 5 | \alias{createDataSet,ANY,ANY,missing,missing-method} 6 | \alias{createDataSet.default} 7 | \title{Create \code{\linkS4class{DataSet}} using data and targets.} 8 | \usage{ 9 | \S4method{createDataSet}{ANY,ANY,missing,missing}(data, targets, formula, 10 | dataSet, ...) 11 | } 12 | \arguments{ 13 | \item{data}{Input data, possibly also target data if a formula is used.} 14 | 15 | \item{targets}{Target data.} 16 | 17 | \item{formula}{Model \code{\link{formula}}.} 18 | 19 | \item{dataSet}{\code{\linkS4class{DataSet}} to be used as the basis for the 20 | new one} 21 | 22 | \item{...}{Further parameters.} 23 | } 24 | \description{ 25 | Create \code{\linkS4class{DataSet}} using data and targets. 26 | } 27 | \seealso{ 28 | \link{createDataSet} 29 | } 30 | \keyword{internal} 31 | 32 | -------------------------------------------------------------------------------- /man/readMNIST.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/mnist.R 3 | \name{readMNIST} 4 | \alias{readMNIST} 5 | \title{Function for generating .RData files of the MNIST Database} 6 | \usage{ 7 | readMNIST(folder) 8 | } 9 | \arguments{ 10 | \item{folder}{The location of the MNIST-Database files.} 11 | } 12 | \description{ 13 | This function reads the MNIST-Database, randomized it and saves it in the 14 | files "train" for the training data and "test" for test data. 15 | } 16 | \details{ 17 | When the data is read the variables for the training data is \code{trainData} 18 | and \code{trainLabels} and for the test data \code{testData} and 19 | \code{testLabels}. To start the function 20 | the files "train-images-idx3-ubyte", "train-labels-idx1-ubyte', 21 | "t10k-images-idx3-ubyte", and "t10k-labels-idx1-ubyte" have to be in the 22 | folder given by the parameter \code{folder}. The folder name must end with 23 | a slash. 24 | } 25 | \keyword{internal} 26 | 27 | -------------------------------------------------------------------------------- /man/loadDArch.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/loadDArch.R 3 | \docType{methods} 4 | \name{loadDArch} 5 | \alias{loadDArch} 6 | \title{Loads a DArch network} 7 | \usage{ 8 | loadDArch(name = "darch") 9 | } 10 | \arguments{ 11 | \item{name}{The name of the file without the ending ".net".} 12 | } 13 | \value{ 14 | \code{DArch} instance: the loaded deep architecture 15 | } 16 | \description{ 17 | Convenience function for loading \code{\linkS4class{DArch}} instances from 18 | disk. There is no special re-initialization logic, so it is not necessary to 19 | use this function, you can also just use \code{\link{load}}. 20 | } 21 | \details{ 22 | Loads the DArch object from the filename given through the parameter \code{name} 23 | plus the ending ".net". 24 | 25 | Make sure when you load a DArch object that every file written by 26 | the \code{\link{saveDArch}} function is in the working directory. 27 | } 28 | \seealso{ 29 | \code{\link{saveDArch}} 30 | } 31 | \keyword{internal} 32 | 33 | -------------------------------------------------------------------------------- /man/tanhUnit.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darchUnitFunctions.R 3 | \name{tanhUnit} 4 | \alias{tanhUnit} 5 | \title{Continuous Tan-Sigmoid unit function.} 6 | \usage{ 7 | tanhUnit(input, ...) 8 | } 9 | \arguments{ 10 | \item{input}{Input for the activation function.} 11 | 12 | \item{...}{Additional parameters, not used.} 13 | } 14 | \value{ 15 | A list with the activation in the first entry and the derivative of 16 | the transfer function in the second entry. 17 | } 18 | \description{ 19 | Calculates the unit activations and returns them in a list. 20 | } 21 | \examples{ 22 | \dontrun{ 23 | data(iris) 24 | model <- darch(Species ~ ., iris, darch.unitFunction = "tanhUnit") 25 | } 26 | } 27 | \seealso{ 28 | Other darch unit functions: \code{\link{exponentialLinearUnit}}, 29 | \code{\link{linearUnit}}, \code{\link{maxoutUnit}}, 30 | \code{\link{rectifiedLinearUnit}}, 31 | \code{\link{sigmoidUnit}}, \code{\link{softmaxUnit}}, 32 | \code{\link{softplusUnit}} 33 | } 34 | 35 | -------------------------------------------------------------------------------- /man/createDataSet-ANY-ANY-missing-DataSet-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/dataset.R 3 | \docType{methods} 4 | \name{createDataSet,ANY,ANY,missing,DataSet-method} 5 | \alias{createDataSet,ANY,ANY,missing,DataSet-method} 6 | \alias{createDataSet.DataSet} 7 | \title{Create new \code{\linkS4class{DataSet}} by filling an existing one with new 8 | data.} 9 | \usage{ 10 | \S4method{createDataSet}{ANY,ANY,missing,DataSet}(data, targets, formula, 11 | dataSet, ...) 12 | } 13 | \arguments{ 14 | \item{data}{Input data, possibly also target data if a formula is used.} 15 | 16 | \item{targets}{Target data.} 17 | 18 | \item{formula}{Model \code{\link{formula}}.} 19 | 20 | \item{dataSet}{\code{\linkS4class{DataSet}} that is used as a basis for the 21 | new one} 22 | 23 | \item{...}{Further parameters.} 24 | } 25 | \description{ 26 | Create new \code{\linkS4class{DataSet}} by filling an existing one with new 27 | data. 28 | } 29 | \seealso{ 30 | \link{createDataSet} 31 | } 32 | \keyword{internal} 33 | 34 | -------------------------------------------------------------------------------- /man/mseError.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/errorFunctions.R 3 | \name{mseError} 4 | \alias{mseError} 5 | \title{Mean squared error function} 6 | \usage{ 7 | mseError(original, estimate) 8 | } 9 | \arguments{ 10 | \item{original}{The original data matrix.} 11 | 12 | \item{estimate}{The calculated data matrix.} 13 | } 14 | \value{ 15 | A list with the name of the error function in the first entry and the 16 | error value in the second entry. 17 | } 18 | \description{ 19 | The function calculates the mean squared error (MSE) from the \code{original} 20 | and \code{estimate} parameters. 21 | } 22 | \details{ 23 | This function is a valid value for both \code{\link{darch}} parameters 24 | \code{rbm.errorFunction} and \code{darch.errorFunction}. 25 | } 26 | \examples{ 27 | \dontrun{ 28 | data(iris) 29 | model <- darch(Species ~ ., iris, rbm.errorFunction = "mseError", 30 | darch.errorFunction = "mseError") 31 | } 32 | } 33 | \seealso{ 34 | Other error functions: \code{\link{crossEntropyError}}, 35 | \code{\link{rmseError}} 36 | } 37 | 38 | -------------------------------------------------------------------------------- /R/autosave.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Johannes Rueckert 2 | # 3 | # This file is part of darch. 4 | # 5 | # darch is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # darch is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with darch. If not, see . 17 | 18 | autosave <- function(darch, dir, trim, digits) 19 | { 20 | # Make sure the directory exists 21 | prepareBenchmarkDirectory(dir, save = T, continue = T) 22 | 23 | fileName <- fileName <- paste0(dir, "/autosave_", 24 | formatC(darch@epochs, width = digits, flag = "0")) 25 | 26 | saveDArch(darch, fileName, trim) 27 | } -------------------------------------------------------------------------------- /src/helpers.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | using namespace Rcpp; 5 | using namespace RcppParallel; 6 | 7 | double cppSD(RMatrix::Column column) 8 | { 9 | std::vector inVec(column.begin(), column.end()); 10 | int n = inVec.size(); 11 | double sum = std::accumulate(inVec.begin(), inVec.end(), 0.0); 12 | double mean = sum / inVec.size(); 13 | 14 | for(std::vector::iterator iter = inVec.begin(); 15 | iter != inVec.end(); 16 | ++iter) 17 | { 18 | double temp = (*iter - mean)*(*iter - mean); 19 | *iter = temp; 20 | } 21 | 22 | double sd = std::accumulate(inVec.begin(), inVec.end(), 0.0); 23 | return std::sqrt( sd / (n-1) ); 24 | } 25 | 26 | double normalizeWeightsSum(RMatrix::Column column) 27 | { 28 | std::vector inVec(column.begin(), column.end()); 29 | double sum = 0; 30 | 31 | for(std::vector::iterator iter = inVec.begin(); 32 | iter != inVec.end(); 33 | ++iter) 34 | { 35 | sum += *iter * *iter; 36 | } 37 | 38 | return std::sqrt(sum); 39 | } -------------------------------------------------------------------------------- /man/rmseError.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/errorFunctions.R 3 | \name{rmseError} 4 | \alias{rmseError} 5 | \title{Root-mean-square error function} 6 | \usage{ 7 | rmseError(original, estimate) 8 | } 9 | \arguments{ 10 | \item{original}{The original data matrix.} 11 | 12 | \item{estimate}{The calculated data matrix.} 13 | } 14 | \value{ 15 | A list with the name of the error function in the first entry and the 16 | error value in the second entry. 17 | } 18 | \description{ 19 | The function calculates the root-mean-square error (RMSE) from the 20 | \code{original} and \code{estimate} parameters. 21 | } 22 | \details{ 23 | This function is a valid value for both \code{\link{darch}} parameters 24 | \code{rbm.errorFunction} and \code{darch.errorFunction}. 25 | } 26 | \examples{ 27 | \dontrun{ 28 | data(iris) 29 | model <- darch(Species ~ ., iris, rbm.errorFunction = "rmseError", 30 | darch.errorFunction = "rmseError") 31 | } 32 | } 33 | \seealso{ 34 | Other error functions: \code{\link{crossEntropyError}}, 35 | \code{\link{mseError}} 36 | } 37 | 38 | -------------------------------------------------------------------------------- /examples/example.caret.R: -------------------------------------------------------------------------------- 1 | library(caret) 2 | 3 | # Example for using the caret integration, requires package e1071 4 | example.caret <- function(...) 5 | { 6 | data(iris) 7 | tc <- trainControl(method = "boot", number = 5, allowParallel = F, 8 | verboseIter = T) 9 | 10 | parameters <- data.frame(parameter = c("layers", "bp.learnRate", "darch.unitFunction"), 11 | class = c("character", "numeric", "character"), 12 | label = c("Network structure", "Learning rate", "unitFunction")) 13 | 14 | grid <- function(x, y, len = NULL, search = "grid") 15 | { 16 | df <- expand.grid(layers = c("c(0,20,0)","c(0,10,10,0)","c(0,10,5,5,0)"), bp.learnRate = c(1,2,5,10)) 17 | 18 | df[["darch.unitFunction"]] <- rep(c("c(tanhUnit, softmaxUnit)", "c(tanhUnit, tanhUnit, softmaxUnit)", "c(tanhUnit, tanhUnit, tanhUnit, softmaxUnit)"), 4) 19 | 20 | df 21 | } 22 | 23 | darch <- train(Species ~ ., data = iris, tuneLength = 12, trControl = tc, 24 | method = darchModelInfo(parameters, grid), preProc = c("center", "scale"), 25 | darch.numEpochs = 15, darch.batchSize = 6, testing = T, ...) 26 | 27 | darch 28 | } -------------------------------------------------------------------------------- /man/linearUnit.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darchUnitFunctions.R 3 | \name{linearUnit} 4 | \alias{linearUnit} 5 | \title{Linear unit function with unit derivatives.} 6 | \usage{ 7 | linearUnit(input, ...) 8 | } 9 | \arguments{ 10 | \item{input}{Input for the activation function.} 11 | 12 | \item{...}{Additional parameters, not used.} 13 | } 14 | \value{ 15 | A list with the linear activation in the first entry and the 16 | derivative of the activation in the second entry. 17 | } 18 | \description{ 19 | The function calculates the activation of the units and returns a list, in 20 | which the first entry is the linear activation of the units and the second 21 | entry is the derivative of the transfer function. 22 | } 23 | \examples{ 24 | \dontrun{ 25 | data(iris) 26 | model <- darch(Species ~ ., iris, darch.unitFunction = "linearUnit") 27 | } 28 | } 29 | \seealso{ 30 | Other darch unit functions: \code{\link{exponentialLinearUnit}}, 31 | \code{\link{maxoutUnit}}, 32 | \code{\link{rectifiedLinearUnit}}, 33 | \code{\link{sigmoidUnit}}, \code{\link{softmaxUnit}}, 34 | \code{\link{softplusUnit}}, \code{\link{tanhUnit}} 35 | } 36 | 37 | -------------------------------------------------------------------------------- /man/sigmoidUnit.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darchUnitFunctions.R 3 | \name{sigmoidUnit} 4 | \alias{sigmoidUnit} 5 | \title{Sigmoid unit function with unit derivatives.} 6 | \usage{ 7 | sigmoidUnit(input, ...) 8 | } 9 | \arguments{ 10 | \item{input}{Input for the activation function.} 11 | 12 | \item{...}{Additional parameters, not used.} 13 | } 14 | \value{ 15 | A list with the activation in the first entry and the derivative of 16 | the transfer function in the second entry. 17 | } 18 | \description{ 19 | The function calculates the activation and returns a list which the first 20 | entry is the result through the sigmoid transfer function and the second 21 | entry is the derivative of the transfer function. 22 | } 23 | \examples{ 24 | \dontrun{ 25 | data(iris) 26 | model <- darch(Species ~ ., iris, darch.unitFunction = "sigmoidUnit") 27 | } 28 | } 29 | \seealso{ 30 | Other darch unit functions: \code{\link{exponentialLinearUnit}}, 31 | \code{\link{linearUnit}}, \code{\link{maxoutUnit}}, 32 | \code{\link{rectifiedLinearUnit}}, 33 | \code{\link{softmaxUnit}}, \code{\link{softplusUnit}}, 34 | \code{\link{tanhUnit}} 35 | } 36 | 37 | -------------------------------------------------------------------------------- /R/momentum.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Johannes Rueckert 2 | # 3 | # This file is part of darch. 4 | # 5 | # darch is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # darch is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with darch. If not, see . 17 | 18 | calculateMomentum <- function(initial, final, rampLength, epochsScheduled, epochsTrained) 19 | { 20 | if (rampLength <= 0 || epochsScheduled <= 1 || 21 | epochsTrained >= rampLength * epochsScheduled) 22 | { 23 | momentum <- final 24 | } 25 | else if (epochsTrained <= 1) momentum <- initial 26 | else 27 | { 28 | momentum <- (initial + (final - initial) * (epochsTrained - 1) 29 | / (rampLength * epochsScheduled - 1)) 30 | } 31 | 32 | momentum 33 | } -------------------------------------------------------------------------------- /examples/example.maxout.R: -------------------------------------------------------------------------------- 1 | # MNIST example using dropout and maxout 2 | example.maxout <- function(dataFolder = "data/", downloadMNIST = F, ...) 3 | { 4 | provideMNIST(dataFolder, downloadMNIST) 5 | 6 | load(paste0(dataFolder, "train.RData")) # trainData, trainLabels 7 | load(paste0(dataFolder, "test.RData")) # testData, testLabels 8 | 9 | # only take 1000 samples, otherwise training takes increasingly long 10 | chosenRowsTrain <- sample(1:nrow(trainData), size = 1000) 11 | trainDataSmall <- trainData[chosenRowsTrain,] 12 | trainLabelsSmall <- trainLabels[chosenRowsTrain,] 13 | 14 | darch <- darch(x = trainDataSmall, y = trainLabelsSmall, 15 | xValid = testData, yValid = testLabels, 16 | layers = c(784, 500, 10), 17 | darch.batchSize = 100, 18 | darch.dropout = .5, 19 | darch.dropout.oneMaskPerEpoch = T, 20 | # custom activation functions 21 | darch.unitFunction = c(maxoutUnit, softmaxUnit), 22 | darch.maxout.poolSize = 5, 23 | darch.maxout.unitFunction = exponentialLinearUnit, 24 | darch.elu.alpha = 2, 25 | darch.weightUpdateFunction = c(maxoutWeightUpdate, weightDecayWeightUpdate), 26 | darch.numEpochs = 5, 27 | logLevel = "DEBUG", 28 | ... 29 | ) 30 | 31 | darch 32 | } -------------------------------------------------------------------------------- /man/runDArch.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/runDArch.R 3 | \name{runDArch} 4 | \alias{runDArch} 5 | \title{Forward-propagates data through the network} 6 | \usage{ 7 | runDArch(darch, data, inputLayer = 1, outputLayer = length(darch@layers), 8 | matMult = getParameter(".matMult")) 9 | } 10 | \arguments{ 11 | \item{darch}{A instance of the class \code{\linkS4class{DArch}}.} 12 | 13 | \item{data}{The input data to execute the darch on.} 14 | 15 | \item{inputLayer}{Into which layer the given data is to be fed. Absolute 16 | number starting at 1 for the input layer.} 17 | 18 | \item{outputLayer}{The output of which layer is to be returned, absolute 19 | number starting a 0 for the input layer (i.e. pre-processed data is 20 | returned).} 21 | 22 | \item{matMult}{Function to use for matrix multiplication.} 23 | } 24 | \value{ 25 | The network output 26 | } 27 | \description{ 28 | Runs the \code{DArch} in a feed-forward manner and returns the output. 29 | } 30 | \details{ 31 | Input and output layer can be chosen via the parameters \code{inputLayer} 32 | and \code{outputLayer}. 33 | } 34 | \seealso{ 35 | \code{\link{darch}} 36 | 37 | Other darch execute functions: \code{\link{runDArchDropout}} 38 | } 39 | \keyword{internal} 40 | 41 | -------------------------------------------------------------------------------- /man/preTrainDArch-DArch-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darch.Learn.R 3 | \docType{methods} 4 | \name{preTrainDArch,DArch-method} 5 | \alias{preTrainDArch,DArch-method} 6 | \title{Pre-trains a \code{\linkS4class{DArch}} network} 7 | \usage{ 8 | \S4method{preTrainDArch}{DArch}(darch, dataSet, dataSetValid = NULL, 9 | numEpochs = 1, numCD = 1, lastLayer = 0, isClass = F, 10 | consecutive = T, ...) 11 | } 12 | \arguments{ 13 | \item{darch}{A instance of the class \code{\linkS4class{DArch}}.} 14 | 15 | \item{dataSet}{\code{\linkS4class{DataSet}} to be used for training.} 16 | 17 | \item{dataSetValid}{\code{\linkS4class{DataSet}} to be used for validation.} 18 | 19 | \item{numEpochs}{The number of epochs} 20 | 21 | \item{numCD}{The number of CD iterations} 22 | 23 | \item{lastLayer}{Numeric indicating after which layer to stop training.} 24 | 25 | \item{isClass}{Whether to test pre-trained networks against target data} 26 | 27 | \item{consecutive}{Whether to train RBMs consecutively instead of each one 28 | epoch at a time.} 29 | 30 | \item{...}{Additional parameters for the function \code{\link{trainRBM}}} 31 | } 32 | \description{ 33 | Pre-trains a \code{\linkS4class{DArch}} network 34 | } 35 | \seealso{ 36 | \link{preTrainDArch} 37 | } 38 | \keyword{internal} 39 | 40 | -------------------------------------------------------------------------------- /man/softmaxUnit.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darchUnitFunctions.R 3 | \name{softmaxUnit} 4 | \alias{softmaxUnit} 5 | \title{Softmax unit function with unit derivatives.} 6 | \usage{ 7 | softmaxUnit(input, ...) 8 | } 9 | \arguments{ 10 | \item{input}{Input for the activation function.} 11 | 12 | \item{...}{Additional parameters, not used.} 13 | } 14 | \value{ 15 | A list with the softmax activation in the first entry and the 16 | derivative of the transfer function in the second entry. 17 | } 18 | \description{ 19 | The function calculates the activation of the units and returns a list, in 20 | which the first entry is the result through the softmax transfer function 21 | and the second entry is the derivative of the transfer function. 22 | } 23 | \examples{ 24 | \dontrun{ 25 | data(iris) 26 | model <- darch(Species ~ ., iris, 27 | darch.unitFunction = c("sigmoidUnit", "softmaxUnit")) 28 | } 29 | } 30 | \references{ 31 | http://www.faqs.org/faqs/ai-faq/neural-nets/part2/section-12.html 32 | } 33 | \seealso{ 34 | Other darch unit functions: \code{\link{exponentialLinearUnit}}, 35 | \code{\link{linearUnit}}, \code{\link{maxoutUnit}}, 36 | \code{\link{rectifiedLinearUnit}}, 37 | \code{\link{sigmoidUnit}}, \code{\link{softplusUnit}}, 38 | \code{\link{tanhUnit}} 39 | } 40 | 41 | -------------------------------------------------------------------------------- /man/createDataSet-ANY-missing-formula-missing-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/dataset.R 3 | \docType{methods} 4 | \name{createDataSet,ANY,missing,formula,missing-method} 5 | \alias{createDataSet,ANY,missing,formula,missing-method} 6 | \alias{createDataSet.formula} 7 | \title{Constructor function for \code{\linkS4class{DataSet}} objects.} 8 | \usage{ 9 | \S4method{createDataSet}{ANY,missing,formula,missing}(data, formula, ..., 10 | na.action = na.pass, previous.dataSet = new("DataSet")) 11 | } 12 | \arguments{ 13 | \item{data}{\code{\link{data.frame}} containing the dataset.} 14 | 15 | \item{formula}{Model formula.} 16 | 17 | \item{...}{Additional parameters.} 18 | 19 | \item{na.action}{\code{\link{model.frame}} parameter} 20 | 21 | \item{previous.dataSet}{Existing \code{\linkS4class{DataSet}} from which 22 | parameters are extrated and re-used.} 23 | } 24 | \value{ 25 | The new \code{\linkS4class{DataSet}} object 26 | } 27 | \description{ 28 | Generates a new \code{\linkS4class{DataSet}} object with the given 29 | parameters. 30 | } 31 | \details{ 32 | Initializes a \code{\linkS4class{DataSet}} with data using a 33 | \code{\link{formula}} and saving its parameters for subsequent data. 34 | } 35 | \seealso{ 36 | \link{darch.formula}, \link{createDataSet} 37 | } 38 | \keyword{internal} 39 | 40 | -------------------------------------------------------------------------------- /examples/example.xor_nominal.R: -------------------------------------------------------------------------------- 1 | # XOR with nominal/ordinal data 2 | example.xor_nominal <- function(...) 3 | { 4 | # dataset 5 | trainData <- matrix(c(0,0,0,1,1,0,1,1), ncol = 2, byrow = TRUE) 6 | trainTargets <- matrix(c("zero", "one", "one", "zero"), nrow=4) 7 | # create data frame with column names X1, X2, and trainTargets, which will 8 | # used in the model formula 9 | dataFrame <- data.frame(trainData, trainTargets) 10 | # Convert input data to ordered factors 11 | dataFrame[, c("X1", "X2")] <- lapply(dataFrame[, c("X1", "X2")], 12 | FUN = function(x) { as.ordered(x)}) 13 | 14 | # see XOR example #1 for explanation of the parameter values 15 | darch <- darch(trainTargets ~ ., dataFrame, 16 | layers = c(2, 10, 1), 17 | preProc.orderedToFactor.targets = F, 18 | shuffleTrainData = F, 19 | darch.errorFunction = mseError, 20 | darch.stopErr = .1, 21 | darch.unitFunction = sigmoidUnit, 22 | bp.learnRate = 1, 23 | darch.numEpochs = 1000, 24 | retainData = T, 25 | ... 26 | ) 27 | 28 | e <- darchTest(darch) 29 | cat(paste0("Incorrect classifications on all examples: ", e[3], " (", 30 | e[2], "%)\n")) 31 | 32 | predictions <- predict(darch, newdata = dataFrame, type = "class") 33 | 34 | print(predictions, max.levels = 0) 35 | 36 | darch 37 | } 38 | -------------------------------------------------------------------------------- /man/weightDecayWeightUpdate.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/weightUpdateFunctions.R 3 | \name{weightDecayWeightUpdate} 4 | \alias{weightDecayWeightUpdate} 5 | \title{Updates the weight using weight decay.} 6 | \usage{ 7 | weightDecayWeightUpdate(darch, layerIndex, weightsInc, biasesInc, ..., 8 | weightDecay = getParameter(".darch.weightDecay", 0, darch), 9 | debug = getParameter(".debug", F, darch)) 10 | } 11 | \arguments{ 12 | \item{darch}{\linkS4class{DArch} instance.} 13 | 14 | \item{layerIndex}{Layer index within the network.} 15 | 16 | \item{weightsInc}{Matrix containing scheduled weight updates from the 17 | fine-tuning algorithm.} 18 | 19 | \item{biasesInc}{Bias weight updates.} 20 | 21 | \item{...}{Additional parameters, not used.} 22 | 23 | \item{weightDecay}{Weights are multiplied by (1 - \code{weightDecay}) before 24 | each update. Corresponds to the \code{darch.weightDecay} parameter of 25 | \link{darch.default}.} 26 | 27 | \item{debug}{Internal debugging flag.} 28 | } 29 | \value{ 30 | updated weights 31 | } 32 | \description{ 33 | Multiplies the weights by (1 - \code{weightDecay}) before applying the 34 | scheduled weight changes. 35 | } 36 | \examples{ 37 | \dontrun{ 38 | model <- darch(Species ~ ., iris, c(0, 50, 0), 39 | darch.weightUpdateFunction = "weightDecayWeightUpdate") 40 | } 41 | } 42 | \seealso{ 43 | Other weight update functions: \code{\link{maxoutWeightUpdate}} 44 | } 45 | 46 | -------------------------------------------------------------------------------- /man/generateWeightsUniform.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/generateWeightsFunctions.R 3 | \name{generateWeightsUniform} 4 | \alias{generateWeightsUniform} 5 | \title{Generates a weight matrix using \link{runif}} 6 | \usage{ 7 | generateWeightsUniform(numUnits1, numUnits2, 8 | weights.min = getParameter(".weights.min", -0.1, ...), 9 | weights.max = getParameter(".weights.max", 0.1, ...), ...) 10 | } 11 | \arguments{ 12 | \item{numUnits1}{Number of units in the lower layer.} 13 | 14 | \item{numUnits2}{Number of units in the upper layer.} 15 | 16 | \item{weights.min}{\code{min} parameter to the \link{runif} function.} 17 | 18 | \item{weights.max}{\code{max} parameter to the \link{runif} function.} 19 | 20 | \item{...}{Additional parameters, used for parameter resolution.} 21 | } 22 | \value{ 23 | Weight matrix. 24 | } 25 | \description{ 26 | This function is used to generate random weights and biases using 27 | \link{runif}. 28 | } 29 | \examples{ 30 | \dontrun{ 31 | data(iris) 32 | model <- darch(Species ~ ., iris, generateWeightsFunction = "generateWeightsUniform", 33 | weights.min = -.1, weights.max = .5) 34 | } 35 | } 36 | \seealso{ 37 | Other weight generation functions: \code{\link{generateWeightsGlorotNormal}}, 38 | \code{\link{generateWeightsGlorotUniform}}, 39 | \code{\link{generateWeightsHeNormal}}, 40 | \code{\link{generateWeightsHeUniform}}, 41 | \code{\link{generateWeightsNormal}} 42 | } 43 | 44 | -------------------------------------------------------------------------------- /man/crossEntropyError.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/errorFunctions.R 3 | \name{crossEntropyError} 4 | \alias{crossEntropyError} 5 | \title{Cross entropy error function} 6 | \usage{ 7 | crossEntropyError(original, estimate) 8 | } 9 | \arguments{ 10 | \item{original}{The original data matrix.} 11 | 12 | \item{estimate}{The calculated data matrix.} 13 | } 14 | \value{ 15 | A list with the name of the error function in the first entry and the 16 | error value in the second entry. 17 | } 18 | \description{ 19 | The function calculates the cross entropy error from the \code{original} and 20 | \code{estimate} parameters. 21 | } 22 | \details{ 23 | This function can be used for the \code{darch.errorFunction} parameter of the 24 | \code{\link{darch}} function, but is only a valid error function if used in 25 | combination with the \code{\link{softmaxUnit}} activation function! It is not 26 | a valid value for the parameter \code{rbm.errorFunction}. 27 | } 28 | \examples{ 29 | \dontrun{ 30 | data(iris) 31 | model <- darch(Species ~ ., iris, darch.errorFunction = "crossEntropyError") 32 | } 33 | } 34 | \references{ 35 | Rubinstein, R.Y., Kroese, D.P. (2004). The Cross-Entropy Method: 36 | A Unified Approach to Combinatorial Optimization, Monte-Carlo Simulation, 37 | and Machine Learning, Springer-Verlag, New York. 38 | } 39 | \seealso{ 40 | Other error functions: \code{\link{mseError}}, 41 | \code{\link{rmseError}} 42 | } 43 | 44 | -------------------------------------------------------------------------------- /man/sigmoidUnitRbm.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/rbmUnitFunctions.R 3 | \name{sigmoidUnitRbm} 4 | \alias{sigmoidUnitRbm} 5 | \title{Calculates the RBM neuron output with the sigmoid function} 6 | \usage{ 7 | sigmoidUnitRbm(rbm, data, biases, weights, runParams, 8 | matMult = getParameter(".matMult", ...), ...) 9 | } 10 | \arguments{ 11 | \item{rbm}{An instance of the class \code{\linkS4class{RBM}}.} 12 | 13 | \item{data}{A matrix with the data for the calculations.} 14 | 15 | \item{biases}{The biases for the calculations.} 16 | 17 | \item{weights}{The weight matrix for the calculations.} 18 | 19 | \item{runParams}{Parameters which indicates the status of the training.} 20 | 21 | \item{matMult}{Matrix multiplication function.} 22 | 23 | \item{...}{Additional parameters, not used.} 24 | } 25 | \value{ 26 | The real value and binary activations for the units 27 | } 28 | \description{ 29 | Calculates the RBM neuron output with the sigmoid function from input saved 30 | in \code{data}. 31 | } 32 | \details{ 33 | The return value is a list with the output of the sigmoid function 34 | as first entry and binary representation calculated through a comparison of 35 | the output with random numbers. The random numbers a generated with the 36 | function \code{\link{runif}}. 37 | } 38 | \seealso{ 39 | Other RBM unit functions: \code{\link{linearUnitRbm}}, 40 | \code{\link{tanhUnitRbm}} 41 | } 42 | \keyword{internal} 43 | 44 | -------------------------------------------------------------------------------- /R/net.Class.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2016 Martin Drees 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | 19 | #' Abstract class for neural networks. 20 | #' 21 | #' This is an abstract class for neural networks. It provides some 22 | #' functionalities used in more than one network type. 23 | #' 24 | #' @slot epochs Number of epochs the network has been trained for. 25 | #' @slot stats Training statistics. 26 | #' @slot parameters List of parameters which do not change throughout training. 27 | #' @family darch classes 28 | #' @author Martin Drees 29 | #' @exportClass Net 30 | #' @rdname Net 31 | #' @keywords internal 32 | setClass( 33 | Class = "Net", 34 | representation = representation( 35 | epochs = "numeric", 36 | stats = "list", 37 | parameters = "list" 38 | ) 39 | ) -------------------------------------------------------------------------------- /man/linearUnitRbm.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/rbmUnitFunctions.R 3 | \name{linearUnitRbm} 4 | \alias{linearUnitRbm} 5 | \title{Calculates the linear neuron output no transfer function} 6 | \usage{ 7 | linearUnitRbm(rbm, data, biases, weights, runParams, 8 | matMult = getParameter(".matMult", ...), ...) 9 | } 10 | \arguments{ 11 | \item{rbm}{A instance of the class \code{\linkS4class{RBM}}.} 12 | 13 | \item{data}{A matrix with the data for the calculations.} 14 | 15 | \item{biases}{The biases for the calculations.} 16 | 17 | \item{weights}{The weight matrix for the calculations.} 18 | 19 | \item{runParams}{Parameters which indicates the status of the training.} 20 | 21 | \item{matMult}{Matrix multiplication function.} 22 | 23 | \item{...}{Additional parameters, used for paramete resolution.} 24 | } 25 | \value{ 26 | The real value and binary activations for the units 27 | } 28 | \description{ 29 | Calculates the linear neuron output with no transfer function from real 30 | value input saved in \code{data}. 31 | } 32 | \details{ 33 | The return value is a list with the output of the neurons as first 34 | entry and binary representation calculated through a comparison of the 35 | output with random numbers. The random numbers a generated with the 36 | function \code{\link{rnorm}}. 37 | } 38 | \seealso{ 39 | Other RBM unit functions: \code{\link{sigmoidUnitRbm}}, 40 | \code{\link{tanhUnitRbm}} 41 | } 42 | \keyword{internal} 43 | 44 | -------------------------------------------------------------------------------- /man/runDArchDropout.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/runDArch.R 3 | \name{runDArchDropout} 4 | \alias{runDArchDropout} 5 | \title{Forward-propagate data through the network with dropout inference} 6 | \usage{ 7 | runDArchDropout(darch, data, inputLayer = 1, 8 | outputLayer = length(darch@layers), matMult = getParameter(".matMult"), 9 | dropout = getParameter(".darch.dropout"), 10 | iterations = getParameter(".darch.dropout.momentMatching")) 11 | } 12 | \arguments{ 13 | \item{darch}{A instance of the class \code{\linkS4class{DArch}}.} 14 | 15 | \item{data}{The input data to execute the darch on.} 16 | 17 | \item{inputLayer}{Into which layer the given data is to be fed. Absolute 18 | number starting at 1 for the input layer.} 19 | 20 | \item{outputLayer}{The output of which layer is to be returned, absolute 21 | number starting a 0 for the input layer (i.e. pre-processed data is 22 | returned).} 23 | 24 | \item{matMult}{Function to use for matrix multiplication.} 25 | 26 | \item{dropout}{Dropout rates for the layers.} 27 | 28 | \item{iterations}{If greater than 0, the numbr of iterations to use for 29 | moment matching dropout inference.} 30 | } 31 | \value{ 32 | The network output. 33 | } 34 | \description{ 35 | If dropout was disabled, \code{\link{runDArch}} will be called instead. 36 | } 37 | \seealso{ 38 | \code{\link{darch}} 39 | 40 | Other darch execute functions: \code{\link{runDArch}} 41 | } 42 | \keyword{internal} 43 | 44 | -------------------------------------------------------------------------------- /examples/example.iris.R: -------------------------------------------------------------------------------- 1 | # IRIS example using RPROP fine-tuning and autosaving 2 | example.iris <- function(...) 3 | { 4 | data(iris) 5 | 6 | darch <- darch(Species ~ ., iris, 7 | preProc.params = list("method" = c("scale", "center")), 8 | normalizeWeights = T, 9 | normalizeWeightsBound = 1, 10 | layers = 20, # one hidden layer with 20 neurons 11 | darch.batchSize = 30, 12 | darch.fineTuneFunction = "rpropagation", 13 | darch.unitFunction = c("tanhUnit", "softmaxUnit"), 14 | darch.stopValidClassErr = 0, 15 | darch.stopValidErr = .15, 16 | bootstrap = T, 17 | bootstrap.unique = F, 18 | rprop.incFact = 1.3, 19 | rprop.decFact = .7, 20 | rprop.initDelta = .1, 21 | rprop.maxDelta = 5, 22 | rprop.method = "iRprop-", 23 | rprop.minDelta = 1e-5, 24 | autosave = T, 25 | autosave.dir = "darch.autosave", 26 | autosave.epochs = 10, 27 | autosave.trim = T, 28 | ... 29 | ) 30 | 31 | # The predict function can be used to get the network output for a new set of 32 | # data, it will even convert the output back to the original class labels 33 | predictions <- predict(darch, newdata = iris, type = "class") 34 | 35 | # And these labels can then easily be compared to the correct ones 36 | numIncorrect <- sum(predictions != iris[,5]) 37 | cat(paste0("Incorrect classifications on all examples: ", numIncorrect, " (", 38 | round(numIncorrect/nrow(iris)*100, 2), "%)\n")) 39 | 40 | darch 41 | } 42 | -------------------------------------------------------------------------------- /man/tanhUnitRbm.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/rbmUnitFunctions.R 3 | \name{tanhUnitRbm} 4 | \alias{tanhUnitRbm} 5 | \title{Calculates the neuron output with the hyperbolic tangent function} 6 | \usage{ 7 | tanhUnitRbm(rbm, data, biases, weights, runParams, 8 | matMult = getParameter(".matMult", ...), ...) 9 | } 10 | \arguments{ 11 | \item{rbm}{A instance of the class \code{\linkS4class{RBM}}.} 12 | 13 | \item{data}{A matrix with the data for the calculations.} 14 | 15 | \item{biases}{The biases for the calculations.} 16 | 17 | \item{weights}{The weight matrix for the calculations.} 18 | 19 | \item{runParams}{Parameters which indicates the status of the training.} 20 | 21 | \item{matMult}{Matrix multiplication function.} 22 | 23 | \item{...}{Additional parameters, used for parameter resolution.} 24 | } 25 | \value{ 26 | The real value and binary (-1,1) activations for the units. 27 | } 28 | \description{ 29 | Calculates the neuron output with the hyperbolic tangent function from 30 | input in \code{data}. 31 | } 32 | \details{ 33 | The return value is a list with the output of the hyperbolic 34 | tangent function as first entry and binary (-1,1) representation calculated 35 | through a comparison ofthe output with random numbers. The random numbers a 36 | generated with the function \code{\link{runif}}. 37 | } 38 | \seealso{ 39 | Other RBM unit functions: \code{\link{linearUnitRbm}}, 40 | \code{\link{sigmoidUnitRbm}} 41 | } 42 | \keyword{internal} 43 | 44 | -------------------------------------------------------------------------------- /man/generateWeightsNormal.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/generateWeightsFunctions.R 3 | \name{generateWeightsNormal} 4 | \alias{generateWeightsNormal} 5 | \title{Generates a weight matrix using \link{rnorm}.} 6 | \usage{ 7 | generateWeightsNormal(numUnits1, numUnits2, 8 | weights.mean = getParameter(".weights.mean", 0, ...), 9 | weights.sd = getParameter(".weights.sd", 0.01, ...), ...) 10 | } 11 | \arguments{ 12 | \item{numUnits1}{Number of units in the lower layer.} 13 | 14 | \item{numUnits2}{Number of units in the upper layer.} 15 | 16 | \item{weights.mean}{\code{mean} parameter to the \link{rnorm} function.} 17 | 18 | \item{weights.sd}{\code{sd} parameter to the \link{rnorm} function.} 19 | 20 | \item{...}{Additional parameters, used for parameter resolution.} 21 | } 22 | \value{ 23 | Weight matrix. 24 | } 25 | \description{ 26 | This function is the standard method for generating weights for instances of 27 | \code{\linkS4class{Net}}. It uses \code{\link{rnorm}} to do so. 28 | } 29 | \examples{ 30 | \dontrun{ 31 | data(iris) 32 | model <- darch(Species ~ ., iris, generateWeightsFunction = "generateWeightsNormal", 33 | weights.mean = .1, weights.sd = .05) 34 | } 35 | } 36 | \seealso{ 37 | Other weight generation functions: \code{\link{generateWeightsGlorotNormal}}, 38 | \code{\link{generateWeightsGlorotUniform}}, 39 | \code{\link{generateWeightsHeNormal}}, 40 | \code{\link{generateWeightsHeUniform}}, 41 | \code{\link{generateWeightsUniform}} 42 | } 43 | 44 | -------------------------------------------------------------------------------- /man/generateWeightsGlorotUniform.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/generateWeightsFunctions.R 3 | \name{generateWeightsGlorotUniform} 4 | \alias{generateWeightsGlorotUniform} 5 | \title{Glorot uniform weight initialization} 6 | \usage{ 7 | generateWeightsGlorotUniform(numUnits1, numUnits2, ...) 8 | } 9 | \arguments{ 10 | \item{numUnits1}{Number of units in the lower layer.} 11 | 12 | \item{numUnits2}{Number of units in the upper layer.} 13 | 14 | \item{...}{Additional parameters, used for parameter resolution and passed 15 | to \code{\link{generateWeightsUniform}}.} 16 | } 17 | \value{ 18 | Weight matrix. 19 | } 20 | \description{ 21 | This function is used to generate random weights and biases using 22 | Glorot uniform weight initialization as described in 23 | Glorot & Bengio, AISTATS 2010. 24 | } 25 | \examples{ 26 | \dontrun{ 27 | data(iris) 28 | model <- darch(Species ~ ., iris, generateWeightsFunction = "generateWeightsGlorotUniform") 29 | } 30 | } 31 | \references{ 32 | Glorot, Xavier and Yoshua Bengio (2010). "Understanding the 33 | difficulty of training deep feedforward neural networks". 34 | In: International conference on artificial intelligence and statistics, pp. 249-256 35 | } 36 | \seealso{ 37 | Other weight generation functions: \code{\link{generateWeightsGlorotNormal}}, 38 | \code{\link{generateWeightsHeNormal}}, 39 | \code{\link{generateWeightsHeUniform}}, 40 | \code{\link{generateWeightsNormal}}, 41 | \code{\link{generateWeightsUniform}} 42 | } 43 | 44 | -------------------------------------------------------------------------------- /man/plot.DArch.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/plot.R 3 | \name{plot.DArch} 4 | \alias{plot.DArch} 5 | \title{Plot \code{\linkS4class{DArch}} statistics or structure.} 6 | \usage{ 7 | \method{plot}{DArch}(x, y = "raw", ..., type = y) 8 | } 9 | \arguments{ 10 | \item{x}{\code{\linkS4class{DArch}} instance.} 11 | 12 | \item{y}{See \code{type}.} 13 | 14 | \item{...}{Additional parameters, passed to plotting functions.} 15 | 16 | \item{type}{Which type of plot to create, one of \code{raw}, 17 | \code{class}, \code{time}, \code{momentum}, and \code{net}.} 18 | } 19 | \value{ 20 | The plotted graph. 21 | } 22 | \description{ 23 | This function provides different plots depending on the \code{type} 24 | parameter: 25 | } 26 | \details{ 27 | \itemize{ 28 | \item raw. Plots the raw network error (e.g. MSE), this is the default 29 | \item class. Plots the classification error 30 | \item time. Plots the times needed for each epoch 31 | \item momentum. Plots the momentum ramp 32 | \item net. Calls \code{\link[NeuralNetTools]{plotnet}} to plot the network 33 | } 34 | } 35 | \examples{ 36 | \dontrun{ 37 | data(iris) 38 | model <- darch(Species ~ ., iris) 39 | plot(model) 40 | plot(model, "class") 41 | plot(model, "time") 42 | plot(model, "momentum") 43 | plot(model, "net") 44 | } 45 | } 46 | \seealso{ 47 | Other darch interface functions: \code{\link{darchBench}}, 48 | \code{\link{darchTest}}, \code{\link{darch}}, 49 | \code{\link{predict.DArch}}, \code{\link{print.DArch}} 50 | } 51 | 52 | -------------------------------------------------------------------------------- /R/net.Getter.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2016 Martin Drees 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | 19 | #' Returns the current momentum of the \code{Net}. 20 | #' 21 | #' @param net A instance of the class \code{Net}. 22 | #' 23 | #' @seealso \code{\linkS4class{Net}} 24 | #' @include net.Class.R 25 | #' @keywords internal 26 | setGeneric("getMomentum",function(net){standardGeneric("getMomentum")}) 27 | 28 | setMethod( 29 | f = "getMomentum", 30 | signature = "Net", 31 | definition = function(net) 32 | { 33 | # TODO optimize for speed 34 | calculateMomentum(getParameter(".darch.initialMomentum", net = net), 35 | getParameter(".darch.finalMomentum", net = net), 36 | getParameter(".darch.momentumRampLength", net = net), 37 | getParameter(".darch.epochsScheduled", net = net), net@epochs) 38 | } 39 | ) 40 | -------------------------------------------------------------------------------- /man/generateWeightsHeUniform.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/generateWeightsFunctions.R 3 | \name{generateWeightsHeUniform} 4 | \alias{generateWeightsHeUniform} 5 | \title{He uniform weight initialization} 6 | \usage{ 7 | generateWeightsHeUniform(numUnits1, numUnits2, ...) 8 | } 9 | \arguments{ 10 | \item{numUnits1}{Number of units in the lower layer.} 11 | 12 | \item{numUnits2}{Number of units in the upper layer.} 13 | 14 | \item{...}{Additional parameters, used for parameter resolution and passed 15 | to \code{\link{generateWeightsUniform}}.} 16 | } 17 | \value{ 18 | Weight matrix. 19 | } 20 | \description{ 21 | This function is used to generate random weights and biases using 22 | He uniform weight initialization as described in 23 | He et al., \url{http://arxiv.org/abs/1502.01852}. 24 | } 25 | \examples{ 26 | \dontrun{ 27 | data(iris) 28 | model <- darch(Species ~ ., iris, generateWeightsFunction = "generateWeightsHeUniform") 29 | } 30 | } 31 | \references{ 32 | He, Kaiming, Xiangyu Zhang, Shaoqing Ren, and Jian Sun (2015). 33 | "Delving Deep into Rectifiers: Surpassing Human-Level Performance on 34 | ImageNet Classification". In: CoRR abs/1502.01852. 35 | URL: http://arxiv.org/abs/1502.01852 36 | } 37 | \seealso{ 38 | Other weight generation functions: \code{\link{generateWeightsGlorotNormal}}, 39 | \code{\link{generateWeightsGlorotUniform}}, 40 | \code{\link{generateWeightsHeNormal}}, 41 | \code{\link{generateWeightsNormal}}, 42 | \code{\link{generateWeightsUniform}} 43 | } 44 | 45 | -------------------------------------------------------------------------------- /man/softplusUnit.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darchUnitFunctions.R 3 | \name{softplusUnit} 4 | \alias{softplusUnit} 5 | \title{Softplus unit function with unit derivatives.} 6 | \usage{ 7 | softplusUnit(input, ...) 8 | } 9 | \arguments{ 10 | \item{input}{Input for the activation function.} 11 | 12 | \item{...}{Additional parameters, not used.} 13 | } 14 | \value{ 15 | A list with the softplus activation in the first entry and 16 | the derivative of the activation in the second entry. 17 | } 18 | \description{ 19 | The function calculates the activation of the units and returns a list, in 20 | which the first entry is the softmax activation of the units and the second 21 | entry is the derivative of the transfer function. Softplus is a smoothed 22 | version of the rectified linear activation function. 23 | } 24 | \examples{ 25 | \dontrun{ 26 | data(iris) 27 | model <- darch(Species ~ ., iris, darch.unitFunction = "softplusUnit") 28 | } 29 | } 30 | \references{ 31 | Dugas, Charles, Yoshua Bengio, Francois Belisle, Claude Nadeau, 32 | and Rene Garcia (2001). "Incorporating Second-Order Functional Knowledge for 33 | Better Option Pricing". In: Advances in Neural Information Processing 34 | Systems, pp. 472-478. 35 | } 36 | \seealso{ 37 | Other darch unit functions: \code{\link{exponentialLinearUnit}}, 38 | \code{\link{linearUnit}}, \code{\link{maxoutUnit}}, 39 | \code{\link{rectifiedLinearUnit}}, 40 | \code{\link{sigmoidUnit}}, \code{\link{softmaxUnit}}, 41 | \code{\link{tanhUnit}} 42 | } 43 | 44 | -------------------------------------------------------------------------------- /man/darchTest.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/test.R 3 | \name{darchTest} 4 | \alias{darchTest} 5 | \title{Test classification network.} 6 | \usage{ 7 | darchTest(darch, newdata = NULL, targets = T) 8 | } 9 | \arguments{ 10 | \item{darch}{\code{\linkS4class{DArch}} instance.} 11 | 12 | \item{newdata}{New data to use, \code{NULL} to use training data.} 13 | 14 | \item{targets}{Labels for the \code{data}, \code{NULL} to use training 15 | labels (only possible when \code{data} is \code{NULL} as well).} 16 | } 17 | \value{ 18 | Vector containing error function output, percentage of incorrect 19 | classifications and absolute number of incorrect classifications. 20 | } 21 | \description{ 22 | Forward-propagate given data through the deep neural network and return 23 | classification accuracy using the given labels. 24 | } 25 | \details{ 26 | This is primarily a convenience function similar to \link{predict.DArch} with 27 | classification performance measurements instead of network output, 28 | and it returns a list of accuracy indicators (raw network error, percentage 29 | of incorrect classifications and absolute number of incorrect 30 | classifications). 31 | } 32 | \examples{ 33 | \dontrun{ 34 | data(iris) 35 | model <- darch(Species ~ ., iris, retainData = T) 36 | classificationStats <- darchTest(model) 37 | } 38 | } 39 | \seealso{ 40 | Other darch interface functions: \code{\link{darchBench}}, 41 | \code{\link{darch}}, \code{\link{plot.DArch}}, 42 | \code{\link{predict.DArch}}, \code{\link{print.DArch}} 43 | } 44 | 45 | -------------------------------------------------------------------------------- /examples/example.dropconnect.R: -------------------------------------------------------------------------------- 1 | # MNIST example with pre-training 2 | example.dropconnect <- function(dataFolder = "data/", downloadMNIST = F, ...) 3 | { 4 | # Make sure to prove the correct folder if you have already downloaded the 5 | # MNIST data somewhere, or otherwise set downloadMNIST to TRUE 6 | provideMNIST(dataFolder, downloadMNIST) 7 | 8 | # Load MNIST data 9 | load(paste0(dataFolder, "train.RData")) # trainData, trainLabels 10 | load(paste0(dataFolder, "test.RData")) # testData, testLabels 11 | 12 | # only take 1000 samples, otherwise training takes increasingly long 13 | chosenRowsTrain <- sample(1:nrow(trainData), size=1000) 14 | trainDataSmall <- trainData[chosenRowsTrain,] 15 | trainLabelsSmall <- trainLabels[chosenRowsTrain,] 16 | 17 | darch <- darch(trainDataSmall, trainLabelsSmall, 18 | layers = c(784,100,10), 19 | bootstrap = T, 20 | bootstrap.unique = F, 21 | darch.batchSize = 100, 22 | darch.dropout = .25, 23 | darch.dropout.dropConnect = T, 24 | darch.dropout.momentMatching = 10, 25 | bp.learnRate = 1, 26 | darch.unitFunction = c(rectifiedLinearUnit, softmaxUnit), 27 | darch.numEpochs = 25, 28 | ... 29 | ) 30 | 31 | predictions <- predict(darch, newdata = testData, type = "class") 32 | 33 | labels <- cbind(predictions, testLabels) 34 | numIncorrect <- sum(apply(labels, 1, function(i) { any(i[1:10] != i[11:20]) })) 35 | cat(paste0("Incorrect classifications on test data: ", numIncorrect, 36 | " (", round(numIncorrect/nrow(testLabels)*100, 2), "%)\n")) 37 | 38 | darch 39 | } 40 | -------------------------------------------------------------------------------- /man/exponentialLinearUnit.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darchUnitFunctions.R 3 | \name{exponentialLinearUnit} 4 | \alias{exponentialLinearUnit} 5 | \title{Exponential linear unit (ELU) function with unit derivatives.} 6 | \usage{ 7 | exponentialLinearUnit(input, alpha = getParameter(".darch.elu.alpha", 1, ...), 8 | ...) 9 | } 10 | \arguments{ 11 | \item{input}{Input for the activation function.} 12 | 13 | \item{alpha}{ELU hyperparameter.} 14 | 15 | \item{...}{Additional parameters.} 16 | } 17 | \value{ 18 | A list with the ELU activation in the first entry and 19 | the derivative of the activation in the second entry. 20 | } 21 | \description{ 22 | The function calculates the activation of the units and returns a list, in 23 | which the first entry is the exponential linear activation of the units and 24 | the second entry is the derivative of the transfer function. 25 | } 26 | \examples{ 27 | \dontrun{ 28 | data(iris) 29 | model <- darch(Species ~ ., iris, darch.unitFunction = "exponentialLinearUnit", 30 | darch.elu.alpha = 2) 31 | } 32 | } 33 | \references{ 34 | Clevert, Djork-Arne, Thomas Unterthiner, and Sepp Hochreiter 35 | (2015). "Fast and Accurate Deep Network Learning by Exponential Linear Units 36 | (ELUs)". In: CoRR abs/1511.07289. URL : http://arxiv.org/abs/1511.07289 37 | } 38 | \seealso{ 39 | Other darch unit functions: \code{\link{linearUnit}}, 40 | \code{\link{maxoutUnit}}, 41 | \code{\link{rectifiedLinearUnit}}, 42 | \code{\link{sigmoidUnit}}, \code{\link{softmaxUnit}}, 43 | \code{\link{softplusUnit}}, \code{\link{tanhUnit}} 44 | } 45 | 46 | -------------------------------------------------------------------------------- /R/darch.Getter.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2016 Martin Drees 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | 19 | #' Returns the dropout mask for the given layer 20 | #' 21 | #' The dropout mask is applied to the weights between layer i and i+1, for 0 < i 22 | #' < numLayers. For i = 0, the dropout mask for the input layer is returned, 23 | #' which will be applied to the initial input data. 24 | #' 25 | #' @param darch A instance of the class \code{\linkS4class{DArch}}. 26 | #' @param i Layer index or 0 for input dropout mask. 27 | #' @return Dropout mask for the given layer. 28 | #' @seealso \code{\linkS4class{DArch}} 29 | #' @include darch.Class.R 30 | #' @keywords internal 31 | setGeneric("getDropoutMask",function(darch, i){standardGeneric("getDropoutMask")}) 32 | 33 | setMethod( 34 | f="getDropoutMask", 35 | signature="DArch", 36 | definition=function(darch, i){ 37 | return (darch@dropoutMasks[[i+1]]) 38 | } 39 | ) -------------------------------------------------------------------------------- /R/config.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Johannes Rueckert 2 | # 3 | # This file is part of darch. 4 | # 5 | # darch is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # darch is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with darch. If not, see . 17 | 18 | #' @include darch.Class.R 19 | configureDArch <- function(darch) 20 | { 21 | numLayers <- length(getParameter(".layers")) 22 | 23 | # per-layer configuration 24 | for (i in 1:(numLayers - 1)) 25 | { 26 | # Layer functions 27 | darch@layers[[i]][["unitFunction"]] <- 28 | getParameter(".darch.unitFunction")[[i]] 29 | 30 | # Weight update functions 31 | darch@layers[[i]][["weightUpdateFunction"]] <- 32 | getParameter(".darch.weightUpdateFunction")[[i]] 33 | } 34 | 35 | dropout <- getParameter(".darch.dropout") 36 | 37 | # Execute function 38 | if (length(dropout) <= 1 || 39 | all(dropout[2:length(dropout)] == 0)) 40 | { 41 | darch@parameters[[".darch.executeFunction"]] <- runDArch 42 | } 43 | else 44 | { 45 | darch@parameters[[".darch.executeFunction"]] <- runDArchDropout 46 | } 47 | 48 | darch 49 | } -------------------------------------------------------------------------------- /examples/example.xor.R: -------------------------------------------------------------------------------- 1 | # XOR example with logical targets and a custom weight generation function 2 | 3 | example.xor <- function(...) 4 | { 5 | genWeightsExample <- function (numUnits1, numUnits2, ...) 6 | { 7 | generateWeightsGlorotUniform(numUnits1, numUnits2, weights.mean=.1, ...) 8 | } 9 | 10 | # dataset 11 | trainData <- matrix(c(0,0,0,1,1,0,1,1), ncol = 2, byrow = TRUE) 12 | trainTargets <- matrix(c(FALSE,TRUE,TRUE,FALSE), nrow = 4) 13 | 14 | darch <- darch(trainData, trainTargets, 15 | # Note how you can pass deparsed values here, these will be parsed 16 | layers = "c(0,10,0)", 17 | generateWeightsFunction = # multiple weight generation functions 18 | c(genWeightsExample, generateWeightsGlorotNormal), 19 | bp.learnRate = c(2,3), 20 | darch.nesterovMomentum = F, 21 | darch.errorFunction = rmseError, 22 | # Stop when the network classifies all of the training examples correctly 23 | darch.stopClassErr = 0, 24 | darch.returnBestModel = F, # returns the last model and not the best 25 | darch.numEpochs = 1000, 26 | retainData = T, 27 | ... 28 | ) 29 | 30 | # darchTest() can be used to obtain information about the classification 31 | # performance 32 | e <- darchTest(darch) 33 | cat(paste0("Incorrect classifications on all examples: ", e[3], " (", 34 | e[2], "%)\n")) 35 | 36 | # the predict function can be used to get the network output for a new set of 37 | # data, it will even convert the output back to the original character labels 38 | predictions <- predict(darch, type = "bin") 39 | 40 | print(predictions) 41 | 42 | darch 43 | } 44 | -------------------------------------------------------------------------------- /NAMESPACE: -------------------------------------------------------------------------------- 1 | # Generated by roxygen2: do not edit by hand 2 | 3 | S3method(darch,DataSet) 4 | S3method(darch,default) 5 | S3method(darch,formula) 6 | S3method(plot,DArch) 7 | S3method(predict,DArch) 8 | S3method(print,DArch) 9 | export(addLayer) 10 | export(addLayerField) 11 | export(backpropagation) 12 | export(contr.ltfr) 13 | export(crossEntropyError) 14 | export(darch) 15 | export(darchBench) 16 | export(darchModelInfo) 17 | export(darchTest) 18 | export(exponentialLinearUnit) 19 | export(generateWeightsGlorotNormal) 20 | export(generateWeightsGlorotUniform) 21 | export(generateWeightsHeNormal) 22 | export(generateWeightsHeUniform) 23 | export(generateWeightsNormal) 24 | export(generateWeightsUniform) 25 | export(linearUnit) 26 | export(linearUnitRbm) 27 | export(loadDArch) 28 | export(maxoutUnit) 29 | export(maxoutWeightUpdate) 30 | export(minimizeAutoencoder) 31 | export(minimizeClassifier) 32 | export(mseError) 33 | export(provideMNIST) 34 | export(rbmUpdate) 35 | export(readMNIST) 36 | export(rectifiedLinearUnit) 37 | export(rmseError) 38 | export(rpropagation) 39 | export(saveDArch) 40 | export(setDarchParams) 41 | export(setLogLevel) 42 | export(sigmoidUnit) 43 | export(sigmoidUnitRbm) 44 | export(softmaxUnit) 45 | export(softplusUnit) 46 | export(tanhUnit) 47 | export(tanhUnitRbm) 48 | export(weightDecayWeightUpdate) 49 | exportClasses(DArch) 50 | exportClasses(DataSet) 51 | exportClasses(Net) 52 | exportClasses(RBM) 53 | exportMethods(addLayer) 54 | exportMethods(addLayerField) 55 | import(caret) 56 | import(ggplot2) 57 | import(methods) 58 | import(stats) 59 | importFrom(Rcpp,sourceCpp) 60 | importFrom(RcppParallel, RcppParallelLibs) 61 | useDynLib(darch) 62 | -------------------------------------------------------------------------------- /R/saveDArch.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2016 Martin Drees 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | 19 | #' Saves a DArch network 20 | #' 21 | #' Saves the DArch object to the filename given through the parameter \code{name} 22 | #' plus the ending ".net". 23 | #' 24 | #' @details If the parameter \code{saveRBM} is \code{FALSE} the field 25 | #' \code{rbmList} of the DArch object is overwritten by an empty list. 26 | #' 27 | #' @param darch An instance of the class \code{\linkS4class{DArch}}. 28 | #' @param name The name for the file. Default value is "darch". 29 | #' @param trim Whether to trim the model before saving. 30 | #' @seealso \code{\link{loadDArch}} 31 | #' @include darch.Class.R 32 | #' @export 33 | #' @keywords internal 34 | saveDArch <- function(darch, name="darch", trim = F) 35 | { 36 | if (trim) 37 | { 38 | darch@dataSet <- NULL 39 | darch@layers <- list() 40 | } 41 | 42 | save(darch, file = paste(name, ".net", sep = "")) 43 | } 44 | -------------------------------------------------------------------------------- /man/generateWeightsGlorotNormal.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/generateWeightsFunctions.R 3 | \name{generateWeightsGlorotNormal} 4 | \alias{generateWeightsGlorotNormal} 5 | \title{Glorot normal weight initialization} 6 | \usage{ 7 | generateWeightsGlorotNormal(numUnits1, numUnits2, 8 | weights.mean = getParameter(".weights.mean", 0, ...), ...) 9 | } 10 | \arguments{ 11 | \item{numUnits1}{Number of units in the lower layer.} 12 | 13 | \item{numUnits2}{Number of units in the upper layer.} 14 | 15 | \item{weights.mean}{\code{mean} parameter to the \link{rnorm} function.} 16 | 17 | \item{...}{Additional parameters, used for parameter resolution and passed 18 | to \code{\link{generateWeightsNormal}}.} 19 | } 20 | \value{ 21 | Weight matrix. 22 | } 23 | \description{ 24 | This function is used to generate random weights and biases using 25 | Glorot normal weight initialization as described in 26 | Glorot & Bengio, AISTATS 2010. 27 | } 28 | \examples{ 29 | \dontrun{ 30 | data(iris) 31 | model <- darch(Species ~ ., iris, generateWeightsFunction = "generateWeightsGlorotNormal", 32 | weights.mean = .1) 33 | } 34 | } 35 | \references{ 36 | Glorot, Xavier and Yoshua Bengio (2010). "Understanding the 37 | difficulty of training deep feedforward neural networks". 38 | In: International conference on artificial intelligence and statistics, pp. 249-256 39 | } 40 | \seealso{ 41 | Other weight generation functions: \code{\link{generateWeightsGlorotUniform}}, 42 | \code{\link{generateWeightsHeNormal}}, 43 | \code{\link{generateWeightsHeUniform}}, 44 | \code{\link{generateWeightsNormal}}, 45 | \code{\link{generateWeightsUniform}} 46 | } 47 | 48 | -------------------------------------------------------------------------------- /man/generateWeightsHeNormal.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/generateWeightsFunctions.R 3 | \name{generateWeightsHeNormal} 4 | \alias{generateWeightsHeNormal} 5 | \title{He normal weight initialization} 6 | \usage{ 7 | generateWeightsHeNormal(numUnits1, numUnits2, 8 | weights.mean = getParameter(".weights.mean", 0, ...), ...) 9 | } 10 | \arguments{ 11 | \item{numUnits1}{Number of units in the lower layer.} 12 | 13 | \item{numUnits2}{Number of units in the upper layer.} 14 | 15 | \item{weights.mean}{\code{mean} parameter to the \link{rnorm} function.} 16 | 17 | \item{...}{Additional parameters, used for parameter resolution and passed 18 | to \code{\link{generateWeightsNormal}}.} 19 | } 20 | \value{ 21 | Weight matrix. 22 | } 23 | \description{ 24 | This function is used to generate random weights and biases using 25 | He normal weight initialization as described in 26 | He et al., \url{http://arxiv.org/abs/1502.01852}. 27 | } 28 | \examples{ 29 | \dontrun{ 30 | data(iris) 31 | model <- darch(Species ~ ., iris, generateWeightsFunction = "generateWeightsHeNormal", 32 | weights.mean = .1) 33 | } 34 | } 35 | \references{ 36 | He, Kaiming, Xiangyu Zhang, Shaoqing Ren, and Jian Sun (2015). 37 | "Delving Deep into Rectifiers: Surpassing Human-Level Performance on 38 | ImageNet Classification". In: CoRR abs/1502.01852. 39 | URL: http://arxiv.org/abs/1502.01852 40 | } 41 | \seealso{ 42 | Other weight generation functions: \code{\link{generateWeightsGlorotNormal}}, 43 | \code{\link{generateWeightsGlorotUniform}}, 44 | \code{\link{generateWeightsHeUniform}}, 45 | \code{\link{generateWeightsNormal}}, 46 | \code{\link{generateWeightsUniform}} 47 | } 48 | 49 | -------------------------------------------------------------------------------- /man/preTrainDArch.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darch.Learn.R 3 | \name{preTrainDArch} 4 | \alias{preTrainDArch} 5 | \title{Pre-trains a \code{\linkS4class{DArch}} network} 6 | \usage{ 7 | preTrainDArch(darch, dataSet, dataSetValid = NULL, numEpochs = 1, 8 | numCD = 1, lastLayer = 0, isClass = F, consecutive = T, ...) 9 | } 10 | \arguments{ 11 | \item{darch}{A instance of the class \code{\linkS4class{DArch}}.} 12 | 13 | \item{dataSet}{\code{\linkS4class{DataSet}} to be used for training.} 14 | 15 | \item{dataSetValid}{\code{\linkS4class{DataSet}} to be used for validation.} 16 | 17 | \item{numEpochs}{The number of epochs} 18 | 19 | \item{numCD}{The number of CD iterations} 20 | 21 | \item{lastLayer}{Numeric indicating after which layer to stop training.} 22 | 23 | \item{isClass}{Whether to test pre-trained networks against target data} 24 | 25 | \item{consecutive}{Whether to train RBMs consecutively instead of each one 26 | epoch at a time.} 27 | 28 | \item{...}{Additional parameters for the function \code{\link{trainRBM}}} 29 | } 30 | \value{ 31 | Trained \code{\linkS4class{DArch}} instance 32 | } 33 | \description{ 34 | This function pre-trains a \code{\linkS4class{DArch}} network with the 35 | contrastive divergence method 36 | } 37 | \details{ 38 | The function runs for every \code{\linkS4class{RBM}} in the 39 | attribute \code{rbmList} the training function \code{\link{trainRBM}} 40 | copies after the training the weights and biases into the corresponding 41 | layer of the \code{\linkS4class{DArch}} network. 42 | } 43 | \seealso{ 44 | \code{\linkS4class{DArch}}, \code{\linkS4class{RBM}}, 45 | \code{\link{trainRBM}} 46 | } 47 | \keyword{internal} 48 | 49 | -------------------------------------------------------------------------------- /man/rectifiedLinearUnit.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darchUnitFunctions.R 3 | \name{rectifiedLinearUnit} 4 | \alias{rectifiedLinearUnit} 5 | \title{Rectified linear unit function with unit derivatives.} 6 | \usage{ 7 | rectifiedLinearUnit(input, ...) 8 | } 9 | \arguments{ 10 | \item{input}{Input for the activation function.} 11 | 12 | \item{...}{Additional parameters, not used.} 13 | } 14 | \value{ 15 | A list with the rectified linear activation in the first entry and 16 | the derivative of the activation in the second entry. 17 | } 18 | \description{ 19 | The function calculates the activation of the units and returns a list, in 20 | which the first entry is the rectified linear activation of the units and 21 | the second entry is the derivative of the transfer function. 22 | } 23 | \examples{ 24 | \dontrun{ 25 | data(iris) 26 | model <- darch(Species ~ ., iris, darch.unitFunction = "rectifiedLinearUnit") 27 | } 28 | } 29 | \references{ 30 | Glorot, Xavier, Antoine Bordes, and Yoshua Bengio (2011). "Deep 31 | Sparse Rectifier Neural Networks". In: Proceedings of the Fourteenth 32 | International Conference on Artificial Intelligence and Statistics 33 | (AISTATS-11). Ed. by Geoffrey J. Gordon and David B. Dunson. Vol. 15. 34 | Journal of Machine Learning Research - Workshop and Conference Proceedings, 35 | pp. 315-323. 36 | URL : http://www.jmlr.org/proceedings/papers/v15/glorot11a/glorot11a.pdf 37 | } 38 | \seealso{ 39 | Other darch unit functions: \code{\link{exponentialLinearUnit}}, 40 | \code{\link{linearUnit}}, \code{\link{maxoutUnit}}, 41 | \code{\link{sigmoidUnit}}, \code{\link{softmaxUnit}}, 42 | \code{\link{softplusUnit}}, \code{\link{tanhUnit}} 43 | } 44 | 45 | -------------------------------------------------------------------------------- /R/loadDArch.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2016 Martin Drees 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | 19 | #' Loads a DArch network 20 | #' 21 | #' Convenience function for loading \code{\linkS4class{DArch}} instances from 22 | #' disk. There is no special re-initialization logic, so it is not necessary to 23 | #' use this function, you can also just use \code{\link{load}}. 24 | #' 25 | #' Loads the DArch object from the filename given through the parameter \code{name} 26 | #' plus the ending ".net". 27 | #' 28 | #' @details Make sure when you load a DArch object that every file written by 29 | #' the \code{\link{saveDArch}} function is in the working directory. 30 | #' 31 | #' @param name The name of the file without the ending ".net". 32 | #' @return \code{DArch} instance: the loaded deep architecture 33 | #' @seealso \code{\link{saveDArch}} 34 | #' @include darch.Class.R 35 | #' @export 36 | #' @keywords internal 37 | #' @docType methods 38 | #' @rdname loadDArch 39 | loadDArch <- function(name="darch") 40 | { 41 | local(get(load(paste0(name, ".net")))) 42 | } 43 | -------------------------------------------------------------------------------- /examples/example.paramsList.R: -------------------------------------------------------------------------------- 1 | # IRIS example using the paramsList parameter to specify parameters 2 | example.paramsList <- function(...) 3 | { 4 | data(iris) 5 | 6 | paramsList <- list() 7 | paramsList$rbm.numEpochs = 2 8 | paramsList$preProc.params = list("method" = c("scale", "center")) 9 | paramsList$normalizeWeights = T 10 | paramsList$normalizeWeightsBound = 1 11 | paramsList$layers = 20 # one hidden layer with 20 neurons 12 | paramsList$darch.batchSize = 30 13 | paramsList$darch.fineTuneFunction = "rpropagation" 14 | paramsList$darch.unitFunction = c("tanhUnit", "softmaxUnit") 15 | paramsList$darch.stopValidClassErr = 0 16 | paramsList$darch.stopValidErr = .15 17 | paramsList$bootstrap = T 18 | paramsList$bootstrap.unique = F 19 | paramsList$rprop.incFact = 1.3 20 | paramsList$rprop.decFact = .7 21 | paramsList$rprop.initDelta = .1 22 | paramsList$rprop.maxDelta = 5 23 | paramsList$rprop.method = "iRprop-" 24 | paramsList$rprop.minDelta = 1e-5 25 | paramsList$autosave = T 26 | paramsList$autosave.dir = "darch.autosave" 27 | paramsList$autosave.epochs = 10 28 | paramsList$autosave.trim = T 29 | 30 | darch <- darch(Species ~ ., iris, 31 | paramsList = paramsList, 32 | ... 33 | ) 34 | 35 | # The predict function can be used to get the network output for a new set of 36 | # data, it will even convert the output back to the original class labels 37 | predictions <- predict(darch, newdata = iris, type = "class") 38 | 39 | # And these labels can then easily be compared to the correct ones 40 | numIncorrect <- sum(predictions != iris[,5]) 41 | cat(paste0("Incorrect classifications on all examples: ", numIncorrect, " (", 42 | round(numIncorrect/nrow(iris)*100, 2), "%)\n")) 43 | 44 | darch 45 | } 46 | -------------------------------------------------------------------------------- /man/DArch-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darch.Class.R 3 | \docType{class} 4 | \name{DArch-class} 5 | \alias{DArch-class} 6 | \title{Class for deep architectures} 7 | \description{ 8 | This class represents a model created, and/or configured, and/or trained with 9 | the \code{\link{darch}} function. 10 | It implements deep architectures and provides the ability to train 11 | them with a pre training using contrastive divergence and fine tuning with 12 | backpropagation, resilient backpropagation and conjugate gradients. 13 | } 14 | \details{ 15 | The class inherits all attributes from the class \code{\linkS4class{Net}}. 16 | User-relevant slots include \code{stats} (training statistics), \code{epochs} 17 | (numer of epoch this model was trained for), and \code{parameters} (all 18 | parameters passed to \code{\link{darch}} as well as internal parameters). 19 | } 20 | \section{Slots}{ 21 | 22 | \describe{ 23 | \item{\code{rbmList}}{A list which contains all RBMs for the pre-training.} 24 | 25 | \item{\code{layers}}{A list with the layer information. In the first field are the 26 | weights and in the second field is the unit function.} 27 | 28 | \item{\code{cancel}}{Boolean value which indicates if the network training is 29 | canceled.} 30 | 31 | \item{\code{cancelMessage}}{The message when the execution is canceled.} 32 | 33 | \item{\code{dropoutMasks}}{List of dropout masks, used internally.} 34 | 35 | \item{\code{dataSet}}{\linkS4class{DataSet} instance.} 36 | }} 37 | \author{ 38 | Martin Drees 39 | } 40 | \seealso{ 41 | \code{\linkS4class{Net}}, \code{\linkS4class{RBM}} 42 | 43 | Other darch classes: \code{\link{DataSet-class}}, 44 | \code{\link{Net-class}}, \code{\link{RBM-class}} 45 | } 46 | \keyword{internal} 47 | 48 | -------------------------------------------------------------------------------- /R/darch.Setter.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2016 Martin Drees 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | 19 | #' Set the dropout mask for the given layer. 20 | #' 21 | #' Sets the dropout mask to be applied to the weights between layer i and i+1, 22 | #' for 0 < i < numLayers. For i = 0, sets the dropout mask for the input layer, 23 | #' which will be applied to the initiali input data. 24 | #' 25 | #' @param darch A instance of the class \code{\linkS4class{DArch}}. 26 | #' @param i Layer index or 0 for input layer. 27 | #' @param value Dropout mask for the given layer. 28 | #' @usage setDropoutMask(darch, i) <- value 29 | #' @return The darch with the updated dropout mask 30 | #' @seealso \code{\linkS4class{DArch}} 31 | #' @keywords internal 32 | #' @include darch.Class.R 33 | setGeneric("setDropoutMask<-",function(darch, i, value) 34 | {standardGeneric("setDropoutMask<-")}) 35 | 36 | setReplaceMethod( 37 | f = "setDropoutMask", 38 | signature = "DArch", 39 | definition = function(darch, i, value) 40 | { 41 | darch@dropoutMasks[[i + 1]] <- value 42 | darch 43 | } 44 | ) 45 | -------------------------------------------------------------------------------- /R/makeStartEndPoints.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2016 Martin Drees 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | 19 | # TODO replace with something simpler / more efficient 20 | 21 | #' Makes start- and end-points for the batches. 22 | #' 23 | #' The start- and end-points are used for dividing the data into batches. 24 | #' 25 | #' @details If the data is not divisible by the \code{batchSize} the last batch 26 | #' will contain the rest of the data. 27 | #' The function returns a list with in which the first entry is a list with the 28 | #' values for the start and end points for reading the data matrix. The second 29 | #' entry is the number of batches. 30 | #' 31 | #' @param batchSize Desired batch size 32 | #' @param numRows Number of rows of the data 33 | #' @keywords internal 34 | makeStartEndPoints <- function(batchSize,numRows) 35 | { 36 | numBatches <- ceiling(numRows / batchSize) 37 | batchValues <- list() 38 | batchValues[[1]] <- 0 39 | for(n in 2:(numBatches)) 40 | { 41 | batchValues[[n]] <- (n - 1) * batchSize 42 | } 43 | 44 | batchValues[[length(batchValues) + 1]] <- numRows 45 | 46 | return(list(batchValues, numBatches)) 47 | } 48 | -------------------------------------------------------------------------------- /man/RBM.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/rbm.Class.R 3 | \docType{class} 4 | \name{RBM-class} 5 | \alias{RBM-class} 6 | \title{Class for restricted Boltzmann machines} 7 | \description{ 8 | This class represents a restricted Boltzmann machine. 9 | } 10 | \details{ 11 | The RBM can be trained with the implementation of the 12 | contrastive divergence method \code{\link{trainRBM}}. The class inherits the 13 | attributes from the \code{\linkS4class{Net}}. 14 | } 15 | \section{Slots}{ 16 | 17 | \describe{ 18 | \item{\code{weights}}{Object of class \code{"matrix"}. Weight matrix.} 19 | 20 | \item{\code{weightsInc}}{Object of class \code{"matrix"}. Matrix of update values for 21 | the Weight.} 22 | 23 | \item{\code{visibleBiases}}{Object of class \code{"array"}. Visible biases array.} 24 | 25 | \item{\code{visibleBiasesInc}}{Object of class \code{"array"}. Array of update values 26 | for the visible biases} 27 | 28 | \item{\code{visibleUnitStates}}{Object of class \code{"list"}. States of the visible 29 | units.} 30 | 31 | \item{\code{hiddenBiases}}{Object of class \code{"array"}. Hidden biases array.} 32 | 33 | \item{\code{hiddenBiasesInc}}{Object of class \code{"array"}. Array of update values 34 | for the hidden biases.} 35 | 36 | \item{\code{hiddenUnitStates}}{Object of class \code{"list"}. States of the hidden 37 | units.} 38 | 39 | \item{\code{posPhaseData}}{Object of class \code{"list"}. Attribute to save the 40 | positive phase data during the training.} 41 | 42 | \item{\code{output}}{Object of class \code{"matrix"}. Output matrix of the RBM.} 43 | }} 44 | \author{ 45 | Martin Drees 46 | } 47 | \seealso{ 48 | \code{\linkS4class{Net}}, \code{\linkS4class{DArch}}, 49 | \code{\link{trainRBM}} 50 | 51 | Other darch classes: \code{\link{DArch-class}}, 52 | \code{\link{DataSet-class}}, \code{\link{Net-class}} 53 | } 54 | \keyword{internal} 55 | 56 | -------------------------------------------------------------------------------- /src/softplusUnit.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | */ 19 | 20 | #include 21 | #include 22 | #include "unitFunction.h" 23 | 24 | using namespace RcppParallel; 25 | using namespace Rcpp; 26 | 27 | struct SoftplusUnit : UnitFunction { 28 | 29 | SoftplusUnit(const NumericMatrix input, NumericMatrix activations, 30 | NumericMatrix derivatives) : UnitFunction(input, activations, derivatives) 31 | {} 32 | 33 | void operator()(std::size_t begin_col, std::size_t end_col) 34 | { 35 | for (int i = begin_col; i < end_col; i++) 36 | { 37 | for (int j = 0; j < nrowInput; j++) 38 | { 39 | activations(j, i) = std::log(1 + std::exp(input(j, i))); 40 | derivatives(j, i) = 1/(1 + std::exp(-input(j, i))); 41 | } 42 | } 43 | } 44 | }; 45 | 46 | // [[Rcpp::export]] 47 | List softplusUnitCpp(NumericMatrix input) 48 | { 49 | int nrows = input.nrow(); 50 | int ncols = input.ncol(); 51 | NumericMatrix activations = clone(input); 52 | NumericMatrix derivatives = NumericMatrix(Dimension(nrows, ncols)); 53 | 54 | SoftplusUnit worker(input, activations, derivatives); 55 | 56 | parallelFor(0, ncols, worker); 57 | 58 | return List::create(activations, derivatives); 59 | } 60 | -------------------------------------------------------------------------------- /src/sigmoidUnitFunction.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | */ 19 | 20 | #include 21 | #include 22 | #include "unitFunction.h" 23 | 24 | using namespace RcppParallel; 25 | using namespace Rcpp; 26 | 27 | struct SigmoidUnit : UnitFunction 28 | { 29 | SigmoidUnit(const NumericMatrix input, NumericMatrix activations, 30 | NumericMatrix derivatives) : UnitFunction(input, activations, derivatives) 31 | {} 32 | 33 | void operator()(std::size_t begin_col, std::size_t end_col) 34 | { 35 | for (int i = begin_col; i < end_col; i++) 36 | { 37 | for (int j = 0; j < nrowInput; j++) 38 | { 39 | activations(j, i) = 1/(1 + std::exp(-input(j, i))); 40 | derivatives(j, i) = activations(j, i) * (1 - activations(j, i)); 41 | } 42 | } 43 | } 44 | }; 45 | 46 | // [[Rcpp::export]] 47 | List sigmoidUnitCpp(NumericMatrix input) 48 | { 49 | int nrows = input.nrow(); 50 | int ncols = input.ncol(); 51 | NumericMatrix activations(Dimension(nrows, ncols)); 52 | NumericMatrix derivatives(Dimension(nrows, ncols)); 53 | 54 | SigmoidUnit worker(input, activations, derivatives); 55 | 56 | parallelFor(0, ncols, worker); 57 | 58 | return List::create(activations, derivatives); 59 | } 60 | -------------------------------------------------------------------------------- /man/fineTuneDArch-DArch-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darch.Learn.R 3 | \docType{methods} 4 | \name{fineTuneDArch,DArch-method} 5 | \alias{fineTuneDArch,DArch-method} 6 | \title{Fine tuning function for the deep architecture} 7 | \usage{ 8 | \S4method{fineTuneDArch}{DArch}(darch, dataSet, dataSetValid = NULL, 9 | numEpochs = 1, isClass = TRUE, stopErr = -Inf, stopClassErr = 101, 10 | stopValidErr = -Inf, stopValidClassErr = 101, 11 | shuffleTrainData = getParameter(".shuffleTrainData", T), 12 | debugMode = getParameter(".debug", F), ...) 13 | } 14 | \arguments{ 15 | \item{darch}{A instance of the class \code{\linkS4class{DArch}}.} 16 | 17 | \item{dataSet}{\code{\linkS4class{DataSet}} containing training and 18 | optionally validation and test data.} 19 | 20 | \item{dataSetValid}{\code{\linkS4class{DataSet}} to be used for validation.} 21 | 22 | \item{numEpochs}{The number of training iterations} 23 | 24 | \item{isClass}{Indicates whether the training is for a classification net. 25 | When \code{TRUE} then statistics for classification will be determind. 26 | Default is \code{TRUE}} 27 | 28 | \item{stopErr}{Stop criteria for the error on the train data. Default is 29 | \code{-Inf}} 30 | 31 | \item{stopClassErr}{Stop criteria for the classification error on the train 32 | data. Default is \code{101}} 33 | 34 | \item{stopValidErr}{Stop criteria for the error on the validation data. 35 | Default is \code{-Inf}.} 36 | 37 | \item{stopValidClassErr}{Stop criteria for the classification error on the 38 | validation data. Default is \code{101}.} 39 | 40 | \item{shuffleTrainData}{Whether to shuffle train data before each epoch.} 41 | 42 | \item{debugMode}{Whether to enable debug mode, internal parameter.} 43 | 44 | \item{...}{Additional parameters for the training function} 45 | } 46 | \description{ 47 | Fine tuning function for the deep architecture 48 | } 49 | \seealso{ 50 | \link{fineTuneDArch} 51 | } 52 | \keyword{internal} 53 | 54 | -------------------------------------------------------------------------------- /R/bootstrap.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Johannes Rueckert 2 | # 3 | # This file is part of darch. 4 | # 5 | # darch is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # darch is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with darch. If not, see . 17 | 18 | bootstrapDataSet <- function(dataSet, unique, num = 0) 19 | { 20 | numRows <- nrow(dataSet@data) 21 | bootstrapTrainingSamples <- sample(1:numRows, 22 | if (num > 0) num else numRows, 23 | replace = if (num > 0) F else T) 24 | 25 | if (unique && num == 0) 26 | { 27 | bootstrapTrainingSamples <- unique(bootstrapTrainingSamples) 28 | } 29 | 30 | numTrain <- length(bootstrapTrainingSamples) 31 | bootstrapValidationSamples <- 32 | which(!(1:numRows %in% bootstrapTrainingSamples)) 33 | numValid <- length(bootstrapValidationSamples) 34 | 35 | # TODO validate sizes? 36 | dataSetValid <- dataSet 37 | dataSet@data <- dataSet@data[bootstrapTrainingSamples,, drop = F] 38 | dataSet@targets <- dataSet@targets[bootstrapTrainingSamples,, drop = F] 39 | dataSetValid@data <- dataSetValid@data[bootstrapValidationSamples,, drop = F] 40 | dataSetValid@targets <- 41 | dataSetValid@targets[bootstrapValidationSamples,, drop = F] 42 | 43 | futile.logger::flog.info(paste( 44 | "Bootstrapping is started with %s samples, bootstrapping results in", 45 | "%s training (%s unique) and %s validation samples for this run."), 46 | numRows, numTrain, numRows - numValid, numValid) 47 | 48 | return(list(dataSet, dataSetValid)) 49 | } 50 | -------------------------------------------------------------------------------- /src/normalizeWeights.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | */ 19 | 20 | #include 21 | #include 22 | #include "helpers.h" 23 | 24 | using namespace RcppParallel; 25 | using namespace Rcpp; 26 | 27 | struct NormalizeWeights : public Worker 28 | { 29 | RMatrix weights; 30 | 31 | int colLength; 32 | 33 | float bound; 34 | 35 | NormalizeWeights(NumericMatrix weights, float bound) 36 | : weights(weights), bound(bound) { 37 | colLength = weights.nrow(); 38 | } 39 | 40 | void operator()(std::size_t begin_col, std::size_t end_col) 41 | { 42 | double colSum; 43 | 44 | for (int i = begin_col; i < end_col; i++) 45 | { 46 | colSum = normalizeWeightsSum(weights.column(i)); 47 | 48 | // Normalize after comparing square sum to bound 49 | if (colSum > bound) 50 | { 51 | for (int j = 0; j < colLength; j++) 52 | { 53 | weights(j, i) = weights(j, i) / colSum * bound; 54 | } 55 | } 56 | } 57 | 58 | } 59 | }; 60 | 61 | // edits in-place 62 | // [[Rcpp::export]] 63 | void normalizeWeightsCpp(NumericMatrix weights, float bound) 64 | { 65 | int ncols = weights.ncol(); 66 | NormalizeWeights worker(weights, bound); 67 | 68 | parallelFor(0, ncols, worker); 69 | } 70 | -------------------------------------------------------------------------------- /man/trainRBM.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/rbm.Learn.R 3 | \name{trainRBM} 4 | \alias{trainRBM} 5 | \title{Trains an \code{\linkS4class{RBM}} with contrastive divergence} 6 | \usage{ 7 | trainRBM(rbm, trainData, numEpochs = 1, numCD = 1, shuffleTrainData = T, 8 | ...) 9 | } 10 | \arguments{ 11 | \item{rbm}{A instance of the class \code{\linkS4class{RBM}}.} 12 | 13 | \item{trainData}{The data matrix for the training} 14 | 15 | \item{numEpochs}{The number of training iterations} 16 | 17 | \item{numCD}{Number of contrastive divergence iterations} 18 | 19 | \item{shuffleTrainData}{Whether to shuffle the training data prior to each 20 | epoch.} 21 | 22 | \item{...}{Additional parameters for the unit functions} 23 | } 24 | \description{ 25 | The function trains a restricted Boltzmann machine (\code{\linkS4class{RBM}}) with 26 | the contrastive divergence method. 27 | } 28 | \details{ 29 | This function is build on the basis of the code from G. Hinton et. 30 | al. (http://www.cs.toronto.edu/~hinton/MatlabForSciencePaper.html - last 31 | visit 2016-04-30) for the pre training of deep belief nets. The original 32 | code is located in the files 'rbm.m' and 'rbmhidlinear.m'. It iterates in 33 | every epoche over the batches and calculates the updates for the weights. 34 | If it is the first CD iteration or the CD iterations are finished, the 35 | hidden units are calculated with the real value activations of the visible 36 | units, otherwise with the binary activations. To tell the unit functions 37 | the actual state of the training, the function generates a array with the 38 | following running parameters and passes them to the units: Number of 39 | epochs: "numEpochs", current epochs: "currentEpoch", Number of batches: 40 | "numBatches", current batch: "currentBatch", Maximal CD iterations: 41 | "numCD", current CD iteration: "currentCD", CD is finished: "finishCD". 42 | } 43 | \seealso{ 44 | \code{\linkS4class{RBM}} 45 | } 46 | \keyword{internal} 47 | 48 | -------------------------------------------------------------------------------- /src/dither.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | */ 19 | 20 | #include 21 | #include 22 | #include "helpers.h" 23 | 24 | using namespace RcppParallel; 25 | using namespace Rcpp; 26 | 27 | struct Dither : public Worker 28 | { 29 | RMatrix data; 30 | 31 | const RVector columnMask; 32 | 33 | Dither(NumericMatrix data, const NumericVector columnMask) : 34 | data(data), columnMask(columnMask) 35 | {} 36 | 37 | void operator()(std::size_t begin_col, std::size_t end_col) 38 | { 39 | int nrow = data.nrow(); 40 | double sdColumn, variance; 41 | 42 | for (int i = begin_col; i < end_col; i++) 43 | { 44 | if (columnMask[i] == 1) 45 | { 46 | sdColumn = cppSD(data.column(i)); 47 | variance = sdColumn * sdColumn; 48 | 49 | for (int j = 0; j < nrow; j++) 50 | { 51 | data(j, i) += R::runif(-variance, variance); 52 | } 53 | } 54 | } 55 | } 56 | }; 57 | 58 | // edits in-place 59 | // [[Rcpp::export]] 60 | NumericMatrix ditherCpp(NumericMatrix data, NumericVector columnMask) 61 | { 62 | NumericMatrix output = clone(data); 63 | int ncols = data.ncol(); 64 | Dither worker(output, columnMask); 65 | 66 | parallelFor(0, ncols, worker); 67 | 68 | return output; 69 | } 70 | -------------------------------------------------------------------------------- /R/dropout.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2016 Martin Drees 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | 19 | #' @include darch.Class.R 20 | NULL 21 | 22 | #' Dropout mask generator function. 23 | #' 24 | #' This function generates droput masks as a vector of given length, according 25 | #' to the given dropout rate. 26 | #' 27 | #' @param length length of the dropout mask 28 | #' @param dropoutRate The dropout rate (i.e. percentage of zeros in the mask) 29 | #' @return Matrix containing the dropout mask 30 | #' 31 | #' @keywords internal 32 | generateDropoutMask <- function(length, dropoutRate) 33 | { 34 | sample(0:1, length, replace = T, 35 | prob = c(dropoutRate, 1 - dropoutRate)) 36 | } 37 | 38 | generateDropoutMasksForDarch <- function(darch) 39 | { 40 | numLayers <- length(darch@layers) 41 | dropConnect <- getParameter(".darch.dropout.dropConnect") 42 | dropout <- getParameter(".darch.dropout") 43 | 44 | # generate dropout masks 45 | setDropoutMask(darch, 0) <- 46 | generateDropoutMask(nrow(darch@layers[[1]][["weights"]]) - 1, dropout[1]) 47 | for (i in 1:(numLayers - !dropConnect)) 48 | { 49 | weights <- darch@layers[[i + !dropConnect]][["weights"]] 50 | length <- if (dropConnect) length(weights) else nrow(weights) - 1 51 | setDropoutMask(darch, i) <- 52 | generateDropoutMask(length, dropout[i + 1]) 53 | } 54 | 55 | darch 56 | } -------------------------------------------------------------------------------- /src/applyDropoutMask.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | */ 19 | 20 | #include 21 | #include 22 | 23 | using namespace RcppParallel; 24 | using namespace Rcpp; 25 | 26 | struct ApplyDropoutMask : public Worker 27 | { 28 | const RMatrix input; 29 | 30 | int colLength; 31 | 32 | RMatrix output; 33 | 34 | RVector mask; 35 | 36 | ApplyDropoutMask(const NumericMatrix input, NumericMatrix output, NumericVector mask) 37 | : input(input), output(output), mask(mask) { 38 | colLength = input.nrow(); 39 | } 40 | 41 | void operator()(std::size_t begin_col, std::size_t end_col) 42 | { 43 | for (int i = begin_col; i < end_col; i++) 44 | { 45 | RMatrix::Column colOutput = output.column(i); 46 | RMatrix::Column colInput = input.column(i); 47 | for (int j = 0; j < colLength; j++) 48 | { 49 | colOutput[j] = colInput[j] * mask[i]; 50 | } 51 | } 52 | 53 | } 54 | }; 55 | 56 | // does not edit in-place 57 | // [[Rcpp::export]] 58 | NumericMatrix applyDropoutMaskCpp(NumericMatrix data, NumericVector mask) 59 | { 60 | NumericMatrix cData(clone(data)); 61 | int ncols = cData.ncol(); 62 | 63 | ApplyDropoutMask worker(data, cData, mask); 64 | 65 | parallelFor(0, ncols, worker); 66 | 67 | return cData; 68 | } 69 | -------------------------------------------------------------------------------- /man/maxoutWeightUpdate.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/weightUpdateFunctions.R 3 | \name{maxoutWeightUpdate} 4 | \alias{maxoutWeightUpdate} 5 | \title{Updates the weight on maxout layers} 6 | \usage{ 7 | maxoutWeightUpdate(darch, layerIndex, weightsInc, biasesInc, ..., 8 | weightDecay = getParameter(".darch.weightDecay", 0, darch), 9 | poolSize = getParameter(".darch.maxout.poolSize", 2, darch)) 10 | } 11 | \arguments{ 12 | \item{darch}{\linkS4class{DArch} instance.} 13 | 14 | \item{layerIndex}{Layer index within the network.} 15 | 16 | \item{weightsInc}{Matrix containing scheduled weight updates from the 17 | fine-tuning algorithm.} 18 | 19 | \item{biasesInc}{Bias weight updates.} 20 | 21 | \item{...}{Additional parameters, not used.} 22 | 23 | \item{weightDecay}{Weights are multiplied by (1 - \code{weightDecay}) before 24 | each update. Corresponds to the \code{darch.weightDecay} parameter of 25 | \link{darch.default}.} 26 | 27 | \item{poolSize}{Size of maxout pools, see parameter 28 | \code{darch.maxout.poolSize} of \code{\link{darch}}.} 29 | } 30 | \value{ 31 | The updated weights. 32 | } 33 | \description{ 34 | On maxout layers, only the weights of active units are altered, additionally 35 | all weights within a pool must be the same. 36 | } 37 | \examples{ 38 | \dontrun{ 39 | data(iris) 40 | model <- darch(Species ~ ., iris, c(0, 50, 0), 41 | darch.unitFunction = c("maxoutUnit", "softmaxUnit"), 42 | darch.maxout.poolSize = 5, darch.maxout.unitFunction = "sigmoidUnit", 43 | darch.weightUpdateFunction = c("weightDecayWeightUpdate", "maxoutWeightUpdate")) 44 | } 45 | } 46 | \references{ 47 | Goodfellow, Ian J., David Warde-Farley, Mehdi Mirza, Aaron C. Courville, 48 | and Yoshua Bengio (2013). "Maxout Networks". In: Proceedings of the 30th 49 | International Conference on Machine Learning, ICML 2013, Atlanta, GA, USA, 50 | 16-21 June 2013, pp. 1319-1327. 51 | URL: http://jmlr.org/proceedings/papers/v28/goodfellow13.html 52 | } 53 | \seealso{ 54 | Other weight update functions: \code{\link{weightDecayWeightUpdate}} 55 | } 56 | 57 | -------------------------------------------------------------------------------- /R/newDArch.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2016 Martin Drees 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | 19 | #' Constructor function for \code{\linkS4class{DArch}} objects. 20 | #' 21 | #' Generate a new \code{\linkS4class{DArch}} object with the given parameters. 22 | #' 23 | #' @details 24 | #' This function is used internally only, please call \code{\link{darch}} 25 | #' to create a new \code{\linkS4class{DArch}} instance. 26 | #' 27 | #' @param params Additional parameters as a list, also stored in 28 | #' \code{darch@parameters}. 29 | #' @return The new DArch object 30 | #' @include darch.Class.R 31 | #' @include darch.Setter.R 32 | #' @keywords internal 33 | newDArch <- function(params) 34 | { 35 | darch <- new("DArch") 36 | darch@parameters <- params 37 | layers <- getParameter(".layers") 38 | futile.logger::flog.info("Constructing a network with %s layers (%s neurons).", 39 | length(layers), paste(layers, collapse = ', ')) 40 | darch@stats <- 41 | list("trainErrors" = list("raw" = c(), "class" = c()), 42 | "validErrors" = list("raw" = c(), "class" = c()), 43 | "dot632Errors" = list("raw" = c(), "class" = c()), 44 | "times" = c(), "preTrainTime" = 0, "fineTuneTime" = 0, "validTime" = 0, 45 | "numPatterns" = list("train" = 0, "valid" = 0)) 46 | darch <- generateRBMs(darch) 47 | darch <- configureDArch(darch) 48 | darch 49 | } 50 | -------------------------------------------------------------------------------- /R/rbm.Reset.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2016 Martin Drees 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | 19 | #' Resets the weights and biases of the \code{RBM} object 20 | #' 21 | #' This function resets the weights and biases of the \code{\linkS4class{RBM}} 22 | #' object. 23 | #' 24 | #' @param rbm An instance of class \code{\linkS4class{RBM}}. 25 | #' @param ... Additional arguments. 26 | #' 27 | #' @seealso \code{\linkS4class{RBM}} 28 | #' @keywords internal 29 | #' @include rbm.Class.R 30 | setGeneric( 31 | name = "resetRBM", 32 | def = function(rbm, ...){standardGeneric("resetRBM")} 33 | ) 34 | 35 | setMethod( 36 | f = "resetRBM", 37 | signature = "RBM", 38 | definition = function(rbm, ...) 39 | { 40 | numVisible <- getParameter(".numVisible", net = rbm) 41 | numHidden <- getParameter(".numHidden", net = rbm) 42 | genWeightFunction <- getParameter(".generateWeightsFunction", net = rbm) 43 | 44 | rbm@weights <- genWeightFunction(numVisible, numHidden, ..., net = rbm) 45 | rbm@hiddenBiases <- genWeightFunction(1, numHidden, ..., net = rbm) 46 | rbm@visibleBiases <- genWeightFunction(1, numVisible, ..., net = rbm) 47 | 48 | rbm@weightsInc <- matrix(0, numVisible, numHidden) 49 | rbm@hiddenBiasesInc <- matrix(0, 1, numHidden) 50 | rbm@visibleBiasesInc <- matrix(0, 1, numVisible) 51 | rbm@stats <- list() 52 | 53 | rbm 54 | } 55 | ) -------------------------------------------------------------------------------- /src/rectifiedLinearUnit.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | */ 19 | 20 | #include 21 | #include 22 | #include "unitFunction.h" 23 | 24 | using namespace RcppParallel; 25 | using namespace Rcpp; 26 | 27 | struct RectifiedLinearUnit : UnitFunction { 28 | 29 | RectifiedLinearUnit(const NumericMatrix input, NumericMatrix activations, 30 | NumericMatrix derivatives) : UnitFunction(input, activations, derivatives) 31 | {} 32 | 33 | void operator()(std::size_t begin_col, std::size_t end_col) 34 | { 35 | for (int i = begin_col; i < end_col; i++) 36 | { 37 | for (int j = 0; j < nrowInput; j++) 38 | { 39 | if (input(j, i) <= 0) 40 | { 41 | activations(j, i) = 0; 42 | derivatives(j, i) = 0; 43 | } 44 | else 45 | { 46 | derivatives(j, i) = 1; 47 | } 48 | } 49 | } 50 | } 51 | }; 52 | 53 | // [[Rcpp::export]] 54 | List rectifiedLinearUnitCpp(NumericMatrix input) 55 | { 56 | int nrows = input.nrow(); 57 | int ncols = input.ncol(); 58 | NumericMatrix activations = clone(input); 59 | NumericMatrix derivatives = NumericMatrix(Dimension(nrows, ncols)); 60 | 61 | RectifiedLinearUnit worker(input, activations, derivatives); 62 | 63 | parallelFor(0, ncols, worker); 64 | 65 | return List::create(activations, derivatives); 66 | } 67 | -------------------------------------------------------------------------------- /man/darchModelInfo.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/caret.R 3 | \name{darchModelInfo} 4 | \alias{darchModelInfo} 5 | \title{Creates a custom caret model for \code{darch}.} 6 | \usage{ 7 | darchModelInfo(params = NULL, grid = NULL) 8 | } 9 | \arguments{ 10 | \item{params}{\code{\link{data.frame}} of parameters or \code{NULL} to use a 11 | simple default (bp.learnRate).} 12 | 13 | \item{grid}{Function which procuces a \code{\link{data.frame}} containing a 14 | grid of parameter combinations or \code{NULL} to use a simple default.} 15 | } 16 | \value{ 17 | A valid \code{caret} model which can be passed to 18 | \code{\link[caret]{train}}. 19 | } 20 | \description{ 21 | This function creates a \code{caret} model description to enable training 22 | \code{DArch} instances with the \code{\link[caret]{train}} function. See the 23 | documentation on custom caret models for further information and examples on 24 | how to create valid \code{params} and \code{grid} values. 25 | } 26 | \examples{ 27 | \dontrun{ 28 | data(iris) 29 | tc <- trainControl(method = "boot", number = 5, allowParallel = F, 30 | verboseIter = T) 31 | 32 | parameters <- data.frame(parameter = c("layers", "bp.learnRate", "darch.unitFunction"), 33 | class = c("character", "numeric", "character"), 34 | label = c("Network structure", "Learning rate", "unitFunction")) 35 | 36 | grid <- function(x, y, len = NULL, search = "grid") 37 | { 38 | df <- expand.grid(layers = c("c(0,20,0)","c(0,10,10,0)","c(0,10,5,5,0)"), 39 | bp.learnRate = c(1,2,5,10)) 40 | 41 | df[["darch.unitFunction"]] <- rep(c("c(tanhUnit, softmaxUnit)", 42 | "c(tanhUnit, tanhUnit, softmaxUnit)", 43 | "c(tanhUnit, tanhUnit, tanhUnit, softmaxUnit)"), 4) 44 | 45 | df 46 | } 47 | 48 | caretModel <- train(Species ~ ., data = iris, tuneLength = 12, trControl = tc, 49 | method = darchModelInfo(parameters, grid), preProc = c("center", "scale"), 50 | darch.numEpochs = 15, darch.batchSize = 6, testing = T, ...) 51 | } 52 | } 53 | \seealso{ 54 | \href{https://topepo.github.io/caret/custom_models.html}{Caret custom models} 55 | } 56 | 57 | -------------------------------------------------------------------------------- /examples/example.mnist.R: -------------------------------------------------------------------------------- 1 | # MNIST example with pre-training 2 | example.mnist <- function(dataFolder = "data/", downloadMNIST = F, ...) 3 | { 4 | # Make sure to prove the correct folder if you have already downloaded the 5 | # MNIST data somewhere, or otherwise set downloadMNIST to TRUE 6 | provideMNIST(dataFolder, downloadMNIST) 7 | 8 | # Load MNIST data 9 | load(paste0(dataFolder, "train.RData")) # trainData, trainLabels 10 | load(paste0(dataFolder, "test.RData")) # testData, testLabels 11 | 12 | # only take 1000 samples, otherwise training takes increasingly long 13 | chosenRowsTrain <- sample(1:nrow(trainData), size=1000) 14 | trainDataSmall <- trainData[chosenRowsTrain,] 15 | trainLabelsSmall <- trainLabels[chosenRowsTrain,] 16 | 17 | darch <- darch(trainDataSmall, trainLabelsSmall, 18 | rbm.numEpochs = 5, 19 | rbm.consecutive = F, # each RBM is trained one epoch at a time 20 | rbm.batchSize = 100, 21 | rbm.lastLayer = -1, # don't train output layer 22 | rbm.allData = T, # use bootstrap validation data as well for training 23 | rbm.errorFunction = rmseError, 24 | rbm.initialMomentum = .5, 25 | rbm.finalMomentum = .7, 26 | rbm.learnRate = .1, 27 | rbm.learnRateScale = .98, 28 | rbm.momentumRampLength = .8, 29 | rbm.numCD = 2, 30 | rbm.unitFunction = sigmoidUnitRbm, 31 | rbm.weightDecay = .001, 32 | layers = c(784,100,10), 33 | darch.batchSize = 100, 34 | darch.dither = T, 35 | darch.initialMomentum = .4, 36 | darch.finalMomentum = .9, 37 | darch.momentumRampLength = .75, 38 | bp.learnRate = 1, 39 | bp.learnRateScale = .99, 40 | darch.unitFunction = c(tanhUnit, softmaxUnit), 41 | bootstrap = T, 42 | darch.numEpochs = 20, 43 | gputools = T, # try to use gputools 44 | gputools.deviceId = 0, 45 | ... 46 | ) 47 | 48 | predictions <- predict(darch, newdata=testData, type="class") 49 | 50 | labels <- cbind(predictions, testLabels) 51 | numIncorrect <- sum(apply(labels, 1, function(i) { any(i[1:10] != i[11:20]) })) 52 | cat(paste0("Incorrect classifications on test data: ", numIncorrect, 53 | " (", round(numIncorrect/nrow(testLabels)*100, 2), "%)\n")) 54 | 55 | darch 56 | } 57 | -------------------------------------------------------------------------------- /man/predict.DArch.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/predict.R 3 | \name{predict.DArch} 4 | \alias{predict.DArch} 5 | \title{Forward-propagate data.} 6 | \usage{ 7 | \method{predict}{DArch}(object, ..., newdata = NULL, type = "raw", 8 | inputLayer = 1, outputLayer = 0) 9 | } 10 | \arguments{ 11 | \item{object}{\code{\linkS4class{DArch}} instance} 12 | 13 | \item{...}{Further parameters, if \code{newdata} is \code{NULL}, the first 14 | unnamed parameter will be used for \code{newdata} instead.} 15 | 16 | \item{newdata}{New data to predict, \code{NULL} to return latest network 17 | output} 18 | 19 | \item{type}{Output type, one of: \code{raw}, \code{bin}, \code{class}, or 20 | \code{character}. \code{raw} returns the layer output, \code{bin} returns 21 | \code{1} for every layer output \code{>0.5}, \code{0} otherwise, and 22 | \code{class} returns \code{1} for the output unit with the highest 23 | activation, otherwise \code{0}. Additionally, when using \code{class}, 24 | class labels are returned when available. \code{character} is the same as 25 | \code{class}, except using character vectors instead of factors.} 26 | 27 | \item{inputLayer}{Layer number (\code{> 0}). The data given in 28 | \code{newdata} will be fed into this layer. 29 | Note that absolute numbers count from the input layer, i.e. for 30 | a network with three layers, \code{1} would indicate the input layer.} 31 | 32 | \item{outputLayer}{Layer number (if \code{> 0}) or offset (if \code{<= 0}) 33 | relative to the last layer. The output of the given layer is returned. 34 | Note that absolute numbers count from the input layer, i.e. for 35 | a network with three layers, \code{1} would indicate the input layer.} 36 | } 37 | \value{ 38 | Vector or matrix of networks outputs, output type depending on the 39 | \code{type} parameter. 40 | } 41 | \description{ 42 | Forward-propagate given data through the deep neural network. 43 | } 44 | \examples{ 45 | \dontrun{ 46 | data(iris) 47 | model <- darch(Species ~ ., iris, retainData = T) 48 | predict(model) 49 | } 50 | } 51 | \seealso{ 52 | Other darch interface functions: \code{\link{darchBench}}, 53 | \code{\link{darchTest}}, \code{\link{darch}}, 54 | \code{\link{plot.DArch}}, \code{\link{print.DArch}} 55 | } 56 | 57 | -------------------------------------------------------------------------------- /src/exponentialLinearUnit.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | */ 19 | 20 | #include 21 | #include 22 | #include "unitFunction.h" 23 | 24 | using namespace RcppParallel; 25 | using namespace Rcpp; 26 | 27 | struct ExponentialLinearUnit : UnitFunction { 28 | 29 | const double alpha; 30 | 31 | ExponentialLinearUnit(const NumericMatrix input, NumericMatrix activations, 32 | NumericMatrix derivatives, const double alpha) : 33 | UnitFunction(input, activations, derivatives), 34 | alpha(alpha) 35 | {} 36 | 37 | void operator()(std::size_t begin_col, std::size_t end_col) 38 | { 39 | for (int i = begin_col; i < end_col; i++) 40 | { 41 | for (int j = 0; j < nrowInput; j++) 42 | { 43 | if (input(j, i) <= 0) 44 | { 45 | activations(j, i) = alpha * (std::exp(input(j, i)) - 1); 46 | derivatives(j, i) = activations(j, i) + alpha; 47 | } 48 | else 49 | { 50 | activations(j, i) = input(j, i); 51 | derivatives(j, i) = 1; 52 | } 53 | } 54 | } 55 | } 56 | }; 57 | 58 | // [[Rcpp::export]] 59 | List exponentialLinearUnitCpp(NumericMatrix input, double alpha) 60 | { 61 | int nrows = input.nrow(); 62 | int ncols = input.ncol(); 63 | NumericMatrix activations = clone(input); 64 | NumericMatrix derivatives = NumericMatrix(Dimension(nrows, ncols)); 65 | 66 | ExponentialLinearUnit worker(input, activations, derivatives, alpha); 67 | 68 | parallelFor(0, ncols, worker); 69 | 70 | return List::create(activations, derivatives); 71 | } 72 | -------------------------------------------------------------------------------- /src/softmaxUnitFunction.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | */ 19 | 20 | #include 21 | #include 22 | #include "unitFunction.h" 23 | 24 | using namespace RcppParallel; 25 | using namespace Rcpp; 26 | 27 | struct SoftmaxUnit : UnitFunction 28 | { 29 | SoftmaxUnit(const NumericMatrix input, NumericMatrix activations, 30 | NumericMatrix derivatives) : UnitFunction(input, activations, derivatives) 31 | {} 32 | 33 | void operator()(std::size_t begin_col, std::size_t end_col) 34 | { 35 | double colSum; 36 | 37 | for (int i = begin_col; i < end_col; i++) 38 | { 39 | std::transform(activations.column(i).begin(), activations.column(i).end(), 40 | activations.column(i).begin(), ::exp); 41 | 42 | colSum = std::accumulate(activations.column(i).begin(), 43 | activations.column(i).end(), 0.0); 44 | 45 | for (int j = 0; j < ncolInput; j++) 46 | { 47 | activations(j, i) = activations(j, i) / colSum; 48 | derivatives(j, i) = activations(j, i) * (1 - activations(j, i)); 49 | } 50 | } 51 | } 52 | }; 53 | 54 | // [[Rcpp::export]] 55 | List softmaxUnitCpp(NumericMatrix input) 56 | { 57 | int nrows = input.nrow(); 58 | int ncols = input.ncol(); 59 | NumericMatrix activations = transpose(input); 60 | NumericMatrix derivatives(Dimension(ncols, nrows)); 61 | 62 | SoftmaxUnit worker(input, activations, derivatives); 63 | 64 | parallelFor(0, nrows, worker); 65 | 66 | return List::create(transpose(activations), transpose(derivatives)); 67 | } 68 | -------------------------------------------------------------------------------- /R/RcppExports.R: -------------------------------------------------------------------------------- 1 | # Generated by using Rcpp::compileAttributes() -> do not edit by hand 2 | # Generator token: 10BE3573-1514-4C36-9D1C-5A225CD40393 3 | 4 | applyDropoutMaskCpp <- function(data, mask) { 5 | .Call('_darch_applyDropoutMaskCpp', PACKAGE = 'darch', data, mask) 6 | } 7 | 8 | ditherCpp <- function(data, columnMask) { 9 | .Call('_darch_ditherCpp', PACKAGE = 'darch', data, columnMask) 10 | } 11 | 12 | exponentialLinearUnitCpp <- function(input, alpha) { 13 | .Call('_darch_exponentialLinearUnitCpp', PACKAGE = 'darch', input, alpha) 14 | } 15 | 16 | maxoutUnitCppParallel <- function(activations, derivatives, poolSize, dropoutMask) { 17 | invisible(.Call('_darch_maxoutUnitCppParallel', PACKAGE = 'darch', activations, derivatives, poolSize, dropoutMask)) 18 | } 19 | 20 | maxoutUnitCpp <- function(activations, derivatives, poolSize, dropoutMask) { 21 | invisible(.Call('_darch_maxoutUnitCpp', PACKAGE = 'darch', activations, derivatives, poolSize, dropoutMask)) 22 | } 23 | 24 | maxoutWeightUpdateCpp <- function(inc, poolSize) { 25 | invisible(.Call('_darch_maxoutWeightUpdateCpp', PACKAGE = 'darch', inc, poolSize)) 26 | } 27 | 28 | minimizeCpp <- function(x, f, length, red, dims, data, target, epochSwitch, matMult) { 29 | .Call('_darch_minimizeCpp', PACKAGE = 'darch', x, f, length, red, dims, data, target, epochSwitch, matMult) 30 | } 31 | 32 | normalizeWeightsCpp <- function(weights, bound) { 33 | invisible(.Call('_darch_normalizeWeightsCpp', PACKAGE = 'darch', weights, bound)) 34 | } 35 | 36 | rectifiedLinearUnitCpp <- function(input) { 37 | .Call('_darch_rectifiedLinearUnitCpp', PACKAGE = 'darch', input) 38 | } 39 | 40 | rpropGradientsCpp <- function(gg, gradients) { 41 | invisible(.Call('_darch_rpropGradientsCpp', PACKAGE = 'darch', gg, gradients)) 42 | } 43 | 44 | rpropDeltaCpp <- function(gg, delta, inc, dec, minDelta, maxDelta) { 45 | invisible(.Call('_darch_rpropDeltaCpp', PACKAGE = 'darch', gg, delta, inc, dec, minDelta, maxDelta)) 46 | } 47 | 48 | rpropDeltaWiRpropPlusCpp <- function(gg, deltaW, gradients, delta, newE, oldE) { 49 | invisible(.Call('_darch_rpropDeltaWiRpropPlusCpp', PACKAGE = 'darch', gg, deltaW, gradients, delta, newE, oldE)) 50 | } 51 | 52 | sigmoidUnitCpp <- function(input) { 53 | .Call('_darch_sigmoidUnitCpp', PACKAGE = 'darch', input) 54 | } 55 | 56 | softmaxUnitCpp <- function(input) { 57 | .Call('_darch_softmaxUnitCpp', PACKAGE = 'darch', input) 58 | } 59 | 60 | softplusUnitCpp <- function(input) { 61 | .Call('_darch_softplusUnitCpp', PACKAGE = 'darch', input) 62 | } 63 | 64 | -------------------------------------------------------------------------------- /R/compat.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Johannes Rueckert 2 | # 3 | # This file is part of darch. 4 | # 5 | # darch is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # darch is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with darch. If not, see . 17 | 18 | # parameter compability list 19 | compatibilityList <- function() 20 | { 21 | list( 22 | "scale" = "preProc.params", 23 | "caret.preProcessParams" = "preProc.params", 24 | "preProc.logicalToNumeric" = "preProc.fullRank", 25 | "preProc.logicalToNumeric.targets" = "preProc.fullRank.targets", 26 | "preProc.orderedToNumeric" = "preProc.factorToNumeric", 27 | "preProc.orderedToNumeric.targets" = "preProc.factorToNumeric.targets", 28 | "rbm.trainOutputLayer" = "rbm.lastLayer", 29 | "rbm.learnRateWeights" = "rbm.learnRate", 30 | "rbm.learnRateBiasVisible" = "rbm.learnRate", 31 | "rbm.learnRateBiasHidden" = "rbm.learnRate", 32 | "rbm.weightCost" = "rbm.weightDecay", 33 | "rbm.momentumSwitch" = "rbm.momentumRampLength", 34 | "rbm.visibleUnitFunction" = "rbm.unitFunction", 35 | "rbm.hiddenUnitFunction" = "rbm.unitFunction", 36 | "rbm.genWeightFunction" = "generateWeightsFunction", 37 | "darch.bootstrap" = "bootstrap", 38 | "darch.genWeightFunc" = "generateWeightsFunction", 39 | "darch.logLevel" = "logLevel", 40 | "darch.momentumSwitch" = "darch.momentumRampLength", 41 | "darch.learnRateWeights" = "bp.learnRate", 42 | "darch.learnRateBiases" = "bp.learnRate", 43 | "darch.dropoutHidden" = "darch.dropout", 44 | "darch.dropoutInput" = "darch.dropout", 45 | "darch.dropoutOneMaskPerEpoch" = "darch.dropout.oneMaskPerEpoch", 46 | "darch.layerFunction" = "darch.unitFunction", 47 | "darch.layerFunctions" = "darch.unitFunction", 48 | "darch.layerFunctionDefault" = "darch.unitFunction", 49 | "darch.layerFunction.maxout.poolSize" = "darch.maxout.poolSize", 50 | "darch.unitFunction.maxout.poolSize" = "darch.maxout.poolSize", 51 | "darch.retainData" = "retainData" 52 | ) 53 | } -------------------------------------------------------------------------------- /R/generateRBMs.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2016 Martin Drees 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | 19 | setGeneric( 20 | name = "generateRBMs", 21 | def = function(darch){standardGeneric("generateRBMs")} 22 | ) 23 | 24 | #' Generates the RBMs for the pre-training. 25 | #' 26 | #' Used the layer sizes from the \code{\linkS4class{DArch}} object to create the 27 | #' RBM objects for the pre-training. 28 | #' 29 | #' @param darch An instance of the class \code{DArch} for which RBMs are 30 | #' to be generated. 31 | #' @return The DArch object with the generated RBMs 32 | #' 33 | #' @seealso \code{\linkS4class{DArch}} 34 | #' \code{\linkS4class{RBM}} 35 | #' 36 | #' @include darch.Class.R 37 | #' @include rbm.Class.R 38 | #' @keywords internal 39 | setMethod( 40 | f = "generateRBMs", 41 | signature = "DArch", 42 | definition = function(darch) 43 | { 44 | darch@rbmList <- list() 45 | layers <- getParameter(".layers") 46 | futile.logger::flog.info("Generating RBMs.") 47 | for (i in 1:(length(layers) - 1)) 48 | { 49 | # generate the RBMs 50 | visible <- layers[[i]] 51 | hidden <- layers[[(i + 1)]] 52 | futile.logger::flog.info(paste("Constructing new RBM instance with %s", 53 | "visible and %s hidden units."), visible, hidden) 54 | rbm <- new("RBM") 55 | rbm@parameters <- darch@parameters 56 | rbm@parameters[[".numVisible"]] <- visible 57 | rbm@parameters[[".numHidden"]] <- hidden 58 | rbm@parameters[[".generateWeightsFunction"]] <- 59 | getParameter(".generateWeightsFunction")[[i]] 60 | rbm <- resetRBM(rbm) 61 | darch@rbmList[[i]] <- rbm 62 | darch <- addLayer(darch, rbm@weights, rbm@hiddenBiases, 63 | getParameter(".darch.unitFunction")[[i]], 64 | getParameter(".darch.weightUpdateFunction")[[i]]) 65 | } 66 | 67 | darch 68 | } 69 | ) 70 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | darch 2 | 3 | The darch package is built on the basis of the code from G. E. Hinton and R. R. Salakhutdinov 4 | (available under Matlab Code for deep belief nets (http://www.cs.toronto.edu/~hinton/MatlabForSciencePaper.html 5 | "Matlab for science paper", last visit: 23.06.2015). 6 | 7 | This package is for generating neural networks with many layers (deep architectures) and train them with the method introduced by the publications "A fast learning algorithm for deep belief nets" (G. E. Hinton, S. Osindero, Y. W. Teh) and "Reducing the dimensionality of data with neural networks" (G. E. Hinton, R. R. Salakhutdinov). This method includes a pre training with the contrastive divergence method published by G.E Hinton (2002) and a fine tuning with common known training algorithms like backpropagation or conjugate gradient, as well as more recent techniques like dropout and maxout. 8 | 9 | Visit https://github.com/maddin79/darch for releases, information, bug reports 10 | etc. 11 | 12 | Copyright (C) 2013-2016 Martin Drees and contributors 13 | 14 | References: 15 | 16 | Hinton, G. E., S. Osindero, Y. W. Teh, A fast learning algorithm for deep belief nets, 17 | Neural Computation 18(7), S. 1527-1554, DOI: 18 | [10.1162/neco.2006.18.7.1527](http://dx.doi.org/10.1162/neco.2006.18.7.1527), 2006. 19 | 20 | Hinton, G. E., R. R. Salakhutdinov, Reducing the dimensionality of data with neural 21 | networks, Science 313(5786), S. 504-507, DOI: 22 | [10.1126/science.1127647](http://dx.doi.org/10.1126/science.1127647), 2006. 23 | 24 | Hinton, G. E., Training products of experts by minimizing contrastive divergence, 25 | Neural Computation 14(8), S. 1711-1800, DOI: 26 | [10.1162/089976602760128018](http://dx.doi.org/10.1162/089976602760128018), 2002. 27 | 28 | Hinton, Geoffrey E. et al. (2012). "Improving neural networks by preventing coadaptation of feature detectors". In: Clinical Orthopaedics and Related Research abs/1207.0580. URL : [arxiv.org](http://arxiv.org/abs/1207.0580). 29 | 30 | Goodfellow, Ian J. et al. (2013). "Maxout Networks". In: Proceedings of the 30th International Conference on Machine Learning, ICML 2013, Atlanta, GA, USA, 16-21 June 2013, pp. 1319–1327. URL : [jmlr.org](http://jmlr.org/proceedings/papers/v28/goodfellow13.html). 31 | 32 | Drees, Martin (2013). "Implementierung und Analyse von tiefen Architekturen 33 | in R". German. Master's thesis. Fachhochschule Dortmund. 34 | 35 | Rueckert, Johannes (2015). "Extending the Darch library for deep 36 | architectures". Project thesis. Fachhochschule Dortmund. 37 | URL: [saviola.de](http://static.saviola.de/publications/rueckert_2015.pdf). -------------------------------------------------------------------------------- /R/log.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2016 Martin Drees 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | 19 | # Define our own log function shortcuts for convenience and easy access 20 | # TODO 21 | #TRACE <- futile.logger::flog.trace 22 | #DEBUG <- futile.logger::flog.debug 23 | #INFO <- futile.logger::flog.info 24 | #WARN <- futile.logger::flog.warn 25 | #ERROR <- futile.logger::flog.error 26 | #FATAL <- futile.logger::flog.fatal 27 | 28 | 29 | #' Set the log level. 30 | #' 31 | #' Convenience wrapper for \code{\link[futile.logger]{flog.threshold}} with 32 | #' sanity checking and current level output. 33 | #' 34 | #' The log levels are defined by the \code{\link{futile.logger}} package. 35 | #' The following levels a available: 36 | #' \tabular{ll}{ 37 | #' futile.logger::TRACE\cr 38 | #' futile.logger::DEBUG\cr 39 | #' futile.logger::INFO\cr 40 | #' futile.logger::WARN\cr 41 | #' futile.logger::ERROR\cr 42 | #' futile.logger::FATAL 43 | #' } 44 | #' 45 | #' @param value Log level, must be one of the \code{futile.logger} constants. 46 | #' @export 47 | #' @keywords internal 48 | setLogLevel <- function(value) 49 | { 50 | 51 | if (!is.null(value)) 52 | { 53 | if (value %in% c(futile.logger::TRACE, futile.logger::DEBUG, 54 | futile.logger::INFO, futile.logger::WARN, 55 | futile.logger::ERROR, futile.logger::FATAL, 56 | "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL")) 57 | { 58 | futile.logger::flog.threshold(value) 59 | } 60 | else 61 | { 62 | futile.logger::flog.warn( 63 | "It is not possible to set the log level to %s", value) 64 | } 65 | } 66 | 67 | futile.logger::flog.info("The current log level is: %s", 68 | futile.logger::flog.threshold()) 69 | } 70 | 71 | logNamedList <- function(l, lf = futile.logger::flog.info) 72 | { 73 | for (n in names(l)) 74 | { 75 | # TODO functions? 76 | lf("%s = %s", n, deparseClean(l[[n]])) 77 | } 78 | } -------------------------------------------------------------------------------- /man/minimizeAutoencoder.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/minimizeAutoencoder.R 3 | \name{minimizeAutoencoder} 4 | \alias{minimizeAutoencoder} 5 | \title{Conjugate gradient for a autoencoder network} 6 | \usage{ 7 | minimizeAutoencoder(darch, trainData, targetData, 8 | cg.length = getParameter(".cg.length"), 9 | dropout = getParameter(".darch.dropout"), 10 | dropConnect = getParameter(".darch.dropout.dropConnect"), 11 | matMult = getParameter(".matMult"), debugMode = getParameter(".debug"), 12 | ...) 13 | } 14 | \arguments{ 15 | \item{darch}{A instance of the class \code{\linkS4class{DArch}}.} 16 | 17 | \item{trainData}{The training data matrix.} 18 | 19 | \item{targetData}{The labels for the training data.} 20 | 21 | \item{cg.length}{Numbers of line search} 22 | 23 | \item{dropout}{See \code{darch.dropout} parameter of \code{\link{darch}}.} 24 | 25 | \item{dropConnect}{See \code{darch.dropout.dropConnect} parameter of 26 | \code{\link{darch}}.} 27 | 28 | \item{matMult}{Matrix multiplication function, internal parameter.} 29 | 30 | \item{debugMode}{Whether debug mode is enabled, internal parameter.} 31 | 32 | \item{...}{Further parameters.} 33 | } 34 | \value{ 35 | The trained \code{\linkS4class{DArch}} object. 36 | } 37 | \description{ 38 | This function trains a \code{\linkS4class{DArch}} autoencoder network with the 39 | conjugate gradient method. 40 | } 41 | \details{ 42 | This function is built on the basis of the code from G. Hinton et. al. 43 | (http://www.cs.toronto.edu/~hinton/MatlabForSciencePaper.html - last visit 44 | 2016-04-30) for the fine tuning of deep belief nets. The original code is 45 | located in the files 'backpropclassify.m', 'CG_MNIST.m' and 46 | 'CG_CLASSIFY_INIT.m'. 47 | It implements the fine tuning for a classification net with backpropagation 48 | using a direct translation of the \code{\link{minimize}} function from C. 49 | Rassmussen (available at http://www.gatsby.ucl.ac.uk/~edward/code/minimize/ 50 | - last visit 2016-04-30) to R. 51 | 52 | \code{minimizeAutoencoder} supports dropout but does not use the weight 53 | update function as defined via the \code{darch.weightUpdateFunction} 54 | parameter of \code{\link{darch}}, so that weight decay, momentum etc. are not 55 | supported. 56 | } 57 | \examples{ 58 | \dontrun{ 59 | data(iris) 60 | model <- darch(Species ~ ., iris, c(6,10,2,10,6), darch.isClass = F, 61 | preProc.params = list(method=c("center", "scale")), 62 | darch.numEpochs = 20, darch.batchSize = 6, darch.unitFunction = tanhUnit 63 | darch.fineTuneFunction = "minimizeAutoencoder") 64 | } 65 | } 66 | \seealso{ 67 | \code{\link{darch}}, \code{\link{fineTuneDArch}} 68 | 69 | Other fine-tuning functions: \code{\link{backpropagation}}, 70 | \code{\link{minimizeClassifier}}, 71 | \code{\link{rpropagation}} 72 | } 73 | 74 | -------------------------------------------------------------------------------- /R/darch.Class.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2016 Martin Drees 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | 19 | #' Class for deep architectures 20 | #' 21 | #' This class represents a model created, and/or configured, and/or trained with 22 | #' the \code{\link{darch}} function. 23 | #' It implements deep architectures and provides the ability to train 24 | #' them with a pre training using contrastive divergence and fine tuning with 25 | #' backpropagation, resilient backpropagation and conjugate gradients. 26 | #' 27 | #' The class inherits all attributes from the class \code{\linkS4class{Net}}. 28 | #' User-relevant slots include \code{stats} (training statistics), \code{epochs} 29 | #' (numer of epoch this model was trained for), and \code{parameters} (all 30 | #' parameters passed to \code{\link{darch}} as well as internal parameters). 31 | #' 32 | #' @slot rbmList A list which contains all RBMs for the pre-training. 33 | #' @slot layers A list with the layer information. In the first field are the 34 | #' weights and in the second field is the unit function. 35 | #' @slot cancel Boolean value which indicates if the network training is 36 | #' canceled. 37 | #' @slot cancelMessage The message when the execution is canceled. 38 | #' @slot dropoutMasks List of dropout masks, used internally. 39 | #' @slot dataSet \linkS4class{DataSet} instance. 40 | #' @seealso \code{\linkS4class{Net}}, \code{\linkS4class{RBM}} 41 | #' @author Martin Drees 42 | #' @include net.Class.R 43 | #' @exportClass DArch 44 | #' @rdname DArch-class 45 | #' @family darch classes 46 | #' @keywords internal 47 | setClass( 48 | Class = "DArch", 49 | representation = representation( 50 | rbmList = "list", 51 | layers = "list", 52 | cancel = "logical", 53 | cancelMessage = "character", 54 | dropoutMasks = "list", 55 | dataSet = "ANY" 56 | ), 57 | contains = "Net" 58 | ) 59 | 60 | setMethod("initialize","DArch", 61 | function(.Object){ 62 | .Object@epochs <- 0 63 | .Object@cancel <- FALSE 64 | .Object@cancelMessage <- "No reason specified." 65 | .Object@dataSet <- NULL 66 | .Object@parameters <- list() 67 | return(.Object) 68 | } 69 | ) 70 | -------------------------------------------------------------------------------- /R/rbm.Class.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2016 Martin Drees 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | 19 | #' Class for restricted Boltzmann machines 20 | #' 21 | #' This class represents a restricted Boltzmann machine. 22 | #' 23 | #' The RBM can be trained with the implementation of the 24 | #' contrastive divergence method \code{\link{trainRBM}}. The class inherits the 25 | #' attributes from the \code{\linkS4class{Net}}. 26 | #' 27 | #' @slot weights Object of class \code{"matrix"}. Weight matrix. 28 | #' @slot weightsInc Object of class \code{"matrix"}. Matrix of update values for 29 | #' the Weight. 30 | #' @slot visibleBiases Object of class \code{"array"}. Visible biases array. 31 | #' @slot visibleBiasesInc Object of class \code{"array"}. Array of update values 32 | #' for the visible biases 33 | #' @slot visibleUnitStates Object of class \code{"list"}. States of the visible 34 | #' units. 35 | #' @slot hiddenBiases Object of class \code{"array"}. Hidden biases array. 36 | #' @slot hiddenBiasesInc Object of class \code{"array"}. Array of update values 37 | #' for the hidden biases. 38 | #' @slot hiddenUnitStates Object of class \code{"list"}. States of the hidden 39 | #' units. 40 | #' @slot posPhaseData Object of class \code{"list"}. Attribute to save the 41 | #' positive phase data during the training. 42 | #' @slot output Object of class \code{"matrix"}. Output matrix of the RBM. 43 | #' @seealso \code{\linkS4class{Net}}, \code{\linkS4class{DArch}}, 44 | #' \code{\link{trainRBM}} 45 | #' @author Martin Drees 46 | #' @include net.Class.R 47 | #' @exportClass RBM 48 | #' @keywords internal 49 | #' @family darch classes 50 | #' @rdname RBM 51 | setClass( 52 | Class = "RBM", 53 | representation = representation( 54 | weights = "matrix", 55 | weightsInc = "matrix", 56 | visibleBiases = "array", 57 | visibleBiasesInc = "array", 58 | visibleUnitStates = "list", 59 | hiddenBiases = "array", 60 | hiddenBiasesInc = "array", 61 | hiddenUnitStates = "list", 62 | posPhaseData = "list", 63 | output = "matrix" 64 | ), 65 | contains = "Net" 66 | ) 67 | 68 | setMethod("initialize","RBM", 69 | function(.Object){ 70 | .Object@epochs <- 0 71 | .Object@parameters <- list() 72 | return(.Object) 73 | } 74 | ) 75 | -------------------------------------------------------------------------------- /man/fineTuneDArch.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darch.Learn.R 3 | \name{fineTuneDArch} 4 | \alias{fineTuneDArch} 5 | \title{Fine tuning function for the deep architecture} 6 | \usage{ 7 | fineTuneDArch(darch, dataSet, dataSetValid = NULL, numEpochs = 1, 8 | isClass = TRUE, stopErr = -Inf, stopClassErr = 101, 9 | stopValidErr = -Inf, stopValidClassErr = 101, shuffleTrainData = T, 10 | debugMode = F, ...) 11 | } 12 | \arguments{ 13 | \item{darch}{A instance of the class \code{\linkS4class{DArch}}.} 14 | 15 | \item{dataSet}{\code{\linkS4class{DataSet}} containing training and 16 | optionally validation and test data.} 17 | 18 | \item{dataSetValid}{\code{\linkS4class{DataSet}} to be used for validation.} 19 | 20 | \item{numEpochs}{The number of training iterations} 21 | 22 | \item{isClass}{Indicates whether the training is for a classification net. 23 | When \code{TRUE} then statistics for classification will be determind. 24 | Default is \code{TRUE}} 25 | 26 | \item{stopErr}{Stop criteria for the error on the train data. Default is 27 | \code{-Inf}} 28 | 29 | \item{stopClassErr}{Stop criteria for the classification error on the train 30 | data. Default is \code{101}} 31 | 32 | \item{stopValidErr}{Stop criteria for the error on the validation data. 33 | Default is \code{-Inf}.} 34 | 35 | \item{stopValidClassErr}{Stop criteria for the classification error on the 36 | validation data. Default is \code{101}.} 37 | 38 | \item{shuffleTrainData}{Whether to shuffle train data before each epoch.} 39 | 40 | \item{debugMode}{Whether to enable debug mode, internal parameter.} 41 | 42 | \item{...}{Additional parameters for the training function} 43 | 44 | \item{bootstrap}{Whether to use bootstrapping to create validation data.} 45 | } 46 | \value{ 47 | Trained \code{\linkS4class{DArch}} instance. 48 | } 49 | \description{ 50 | The fine tuning function for deep architectures. This function use the 51 | function saved in the attribute \code{fineTuneFunction} to train the deep 52 | architecture. 53 | } 54 | \details{ 55 | The function trains the given network \code{darch} with the function 56 | saved in the attribute \code{fineTuneFunction} of the 57 | \code{\linkS4class{DArch}}-Object. The data and classes for validation and 58 | testing are optional. If they are provided the network will be executed 59 | with this datasets and statistics will be calculated. This statistics are 60 | saved in the \code{stats} attribute (see \code{\linkS4class{Net}}). Also it 61 | is possible to set stop criteria for the training on the error 62 | (\code{stopErr}, \code{stopValidErr}) or the correct classifications 63 | (\code{stopClassErr}, \code{stopValidClassErr}) of the training or 64 | validation dataset. 65 | } 66 | \seealso{ 67 | \code{\linkS4class{DArch}}, \code{\linkS4class{Net}}, 68 | \code{\link{backpropagation}}, \code{\link{rpropagation}}, 69 | \code{\link{minimizeAutoencoder}}, \code{\link{minimizeClassifier}} 70 | } 71 | \keyword{internal} 72 | 73 | -------------------------------------------------------------------------------- /man/maxoutUnit.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/darchUnitFunctions.R 3 | \name{maxoutUnit} 4 | \alias{maxoutUnit} 5 | \title{Maxout / LWTA unit function} 6 | \usage{ 7 | maxoutUnit(input, ..., poolSize = getParameter(".darch.maxout.poolSize", 2, 8 | ...), unitFunc = getParameter(".darch.maxout.unitFunction", linearUnit, 9 | ...), dropoutMask = vector()) 10 | } 11 | \arguments{ 12 | \item{input}{Input for the activation function.} 13 | 14 | \item{...}{Additional parameters, passed on to inner unit function.} 15 | 16 | \item{poolSize}{The size of each maxout pool.} 17 | 18 | \item{unitFunc}{Inner unit function for maxout.} 19 | 20 | \item{dropoutMask}{Vector containing the dropout mask.} 21 | } 22 | \value{ 23 | A list with the maxout activation in the first entry and the 24 | derivative of the transfer function in the second entry. 25 | } 26 | \description{ 27 | The function calculates the activation of the units and returns a list, in 28 | which the first entry is the result through the maxout transfer function and 29 | the second entry is the derivative of the transfer function. 30 | } 31 | \details{ 32 | Maxout sets the activations of all neurons but the one with the highest 33 | activation within a pool to \code{0}. If this is used without 34 | \link{maxoutWeightUpdate}, it becomes the local-winner-takes-all algorithm, 35 | as the only difference between the two is that outgoing weights are shared 36 | for maxout. 37 | } 38 | \examples{ 39 | \dontrun{ 40 | data(iris) 41 | # LWTA: 42 | model <- darch(Species ~ ., iris, c(0, 50, 0), 43 | darch.unitFunction = c("maxoutUnit", "softmaxUnit"), 44 | darch.maxout.poolSize = 5, darch.maxout.unitFunction = "sigmoidUnit") 45 | # Maxout: 46 | model <- darch(Species ~ ., iris, c(0, 50, 0), 47 | darch.unitFunction = c("maxoutUnit", "softmaxUnit"), 48 | darch.maxout.poolSize = 5, darch.maxout.unitFunction = "sigmoidUnit", 49 | darch.weightUpdateFunction = c("weightDecayWeightUpdate", "maxoutWeightUpdate")) 50 | } 51 | } 52 | \references{ 53 | Srivastava, Rupesh Kumar, Jonathan Masci, Sohrob Kazerounian, 54 | Faustino Gomez, and Juergen Schmidhuber (2013). "Compete to Compute". In: 55 | Advances in Neural Information Processing Systems 26. Ed. by C.J.C. Burges, 56 | L. Bottou, M. Welling, Z. Ghahramani, and K.Q. Weinberger. 57 | Curran Associates, Inc., pp. 2310-2318. 58 | URL: http://papers.nips.cc/paper/5059-compete-to-compute.pdf 59 | 60 | Goodfellow, Ian J., David Warde-Farley, Mehdi Mirza, Aaron C. Courville, 61 | and Yoshua Bengio (2013). "Maxout Networks". In: Proceedings of the 30th 62 | International Conference on Machine Learning, ICML 2013, Atlanta, GA, USA, 63 | 16-21 June 2013, pp. 1319-1327. 64 | URL: http://jmlr.org/proceedings/papers/v28/goodfellow13.html 65 | } 66 | \seealso{ 67 | Other darch unit functions: \code{\link{exponentialLinearUnit}}, 68 | \code{\link{linearUnit}}, 69 | \code{\link{rectifiedLinearUnit}}, 70 | \code{\link{sigmoidUnit}}, \code{\link{softmaxUnit}}, 71 | \code{\link{softplusUnit}}, \code{\link{tanhUnit}} 72 | } 73 | 74 | -------------------------------------------------------------------------------- /man/darchBench.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/benchmark.R 3 | \name{darchBench} 4 | \alias{darchBench} 5 | \title{Benchmarking wrapper for \code{darch}} 6 | \usage{ 7 | darchBench(..., bench.times = 1, bench.save = F, 8 | bench.dir = "./darch.benchmark", bench.continue = T, bench.delete = F, 9 | bench.seeds = NULL, output.capture = bench.save, logLevel = NULL) 10 | } 11 | \arguments{ 12 | \item{...}{Parameters to the \code{\link{darch}} function} 13 | 14 | \item{bench.times}{How many benchmark runs to perform} 15 | 16 | \item{bench.save}{Whether to save benchmarking results to a directory} 17 | 18 | \item{bench.dir}{Path (relative or absolute) including directory where 19 | benchmark results are saved if \code{bench.save} is true} 20 | 21 | \item{bench.continue}{Whether the benchmark is to be continued from an 22 | earlier run. If \code{TRUE}, existing benchmark results are looked for in 23 | the directory given in \code{bench.dir} and new results are appended. 24 | If both this and \code{bench.continue} are \code{FALSE} and 25 | the directory given in \code{bench.dir} does already exist, the training 26 | will be aborted with an error.} 27 | 28 | \item{bench.delete}{Whether to delete the contents of \code{bench.dir} if 29 | \code{bench.continue} is \code{FALSE}. Caution: This will attempt to delete 30 | ALL files in the given directory, use at your own risk!} 31 | 32 | \item{bench.seeds}{Vector of seeds, one for each run. Will be passed to 33 | \code{\link{darch}}.} 34 | 35 | \item{output.capture}{Whether to capture R output in \code{.Rout} files in 36 | the given directory. This is the only way of gaining access to the R 37 | output since the foreach loop will not print anything to the console. Will 38 | be ignored if \code{bench.save} is \code{FALSE}.} 39 | 40 | \item{logLevel}{\code{\link{futile.logger}} log level. Uses the currently 41 | set log level by default, which is \code{futile.logger::flog.info} if it 42 | was not changed. Other available levels include, from least to most 43 | verbose: \code{FATAL}, \code{ERROR}, \code{WARN}, \code{DEBUG}, and 44 | \code{TRACE}.} 45 | } 46 | \value{ 47 | List of \code{DArch} instances; the results of each call to 48 | \code{darch}. 49 | } 50 | \description{ 51 | Simple benchmarking function which wraps around the \code{\link{darch}} 52 | function for users who can't or don't want to use the caret package for 53 | benchmarking. This function requires the \code{foreach} 54 | package to work, and will perform parallel benchmarks if an appropriate 55 | backend was registered beforehand. 56 | } 57 | \examples{ 58 | \dontrun{ 59 | data(iris) 60 | modelList <- darchBench(Species ~ ., iris, c(0, 50, 0), 61 | preProc.params = list(method = c("center", "scale")), 62 | darch.unitFunction = c("sigmoidUnit", "softmaxUnit"), 63 | darch.numEpochs = 30, bench.times = 10, bench.save = T) 64 | } 65 | } 66 | \seealso{ 67 | Other darch interface functions: \code{\link{darchTest}}, 68 | \code{\link{darch}}, \code{\link{plot.DArch}}, 69 | \code{\link{predict.DArch}}, \code{\link{print.DArch}} 70 | } 71 | 72 | -------------------------------------------------------------------------------- /R/darch.Add.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2016 Martin Drees 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | 19 | #' @include darch.Class.R 20 | NULL 21 | 22 | #' Adds a layer to the \code{\linkS4class{DArch}} object 23 | #' 24 | #' Adds a layer to the given \code{\linkS4class{DArch}} object. The parameter 25 | #' weights and biases will be put together in one matrix. 26 | #' 27 | #' @param darch An instance of the class \code{\linkS4class{DArch}}. 28 | #' @param weights The weights for the layer. 29 | #' @param biases The biases for the layer. 30 | #' @param unitFunction The functions of the units in the layer. 31 | #' 32 | #' @seealso \code{\linkS4class{DArch}} 33 | #' 34 | #' @keywords internal 35 | #' @export 36 | setGeneric("addLayer",function(darch, weights, biases, unitFunction, weightUpdateFunction){standardGeneric("addLayer")}) 37 | 38 | #' Adds a layer to the \code{\linkS4class{DArch}} object 39 | #' 40 | #' @inheritParams addLayer 41 | #' @seealso \code{\link{addLayer}} 42 | #' @keywords internal 43 | #' @export 44 | setMethod( 45 | f = "addLayer", 46 | signature = "DArch", 47 | definition = function(darch, weights, biases, unitFunction, weightUpdateFunction){ 48 | numLayers <- length(darch@layers) 49 | w <- rbind(weights, biases) 50 | layer <- list() 51 | layer[["weights"]] <- w 52 | layer[["unitFunction"]] <- unitFunction 53 | layer[["weightUpdateFunction"]] <- weightUpdateFunction 54 | darch@layers[[numLayers + 1]] <- layer 55 | return(darch) 56 | } 57 | ) 58 | 59 | #' Adds a field to a layer 60 | #' 61 | #' Adds a field to the layer given by the index. 62 | #' 63 | #' @param darch An instance of the class \code{\linkS4class{DArch}}. 64 | #' @param index The position of the layer. 65 | #' @param field The new field for the layer. 66 | #' 67 | #' @seealso \code{\linkS4class{DArch}} 68 | #' 69 | #' @keywords internal 70 | #' @export 71 | setGeneric("addLayerField",function(darch, index, field){standardGeneric("addLayerField")}) 72 | 73 | #' Adds a field to a layer 74 | #' 75 | #' @inheritParams addLayerField 76 | #' @seealso \code{\link{addLayerField}} 77 | #' @keywords internal 78 | #' @export 79 | setMethod( 80 | f = "addLayerField", 81 | signature = "DArch", 82 | definition = function(darch, index, field){ 83 | num <- length(darch@layers[[index]]) 84 | darch@layers[[index]][[num + 1]] <- field 85 | return(darch) 86 | } 87 | ) 88 | -------------------------------------------------------------------------------- /DESCRIPTION: -------------------------------------------------------------------------------- 1 | Package: darch 2 | Type: Package 3 | Title: Package for Deep Architectures and Restricted Boltzmann Machines 4 | Version: 0.13.0 5 | Date: 2018-05-03 6 | Author: Martin Drees [aut, cre, cph], 7 | Johannes Rueckert [ctb], 8 | Christoph M. Friedrich [ctb], 9 | Geoffrey Hinton [cph], 10 | Ruslan Salakhutdinov [cph], 11 | Carl Edward Rasmussen [cph], 12 | Maintainer: Johannes Rueckert 13 | Description: The darch package is built on the basis of the code from G. E. 14 | Hinton and R. R. Salakhutdinov (available under Matlab Code for deep belief 15 | nets). This package is for generating neural networks with many layers (deep 16 | architectures) and train them with the method introduced by the publications 17 | "A fast learning algorithm for deep belief nets" (G. E. Hinton, S. Osindero, 18 | Y. W. Teh (2006) ) and "Reducing the 19 | dimensionality of data with neural networks" (G. E. Hinton, R. R. 20 | Salakhutdinov (2006) ). This method includes a 21 | pre training with the contrastive divergence method published by G.E Hinton 22 | (2002) and a fine tuning with common known 23 | training algorithms like backpropagation or conjugate gradients. 24 | Additionally, supervised fine-tuning can be enhanced with maxout and 25 | dropout, two recently developed techniques to improve fine-tuning for deep 26 | learning. 27 | License: GPL (>= 2) | file LICENSE 28 | URL: https://github.com/maddin79/darch 29 | BugReports: https://github.com/maddin79/darch/issues 30 | Depends: 31 | R (>= 3.0.0) 32 | Imports: 33 | stats, 34 | methods, 35 | ggplot2, 36 | reshape2, 37 | futile.logger (>= 1.4.1), 38 | caret, 39 | Rcpp (>= 0.12.3), 40 | RcppParallel (>= 4.3.20) 41 | LinkingTo: Rcpp, RcppParallel 42 | SystemRequirements: GNU make 43 | Suggests: 44 | foreach, 45 | doRNG, 46 | NeuralNetTools, 47 | gputools, 48 | testthat, 49 | plyr (>= 1.8.3.9000) 50 | Collate: 51 | 'RcppExports.R' 52 | 'autosave.R' 53 | 'net.Class.R' 54 | 'darch.Class.R' 55 | 'backpropagation.R' 56 | 'benchmark.R' 57 | 'bootstrap.R' 58 | 'caret.R' 59 | 'compat.R' 60 | 'config.R' 61 | 'darch.Add.R' 62 | 'darch.Getter.R' 63 | 'dataset.R' 64 | 'darch.Learn.R' 65 | 'darch.R' 66 | 'darch.Setter.R' 67 | 'darchUnitFunctions.R' 68 | 'dropout.R' 69 | 'errorFunctions.R' 70 | 'rbm.Class.R' 71 | 'generateRBMs.R' 72 | 'generateWeightsFunctions.R' 73 | 'loadDArch.R' 74 | 'log.R' 75 | 'makeStartEndPoints.R' 76 | 'minimize.R' 77 | 'minimizeAutoencoder.R' 78 | 'minimizeClassifier.R' 79 | 'mnist.R' 80 | 'momentum.R' 81 | 'net.Getter.R' 82 | 'newDArch.R' 83 | 'params.R' 84 | 'plot.R' 85 | 'predict.R' 86 | 'print.R' 87 | 'rbm.Learn.R' 88 | 'rbm.Reset.R' 89 | 'rbmUnitFunctions.R' 90 | 'rbmUpdate.R' 91 | 'rpropagation.R' 92 | 'runDArch.R' 93 | 'saveDArch.R' 94 | 'test.R' 95 | 'util.R' 96 | 'weightUpdateFunctions.R' 97 | RoxygenNote: 5.0.1 98 | -------------------------------------------------------------------------------- /man/backpropagation.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/backpropagation.R 3 | \name{backpropagation} 4 | \alias{backpropagation} 5 | \title{Backpropagation learning function} 6 | \usage{ 7 | backpropagation(darch, trainData, targetData, 8 | bp.learnRate = getParameter(".bp.learnRate", rep(1, times = 9 | length(darch@layers))), 10 | bp.learnRateScale = getParameter(".bp.learnRateScale"), 11 | nesterovMomentum = getParameter(".darch.nesterovMomentum"), 12 | dropout = getParameter(".darch.dropout", rep(0, times = length(darch@layers) 13 | + 1), darch), dropConnect = getParameter(".darch.dropout.dropConnect"), 14 | matMult = getParameter(".matMult"), debugMode = getParameter(".debug", F), 15 | ...) 16 | } 17 | \arguments{ 18 | \item{darch}{An instance of the class \code{\linkS4class{DArch}}.} 19 | 20 | \item{trainData}{The training data (inputs).} 21 | 22 | \item{targetData}{The target data (outputs).} 23 | 24 | \item{bp.learnRate}{Learning rates for backpropagation, length is either one 25 | or the same as the number of weight matrices when using different learning 26 | rates for each layer.} 27 | 28 | \item{bp.learnRateScale}{The learn rate is multiplied by this value after 29 | each epoch.} 30 | 31 | \item{nesterovMomentum}{See \code{darch.nesterovMomentum} parameter of 32 | \code{\link{darch}}.} 33 | 34 | \item{dropout}{See \code{darch.dropout} parameter of \code{\link{darch}}.} 35 | 36 | \item{dropConnect}{See \code{darch.dropout.dropConnect} parameter of 37 | \code{\link{darch}}.} 38 | 39 | \item{matMult}{Matrix multiplication function, internal parameter.} 40 | 41 | \item{debugMode}{Whether debug mode is enabled, internal parameter.} 42 | 43 | \item{...}{Further parameters.} 44 | } 45 | \value{ 46 | The trained deep architecture 47 | } 48 | \description{ 49 | This function provides the backpropagation algorithm for deep architectures. 50 | } 51 | \details{ 52 | The only backpropagation-specific, user-relevant parameters are 53 | \code{bp.learnRate} and \code{bp.learnRateScale}; they can be passed to the 54 | \code{\link{darch}} function when enabling \code{backpropagation} as the 55 | fine-tuning function. \code{bp.learnRate} defines the backpropagation 56 | learning rate and can either be specified as a single scalar or as a vector 57 | with one entry for each weight matrix, allowing for per-layer learning rates. 58 | \code{bp.learnRateScale} is a single scalar which contains a scaling factor 59 | for the learning rate(s) which will be applied after each epoch. 60 | 61 | Backpropagation supports dropout and uses the weight update function as 62 | defined via the \code{darch.weightUpdateFunction} parameter of 63 | \code{\link{darch}}. 64 | } 65 | \examples{ 66 | \dontrun{ 67 | data(iris) 68 | model <- darch(Species ~ ., iris, darch.fineTuneFunction = "backpropagation") 69 | } 70 | } 71 | \references{ 72 | Rumelhart, D., G. E. Hinton, R. J. Williams, Learning 73 | representations by backpropagating errors, Nature 323, S. 533-536, DOI: 74 | 10.1038/323533a0, 1986. 75 | } 76 | \seealso{ 77 | \code{\link{darch}} 78 | 79 | Other fine-tuning functions: \code{\link{minimizeAutoencoder}}, 80 | \code{\link{minimizeClassifier}}, 81 | \code{\link{rpropagation}} 82 | } 83 | 84 | -------------------------------------------------------------------------------- /man/minimizeClassifier.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/minimizeClassifier.R 3 | \docType{methods} 4 | \name{minimizeClassifier} 5 | \alias{minimizeClassifier} 6 | \title{Conjugate gradient for a classification network} 7 | \usage{ 8 | minimizeClassifier(darch, trainData, targetData, 9 | cg.length = getParameter(".cg.length"), 10 | cg.switchLayers = getParameter(".cg.length"), 11 | dropout = getParameter(".darch.dropout"), 12 | dropConnect = getParameter(".darch.dropout.dropConnect"), 13 | matMult = getParameter(".matMult"), debugMode = getParameter(".debug"), 14 | ...) 15 | } 16 | \arguments{ 17 | \item{darch}{A instance of the class \code{\linkS4class{DArch}}.} 18 | 19 | \item{trainData}{The training data matrix.} 20 | 21 | \item{targetData}{The labels for the training data.} 22 | 23 | \item{cg.length}{Numbers of line search} 24 | 25 | \item{cg.switchLayers}{Indicates when to train the full network instead of 26 | only the upper two layers} 27 | 28 | \item{dropout}{See \code{darch.dropout} parameter of \code{\link{darch}}.} 29 | 30 | \item{dropConnect}{See \code{darch.dropout.dropConnect} parameter of 31 | \code{\link{darch}}.} 32 | 33 | \item{matMult}{Matrix multiplication function, internal parameter.} 34 | 35 | \item{debugMode}{Whether debug mode is enabled, internal parameter.} 36 | 37 | \item{...}{Further parameters.} 38 | } 39 | \value{ 40 | The trained \code{\linkS4class{DArch}} object. 41 | } 42 | \description{ 43 | This function trains a \code{\linkS4class{DArch}} classifier network with the 44 | conjugate gradient method. 45 | } 46 | \details{ 47 | This function is build on the basis of the code from G. Hinton et. al. 48 | (http://www.cs.toronto.edu/~hinton/MatlabForSciencePaper.html - last visit 49 | 2016-04-30) for the fine tuning of deep belief nets. The original code is 50 | located in the files 'backpropclassify.m', 'CG_MNIST.m' and 51 | 'CG_CLASSIFY_INIT.m'. 52 | It implements the fine tuning for a classification net with backpropagation 53 | using a direct translation of the \code{\link{minimize}} function from C. 54 | Rassmussen (available at http://www.gatsby.ucl.ac.uk/~edward/code/minimize/ 55 | - last visit 2016-04-30) to R. 56 | The parameter \code{cg.switchLayers} is for the switch between two training 57 | type. Like in the original code, the top two layers can be trained alone 58 | until \code{epoch} is equal to \code{epochSwitch}. Afterwards the entire 59 | network will be trained. 60 | 61 | \code{minimizeClassifier} supports dropout but does not use the weight 62 | update function as defined via the \code{darch.weightUpdateFunction} 63 | parameter of \code{\link{darch}}, so that weight decay, momentum etc. are not 64 | supported. 65 | } 66 | \examples{ 67 | \dontrun{ 68 | data(iris) 69 | model <- darch(Species ~ ., iris, 70 | preProc.params = list(method = c("center", "scale")), 71 | darch.unitFunction = c("sigmoidUnit", "softmaxUnit"), 72 | darch.fineTuneFunction = "minimizeClassifier", 73 | cg.length = 3, cg.switchLayers = 5) 74 | } 75 | } 76 | \seealso{ 77 | \code{\link{darch}}, \code{\link{fineTuneDArch}} 78 | 79 | Other fine-tuning functions: \code{\link{backpropagation}}, 80 | \code{\link{minimizeAutoencoder}}, 81 | \code{\link{rpropagation}} 82 | } 83 | 84 | -------------------------------------------------------------------------------- /R/util.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015-2018 Johannes Rueckert 2 | # 3 | # This file is part of darch. 4 | # 5 | # darch is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # darch is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with darch. If not, see . 17 | 18 | # Utility functions 19 | 20 | # Convert given function name to actual function or return NULL if not found 21 | # TODO use match.fun and disregard functions not exported from other packages or 22 | # duplicates? 23 | characterToFunction <- function(funcName) 24 | { 25 | if (is.function(funcName)) 26 | { 27 | return (funcName) 28 | } 29 | 30 | results <- utils::getAnywhere(funcName) 31 | 32 | if (length(results$objs) == 0) 33 | { 34 | return (NULL) 35 | } 36 | 37 | func <- NULL 38 | funcsFound <- 0 39 | funcChosen <- NULL 40 | 41 | for (i in 1:length(results$objs)) 42 | { 43 | if (is.function(results$objs[[i]])) 44 | { 45 | if (funcsFound == 0) 46 | { 47 | func <- results$objs[[i]] 48 | funcChosen <- i 49 | } 50 | 51 | funcsFound <- if (results$dups[i]) funcsFound else funcsFound + 1 52 | } 53 | } 54 | 55 | if (funcsFound > 1) 56 | { 57 | futile.logger::flog.warn(paste("%s different functions matching '%s' were", 58 | "found, using the one from \"%s\". Pass function directly to use a", 59 | "different one"), funcsFound, funcName, results$where[funcChosen]) 60 | } 61 | 62 | func 63 | } 64 | 65 | # Find function in a list of function names by comparing function bodies; 66 | # returns function name if found, NULL otherwise 67 | functionToCharacter <- function(needle, default = "unknown function", 68 | package = "package:darch") 69 | { 70 | needleBody <- body(needle) 71 | needleBodyLength <- length(needleBody) 72 | 73 | for (functionName in utils::lsf.str(package)) 74 | { 75 | functionBody <- body(functionName) 76 | 77 | if (needleBodyLength == length(functionBody) 78 | && length(intersect(as.character(needleBody), as.character(functionBody))) 79 | == needleBodyLength) 80 | { 81 | return(functionName) 82 | } 83 | } 84 | 85 | return(default) 86 | } 87 | 88 | # TODO solve better 89 | getErrorFunctionName <- function(func) 90 | { 91 | func(matrix(0), matrix(0))[[1]] 92 | } 93 | 94 | # Helper to check whether a variable is exactly FALSE 95 | is.logical.length1 <- function(variable, logical) 96 | { 97 | return(length(variable) == 1 && is.logical(variable) && variable == logical) 98 | } 99 | 100 | # clean deparsing on only one line and without structure() constructs for lists 101 | deparseClean <- function(s) 102 | { 103 | paste(deparse(s, control = c("keepInteger")), collapse = "") 104 | } -------------------------------------------------------------------------------- /man/minimize.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/minimize.R 3 | \name{minimize} 4 | \alias{minimize} 5 | \title{Minimize a differentiable multivariate function.} 6 | \usage{ 7 | minimize(X, f, length, red, dims, data, target, epochSwitch, matMult) 8 | } 9 | \arguments{ 10 | \item{X}{Starting point. An Array of the weights.} 11 | 12 | \item{f}{Function for calculating the function value and the 13 | partial derivatives} 14 | 15 | \item{length}{Maximum number of line searches or maximum allowed number of 16 | function evaluations if negative} 17 | 18 | \item{red}{Expected reduction in function value in the first search.} 19 | 20 | \item{dims}{Parameter to the function f.} 21 | 22 | \item{data}{Parameter to the function f.} 23 | 24 | \item{target}{Parameter to the function f.} 25 | 26 | \item{epochSwitch}{Parameter to the function f.} 27 | 28 | \item{matMult}{Matrix multiplication function.} 29 | } 30 | \value{ 31 | The function returns the found solution "X", a vector of function 32 | values "fX" indicating the progress made and "i" the number of iterations 33 | (line searches or function evaluations, depending on the sign of "length") 34 | used. 35 | } 36 | \description{ 37 | This function is a direct translation from the Matlab source code of the 38 | minimize function from Carl Edward Rasmussen. 39 | } 40 | \details{ 41 | Minimize a differentiable multivariate function. 42 | 43 | Usage: [X, fX, i] <- minimize(X, f, length, P1, P2, P3, ... ) 44 | 45 | where the starting point is given by "X" (D by 1), and the function named in 46 | the string "f", must return a function value and a vector of partial 47 | derivatives of f wrt X, the "length" gives the length of the run: if it is 48 | positive, it gives the maximum number of line searches, if negative its 49 | absolute gives the maximum allowed number of function evaluations. You can 50 | (optionally) give "length" a second component, which will indicate the 51 | reduction in function value to be expected in the first line-search (defaults 52 | to 1.0). The parameters P1, P2, P3, ... are passed on to the function f. 53 | 54 | The function returns when either its length is up, or if no further progress 55 | can be made (ie, we are at a (local) minimum, or so close that due to 56 | numerical problems, we cannot get any closer). NOTE: If the function 57 | terminates within a few iterations, it could be an indication that the 58 | function values and derivatives are not consistent (i.e., there may be a bug in 59 | the implementation of your "f" function). The function returns the found 60 | solution "X", a vector of function values "fX" indicating the progress made 61 | and "i" the number of iterations (line searches or function evaluations, 62 | depending on the sign of "length") used. 63 | The Polack-Ribiere flavour of conjugate gradients is used to compute search 64 | directions, and a line search using quadratic and cubic polynomial 65 | approximations and the Wolfe-Powell stopping criteria is used together with 66 | the slope ratio method for guessing initial step sizes. Additionally a bunch 67 | of checks are made to make sure that exploration is taking place and that 68 | extrapolation will not be unboundedly large. 69 | See also: checkgrad 70 | Copyright (C) 2001 - 2006 by Carl Edward Rasmussen (2006-09-08). 71 | } 72 | \seealso{ 73 | \code{\link{minimizeAutoencoder}}, \code{\link{minimizeClassifier}} 74 | } 75 | \keyword{internal} 76 | 77 | -------------------------------------------------------------------------------- /R/rbmUpdate.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2016 Martin Drees 2 | # 3 | # This file is part of darch. 4 | # 5 | # darch is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # darch is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with darch. If not, see . 17 | 18 | #' Function for updating the weights and biases of an \code{RBM} 19 | #' 20 | #' This function updates the weights and biases for an \code{\linkS4class{RBM}} 21 | #' network. It is saved in the attribute \code{updateFunction} of the 22 | #' \code{RBM} object and called from the training function 23 | #' \code{\link{trainRBM}}. 24 | #' 25 | #' @param rbm A instance of the class \code{\linkS4class{RBM}}. 26 | #' @param matMult Matrix multiplication function. 27 | #' @param ... Additional arguments. 28 | #' @return The updated \code{\linkS4class{RBM}}. 29 | #' @export 30 | #' @seealso \code{\linkS4class{RBM}} 31 | #' @keywords internal 32 | rbmUpdate <- function(rbm, matMult = getParameter(".matMult", net = rbm)) 33 | { 34 | # get parameters 35 | momentum <- getMomentum(rbm) 36 | weightsInc <- rbm@weightsInc 37 | weights <- rbm@weights 38 | visibleBiasInc <- rbm@visibleBiasesInc 39 | visibleBiases <- rbm@visibleBiases 40 | hiddenBiasInc <- rbm@hiddenBiasesInc 41 | hiddenBiases <- rbm@hiddenBiases 42 | data <- rbm@posPhaseData[[1]] 43 | posHiddenProbs <- rbm@posPhaseData[[2]][[1]] 44 | negativeData <- rbm@visibleUnitStates[[1]] 45 | negHiddenProbs <- rbm@hiddenUnitStates[[1]] 46 | learnRate <- (getParameter(".rbm.learnRate", net = rbm) 47 | * getParameter(".rbm.learnRateScale", net = rbm) ^ rbm@epochs 48 | * (1 - momentum)) 49 | weightDecay <- getParameter(".rbm.weightDecay", net = rbm) 50 | batchSize <- getParameter(".rbm.batchSize", net = rbm) 51 | 52 | # positive phase 53 | posProducts <- matMult(t(data), posHiddenProbs) 54 | posHiddienAct <- colSums(posHiddenProbs) 55 | posVisibleAct <- colSums(data) 56 | 57 | # negative phase 58 | negProducts <- matMult(t(negativeData), negHiddenProbs) 59 | negHiddienAct <- colSums(negHiddenProbs) 60 | negVisibleAct <- colSums(negativeData) 61 | 62 | # update 63 | weightsInc <- momentum * weightsInc + learnRate * 64 | (((posProducts - negProducts) / batchSize) - weightDecay * weights) 65 | visibleBiasInc <- momentum * visibleBiasInc + (learnRate / batchSize) * 66 | (posVisibleAct - negVisibleAct - visibleBiases * weightDecay) 67 | hiddenBiasInc <- momentum * hiddenBiasInc + (learnRate / batchSize) * 68 | (posHiddienAct - negHiddienAct - hiddenBiases * weightDecay) 69 | 70 | weights <- weights + weightsInc 71 | 72 | if (getParameter(".normalizeWeights", net = rbm)) 73 | { 74 | normalizeWeightsCpp(weights, getParameter(".normalizeWeightsBound", 75 | net = rbm)) 76 | } 77 | 78 | rbm@weights <- weights 79 | rbm@visibleBiases <- visibleBiases + visibleBiasInc 80 | rbm@hiddenBiases <- hiddenBiases + hiddenBiasInc 81 | rbm@weightsInc <- weightsInc 82 | rbm@visibleBiasesInc <- visibleBiasInc 83 | rbm@hiddenBiasesInc <- hiddenBiasInc 84 | 85 | rbm 86 | } -------------------------------------------------------------------------------- /R/test.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015-2018 Johannes Rueckert 2 | # 3 | # This file is part of darch. 4 | # 5 | # darch is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # darch is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with darch. If not, see . 17 | 18 | #' Test classification network. 19 | #' 20 | #' Forward-propagate given data through the deep neural network and return 21 | #' classification accuracy using the given labels. 22 | #' 23 | #' This is primarily a convenience function similar to \link{predict.DArch} with 24 | #' classification performance measurements instead of network output, 25 | #' and it returns a list of accuracy indicators (raw network error, percentage 26 | #' of incorrect classifications and absolute number of incorrect 27 | #' classifications). 28 | #' 29 | #' @param darch \code{\linkS4class{DArch}} instance. 30 | #' @param newdata New data to use, \code{NULL} to use training data. 31 | #' @param targets Labels for the \code{data}, \code{NULL} to use training 32 | #' labels (only possible when \code{data} is \code{NULL} as well). 33 | #' @return Vector containing error function output, percentage of incorrect 34 | #' classifications and absolute number of incorrect classifications. 35 | #' @examples 36 | #' \dontrun{ 37 | #' data(iris) 38 | #' model <- darch(Species ~ ., iris, retainData = T) 39 | #' classificationStats <- darchTest(model) 40 | #' } 41 | #' @export 42 | #' @family darch interface functions 43 | darchTest <- function(darch, newdata = NULL, targets = T) 44 | { 45 | if (is.null(newdata)) 46 | { 47 | if (!getParameter(".retainData")) 48 | { 49 | stop(futile.logger::flog.error("No data available for prediction")) 50 | } 51 | 52 | dataSet <- darch@dataSet 53 | } 54 | else 55 | { 56 | if (is.null(darch@dataSet@formula) && is.logical.length1(targets, T)) 57 | { 58 | stop(futile.logger::flog.error( 59 | "No target data provided for classification test")) 60 | } 61 | 62 | dataSet <- createDataSet(data = newdata, 63 | targets = targets, 64 | dataSet = darch@dataSet) 65 | } 66 | 67 | e <- testDArch(darch, dataSet@data, dataSet@targets, "All Data", 68 | getParameter(".darch.isClass")) 69 | list("error" = e[1], "percentIncorrect" = e[2], "numIncorrect" = e[3]) 70 | } 71 | 72 | # TODO documentation 73 | testDArch <- function(darch, data, targets, dataType, isClass) 74 | { 75 | execOut <- getParameter(".darch.executeFunction")(darch, data) 76 | fullRank <- getParameter(".preProc.fullRank.targets") 77 | 78 | tError <- getParameter(".darch.errorFunction")(targets, execOut) 79 | classError <- NA 80 | numIncorrect <- NA 81 | if (isClass) 82 | { 83 | rows <- nrow(targets) 84 | cols <- ncol(targets) 85 | execOut <- 86 | (if (cols > 1 && !fullRank) 87 | diag(cols)[max.col(execOut, ties.method = "first"),] 88 | else (execOut > .5)*1) 89 | numIncorrect <- sum(rowMeans(execOut == targets) < 1) 90 | classError <- numIncorrect / rows * 100 91 | futile.logger::flog.info("Classification error on %s: %s%% (%s/%s)", 92 | dataType, round(classError, 2), numIncorrect, rows) 93 | } 94 | 95 | futile.logger::flog.info("%s %s: %.3f", dataType, tError[[1]], tError[[2]]) 96 | 97 | c(tError[[2]], classError, numIncorrect) 98 | } -------------------------------------------------------------------------------- /R/errorFunctions.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2016 Martin Drees 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | 19 | #' Mean squared error function 20 | #' 21 | #' The function calculates the mean squared error (MSE) from the \code{original} 22 | #' and \code{estimate} parameters. 23 | #' 24 | #' This function is a valid value for both \code{\link{darch}} parameters 25 | #' \code{rbm.errorFunction} and \code{darch.errorFunction}. 26 | #' 27 | #' @param original The original data matrix. 28 | #' @param estimate The calculated data matrix. 29 | #' @return A list with the name of the error function in the first entry and the 30 | #' error value in the second entry. 31 | #' @examples 32 | #' \dontrun{ 33 | #' data(iris) 34 | #' model <- darch(Species ~ ., iris, rbm.errorFunction = "mseError", 35 | #' darch.errorFunction = "mseError") 36 | #' } 37 | #' @family error functions 38 | #' @export 39 | mseError <- function(original, estimate) 40 | { 41 | # TODO colMeans? .5*? 42 | list("MSE", sum(colMeans((original - estimate) ^ 2))) 43 | } 44 | 45 | #' Root-mean-square error function 46 | #' 47 | #' The function calculates the root-mean-square error (RMSE) from the 48 | #' \code{original} and \code{estimate} parameters. 49 | #' 50 | #' This function is a valid value for both \code{\link{darch}} parameters 51 | #' \code{rbm.errorFunction} and \code{darch.errorFunction}. 52 | #' 53 | #' @param original The original data matrix. 54 | #' @param estimate The calculated data matrix. 55 | #' @return A list with the name of the error function in the first entry and the 56 | #' error value in the second entry. 57 | #' @examples 58 | #' \dontrun{ 59 | #' data(iris) 60 | #' model <- darch(Species ~ ., iris, rbm.errorFunction = "rmseError", 61 | #' darch.errorFunction = "rmseError") 62 | #' } 63 | #' @family error functions 64 | #' @export 65 | rmseError <- function(original, estimate) 66 | { 67 | list("RMSE", sum(sqrt(colMeans((original - estimate) ^ 2)))) 68 | } 69 | 70 | #' Cross entropy error function 71 | #' 72 | #' The function calculates the cross entropy error from the \code{original} and 73 | #' \code{estimate} parameters. 74 | #' 75 | #' This function can be used for the \code{darch.errorFunction} parameter of the 76 | #' \code{\link{darch}} function, but is only a valid error function if used in 77 | #' combination with the \code{\link{softmaxUnit}} activation function! It is not 78 | #' a valid value for the parameter \code{rbm.errorFunction}. 79 | #' 80 | #' @param original The original data matrix. 81 | #' @param estimate The calculated data matrix. 82 | #' @return A list with the name of the error function in the first entry and the 83 | #' error value in the second entry. 84 | #' @examples 85 | #' \dontrun{ 86 | #' data(iris) 87 | #' model <- darch(Species ~ ., iris, darch.errorFunction = "crossEntropyError") 88 | #' } 89 | #' @family error functions 90 | #' @references Rubinstein, R.Y., Kroese, D.P. (2004). The Cross-Entropy Method: 91 | #' A Unified Approach to Combinatorial Optimization, Monte-Carlo Simulation, 92 | #' and Machine Learning, Springer-Verlag, New York. 93 | #' @export 94 | crossEntropyError <- function(original, estimate) 95 | { 96 | # C = - sum [all cases and outputs] (d*log(y) + (1-d)*log(1-y) ) 97 | c <- -sum(colMeans(original * log(pmax(estimate, .Machine$double.eps)) + 98 | (1 - original) * log(pmax(1 - estimate, .Machine$double.eps)))) 99 | #c <- -sum(original*log(estimate)) 100 | list("Cross Entropy error",c) 101 | } 102 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Downloads from the RStudio CRAN mirror](http://cranlogs.r-pkg.org/badges/darch)](http://cran.rstudio.com/package=darch) 2 | 3 | 4 | darch 5 | ===== 6 | 7 | #### Create deep architectures in the R programming language 8 | 9 | 10 | ### Installation 11 | 12 | The previous stable version of `darch` (0.12.0) can be installed from CRAN using 13 | 14 | ```R 15 | install.packages("darch") 16 | ``` 17 | 18 | When using [devtools](https://github.com/hadley/devtools/), the latest git version (identifiable by a version number ending in something greater than or equal to 9000 and by the fact that it is regularly broken) can be installed using 19 | 20 | ```R 21 | install_github("maddin79/darch") 22 | ``` 23 | 24 | or, if you want the latest stable version (0.13.0), 25 | 26 | ```R 27 | install_github("maddin79/darch@v0.13.0") 28 | ``` 29 | 30 | Then, use `?darch` to view its documentation or `example("darch")` to view some simple examples. For further examples, see the [examples directory](https://github.com/maddin79/darch/tree/master/examples). 31 | 32 | ### Building 33 | 34 | Assuming you use RStudio, open the darch.Rproj after cloning the repository. Make sure you have `gcc-fortran` installed (part of the `r-base-dev` package on ubuntu). Now install the dependencies: 35 | 36 | ```R 37 | install.packages(c("ggplot2", "reshape2", "futile.logger", "caret", "Rcpp", "RcppParallel")) 38 | ``` 39 | 40 | You can then build the package using `Build -> Clean and Rebuild`. 41 | 42 | ### About 43 | 44 | The darch package is built on the basis of the code from G. E. Hinton and R. R. Salakhutdinov 45 | (available under [Matlab Code for deep belief nets](http://www.cs.toronto.edu/~hinton/MatlabForSciencePaper.html 46 | "Matlab for science paper") : last visit: 12.11.2015). 47 | 48 | This package is for generating neural networks with many layers (deep architectures) and train them with the method introduced by the publications "A fast learning algorithm for deep belief nets" (G. E. Hinton, S. Osindero, Y. W. Teh) and "Reducing the dimensionality of data with neural networks" (G. E. Hinton, R. R. Salakhutdinov). This method includes a pre training with the contrastive divergence method published by G.E Hinton (2002) and a fine tuning with common known training algorithms like backpropagation or conjugate gradient, as well as more recent techniques like dropout and maxout. 49 | 50 | Copyright (C) 2013-2016 Martin Drees and contributors 51 | 52 | #### References 53 | Hinton, G. E., S. Osindero, Y. W. Teh, A fast learning algorithm for deep belief nets, 54 | Neural Computation 18(7), S. 1527-1554, DOI: 55 | [10.1162/neco.2006.18.7.1527](http://dx.doi.org/10.1162/neco.2006.18.7.1527), 2006. 56 | 57 | Hinton, G. E., R. R. Salakhutdinov, Reducing the dimensionality of data with neural 58 | networks, Science 313(5786), S. 504-507, DOI: 59 | [10.1126/science.1127647](http://dx.doi.org/10.1126/science.1127647), 2006. 60 | 61 | Hinton, G. E., Training products of experts by minimizing contrastive divergence, 62 | Neural Computation 14(8), S. 1711-1800, DOI: 63 | [10.1162/089976602760128018](http://dx.doi.org/10.1162/089976602760128018), 2002. 64 | 65 | Hinton, Geoffrey E. et al. (2012). "Improving neural networks by preventing coadaptation of feature detectors". In: Clinical Orthopaedics and Related Research abs/1207.0580. URL : [arxiv.org](http://arxiv.org/abs/1207.0580). 66 | 67 | Goodfellow, Ian J. et al. (2013). "Maxout Networks". In: Proceedings of the 30th International Conference on Machine Learning, ICML 2013, Atlanta, GA, USA, 16-21 June 2013, pp. 1319–1327. URL : [jmlr.org](http://jmlr.org/proceedings/papers/v28/goodfellow13.html). 68 | 69 | Drees, Martin (2013). "Implementierung und Analyse von tiefen Architekturen 70 | in R". German. Master's thesis. Fachhochschule Dortmund. 71 | 72 | Rueckert, Johannes (2015). "Extending the Darch library for deep 73 | architectures". Project thesis. Fachhochschule Dortmund. 74 | URL: [saviola.de](http://static.saviola.de/publications/rueckert_2015.pdf). 75 | 76 | Rueckert, Johannes (20116). "Toward State-of-the-Art Deep Learning in R: 77 | darch 1.0". Master's thesis. Fachhochschule Dortmund. 78 | URL: [saviola.de](http://static.saviola.de/publications/rueckert_2016.pdf). 79 | (This describes version [0.12.0](https://github.com/maddin79/darch/releases/tag/v0.12.0)) 80 | -------------------------------------------------------------------------------- /R/runDArch.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2016 Martin Drees 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | 19 | #' @include darch.Class.R 20 | NULL 21 | 22 | #' Forward-propagates data through the network 23 | #' 24 | #' Runs the \code{DArch} in a feed-forward manner and returns the output. 25 | #' 26 | #' Input and output layer can be chosen via the parameters \code{inputLayer} 27 | #' and \code{outputLayer}. 28 | #' 29 | #' @param darch A instance of the class \code{\linkS4class{DArch}}. 30 | #' @param data The input data to execute the darch on. 31 | #' @param inputLayer Into which layer the given data is to be fed. Absolute 32 | #' number starting at 1 for the input layer. 33 | #' @param outputLayer The output of which layer is to be returned, absolute 34 | #' number starting a 0 for the input layer (i.e. pre-processed data is 35 | #' returned). 36 | #' @param matMult Function to use for matrix multiplication. 37 | #' @return The network output 38 | #' 39 | #' @seealso \code{\link{darch}} 40 | #' @family darch execute functions 41 | #' @keywords internal 42 | runDArch <- function(darch, data, inputLayer = 1, 43 | outputLayer = length(darch@layers), 44 | matMult = getParameter(".matMult")) 45 | { 46 | layers <- darch@layers 47 | numRows <- nrow(data) 48 | 49 | if (outputLayer == 0) 50 | { 51 | return(data) 52 | } 53 | 54 | for (i in inputLayer:outputLayer) 55 | { 56 | data <- cbind(data,rep(1,numRows)) 57 | data <- layers[[i]][["unitFunction"]](matMult(data, 58 | layers[[i]][["weights"]]), net = darch)[[1]] 59 | } 60 | 61 | data 62 | } 63 | 64 | #' Forward-propagate data through the network with dropout inference 65 | #' 66 | #' 67 | #' If dropout was disabled, \code{\link{runDArch}} will be called instead. 68 | #' 69 | #' @param dropout Dropout rates for the layers. 70 | #' @param iterations If greater than 0, the numbr of iterations to use for 71 | #' moment matching dropout inference. 72 | #' @return The network output. 73 | #' 74 | #' @inheritParams runDArch 75 | #' @seealso \code{\link{darch}} 76 | #' @family darch execute functions 77 | #' @keywords internal 78 | runDArchDropout <- function(darch, data, inputLayer = 1, 79 | outputLayer = length(darch@layers), matMult = getParameter(".matMult"), 80 | dropout = getParameter(".darch.dropout"), 81 | iterations = getParameter(".darch.dropout.momentMatching")) 82 | { 83 | layers <- darch@layers 84 | numRows <- nrow(data) 85 | # In case DropConnect is disabled, we append 0 for the last layer 86 | dropout <- c(dropout, 0) 87 | 88 | if (outputLayer == 0) 89 | { 90 | return(data) 91 | } 92 | 93 | for (i in inputLayer:outputLayer) 94 | { 95 | data <- cbind(data, rep(1, numRows)) 96 | input <- matMult(data, (1 - dropout[i + 1]) * layers[[i]][["weights"]]) 97 | 98 | if (iterations > 0) 99 | { 100 | E <- as.vector(input) 101 | V <- as.vector(dropout[i + 1] * (1 - dropout[i + 1]) * 102 | (matMult(data ^ 2, layers[[i]][["weights"]] ^ 2))) 103 | n <- length(E) 104 | 105 | ret <- matrix(rep(0, n), nrow = numRows) 106 | 107 | for (j in 1:iterations) 108 | { 109 | ret <- ret + layers[[i]][["unitFunction"]](matrix(rnorm(n, E, V), 110 | nrow = numRows), darch = darch)[[1]] 111 | } 112 | 113 | data <- ret/iterations 114 | } 115 | else 116 | { 117 | data <- layers[[i]][["unitFunction"]](input, net = darch)[[1]] 118 | } 119 | } 120 | 121 | data 122 | } -------------------------------------------------------------------------------- /src/rpropagation.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | # Copyright (C) 2015-2018 Johannes Rueckert 3 | # 4 | # This file is part of darch. 5 | # 6 | # darch is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # darch is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with darch. If not, see . 18 | */ 19 | 20 | #include 21 | #include 22 | 23 | using namespace RcppParallel; 24 | using namespace Rcpp; 25 | 26 | struct RpropGradients : public Worker 27 | { 28 | const RMatrix gg; 29 | 30 | RMatrix gradients; 31 | 32 | int colLength; 33 | 34 | RpropGradients(const NumericMatrix gg, NumericMatrix gradients) 35 | : gg(gg), gradients(gradients) { 36 | colLength = gg.nrow(); 37 | } 38 | 39 | void operator()(std::size_t begin_col, std::size_t end_col) 40 | { 41 | for (int i = begin_col; i < end_col; i++) 42 | { 43 | for (int j = 0; j < colLength; j++) 44 | { 45 | if (gg(j, i) < 0) gradients(j, i) = 0; 46 | } 47 | } 48 | } 49 | }; 50 | 51 | // edits in-place 52 | // [[Rcpp::export]] 53 | void rpropGradientsCpp(NumericMatrix gg, NumericMatrix gradients) 54 | { 55 | int ncols = gg.ncol(); 56 | RpropGradients worker(gg, gradients); 57 | 58 | parallelFor(0, ncols, worker); 59 | } 60 | 61 | struct RpropDelta : public Worker 62 | { 63 | const RMatrix gg; 64 | 65 | RMatrix delta; 66 | 67 | double inc, dec, minDelta, maxDelta; 68 | 69 | int colLength; 70 | 71 | RpropDelta(const NumericMatrix gg, NumericMatrix delta, double inc, 72 | double dec, double minDelta, double maxDelta) : 73 | gg(gg), delta(delta), inc(inc), dec(dec), minDelta(minDelta), 74 | maxDelta(maxDelta) { 75 | colLength = gg.nrow(); 76 | } 77 | 78 | void operator()(std::size_t begin_col, std::size_t end_col) 79 | { 80 | for (int i = begin_col; i < end_col; i++) 81 | { 82 | for (int j = 0; j < colLength; j++) 83 | { 84 | if (gg(j, i) < 0) delta(j, i) = std::max(delta(j, i) * dec, minDelta); 85 | else if (gg(j, i) > 0) delta(j, i) = std::min(delta(j, i) * inc, maxDelta); 86 | } 87 | } 88 | } 89 | }; 90 | 91 | // edits in-place 92 | // [[Rcpp::export]] 93 | void rpropDeltaCpp(NumericMatrix gg, NumericMatrix delta, double inc, 94 | double dec, double minDelta, double maxDelta) 95 | { 96 | int ncols = gg.ncol(); 97 | RpropDelta worker(gg, delta, inc, dec, minDelta, maxDelta); 98 | 99 | parallelFor(0, ncols, worker); 100 | } 101 | 102 | struct RpropDeltaWiRpropPlus : public Worker 103 | { 104 | const RMatrix gg, gradients, delta; 105 | 106 | RMatrix deltaW; 107 | 108 | const double newE, oldE; 109 | 110 | int colLength; 111 | 112 | RpropDeltaWiRpropPlus(const NumericMatrix gg, NumericMatrix deltaW, 113 | const NumericMatrix gradients, const NumericMatrix delta, 114 | const double newE, const double oldE) : 115 | gg(gg), deltaW(deltaW), gradients(gradients), delta(delta), newE(newE), 116 | oldE(oldE) 117 | { 118 | colLength = gg.nrow(); 119 | } 120 | 121 | void operator()(std::size_t begin_col, std::size_t end_col) 122 | { 123 | for (int i = begin_col; i < end_col; i++) 124 | { 125 | for (int j = 0; j < colLength; j++) 126 | { 127 | if (gg(j, i) >= 0) 128 | deltaW(j, i) = 129 | ((gradients(j, i) < 0) - (gradients(j, i) > 0)) * delta(j, i); 130 | else if (newE > oldE) deltaW(j, i) = -deltaW(j, i); 131 | else deltaW(j, i) = 0; 132 | } 133 | } 134 | } 135 | }; 136 | 137 | // edits in-place 138 | // [[Rcpp::export]] 139 | void rpropDeltaWiRpropPlusCpp(const NumericMatrix gg, 140 | NumericMatrix deltaW, const NumericMatrix gradients, 141 | const NumericMatrix delta, const double newE, const double oldE) 142 | { 143 | int ncols = gg.ncol(); 144 | RpropDeltaWiRpropPlus worker(gg, deltaW, gradients, delta, newE, oldE); 145 | 146 | parallelFor(0, ncols, worker); 147 | } 148 | -------------------------------------------------------------------------------- /R/caret.R: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Johannes Rueckert 2 | # 3 | # This file is part of darch. 4 | # 5 | # darch is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # darch is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with darch. If not, see . 17 | 18 | #' Creates a custom caret model for \code{darch}. 19 | #' 20 | #' This function creates a \code{caret} model description to enable training 21 | #' \code{DArch} instances with the \code{\link[caret]{train}} function. See the 22 | #' documentation on custom caret models for further information and examples on 23 | #' how to create valid \code{params} and \code{grid} values. 24 | #' 25 | #' @seealso \href{https://topepo.github.io/caret/custom_models.html}{Caret custom models} 26 | #' @param params \code{\link{data.frame}} of parameters or \code{NULL} to use a 27 | #' simple default (bp.learnRate). 28 | #' @param grid Function which procuces a \code{\link{data.frame}} containing a 29 | #' grid of parameter combinations or \code{NULL} to use a simple default. 30 | #' @return A valid \code{caret} model which can be passed to 31 | #' \code{\link[caret]{train}}. 32 | #' @examples 33 | #' \dontrun{ 34 | #' data(iris) 35 | #' tc <- trainControl(method = "boot", number = 5, allowParallel = F, 36 | #' verboseIter = T) 37 | #' 38 | #' parameters <- data.frame(parameter = c("layers", "bp.learnRate", "darch.unitFunction"), 39 | #' class = c("character", "numeric", "character"), 40 | #' label = c("Network structure", "Learning rate", "unitFunction")) 41 | #' 42 | #' grid <- function(x, y, len = NULL, search = "grid") 43 | #' { 44 | #' df <- expand.grid(layers = c("c(0,20,0)","c(0,10,10,0)","c(0,10,5,5,0)"), 45 | #' bp.learnRate = c(1,2,5,10)) 46 | #' 47 | #' df[["darch.unitFunction"]] <- rep(c("c(tanhUnit, softmaxUnit)", 48 | #' "c(tanhUnit, tanhUnit, softmaxUnit)", 49 | #' "c(tanhUnit, tanhUnit, tanhUnit, softmaxUnit)"), 4) 50 | #' 51 | #' df 52 | #' } 53 | #' 54 | #' caretModel <- train(Species ~ ., data = iris, tuneLength = 12, trControl = tc, 55 | #' method = darchModelInfo(parameters, grid), preProc = c("center", "scale"), 56 | #' darch.numEpochs = 15, darch.batchSize = 6, testing = T, ...) 57 | #' } 58 | #' @export 59 | darchModelInfo <- function(params = NULL, grid = NULL) 60 | { 61 | r <- list(type = c("Classification", "Regression"), 62 | library = "darch") 63 | #loop = NULL) 64 | 65 | if (is.null(params)) 66 | { 67 | params <- data.frame(parameter = c("bp.learnRate"), 68 | class = c("numeric"), 69 | label = c("Learn rate")) 70 | } 71 | 72 | r$parameters <- params 73 | 74 | if (is.null(grid)) 75 | { 76 | grid <- function(x, y, len = NULL, search = "grid") 77 | { 78 | if (search == "grid") 79 | { 80 | out <- data.frame(darch.learnRate = seq.int(.1,5,.1)[1:len]) 81 | } 82 | else 83 | { 84 | out <- data.frame(darch.learnRate = runif(len, .01, 2)) 85 | } 86 | 87 | out 88 | } 89 | } 90 | 91 | r$grid <- grid 92 | 93 | r$fit <- function(x, y, wts, param, lev, last, weights, classProbs, ...) 94 | { 95 | darch(x, y, paramsList = as.list(param), ...) 96 | } 97 | 98 | r$predict <- function(modelFit, newdata, preProc = NULL, submodels = NULL) 99 | { 100 | # TODO switch between class and raw when not dealing with classification? 101 | predict(modelFit, newdata = newdata, type = "character") 102 | } 103 | 104 | r$prob <- function(darch, newdata, preProc = NULL, submodels = NULL) 105 | { 106 | predict(darch, newdata = newdata, type = "raw") 107 | } 108 | 109 | r$levels <- function(darch) 110 | { 111 | if (is.null(darch@dataSet@parameters$dummyVarsTargets$lvls)) NULL else 112 | darch@dataSet@parameters$dummyVarsTargets$lvls[[1]] 113 | } 114 | 115 | r 116 | } 117 | 118 | #' Wrapper for \code{\link[caret]{contr.ltfr}} 119 | #' 120 | #' Simply redirects the call to \code{\link[caret]{contr.ltfr}}, this is done 121 | #' to avoid errors when the caret namespace is not attached. 122 | #' 123 | #' @param ... \code{\link[caret]{contr.ltfr}} parameters. 124 | #' @export 125 | #' @keywords internal 126 | contr.ltfr <- function(...) 127 | { 128 | caret::contr.ltfr(...) 129 | } --------------------------------------------------------------------------------