├── .gitignore ├── R ├── KERAS_train_MNIST_CNN.r ├── LIVE_webcam_CNN_prediction.r ├── LIVE_webcam_CNN_single_blob_prediction.r ├── LIVE_webcam_MNIST_CNN_predict.R ├── LIVE_webcam_blob_detection.r ├── MNIST_CNN_video_predict.r └── Video_balloon_blob.R ├── README.md └── video ├── balloon1.3gp └── videoplayback.mp4 /.gitignore: -------------------------------------------------------------------------------- 1 | .Rproj.user 2 | .Rhistory 3 | .RData 4 | .Ruserdata 5 | rvision_examples.Rproj 6 | -------------------------------------------------------------------------------- /R/KERAS_train_MNIST_CNN.r: -------------------------------------------------------------------------------- 1 | library(keras) 2 | 3 | # Data Preparation ----------------------------------------------------- 4 | 5 | batch_size <- 128 6 | num_classes <- 10 7 | epochs <- 5 8 | 9 | # Input image dimensions 10 | img_rows <- 28 11 | img_cols <- 28 12 | 13 | # The data, shuffled and split between train and test sets 14 | mnist <- dataset_mnist() 15 | x_train <- mnist$train$x 16 | y_train <- mnist$train$y 17 | x_test <- mnist$test$x 18 | y_test <- mnist$test$y 19 | 20 | # Redefine dimension of train/test inputs 21 | x_train <- array_reshape(x_train, c(nrow(x_train), img_rows, img_cols, 1)) 22 | x_test <- array_reshape(x_test, c(nrow(x_test), img_rows, img_cols, 1)) 23 | input_shape <- c(img_rows, img_cols, 1) 24 | 25 | # Transform RGB values into [0,1] range 26 | x_train <- x_train / 255 27 | x_test <- x_test / 255 28 | 29 | cat('x_train_shape:', dim(x_train), '\n') 30 | cat(nrow(x_train), 'train samples\n') 31 | cat(nrow(x_test), 'test samples\n') 32 | 33 | # Convert class vectors to binary class matrices 34 | y_train <- to_categorical(y_train, num_classes) 35 | y_test <- to_categorical(y_test, num_classes) 36 | 37 | # Define MNIST_model ----------------------------------------------------------- 38 | 39 | # Define MNIST_model 40 | MNIST_model <- keras_MNIST_model_sequential() %>% 41 | layer_conv_2d(filters = 32, kernel_size = c(3,3), activation = 'relu', 42 | input_shape = input_shape) %>% 43 | layer_conv_2d(filters = 64, kernel_size = c(3,3), activation = 'relu') %>% 44 | layer_max_pooling_2d(pool_size = c(2, 2)) %>% 45 | layer_dropout(rate = 0.25) %>% 46 | layer_flatten() %>% 47 | layer_dense(units = 128, activation = 'relu') %>% 48 | layer_dropout(rate = 0.5) %>% 49 | layer_dense(units = num_classes, activation = 'softmax') 50 | 51 | # Compile MNIST_model 52 | MNIST_model %>% compile( 53 | loss = loss_categorical_crossentropy, 54 | optimizer = optimizer_adadelta(), 55 | metrics = c('accuracy') 56 | ) 57 | 58 | # Train MNIST_model 59 | MNIST_model %>% fit( 60 | x_train, y_train, 61 | batch_size = batch_size, 62 | epochs = epochs, 63 | validation_split = 0.2 64 | ) 65 | 66 | MNIST_model %>% evaluate(x_test,y_test) 67 | -------------------------------------------------------------------------------- /R/LIVE_webcam_CNN_prediction.r: -------------------------------------------------------------------------------- 1 | library("Rvision") 2 | library("dplyr") 3 | library("keras") 4 | 5 | 6 | # instantiate the model 7 | CNN_model <- application_resnet50(weights = 'imagenet') 8 | 9 | my_stream <- stream(0) 10 | newDisplay("Live test", 400, 400) 11 | old <- Sys.time() 12 | 13 | while(TRUE){ 14 | img <- readNext(my_stream) 15 | blob <- img %>% 16 | split() %>% 17 | '['(c(3,1)) %>% 18 | do.call(function(x, y) (x - y) > 50, .) %>% 19 | morph("close", iterations = 3) %>% 20 | # medianBlur(21) %>% 21 | simpleBlobDetector(max_area = Inf, min_area = 1000, blob_color = 255, 22 | filter_by_convexity = FALSE, 23 | filter_by_inertia = FALSE, min_threshold = 0) 24 | 25 | # cat(i,"-", blob$id, ",", blob$x, ",", blob$x, ",", blob$size, "\n") 26 | new <- Sys.time() 27 | 28 | if(nrow(blob) > 0){ 29 | img_no_blob <- img # copy of image without annotated bounding box for prediction 30 | bbb = 10 31 | x1 = blob$x[1] + bbb + blob$size[1]/2 32 | y1 = blob$y[1] + bbb + blob$size[1]/2 33 | x2 = blob$x[1] - bbb - blob$size[1]/2 34 | y2 = blob$y[1] - bbb - blob$size[1]/2 35 | drawRectangle(img, x1,y1,x2,y2, thickness = 2) 36 | 37 | img32 <- resize(img_no_blob, 224,224) 38 | img32_array <- image_to_array(as.matrix(img32)) 39 | img32_array <- array_reshape(img32_array, c(1, dim(img32_array))) 40 | img32_proc <- imagenet_preprocess_input(img32_array) 41 | preds <- CNN_model %>% predict(img32_proc) 42 | bb1 <- imagenet_decode_predictions(preds, top = 1)[[1]][,2:3] 43 | print(bb1) 44 | 45 | drawText(img, paste0(format(as.character(bb1[1])), 46 | "-", format(round(as.numeric(bb1[2]),2), digits = 2, nsmall = 0), 47 | "%"), 48 | x = 10, y = 10, font_face = 'plain', font_scale = 3.5, 49 | color = "red", thickness = 3) 50 | } 51 | old <- new 52 | 53 | display(img, "Live test", 1, 360*2, 640*2) 54 | } 55 | 56 | destroyDisplay("Live test") 57 | release(my_stream) 58 | -------------------------------------------------------------------------------- /R/LIVE_webcam_CNN_single_blob_prediction.r: -------------------------------------------------------------------------------- 1 | library("Rvision") 2 | library("dplyr") 3 | library("magick") 4 | library("keras") 5 | 6 | ######## Not complete: 7 | # need to extract blob and predict on that, not the entire image 8 | 9 | # instantiate the model, this may take a few moments... 10 | model <- application_resnet50(weights = 'imagenet') 11 | 12 | my_stream <- stream(0) 13 | newDisplay("Live test", 400, 400) 14 | old <- Sys.time() 15 | 16 | while(TRUE){ 17 | img <- readNext(my_stream) 18 | blob <- img %>% 19 | split() %>% 20 | '['(c(3,1)) %>% 21 | do.call(function(x, y) (x - y) > 50, .) %>% 22 | morph("close", iterations = 3) %>% 23 | # medianBlur(21) %>% 24 | simpleBlobDetector(max_area = Inf, min_area = 1000, blob_color = 255, 25 | filter_by_convexity = FALSE, 26 | filter_by_inertia = FALSE, min_threshold = 0) 27 | 28 | # cat(i,"-", blob$id, ",", blob$x, ",", blob$x, ",", blob$size, "\n") 29 | new <- Sys.time() 30 | 31 | if(nrow(blob) > 0){ 32 | # for(i in 1:nrow(blob)){ 33 | # drawRectangle(img, blob$x[i] - 1 + blob$size[i]/2, 34 | # blob$y[i] - 1 + blob$size[i]/2, 35 | # blob$x[i] - 1 - blob$size[i]/2, 36 | # blob$y[i] - 1 - blob$size[i]/2, thickness = 2) 37 | # } 38 | img_no_blob <- img # copy of image without annotated bounding box for prediction 39 | bbb = 10 40 | x1 = blob$x[1] + bbb + blob$size[1]/2 41 | y1 = blob$y[1] + bbb + blob$size[1]/2 42 | x2 = blob$x[1] - bbb - blob$size[1]/2 43 | y2 = blob$y[1] - bbb - blob$size[1]/2 44 | drawRectangle(img, x1,y1,x2,y2, thickness = 2) 45 | 46 | # f <- function(m) t(m)[,nrow(m):1] 47 | blob1_img <- as.matrix(img) 48 | # blob1_t <- f(blob1_img[,,1:3]) 49 | # image.default(blob1_t[x2:x1,y2:y1]) 50 | # blob1_bb <- blob1_img[x2:x1,y2:y1,] 51 | 52 | img32 <- resize(img_no_blob, 224,224) 53 | img32_array <- image_to_array(as.matrix(img32)) 54 | img32_array <- array_reshape(img32_array, c(1, dim(img32_array))) 55 | img32_proc <- imagenet_preprocess_input(img32_array) 56 | preds <- model %>% predict(img32_proc) 57 | bb1 <- imagenet_decode_predictions(preds, top = 1)[[1]][,2:3] 58 | print(bb1) 59 | 60 | drawText(img, paste0(format(as.character(bb1[1])), 61 | "-", format(round(as.numeric(bb1[2]),2), digits = 2, nsmall = 0), 62 | "%"), 63 | x = 10, y = 10, font_face = 'plain', font_scale = 3.5, 64 | color = "red", thickness = 3) 65 | } 66 | old <- new 67 | 68 | display(img, "Live test", 1, 360*2, 640*2) 69 | } 70 | 71 | destroyDisplay("Live test") 72 | release(my_stream) 73 | -------------------------------------------------------------------------------- /R/LIVE_webcam_MNIST_CNN_predict.R: -------------------------------------------------------------------------------- 1 | library("Rvision") 2 | library("dplyr") 3 | 4 | 5 | my_stream <- stream(0) 6 | newDisplay("Live test", 400, 400) 7 | old <- Sys.time() 8 | 9 | while(TRUE){ 10 | img <- readNext(my_stream) 11 | blob <- img %>% 12 | split() %>% 13 | '['(c(3,1)) %>% 14 | do.call(function(x, y) (x - y) > 50, .) %>% 15 | morph("close", iterations = 3) %>% 16 | # medianBlur(21) %>% 17 | simpleBlobDetector(max_area = Inf, min_area = 1000, blob_color = 255, 18 | filter_by_convexity = FALSE, 19 | filter_by_inertia = FALSE, min_threshold = 0) 20 | 21 | # cat(i,"-", blob$id, ",", blob$x, ",", blob$x, ",", blob$size, "\n") 22 | new <- Sys.time() 23 | 24 | if(nrow(blob) > 0){ 25 | img_no_blob <- img # copy of image without annotated bounding box for prediction 26 | bbb = 10 27 | x1 = blob$x[1] + bbb + blob$size[1]/2 28 | y1 = blob$y[1] + bbb + blob$size[1]/2 29 | x2 = blob$x[1] - bbb - blob$size[1]/2 30 | y2 = blob$y[1] - bbb - blob$size[1]/2 31 | drawRectangle(img, x1,y1,x2,y2, thickness = 2) 32 | 33 | img32 <- resize(img_no_blob, 28,28) 34 | img32_gray <- changeColorSpace(img32,"GRAY") 35 | img32_array <- image_to_array(as.matrix(img32_gray)) 36 | img32_array <- array_reshape(img32_array, c(1, dim(img32_array))) 37 | preds <- MNIST_model %>% 38 | predict(img32_array) %>% 39 | data.frame() %>% 40 | magrittr::set_colnames(seq(0,9,1)) 41 | preds$top <- as.numeric(colnames(preds)[max.col(preds,ties.method="first")]) 42 | print(preds[1,]) 43 | 44 | drawText(img, paste0("pred: ", format(preds$top[1], digits = 2, nsmall = 0)), 45 | x = 30, y = 30, font_face = 'plain', font_scale = 4, 46 | color = "red", thickness = 3) 47 | } 48 | old <- new 49 | 50 | display(img, "Live test", 1, 360*2, 640*2) 51 | } 52 | 53 | destroyDisplay("Live test") 54 | release(my_stream) 55 | -------------------------------------------------------------------------------- /R/LIVE_webcam_blob_detection.r: -------------------------------------------------------------------------------- 1 | library("Rvision") 2 | library("dplyr") 3 | 4 | my_stream <- stream(0) # 0 will start your default webcam in general. 5 | 6 | on <- TRUE 7 | frames = 400 8 | frame = 1 9 | old <- Sys.time() 10 | 11 | while(isTRUE(on)){ 12 | 13 | img <- readNext(my_stream) 14 | blob <- img %>% 15 | split() %>% 16 | '['(c(3,1)) %>% 17 | do.call(function(x, y) (x - y) > 50, .) %>% 18 | morph("close", iterations = 3) %>% 19 | simpleBlobDetector(max_area = Inf, min_area = 1000, blob_color = 255, 20 | filter_by_convexity = FALSE, 21 | filter_by_inertia = FALSE, min_threshold = 0) 22 | 23 | # cat(i,"-", blob$id, ",", blob$x, ",", blob$x, ",", blob$size, "\n") 24 | new <- Sys.time() 25 | 26 | drawRectangle(img, blob$x[1] - 1 + blob$size[1]/2, 27 | blob$y[1] - 1 + blob$size[1]/2, 28 | blob$x[1] - 1 - blob$size[1]/2, 29 | blob$y[1] - 1 - blob$size[1]/2, thickness = 2) 30 | drawText(img, paste0("x: ", format(blob$x[1] - 1, digits = 2, nsmall = 2), 31 | "; y ", format(blob$y[1] - 1, digits = 2, nsmall = 2), 32 | "; fps: ", format(1 / as.numeric(new - old), 33 | digits = 2, nsmall = 2)), 34 | x = 10, y = 10, font_face = 'plain', font_scale = 0.5, 35 | color = "black", thickness = 1) 36 | 37 | old <- new 38 | 39 | display(img, "Live test", 1, 144*4, 176*4) 40 | frame = frame + 1 41 | if(frame == frames){ 42 | on <- FALSE 43 | } 44 | } 45 | release(my_stream) 46 | -------------------------------------------------------------------------------- /R/MNIST_CNN_video_predict.r: -------------------------------------------------------------------------------- 1 | library("Rvision") 2 | library("dplyr") 3 | library("reticulate") 4 | 5 | ### Requires MNIST model trained on 28x28 grayscare MNIST dataset, see: KERAS_train_MNIST_CNN.r 6 | 7 | my_vid <- video("./Video/videoplayback.mp4") 8 | old <- Sys.time() 9 | 10 | for(i in 1:nframes(my_vid)){ 11 | 12 | img <- readNext(my_vid) 13 | new <- Sys.time() 14 | 15 | img32 <- resize(img, 28,28) 16 | img32_gray <- changeColorSpace(img32,"GRAY") 17 | img32_scale <- 1-(as.matrix(img32_gray)/255) # scale colors, still not 100% 18 | img32_array <- reticulate::array_reshape(img32_scale, c(1, 28, 28, 1)) 19 | results <- model %>% predict(img32_array) %>% 20 | data.frame() %>% 21 | magrittr::set_colnames(seq(0,9,1)) 22 | results$top <- as.numeric(colnames(results)[max.col(results,ties.method="first")]) 23 | print(results[1,]) 24 | 25 | drawText(img, paste0("pred: ", format(results$top[1], digits = 2, nsmall = 0)), 26 | x = 30, y = 30, font_face = 'plain', font_scale = 4, 27 | color = "red", thickness = 3) 28 | 29 | old <- new 30 | 31 | display(img, "Live test", 1, 144*4, 176*4) 32 | } 33 | 34 | 35 | -------------------------------------------------------------------------------- /R/Video_balloon_blob.R: -------------------------------------------------------------------------------- 1 | library("Rvision") 2 | library("dplyr") 3 | 4 | my_vid <- video("./Video/balloon1.3gp") 5 | old <- Sys.time() 6 | 7 | for(i in 1:nframes(my_vid)){ 8 | 9 | img <- readNext(my_vid) 10 | blob <- img %>% 11 | split() %>% 12 | '['(c(3,1)) %>% 13 | do.call(function(x, y) (x - y) > 50, .) %>% 14 | morph("close", iterations = 3) %>% 15 | simpleBlobDetector(max_area = Inf, min_area = 1000, blob_color = 255, 16 | filter_by_convexity = FALSE, 17 | filter_by_inertia = FALSE, min_threshold = 0) 18 | 19 | cat(i,"-", blob$id, ",", blob$x, ",", blob$x, ",", blob$size, "\n") 20 | new <- Sys.time() 21 | 22 | drawRectangle(img, blob$x[1] - 1 + blob$size[1]/2, 23 | blob$y[1] - 1 + blob$size[1]/2, 24 | blob$x[1] - 1 - blob$size[1]/2, 25 | blob$y[1] - 1 - blob$size[1]/2, thickness = 2) 26 | drawText(img, paste0("x: ", format(blob$x[1] - 1, digits = 2, nsmall = 2), 27 | "; y ", format(blob$y[1] - 1, digits = 2, nsmall = 2), 28 | "; fps: ", format(1 / as.numeric(new - old), 29 | digits = 2, nsmall = 2)), 30 | x = 10, y = 10, font_face = 'plain', font_scale = 0.5, 31 | color = "black", thickness = 1) 32 | 33 | old <- new 34 | 35 | display(img, "Live test", 1, 144*4, 176*4) 36 | } 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 | # Examples of using the Rvision package 5 | ### This is a living repo and the examples are subject to change and sometimes not working 6 | 7 | This repo is my stash of code examples using the R package [`Rvision`](https://github.com/swarm-lab/Rvision). `Rvision` is a really great package out of the [Swarm Lab](http://www.theswarmlab.com/) at the [New Jersey Institute of Technology](www.njit.edu). This package ports aspects of the massive [`OpenCV` Library](https://opencv.org/) for machine vision into R; a new and needed addition! I started experimenting with this package soon after its release and found it very fun to work with. This repo contains my code and examples of working with the `Rvision` package in conjunction with [`Tensorflow`](https://tensorflow.rstudio.com/) and [`Keras`](https://keras.rstudio.com/) machine learning libraries. 8 | 9 | The basis for much of this code is a series of tweets and discussion with the Swarm Lab PI and primary package author [Simon Garnier](https://twitter.com/sjmgarnier). I am not affiliated with the Swarm Lab or the `Rvision` package. Also note, the examples here have a number of dependencies that require some attention. `Rvision` itself requires the `Rtools` tool-chain and `CMake`. The `Tensorflow` and `keras` installs range from pain-free to a total pain in the butt. See the package links above for installation details. 10 | 11 | There is still a ton of work to be done here and these examples are simplistic, but enjoyable to put together. 12 | 13 | ## Code in this repo 14 | 15 | 16 | * Video_balloon_blob.R - single object tracking example 17 | * MNIST_CNN_video_predict.r - use CNN trained on MNIST to classify numbers in video 18 | * LIVE_webcam_CNN_single_blob_prediction.r - NOT FINISHED - object tracking and CNN prediction with ResNet50 19 | * LIVE_webcam_CNN_prediction.r - ResNet50 CNN classification of video frames 20 | * LIVE_webcam_blob_detection.r - object tracking from webcam 21 | * KERAS_train_MNIST_CNN.r - script to train MNIST CNN with Keras (from Keras examples) 22 | * LIVE_webcam_MNIST_CNN_predict.r - webcam stream with MNIST trained model - experimental... 23 | 24 | 25 | -------------------------------------------------------------------------------- /video/balloon1.3gp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrecos/rvision_examples/bd3b19459566b34ac5280ffa028dc1a6054bdf31/video/balloon1.3gp -------------------------------------------------------------------------------- /video/videoplayback.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrecos/rvision_examples/bd3b19459566b34ac5280ffa028dc1a6054bdf31/video/videoplayback.mp4 --------------------------------------------------------------------------------