├── README.md └── magpie_system.py /README.md: -------------------------------------------------------------------------------- 1 | # Magpie-Recycling-Project 2 | In this repository you can find the code used in the Magpie Recycling Project. For those of you unaware of what this project is about, go check out this youtube channel: https://www.youtube.com/channel/UCEhPdSIMrLNynfwnkibVzZg 3 | 4 | Feel free to contact us for questions regarding the project or the code found here via: skataprojekt@gmail.com 5 | -------------------------------------------------------------------------------- /magpie_system.py: -------------------------------------------------------------------------------- 1 | import multiprocessing 2 | from datetime import datetime 3 | import numpy as np 4 | import cv2 5 | import subprocess 6 | 7 | # Function that restart the computer to handle problem with USB cards getting full 8 | def restart_system(s): 9 | import time 10 | print('Starting windows restart timer at: {}'.format(datetime.now().strftime("%Y_%m_%d#%H-%M-%S"))) 11 | time.sleep(s) 12 | 13 | import os 14 | print('Windows restart!') 15 | os.system('shutdown -t 0 -r -f') 16 | 17 | # Function that calls relay_run_feeder.exe which is a executable that tell the USB-relay to open and then close the ports for the feeder 18 | def relay_run_feeder(): 19 | subprocess.call(['Path to: relay_run_feeder.exe']) 20 | 21 | # Function that calibrate the exposure in order to get good footage 22 | def calibrate_exposure(target_pixel_sum, tolerance, current_exposure): 23 | 24 | exposure = current_exposure 25 | 26 | # Setting up camera 27 | cap = cv2.VideoCapture(0) 28 | cap.set(cv2.CAP_PROP_FPS, 30) 29 | cap.set(cv2.CAP_PROP_BUFFERSIZE, 3) 30 | cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) 31 | cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) 32 | cap.set(cv2.CAP_PROP_AUTO_EXPOSURE, 0.25) 33 | cap.set(cv2.CAP_PROP_EXPOSURE, exposure) 34 | 35 | 36 | stop_frame = 30 # Frame n to stop at 37 | frame_n = 0 # Frame count 38 | 39 | # Camera loop that stop at stop_frame, used to collect a pixel sum after camera has stabilized 40 | while frame_n < stop_frame: 41 | ret, frame = cap.read() 42 | if ret: 43 | pixel_sum = np.sum(frame) 44 | frame_n += 1 45 | 46 | cap.release() 47 | 48 | # Check if the pixel sum is high or low, in other words decide whether to lower or higher the exposure 49 | try: 50 | if pixel_sum < target_pixel_sum - tolerance * target_pixel_sum and exposure < 15: 51 | exposure += 1 52 | elif pixel_sum > target_pixel_sum + tolerance * target_pixel_sum: 53 | exposure -= 1 54 | 55 | return exposure 56 | 57 | except TypeError: 58 | return exposure 59 | 60 | # Function that write video 61 | def write_video(filename, frames, fps): 62 | shape_frame = frames[0].shape 63 | fourcc = cv2.VideoWriter_fourcc(*'MJPG') 64 | writer = cv2.VideoWriter(filename, fourcc, fps, (shape_frame[1], shape_frame[0]), isColor=True) 65 | 66 | for frame in frames: 67 | writer.write(frame) 68 | 69 | writer.release() 70 | 71 | def detect_dropped_object(conn_bool, exposure): 72 | ''' 73 | This function uses the camera inside the machine to detect if an item has been dropped. When the script is running 74 | this is a subprocess which sends a boolean to the other subprocess (the function responsible for the outdoor camera) 75 | when it is triggered. We are using a very simple movement detection, we compare the current frame with a background 76 | to detect changes. It takes the connection of a pipe to send a boolean through and exposure of the camera as input 77 | parameters. 78 | ''' 79 | 80 | # Setting up the camera 81 | cap = cv2.VideoCapture(1) 82 | cap.set(cv2.CAP_PROP_FPS, 30) 83 | cap.set(cv2.CAP_PROP_AUTO_EXPOSURE, 0.25) 84 | cap.set(cv2.CAP_PROP_EXPOSURE, exposure) 85 | #cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) 86 | #cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) 87 | #cap.set(cv2.CAP_PROP_AUTOFOCUS, 0) 88 | #cap.set(cv2.CAP_PROP_FOCUS, 10) 89 | 90 | print('Starting inside camera with exposure: ' + str(exposure)) 91 | 92 | # Some parameters for the function 93 | frame_n = 0 # Frame count 94 | background_update = 10 # x frames then update 95 | start_frame = 30 * 6 # What frame_n to start, because the background has to stabilize and to fill the buffer. Has to be bigger or same as buffer_len 96 | background_th = 20 # Collect pixel values > x 97 | object_detection_th = 950000 # Threshold value for what is regarded an object falling down 98 | triggered = False # Boolean for multiprocess pipe 99 | 100 | # Camera loop 101 | while True: 102 | # Capture frame-by-frame 103 | ret, frame = cap.read() 104 | 105 | if ret: 106 | # Update background to current frame 107 | if frame_n % background_update == 0 or frame_n == start_frame: 108 | background = frame 109 | 110 | # After camera have stabilized, subtract background and sum the pixel intensity 111 | if frame_n > start_frame: 112 | img = cv2.subtract(frame, background) 113 | flat_img = img.flatten() 114 | img_filtered = flat_img[np.where(flat_img > background_th)] 115 | img_sum = float(np.sum(img_filtered)) 116 | 117 | # Calculate difference in pixel sum from the last pixel sum to detect change (movement) 118 | if frame_n == start_frame + 1: 119 | last_sum = img_sum 120 | diff = img_sum - last_sum 121 | last_sum = img_sum 122 | 123 | # Check if difference is big enough to be regarded as an object falling down 124 | if diff > object_detection_th: 125 | triggered = True 126 | conn_bool.send(triggered) 127 | conn_bool.close() 128 | print('Detection triggered!') 129 | break 130 | 131 | frame_n += 1 132 | conn_bool.send(triggered) 133 | 134 | elif not ret: 135 | break 136 | 137 | # Release camera capture 138 | cap.release() 139 | 140 | def capture_footage(conn_frames, conn_bool, exposure): 141 | ''' 142 | This function uses the camera outside of the machine to capture the footage of what triggered the function that 143 | detects dropped objects. In order to get footage before and after the detection of dropped object we use a 144 | 'frame buffer'. It takes 2 pipe connections, one to receive boolean from the 'detect dropped object' 145 | function and one to send the frames through as well as the exposure as input parametres. 146 | ''' 147 | 148 | # Setting up the camera 149 | cap = cv2.VideoCapture(0) 150 | cap.set(cv2.CAP_PROP_FPS, 30) 151 | cap.set(cv2.CAP_PROP_AUTO_EXPOSURE, 0.25) 152 | cap.set(cv2.CAP_PROP_EXPOSURE, exposure) 153 | cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) 154 | cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) 155 | cap.set(cv2.CAP_PROP_AUTOFOCUS, 0) 156 | cap.set(cv2.CAP_PROP_FOCUS, 10) 157 | 158 | print('Starting outside camera with exposure: ' + str(exposure)) 159 | 160 | frame_n = 0 # Frame count 161 | start_frame = 30 * 6 # What frame_n to start, because the background has to stabilize and to fill the buffer. Has to be bigger or same as buffer_len 162 | frame_n_detection = 0 # Frame count since detection 163 | recording_len = 30 * 10 # How many frames should be captured after detection triggered, if fps = 30 then 300 frames = 10 seconds 164 | buffer_len = start_frame # How many frames should be captured before detection triggered 165 | buffer_split = None # Used to know where to split buffer to arrange the frames in proper order for footage 166 | pipe_bool = False # Boolean from pipe from 'detect dropped object' function 167 | error_bool = False # Boolean to check if error handling triggered 168 | triggered = False # Boolean to stop filling buffer when object detection is triggered 169 | 170 | # Arrays to store frames 171 | frames_recording = np.zeros((recording_len, 480, 640, 3), dtype=np.uint8) 172 | frames_buffer = np.zeros((buffer_len, 480, 640, 3), dtype=np.uint8) 173 | 174 | # Camera loop 175 | while True: 176 | # Capture frame-by-frame 177 | ret, frame = cap.read() 178 | 179 | if ret: 180 | # Filling up the buffer 181 | if not triggered: 182 | frames_buffer[frame_n % buffer_len] = frame 183 | 184 | # If 'detect dropped object' function is triggered stop filling buffer and start recording the aftermath 185 | if not triggered and pipe_bool: 186 | triggered = True 187 | buffer_split = frame_n % buffer_len 188 | feeder.start() # Start subprocess which run feeder 189 | print("Triggered at: ", datetime.now().strftime("%Y_%m_%d#%H-%M-%S")) 190 | frame_n_detection = 0 191 | 192 | # Getting frames for aftermath recording 193 | if triggered: 194 | # Saving frames for writing recording 195 | frames_recording[frame_n_detection] = frame 196 | frame_n_detection += 1 197 | 198 | # Stop recording if all frames are collected 199 | if recording_len == frame_n_detection and triggered: 200 | break 201 | 202 | frame_n += 1 203 | 204 | # Check if 'detect dropped object' function is triggered 205 | if not pipe_bool: 206 | pipe_bool = conn_bool.recv() 207 | 208 | elif not ret: 209 | error_bool = True 210 | break 211 | 212 | # Release camera capture 213 | cap.release() 214 | 215 | if error_bool: 216 | print('Error handling triggered!') 217 | 218 | if not error_bool: 219 | print('Stopped recording!') 220 | 221 | feeder.join() # Wait and make sure the feeder has stopped 222 | 223 | # Arrange buffer and concatenate all frames 224 | fixed_buffer = np.concatenate((frames_buffer[buffer_split + 1:, :, :, :], frames_buffer[:buffer_split + 1, :, :, :]), axis=0) 225 | all_frames = np.concatenate((fixed_buffer, frames_recording), axis=0) 226 | 227 | conn_frames.send(all_frames) 228 | conn_frames.close() 229 | 230 | 231 | # Creating multiprocess for feeder and creating pipes 232 | feeder = multiprocessing.Process(target=relay_run_feeder) 233 | 234 | parent_conn_frames, child_conn_frames = multiprocessing.Pipe() 235 | parent_conn_bool, child_conn_bool = multiprocessing.Pipe() 236 | 237 | 238 | t_restart = 60 * 60 * 8 # x hours between restart 239 | restart_timer = multiprocessing.Process(target=restart_system, args=(t_restart,)) # Creating multiprocess 240 | 241 | current_exposure = -12 # Exposure parameter 242 | 243 | if __name__ == '__main__': 244 | 245 | restart_timer.start() 246 | 247 | # System loop 248 | while True: 249 | 250 | temp_exposure = calibrate_exposure(9*10**7, 0.3, current_exposure) 251 | 252 | indCam = multiprocessing.Process(target=detect_dropped_object, args=(child_conn_bool, -9,)) 253 | outCam = multiprocessing.Process(target=capture_footage, args=(child_conn_frames, parent_conn_bool, temp_exposure,)) 254 | 255 | indCam.start() 256 | outCam.start() 257 | 258 | try: 259 | video_frames = parent_conn_frames.recv() 260 | write_video('path to video folder/{}.avi'.format(datetime.now().strftime("%Y_%m_%d#%H-%M-%S")), video_frames , 30) 261 | 262 | indCam.join() 263 | outCam.join() 264 | 265 | print('Video saved.') 266 | current_exposure = temp_exposure 267 | 268 | except: 269 | pass 270 | --------------------------------------------------------------------------------