├── main.c ├── fastpub.h ├── README.md └── fastpub.c /main.c: -------------------------------------------------------------------------------- 1 | #define _POSIX_C_SOURCE 200809L 2 | 3 | #include 4 | #include 5 | 6 | #include "fastpub.h" 7 | 8 | void publisher() { 9 | struct fastpub *fp = fastpub_pubopen("fastpub_test", 1024, 1); 10 | 11 | printf("publisher shm @ %p\n", fp->info); 12 | 13 | for (long int i = 0; i < 10000000L; i++) { 14 | int *buffer = fastpub_nextbuf(fp); 15 | buffer[0] = i; 16 | usleep(100); 17 | fastpub_publish(fp); 18 | } 19 | 20 | fastpub_close(fp); 21 | } 22 | 23 | void subscriber() { 24 | struct fastpub *fp = fastpub_subopen("fastpub_test"); 25 | 26 | printf("subscriber shm @ %p\n", fp->info); 27 | 28 | for (int i = 0;; i++) { 29 | int *buffer = fastpub_next_update(fp); 30 | 31 | printf("got updated frame %d\n", buffer[0]); 32 | 33 | if (buffer[0] == 9999999) { 34 | fastpub_release(fp, buffer); 35 | break; 36 | } 37 | 38 | fastpub_release(fp, buffer); 39 | } 40 | 41 | fastpub_close(fp); 42 | } 43 | 44 | int main(int argc, char **argv) { 45 | 46 | if (fork()) { 47 | publisher(); 48 | } 49 | else { 50 | subscriber(); 51 | } 52 | 53 | return 0; 54 | } 55 | -------------------------------------------------------------------------------- /fastpub.h: -------------------------------------------------------------------------------- 1 | #ifndef __FASTPUB_H 2 | #define __FASTPUB_H 3 | 4 | /* 5 | * FastPub - Zero-Copy Non-Blocking Shared Memory Publisher/Subscriber IPC Library 6 | */ 7 | 8 | #include 9 | #include 10 | 11 | struct fastpub { 12 | void *info; 13 | int shmfd; 14 | size_t shm_size; 15 | size_t buffer_size; 16 | size_t max_subscribers; 17 | bool publisher; 18 | char *name; 19 | }; 20 | 21 | /* 22 | * Publisher functions 23 | * 24 | * struct fastpub *fp = fastpub_pubopen("foo", 1024, 1); 25 | * while (1) { 26 | * char *buffer = fastpub_nextbuf(fp); // buffer has 1024 bytes 27 | * ... 28 | * fastpub_publish(fp); 29 | * } 30 | * fastpub_close(fp); 31 | * 32 | * fastpub_init("foo", 1024, 1); 33 | */ 34 | struct fastpub *fastpub_pubopen(const char *name, size_t buffer_size, size_t max_subscribers); 35 | void *fastpub_nextbuf(struct fastpub *); 36 | void fastpub_publish(struct fastpub *); 37 | 38 | /* 39 | * Subscriber functions 40 | * 41 | * struct fastpub *fp = fastpub_subopen("foo"); 42 | * while (1) { 43 | * char *buffer = fastpub_current(fp); // buffer has fp->buffer_size bytes 44 | * ... 45 | * fastpub_release(fp, buffer); 46 | * } 47 | * fastpub_close(fp); 48 | */ 49 | struct fastpub *fastpub_subopen(const char *name); 50 | void *fastpub_next_update(struct fastpub *); 51 | void *fastpub_current(struct fastpub *); 52 | void fastpub_release(struct fastpub *, void *buffer); 53 | 54 | void fastpub_close(struct fastpub *); 55 | 56 | #endif//__FASTPUB_H 57 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | FastPub 2 | ======= 3 | 4 | Zero-copy non-blocking shared memory publisher/subscriber IPC library 5 | 6 | This is the result of me needing a shared memory publisher-subscriber protocol that would not have any copying overhead at all. The protocol is conceptually pretty simple, and I hope to improve the interface a lot in the future. 7 | 8 | The protocol's core algorithm is essentially an extension of triple buffering, with the addition of multiple readers. The writer writes to one buffer, while a "current" buffer is always available for readers to grab. When all readers release a buffer that is not current, it falls into a "slack" pool, where the writer can grab it when it needs to produce a new buffer. As long as there are n + 2 total buffers, where n is the number of readers, the writer will always have at least one slack buffer available and the readers will always have the current buffer to grab, so neither side will ever block. For the trivial case of one reader, this is exactly triple buffering. 9 | 10 | The buffers are all preallocated and stay in the shared memory region, and are also never copied, which means that this algorithm should have pretty close to optimal performance for just moving buffers of data between processes. The strict no-blocking guarantee makes it suitable for soft realtime applications, like realtime computer vision and robot control (which is the original intended purpose). Of course, it will readily (but safely) drop buffers that are not picked up by subscribers, and will only provide the most recent update at any one time, so it is not a general-purpose IPC mechanism by any means. 11 | -------------------------------------------------------------------------------- /fastpub.c: -------------------------------------------------------------------------------- 1 | #define _POSIX_C_SOURCE 200809L 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "fastpub.h" 16 | 17 | struct fastpub_info { 18 | uint32_t buffer_size; 19 | uint32_t buffer_count; 20 | 21 | uint32_t ready; 22 | uint32_t next; // struct fastpub_buffer * 23 | uint32_t current; // struct fastpub_buffer * 24 | uint32_t slack; // struct fastpub_buffer * 25 | 26 | pthread_mutex_t mutex; 27 | pthread_mutexattr_t mutex_attr; 28 | 29 | pthread_cond_t update_cond; 30 | pthread_condattr_t update_cond_attr; 31 | } __attribute__((aligned(64))); 32 | 33 | struct fastpub_buffer { 34 | uint32_t refcount; 35 | uint32_t next; // struct fastpub_buffer * 36 | } __attribute__((aligned(64))); 37 | 38 | struct fastpub *fastpub_pubopen(const char *name, size_t buffer_size, size_t max_subscribers) { 39 | 40 | // allocate descriptor structure 41 | struct fastpub *self = malloc(sizeof(struct fastpub)); 42 | if (!self) { 43 | return NULL; 44 | } 45 | 46 | self->buffer_size = buffer_size; 47 | self->max_subscribers = max_subscribers; 48 | self->publisher = true; 49 | self->name = strdup(name); 50 | 51 | // open shared memory file 52 | int shmfd = shm_open(name, O_RDWR | O_CREAT, 0660); 53 | self->shmfd = shmfd; 54 | if (shmfd < 0) { 55 | free(self->name); 56 | free(self); 57 | return NULL; 58 | } 59 | 60 | // resize shared memory file 61 | self->shm_size = sizeof(struct fastpub_info) + 62 | (sizeof(struct fastpub_buffer) + buffer_size) * (max_subscribers + 2); 63 | if (ftruncate(shmfd, self->shm_size)) { 64 | free(self->name); 65 | free(self); 66 | shm_unlink(name); 67 | return NULL; 68 | } 69 | 70 | // map shared memory file 71 | struct fastpub_info *info = mmap(NULL, self->shm_size, 72 | PROT_READ | PROT_WRITE, MAP_SHARED, shmfd, 0); 73 | self->info = info; 74 | if (!self->info) { 75 | free(self->name); 76 | free(self); 77 | shm_unlink(name); 78 | return NULL; 79 | } 80 | 81 | // create memory layout 82 | // the first block is next, and remaining ones are put in slack 83 | info->ready = 0; 84 | info->buffer_size = buffer_size; 85 | info->buffer_count = max_subscribers + 2; 86 | info->next = sizeof(struct fastpub_info); 87 | info->slack = sizeof(struct fastpub_info) + sizeof(struct fastpub_buffer) + buffer_size; 88 | for (size_t i = 0; i < max_subscribers + 1; i++) { 89 | struct fastpub_buffer *buffer = (void*) 90 | (info->slack + (uintptr_t) info + 91 | (sizeof(struct fastpub_buffer) + buffer_size) * i); 92 | buffer->refcount = 0; 93 | if (i != max_subscribers) { 94 | buffer->next = info->slack + 95 | (sizeof(struct fastpub_buffer) + buffer_size) * (i + 1); 96 | } 97 | else { 98 | buffer->next = 0; 99 | } 100 | } 101 | 102 | // initialize mutex 103 | { 104 | pthread_mutexattr_init(&info->mutex_attr); 105 | pthread_mutexattr_setpshared(&info->mutex_attr, PTHREAD_PROCESS_SHARED); 106 | pthread_mutex_init(&info->mutex, &info->mutex_attr); 107 | 108 | pthread_condattr_init(&info->update_cond_attr); 109 | pthread_condattr_setpshared(&info->update_cond_attr, PTHREAD_PROCESS_SHARED); 110 | pthread_cond_init(&info->update_cond, &info->update_cond_attr); 111 | 112 | info->ready = 0x40404040; 113 | } 114 | 115 | return self; 116 | } 117 | 118 | struct fastpub *fastpub_subopen(const char *name) { 119 | 120 | // allocate descriptor structure 121 | struct fastpub *self = malloc(sizeof(struct fastpub)); 122 | if (!self) { 123 | return NULL; 124 | } 125 | 126 | retry: 127 | 128 | self->publisher = false; 129 | self->name = strdup(name); 130 | 131 | // open shared memory file 132 | int shmfd = shm_open(name, O_RDWR | O_CREAT, 0660); 133 | self->shmfd = shmfd; 134 | if (shmfd < 0) { 135 | free(self->name); 136 | free(self); 137 | return NULL; 138 | } 139 | 140 | // resize shared memory file 141 | self->shm_size = lseek(shmfd, 0, SEEK_END) + 1; 142 | 143 | if (self->shm_size == 0) { 144 | close(shmfd); 145 | usleep(10000); 146 | goto retry; 147 | } 148 | 149 | // map shared memory file 150 | struct fastpub_info *info = mmap(NULL, self->shm_size, 151 | PROT_READ | PROT_WRITE, MAP_SHARED, shmfd, 0); 152 | self->info = info; 153 | 154 | if (!self->info) { 155 | free(self->name); 156 | free(self); 157 | shm_unlink(name); 158 | return NULL; 159 | } 160 | 161 | while (info->ready != 0x40404040) usleep(10000); 162 | 163 | return self; 164 | } 165 | 166 | void *fastpub_nextbuf(struct fastpub *self) { 167 | struct fastpub_info *info = self->info; 168 | return (void*) ((uintptr_t) info + info->next + sizeof(struct fastpub_buffer)); 169 | } 170 | 171 | static void _push_slack(struct fastpub *self, uint32_t offset) { 172 | struct fastpub_info *info = self->info; 173 | 174 | ((struct fastpub_buffer*) ((uintptr_t) info + offset))->next = info->slack; 175 | info->slack = offset; 176 | } 177 | 178 | static uint32_t _pop_slack(struct fastpub *self) { 179 | struct fastpub_info *info = self->info; 180 | 181 | struct fastpub_buffer *block = (void*) ((uintptr_t) info + info->slack); 182 | uint32_t block_offset = info->slack; 183 | info->slack = block->next; 184 | return block_offset; 185 | } 186 | 187 | void fastpub_publish(struct fastpub *self) { 188 | struct fastpub_info *info = self->info; 189 | 190 | pthread_mutex_lock(&info->mutex); 191 | struct fastpub_buffer *current = (void*) ((uintptr_t) info + info->current); 192 | struct fastpub_buffer *next = (void*) ((uintptr_t) info + info->next); 193 | 194 | // if --current.refcount == 0, push_slack(current) 195 | if (--current->refcount == 0) { 196 | _push_slack(self, info->current); 197 | } 198 | 199 | // current <- next 200 | next->refcount = 1; 201 | info->current = info->next; 202 | 203 | // next <- pop_slack() 204 | info->next = _pop_slack(self); 205 | pthread_mutex_unlock(&info->mutex); 206 | 207 | pthread_cond_broadcast(&info->update_cond); 208 | } 209 | 210 | void *fastpub_current(struct fastpub *self) { 211 | struct fastpub_info *info = self->info; 212 | 213 | pthread_mutex_lock(&info->mutex); 214 | struct fastpub_buffer *current = (void*) ((uintptr_t) info + info->current); 215 | current->refcount++; 216 | pthread_mutex_unlock(&info->mutex); 217 | 218 | return (void*) ((uintptr_t) current + sizeof(struct fastpub_buffer)); 219 | } 220 | 221 | void fastpub_release(struct fastpub *self, void *buffer) { 222 | struct fastpub_info *info = self->info; 223 | 224 | pthread_mutex_lock(&info->mutex); 225 | struct fastpub_buffer *fpbuffer = (void*) ((intptr_t) buffer - sizeof(struct fastpub_buffer)); 226 | if (--fpbuffer->refcount == 0) { 227 | _push_slack(self, (uint32_t) ((uintptr_t) fpbuffer - (uintptr_t) self->info)); 228 | } 229 | pthread_mutex_unlock(&info->mutex); 230 | } 231 | 232 | void *fastpub_next_update(struct fastpub *self) { 233 | struct fastpub_info *info = self->info; 234 | 235 | pthread_mutex_lock(&info->mutex); 236 | pthread_cond_wait(&info->update_cond, &info->mutex); 237 | struct fastpub_buffer *current = (void*) ((uintptr_t) info + info->current); 238 | current->refcount++; 239 | pthread_mutex_unlock(&info->mutex); 240 | 241 | return (void*) ((uintptr_t) current + sizeof(struct fastpub_buffer)); 242 | } 243 | 244 | void fastpub_close(struct fastpub *self) { 245 | 246 | // if (self->publisher) { 247 | // shm_unlink(self->name); 248 | // } 249 | 250 | munmap(self->info, self->shm_size); 251 | close(self->shmfd); 252 | free(self->name); 253 | free(self); 254 | } 255 | --------------------------------------------------------------------------------