├── .gitignore ├── README.md ├── bwenc.c └── eval.sh /.gitignore: -------------------------------------------------------------------------------- 1 | data/ 2 | data.zip -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This is an attempt at the [Neuralink Compression Challenge](https://content.neuralink.com/compression-challenge/README.html). It meets almost all target criteria: 2 | 3 | - lossless: reproduces input data exactly 4 | - no algorithmic delay: samples can be immediately encoded and sent 5 | - simple enough to run on a gameboy 6 | 7 | It _just_ misses the compression target (200x) by two orders of magnitude: the compression rate of this solution is 3.35x for the provided data set. 8 | 9 | Output from `eval.sh`: 10 | 11 | ``` 12 | All recordings successfully compressed. 13 | Original size (bytes): 146800526 14 | Compressed size (bytes): 43774883 15 | Compression ratio: 3.35 16 | ``` 17 | 18 | Compression is done by: 19 | 20 | 1. set `previous_sample = 0`, `rice_k = 3` 21 | 2. take the difference of `previous_sample` and current `sample` 22 | 3. encode the difference using [rice coding](https://en.wikipedia.org/wiki/Golomb_coding#Rice_coding) 23 | 4. estimate next `rice_k` based on bit length of this encode 24 | 5. go to 2. 25 | 26 | I believe this is the "optimal" solution in terms of complexity and results. With a bit of prediction (similar to e.g. [qoa](https://github.com/phoboslab/qoa)) you can get to 3.4x. With entropy coding you _might_ get to 3.5x. In any case, what remains after the initial prediction (`residual = sample - previous_sample`) is very close to random noise, and noise famously compresses rather badly. I'd be very surprised if we see any solutions approaching (or even exceeding) 4x. 27 | 28 | In conclusion, this challenge is either dishonest or ignorant. -------------------------------------------------------------------------------- /bwenc.c: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (c) 2024, Dominic Szablewski - https://phoboslab.org 4 | SPDX-License-Identifier: MIT 5 | 6 | Command line tool to compress neuralink samples 7 | 8 | Compile with: 9 | gcc bwenc.c -std=c99 -lm -O3 -o bwenc 10 | 11 | Usage: 12 | ./bwenc in.wav comp.bw 13 | ./bwenc comp.bw decomp.wav 14 | 15 | */ 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #define STRINGIFY(x) #x 24 | #define TOSTRING(x) STRINGIFY(x) 25 | #define ABORT(...) \ 26 | printf("Abort at line " TOSTRING(__LINE__) ": " __VA_ARGS__); \ 27 | printf("\n"); \ 28 | exit(1) 29 | #define ASSERT(TEST, ...) \ 30 | if (!(TEST)) { \ 31 | ABORT(__VA_ARGS__); \ 32 | } 33 | 34 | #define STR_ENDS_WITH(S, E) (strcmp(S + strlen(S) - (sizeof(E)-1), E) == 0) 35 | 36 | typedef struct { 37 | uint32_t channels; 38 | uint32_t samplerate; 39 | uint32_t samples; 40 | } samples_t; 41 | 42 | 43 | /* ----------------------------------------------------------------------------- 44 | WAV reader / writer */ 45 | 46 | #define WAV_CHUNK_ID(S) \ 47 | (((uint32_t)(S[3])) << 24 | ((uint32_t)(S[2])) << 16 | \ 48 | ((uint32_t)(S[1])) << 8 | ((uint32_t)(S[0]))) 49 | 50 | void fwrite_u32_le(uint32_t v, FILE *fh) { 51 | uint8_t buf[sizeof(uint32_t)]; 52 | buf[0] = 0xff & (v ); 53 | buf[1] = 0xff & (v >> 8); 54 | buf[2] = 0xff & (v >> 16); 55 | buf[3] = 0xff & (v >> 24); 56 | int wrote = fwrite(buf, sizeof(uint32_t), 1, fh); 57 | ASSERT(wrote, "Write error"); 58 | } 59 | 60 | void fwrite_u16_le(unsigned short v, FILE *fh) { 61 | uint8_t buf[sizeof(unsigned short)]; 62 | buf[0] = 0xff & (v ); 63 | buf[1] = 0xff & (v >> 8); 64 | int wrote = fwrite(buf, sizeof(unsigned short), 1, fh); 65 | ASSERT(wrote, "Write error"); 66 | } 67 | 68 | uint32_t fread_u32_le(FILE *fh) { 69 | uint8_t buf[sizeof(uint32_t)]; 70 | int read = fread(buf, sizeof(uint32_t), 1, fh); 71 | ASSERT(read, "Read error or unexpected end of file"); 72 | return (buf[3] << 24) | (buf[2] << 16) | (buf[1] << 8) | buf[0]; 73 | } 74 | 75 | unsigned short fread_u16_le(FILE *fh) { 76 | uint8_t buf[sizeof(unsigned short)]; 77 | int read = fread(buf, sizeof(unsigned short), 1, fh); 78 | ASSERT(read, "Read error or unexpected end of file"); 79 | return (buf[1] << 8) | buf[0]; 80 | } 81 | 82 | int wav_write(const char *path, short *sample_data, samples_t *desc) { 83 | uint32_t data_size = desc->samples * desc->channels * sizeof(short); 84 | uint32_t samplerate = desc->samplerate; 85 | short bits_per_sample = 16; 86 | short channels = desc->channels; 87 | 88 | // Lifted from https://www.jonolick.com/code.html - public domain 89 | // Made endian agnostic using fwrite_u*() 90 | FILE *fh = fopen(path, "wb"); 91 | ASSERT(fh, "Can't open %s for writing", path); 92 | fwrite("RIFF", 1, 4, fh); 93 | fwrite_u32_le(data_size + 44 - 8, fh); 94 | fwrite("WAVEfmt \x10\x00\x00\x00\x01\x00", 1, 14, fh); 95 | fwrite_u16_le(channels, fh); 96 | fwrite_u32_le(samplerate, fh); 97 | fwrite_u32_le(channels * samplerate * bits_per_sample/8, fh); 98 | fwrite_u16_le(channels * bits_per_sample/8, fh); 99 | fwrite_u16_le(bits_per_sample, fh); 100 | fwrite("data", 1, 4, fh); 101 | fwrite_u32_le(data_size, fh); 102 | fwrite((void*)sample_data, data_size, 1, fh); 103 | fclose(fh); 104 | return data_size + 44 - 8; 105 | } 106 | 107 | short *wav_read(const char *path, samples_t *desc) { 108 | FILE *fh = fopen(path, "rb"); 109 | ASSERT(fh, "Can't open %s for reading", path); 110 | 111 | uint32_t container_type = fread_u32_le(fh); 112 | ASSERT(container_type == WAV_CHUNK_ID("RIFF"), "Not a RIFF container"); 113 | 114 | uint32_t wav_size = fread_u32_le(fh); 115 | uint32_t wavid = fread_u32_le(fh); 116 | ASSERT(wavid == WAV_CHUNK_ID("WAVE"), "No WAVE id found"); 117 | 118 | uint32_t data_size = 0; 119 | uint32_t format_length = 0; 120 | uint32_t format_type = 0; 121 | uint32_t channels = 0; 122 | uint32_t samplerate = 0; 123 | uint32_t byte_rate = 0; 124 | uint32_t block_align = 0; 125 | uint32_t bits_per_sample = 0; 126 | 127 | // Find the fmt and data chunk, skip all others 128 | while (1) { 129 | uint32_t chunk_type = fread_u32_le(fh); 130 | uint32_t chunk_size = fread_u32_le(fh); 131 | 132 | if (chunk_type == WAV_CHUNK_ID("fmt ")) { 133 | ASSERT(chunk_size == 16 || chunk_size == 18, "WAV fmt chunk size missmatch"); 134 | 135 | format_type = fread_u16_le(fh); 136 | channels = fread_u16_le(fh); 137 | samplerate = fread_u32_le(fh); 138 | byte_rate = fread_u32_le(fh); 139 | block_align = fread_u16_le(fh); 140 | bits_per_sample = fread_u16_le(fh); 141 | 142 | if (chunk_size == 18) { 143 | unsigned short extra_params = fread_u16_le(fh); 144 | ASSERT(extra_params == 0, "WAV fmt extra params not supported"); 145 | } 146 | } 147 | else if (chunk_type == WAV_CHUNK_ID("data")) { 148 | data_size = chunk_size; 149 | break; 150 | } 151 | else { 152 | int seek_result = fseek(fh, chunk_size, SEEK_CUR); 153 | ASSERT(seek_result == 0, "Malformed RIFF header"); 154 | } 155 | } 156 | 157 | ASSERT(format_type == 1, "Type in fmt chunk is not PCM"); 158 | ASSERT(bits_per_sample == 16, "Bits per samples != 16"); 159 | ASSERT(data_size, "No data chunk"); 160 | 161 | uint8_t *wav_bytes = malloc(data_size); 162 | ASSERT(wav_bytes, "Malloc for %d bytes failed", data_size); 163 | int read = fread(wav_bytes, data_size, 1, fh); 164 | ASSERT(read, "Read error or unexpected end of file for %d bytes", data_size); 165 | fclose(fh); 166 | 167 | desc->samplerate = samplerate; 168 | desc->samples = data_size / (channels * (bits_per_sample/8)); 169 | desc->channels = channels; 170 | 171 | return (short*)wav_bytes; 172 | } 173 | 174 | 175 | 176 | /* ----------------------------------------------------------------------------- 177 | BRAINWIRE reader / writer */ 178 | 179 | static inline int rice_read(uint8_t *bytes, int *bit_pos, uint32_t k) { 180 | int msbs = 0; 181 | int p = *bit_pos; 182 | while (!(bytes[p >> 3] & (1 << (7-(p & 7))))) { 183 | p++; 184 | msbs++; 185 | } 186 | p++; 187 | 188 | int count = k; 189 | int lsbs = 0; 190 | while (count) { 191 | int remaining = 8 - (p & 7); 192 | int read = remaining < count ? remaining : count; 193 | int shift = remaining - read; 194 | int mask = (0xff >> (8 - read)); 195 | lsbs = (lsbs << read) | ((bytes[p >> 3] & (mask << shift)) >> shift); 196 | p += read; 197 | count -= read; 198 | } 199 | *bit_pos = p; 200 | 201 | int val; 202 | uint32_t uval = (msbs << k) | lsbs; 203 | if (uval & 1) { 204 | val = -((int)(uval >> 1)) - 1; 205 | } 206 | else { 207 | val = (int)(uval >> 1); 208 | } 209 | 210 | return val; 211 | } 212 | 213 | static inline int rice_write(uint8_t *bytes, int *bit_pos, int val, uint32_t k) { 214 | uint32_t uval = val; 215 | uval <<= 1; 216 | uval ^= (val >> 31); 217 | 218 | uint32_t msbs = uval >> k; 219 | uint32_t lsbs = 1 + k; 220 | uint32_t count = msbs + lsbs; 221 | uint32_t pattern = 1 << k; // the unary end bit 222 | pattern |= (uval & ((1 << k)-1)); // the binary LSBs 223 | 224 | int pos = *bit_pos; 225 | while (count) { 226 | int occupied = (pos & 7); 227 | int remaining = 8 - occupied; 228 | int written = remaining < count ? remaining : count; 229 | int bits = 0; 230 | if (count - written < 31) { 231 | bits = (pattern >> (count - written)) << (remaining - written); 232 | bits &= (0xff >> occupied); 233 | } 234 | bytes[pos >> 3] |= bits; 235 | pos += written; 236 | count -= written; 237 | } 238 | *bit_pos = pos; 239 | return msbs + lsbs; 240 | } 241 | 242 | 243 | static inline int brainwire_dequant(int v) { 244 | // Not really sure what's goin on here. The original 10bit data was 245 | // upscaled to 16 bit somehow. It wasn't a simple bit shift. This thing 246 | // here was found through a brute force search and just happens to 247 | // replicate neuralink's original upscale. 248 | if (v >= 0) { 249 | return round(v * 64.061577 + 31.034184); 250 | } 251 | else { 252 | return -round((-v -1) * 64.061577 + 31.034184) - 1; 253 | } 254 | } 255 | 256 | static inline int brainwire_quant(int v) { 257 | return (int)floor(v/64.0); 258 | } 259 | 260 | short *brainwire_read(const char *path, samples_t *desc) { 261 | FILE *fh = fopen(path, "rb"); 262 | ASSERT(fh, "Couldnt open %s for reading", path); 263 | 264 | fseek(fh, 0, SEEK_END); 265 | int size = ftell(fh); 266 | fseek(fh, 0, SEEK_SET); 267 | 268 | uint8_t *bytes = malloc(size); 269 | int bytes_read = fread(bytes, 1, size, fh); 270 | ASSERT(size > 0 && bytes_read == size, "Read failed"); 271 | fclose(fh); 272 | 273 | 274 | int bit_pos = 0; 275 | float rice_k = 3; 276 | 277 | int samples = rice_read(bytes, &bit_pos, 16); 278 | int samplerate = rice_read(bytes, &bit_pos, 16); 279 | short *sample_data = malloc(samples * sizeof(short)); 280 | 281 | int prev_quantized = 0; 282 | for (int i = 0; i < samples; i++) { 283 | int temp = bit_pos; 284 | 285 | int residual = rice_read(bytes, &bit_pos, rice_k); 286 | int quantized = prev_quantized + residual; 287 | prev_quantized = quantized; 288 | sample_data[i] = brainwire_dequant(quantized); 289 | 290 | int encoded_len = bit_pos - temp; 291 | rice_k = rice_k * 0.99 + (encoded_len / 1.55) * 0.01; 292 | } 293 | 294 | desc->channels = 1; 295 | desc->samples = samples; 296 | desc->samplerate = samplerate; 297 | return sample_data; 298 | } 299 | 300 | int brainwire_write(const char *path, short *sample_data, samples_t *desc) { 301 | uint8_t *bytes = malloc(desc->samples * 2); // just to be sure... 302 | memset(bytes, 0, desc->samples * 2); 303 | 304 | int bit_pos = 0; 305 | float rice_k = 3; 306 | 307 | rice_write(bytes, &bit_pos, desc->samples, 16); 308 | rice_write(bytes, &bit_pos, desc->samplerate, 16); 309 | 310 | int prev_quantized = 0; 311 | for (int i = 0; i < desc->samples; i++) { 312 | int quantized = brainwire_quant(sample_data[i]); 313 | int residual = quantized - prev_quantized; 314 | prev_quantized = quantized; 315 | 316 | int encoded_len = rice_write(bytes, &bit_pos, residual, rice_k); 317 | rice_k = rice_k * 0.99 + (encoded_len / 1.55) * 0.01; 318 | } 319 | 320 | int byte_len = (bit_pos + 7) / 8; 321 | FILE *fh = fopen(path, "wb"); 322 | ASSERT(fh, "Couldnt open %s for writing", path); 323 | fwrite(bytes, 1, byte_len, fh); 324 | fclose(fh); 325 | 326 | return byte_len; 327 | } 328 | 329 | 330 | 331 | /* ----------------------------------------------------------------------------- 332 | Main */ 333 | 334 | int main(int argc, char **argv) { 335 | ASSERT(argc >= 3, "\nUsage: bwenc in.{wav,bw} out.{wav,bw}") 336 | 337 | samples_t desc; 338 | short *sample_data = NULL; 339 | 340 | // Decode input 341 | 342 | if (STR_ENDS_WITH(argv[1], ".wav")) { 343 | sample_data = wav_read(argv[1], &desc); 344 | } 345 | else if (STR_ENDS_WITH(argv[1], ".bw")) { 346 | sample_data = brainwire_read(argv[1], &desc); 347 | } 348 | else { 349 | ABORT("Unknown file type for %s", argv[1]); 350 | } 351 | 352 | ASSERT(sample_data, "Can't load/decode %s", argv[1]); 353 | 354 | 355 | // Encode output 356 | 357 | int bytes_written = 0; 358 | double psnr = 1.0/0.0; 359 | if (STR_ENDS_WITH(argv[2], ".wav")) { 360 | bytes_written = wav_write(argv[2], sample_data, &desc); 361 | } 362 | else if (STR_ENDS_WITH(argv[2], ".bw")) { 363 | bytes_written = brainwire_write(argv[2], sample_data, &desc); 364 | } 365 | else { 366 | ABORT("Unknown file type for %s", argv[2]); 367 | } 368 | 369 | ASSERT(bytes_written, "Can't write/encode %s", argv[2]); 370 | free(sample_data); 371 | 372 | printf( 373 | "%s: size: %d kb (%d bytes) = %.2fx compression\n", 374 | argv[2], bytes_written/1024, bytes_written, 375 | (float)(desc.samples * sizeof(short))/(float)bytes_written 376 | ); 377 | 378 | return 0; 379 | } 380 | -------------------------------------------------------------------------------- /eval.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Modified to use the same binary for en-/decode 4 | 5 | rm -rf data 6 | unzip data.zip 7 | 8 | get_file_size() { 9 | find "$1" -printf "%s\n" 10 | } 11 | 12 | total_size_raw=0 13 | coder_size=$(get_file_size bwenc) 14 | total_size_compressed=$((coder_size)) 15 | 16 | for file in data/* 17 | do 18 | echo "Processing $file" 19 | compressed_file_path="${file}.bw" 20 | decompressed_file_path="${file}.dec.wav" 21 | 22 | ./bwenc "$file" "$compressed_file_path" 23 | ./bwenc "$compressed_file_path" "$decompressed_file_path" 24 | 25 | file_size=$(get_file_size "$file") 26 | compressed_size=$(get_file_size "$compressed_file_path") 27 | 28 | if diff -q "$file" "$decompressed_file_path" > /dev/null; then 29 | echo "${file} losslessly compressed from ${file_size} bytes to ${compressed_size} bytes" 30 | else 31 | echo "ERROR: ${file} and ${decompressed_file_path} are different." 32 | exit 1 33 | fi 34 | 35 | total_size_raw=$((total_size_raw + file_size)) 36 | total_size_compressed=$((total_size_compressed + compressed_size)) 37 | done 38 | 39 | compression_ratio=$(echo "scale=2; ${total_size_raw} / ${total_size_compressed}" | bc) 40 | 41 | echo "All recordings successfully compressed." 42 | echo "Original size (bytes): ${total_size_raw}" 43 | echo "Compressed size (bytes): ${total_size_compressed}" 44 | echo "Compression ratio: ${compression_ratio}" 45 | --------------------------------------------------------------------------------