├── .gitignore ├── Makefile ├── README.md ├── alphabet-test ├── audio-common.h ├── badmaths.h ├── blit-helpers.h ├── ccan ├── build_assert │ ├── LICENSE │ ├── _info │ └── build_assert.h ├── cast │ ├── LICENSE │ ├── _info │ └── cast.h ├── compiler │ ├── LICENSE │ ├── _info │ └── compiler.h ├── opt │ ├── .depends │ ├── LICENSE │ ├── _info │ ├── helpers.c │ ├── opt.c │ ├── opt.h │ ├── parse.c │ ├── private.h │ ├── test │ │ ├── compile_ok-const-arg.c │ │ ├── run-add_desc.c │ │ ├── run-checkopt.c │ │ ├── run-consume_words.c │ │ ├── run-correct-reporting.c │ │ ├── run-early.c │ │ ├── run-helpers.c │ │ ├── run-iter.c │ │ ├── run-no-options.c │ │ ├── run-set_alloc.c │ │ ├── run-usage.c │ │ ├── run.c │ │ ├── utils.c │ │ └── utils.h │ └── usage.c ├── tools │ └── configurator │ │ └── configurator.c └── typesafe_cb │ ├── .depends │ ├── LICENSE │ ├── _info │ ├── test │ ├── compile_fail-cast_if_type-promotable.c │ ├── compile_fail-typesafe_cb-int.c │ ├── compile_fail-typesafe_cb.c │ ├── compile_fail-typesafe_cb_cast-multi.c │ ├── compile_fail-typesafe_cb_cast.c │ ├── compile_fail-typesafe_cb_postargs.c │ ├── compile_fail-typesafe_cb_preargs.c │ ├── compile_ok-typesafe_cb-NULL.c │ ├── compile_ok-typesafe_cb-undefined.c │ ├── compile_ok-typesafe_cb-vars.c │ ├── compile_ok-typesafe_cb_cast.c │ └── run.c │ └── typesafe_cb.h ├── charmodel-classify.c ├── charmodel-helpers.h ├── charmodel-init.c ├── charmodel-multi-predict.c ├── charmodel-predict.c ├── charmodel.h ├── classify-gtk ├── classify-simple-test ├── classify-train ├── classify.py ├── classify_stats.py ├── colour.c ├── colour.h ├── colour.py ├── context-helpers.h ├── context-recurse.c ├── convert-saved-net.c ├── find-bird-calls ├── gstclassify.c ├── gstclassify.h ├── gstparrot.c ├── gstparrot.h ├── gstrecur_audio.c ├── gstrecur_audio.h ├── gstrecur_manager.c ├── gstrecur_manager.h ├── gstrecur_video.c ├── gstrecur_video.h ├── gstrnnca.c ├── gstrnnca.h ├── gtk-recur.c ├── gtkdisplay.py ├── licenses ├── CC0 ├── COPYING.mdct ├── GPL-2 ├── LGPL-2 ├── LGPL-2.1 ├── MIT.pycdb └── README ├── local.mak.example.x86_64 ├── mdct.c ├── mdct.h ├── mfcc.c ├── mfcc.h ├── multi-test ├── opt-helpers.h ├── pending_properties.h ├── pgm_dump.h ├── player-common.h ├── plot ├── py-recur-helpers.h ├── py-recur-numpy.c ├── py-recur-text.c ├── py-recur-text.h ├── recur-common.h ├── recur-config.h ├── recur-context.c ├── recur-context.h ├── recur-nn-helpers.h ├── recur-nn-init.c ├── recur-nn-io.c ├── recur-nn.c ├── recur-nn.h ├── recur-rng.h ├── rescale.c ├── rescale.h ├── rnnca-player.c ├── scripts ├── colour-gen ├── compare-nets ├── find-best-nets ├── find-character-set ├── find-feature-means ├── pycdb.py ├── reduce-video.sh ├── rnn_describe ├── rnnca-train.sh ├── shuffler.py ├── switch-blas.sh └── test-doc-labels ├── setup-charmodel.py ├── setup-rnnumpy.py ├── startup ├── README ├── recur-startup.sh ├── recur.desktop.template ├── rnnca-startup.sh └── rnnca.desktop.template ├── test-images ├── Wai1874NgaM-nfc.txt ├── Wai1874NgaM-nfd.txt └── erewhon.txt ├── test ├── charmodel.py ├── multi-text-6c34c563i73-h99-o3650.net ├── rnnumpy.py ├── test_badmath.c ├── test_charmodel_alphabet.c ├── test_dct.c ├── test_exp.c ├── test_fb_backprop.c ├── test_mdct.c ├── test_mfcc_bins.c ├── test_mfcc_table.c ├── test_poisson.c ├── test_random.c ├── test_rescale.c ├── test_simple_rescale.c ├── test_utf8.c ├── test_window.c └── test_window_functions.c ├── text-classify-results.c ├── text-classify.c ├── text-confabulate.c ├── text-cross-entropy.c ├── text-predict.c ├── utf8.h ├── valgrind-python.supp ├── window.h └── xml-lang-classify.c /.gitignore: -------------------------------------------------------------------------------- 1 | #ignore these in git listings 2 | *.pyc 3 | *~ 4 | *.[oadi] 5 | *.so 6 | .emacs* 7 | /junk/ 8 | test_* 9 | !test_*.[ch] 10 | convert-saved-net 11 | /test-images/generated 12 | /test-images/output 13 | /test-images/dickens 14 | /test-images/*.txt 15 | /test-images/more-text/*.txt 16 | !/test-images/erewhon.txt 17 | /test-video 18 | /test-audio 19 | *.p[bgp]m 20 | *.log 21 | /images/ 22 | /*.net 23 | /*.S 24 | /debug/ 25 | /gtk-recur 26 | /nets 27 | /path.h 28 | /ccan/tools/configurator/configurator 29 | /config.h 30 | /rnnca-player 31 | /examples/ 32 | local.mak 33 | /bird-data/ 34 | *.dot 35 | /data/ 36 | /xml-lang-classify 37 | /text-confabulate 38 | /text-cross-entropy 39 | /text-classify 40 | /text-classify-results 41 | /text-predict 42 | /colour-spectrum.h 43 | /TAGS 44 | -------------------------------------------------------------------------------- /alphabet-test: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys, os 3 | 4 | os.environ['LD_LIBRARY_PATH'] = '.' 5 | 6 | import charmodel 7 | 8 | a = charmodel.Alphabet('vwervwr234wrvw ') 9 | print a 10 | print a.alphabet 11 | print a.get_codepoint('2') 12 | print a.get_codepoint('c') 13 | 14 | try: 15 | a.encode_text(1) 16 | except ValueError, e: 17 | print "got expected error %r" % e 18 | 19 | t = a.encode_text('fgdgergegewvwvwg') 20 | 21 | print a.decode_text(t) 22 | 23 | b = charmodel.Alphabet('vwevw ') 24 | 25 | c = charmodel.Alphabet('vcws', utf8=0) 26 | 27 | #print c, dir(c) 28 | 29 | print c.alphabet 30 | print c.get_codepoint('c') 31 | print c.utf8 32 | 33 | 34 | n = charmodel.Net(a, ('a', 'bb', 3), 99) 35 | 36 | print n 37 | print dir(n) 38 | print n.class_name_lut 39 | 40 | print vars(charmodel) 41 | -------------------------------------------------------------------------------- /audio-common.h: -------------------------------------------------------------------------------- 1 | #ifndef HAVE_AUDIO_COMMON_H 2 | #define HAVE_AUDIO_COMMON_H 3 | 4 | #include "recur-common.h" 5 | 6 | /* queue_audio_segment collects up the audio data in a circular buffer, but 7 | leaves the deinterlacing and interpretation till later. This is a 8 | decoupling step because the incoming packet size is unrelated to the 9 | evaluation window size. 10 | */ 11 | static inline void 12 | queue_audio_segment(GstBuffer *buffer, s16 *const queue, const int queue_size, 13 | int *read_offset, int *write_offset) 14 | { 15 | GstMapInfo map; 16 | gst_buffer_map(buffer, &map, GST_MAP_READ); 17 | int len = map.size / sizeof(s16); 18 | 19 | int lag = *write_offset - *read_offset; 20 | if (lag < 0){ 21 | lag += queue_size; 22 | } 23 | if (lag + len > queue_size){ 24 | GST_WARNING("audio queue lag %d + %d seems to exceed queue size %d", 25 | lag, len, queue_size); 26 | } 27 | 28 | if (*write_offset + len < queue_size){ 29 | memcpy(queue + *write_offset, map.data, map.size); 30 | *write_offset += len; 31 | } 32 | else { 33 | int snip = queue_size - *write_offset; 34 | int snip8 = snip * sizeof(s16); 35 | memcpy(queue + *write_offset, map.data, snip8); 36 | memcpy(queue, map.data + snip8, 37 | map.size - snip8); 38 | *write_offset = len - snip; 39 | } 40 | 41 | GST_LOG("queueing audio starting %" PRIu64 ", ending %" PRIu64, 42 | GST_BUFFER_PTS(buffer), GST_BUFFER_PTS(buffer) + GST_BUFFER_DURATION(buffer)); 43 | gst_buffer_unmap(buffer, &map); 44 | } 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /badmaths.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Douglas Bagnall LGPL */ 2 | #ifndef HAVE_BADMATHS_H 3 | #define HAVE_BADMATHS_H 4 | /*XXX it would be possible to vectorise and do 4 at a time */ 5 | 6 | #define SYMMETRIC_EXTREMA_CLAMP(x, extreme, low, high) do { \ 7 | if ((x) <= -(extreme)) return (low); \ 8 | if ((x) >= (extreme)) return (high); } while(0) 9 | 10 | 11 | /*Pade 2, 2 approximant 12 | Mineiro has a faster less accurate method. 13 | */ 14 | static inline float __attribute__ ((always_inline)) 15 | fast_expf(float x){ 16 | int count = 0; 17 | while (fabsf(x) > 0.2){ 18 | x *= 0.125; 19 | count++; 20 | } 21 | float a = ((x + 3) * (x + 3) + 3) / ((x - 3) * (x - 3) + 3); 22 | while(count){ 23 | a *= a; 24 | a *= a; 25 | a *= a; 26 | count--; 27 | } 28 | return a; 29 | } 30 | 31 | #define SIGMOID_SCALE 1.0f 32 | 33 | static inline float 34 | fast_sigmoid(float x){ 35 | return 1.0f / (1.0f + fast_expf(-x * SIGMOID_SCALE)); 36 | } 37 | 38 | static inline void 39 | fast_sigmoid_array(float *dest, float *src, int len) 40 | { 41 | for (int i = 0; i < len; i++){ 42 | dest[i] = 1.0f / (1.0f + fast_expf(-src[i] * SIGMOID_SCALE)); 43 | } 44 | } 45 | 46 | static inline void 47 | fast_sigmoid_byte_array(u8 *dest, const float *src, int len) 48 | { 49 | for (int i = 0; i < len; i++){ 50 | dest[i] = 255.99f / (1.0f + fast_expf(-src[i] * SIGMOID_SCALE)); 51 | } 52 | } 53 | 54 | 55 | static inline float 56 | fast_tanhf(float x) 57 | { 58 | /*based on 59 | http://varietyofsound.wordpress.com/2011/02/14/efficient-tanh-computation-using-lamberts-continued-fraction/ 60 | */ 61 | /*rather than simply clamp, it would be possible to scale and use 62 | identities. but the answer is close to +/- 1 anyway.*/ 63 | SYMMETRIC_EXTREMA_CLAMP(x, 4.97f, -1.0f, 1.0f); 64 | float x2 = x * x; 65 | float a = x * (135135.0f + x2 * (17325.0f + x2 * (378.0f + x2))); 66 | float b = 135135.0f + x2 * (62370.0f + x2 * (3150.0f + x2 * 28.0f)); 67 | return a / b; 68 | } 69 | 70 | 71 | static inline void 72 | softmax(float *restrict dest, const float *restrict src, int len){ 73 | int i; 74 | float sum = 0.0f; 75 | float adj = 0.0f; 76 | float min = src[0]; 77 | float max = src[0]; 78 | const float max_exp = 50.0f; 79 | const float min_exp = -60.0f; 80 | /* for safety, avoid really big numbers. Addition/subtraction of a constant 81 | to the src vector doesn't affect the outcome (difference here becomes 82 | ratio post-exp). */ 83 | for (i = 1; i < len; i++){ 84 | max = MAX(max, src[i]); 85 | min = MIN(min, src[i]); 86 | } 87 | if (max > max_exp){ 88 | adj = max_exp - max; 89 | } 90 | else if (min < min_exp){ 91 | adj = MIN(min_exp - min, max_exp - max); 92 | } 93 | /*shifting most numbers toward zero would actually speed things up, because 94 | fast_expf would have less resizing to do in its loops -- but it is not 95 | clear that the midpoint of max and min will shift most numbers toward 96 | zero. Finding the mean might work better, though it is easy to imagine 97 | pathological cases there. 98 | */ 99 | //else { 100 | // adj = -0.5 * (max + min); 101 | //} 102 | for (i = 0; i < len; i++){ 103 | float f = src[i] + adj; 104 | float x = fast_expf(f); 105 | sum += x; 106 | dest[i] = x; 107 | } 108 | for (i = 0; i < len; i++){ 109 | dest[i] /= sum; 110 | } 111 | } 112 | 113 | static inline int 114 | softmax_best_guess(float *restrict error, const float *restrict src, int len) 115 | { 116 | softmax(error, src, len); 117 | /*softmax error is 0-1. all values should be 0, EXCEPT the right answer, 118 | which should be 1. The passed in error array is overwritten with negated 119 | softmax values. Training error encodes the desired change, i.e., a negative 120 | number in most cases, and 1 - softmax for the correct answer, so in 121 | training situations you want to add one straight afterwards: 122 | 123 | int target = whatever(); 124 | int winner = softmax_best_guess(error, answer, len); 125 | error[target] += 1.0f; 126 | 127 | Sum of softmax is always one, so error sum is always twice target error. 128 | */ 129 | int best_i = 0; 130 | float best_e = error[0]; 131 | error[0] = -best_e; 132 | for (int i = 1; i < len; i++){ 133 | float e = error[i]; 134 | if (e > best_e){ 135 | best_e = e; 136 | best_i = i; 137 | } 138 | error[i] = -e; 139 | } 140 | return best_i; 141 | } 142 | 143 | static inline void 144 | biased_softmax(float *restrict dest, const float *restrict src, int len, float bias){ 145 | if (bias == 0){ 146 | softmax(dest, src, len); 147 | } 148 | else { 149 | float tmp[len]; 150 | softmax(tmp, src, len); 151 | for (int i = 0; i < len; i++){ 152 | tmp[i] = tmp[i] * bias + src[i]; 153 | } 154 | softmax(dest, tmp, len); 155 | } 156 | } 157 | 158 | #endif 159 | -------------------------------------------------------------------------------- /blit-helpers.h: -------------------------------------------------------------------------------- 1 | #ifndef HAVE_BLIT_HELPERS_H 2 | #define HAVE_BLIT_HELPERS_H 3 | #include "badmaths.h" 4 | #include 5 | 6 | static inline void 7 | blit_planar_u8(GstVideoFrame *dest, u8 *planes, 8 | int x_pos, int y_pos, int width, int height, int scale){ 9 | int plane_scale[3] = {2, 1, 1}; 10 | u8*s = planes; 11 | for (int i = 0; i < 3; i++){ 12 | int pscale = scale * plane_scale[i]; 13 | int stride = GST_VIDEO_FRAME_COMP_STRIDE(dest, i); 14 | u8 *plane = GST_VIDEO_FRAME_COMP_DATA(dest, i); 15 | u8 *d = plane + (y_pos * stride + x_pos) * plane_scale[i]; 16 | if (pscale == 1){ 17 | for (int y = 0; y < height; y++){ 18 | memcpy(d, s, width); 19 | d += stride; 20 | s += width; 21 | } 22 | } 23 | else if (pscale == 2) { 24 | for (int y = 0; y < height; y++){ 25 | for (int x = 0; x < width; x++){ 26 | d[2 * x] = d[2 * x + 1] = s[x]; 27 | } 28 | memcpy(d + stride, d, width * 2); 29 | d += stride * 2; 30 | s += width; 31 | } 32 | } 33 | else if (pscale == 4) { 34 | for (int y = 0; y < height; y++){ 35 | for (int x = 0; x < width; x++){ 36 | d[4 * x] = d[4 * x + 1] = d[4 * x + 2] = d[4 * x + 3] = s[x]; 37 | } 38 | memcpy(d + stride, d, width * 4); 39 | memcpy(d + stride * 2, d, width * 4); 40 | memcpy(d + stride * 3, d, width * 4); 41 | d += stride * 4; 42 | s += width; 43 | } 44 | } 45 | else { 46 | for (int y = 0; y < height; y++){ 47 | for (int x = 0; x < width; x++){ 48 | for (int k = 0; k < pscale; k++){ 49 | d[pscale * x + k] = s[x]; 50 | } 51 | } 52 | for (int k = 1; k < pscale; k++){ 53 | memcpy(d + stride * k, d, width * pscale); 54 | } 55 | d += stride * pscale; 56 | s += width; 57 | } 58 | } 59 | } 60 | GST_DEBUG(" added planar u8 something"); 61 | } 62 | 63 | static inline void 64 | blit_planar_float(GstVideoFrame *dest, const float *planes, 65 | int x_pos, int y_pos, int width, int height, int scale, 66 | bool sigmoid_norm){ 67 | int len = width * height * 3; 68 | u8 bytes[len]; 69 | if (sigmoid_norm){ 70 | fast_sigmoid_byte_array(bytes, planes, len); 71 | } 72 | else { 73 | for (int i = 0; i < len; i++){ 74 | bytes[i] = planes[i] * 255.99f; 75 | } 76 | } 77 | blit_planar_u8(dest, bytes, x_pos, y_pos, width, height, scale); 78 | } 79 | 80 | 81 | static inline void 82 | stretch_row(const u8 *restrict src, u8 *restrict dest, 83 | const int s_width, const int d_width) 84 | { 85 | int i = 0; 86 | int j = 0; 87 | int k = 0; 88 | for (; i < d_width; i++, j+= s_width){ 89 | if (j > d_width){ 90 | 91 | j -= d_width; 92 | k++; 93 | } 94 | dest[i] = src[k]; 95 | } 96 | } 97 | 98 | 99 | static inline void 100 | fill_from_planar_u8(GstVideoFrame *frame, const u8 *restrict src, 101 | int s_width, int s_height){ 102 | for (int i = 0; i < 3; i++){ 103 | int stride = GST_VIDEO_FRAME_COMP_STRIDE(frame, i); 104 | int d_width = GST_VIDEO_FRAME_COMP_WIDTH(frame, i); 105 | int d_height = GST_VIDEO_FRAME_COMP_HEIGHT(frame, i); 106 | u8 *plane = GST_VIDEO_FRAME_COMP_DATA(frame, i); 107 | int h = 0; 108 | int j = 0; 109 | int k = 0; 110 | stretch_row(src, plane, s_width, d_width); 111 | u8 *current_row = plane; 112 | for (; h < d_height; h++, j+= s_height){ 113 | if (j > d_height){ 114 | j -= d_height; 115 | k++; 116 | current_row = plane + h * stride; 117 | stretch_row(src + k * s_width, current_row, s_width, d_width); 118 | } 119 | else { 120 | memcpy(plane + h * stride, current_row, d_width); 121 | } 122 | } 123 | src += s_width * s_height; 124 | } 125 | } 126 | 127 | #endif 128 | -------------------------------------------------------------------------------- /ccan/build_assert/LICENSE: -------------------------------------------------------------------------------- 1 | ../../licenses/CC0 -------------------------------------------------------------------------------- /ccan/build_assert/_info: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "config.h" 4 | 5 | /** 6 | * build_assert - routines for build-time assertions 7 | * 8 | * This code provides routines which will cause compilation to fail should some 9 | * assertion be untrue: such failures are preferable to run-time assertions, 10 | * but much more limited since they can only depends on compile-time constants. 11 | * 12 | * These assertions are most useful when two parts of the code must be kept in 13 | * sync: it is better to avoid such cases if possible, but seconds best is to 14 | * detect invalid changes at build time. 15 | * 16 | * For example, a tricky piece of code might rely on a certain element being at 17 | * the start of the structure. To ensure that future changes don't break it, 18 | * you would catch such changes in your code like so: 19 | * 20 | * Example: 21 | * #include 22 | * #include 23 | * 24 | * struct foo { 25 | * char string[5]; 26 | * int x; 27 | * }; 28 | * 29 | * static char *foo_string(struct foo *foo) 30 | * { 31 | * // This trick requires that the string be first in the structure 32 | * BUILD_ASSERT(offsetof(struct foo, string) == 0); 33 | * return (char *)foo; 34 | * } 35 | * 36 | * License: CC0 (Public domain) 37 | * Author: Rusty Russell 38 | */ 39 | int main(int argc, char *argv[]) 40 | { 41 | if (argc != 2) 42 | return 1; 43 | 44 | if (strcmp(argv[1], "depends") == 0) 45 | /* Nothing. */ 46 | return 0; 47 | 48 | return 1; 49 | } 50 | -------------------------------------------------------------------------------- /ccan/build_assert/build_assert.h: -------------------------------------------------------------------------------- 1 | /* CC0 (Public domain) - see LICENSE file for details */ 2 | #ifndef CCAN_BUILD_ASSERT_H 3 | #define CCAN_BUILD_ASSERT_H 4 | 5 | /** 6 | * BUILD_ASSERT - assert a build-time dependency. 7 | * @cond: the compile-time condition which must be true. 8 | * 9 | * Your compile will fail if the condition isn't true, or can't be evaluated 10 | * by the compiler. This can only be used within a function. 11 | * 12 | * Example: 13 | * #include 14 | * ... 15 | * static char *foo_to_char(struct foo *foo) 16 | * { 17 | * // This code needs string to be at start of foo. 18 | * BUILD_ASSERT(offsetof(struct foo, string) == 0); 19 | * return (char *)foo; 20 | * } 21 | */ 22 | #define BUILD_ASSERT(cond) \ 23 | do { (void) sizeof(char [1 - 2*!(cond)]); } while(0) 24 | 25 | /** 26 | * BUILD_ASSERT_OR_ZERO - assert a build-time dependency, as an expression. 27 | * @cond: the compile-time condition which must be true. 28 | * 29 | * Your compile will fail if the condition isn't true, or can't be evaluated 30 | * by the compiler. This can be used in an expression: its value is "0". 31 | * 32 | * Example: 33 | * #define foo_to_char(foo) \ 34 | * ((char *)(foo) \ 35 | * + BUILD_ASSERT_OR_ZERO(offsetof(struct foo, string) == 0)) 36 | */ 37 | #define BUILD_ASSERT_OR_ZERO(cond) \ 38 | (sizeof(char [1 - 2*!(cond)]) - 1) 39 | 40 | #endif /* CCAN_BUILD_ASSERT_H */ 41 | -------------------------------------------------------------------------------- /ccan/cast/LICENSE: -------------------------------------------------------------------------------- 1 | ../../licenses/LGPL-2.1 -------------------------------------------------------------------------------- /ccan/cast/_info: -------------------------------------------------------------------------------- 1 | #include 2 | #include "config.h" 3 | 4 | /** 5 | * cast - routines for safer casting. 6 | * 7 | * Often you want to cast in a limited way, such as removing a const or 8 | * switching between integer types. However, normal casts will work on 9 | * almost any type, making them dangerous when the code changes. 10 | * 11 | * These C++-inspired macros serve two purposes: they make it clear the 12 | * exact reason for the cast, and they also (with some compilers) cause 13 | * errors when misused. 14 | * 15 | * Based on Jan Engelhardt's libHX macros: http://libhx.sourceforge.net/ 16 | * 17 | * Author: Jan Engelhardt 18 | * Maintainer: Rusty Russell 19 | * License: LGPL (v2.1 or any later version) 20 | * 21 | * Example: 22 | * // Given "test" contains "3 t's in 'test string' 23 | * #include 24 | * #include 25 | * #include 26 | * 27 | * // Find char @orig in @str, if @repl, replace them. Return number. 28 | * static size_t find_chars(char *str, char orig, char repl) 29 | * { 30 | * size_t i, count = 0; 31 | * for (i = 0; str[i]; i++) { 32 | * if (str[i] == orig) { 33 | * count++; 34 | * if (repl) 35 | * str[i] = repl; 36 | * } 37 | * } 38 | * return count; 39 | * } 40 | * 41 | * // Terrible hash function. 42 | * static uint64_t hash_string(const unsigned char *str) 43 | * { 44 | * size_t i; 45 | * uint64_t hash = 0; 46 | * for (i = 0; str[i]; i++) 47 | * hash += str[i]; 48 | * return hash; 49 | * } 50 | * 51 | * int main(int argc, char *argv[]) 52 | * { 53 | * uint64_t hash; 54 | * 55 | * // find_chars wants a non-const string, but doesn't 56 | * // need it if repl == 0. 57 | * printf("%zu %c's in 'test string'\n", 58 | * find_chars(cast_const(char *, "test string"), 59 | * argv[1][0], 0), 60 | * argv[1][0]); 61 | * 62 | * // hash_string wants an unsigned char. 63 | * hash = hash_string(cast_signed(unsigned char *, argv[1])); 64 | * 65 | * // Need a long long to hand to printf. 66 | * printf("Hash of '%s' = %llu\n", argv[1], 67 | * cast_static(unsigned long long, hash)); 68 | * return 0; 69 | * } 70 | * 71 | */ 72 | int main(int argc, char *argv[]) 73 | { 74 | /* Expect exactly one argument */ 75 | if (argc != 2) 76 | return 1; 77 | 78 | if (strcmp(argv[1], "depends") == 0) { 79 | printf("ccan/build_assert\n"); 80 | return 0; 81 | } 82 | 83 | return 1; 84 | } 85 | -------------------------------------------------------------------------------- /ccan/compiler/LICENSE: -------------------------------------------------------------------------------- 1 | ../../licenses/CC0 -------------------------------------------------------------------------------- /ccan/compiler/_info: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "config.h" 4 | 5 | /** 6 | * compiler - macros for common compiler extensions 7 | * 8 | * Abstracts away some compiler hints. Currently these include: 9 | * - COLD 10 | * For functions not called in fast paths (aka. cold functions) 11 | * - PRINTF_FMT 12 | * For functions which take printf-style parameters. 13 | * - CONST_FUNCTION 14 | * For functions which return the same value for same parameters. 15 | * - NEEDED 16 | * For functions and variables which must be emitted even if unused. 17 | * - UNNEEDED 18 | * For functions and variables which need not be emitted if unused. 19 | * - UNUSED 20 | * For parameters which are not used. 21 | * - IS_COMPILE_CONSTANT() 22 | * For using different tradeoffs for compiletime vs runtime evaluation. 23 | * 24 | * License: CC0 (Public domain) 25 | * Author: Rusty Russell 26 | * 27 | * Example: 28 | * #include 29 | * #include 30 | * #include 31 | * 32 | * // Example of a (slow-path) logging function. 33 | * static int log_threshold = 2; 34 | * static void COLD PRINTF_FMT(2,3) 35 | * logger(int level, const char *fmt, ...) 36 | * { 37 | * va_list ap; 38 | * va_start(ap, fmt); 39 | * if (level >= log_threshold) 40 | * vfprintf(stderr, fmt, ap); 41 | * va_end(ap); 42 | * } 43 | * 44 | * int main(int argc, char *argv[]) 45 | * { 46 | * if (argc != 1) { 47 | * logger(3, "Don't want %i arguments!\n", argc-1); 48 | * return 1; 49 | * } 50 | * return 0; 51 | * } 52 | */ 53 | int main(int argc, char *argv[]) 54 | { 55 | /* Expect exactly one argument */ 56 | if (argc != 2) 57 | return 1; 58 | 59 | if (strcmp(argv[1], "depends") == 0) { 60 | return 0; 61 | } 62 | 63 | return 1; 64 | } 65 | -------------------------------------------------------------------------------- /ccan/opt/.depends: -------------------------------------------------------------------------------- 1 | ccan/typesafe_cb 2 | ccan/compiler 3 | -------------------------------------------------------------------------------- /ccan/opt/LICENSE: -------------------------------------------------------------------------------- 1 | ../../licenses/GPL-2 -------------------------------------------------------------------------------- /ccan/opt/_info: -------------------------------------------------------------------------------- 1 | #include "config.h" 2 | #include 3 | #include 4 | 5 | /** 6 | * opt - simple command line parsing 7 | * 8 | * Simple but powerful command line parsing. 9 | * 10 | * See Also: 11 | * ccan/autodata 12 | * 13 | * Example: 14 | * #include 15 | * #include 16 | * #include 17 | * 18 | * static bool someflag; 19 | * static int verbose; 20 | * static char *somestring; 21 | * 22 | * static struct opt_table opts[] = { 23 | * OPT_WITHOUT_ARG("--verbose|-v", opt_inc_intval, &verbose, 24 | * "Verbose mode (can be specified more than once)"), 25 | * OPT_WITHOUT_ARG("--someflag", opt_set_bool, &someflag, 26 | * "Set someflag"), 27 | * OPT_WITH_ARG("--somefile=", opt_set_charp, opt_show_charp, 28 | * &somestring, "Set somefile to "), 29 | * OPT_WITHOUT_ARG("--usage|--help|-h", opt_usage_and_exit, 30 | * "args...\nA silly test program.", 31 | * "Print this message."), 32 | * OPT_ENDTABLE 33 | * }; 34 | * 35 | * int main(int argc, char *argv[]) 36 | * { 37 | * int i; 38 | * 39 | * opt_register_table(opts, NULL); 40 | * // For fun, register an extra one. 41 | * opt_register_noarg("--no-someflag", opt_set_invbool, &someflag, 42 | * "Unset someflag"); 43 | * if (!opt_parse(&argc, argv, opt_log_stderr)) 44 | * exit(1); 45 | * 46 | * printf("someflag = %i, verbose = %i, somestring = %s\n", 47 | * someflag, verbose, somestring); 48 | * printf("%u args left over:", argc - 1); 49 | * for (i = 1; i < argc; i++) 50 | * printf(" %s", argv[i]); 51 | * printf("\n"); 52 | * return 0; 53 | * } 54 | * 55 | * License: GPL (v2 or any later version) 56 | * Author: Rusty Russell 57 | */ 58 | int main(int argc, char *argv[]) 59 | { 60 | if (argc != 2) 61 | return 1; 62 | 63 | if (strcmp(argv[1], "depends") == 0) { 64 | printf("ccan/cast\n"); 65 | printf("ccan/compiler\n"); 66 | printf("ccan/typesafe_cb\n"); 67 | return 0; 68 | } 69 | 70 | return 1; 71 | } 72 | -------------------------------------------------------------------------------- /ccan/opt/parse.c: -------------------------------------------------------------------------------- 1 | /* Licensed under GPLv2+ - see LICENSE file for details */ 2 | /* Actual code to parse commandline. */ 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "private.h" 8 | 9 | /* glibc does this as: 10 | /tmp/opt-example: invalid option -- 'x' 11 | /tmp/opt-example: unrecognized option '--long' 12 | /tmp/opt-example: option '--someflag' doesn't allow an argument 13 | /tmp/opt-example: option '--s' is ambiguous 14 | /tmp/opt-example: option requires an argument -- 's' 15 | */ 16 | static int parse_err(void (*errlog)(const char *fmt, ...), 17 | const char *argv0, const char *arg, unsigned len, 18 | const char *problem) 19 | { 20 | errlog("%s: %.*s: %s", argv0, len, arg, problem); 21 | return -1; 22 | } 23 | 24 | static void consume_option(int *argc, char *argv[], unsigned optnum) 25 | { 26 | memmove(&argv[optnum], &argv[optnum+1], 27 | sizeof(argv[optnum]) * (*argc-optnum)); 28 | (*argc)--; 29 | } 30 | 31 | /* Returns 1 if argument consumed, 0 if all done, -1 on error. */ 32 | int parse_one(int *argc, char *argv[], enum opt_type is_early, unsigned *offset, 33 | void (*errlog)(const char *fmt, ...)) 34 | { 35 | unsigned i, arg, len; 36 | const char *o, *optarg = NULL; 37 | char *problem = NULL; 38 | 39 | if (getenv("POSIXLY_CORRECT")) { 40 | /* Don't find options after non-options. */ 41 | arg = 1; 42 | } else { 43 | for (arg = 1; argv[arg]; arg++) { 44 | if (argv[arg][0] == '-') 45 | break; 46 | } 47 | } 48 | 49 | if (!argv[arg] || argv[arg][0] != '-') 50 | return 0; 51 | 52 | /* Special arg terminator option. */ 53 | if (strcmp(argv[arg], "--") == 0) { 54 | consume_option(argc, argv, arg); 55 | return 0; 56 | } 57 | 58 | /* Long options start with -- */ 59 | if (argv[arg][1] == '-') { 60 | assert(*offset == 0); 61 | for (o = first_lopt(&i, &len); o; o = next_lopt(o, &i, &len)) { 62 | if (strncmp(argv[arg] + 2, o, len) != 0) 63 | continue; 64 | if (argv[arg][2 + len] == '=') 65 | optarg = argv[arg] + 2 + len + 1; 66 | else if (argv[arg][2 + len] != '\0') 67 | continue; 68 | break; 69 | } 70 | if (!o) 71 | return parse_err(errlog, argv[0], 72 | argv[arg], strlen(argv[arg]), 73 | "unrecognized option"); 74 | /* For error messages, we include the leading '--' */ 75 | o -= 2; 76 | len += 2; 77 | } else { 78 | /* offset allows us to handle -abc */ 79 | for (o = first_sopt(&i); o; o = next_sopt(o, &i)) { 80 | if (argv[arg][*offset + 1] != *o) 81 | continue; 82 | (*offset)++; 83 | break; 84 | } 85 | if (!o) 86 | return parse_err(errlog, argv[0], 87 | argv[arg], strlen(argv[arg]), 88 | "unrecognized option"); 89 | /* For error messages, we include the leading '-' */ 90 | o--; 91 | len = 2; 92 | } 93 | 94 | if ((opt_table[i].type & ~OPT_EARLY) == OPT_NOARG) { 95 | if (optarg) 96 | return parse_err(errlog, argv[0], o, len, 97 | "doesn't allow an argument"); 98 | if ((opt_table[i].type & OPT_EARLY) == is_early) 99 | problem = opt_table[i].cb(opt_table[i].u.arg); 100 | } else { 101 | if (!optarg) { 102 | /* Swallow any short options as optarg, eg -afile */ 103 | if (*offset && argv[arg][*offset + 1]) { 104 | optarg = argv[arg] + *offset + 1; 105 | *offset = 0; 106 | } else 107 | optarg = argv[arg+1]; 108 | } 109 | if (!optarg) 110 | return parse_err(errlog, argv[0], o, len, 111 | "requires an argument"); 112 | if ((opt_table[i].type & OPT_EARLY) == is_early) 113 | problem = opt_table[i].cb_arg(optarg, 114 | opt_table[i].u.arg); 115 | } 116 | 117 | if (problem) { 118 | parse_err(errlog, argv[0], o, len, problem); 119 | opt_alloc.free(problem); 120 | return -1; 121 | } 122 | 123 | /* If no more letters in that short opt, reset offset. */ 124 | if (*offset && !argv[arg][*offset + 1]) 125 | *offset = 0; 126 | 127 | /* All finished with that option? */ 128 | if (*offset == 0) { 129 | consume_option(argc, argv, arg); 130 | if (optarg && optarg == argv[arg]) 131 | consume_option(argc, argv, arg); 132 | } 133 | return 1; 134 | } 135 | -------------------------------------------------------------------------------- /ccan/opt/private.h: -------------------------------------------------------------------------------- 1 | /* Licensed under GPLv2+ - see LICENSE file for details */ 2 | #ifndef CCAN_OPT_PRIVATE_H 3 | #define CCAN_OPT_PRIVATE_H 4 | 5 | extern struct opt_table *opt_table; 6 | extern unsigned int opt_count, opt_num_short, opt_num_short_arg, opt_num_long; 7 | 8 | extern const char *opt_argv0; 9 | 10 | #define subtable_of(entry) ((const struct opt_table *)((entry)->names)) 11 | 12 | const char *first_sopt(unsigned *i); 13 | const char *next_sopt(const char *names, unsigned *i); 14 | const char *first_lopt(unsigned *i, unsigned *len); 15 | const char *next_lopt(const char *p, unsigned *i, unsigned *len); 16 | 17 | struct opt_alloc { 18 | void *(*alloc)(size_t size); 19 | void *(*realloc)(void *ptr, size_t size); 20 | void (*free)(void *ptr); 21 | }; 22 | extern struct opt_alloc opt_alloc; 23 | 24 | int parse_one(int *argc, char *argv[], enum opt_type is_early, unsigned *offset, 25 | void (*errlog)(const char *fmt, ...)); 26 | 27 | #endif /* CCAN_OPT_PRIVATE_H */ 28 | -------------------------------------------------------------------------------- /ccan/opt/test/compile_ok-const-arg.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | int main(int argc, char *argv[]) 8 | { 9 | opt_register_noarg("-v", opt_version_and_exit, 10 | (const char *)"1.2.3", 11 | (const char *)"Print version"); 12 | return 0; 13 | } 14 | -------------------------------------------------------------------------------- /ccan/opt/test/run-add_desc.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | static void show_10(char buf[OPT_SHOW_LEN], const void *arg) 8 | { 9 | memset(buf, 'X', 10); 10 | buf[10] = '\0'; 11 | } 12 | 13 | static void show_max(char buf[OPT_SHOW_LEN], const void *arg) 14 | { 15 | memset(buf, 'X', OPT_SHOW_LEN); 16 | } 17 | 18 | /* Test add_desc helper. */ 19 | int main(int argc, char *argv[]) 20 | { 21 | struct opt_table opt; 22 | char *ret; 23 | size_t len, max; 24 | 25 | plan_tests(30); 26 | 27 | opt.show = NULL; 28 | opt.names = "01234"; 29 | opt.desc = "0123456789 0"; 30 | opt.type = OPT_NOARG; 31 | len = max = 0; 32 | 33 | /* Fits easily. */ 34 | ret = add_desc(NULL, &len, &max, 10, 30, &opt); 35 | ok1(len < max); 36 | ret[len] = '\0'; 37 | ok1(strcmp(ret, "01234 0123456789 0\n") == 0); 38 | free(ret); len = max = 0; 39 | 40 | /* Name just fits. */ 41 | ret = add_desc(NULL, &len, &max, 7, 30, &opt); 42 | ok1(len < max); 43 | ret[len] = '\0'; 44 | ok1(strcmp(ret, "01234 0123456789 0\n") == 0); 45 | free(ret); len = max = 0; 46 | 47 | /* Name doesn't fit. */ 48 | ret = add_desc(NULL, &len, &max, 6, 30, &opt); 49 | ok1(len < max); 50 | ret[len] = '\0'; 51 | ok1(strcmp(ret, 52 | "01234\n" 53 | " 0123456789 0\n") == 0); 54 | free(ret); len = max = 0; 55 | 56 | /* Description just fits. */ 57 | ret = add_desc(NULL, &len, &max, 7, 19, &opt); 58 | ok1(len < max); 59 | ret[len] = '\0'; 60 | ok1(strcmp(ret, "01234 0123456789 0\n") == 0); 61 | free(ret); len = max = 0; 62 | 63 | /* Description doesn't quite fit. */ 64 | ret = add_desc(NULL, &len, &max, 7, 18, &opt); 65 | ok1(len < max); 66 | ret[len] = '\0'; 67 | ok1(strcmp(ret, 68 | "01234 0123456789\n" 69 | " 0\n") == 0); 70 | free(ret); len = max = 0; 71 | 72 | /* Neither quite fits. */ 73 | ret = add_desc(NULL, &len, &max, 6, 17, &opt); 74 | ok1(len < max); 75 | ret[len] = '\0'; 76 | ok1(strcmp(ret, 77 | "01234\n" 78 | " 0123456789\n" 79 | " 0\n") == 0); 80 | free(ret); len = max = 0; 81 | 82 | /* With show function, fits just. */ 83 | opt.show = show_10; 84 | ret = add_desc(NULL, &len, &max, 7, 41, &opt); 85 | ok1(len < max); 86 | ret[len] = '\0'; 87 | ok1(strcmp(ret, "01234 0123456789 0 (default: XXXXXXXXXX)\n") == 0); 88 | free(ret); len = max = 0; 89 | 90 | /* With show function, just too long. */ 91 | ret = add_desc(NULL, &len, &max, 7, 40, &opt); 92 | ok1(len < max); 93 | ret[len] = '\0'; 94 | ok1(strcmp(ret, 95 | "01234 0123456789 0\n" 96 | " (default: XXXXXXXXXX)\n") == 0); 97 | free(ret); len = max = 0; 98 | 99 | /* With maximal show function, fits just (we assume OPT_SHOW_LEN = 80. */ 100 | opt.show = show_max; 101 | ret = add_desc(NULL, &len, &max, 7, 114, &opt); 102 | ok1(len < max); 103 | ret[len] = '\0'; 104 | ok1(strcmp(ret, "01234 0123456789 0 (default: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX...)\n") == 0); 105 | free(ret); len = max = 0; 106 | 107 | /* With maximal show function, just too long. */ 108 | ret = add_desc(NULL, &len, &max, 7, 113, &opt); 109 | ok1(len < max); 110 | ret[len] = '\0'; 111 | ok1(strcmp(ret, 112 | "01234 0123456789 0\n" 113 | " (default: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX...)\n") == 0); 114 | free(ret); len = max = 0; 115 | 116 | /* With added " ". Fits, just. */ 117 | opt.show = NULL; 118 | opt.type = OPT_HASARG; 119 | ret = add_desc(NULL, &len, &max, 13, 25, &opt); 120 | ok1(len < max); 121 | ret[len] = '\0'; 122 | ok1(strcmp(ret, "01234 0123456789 0\n") == 0); 123 | free(ret); len = max = 0; 124 | 125 | /* With added " ". Name doesn't quite fit. */ 126 | ret = add_desc(NULL, &len, &max, 12, 25, &opt); 127 | ok1(len < max); 128 | ret[len] = '\0'; 129 | ok1(strcmp(ret, 130 | "01234 \n" 131 | " 0123456789 0\n") == 0); 132 | free(ret); len = max = 0; 133 | 134 | /* With added " ". Desc doesn't quite fit. */ 135 | ret = add_desc(NULL, &len, &max, 13, 24, &opt); 136 | ok1(len < max); 137 | ret[len] = '\0'; 138 | ok1(strcmp(ret, 139 | "01234 0123456789\n" 140 | " 0\n") == 0); 141 | free(ret); len = max = 0; 142 | 143 | /* Empty description, with and default. Just fits. */ 144 | opt.show = show_10; 145 | opt.desc = ""; 146 | ret = add_desc(NULL, &len, &max, 13, 35, &opt); 147 | ok1(len < max); 148 | ret[len] = '\0'; 149 | ok1(strcmp(ret, "01234 (default: XXXXXXXXXX)\n") == 0); 150 | free(ret); len = max = 0; 151 | 152 | /* Empty description, with and default. Doesn't quite fit. */ 153 | opt.show = show_10; 154 | opt.desc = ""; 155 | ret = add_desc(NULL, &len, &max, 13, 34, &opt); 156 | ok1(len < max); 157 | ret[len] = '\0'; 158 | ok1(strcmp(ret, 159 | "01234 \n" 160 | " (default: XXXXXXXXXX)\n") == 0); 161 | free(ret); len = max = 0; 162 | 163 | return exit_status(); 164 | } 165 | -------------------------------------------------------------------------------- /ccan/opt/test/run-checkopt.c: -------------------------------------------------------------------------------- 1 | #include "config.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "utils.h" 9 | 10 | /* We don't actually want it to exit... */ 11 | static jmp_buf exited; 12 | #define failmsg save_and_jump 13 | 14 | static void save_and_jump(const char *fmt, ...); 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | static char *output = NULL; 22 | 23 | static int saved_vprintf(const char *fmt, va_list ap) 24 | { 25 | char *p; 26 | int ret = vasprintf(&p, fmt, ap); 27 | 28 | if (output) { 29 | output = realloc(output, strlen(output) + strlen(p) + 1); 30 | strcat(output, p); 31 | free(p); 32 | } else 33 | output = p; 34 | return ret; 35 | } 36 | 37 | static void save_and_jump(const char *fmt, ...) 38 | { 39 | va_list ap; 40 | 41 | va_start(ap, fmt); 42 | saved_vprintf(fmt, ap); 43 | va_end(ap); 44 | longjmp(exited, 1); 45 | } 46 | 47 | static void reset(void) 48 | { 49 | free(output); 50 | output = NULL; 51 | free(opt_table); 52 | opt_table = NULL; 53 | opt_count = opt_num_short = opt_num_short_arg = opt_num_long = 0; 54 | } 55 | 56 | int main(int argc, char *argv[]) 57 | { 58 | int exitval; 59 | 60 | plan_tests(14); 61 | 62 | exitval = setjmp(exited); 63 | if (exitval == 0) { 64 | /* Bad type. */ 65 | _opt_register("-a", OPT_SUBTABLE, (void *)opt_version_and_exit, 66 | NULL, NULL, "1.2.3", ""); 67 | fail("_opt_register returned?"); 68 | } else { 69 | ok1(exitval == 1); 70 | ok1(strstr(output, "Option -a: unknown entry type")); 71 | } 72 | reset(); 73 | 74 | exitval = setjmp(exited); 75 | if (exitval == 0) { 76 | /* NULL description. */ 77 | opt_register_noarg("-a", test_noarg, "", NULL); 78 | fail("_opt_register returned?"); 79 | } else { 80 | ok1(exitval == 1); 81 | ok1(strstr(output, "Option -a: description cannot be NULL")); 82 | } 83 | reset(); 84 | 85 | exitval = setjmp(exited); 86 | if (exitval == 0) { 87 | /* Bad option name. */ 88 | opt_register_noarg("a", test_noarg, "", ""); 89 | fail("_opt_register returned?"); 90 | } else { 91 | ok1(exitval == 1); 92 | ok1(strstr(output, "Option a: does not begin with '-'")); 93 | } 94 | 95 | reset(); 96 | 97 | exitval = setjmp(exited); 98 | if (exitval == 0) { 99 | /* Bad option name. */ 100 | opt_register_noarg("--", test_noarg, "", ""); 101 | fail("_opt_register returned?"); 102 | } else { 103 | ok1(exitval == 1); 104 | ok1(strstr(output, "Option --: invalid long option '--'")); 105 | } 106 | 107 | reset(); 108 | 109 | exitval = setjmp(exited); 110 | if (exitval == 0) { 111 | /* Bad option name. */ 112 | opt_register_noarg("--a|-aaa", test_noarg, "", ""); 113 | fail("_opt_register returned?"); 114 | } else { 115 | ok1(exitval == 1); 116 | ok1(strstr(output, 117 | "Option --a|-aaa: invalid short option '-aaa'")); 118 | } 119 | reset(); 120 | 121 | exitval = setjmp(exited); 122 | if (exitval == 0) { 123 | /* Documentation for non-optios. */ 124 | opt_register_noarg("--a foo", test_noarg, "", ""); 125 | fail("_opt_register returned?"); 126 | } else { 127 | ok1(exitval == 1); 128 | ok1(strstr(output, 129 | "Option --a foo: does not take arguments 'foo'")); 130 | } 131 | reset(); 132 | 133 | exitval = setjmp(exited); 134 | if (exitval == 0) { 135 | /* Documentation for non-optios. */ 136 | opt_register_noarg("--a=foo", test_noarg, "", ""); 137 | fail("_opt_register returned?"); 138 | } else { 139 | ok1(exitval == 1); 140 | ok1(strstr(output, 141 | "Option --a=foo: does not take arguments 'foo'")); 142 | } 143 | return exit_status(); 144 | } 145 | -------------------------------------------------------------------------------- /ccan/opt/test/run-consume_words.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | /* Test consume_words helper. */ 8 | int main(int argc, char *argv[]) 9 | { 10 | size_t prefix, len; 11 | bool start = true; 12 | 13 | plan_tests(27); 14 | 15 | /* Every line over width. */ 16 | len = consume_words("hello world", 1, &prefix, &start); 17 | ok1(prefix == 0); 18 | ok1(!start); 19 | ok1(len == strlen("hello")); 20 | len = consume_words(" world", 1, &prefix, &start); 21 | ok1(prefix == 1); 22 | ok1(len == strlen("world")); 23 | ok1(!start); 24 | ok1(consume_words("", 1, &prefix, &start) == 0); 25 | 26 | /* Same with width where won't both fit. */ 27 | start = true; 28 | len = consume_words("hello world", 5, &prefix, &start); 29 | ok1(!start); 30 | ok1(prefix == 0); 31 | ok1(len == strlen("hello")); 32 | len = consume_words(" world", 5, &prefix, &start); 33 | ok1(!start); 34 | ok1(prefix == 1); 35 | ok1(len == strlen("world")); 36 | ok1(consume_words("", 5, &prefix, &start) == 0); 37 | 38 | start = true; 39 | len = consume_words("hello world", 11, &prefix, &start); 40 | ok1(!start); 41 | ok1(prefix == 0); 42 | ok1(len == strlen("hello world")); 43 | ok1(consume_words("", 11, &prefix, &start) == 0); 44 | 45 | /* Now try a literal, should not be broken */ 46 | start = true; 47 | len = consume_words(" hello world", 5, &prefix, &start); 48 | ok1(!start); 49 | ok1(prefix == 1); 50 | ok1(len == strlen("hello world")); 51 | 52 | /* A literal after an explicit \n also not broken */ 53 | start = true; 54 | len = consume_words("hi\n hello world", 5, &prefix, &start); 55 | ok1(start); 56 | ok1(prefix == 0); 57 | ok1(len == strlen("hi\n")); 58 | len = consume_words(" hello world", 5, &prefix, &start); 59 | ok1(!start); 60 | ok1(prefix == 1); 61 | ok1(len == strlen("hello world")); 62 | 63 | return exit_status(); 64 | } 65 | -------------------------------------------------------------------------------- /ccan/opt/test/run-correct-reporting.c: -------------------------------------------------------------------------------- 1 | /* Make sure when multiple equivalent options, correct one is used for errors */ 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "utils.h" 10 | 11 | int main(int argc, char *argv[]) 12 | { 13 | plan_tests(12); 14 | 15 | /* --aaa without args. */ 16 | opt_register_arg("-a|--aaa", test_arg, NULL, "aaa", ""); 17 | ok1(!parse_args(&argc, &argv, "--aaa", NULL)); 18 | ok1(strstr(err_output, ": --aaa: requires an argument")); 19 | free(err_output); 20 | err_output = NULL; 21 | ok1(!parse_args(&argc, &argv, "-a", NULL)); 22 | ok1(strstr(err_output, ": -a: requires an argument")); 23 | free(err_output); 24 | err_output = NULL; 25 | 26 | /* Multiple */ 27 | opt_register_arg("--bbb|-b|-c|--ccc", test_arg, NULL, "aaa", ""); 28 | ok1(!parse_args(&argc, &argv, "--bbb", NULL)); 29 | ok1(strstr(err_output, ": --bbb: requires an argument")); 30 | free(err_output); 31 | err_output = NULL; 32 | ok1(!parse_args(&argc, &argv, "-b", NULL)); 33 | ok1(strstr(err_output, ": -b: requires an argument")); 34 | free(err_output); 35 | err_output = NULL; 36 | ok1(!parse_args(&argc, &argv, "-c", NULL)); 37 | ok1(strstr(err_output, ": -c: requires an argument")); 38 | free(err_output); 39 | err_output = NULL; 40 | ok1(!parse_args(&argc, &argv, "--ccc", NULL)); 41 | ok1(strstr(err_output, ": --ccc: requires an argument")); 42 | free(err_output); 43 | err_output = NULL; 44 | 45 | /* parse_args allocates argv */ 46 | free(argv); 47 | return exit_status(); 48 | } 49 | 50 | -------------------------------------------------------------------------------- /ccan/opt/test/run-early.c: -------------------------------------------------------------------------------- 1 | /* With errlog == NULL, we never get a "failure". */ 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "utils.h" 9 | 10 | struct opt_table some_early_table[] = { 11 | OPT_EARLY_WITHOUT_ARG("--verbose|-v", test_noarg, 12 | "vvv", "Description of verbose"), 13 | OPT_EARLY_WITH_ARG("--debug|-d", test_arg, show_arg, 14 | "ddd", "Description of debug"), 15 | OPT_WITHOUT_ARG("-h|--hhh", test_noarg, "hhh", "Description of hhh"), 16 | OPT_ENDTABLE 17 | }; 18 | 19 | int main(int argc, char *argv[]) 20 | { 21 | const char *myname = argv[0]; 22 | 23 | plan_tests(37); 24 | 25 | /* Simple short arg.*/ 26 | opt_register_noarg("-a", test_noarg, NULL, "All"); 27 | opt_register_early_noarg("-b", test_noarg, NULL, "All"); 28 | 29 | /* Early parsing doesn't mangle. */ 30 | ok1(parse_early_args(&argc, &argv, "-a", NULL)); 31 | ok1(argc == 2); 32 | ok1(argv[0] == myname); 33 | ok1(strcmp(argv[1], "-a") == 0); 34 | ok1(argv[2] == NULL); 35 | ok1(test_cb_called == 0); 36 | 37 | /* ... even if it processes arg. */ 38 | ok1(parse_early_args(&argc, &argv, "-b", NULL)); 39 | ok1(argc == 2); 40 | ok1(argv[0] == myname); 41 | ok1(strcmp(argv[1], "-b") == 0); 42 | ok1(argv[2] == NULL); 43 | ok1(test_cb_called == 1); 44 | 45 | ok1(parse_early_args(&argc, &argv, "-ab", NULL)); 46 | ok1(argc == 2); 47 | ok1(argv[0] == myname); 48 | ok1(strcmp(argv[1], "-ab") == 0); 49 | ok1(argv[2] == NULL); 50 | ok1(test_cb_called == 2); 51 | 52 | ok1(parse_args(&argc, &argv, "-ab", NULL)); 53 | ok1(argc == 1); 54 | ok1(argv[0] == myname); 55 | ok1(argv[1] == NULL); 56 | ok1(test_cb_called == 3); 57 | 58 | opt_register_table(some_early_table, "Some early args"); 59 | ok1(parse_early_args(&argc, &argv, "--verbose", "-dddd", "-h", NULL)); 60 | ok1(argc == 4); 61 | ok1(argv[0] == myname); 62 | ok1(strcmp(argv[1], "--verbose") == 0); 63 | ok1(strcmp(argv[2], "-dddd") == 0); 64 | ok1(strcmp(argv[3], "-h") == 0); 65 | ok1(argv[4] == NULL); 66 | ok1(test_cb_called == 5); 67 | 68 | ok1(parse_args(&argc, &argv, "--verbose", "-d", "ddd", "-h", NULL)); 69 | ok1(argc == 1); 70 | ok1(argv[0] == myname); 71 | ok1(argv[1] == NULL); 72 | ok1(test_cb_called == 6); 73 | 74 | /* parse_args allocates argv */ 75 | free(argv); 76 | return exit_status(); 77 | } 78 | -------------------------------------------------------------------------------- /ccan/opt/test/run-iter.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "utils.h" 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | /* Test iterators. */ 13 | int main(int argc, char *argv[]) 14 | { 15 | unsigned j, i, len = 0; 16 | const char *p; 17 | 18 | plan_tests(37 * 2); 19 | for (j = 0; j < 2; j ++) { 20 | reset_options(); 21 | /* Giving subtable a title makes an extra entry! */ 22 | opt_register_table(subtables, j == 0 ? NULL : "subtable"); 23 | 24 | p = first_lopt(&i, &len); 25 | ok1(i == j + 0); 26 | ok1(len == 3); 27 | ok1(strncmp(p, "jjj", len) == 0); 28 | p = next_lopt(p, &i, &len); 29 | ok1(i == j + 0); 30 | ok1(len == 3); 31 | ok1(strncmp(p, "lll", len) == 0); 32 | p = next_lopt(p, &i, &len); 33 | ok1(i == j + 1); 34 | ok1(len == 3); 35 | ok1(strncmp(p, "mmm", len) == 0); 36 | p = next_lopt(p, &i, &len); 37 | ok1(i == j + 5); 38 | ok1(len == 3); 39 | ok1(strncmp(p, "ddd", len) == 0); 40 | p = next_lopt(p, &i, &len); 41 | ok1(i == j + 6); 42 | ok1(len == 3); 43 | ok1(strncmp(p, "eee", len) == 0); 44 | p = next_lopt(p, &i, &len); 45 | ok1(i == j + 7); 46 | ok1(len == 3); 47 | ok1(strncmp(p, "ggg", len) == 0); 48 | p = next_lopt(p, &i, &len); 49 | ok1(i == j + 8); 50 | ok1(len == 3); 51 | ok1(strncmp(p, "hhh", len) == 0); 52 | p = next_lopt(p, &i, &len); 53 | ok1(!p); 54 | 55 | p = first_sopt(&i); 56 | ok1(i == j + 0); 57 | ok1(*p == 'j'); 58 | p = next_sopt(p, &i); 59 | ok1(i == j + 0); 60 | ok1(*p == 'l'); 61 | p = next_sopt(p, &i); 62 | ok1(i == j + 1); 63 | ok1(*p == 'm'); 64 | p = next_sopt(p, &i); 65 | ok1(i == j + 2); 66 | ok1(*p == 'a'); 67 | p = next_sopt(p, &i); 68 | ok1(i == j + 3); 69 | ok1(*p == 'b'); 70 | p = next_sopt(p, &i); 71 | ok1(i == j + 7); 72 | ok1(*p == 'g'); 73 | p = next_sopt(p, &i); 74 | ok1(i == j + 8); 75 | ok1(*p == 'h'); 76 | p = next_sopt(p, &i); 77 | ok1(!p); 78 | } 79 | 80 | return exit_status(); 81 | } 82 | -------------------------------------------------------------------------------- /ccan/opt/test/run-no-options.c: -------------------------------------------------------------------------------- 1 | /* Make sure we still work with no options registered */ 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "utils.h" 9 | 10 | int main(int argc, char *argv[]) 11 | { 12 | const char *myname = argv[0]; 13 | 14 | plan_tests(7); 15 | 16 | /* Simple short arg.*/ 17 | ok1(!parse_args(&argc, &argv, "-a", NULL)); 18 | /* Simple long arg.*/ 19 | ok1(!parse_args(&argc, &argv, "--aaa", NULL)); 20 | 21 | /* Extra arguments preserved. */ 22 | ok1(parse_args(&argc, &argv, "extra", "args", NULL)); 23 | ok1(argc == 3); 24 | ok1(argv[0] == myname); 25 | ok1(strcmp(argv[1], "extra") == 0); 26 | ok1(strcmp(argv[2], "args") == 0); 27 | 28 | /* parse_args allocates argv */ 29 | free(argv); 30 | 31 | return exit_status(); 32 | } 33 | 34 | -------------------------------------------------------------------------------- /ccan/opt/test/run-usage.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "utils.h" 7 | 8 | /* Ensure width is sane. */ 9 | static const char *getenv_override(const char *name) 10 | { 11 | return "100"; 12 | } 13 | 14 | #define getenv getenv_override 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | static char *my_cb(void *p) 22 | { 23 | return NULL; 24 | } 25 | 26 | /* Test helpers. */ 27 | int main(int argc, char *argv[]) 28 | { 29 | char *output; 30 | char *longname = strdup("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); 31 | char *shortname = strdup("shortname"); 32 | 33 | plan_tests(51); 34 | opt_register_table(subtables, NULL); 35 | opt_register_noarg("--kkk|-k", my_cb, NULL, "magic kkk option"); 36 | opt_register_noarg("-?", opt_usage_and_exit, "...", 37 | "This message"); 38 | opt_register_arg("--longname", opt_set_charp, opt_show_charp, 39 | &longname, "a really long option default"); 40 | opt_register_arg("--shortname", opt_set_charp, opt_show_charp, 41 | &shortname, "a short option default"); 42 | output = opt_usage("my name", "ExTrA Args"); 43 | diag("%s", output); 44 | ok1(strstr(output, "Usage: my name")); 45 | ok1(strstr(output, "--jjj|-j|--lll|-l ")); 46 | ok1(strstr(output, "ExTrA Args")); 47 | ok1(strstr(output, "-a ")); 48 | ok1(strstr(output, " Description of a\n")); 49 | ok1(strstr(output, "-b ")); 50 | ok1(strstr(output, " Description of b (default: b)\n")); 51 | ok1(strstr(output, "--ddd ")); 52 | ok1(strstr(output, " Description of ddd\n")); 53 | ok1(strstr(output, "--eee ")); 54 | ok1(strstr(output, " (default: eee)\n")); 55 | ok1(strstr(output, "long table options:\n")); 56 | ok1(strstr(output, "--ggg|-g ")); 57 | ok1(strstr(output, " Description of ggg\n")); 58 | ok1(strstr(output, "-h|--hhh ")); 59 | ok1(strstr(output, " Description of hhh\n")); 60 | ok1(strstr(output, "--kkk|-k")); 61 | ok1(strstr(output, "magic kkk option")); 62 | /* This entry is hidden. */ 63 | ok1(!strstr(output, "--mmm|-m")); 64 | free(output); 65 | 66 | /* NULL should use string from registered options. */ 67 | output = opt_usage("my name", NULL); 68 | diag("%s", output); 69 | ok1(strstr(output, "Usage: my name")); 70 | ok1(strstr(output, "--jjj|-j|--lll|-l ")); 71 | ok1(strstr(output, "...")); 72 | ok1(strstr(output, "-a ")); 73 | ok1(strstr(output, " Description of a\n")); 74 | ok1(strstr(output, "-b ")); 75 | ok1(strstr(output, " Description of b (default: b)\n")); 76 | ok1(strstr(output, "--ddd ")); 77 | ok1(strstr(output, " Description of ddd\n")); 78 | ok1(strstr(output, "--eee ")); 79 | ok1(strstr(output, " (default: eee)\n")); 80 | ok1(strstr(output, "long table options:\n")); 81 | ok1(strstr(output, "--ggg|-g ")); 82 | ok1(strstr(output, " Description of ggg\n")); 83 | ok1(strstr(output, "-h|--hhh ")); 84 | ok1(strstr(output, " Description of hhh\n")); 85 | ok1(strstr(output, "--kkk|-k")); 86 | ok1(strstr(output, "magic kkk option")); 87 | ok1(strstr(output, "--longname")); 88 | ok1(strstr(output, "a really long option default")); 89 | ok1(strstr(output, "(default: \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"...)")); 90 | ok1(strstr(output, "--shortname")); 91 | ok1(strstr(output, "a short option default")); 92 | ok1(strstr(output, "(default: \"shortname\")")); 93 | /* This entry is hidden. */ 94 | ok1(!strstr(output, "--mmm|-m")); 95 | free(output); 96 | 97 | reset_options(); 98 | /* Empty table test. */ 99 | output = opt_usage("nothing", NULL); 100 | ok1(strstr(output, "Usage: nothing \n")); 101 | free(output); 102 | 103 | /* No short args. */ 104 | opt_register_noarg("--aaa", test_noarg, NULL, "AAAAll"); 105 | output = opt_usage("onearg", NULL); 106 | ok1(strstr(output, "Usage: onearg \n")); 107 | ok1(strstr(output, "--aaa")); 108 | ok1(strstr(output, "AAAAll")); 109 | free(output); 110 | 111 | reset_options(); 112 | /* Valgrind nails this to 100 anyway :( */ 113 | setenv("COLUMNS", "100", 1); 114 | opt_register_noarg("--long", my_cb, NULL, "Extremely long option which requires more than one line for its full description to be shown in the usage message."); 115 | opt_register_noarg("--split", my_cb, NULL, "New line in\nlong option which requires more than one line for its full description to be shown in the usage message."); 116 | output = opt_usage("longarg", NULL); 117 | diag("%s", output); 118 | ok1(strstr(output, "Usage: longarg \n")); 119 | ok1(strstr(output, "\n" 120 | "--long Extremely long option which requires more than one line for its full description to be\n" 121 | " shown in the usage message.\n")); 122 | ok1(strstr(output, "\n" 123 | "--split New line in\n" 124 | " long option which requires more than one line for its full description to be shown in the\n" 125 | " usage message.\n")); 126 | free(output); 127 | 128 | free(shortname); 129 | free(longname); 130 | return exit_status(); 131 | } 132 | -------------------------------------------------------------------------------- /ccan/opt/test/utils.c: -------------------------------------------------------------------------------- 1 | #include "config.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "utils.h" 9 | 10 | unsigned int test_cb_called; 11 | char *test_noarg(void *arg) 12 | { 13 | test_cb_called++; 14 | return NULL; 15 | } 16 | 17 | char *test_arg(const char *optarg, const char *arg) 18 | { 19 | test_cb_called++; 20 | ok1(strcmp(optarg, arg) == 0); 21 | return NULL; 22 | } 23 | 24 | void show_arg(char buf[OPT_SHOW_LEN], const char *arg) 25 | { 26 | strncpy(buf, arg, OPT_SHOW_LEN); 27 | } 28 | 29 | char *err_output = NULL; 30 | 31 | void save_err_output(const char *fmt, ...) 32 | { 33 | va_list ap; 34 | char *p; 35 | 36 | va_start(ap, fmt); 37 | /* Check return, for fascist gcc */ 38 | if (vasprintf(&p, fmt, ap) == -1) 39 | p = NULL; 40 | va_end(ap); 41 | 42 | if (err_output) { 43 | err_output = realloc(err_output, 44 | strlen(err_output) + strlen(p) + 1); 45 | strcat(err_output, p); 46 | free(p); 47 | } else 48 | err_output = p; 49 | } 50 | 51 | void reset_options(void) 52 | { 53 | opt_free_table(); 54 | free(err_output); 55 | err_output = NULL; 56 | } 57 | 58 | static bool allocated = false; 59 | 60 | bool parse_args(int *argc, char ***argv, ...) 61 | { 62 | char **a; 63 | va_list ap; 64 | 65 | va_start(ap, argv); 66 | *argc = 1; 67 | a = malloc(sizeof(*a) * (*argc + 1)); 68 | a[0] = (*argv)[0]; 69 | while ((a[*argc] = va_arg(ap, char *)) != NULL) { 70 | (*argc)++; 71 | a = realloc(a, sizeof(*a) * (*argc + 1)); 72 | } 73 | 74 | if (allocated) 75 | free(*argv); 76 | 77 | *argv = a; 78 | allocated = true; 79 | 80 | return opt_parse(argc, *argv, save_err_output); 81 | } 82 | 83 | bool parse_early_args(int *argc, char ***argv, ...) 84 | { 85 | char **a; 86 | va_list ap; 87 | 88 | va_start(ap, argv); 89 | *argc = 1; 90 | a = malloc(sizeof(*a) * (*argc + 1)); 91 | a[0] = (*argv)[0]; 92 | while ((a[*argc] = va_arg(ap, char *)) != NULL) { 93 | (*argc)++; 94 | a = realloc(a, sizeof(*a) * (*argc + 1)); 95 | } 96 | 97 | if (allocated) 98 | free(*argv); 99 | 100 | *argv = a; 101 | allocated = true; 102 | 103 | return opt_early_parse(*argc, *argv, save_err_output); 104 | } 105 | 106 | struct opt_table short_table[] = { 107 | /* Short opts, different args. */ 108 | OPT_WITHOUT_ARG("-a", test_noarg, "a", "Description of a"), 109 | OPT_WITH_ARG("-b", test_arg, show_arg, "b", "Description of b"), 110 | OPT_ENDTABLE 111 | }; 112 | 113 | struct opt_table long_table[] = { 114 | /* Long opts, different args. */ 115 | OPT_WITHOUT_ARG("--ddd", test_noarg, "ddd", "Description of ddd"), 116 | OPT_WITH_ARG("--eee ", test_arg, show_arg, "eee", ""), 117 | OPT_ENDTABLE 118 | }; 119 | 120 | struct opt_table long_and_short_table[] = { 121 | /* Short and long, different args. */ 122 | OPT_WITHOUT_ARG("--ggg|-g", test_noarg, "ggg", "Description of ggg"), 123 | OPT_WITH_ARG("-h|--hhh", test_arg, NULL, "hhh", "Description of hhh"), 124 | OPT_ENDTABLE 125 | }; 126 | 127 | /* Sub-table test. */ 128 | struct opt_table subtables[] = { 129 | /* Two short, and two long long, no description */ 130 | OPT_WITH_ARG("--jjj|-j|--lll|-l", test_arg, show_arg, "jjj", ""), 131 | /* Hidden option */ 132 | OPT_WITH_ARG("--mmm|-m", test_arg, show_arg, "mmm", opt_hidden), 133 | OPT_SUBTABLE(short_table, NULL), 134 | OPT_SUBTABLE(long_table, "long table options"), 135 | OPT_SUBTABLE(long_and_short_table, NULL), 136 | OPT_ENDTABLE 137 | }; 138 | -------------------------------------------------------------------------------- /ccan/opt/test/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef CCAN_OPT_TEST_UTILS_H 2 | #define CCAN_OPT_TEST_UTILS_H 3 | #include 4 | #include 5 | 6 | bool parse_args(int *argc, char ***argv, ...); 7 | bool parse_early_args(int *argc, char ***argv, ...); 8 | extern char *err_output; 9 | void save_err_output(const char *fmt, ...); 10 | void reset_options(void); 11 | 12 | extern unsigned int test_cb_called; 13 | char *test_noarg(void *arg); 14 | char *test_arg(const char *optarg, const char *arg); 15 | void show_arg(char buf[OPT_SHOW_LEN], const char *arg); 16 | 17 | extern struct opt_table short_table[]; 18 | extern struct opt_table long_table[]; 19 | extern struct opt_table long_and_short_table[]; 20 | extern struct opt_table subtables[]; 21 | #endif /* CCAN_OPT_TEST_UTILS_H */ 22 | -------------------------------------------------------------------------------- /ccan/typesafe_cb/.depends: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/douglasbagnall/recur/fc1eb38e14be07a17f2fd765e9925873556d2546/ccan/typesafe_cb/.depends -------------------------------------------------------------------------------- /ccan/typesafe_cb/LICENSE: -------------------------------------------------------------------------------- 1 | ../../licenses/CC0 -------------------------------------------------------------------------------- /ccan/typesafe_cb/_info: -------------------------------------------------------------------------------- 1 | #include "config.h" 2 | #include 3 | #include 4 | 5 | /** 6 | * typesafe_cb - macros for safe callbacks. 7 | * 8 | * The basis of the typesafe_cb header is typesafe_cb_cast(): a 9 | * conditional cast macro. If an expression exactly matches a given 10 | * type, it is cast to the target type, otherwise it is left alone. 11 | * 12 | * This allows us to create functions which take a small number of 13 | * specific types, rather than being forced to use a void *. In 14 | * particular, it is useful for creating typesafe callbacks as the 15 | * helpers typesafe_cb(), typesafe_cb_preargs() and 16 | * typesafe_cb_postargs() demonstrate. 17 | * 18 | * The standard way of passing arguments to callback functions in C is 19 | * to use a void pointer, which the callback then casts back to the 20 | * expected type. This unfortunately subverts the type checking the 21 | * compiler would perform if it were a direct call. Here's an example: 22 | * 23 | * static void my_callback(void *_obj) 24 | * { 25 | * struct obj *obj = _obj; 26 | * ... 27 | * } 28 | * ... 29 | * register_callback(my_callback, &my_obj); 30 | * 31 | * If we wanted to use the natural type for my_callback (ie. "void 32 | * my_callback(struct obj *obj)"), we could make register_callback() 33 | * take a void * as its first argument, but this would subvert all 34 | * type checking. We really want register_callback() to accept only 35 | * the exactly correct function type to match the argument, or a 36 | * function which takes a void *. 37 | * 38 | * This is where typesafe_cb() comes in: it uses typesafe_cb_cast() to 39 | * cast the callback function if it matches the argument type: 40 | * 41 | * void _register_callback(void (*cb)(void *arg), void *arg); 42 | * #define register_callback(cb, arg) \ 43 | * _register_callback(typesafe_cb(void, void *, (cb), (arg)), \ 44 | * (arg)) 45 | * 46 | * On compilers which don't support the extensions required 47 | * typesafe_cb_cast() and friend become an unconditional cast, so your 48 | * code will compile but you won't get type checking. 49 | * 50 | * Example: 51 | * #include 52 | * #include 53 | * #include 54 | * 55 | * // Generic callback infrastructure. 56 | * struct callback { 57 | * struct callback *next; 58 | * int value; 59 | * int (*callback)(int value, void *arg); 60 | * void *arg; 61 | * }; 62 | * static struct callback *callbacks; 63 | * 64 | * static void _register_callback(int value, int (*cb)(int, void *), 65 | * void *arg) 66 | * { 67 | * struct callback *new = malloc(sizeof(*new)); 68 | * new->next = callbacks; 69 | * new->value = value; 70 | * new->callback = cb; 71 | * new->arg = arg; 72 | * callbacks = new; 73 | * } 74 | * #define register_callback(value, cb, arg) \ 75 | * _register_callback(value, \ 76 | * typesafe_cb_preargs(int, void *, \ 77 | * (cb), (arg), int),\ 78 | * (arg)) 79 | * 80 | * static struct callback *find_callback(int value) 81 | * { 82 | * struct callback *i; 83 | * 84 | * for (i = callbacks; i; i = i->next) 85 | * if (i->value == value) 86 | * return i; 87 | * return NULL; 88 | * } 89 | * 90 | * // Define several silly callbacks. Note they don't use void *! 91 | * #define DEF_CALLBACK(name, op) \ 92 | * static int name(int val, int *arg) \ 93 | * { \ 94 | * printf("%s", #op); \ 95 | * return val op *arg; \ 96 | * } 97 | * DEF_CALLBACK(multiply, *); 98 | * DEF_CALLBACK(add, +); 99 | * DEF_CALLBACK(divide, /); 100 | * DEF_CALLBACK(sub, -); 101 | * DEF_CALLBACK(or, |); 102 | * DEF_CALLBACK(and, &); 103 | * DEF_CALLBACK(xor, ^); 104 | * DEF_CALLBACK(assign, =); 105 | * 106 | * // Silly game to find the longest chain of values. 107 | * int main(int argc, char *argv[]) 108 | * { 109 | * int i, run = 1, num = argv[1] ? atoi(argv[1]) : 0; 110 | * 111 | * for (i = 1; i < 1024;) { 112 | * // Since run is an int, compiler checks "add" does too. 113 | * register_callback(i++, add, &run); 114 | * register_callback(i++, divide, &run); 115 | * register_callback(i++, sub, &run); 116 | * register_callback(i++, multiply, &run); 117 | * register_callback(i++, or, &run); 118 | * register_callback(i++, and, &run); 119 | * register_callback(i++, xor, &run); 120 | * register_callback(i++, assign, &run); 121 | * } 122 | * 123 | * printf("%i ", num); 124 | * while (run < 56) { 125 | * struct callback *cb = find_callback(num % i); 126 | * if (!cb) { 127 | * printf("-> STOP\n"); 128 | * return 1; 129 | * } 130 | * num = cb->callback(num, cb->arg); 131 | * printf("->%i ", num); 132 | * run++; 133 | * } 134 | * printf("-> Winner!\n"); 135 | * return 0; 136 | * } 137 | * 138 | * License: CC0 (Public domain) 139 | * Author: Rusty Russell 140 | */ 141 | int main(int argc, char *argv[]) 142 | { 143 | if (argc != 2) 144 | return 1; 145 | 146 | if (strcmp(argv[1], "depends") == 0) { 147 | return 0; 148 | } 149 | 150 | return 1; 151 | } 152 | -------------------------------------------------------------------------------- /ccan/typesafe_cb/test/compile_fail-cast_if_type-promotable.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | static void _set_some_value(void *val) 5 | { 6 | } 7 | 8 | #define set_some_value(expr) \ 9 | _set_some_value(typesafe_cb_cast(void *, long, (expr))) 10 | 11 | int main(int argc, char *argv[]) 12 | { 13 | #ifdef FAIL 14 | bool x = 0; 15 | #if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P 16 | #error "Unfortunately we don't fail if typesafe_cb_cast is a noop." 17 | #endif 18 | #else 19 | long x = 0; 20 | #endif 21 | set_some_value(x); 22 | return 0; 23 | } 24 | -------------------------------------------------------------------------------- /ccan/typesafe_cb/test/compile_fail-typesafe_cb-int.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | void _callback(void (*fn)(void *arg), void *arg); 5 | void _callback(void (*fn)(void *arg), void *arg) 6 | { 7 | fn(arg); 8 | } 9 | 10 | /* Callback is set up to warn if arg isn't a pointer (since it won't 11 | * pass cleanly to _callback's second arg. */ 12 | #define callback(fn, arg) \ 13 | _callback(typesafe_cb(void, (fn), (arg)), (arg)) 14 | 15 | void my_callback(int something); 16 | void my_callback(int something) 17 | { 18 | } 19 | 20 | int main(int argc, char *argv[]) 21 | { 22 | #ifdef FAIL 23 | /* This fails due to arg, not due to cast. */ 24 | callback(my_callback, 100); 25 | #endif 26 | return 0; 27 | } 28 | -------------------------------------------------------------------------------- /ccan/typesafe_cb/test/compile_fail-typesafe_cb.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | static void _register_callback(void (*cb)(void *arg), void *arg) 5 | { 6 | } 7 | 8 | #define register_callback(cb, arg) \ 9 | _register_callback(typesafe_cb(void, void *, (cb), (arg)), (arg)) 10 | 11 | static void my_callback(char *p) 12 | { 13 | } 14 | 15 | int main(int argc, char *argv[]) 16 | { 17 | char str[] = "hello world"; 18 | #ifdef FAIL 19 | int *p; 20 | #if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P 21 | #error "Unfortunately we don't fail if typesafe_cb_cast is a noop." 22 | #endif 23 | #else 24 | char *p; 25 | #endif 26 | p = NULL; 27 | 28 | /* This should work always. */ 29 | register_callback(my_callback, str); 30 | 31 | /* This will fail with FAIL defined */ 32 | register_callback(my_callback, p); 33 | return 0; 34 | } 35 | -------------------------------------------------------------------------------- /ccan/typesafe_cb/test/compile_fail-typesafe_cb_cast-multi.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | struct foo { 5 | int x; 6 | }; 7 | 8 | struct bar { 9 | int x; 10 | }; 11 | 12 | struct baz { 13 | int x; 14 | }; 15 | 16 | struct any { 17 | int x; 18 | }; 19 | 20 | struct other { 21 | int x; 22 | }; 23 | 24 | static void take_any(struct any *any) 25 | { 26 | } 27 | 28 | int main(int argc, char *argv[]) 29 | { 30 | #ifdef FAIL 31 | struct other 32 | #if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P 33 | #error "Unfortunately we don't fail if typesafe_cb_cast is a noop." 34 | #endif 35 | #else 36 | struct foo 37 | #endif 38 | *arg = NULL; 39 | take_any(typesafe_cb_cast3(struct any *, 40 | struct foo *, struct bar *, struct baz *, 41 | arg)); 42 | return 0; 43 | } 44 | -------------------------------------------------------------------------------- /ccan/typesafe_cb/test/compile_fail-typesafe_cb_cast.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void _set_some_value(void *val); 4 | 5 | void _set_some_value(void *val) 6 | { 7 | } 8 | 9 | #define set_some_value(expr) \ 10 | _set_some_value(typesafe_cb_cast(void *, unsigned long, (expr))) 11 | 12 | int main(int argc, char *argv[]) 13 | { 14 | #ifdef FAIL 15 | int x = 0; 16 | set_some_value(x); 17 | #if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P 18 | #error "Unfortunately we don't fail if typesafe_cb_cast is a noop." 19 | #endif 20 | #else 21 | void *p = 0; 22 | set_some_value(p); 23 | #endif 24 | return 0; 25 | } 26 | -------------------------------------------------------------------------------- /ccan/typesafe_cb/test/compile_fail-typesafe_cb_postargs.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | static void _register_callback(void (*cb)(void *arg, int x), void *arg) 5 | { 6 | } 7 | #define register_callback(cb, arg) \ 8 | _register_callback(typesafe_cb_postargs(void, void *, (cb), (arg), int), (arg)) 9 | 10 | static void my_callback(char *p, int x) 11 | { 12 | } 13 | 14 | int main(int argc, char *argv[]) 15 | { 16 | #ifdef FAIL 17 | int *p; 18 | #if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P 19 | #error "Unfortunately we don't fail if typesafe_cb_cast is a noop." 20 | #endif 21 | #else 22 | char *p; 23 | #endif 24 | p = NULL; 25 | register_callback(my_callback, p); 26 | return 0; 27 | } 28 | -------------------------------------------------------------------------------- /ccan/typesafe_cb/test/compile_fail-typesafe_cb_preargs.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | static void _register_callback(void (*cb)(int x, void *arg), void *arg) 5 | { 6 | } 7 | 8 | #define register_callback(cb, arg) \ 9 | _register_callback(typesafe_cb_preargs(void, void *, (cb), (arg), int), (arg)) 10 | 11 | static void my_callback(int x, char *p) 12 | { 13 | } 14 | 15 | int main(int argc, char *argv[]) 16 | { 17 | #ifdef FAIL 18 | int *p; 19 | #if !HAVE_TYPEOF||!HAVE_BUILTIN_CHOOSE_EXPR||!HAVE_BUILTIN_TYPES_COMPATIBLE_P 20 | #error "Unfortunately we don't fail if typesafe_cb_cast is a noop." 21 | #endif 22 | #else 23 | char *p; 24 | #endif 25 | p = NULL; 26 | register_callback(my_callback, p); 27 | return 0; 28 | } 29 | -------------------------------------------------------------------------------- /ccan/typesafe_cb/test/compile_ok-typesafe_cb-NULL.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | /* NULL args for callback function should be OK for normal and _def. */ 5 | 6 | static void _register_callback(void (*cb)(const void *arg), const void *arg) 7 | { 8 | } 9 | 10 | #define register_callback(cb, arg) \ 11 | _register_callback(typesafe_cb(void, const void *, (cb), (arg)), (arg)) 12 | 13 | int main(int argc, char *argv[]) 14 | { 15 | register_callback(NULL, "hello world"); 16 | return 0; 17 | } 18 | -------------------------------------------------------------------------------- /ccan/typesafe_cb/test/compile_ok-typesafe_cb-undefined.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | /* const args in callbacks should be OK. */ 5 | 6 | static void _register_callback(void (*cb)(void *arg), void *arg) 7 | { 8 | } 9 | 10 | #define register_callback(cb, arg) \ 11 | _register_callback(typesafe_cb(void, void *, (cb), (arg)), (arg)) 12 | 13 | static void _register_callback_pre(void (*cb)(int x, void *arg), void *arg) 14 | { 15 | } 16 | 17 | #define register_callback_pre(cb, arg) \ 18 | _register_callback_pre(typesafe_cb_preargs(void, void *, (cb), (arg), int), (arg)) 19 | 20 | static void _register_callback_post(void (*cb)(void *arg, int x), void *arg) 21 | { 22 | } 23 | 24 | #define register_callback_post(cb, arg) \ 25 | _register_callback_post(typesafe_cb_postargs(void, void *, (cb), (arg), int), (arg)) 26 | 27 | struct undefined; 28 | 29 | static void my_callback(struct undefined *undef) 30 | { 31 | } 32 | 33 | static void my_callback_pre(int x, struct undefined *undef) 34 | { 35 | } 36 | 37 | static void my_callback_post(struct undefined *undef, int x) 38 | { 39 | } 40 | 41 | int main(int argc, char *argv[]) 42 | { 43 | struct undefined *handle = NULL; 44 | 45 | register_callback(my_callback, handle); 46 | register_callback_pre(my_callback_pre, handle); 47 | register_callback_post(my_callback_post, handle); 48 | return 0; 49 | } 50 | -------------------------------------------------------------------------------- /ccan/typesafe_cb/test/compile_ok-typesafe_cb-vars.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | /* const args in callbacks should be OK. */ 5 | 6 | static void _register_callback(void (*cb)(void *arg), void *arg) 7 | { 8 | } 9 | 10 | #define register_callback(cb, arg) \ 11 | _register_callback(typesafe_cb(void, void *, (cb), (arg)), (arg)) 12 | 13 | static void _register_callback_pre(void (*cb)(int x, void *arg), void *arg) 14 | { 15 | } 16 | 17 | #define register_callback_pre(cb, arg) \ 18 | _register_callback_pre(typesafe_cb_preargs(void, void *, (cb), (arg), int), (arg)) 19 | 20 | static void _register_callback_post(void (*cb)(void *arg, int x), void *arg) 21 | { 22 | } 23 | 24 | #define register_callback_post(cb, arg) \ 25 | _register_callback_post(typesafe_cb_postargs(void, void *, (cb), (arg), int), (arg)) 26 | 27 | struct undefined; 28 | 29 | static void my_callback(struct undefined *undef) 30 | { 31 | } 32 | 33 | static void my_callback_pre(int x, struct undefined *undef) 34 | { 35 | } 36 | 37 | static void my_callback_post(struct undefined *undef, int x) 38 | { 39 | } 40 | 41 | int main(int argc, char *argv[]) 42 | { 43 | struct undefined *handle = NULL; 44 | void (*cb)(struct undefined *undef) = my_callback; 45 | void (*pre)(int x, struct undefined *undef) = my_callback_pre; 46 | void (*post)(struct undefined *undef, int x) = my_callback_post; 47 | 48 | register_callback(cb, handle); 49 | register_callback_pre(pre, handle); 50 | register_callback_post(post, handle); 51 | return 0; 52 | } 53 | -------------------------------------------------------------------------------- /ccan/typesafe_cb/test/compile_ok-typesafe_cb_cast.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | struct foo { 5 | int x; 6 | }; 7 | 8 | struct bar { 9 | int x; 10 | }; 11 | 12 | struct baz { 13 | int x; 14 | }; 15 | 16 | struct any { 17 | int x; 18 | }; 19 | 20 | static void take_any(struct any *any) 21 | { 22 | } 23 | 24 | int main(int argc, char *argv[]) 25 | { 26 | /* Otherwise we get unused warnings for these. */ 27 | struct foo *foo = NULL; 28 | struct bar *bar = NULL; 29 | struct baz *baz = NULL; 30 | 31 | take_any(typesafe_cb_cast3(struct any *, 32 | struct foo *, struct bar *, struct baz *, 33 | foo)); 34 | take_any(typesafe_cb_cast3(struct any *, 35 | struct foo *, struct bar *, struct baz *, 36 | bar)); 37 | take_any(typesafe_cb_cast3(struct any *, 38 | struct foo *, struct bar *, struct baz *, 39 | baz)); 40 | return 0; 41 | } 42 | -------------------------------------------------------------------------------- /ccan/typesafe_cb/test/run.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | static char dummy = 0; 7 | 8 | /* The example usage. */ 9 | static void _set_some_value(void *val) 10 | { 11 | ok1(val == &dummy); 12 | } 13 | 14 | #define set_some_value(expr) \ 15 | _set_some_value(typesafe_cb_cast(void *, unsigned long, (expr))) 16 | 17 | static void _callback_onearg(void (*fn)(void *arg), void *arg) 18 | { 19 | fn(arg); 20 | } 21 | 22 | static void _callback_preargs(void (*fn)(int a, int b, void *arg), void *arg) 23 | { 24 | fn(1, 2, arg); 25 | } 26 | 27 | static void _callback_postargs(void (*fn)(void *arg, int a, int b), void *arg) 28 | { 29 | fn(arg, 1, 2); 30 | } 31 | 32 | #define callback_onearg(cb, arg) \ 33 | _callback_onearg(typesafe_cb(void, void *, (cb), (arg)), (arg)) 34 | 35 | #define callback_preargs(cb, arg) \ 36 | _callback_preargs(typesafe_cb_preargs(void, void *, (cb), (arg), int, int), (arg)) 37 | 38 | #define callback_postargs(cb, arg) \ 39 | _callback_postargs(typesafe_cb_postargs(void, void *, (cb), (arg), int, int), (arg)) 40 | 41 | static void my_callback_onearg(char *p) 42 | { 43 | ok1(strcmp(p, "hello world") == 0); 44 | } 45 | 46 | static void my_callback_preargs(int a, int b, char *p) 47 | { 48 | ok1(a == 1); 49 | ok1(b == 2); 50 | ok1(strcmp(p, "hello world") == 0); 51 | } 52 | 53 | static void my_callback_postargs(char *p, int a, int b) 54 | { 55 | ok1(a == 1); 56 | ok1(b == 2); 57 | ok1(strcmp(p, "hello world") == 0); 58 | } 59 | 60 | /* This is simply a compile test; we promised typesafe_cb_cast can be in a 61 | * static initializer. */ 62 | struct callback_onearg 63 | { 64 | void (*fn)(void *arg); 65 | const void *arg; 66 | }; 67 | 68 | struct callback_onearg cb_onearg 69 | = { typesafe_cb(void, void *, my_callback_onearg, (char *)(intptr_t)"hello world"), 70 | "hello world" }; 71 | 72 | struct callback_preargs 73 | { 74 | void (*fn)(int a, int b, void *arg); 75 | const void *arg; 76 | }; 77 | 78 | struct callback_preargs cb_preargs 79 | = { typesafe_cb_preargs(void, void *, my_callback_preargs, 80 | (char *)(intptr_t)"hi", int, int), "hi" }; 81 | 82 | struct callback_postargs 83 | { 84 | void (*fn)(void *arg, int a, int b); 85 | const void *arg; 86 | }; 87 | 88 | struct callback_postargs cb_postargs 89 | = { typesafe_cb_postargs(void, void *, my_callback_postargs, 90 | (char *)(intptr_t)"hi", int, int), "hi" }; 91 | 92 | int main(int argc, char *argv[]) 93 | { 94 | void *p = &dummy; 95 | unsigned long l = (unsigned long)p; 96 | char str[] = "hello world"; 97 | 98 | plan_tests(2 + 1 + 3 + 3); 99 | set_some_value(p); 100 | set_some_value(l); 101 | 102 | callback_onearg(my_callback_onearg, str); 103 | 104 | callback_preargs(my_callback_preargs, str); 105 | 106 | callback_postargs(my_callback_postargs, str); 107 | 108 | return exit_status(); 109 | } 110 | -------------------------------------------------------------------------------- /charmodel-helpers.h: -------------------------------------------------------------------------------- 1 | #ifndef HAVE_CHARMODEL_HELPERS_H 2 | #define HAVE_CHARMODEL_HELPERS_H 3 | 4 | #include "recur-nn.h" 5 | #include "recur-nn-helpers.h" 6 | #include 7 | #include "path.h" 8 | #include "badmaths.h" 9 | #include "charmodel.h" 10 | 11 | static inline float 12 | capped_log2f(float x){ 13 | return (x < 1e-30f) ? -100.0f : log2f(x); 14 | } 15 | 16 | static inline float* 17 | one_hot_opinion(RecurNN *net, int hot, float presynaptic_noise){ 18 | float *inputs; 19 | int len; 20 | if (net->bottom_layer){ 21 | inputs = net->bottom_layer->inputs; 22 | len = net->bottom_layer->input_size; 23 | } 24 | else{ 25 | inputs = net->real_inputs; 26 | len = net->input_size; 27 | } 28 | 29 | //XXX could just set the previous one to zero (i.e. remember it) 30 | memset(inputs, 0, len * sizeof(float)); 31 | inputs[hot] = 1.0f; 32 | return rnn_opinion(net, NULL, presynaptic_noise); 33 | } 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /classify-gtk: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright 2014 Douglas Bagnall LGPL 3 | import argparse 4 | from gtkdisplay import run 5 | 6 | from classify import GTKClassifier, gst_init 7 | from classify import add_common_args, process_common_args 8 | 9 | def main(): 10 | gst_init() 11 | parser = argparse.ArgumentParser() 12 | prop_names = add_common_args(parser) 13 | parser.add_argument('-C', '--first-n', type=int, default=0, 14 | help="look at this many files from the audio directory") 15 | parser.add_argument('-R', '--reverse', action='store_true', 16 | help="reverse the order of classes") 17 | parser.add_argument('files', metavar='FILE', nargs='*', 18 | help='files to process before --audio-directory') 19 | parser.add_argument('--min-changes', type=int, default=0, 20 | help="only test files with at least this many class switches") 21 | parser.add_argument('-M', '--microphone', action="store_true", 22 | help="Use the microphone instead of files") 23 | parser.add_argument('--full-screen', action="store_true", 24 | help="Use the entire screen") 25 | 26 | args = parser.parse_args() 27 | 28 | if args.microphone: 29 | c = GTKClassifier(sinkname='fakesink', srcname='autoaudiosrc') 30 | process_common_args(c, args, prop_names, timed=False, load_files=False) 31 | run(c, [], args.reverse) 32 | else: 33 | c = GTKClassifier(sinkname='autoaudiosink', srcname='filesrc', 34 | filetype=args.filetype) 35 | if args.min_changes: 36 | files = process_common_args(c, args, prop_names, timed=True) 37 | files = [x for x in files 38 | if len(x.timings) >= args.min_changes] 39 | else: 40 | #by default ignore timings 41 | files = process_common_args(c, args, prop_names, timed=False) 42 | 43 | files = args.files[:] + [x.fullname for x in files] 44 | run(c, files, args.reverse, args.full_screen) 45 | 46 | main() 47 | -------------------------------------------------------------------------------- /colour.c: -------------------------------------------------------------------------------- 1 | #include "colour.h" 2 | #include "recur-common.h" 3 | 4 | const char *heat_colour_range[] = { 5 | "\033[38;5;17m", 6 | "\033[38;5;17m", 7 | "\033[38;5;18m", 8 | "\033[38;5;18m", 9 | "\033[38;5;19m", 10 | "\033[38;5;19m", 11 | "\033[38;5;57m", 12 | "\033[38;5;56m", 13 | "\033[38;5;55m", 14 | "\033[38;5;54m", 15 | "\033[38;5;53m", 16 | "\033[38;5;52m", 17 | "\033[38;5;90m", 18 | "\033[38;5;89m", 19 | "\033[38;5;88m", 20 | "\033[38;5;160m", 21 | "\033[38;5;196m", 22 | "\033[38;5;202m", 23 | "\033[38;5;208m", 24 | "\033[38;5;214m", 25 | "\033[38;5;220m", 26 | "\033[38;5;226m", 27 | "\033[38;5;190m", 28 | "\033[38;5;154m", 29 | "\033[38;5;118m", 30 | "\033[38;5;82m", 31 | "\033[38;5;46m", 32 | "\033[38;5;48m", 33 | "\033[38;5;49m", 34 | "\033[38;5;51m" 35 | }; 36 | 37 | const char *heat_colour_range_rev[] = { 38 | "\033[38;5;88m\033[48;5;17m", 39 | "\033[38;5;89m\033[48;5;17m", 40 | "\033[38;5;82m\033[48;5;18m", 41 | "\033[38;5;81m\033[48;5;18m", 42 | "\033[38;5;16m\033[48;5;19m", 43 | "\033[38;5;16m\033[48;5;19m", 44 | "\033[38;5;16m\033[48;5;57m", 45 | "\033[38;5;16m\033[48;5;56m", 46 | "\033[38;5;16m\033[48;5;55m", 47 | "\033[38;5;16m\033[48;5;54m", 48 | "\033[38;5;16m\033[48;5;53m", 49 | "\033[38;5;16m\033[48;5;52m", 50 | "\033[38;5;16m\033[48;5;90m", 51 | "\033[38;5;16m\033[48;5;89m", 52 | "\033[38;5;16m\033[48;5;88m", 53 | "\033[38;5;16m\033[48;5;160m", 54 | "\033[38;5;16m\033[48;5;196m", 55 | "\033[38;5;16m\033[48;5;202m", 56 | "\033[38;5;16m\033[48;5;208m", 57 | "\033[38;5;16m\033[48;5;214m", 58 | "\033[38;5;16m\033[48;5;220m", 59 | "\033[38;5;16m\033[48;5;226m", 60 | "\033[38;5;16m\033[48;5;190m", 61 | "\033[38;5;16m\033[48;5;154m", 62 | "\033[38;5;16m\033[48;5;118m", 63 | "\033[38;5;16m\033[48;5;82m", 64 | "\033[38;5;16m\033[48;5;46m", 65 | "\033[38;5;16m\033[48;5;48m", 66 | "\033[38;5;16m\033[48;5;49m", 67 | "\033[38;5;16m\033[48;5;51m" 68 | }; 69 | 70 | const char * colourise_float01(float f, bool rev) 71 | { 72 | const char **array; 73 | uint len; 74 | if (rev) { 75 | len = ARRAY_LEN(heat_colour_range_rev); 76 | array = heat_colour_range_rev; 77 | } else { 78 | len = ARRAY_LEN(heat_colour_range); 79 | array = heat_colour_range; 80 | } 81 | if ( f < 0) { 82 | f = 0; 83 | } else if (f >= 1) { 84 | f = 0.999f; 85 | } 86 | uint i = MIN(f * len, len - 1); 87 | return array[i]; 88 | } 89 | -------------------------------------------------------------------------------- /colour.h: -------------------------------------------------------------------------------- 1 | /*Colour strings for ANSI compatible terminals.*/ 2 | #ifndef __COLOUR_H__ 3 | #define __COLOUR_H__ 4 | 5 | #include 6 | 7 | #define C_NORMAL "\033[00m" 8 | #define BG_NORMAL "\043[00m" 9 | #define C_DARK_RED "\033[00;31m" 10 | #define C_RED "\033[01;31m" 11 | #define C_DARK_GREEN "\033[00;32m" 12 | #define C_GREEN "\033[01;32m" 13 | #define C_YELLOW "\033[01;33m" 14 | #define C_DARK_YELLOW "\033[00;33m" 15 | #define C_DARK_BLUE "\033[00;34m" 16 | #define C_BLUE "\033[01;34m" 17 | #define C_PURPLE "\033[00;35m" 18 | #define C_MAGENTA "\033[01;35m" 19 | #define C_DARK_CYAN "\033[00;36m" 20 | #define C_CYAN "\033[01;36m" 21 | #define C_GREY "\033[00;37m" 22 | #define C_WHITE "\033[01;37m" 23 | 24 | #define C_REV_RED "\033[01;41m" 25 | 26 | #define C_ITALIC "\x1B[3m" 27 | #define C_STANDARD "\x1B[23m" 28 | 29 | 30 | const char * colourise_float01(float f, bool rev); 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /colour.py: -------------------------------------------------------------------------------- 1 | _FOREGROUND = "\033[38;5;%sm" 2 | _BACKGROUND = "\033[48;5;%sm" 3 | 4 | C_NORMAL = "\033[00m" 5 | DARK_RED = "\033[00;31m" 6 | RED = "\033[01;31m" 7 | DARK_GREEN = "\033[00;32m" 8 | GREEN = "\033[01;32m" 9 | YELLOW = "\033[01;33m" 10 | DARK_YELLOW = "\033[00;33m" 11 | DARK_BLUE = "\033[00;34m" 12 | BLUE = "\033[01;34m" 13 | PURPLE = "\033[00;35m" 14 | MAGENTA = "\033[01;35m" 15 | DARK_CYAN = "\033[00;36m" 16 | CYAN = "\033[01;36m" 17 | GREY = "\033[00;37m" 18 | WHITE = "\033[01;37m" 19 | 20 | REV_RED = "\033[01;41m" 21 | 22 | def combo(foreground, background): 23 | return _BACKGROUND % background + _FOREGROUND % foreground 24 | 25 | COLOURS = { 26 | "Z": C_NORMAL, 27 | "g": GREEN, 28 | "G": DARK_GREEN, 29 | "r": RED, 30 | "R": DARK_RED, 31 | "M": MAGENTA, 32 | "P": PURPLE, 33 | "C": CYAN, 34 | "Y": YELLOW, 35 | "W": WHITE, 36 | } 37 | 38 | _spectrum = (range(160, 196, 6) + 39 | range(226, 190, -6) + 40 | range(124, 128, 1) + 41 | range(128, 164, 6) + 42 | range(122, 90, -6) + 43 | range(91, 88, -1) + 44 | range(161, 166, 1) + 45 | range(201, 196, -1) + 46 | range(201, 196, -1) + 47 | range(130, 160, 6) + 48 | range(118, 88, -6)) 49 | 50 | SPECTRUM = [_FOREGROUND % x for x in _spectrum] 51 | BACKGROUND_SPECTRUM = [_BACKGROUND % x for x in _spectrum] 52 | 53 | SCALE_30 = [_BACKGROUND % '16' + _FOREGROUND % x 54 | for x in 55 | (17, 17, 18, 18, 19, 19, 56 | 57, 56, 55, 54, 53, 52, 57 | 90, 89, 88, 160, 196, 202, 58 | 208, 214, 220, 226, 190, 154, 59 | 118, 82, 46, 48, 49, 51)] 60 | 61 | SCALE_12 = [COLOURS[x] for x in 'PPrRYYGGgCCW'] 62 | SCALE_11 = SCALE_12[:-1] 63 | 64 | 65 | def colouriser(colour_scale): 66 | c_scale = len(colour_scale) * 0.9999 67 | c_max = int(c_scale) 68 | def colourise(val): 69 | i = min(int(val * c_scale), c_max) 70 | return colour_scale[max(i, 0)] 71 | return colourise 72 | -------------------------------------------------------------------------------- /context-helpers.h: -------------------------------------------------------------------------------- 1 | #ifndef HAVE_CONTEXT_HELPERS_H 2 | #define HAVE_CONTEXT_HELPERS_H 3 | #include "recur-context.h" 4 | #include "badmaths.h" 5 | #include 6 | 7 | static inline void 8 | dump_frame(GstVideoFrame *f){ 9 | GstVideoInfo *vi = &f->info; 10 | GST_DEBUG("Dumping frame %p", f); 11 | GST_DEBUG("flags %d buffer %p meta %p id %d", 12 | f->flags, f->buffer, f->meta, f->id); 13 | GST_DEBUG("data %p %p %p", 14 | f->data[0], f->data[1], f->data[2]); 15 | 16 | GST_DEBUG("width %d height %d size %zu", 17 | vi->width, vi->height, vi->size); 18 | GST_DEBUG("width %d height %d size %zu", 19 | vi->width, vi->height, vi->size); 20 | } 21 | 22 | static inline void blank_frame(RecurContext *context, GstVideoFrame *dest){ 23 | for (uint i = 0; i < GST_VIDEO_INFO_N_PLANES(&context->video_info); i++){ 24 | GST_DEBUG("i is %d", i); 25 | u8 *data = GST_VIDEO_FRAME_PLANE_DATA(dest, i); 26 | uint size = GST_VIDEO_FRAME_COMP_STRIDE(dest, i) * \ 27 | GST_VIDEO_FRAME_COMP_HEIGHT(dest, i); 28 | GST_DEBUG("plane data %p size %u", data, size); 29 | memset(data, i == 0 ? 60 : 127, size); 30 | } 31 | } 32 | 33 | #define OUTPUT_SCALE 4 34 | 35 | 36 | static inline void 37 | blit_thumbnail(RecurContext *context, GstVideoFrame *dest, int x_pos, int y_pos){ 38 | /* write to dest frame */ 39 | blank_frame(context, dest); 40 | RecurFrame *f = &context->frame_queue[context->fq_head]; 41 | int scale[3] = {2, 1, 1}; 42 | u8 *s = (u8*)f->Y; 43 | for (int i = 0; i < 3; i++){ 44 | int stride = GST_VIDEO_FRAME_COMP_STRIDE(dest, i); 45 | u8 *plane = GST_VIDEO_FRAME_COMP_DATA(dest, i); 46 | u8 *d = plane + (y_pos * stride + x_pos) * scale[i]; 47 | if (scale[i] == 1){ 48 | for (int y = 0; y < RECUR_WORKING_HEIGHT; y++){ 49 | memcpy(d, s, RECUR_WORKING_WIDTH); 50 | d += stride; 51 | s += RECUR_WORKING_WIDTH; 52 | } 53 | } 54 | else if (scale[i] == 2) { 55 | for (int y = 0; y < RECUR_WORKING_HEIGHT; y++){ 56 | for (int x = 0; x < RECUR_WORKING_WIDTH; x++){ 57 | d[2 * x] = d[2 * x + 1] = s[x]; 58 | } 59 | memcpy(d + stride, d, RECUR_WORKING_WIDTH * 2); 60 | d += stride * 2; 61 | s += RECUR_WORKING_WIDTH; 62 | } 63 | } 64 | } 65 | GST_DEBUG(" added thumbnails"); 66 | } 67 | #endif 68 | -------------------------------------------------------------------------------- /convert-saved-net.c: -------------------------------------------------------------------------------- 1 | #include "recur-nn.h" 2 | 3 | int 4 | main(int argc, char *argv[]) 5 | { 6 | RecurNN *net = rnn_load_net(argv[1]); 7 | net->flags &= ~RNN_NET_FLAG_OWN_BPTT; 8 | rnn_save_net(net, argv[2], 1); 9 | } 10 | -------------------------------------------------------------------------------- /gstclassify.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Douglas Bagnall LGPL */ 2 | #ifndef __GST_AUDIO_CLASSIFY_H__ 3 | #define __GST_AUDIO_CLASSIFY_H__ 4 | 5 | #include 6 | #include 7 | #include "recur-common.h" 8 | #include "recur-nn.h" 9 | #include "mfcc.h" 10 | #include "badmaths.h" 11 | #include "pgm_dump.h" 12 | 13 | G_BEGIN_DECLS 14 | 15 | #define CLASSIFY_MAX_CHANNELS 1000 16 | #define CLASSIFY_MIN_CHANNELS 1 17 | #define CLASSIFY_RATE 8000 18 | #define CLASSIFY_VALUE_SIZE 2 19 | 20 | #define CLASSIFY_N_FFT_BINS 32 21 | 22 | #define CLASSIFY_RNN_FLAGS (RNN_NET_FLAG_STANDARD | \ 23 | RNN_NET_FLAG_BPTT_ADAPTIVE_MIN_ERROR) 24 | 25 | #define CLASSIFY_FORMAT "S16LE" 26 | /*sizeof(S16LE)*/ 27 | typedef s16 audio_sample; 28 | #define RECUR_AUDIO_BITS (8 * sizeof(audio_sample)) 29 | 30 | #define PERIODIC_SAVE_NET 0 31 | #define TRY_RELOAD 1 32 | 33 | #define PERIODIC_PGM_DUMP 2047 34 | #define PERIODIC_PGM_DUMPEES "how ihw biw" 35 | #define PGM_DUMP_FEATURES 0 36 | 37 | #define CLASSIFY_QUEUE_FACTOR 30 38 | 39 | #define GST_TYPE_CLASSIFY (gst_classify_get_type()) 40 | #define GST_CLASSIFY(obj) \ 41 | (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_CLASSIFY,GstClassify)) 42 | #define GST_CLASSIFY_CLASS(klass) \ 43 | (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_CLASSIFY,GstClassifyClass)) 44 | #define GST_IS_CLASSIFY(obj) \ 45 | (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_CLASSIFY)) 46 | #define GST_IS_CLASSIFY_CLASS(klass) \ 47 | (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_CLASSIFY)) 48 | 49 | 50 | typedef struct _GstClassify GstClassify; 51 | typedef struct _GstClassifyClass GstClassifyClass; 52 | 53 | /*Do not reorder metadata, or insert things in the middle, even though the 54 | packing is loose and the order haphazard. Only add things at the end, and 55 | keep the same order in the metadata strings. 56 | */ 57 | struct ClassifyMetadata { 58 | const char *classes; 59 | float min_freq; 60 | float max_freq; 61 | float knee_freq; 62 | int mfccs; 63 | int window_size; 64 | const char *basename; 65 | int delta_features; 66 | float focus_freq; 67 | float lag; 68 | int intensity_feature; 69 | float confirmation_lag; 70 | const char *features_offset; 71 | const char *features_scale; 72 | }; 73 | 74 | typedef struct _ClassifyClassEvent { 75 | int channel; 76 | int class_group; 77 | int window_no; 78 | int target; 79 | } ClassifyClassEvent; 80 | 81 | typedef struct _ClassifyClassGroup { 82 | int offset; 83 | int n_classes; 84 | char *classes; 85 | } ClassifyClassGroup; 86 | 87 | 88 | typedef struct _ClassifyChannel 89 | { 90 | RecurNN *net; 91 | float *features; 92 | float *prev_features; 93 | float *pcm_now; 94 | float *pcm_next; 95 | int *group_target; 96 | int *group_winner; 97 | TemporalPPM *mfcc_image; 98 | } ClassifyChannel; 99 | 100 | struct _GstClassify 101 | { 102 | GstAudioFilter audiofilter; 103 | GstAudioInfo *info; 104 | RecurNN *net; 105 | RecurNN **subnets; 106 | ClassifyChannel *channels; 107 | int n_channels; 108 | int n_groups; 109 | ClassifyClassGroup *class_groups; 110 | s16 *audio_queue; 111 | int read_offset; 112 | int write_offset; 113 | RecurAudioBinner *mfcc_factory; 114 | const char *net_filename; 115 | const char *basename; 116 | int queue_size; 117 | int mfccs; 118 | ClassifyClassEvent *class_events; 119 | int n_class_events; 120 | int class_events_index; 121 | float momentum_soft_start; 122 | int window_size; 123 | int window_no; 124 | int ignored_windows; 125 | int training; 126 | int random_alignment; 127 | int learning_style; 128 | float *error_weight; 129 | GValue *pending_properties; 130 | TemporalPPM *error_image; 131 | int delta_features; 132 | int intensity_feature; 133 | float lag; 134 | float confirmation_lag; 135 | float weight_noise; 136 | float *feature_offsets; 137 | int n_feature_offsets; 138 | float *feature_scales; 139 | int n_feature_scales; 140 | FILE *features_file; 141 | u32 *seen_training_counts; 142 | u32 *used_training_counts; 143 | float balanced_training_bias; 144 | }; 145 | 146 | struct _GstClassifyClass 147 | { 148 | GstAudioFilterClass parent_class; 149 | }; 150 | 151 | GType gst_classify_get_type(void); 152 | 153 | 154 | G_END_DECLS 155 | #endif /* __GST_AUDIO_CLASSIFY_H__ */ 156 | -------------------------------------------------------------------------------- /gstparrot.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Douglas Bagnall LGPL */ 2 | #ifndef __GST_AUDIO_PARROT_H__ 3 | #define __GST_AUDIO_PARROT_H__ 4 | 5 | #include 6 | #include 7 | #include "recur-common.h" 8 | #include "recur-nn.h" 9 | #include "mdct.h" 10 | #include "mfcc.h" 11 | #include "badmaths.h" 12 | #include "pgm_dump.h" 13 | 14 | G_BEGIN_DECLS 15 | 16 | #define PARROT_RNG_SEED 11 17 | #define PARROT_BPTT_DEPTH 30 18 | #define LEARN_RATE 0.0003 19 | #define MOMENTUM 0.95 20 | 21 | #define PARROT_MAX_CHANNELS 2000 22 | #define PARROT_MIN_CHANNELS 1 23 | #define PARROT_RATE 16000 24 | #define PARROT_USE_MFCCS 0 25 | 26 | #define PARROT_N_FFT_BINS 40 27 | #define PARROT_WINDOW_SIZE 512 28 | 29 | #if PARROT_USE_MFCCS 30 | #define PARROT_N_FEATURES 20 31 | #else 32 | #define PARROT_N_FEATURES (PARROT_WINDOW_SIZE / 2) 33 | #endif 34 | 35 | #define PARROT_MFCC_MIN_FREQ 20 36 | #define PARROT_MFCC_MAX_FREQ (PARROT_RATE * 0.499) 37 | #define PARROT_MFCC_KNEE_FREQ 700 38 | #define PARROT_MFCC_FOCUS_FREQ 0 39 | #define PARROT_PRESYNAPTIC_NOISE 0 40 | 41 | #define PARROT_EXTRA_FLAGS (RNN_NET_FLAG_BPTT_ADAPTIVE_MIN_ERROR ) 42 | 43 | #define PARROT_RNN_FLAGS (RNN_NET_FLAG_STANDARD | PARROT_EXTRA_FLAGS) 44 | 45 | #define PARROT_FORMAT "S16LE" 46 | #define RECUR_AUDIO_BITS (16) 47 | 48 | #define PERIODIC_SAVE_NET 1 49 | #define TRY_RELOAD 1 50 | 51 | #define PERIODIC_PGM_DUMP 0 52 | #define PGM_DUMP_LEARN 0 53 | #define PGM_DUMP_OUT 1 54 | 55 | #define PGM_DUMP_FEATURES (PGM_DUMP_LEARN | PGM_DUMP_OUT) 56 | 57 | #define PARROT_VALUE_SIZE 2 58 | 59 | #define GST_TYPE_PARROT (gst_parrot_get_type()) 60 | #define GST_PARROT(obj) \ 61 | (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_PARROT,GstParrot)) 62 | #define GST_PARROT_CLASS(klass) \ 63 | (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_PARROT,GstParrotClass)) 64 | #define GST_IS_PARROT(obj) \ 65 | (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_PARROT)) 66 | #define GST_IS_PARROT_CLASS(klass) \ 67 | (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_PARROT)) 68 | 69 | /*how many chunks in the incoming buffer */ 70 | #define PARROT_QUEUE_N_CHUNKS 30 71 | 72 | typedef struct _GstParrot GstParrot; 73 | typedef struct _GstParrotClass GstParrotClass; 74 | 75 | typedef struct _ParrotChannel 76 | { 77 | RecurNN *train_net; 78 | RecurNN *dream_net; 79 | float *pcm_now; 80 | float *pcm_prev; 81 | float *play_now; 82 | float *play_prev; 83 | float *mdct_now; 84 | float *mdct_prev; 85 | TemporalPPM *mfcc_image; 86 | TemporalPPM *pcm_image; 87 | TemporalPPM *pcm_image2; 88 | TemporalPPM *dct_image; 89 | TemporalPPM *answer_image; 90 | } ParrotChannel; 91 | 92 | struct _GstParrot 93 | { 94 | GstAudioFilter audiofilter; 95 | RecurNN *net; 96 | RecurNN **training_nets; 97 | ParrotChannel *channels; 98 | int n_channels; 99 | int queue_size; 100 | s16 *incoming_queue; 101 | int incoming_start; 102 | int incoming_end; 103 | s16 *outgoing_queue; 104 | int outgoing_start; 105 | int outgoing_end; 106 | mdct_lookup mdct_lut; 107 | RecurAudioBinner *mfcc_factory; 108 | float *window; 109 | 110 | int training; 111 | int playing; 112 | 113 | char *net_filename; 114 | char *pending_logfile; 115 | int hidden_size; 116 | float learn_rate; 117 | }; 118 | 119 | 120 | struct _GstParrotClass 121 | { 122 | GstAudioFilterClass parent_class; 123 | }; 124 | 125 | GType gst_parrot_get_type(void); 126 | 127 | 128 | G_END_DECLS 129 | #endif /* __GST_AUDIO_PARROT_H__ */ 130 | -------------------------------------------------------------------------------- /gstrecur_audio.c: -------------------------------------------------------------------------------- 1 | /* GStreamer 2 | * Copyright (C) <2013> Douglas Bagnall 3 | * 4 | */ 5 | 6 | #include "gstrecur_audio.h" 7 | #include 8 | #include 9 | 10 | GST_DEBUG_CATEGORY_STATIC (recur_audio_debug); 11 | #define GST_CAT_DEFAULT recur_audio_debug 12 | 13 | /* GstRecurAudio signals and args */ 14 | enum 15 | { 16 | /* FILL ME */ 17 | LAST_SIGNAL 18 | }; 19 | 20 | enum 21 | { 22 | PROP_0, 23 | }; 24 | 25 | /* static_functions */ 26 | static void gst_recur_audio_class_init(GstRecurAudioClass *g_class); 27 | static void gst_recur_audio_init(GstRecurAudio *self); 28 | static void gst_recur_audio_set_property(GObject *object, guint prop_id, const GValue *value, GParamSpec *pspec); 29 | static void gst_recur_audio_get_property(GObject *object, guint prop_id, GValue *value, GParamSpec *pspec); 30 | static GstFlowReturn gst_recur_audio_transform(GstBaseTransform *base, GstBuffer *inbuf, GstBuffer *outbuf); 31 | static gboolean gst_recur_audio_setup(GstAudioFilter * filter, const GstAudioInfo * info); 32 | 33 | #define gst_recur_audio_parent_class parent_class 34 | G_DEFINE_TYPE (GstRecurAudio, gst_recur_audio, GST_TYPE_AUDIO_FILTER) 35 | 36 | /* Clean up */ 37 | static void 38 | gst_recur_audio_finalize (GObject * obj){ 39 | GST_DEBUG("in gst_recur_audio_finalize!\n"); 40 | //GstRecurAudio *self = GST_RECUR_AUDIO(obj); 41 | } 42 | 43 | static void 44 | gst_recur_audio_class_init (GstRecurAudioClass * klass) 45 | { 46 | GST_DEBUG_CATEGORY_INIT (recur_audio_debug, "recur_audio", RECUR_LOG_COLOUR, 47 | "recur audio"); 48 | GObjectClass *gobject_class = G_OBJECT_CLASS (klass); 49 | GstElementClass *element_class = GST_ELEMENT_CLASS (klass); 50 | GstBaseTransformClass *trans_class = GST_BASE_TRANSFORM_CLASS (klass); 51 | GstAudioFilterClass *af_class = GST_AUDIO_FILTER_CLASS (klass); 52 | gobject_class->set_property = gst_recur_audio_set_property; 53 | gobject_class->get_property = gst_recur_audio_get_property; 54 | gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_recur_audio_finalize); 55 | /*16kHz, single channel, 16 bit signed little endian PCM*/ 56 | GstCaps *caps = gst_caps_new_simple ("audio/x-raw", 57 | "format", G_TYPE_STRING, RECUR_AUDIO_FORMAT, 58 | "rate", G_TYPE_INT, RECUR_AUDIO_RATE, 59 | "channels", G_TYPE_INT, RECUR_AUDIO_CHANNELS, 60 | NULL); 61 | 62 | gst_audio_filter_class_add_pad_templates (af_class, caps); 63 | 64 | gst_element_class_set_static_metadata (element_class, 65 | "Recur audio sub-element", 66 | "Filter/Audio", 67 | "Mangles audio", 68 | "Douglas Bagnall "); 69 | 70 | trans_class->transform = GST_DEBUG_FUNCPTR (gst_recur_audio_transform); 71 | af_class->setup = GST_DEBUG_FUNCPTR (gst_recur_audio_setup); 72 | GST_INFO("gst audio class init\n"); 73 | } 74 | 75 | static void 76 | gst_recur_audio_init (GstRecurAudio * self) 77 | { 78 | GST_INFO("gst recur_audio init\n"); 79 | } 80 | 81 | static gboolean 82 | gst_recur_audio_setup(GstAudioFilter * base, const GstAudioInfo * info){ 83 | GST_INFO("gst audio setup\n"); 84 | GstRecurAudio *self = GST_RECUR_AUDIO(base); 85 | GST_DEBUG_OBJECT (self, 86 | "info: %" GST_PTR_FORMAT, info); 87 | return TRUE; 88 | } 89 | 90 | static inline void 91 | set_string_prop(const GValue *value, const char **target){ 92 | const char *s = g_value_dup_string(value); 93 | size_t len = strlen(s); 94 | if(len){ 95 | *target = s; 96 | } 97 | } 98 | 99 | static void 100 | gst_recur_audio_set_property (GObject * object, guint prop_id, const GValue * value, 101 | GParamSpec * pspec) 102 | { 103 | //GstRecurAudio *self = GST_RECUR_AUDIO (object); 104 | GST_DEBUG("gst_recur_audio_set_property\n"); 105 | if (value){ 106 | switch (prop_id) { 107 | default: 108 | G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); 109 | break; 110 | } 111 | } 112 | } 113 | 114 | static void 115 | gst_recur_audio_get_property (GObject * object, guint prop_id, GValue * value, 116 | GParamSpec * pspec) 117 | { 118 | //GstRecurAudio *self = GST_RECUR_AUDIO (object); 119 | 120 | switch (prop_id) { 121 | default: 122 | G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); 123 | break; 124 | } 125 | } 126 | 127 | 128 | static GstFlowReturn 129 | gst_recur_audio_transform (GstBaseTransform * base, GstBuffer *inbuf, GstBuffer *outbuf) 130 | { 131 | GstRecurAudio *self = GST_RECUR_AUDIO(base); 132 | GstFlowReturn ret = GST_FLOW_OK; 133 | recur_queue_audio_segment(self->context, inbuf); 134 | recur_fill_audio_segment(self->context, outbuf); 135 | GST_LOG("recur_audio_transform returning OK"); 136 | //exit: 137 | return ret; 138 | } 139 | 140 | void 141 | gst_recur_audio_register_context (GstRecurAudio * self, RecurContext *context) 142 | { 143 | self->context = context; 144 | GST_INFO("audio_register_context\n"); 145 | } 146 | -------------------------------------------------------------------------------- /gstrecur_audio.h: -------------------------------------------------------------------------------- 1 | #ifndef __GST_AUDIO_RECUR_AUDIO_H__ 2 | #define __GST_AUDIO_RECUR_AUDIO_H__ 3 | 4 | #include 5 | #include 6 | #include "recur-context.h" 7 | #include "recur-common.h" 8 | 9 | G_BEGIN_DECLS 10 | #define GST_TYPE_RECUR_AUDIO (gst_recur_audio_get_type()) 11 | #define GST_RECUR_AUDIO(obj) \ 12 | (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_RECUR_AUDIO,GstRecurAudio)) 13 | #define GST_RECUR_AUDIO_CLASS(klass) \ 14 | (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_RECUR_AUDIO,GstRecurAudioClass)) 15 | #define GST_IS_RECUR_AUDIO(obj) \ 16 | (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_RECUR_AUDIO)) 17 | #define GST_IS_RECUR_AUDIO_CLASS(klass) \ 18 | (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_RECUR_AUDIO)) 19 | 20 | 21 | typedef struct _GstRecurAudio GstRecurAudio; 22 | typedef struct _GstRecurAudioClass GstRecurAudioClass; 23 | 24 | struct _GstRecurAudio 25 | { 26 | GstAudioFilter audiofilter; 27 | RecurContext *context; 28 | }; 29 | 30 | 31 | struct _GstRecurAudioClass 32 | { 33 | GstAudioFilterClass parent_class; 34 | }; 35 | 36 | GType gst_recur_audio_get_type(void); 37 | 38 | void gst_recur_audio_register_context (GstRecurAudio * self, RecurContext *context); 39 | 40 | 41 | G_END_DECLS 42 | #endif /* __GST_AUDIO_RECUR_AUDIO_H__ */ 43 | -------------------------------------------------------------------------------- /gstrecur_manager.h: -------------------------------------------------------------------------------- 1 | #ifndef __GST_MANAGER_RECUR_MANAGER_H__ 2 | #define __GST_MANAGER_RECUR_MANAGER_H__ 3 | 4 | #include 5 | #include "recur-context.h" 6 | 7 | G_BEGIN_DECLS 8 | #define GST_TYPE_RECUR_MANAGER \ 9 | (gst_recur_manager_get_type()) 10 | #define GST_RECUR_MANAGER(obj) \ 11 | (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_RECUR_MANAGER,GstRecurManager)) 12 | #define GST_RECUR_MANAGER_CLASS(klass) \ 13 | (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_RECUR_MANAGER,GstRecurManagerClass)) 14 | #define GST_IS_RECUR_MANAGER(obj) \ 15 | (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_RECUR_MANAGER)) 16 | #define GST_IS_RECUR_MANAGER_CLASS(klass) \ 17 | (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_RECUR_MANAGER)) 18 | 19 | // 20 | 21 | typedef struct _GstRecurManager GstRecurManager; 22 | typedef struct _GstRecurManagerClass GstRecurManagerClass; 23 | 24 | struct _GstRecurManager 25 | { 26 | GstBin parent; 27 | RecurContext *context; 28 | }; 29 | 30 | 31 | struct _GstRecurManagerClass 32 | { 33 | GstBinClass parent_class; 34 | }; 35 | 36 | GType gst_recur_manager_get_type(void); 37 | 38 | G_END_DECLS 39 | #endif /* __GST_MANAGER_RECUR_MANAGER_H__ */ 40 | -------------------------------------------------------------------------------- /gstrecur_video.c: -------------------------------------------------------------------------------- 1 | /* GStreamer 2 | * Copyright (C) <2013> Douglas Bagnall 3 | * 4 | */ 5 | 6 | #include "gstrecur_video.h" 7 | #include "recur-common.h" 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | 14 | GST_DEBUG_CATEGORY_STATIC (recur_video_debug); 15 | #define GST_CAT_DEFAULT recur_video_debug 16 | 17 | /* GstRecurVideo signals and args */ 18 | enum 19 | { 20 | /* FILL ME */ 21 | LAST_SIGNAL 22 | }; 23 | 24 | enum 25 | { 26 | PROP_0, 27 | }; 28 | 29 | 30 | /* static_functions */ 31 | static void gst_recur_video_class_init(GstRecurVideoClass *g_class); 32 | static void gst_recur_video_init(GstRecurVideo *self); 33 | static void gst_recur_video_set_property(GObject *object, guint prop_id, const GValue *value, GParamSpec *pspec); 34 | static void gst_recur_video_get_property(GObject *object, guint prop_id, GValue *value, GParamSpec *pspec); 35 | static GstFlowReturn gst_recur_video_transform_frame(GstVideoFilter *base, GstVideoFrame *inbuf, GstVideoFrame *outbuf); 36 | 37 | static gboolean set_info (GstVideoFilter *filter, 38 | GstCaps *incaps, GstVideoInfo *in_info, 39 | GstCaps *outcaps, GstVideoInfo *out_info); 40 | 41 | #define VIDEO_FORMATS " { I420 } " 42 | 43 | static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", 44 | GST_PAD_SRC, 45 | GST_PAD_ALWAYS, 46 | GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (VIDEO_FORMATS)) 47 | ); 48 | 49 | static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", 50 | GST_PAD_SINK, 51 | GST_PAD_ALWAYS, 52 | GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (VIDEO_FORMATS)) 53 | ); 54 | 55 | #define gst_recur_video_parent_class parent_class 56 | G_DEFINE_TYPE (GstRecurVideo, gst_recur_video, GST_TYPE_VIDEO_FILTER) 57 | 58 | /* Clean up */ 59 | static void 60 | gst_recur_video_finalize (GObject * obj){ 61 | GST_DEBUG("in gst_recur_video_finalize!\n"); 62 | //GstRecurVideo *self = GST_RECUR_VIDEO(obj); 63 | } 64 | 65 | static void 66 | gst_recur_video_class_init (GstRecurVideoClass * g_class) 67 | { 68 | //GstBaseTransformClass *trans_class = GST_BASE_TRANSFORM_CLASS (g_class); 69 | GstElementClass *gstelement_class = (GstElementClass *) g_class; 70 | 71 | GST_DEBUG_CATEGORY_INIT (recur_video_debug, "recur_video", RECUR_LOG_COLOUR, 72 | "recur video"); 73 | 74 | GObjectClass *gobject_class = G_OBJECT_CLASS (g_class); 75 | GstVideoFilterClass *vf_class = GST_VIDEO_FILTER_CLASS (g_class); 76 | 77 | gobject_class->set_property = gst_recur_video_set_property; 78 | gobject_class->get_property = gst_recur_video_get_property; 79 | gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_recur_video_finalize); 80 | 81 | gst_element_class_add_pad_template (gstelement_class, 82 | gst_static_pad_template_get (&sink_factory)); 83 | gst_element_class_add_pad_template (gstelement_class, 84 | gst_static_pad_template_get (&src_factory)); 85 | 86 | gst_element_class_set_static_metadata (gstelement_class, 87 | "Recur video sub-element", 88 | "Filter/Video", 89 | "Mangles video", 90 | "Douglas Bagnall "); 91 | 92 | vf_class->transform_frame = GST_DEBUG_FUNCPTR (gst_recur_video_transform_frame); 93 | vf_class->set_info = GST_DEBUG_FUNCPTR (set_info); 94 | GST_INFO("gst class init\n"); 95 | } 96 | 97 | static void 98 | gst_recur_video_init (GstRecurVideo * self) 99 | { 100 | //gst_element_create_all_pads(GST_ELEMENT(self)); 101 | GST_INFO("gst recur_video init\n"); 102 | } 103 | 104 | 105 | static gboolean 106 | set_info (GstVideoFilter *filter, 107 | GstCaps *incaps, GstVideoInfo *in_info, 108 | GstCaps *outcaps, GstVideoInfo *out_info) 109 | { 110 | GstRecurVideo *self = GST_RECUR_VIDEO (filter); 111 | recur_context_set_video_properties(self->context, in_info); 112 | return TRUE; 113 | } 114 | 115 | 116 | static inline void 117 | set_string_prop(const GValue *value, const char **target){ 118 | const char *s = g_value_dup_string(value); 119 | size_t len = strlen(s); 120 | if(len){ 121 | *target = s; 122 | } 123 | } 124 | 125 | static void 126 | gst_recur_video_set_property (GObject * object, guint prop_id, const GValue * value, 127 | GParamSpec * pspec) 128 | { 129 | //GstRecurVideo *self = GST_RECUR_VIDEO (object); 130 | GST_DEBUG("gst_recur_video_set_property\n"); 131 | if (value){ 132 | switch (prop_id) { 133 | default: 134 | G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); 135 | break; 136 | } 137 | } 138 | } 139 | 140 | static void 141 | gst_recur_video_get_property (GObject * object, guint prop_id, GValue * value, 142 | GParamSpec * pspec) 143 | { 144 | //GstRecurVideo *self = GST_RECUR_VIDEO (object); 145 | 146 | switch (prop_id) { 147 | default: 148 | G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); 149 | break; 150 | } 151 | } 152 | 153 | 154 | static GstFlowReturn 155 | gst_recur_video_transform_frame (GstVideoFilter *filter, 156 | GstVideoFrame *inframe, GstVideoFrame *outframe) 157 | { 158 | GstRecurVideo *self = GST_RECUR_VIDEO(filter); 159 | recur_queue_video_buffer(self->context, inframe->buffer); 160 | recur_fill_video_frame(self->context, outframe); 161 | GST_LOG("recur_video_transform returning OK"); 162 | return GST_FLOW_OK; 163 | } 164 | 165 | void 166 | gst_recur_video_register_context (GstRecurVideo * self, RecurContext *context) 167 | { 168 | self->context = context; 169 | GST_INFO("video_register_context\n"); 170 | } 171 | -------------------------------------------------------------------------------- /gstrecur_video.h: -------------------------------------------------------------------------------- 1 | #ifndef __GST_VIDEO_RECUR_VIDEO_H__ 2 | #define __GST_VIDEO_RECUR_VIDEO_H__ 3 | 4 | #include 5 | #include "recur-context.h" 6 | 7 | G_BEGIN_DECLS 8 | #define GST_TYPE_RECUR_VIDEO \ 9 | (gst_recur_video_get_type()) 10 | #define GST_RECUR_VIDEO(obj) \ 11 | (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_RECUR_VIDEO,GstRecurVideo)) 12 | #define GST_RECUR_VIDEO_CLASS(klass) \ 13 | (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_RECUR_VIDEO,GstRecurVideoClass)) 14 | #define GST_IS_RECUR_VIDEO(obj) \ 15 | (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_RECUR_VIDEO)) 16 | #define GST_IS_RECUR_VIDEO_CLASS(klass) \ 17 | (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_RECUR_VIDEO)) 18 | 19 | 20 | typedef struct _GstRecurVideo GstRecurVideo; 21 | typedef struct _GstRecurVideoClass GstRecurVideoClass; 22 | 23 | struct _GstRecurVideo 24 | { 25 | GstVideoFilter videofilter; 26 | RecurContext *context; 27 | }; 28 | 29 | struct _GstRecurVideoClass 30 | { 31 | GstVideoFilterClass parent_class; 32 | }; 33 | 34 | GType gst_recur_video_get_type(void); 35 | 36 | void gst_recur_video_register_context (GstRecurVideo * self, RecurContext *context); 37 | 38 | G_END_DECLS 39 | #endif /* __GST_VIDEO_RECUR_VIDEO_H__ */ 40 | -------------------------------------------------------------------------------- /gstrnnca.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Douglas Bagnall LGPL */ 2 | #ifndef __GOT_RNNCA_H__ 3 | #define __GOT_RNNCA_H__ 4 | 5 | #include 6 | #include "recur-common.h" 7 | #include "recur-nn.h" 8 | #include "badmaths.h" 9 | #include "pgm_dump.h" 10 | 11 | G_BEGIN_DECLS 12 | 13 | #define TRY_RELOAD 1 14 | #define RNNCA_WIDTH 144 15 | #define RNNCA_HEIGHT 96 16 | 17 | #define RNNCA_HISTORY_SAMPLES 100 18 | #define RNNCA_HISTORY_RATE 0.1 19 | #define RNNCA_HISTORY_SEEMS_STUCK (200 * RNNCA_HISTORY_RATE) 20 | 21 | #define RNNCA_RNG_SEED 11 22 | #define RNNCA_BPTT_DEPTH 10 23 | 24 | #define RNNCA_DO_TEMPORAL_LOGGING 0 25 | #define RNNCA_PRESYNAPTIC_NOISE 0 26 | 27 | #define LONG_WALK 0 28 | 29 | #if LONG_WALK 30 | #define RNNCA_EXTRA_FLAGS ( RNN_COND_USE_RAND | RNN_COND_USE_SCALE \ 31 | | RNN_COND_USE_TALL_POPPY | RNN_NET_FLAG_LOG_WEIGHT_SUM \ 32 | ) 33 | #define RNNCA_N_TRAINERS 20 34 | #else 35 | #define RNNCA_EXTRA_FLAGS ( RNN_COND_USE_SCALE | RNN_NET_FLAG_LOG_WEIGHT_SUM ) 36 | #define RNNCA_N_TRAINERS 200 37 | #endif 38 | 39 | #define RNNCA_RNN_FLAGS (RNN_NET_FLAG_STANDARD | RNNCA_EXTRA_FLAGS) 40 | 41 | #define PERIODIC_PGM_DUMP 0 42 | #define SPECIFIC_PGM_DUMP 0 43 | #define PERIODIC_SAVE_NET 511 44 | 45 | #define PERIODIC_CHECK_STASIS 1 46 | #define PERIODIC_SHUFFLE_TRAINERS 7 47 | #define PGM_DUMP_CHANGED_MASK 0 48 | 49 | #define RNNCA_DEFAULT_PATTERN "Y00120111C0111" 50 | 51 | static const int RNNCA_POSITIONAL_LEN = 2; 52 | 53 | typedef struct _RnncaFrame { 54 | u8 *Y; 55 | u8 *Cb; 56 | u8 *Cr; 57 | } RnncaFrame; 58 | 59 | 60 | typedef struct _RnncaTrainer { 61 | RecurNN *net; 62 | int x; 63 | int y; 64 | } RnncaTrainer; 65 | 66 | typedef struct _RnncaPixelHistory { 67 | int offset; 68 | int hits; 69 | int colour; 70 | } RnncaPixelHistory; 71 | 72 | typedef struct _GstRnnca GstRnnca; 73 | typedef struct _GstRnncaClass GstRnncaClass; 74 | 75 | struct _GstRnnca 76 | { 77 | GstVideoFilter videofilter; 78 | RecurNN *net; 79 | int current_frame; 80 | float pending_learn_rate; 81 | int osdebug; 82 | int playing; 83 | RnncaFrame *frame_prev; 84 | RnncaFrame *frame_now; 85 | RnncaFrame *play_frame; 86 | RecurNN **constructors; 87 | RnncaTrainer *trainers; 88 | RecurNN **train_nets; 89 | char *net_filename; 90 | int n_trainers; 91 | int hidden_size; 92 | char *pending_logfile; 93 | int training; 94 | u8 *training_map; 95 | int edges; 96 | float momentum; 97 | int momentum_soft_start; 98 | RnncaPixelHistory *history; 99 | TemporalPPM **temporal_ppms; 100 | int *offsets_Y; 101 | int len_Y; 102 | int *offsets_C; 103 | int len_C; 104 | int len_pos; 105 | char *offset_pattern; 106 | }; 107 | 108 | struct _GstRnncaClass 109 | { 110 | GstVideoFilterClass parent_class; 111 | }; 112 | 113 | #define GST_TYPE_RNNCA (gst_rnnca_get_type()) 114 | #define GST_RNNCA(obj) \ 115 | (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_RNNCA,GstRnnca)) 116 | #define GST_RNNCA_CLASS(klass) \ 117 | (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_RNNCA,GstRnncaClass)) 118 | #define GST_IS_RNNCA(obj) \ 119 | (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_RNNCA)) 120 | #define GST_IS_RNNCA_CLASS(klass) \ 121 | (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_RNNCA)) 122 | 123 | GType gst_rnnca_get_type(void); 124 | 125 | 126 | G_END_DECLS 127 | #endif /* __GOT_RNNCA_H__ */ 128 | -------------------------------------------------------------------------------- /gtk-recur.c: -------------------------------------------------------------------------------- 1 | /*Copyright 2014 Douglas Bagnall LGPL 2 | 3 | This is a gtk app that runs the recur plugin. Try --help. 4 | */ 5 | 6 | #include "player-common.h" 7 | 8 | #define WIDTH 800 9 | #define HEIGHT 600 10 | 11 | #define VID_LAGOS "movies/louis-theroux-lagos/louis.theroux.law.and.disorder.in.lagos.ws.pdtv.xvid-waters.avi" 12 | 13 | #define VID_TEARS "F30275.mov" 14 | 15 | static const char *PIPELINE_TEMPLATE = ("uridecodebin name=src " 16 | " ! videoscale method=nearest-neighbour ! videoconvert" 17 | " ! video/x-raw, format=I420, width=" QUOTE(WIDTH) ", height=" QUOTE(HEIGHT) 18 | " ! recur_manager name=recur osdebug=0 ! videoconvert" 19 | " ! xvimagesink name=videosink force-aspect-ratio=false" 20 | " recur. ! audio/x-raw ! fakesink" 21 | " src. ! audioconvert ! audioresample ! recur."); 22 | 23 | static gboolean option_fullscreen = FALSE; 24 | static gint option_width = WIDTH; 25 | static gint option_height = HEIGHT; 26 | static char *option_uri = URI_PREFIX VID_LAGOS; 27 | 28 | static GOptionEntry entries[] = 29 | { 30 | { "full-screen", 'f', 0, G_OPTION_ARG_NONE, &option_fullscreen, 31 | "run full screen", NULL }, 32 | { "width", 'w', 0, G_OPTION_ARG_INT, &option_width, "width of each screen", NULL }, 33 | { "height", 'h', 0, G_OPTION_ARG_INT, &option_height, "height of screen", NULL }, 34 | { "uri", 'u', 0, G_OPTION_ARG_FILENAME, &option_uri, "URI to play", NULL }, 35 | { NULL, 0, 0, 0, NULL, NULL, NULL } 36 | }; 37 | 38 | static gboolean 39 | key_press_event_cb(GtkWidget *widget, GdkEventKey *event, gpointer data) 40 | { 41 | switch (event->keyval){ 42 | case 'f': 43 | toggle_fullscreen(widget); 44 | break; 45 | case 'q': 46 | g_signal_emit_by_name(widget, "destroy"); 47 | break; 48 | default: 49 | break; 50 | } 51 | return TRUE; 52 | } 53 | 54 | static void 55 | video_widget_realize_cb (GtkWidget * widget, gpointer data) 56 | { 57 | gulong xid = GDK_WINDOW_XID (gtk_widget_get_window (widget)); 58 | video_window_handle = xid; 59 | 60 | //static const GdkColor black = {0, 0, 0, 0}; 61 | gtk_window_set_default_size(GTK_WINDOW(widget), option_width, option_height); 62 | hide_mouse(widget); 63 | if (option_fullscreen){ 64 | gtk_window_fullscreen(GTK_WINDOW(widget)); 65 | } 66 | } 67 | 68 | gboolean bus_callback(GstBus *bus, GstMessage *msg, GMainLoop *loop) 69 | { 70 | switch (GST_MESSAGE_TYPE(msg)) 71 | { 72 | case GST_MESSAGE_EOS: 73 | g_main_loop_quit(loop); 74 | gtk_main_quit(); 75 | break; 76 | default: 77 | break; 78 | } 79 | return TRUE; 80 | } 81 | 82 | static void 83 | src_drained_cb(GstElement *src, GMainLoop *loop) 84 | { 85 | g_main_loop_quit(loop); 86 | gtk_main_quit(); 87 | } 88 | 89 | 90 | static GstElement * 91 | make_pipeline(GError **parse_error, GMainLoop *loop){ 92 | GstElement *pipeline = gst_parse_launch(PIPELINE_TEMPLATE, 93 | parse_error); 94 | GstElement *src = gst_bin_get_by_name(GST_BIN(pipeline), "src"); 95 | g_object_set(G_OBJECT(src), 96 | "uri", option_uri, 97 | NULL); 98 | DEBUG("uri is %s", option_uri); 99 | 100 | g_signal_connect(src, "drained", 101 | G_CALLBACK(src_drained_cb), loop); 102 | 103 | return pipeline; 104 | } 105 | 106 | static void 107 | options(int argc, char *argv[]){ 108 | GOptionGroup *gst_opts = gst_init_get_option_group(); 109 | GOptionGroup *gtk_opts = gtk_get_option_group(TRUE); 110 | GOptionContext *ctx = g_option_context_new("...!"); 111 | g_option_context_add_main_entries(ctx, entries, NULL); 112 | g_option_context_add_group(ctx, gst_opts); 113 | g_option_context_add_group(ctx, gtk_opts); 114 | GError *error = NULL; 115 | if (!g_option_context_parse(ctx, &argc, &argv, &error)){ 116 | g_print ("Error initializing: %s\n", GST_STR_NULL(error->message)); 117 | exit (1); 118 | } 119 | g_option_context_free(ctx); 120 | } 121 | 122 | void destroy_cb(GtkWidget * widget, GMainLoop *loop) 123 | { 124 | g_main_loop_quit(loop); 125 | gtk_main_quit(); 126 | } 127 | 128 | gint main (int argc, char *argv[]) 129 | { 130 | options(argc, argv); 131 | GMainLoop *loop = g_main_loop_new(NULL, FALSE); 132 | GtkWidget *window = gtk_window_new(GTK_WINDOW_TOPLEVEL); 133 | 134 | /*Tell gstreamer to look locally for the plugin*/ 135 | GstRegistry *registry; 136 | registry = gst_registry_get(); 137 | gst_registry_scan_path(registry, "plugins"); 138 | 139 | GError *parse_error = NULL; 140 | GstElement *pipeline = make_pipeline(&parse_error, loop); 141 | DEBUG("pipeline is %p", pipeline); 142 | 143 | GstBus *bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline)); 144 | gst_bus_set_sync_handler(bus, (GstBusSyncHandler)sync_bus_call, NULL, NULL); 145 | gst_bus_add_watch(bus, (GstBusFunc)bus_callback, loop); 146 | gst_object_unref(bus); 147 | 148 | gst_element_set_state(pipeline, GST_STATE_PLAYING); 149 | 150 | g_signal_connect(G_OBJECT(window), "key-press-event", 151 | G_CALLBACK(key_press_event_cb), NULL); 152 | 153 | DEBUG("pipeline is %p", pipeline); 154 | 155 | g_signal_connect(G_OBJECT(window), "destroy", 156 | G_CALLBACK(destroy_cb), loop); 157 | 158 | g_signal_connect(window, "realize", 159 | G_CALLBACK(video_widget_realize_cb), NULL); 160 | 161 | gtk_widget_show_all(window); 162 | //hide_mouse(window); 163 | 164 | g_main_loop_run(loop); 165 | 166 | gst_element_set_state (pipeline, GST_STATE_NULL); 167 | gst_object_unref (pipeline); 168 | return 0; 169 | } 170 | -------------------------------------------------------------------------------- /gtkdisplay.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Douglas Bagnall LGPL 2 | 3 | # used by classify-gtk 4 | 5 | import gi 6 | gi.require_version('Gst', '1.0') 7 | from gi.repository import Gtk, Gdk 8 | 9 | class ClassifierDisplay(Gtk.DrawingArea): 10 | results = [] 11 | def __init__(self, classifier, size): 12 | Gtk.DrawingArea.__init__(self) 13 | self.size = size 14 | self.set_size_request (size, size) 15 | self.classifier = classifier 16 | self.connect('draw', self.on_draw) 17 | 18 | def notify_results(self, results): 19 | self.results.append(results) 20 | self.queue_draw() 21 | 22 | def on_draw(self, widget, cr): 23 | if not self.results: 24 | return 25 | cr.save() 26 | w = widget.get_allocated_width() 27 | h = widget.get_allocated_height() 28 | cr.translate (w / 2, h / 2) 29 | cr.scale(w * 0.48, h * 0.48) 30 | 31 | #draw a centre line 32 | cr.set_line_width(0.05) 33 | cr.set_dash((0.01, 0.02)) 34 | cr.set_source_rgb(0.8, 0.8, 0.8) 35 | cr.move_to(0, -0.5) 36 | cr.line_to(0, 0.4) 37 | cr.stroke() 38 | 39 | cr.set_dash(()) 40 | step = 1.0 / len(self.results) 41 | i = step 42 | for winner, scores in self.results: 43 | cr.set_line_width(i * 0.08) 44 | i += step 45 | if len(scores) == 2: 46 | m, e = scores 47 | p = m - e 48 | cr.set_source_rgb(abs(p), m, e) 49 | cr.move_to(p, 0.1) 50 | cr.line_to(p, 0) 51 | 52 | elif len(scores) == 3: 53 | m, e, n = scores 54 | p = m - e 55 | cr.set_source_rgb(n, m, e) 56 | cr.move_to(p, n - 0.1) 57 | cr.line_to(p, n - 0.2) 58 | 59 | cr.stroke() 60 | cr.restore() 61 | self.results = [] 62 | 63 | 64 | def window_stop(window): 65 | window.classifier.stop() 66 | Gtk.main_quit() 67 | 68 | def on_key_press_event(widget, event): 69 | keyname = Gdk.keyval_name(event.keyval).lower() 70 | ctrl = event.state & Gdk.ModifierType.CONTROL_MASK 71 | if keyname == 'q' or (ctrl and keyname == 'w'): 72 | window_stop(widget) 73 | if keyname == 'n': 74 | widget.classifier.load_next_file() 75 | if keyname == 'right': 76 | widget.classifier.seek_relative(5) 77 | if keyname == 'left': 78 | widget.classifier.seek_relative(-5) 79 | 80 | #print "Key %s (%d) was pressed" % (keyname, event.keyval) 81 | 82 | 83 | def run(classifier, files, reverse=False, fullscreen=False): 84 | window = Gtk.Window() 85 | window.set_title ("RNN Classifier") 86 | app = ClassifierDisplay(classifier, 300) 87 | classifier.widget = app 88 | window.add(app) 89 | #XXX mucky 90 | window.classifier = classifier 91 | #print dir(window) 92 | window.connect_after('destroy', window_stop) 93 | window.connect('key_press_event', on_key_press_event) 94 | window.show_all() 95 | if fullscreen: 96 | window.fullscreen() 97 | 98 | classifier.run(files, reverse) 99 | Gtk.main() 100 | -------------------------------------------------------------------------------- /licenses/COPYING.mdct: -------------------------------------------------------------------------------- 1 | Copyright (c) 2002-2008 Xiph.org Foundation 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions 5 | are met: 6 | 7 | - Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | 10 | - Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | - Neither the name of the Xiph.org Foundation nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION 22 | OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /licenses/MIT.pycdb: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009-2015 David Wilson 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 17 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 18 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 19 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 20 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /licenses/README: -------------------------------------------------------------------------------- 1 | Which licence applies where: 2 | 3 | * CC0 applies to some of the ccan modules (see ccan/*/LICENSE). 4 | 5 | * GPL-2+ applies to ccan/opt and text-predict which depends on 6 | ccan/opt. It doesn't affect the Gstreamer plug-ins. 7 | 8 | * mdct.c is from libvorbis and comes with a BSD license (see 9 | COPYING.mdct). 10 | 11 | * The MIT license applies to scripts/pycdb.py, which is derived from 12 | David Wilson's python-pure-cdb module. 13 | 14 | * Everything else can be used under the terms of any version of the 15 | Gnu LGPL. 16 | -------------------------------------------------------------------------------- /local.mak.example.x86_64: -------------------------------------------------------------------------------- 1 | DEV_CFLAGS = -ggdb -Ofast -fno-inline -pg 2 | ARCH_CFLAGS = -fPIC -DPIC -m64 -D_FILE_OFFSET_BITS=64 3 | 4 | #recur-nn.o works better with -fprefetch-loop-arrays 5 | NN_SPECIAL_FLAGS = -fprefetch-loop-arrays 6 | 7 | ARCH = x86_64 8 | 9 | # set USE_CBLAS to use BLAS library instead of specialised routines 10 | # (CBLAS is slower in tests on x64-64) 11 | 12 | #USE_CBLAS = 1 13 | 14 | # include -DVECTOR_ALL_THE_WAY in LOCAL_FLAGS to use explicit 15 | # vectorisation in places where GCC 4.7+ can already deduce efficient 16 | # vectorisation. This helps Clang in one or two places. On the other 17 | # hand, removing -DVECTOR removes vectorisation assistance that GCC 18 | # 4.8 still needs. 19 | 20 | LOCAL_FLAGS = -DVECTOR 21 | #LOCAL_FLAGS = -DVECTOR_ALL_THE_WAY -DVECTOR 22 | -------------------------------------------------------------------------------- /mdct.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is originally from Xiph.org's libvorbis. For license terms, see 3 | * COPYING.mdct. (BSDish). 4 | * 5 | * COPYRIGHT 1994-2009 Xiph.Org Foundation http://www.xiph.org/ 6 | * 7 | * lightly modified by Douglas Bagnall 8 | */ 9 | 10 | #ifndef _OGG_mdct_H_ 11 | #define _OGG_mdct_H_ 12 | 13 | /*#define MDCT_INTEGERIZED <- be warned there could be some hurt left here*/ 14 | #ifdef MDCT_INTEGERIZED 15 | 16 | #define DATA_TYPE int 17 | #define REG_TYPE register int 18 | #define TRIGBITS 14 19 | #define cPI3_8 6270 20 | #define cPI2_8 11585 21 | #define cPI1_8 15137 22 | 23 | #define FLOAT_CONV(x) ((int)((x)*(1<>TRIGBITS) 25 | #define HALVE(x) ((x)>>1) 26 | 27 | #else 28 | 29 | #define DATA_TYPE float 30 | #define REG_TYPE float 31 | #define cPI3_8 .38268343236508977175F 32 | #define cPI2_8 .70710678118654752441F 33 | #define cPI1_8 .92387953251128675613F 34 | 35 | #define FLOAT_CONV(x) (x) 36 | #define MULT_NORM(x) (x) 37 | #define HALVE(x) ((x)*.5f) 38 | 39 | #endif 40 | 41 | 42 | typedef struct { 43 | int n; 44 | int log2n; 45 | 46 | DATA_TYPE *trig; 47 | int *bitrev; 48 | 49 | DATA_TYPE scale; 50 | } mdct_lookup; 51 | 52 | extern void mdct_init(mdct_lookup *lookup,int n); 53 | extern void mdct_clear(mdct_lookup *l); 54 | extern void mdct_forward(mdct_lookup *init, DATA_TYPE *in, DATA_TYPE *out); 55 | extern void mdct_backward(mdct_lookup *init, DATA_TYPE *in, DATA_TYPE *out); 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /mfcc.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Douglas Bagnall LGPL */ 2 | #ifndef HAVE_MFCC_H 3 | #define HAVE_MFCC_H 4 | 5 | #include "recur-common.h" 6 | #include 7 | 8 | 9 | typedef struct _RecurAudioBinSlope RecurAudioBinSlope; 10 | 11 | struct _RecurAudioBinSlope { 12 | int left; 13 | int right; 14 | float left_fraction; 15 | float right_fraction; 16 | float log_scale; 17 | float slope; 18 | }; 19 | 20 | typedef struct _RecurAudioBinner { 21 | GstFFTF32 *fft; 22 | float *pcm_data; 23 | GstFFTF32Complex *freq_data; 24 | RecurAudioBinSlope *slopes; 25 | int window_size; 26 | int n_bins; 27 | const float *mask; 28 | float *fft_bins; 29 | float *dct_bins; 30 | int window_type; 31 | int value_size; 32 | } RecurAudioBinner; 33 | 34 | enum { 35 | RECUR_WINDOW_NONE = 0, 36 | RECUR_WINDOW_HANN = 1, 37 | RECUR_WINDOW_VORBIS, 38 | RECUR_WINDOW_MP3, 39 | }; 40 | 41 | float * 42 | recur_extract_log_freq_bins(RecurAudioBinner *ab, float *data); 43 | 44 | float * 45 | recur_extract_mfccs(RecurAudioBinner *ab, float *data); 46 | 47 | RecurAudioBinSlope * 48 | recur_bin_slopes_new(const int n_bins, const int fft_len, 49 | const float fmin, const float fmax, const float fknee, 50 | const float ffocus, const float audio_rate) __attribute__ ((malloc)); 51 | 52 | 53 | RecurAudioBinner * 54 | recur_audio_binner_new(int window_size, int window_type, 55 | int n_bins, 56 | float min_freq, 57 | float max_freq, 58 | float knee_freq, 59 | float focus_freq, 60 | float audio_rate, 61 | float scale, 62 | int value_size 63 | ); 64 | 65 | void recur_audio_binner_delete(RecurAudioBinner *ab); 66 | 67 | void recur_window_init(float *mask, int len, int type, float scale); 68 | 69 | 70 | void recur_dct(const float *restrict input, float *restrict output, int len); 71 | void recur_dct_cached(const float *restrict input, float *restrict output, int len); 72 | void recur_idct(const float *restrict input, float *restrict output, int len); 73 | #endif 74 | -------------------------------------------------------------------------------- /opt-helpers.h: -------------------------------------------------------------------------------- 1 | /* Copyright (C) 2014 Douglas Bagnall GPL2+ 2 | 3 | This thinly wraps and extends ccan/opt/opt.h. 4 | 5 | All files that include this are licensed under the GNU General Public License, 6 | version 2 or greater, due to the ccan/opt link. 7 | */ 8 | #ifndef HAVE_OPT_HELPERS_H 9 | #define HAVE_OPT_HELPERS_H 10 | 11 | #include "recur-nn.h" 12 | #include "recur-nn-helpers.h" 13 | #include "ccan/opt/opt.h" 14 | #include 15 | 16 | #define IN_RANGE_01(x) (((x) >= 0.0f) && ((x) <= 1.0f)) 17 | 18 | /*restrict to 0-1 range (mostly for probabilities)*/ 19 | static UNUSED char * 20 | opt_set_floatval01(const char *arg, float *f){ 21 | char *msg = opt_set_floatval(arg, f); 22 | if (msg == NULL && ! IN_RANGE_01(*f)){ 23 | char *s; 24 | if (asprintf(&s, "We want a number between 0 and 1, not '%s'", arg) > 0){ 25 | return s; 26 | } 27 | } 28 | return msg; 29 | } 30 | 31 | #endif 32 | -------------------------------------------------------------------------------- /pending_properties.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Douglas Bagnall LGPL */ 2 | #ifndef HAVE_PENDING_PROPERTIES_H 3 | #define HAVE_PENDING_PROPERTIES_H 4 | 5 | #include "gstclassify.h" 6 | #include "recur-common.h" 7 | #include 8 | 9 | #if 1 10 | #define PENDING_PROP(self, prop) (&(self)->pending_properties[prop]) 11 | #else 12 | static inline GValue* 13 | PENDING_PROP(GstClassify *self, int prop) { 14 | STDERR_DEBUG("prop is %d", prop); 15 | return &self->pending_properties[prop]; 16 | } 17 | #endif 18 | 19 | 20 | #define PP_IS_SET(self, id) (G_IS_VALUE(PENDING_PROP(self, id))) 21 | 22 | #define PP_GET_FLOAT(self, id, def) get_gvalue_float(PENDING_PROP(self, id), def) 23 | #define PP_GET_INT(self, id, def) get_gvalue_int(PENDING_PROP(self, id), def) 24 | #define PP_GET_STRING(self, id, def) get_gvalue_string(PENDING_PROP(self, id), def) 25 | #define PP_GET_BOOLEAN(self, id, def) get_gvalue_boolean(PENDING_PROP(self, id), def) 26 | 27 | #define RESET_OR_INIT_GV(v, t) ((G_IS_VALUE(v)) ? g_value_reset(v) : \ 28 | g_value_init((v), (t))) 29 | 30 | 31 | static inline void 32 | copy_gvalue(GValue *dest, const GValue *src) 33 | { 34 | /*XXX does this reclaim a string value? who knows */ 35 | RESET_OR_INIT_GV(dest, G_VALUE_TYPE(src)); 36 | g_value_copy(src, dest); 37 | } 38 | 39 | static inline void 40 | warn_if_set(const GValue *v){ 41 | if (G_IS_VALUE(v)){ 42 | char *contents = g_strdup_value_contents(v); 43 | GST_WARNING("gvalue %p appears an unexpected type: %s", v, contents); 44 | free(contents); 45 | } 46 | } 47 | 48 | 49 | static inline const char * 50 | get_gvalue_string(GValue *v, const char *_default){ 51 | if (! G_VALUE_HOLDS_STRING(v)){ 52 | warn_if_set(v); 53 | return _default; 54 | } 55 | return g_value_get_string(v); 56 | } 57 | 58 | static inline gboolean 59 | get_gvalue_boolean(GValue *v, const gboolean _default){ 60 | if (! G_VALUE_HOLDS_BOOLEAN(v)){ 61 | warn_if_set(v); 62 | return _default; 63 | } 64 | return g_value_get_boolean(v); 65 | } 66 | 67 | static inline int 68 | get_gvalue_int(GValue *v, const int _default){ 69 | if (! G_VALUE_HOLDS_INT(v)){ 70 | warn_if_set(v); 71 | return _default; 72 | } 73 | return g_value_get_int(v); 74 | } 75 | 76 | static inline u64 77 | get_gvalue_u64(GValue *v, const u64 _default){ 78 | if (! G_VALUE_HOLDS_UINT64(v)){ 79 | warn_if_set(v); 80 | return _default; 81 | } 82 | return g_value_get_uint64(v); 83 | } 84 | 85 | static inline float 86 | get_gvalue_float(const GValue *v, const float _default){ 87 | if (! G_VALUE_HOLDS_FLOAT(v)){ 88 | warn_if_set(v); 89 | return _default; 90 | } 91 | return g_value_get_float(v); 92 | } 93 | 94 | static inline char * 95 | steal_gvalue_string(GValue *v){ 96 | if (! G_VALUE_HOLDS_STRING(v)){ 97 | warn_if_set(v); 98 | return NULL; 99 | } 100 | char *s = g_value_dup_string(v); 101 | g_value_unset(v); 102 | return s; 103 | } 104 | 105 | static inline void 106 | set_gvalue_float(GValue *v, const float f){ 107 | RESET_OR_INIT_GV(v, G_TYPE_FLOAT); 108 | g_value_set_float(v, f); 109 | } 110 | 111 | #endif 112 | -------------------------------------------------------------------------------- /player-common.h: -------------------------------------------------------------------------------- 1 | #ifndef HAVE_PLAYER_COMMON_H 2 | #define HAVE_PLAYER_COMMON_H 1 3 | 4 | #define URI_PREFIX "file://" TEST_VIDEO_DIR "/" 5 | 6 | #include 7 | #include 8 | #include "recur-common.h" 9 | #include 10 | #include 11 | #include "path.h" 12 | 13 | static inline void 14 | toggle_fullscreen(GtkWidget *widget){ 15 | GdkWindow *gdk_window = gtk_widget_get_window(widget); 16 | GdkWindowState state = gdk_window_get_state(gdk_window); 17 | if (state & GDK_WINDOW_STATE_FULLSCREEN){ 18 | gtk_window_unfullscreen(GTK_WINDOW(widget)); 19 | } 20 | else{ 21 | gtk_window_fullscreen(GTK_WINDOW(widget)); 22 | } 23 | } 24 | 25 | static inline void 26 | hide_mouse(GtkWidget *widget){ 27 | GdkWindow *w = gtk_widget_get_window(widget); 28 | GdkDisplay *display = gdk_display_get_default(); 29 | GdkCursor *cursor = gdk_cursor_new_for_display(display, GDK_BLANK_CURSOR); 30 | gdk_window_set_cursor(w, cursor); 31 | g_object_unref (cursor); 32 | } 33 | 34 | static inline void 35 | set_up_loop(GstElement *source, int flags){ 36 | DEBUG("loooooping\n"); 37 | if (! gst_element_seek(source, 1.0, GST_FORMAT_TIME, 38 | flags, 39 | GST_SEEK_TYPE_SET, 0, 40 | GST_SEEK_TYPE_NONE, GST_CLOCK_TIME_NONE 41 | )){ 42 | GST_WARNING("Seek failed!\n"); 43 | } 44 | } 45 | 46 | static guintptr video_window_handle = 0; 47 | 48 | static GstBusSyncReply 49 | sync_bus_call(GstBus *bus, GstMessage *msg, gpointer data) 50 | { 51 | if (!gst_is_video_overlay_prepare_window_handle_message (msg)) 52 | return GST_BUS_PASS; 53 | 54 | if (video_window_handle != 0) { 55 | GstVideoOverlay *overlay; 56 | // GST_MESSAGE_SRC (msg) will be the video sink element 57 | overlay = GST_VIDEO_OVERLAY (GST_MESSAGE_SRC (msg)); 58 | gst_video_overlay_set_window_handle (overlay, video_window_handle); 59 | } 60 | else { 61 | g_warning ("Should have obtained video_window_handle by now!"); 62 | } 63 | gst_message_unref (msg); 64 | return GST_BUS_DROP; 65 | } 66 | 67 | #endif 68 | -------------------------------------------------------------------------------- /py-recur-helpers.h: -------------------------------------------------------------------------------- 1 | #ifndef HAVE_PY_RECUR_HELPERS_H 2 | #define HAVE_PY_RECUR_HELPERS_H 3 | #include 4 | #include "structmember.h" 5 | #include "structseq.h" 6 | #include "recur-nn.h" 7 | 8 | #define BaseNet_HEAD \ 9 | PyObject_HEAD \ 10 | RecurNN *net; \ 11 | rnn_learning_method learning_method; \ 12 | float momentum; \ 13 | int batch_size; \ 14 | const char *filename; \ 15 | 16 | typedef struct { 17 | BaseNet_HEAD 18 | } BaseNet; 19 | 20 | 21 | #define RNNPY_BASE_NET(x) ((BaseNet *)(x)) 22 | 23 | static PyObject * 24 | BaseNet_save(BaseNet *self, PyObject *args, PyObject *kwds) 25 | { 26 | RecurNN *net = self->net; 27 | const char *filename = NULL; 28 | int backup = 1; 29 | 30 | static char *kwlist[] = {"filename", /* z */ 31 | "backup", /* i */ 32 | NULL}; 33 | 34 | if (!PyArg_ParseTupleAndKeywords(args, kwds, "|zi", kwlist, 35 | &filename, 36 | &backup 37 | )){ 38 | return NULL; 39 | } 40 | if (filename == NULL){ 41 | filename = self->filename; 42 | } 43 | int r = rnn_save_net(net, filename, backup); 44 | if (r){ 45 | return PyErr_Format(PyExc_IOError, "could not save to %s", 46 | filename); 47 | } 48 | return Py_BuildValue(""); 49 | } 50 | 51 | static int 52 | set_net_filename(BaseNet *self, const char *filename, const char *basename, 53 | char *metadata) 54 | { 55 | char s[1000]; 56 | RecurNN *net = self->net; 57 | if (filename){ 58 | self->filename = strdup(filename); 59 | } 60 | else { 61 | u32 sig = rnn_hash32(metadata); 62 | int wrote = snprintf(s, sizeof(s), "%s-%0" PRIx32 "i%d-h%d-o%d.net", 63 | basename, sig, net->input_size, net->hidden_size, net->output_size); 64 | if (wrote >= sizeof(s)){ 65 | PyErr_Format(PyExc_ValueError, 66 | "filename is trying to be too long!"); 67 | return -1; 68 | } 69 | self->filename = strdup(s); 70 | } 71 | return 0; 72 | } 73 | 74 | /* Net_{get,set}float_{rnn,bptt}. These access float attributes that are 75 | pointed to via an integer offset into the struct. 76 | 77 | Without this we'd need separate access functions for each attribute. 78 | */ 79 | 80 | static UNUSED PyObject * 81 | Net_getfloat_rnn(BaseNet *self, int *offset) 82 | { 83 | void *addr = ((void *)self->net) + *offset; 84 | float f = *(float *)addr; 85 | return PyFloat_FromDouble((double)f); 86 | } 87 | 88 | static UNUSED int 89 | Net_setfloat_rnn(BaseNet *self, PyObject *value, int *offset) 90 | { 91 | PyObject *pyfloat = PyNumber_Float(value); 92 | if (pyfloat == NULL){ 93 | return -1; 94 | } 95 | void *addr = ((void *)self->net) + *offset; 96 | float f = PyFloat_AS_DOUBLE(pyfloat); 97 | *(float *)addr = f; 98 | return 0; 99 | } 100 | 101 | static UNUSED PyObject * 102 | Net_getfloat_bptt(BaseNet *self, int *offset) 103 | { 104 | void *addr = ((void *)self->net->bptt) + *offset; 105 | float f = *(float *)addr; 106 | return PyFloat_FromDouble((double)f); 107 | } 108 | 109 | static UNUSED int 110 | Net_setfloat_bptt(BaseNet *self, PyObject *value, int *offset) 111 | { 112 | PyObject *pyfloat = PyNumber_Float(value); 113 | if (pyfloat == NULL){ 114 | return -1; 115 | } 116 | void *addr = ((void *)self->net->bptt) + *offset; 117 | float f = PyFloat_AS_DOUBLE(pyfloat); 118 | *(float *)addr = f; 119 | return 0; 120 | } 121 | 122 | 123 | static int add_module_constants(PyObject* m) 124 | { 125 | int r = 0; 126 | 127 | #define ADD_INT_CONSTANT(x) (PyModule_AddIntConstant(m, QUOTE(x), (RNN_ ##x))) 128 | 129 | r = r || ADD_INT_CONSTANT(MOMENTUM_WEIGHTED); 130 | r = r || ADD_INT_CONSTANT(MOMENTUM_NESTEROV); 131 | r = r || ADD_INT_CONSTANT(MOMENTUM_SIMPLIFIED_NESTEROV); 132 | r = r || ADD_INT_CONSTANT(MOMENTUM_CLASSICAL); 133 | r = r || ADD_INT_CONSTANT(ADAGRAD); 134 | r = r || ADD_INT_CONSTANT(ADADELTA); 135 | r = r || ADD_INT_CONSTANT(RPROP); 136 | 137 | r = r || ADD_INT_CONSTANT(RELU); 138 | r = r || ADD_INT_CONSTANT(RESQRT); 139 | r = r || ADD_INT_CONSTANT(RECLIP20); 140 | 141 | r = r || ADD_INT_CONSTANT(INIT_ZERO); 142 | r = r || ADD_INT_CONSTANT(INIT_FLAT); 143 | r = r || ADD_INT_CONSTANT(INIT_FAN_IN); 144 | r = r || ADD_INT_CONSTANT(INIT_RUNS); 145 | 146 | #undef ADD_INT_CONSTANT 147 | 148 | return r; 149 | } 150 | 151 | 152 | #endif 153 | -------------------------------------------------------------------------------- /py-recur-text.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2015 Douglas Bagnall LGPL 2 | 3 | Python bindings for text RNNs. */ 4 | #ifndef __PY_RECUR_TEXT_H__ 5 | #define __PY_RECUR_TEXT_H__ 6 | 7 | #include 8 | #include "structmember.h" 9 | #include "structseq.h" 10 | #endif 11 | -------------------------------------------------------------------------------- /recur-config.h: -------------------------------------------------------------------------------- 1 | /* gst bits */ 2 | #ifndef _HAVE_RECUR_CONFIG_H 3 | #define _HAVE_RECUR_CONFIG_H 4 | 5 | #define GST_LICENSE "LGPL" 6 | #define GST_PACKAGE_NAME "GStreamer RNN plugin" 7 | #define GST_PACKAGE_ORIGIN "http://halo.gen.nz/" 8 | 9 | #ifndef VERSION 10 | #define VERSION "1" 11 | #endif 12 | 13 | #ifndef PACKAGE 14 | #define PACKAGE "recur" 15 | #endif 16 | 17 | #endif 18 | -------------------------------------------------------------------------------- /recur-nn-helpers.h: -------------------------------------------------------------------------------- 1 | /* Copyright (C) 2013 Douglas Bagnall LGPL 2 | 3 | These are mostly-private variables and helpers for recur-nn. 4 | */ 5 | #ifndef _GOT_RECUR_NN_HELPERS_H 6 | #define _GOT_RECUR_NN_HELPERS_H 1 7 | 8 | #include "recur-nn.h" 9 | 10 | #if USE_CBLAS 11 | #include 12 | #endif 13 | #include "pgm_dump.h" 14 | 15 | #if VECTOR 16 | typedef float v4ss __attribute__ ((vector_size (16))) __attribute__ ((aligned (16))); 17 | #endif 18 | 19 | #define ALIGNED_SIZEOF(x) ((sizeof(x) + 15UL) & ~15UL) 20 | #define ALIGNED_VECTOR_LEN(x, type) ((((x) * sizeof(type) + 15UL) & ~15UL) / sizeof(type)) 21 | 22 | static inline void 23 | scale_aligned_array(float *array, int len, float scale) 24 | { 25 | /*Using cblas might create denormal numbers -- unless the blas library has 26 | been compiled with -ffastmath. */ 27 | #if ! USE_CBLAS 28 | ASSUME_ALIGNED(array); 29 | for (int i = 0; i < len; i++){ 30 | array[i] *= scale; 31 | } 32 | #else 33 | cblas_sscal(len, scale, array, 1); 34 | #endif 35 | } 36 | 37 | static inline void 38 | add_aligned_arrays(float *restrict dest, int len, const float *restrict src, 39 | const float scale) 40 | { 41 | /*dest = dest + src * scale 42 | cblas_saxpy can do it. */ 43 | #if ! USE_CBLAS 44 | //XXX a prefetch would help 45 | ASSUME_ALIGNED(dest); 46 | ASSUME_ALIGNED(src); 47 | #if VECTOR_ALL_THE_WAY && 0 48 | len >>= 2; 49 | v4ss *vd = (v4ss*)dest; 50 | v4ss *vs = (v4ss*)src; 51 | if (scale != 1.0f){ 52 | v4ss v_scale = {scale, scale, scale, scale}; 53 | for (int i = 0; i < len; i++){ 54 | __builtin_prefetch(&vs[i + 3]); 55 | __builtin_prefetch(&vd[i + 3]); 56 | vd[i] += vs[i] * v_scale; 57 | } 58 | } 59 | else { 60 | for (int i = 0; i < len; i++){ 61 | __builtin_prefetch(&vs[i + 3]); 62 | __builtin_prefetch(&vd[i + 3]); 63 | vd[i] += vs[i]; 64 | } 65 | } 66 | 67 | #else /*not VECTOR_ALL_THE_WAY*/ 68 | if (scale == 1.0f){ 69 | for (int i = 0; i < len; i++){ 70 | dest[i] += src[i]; 71 | } 72 | } 73 | else { 74 | for (int i = 0; i < len; i++){ 75 | dest[i] += src[i] * scale; 76 | } 77 | } 78 | #endif 79 | #else 80 | cblas_saxpy(len, scale, src, 1, dest, 1); 81 | #endif 82 | } 83 | 84 | static inline void 85 | perforate_array(float *array, int len, float dropout, rand_ctx *rng){ 86 | int i; 87 | if (dropout == 0.5f){ /*special case using far fewer random numbers*/ 88 | for (i = 0; i < len;){ 89 | u64 bits = rand64(rng); 90 | int end = i + MIN(64, len - i); 91 | for (; i < end; i++){ 92 | array[i] = (bits & 1) ? array[i] : 0; 93 | } 94 | } 95 | } 96 | else { 97 | for (i = 0; i < len; i++){ 98 | /*XXX could use randomise_float_pair() for possibly marginal speedup*/ 99 | array[i] = (rand_double(rng) > dropout) ? array[i] : 0.0f; 100 | } 101 | } 102 | } 103 | 104 | static inline float 105 | soft_clip(float sum, float halfmax){ 106 | if (halfmax == 0){ 107 | /*there is nothing sensible to do*/ 108 | return sum; 109 | } 110 | float x = sum / halfmax; 111 | float fudge = 0.99 + x * x / 100; 112 | return 2.0f * x / (1 + x * x * fudge); 113 | } 114 | 115 | static inline float 116 | softclip_scale(float sum, float halfmax, float *array, int len){ 117 | ASSUME_ALIGNED(array); 118 | if (sum > halfmax){ 119 | float scale = soft_clip(sum, halfmax); 120 | scale_aligned_array(array, len, scale); 121 | return scale * sum; 122 | } 123 | return sum; 124 | } 125 | 126 | static inline void 127 | zero_small_numbers(float *array, int len) 128 | { 129 | ASSUME_ALIGNED(array); 130 | for (int i = 0; i < len; i++){ 131 | array[i] = (fabsf(array[i]) > 1e-34f) ? array[i] : 0.0f; 132 | } 133 | } 134 | 135 | static inline float 136 | abs_sum_aligned_array(const float *array, int len) 137 | { 138 | float sum = 0.0f; 139 | ASSUME_ALIGNED(array); 140 | for (int i = 0; i < len; i++){ 141 | sum += fabsf(array[i]); 142 | } 143 | return sum; 144 | } 145 | 146 | #define ZERO_WITH_MEMSET 0 147 | 148 | static inline void 149 | zero_aligned_array(float *array, int size){ 150 | ASSUME_ALIGNED(array); 151 | #if ZERO_WITH_MEMSET 152 | memset(array, 0, size * sizeof(float)); 153 | #else 154 | int i; 155 | #if VECTOR 156 | size >>= 2; 157 | v4ss *v_array = (v4ss*)array; 158 | v4ss vz = {0, 0, 0, 0}; 159 | for (i = 0; i < size; i++){ 160 | v_array[i] = vz; 161 | } 162 | #else 163 | for (i = 0; i < size; i++){ 164 | array[i] = 0.0f; 165 | } 166 | #endif 167 | #endif 168 | } 169 | 170 | static inline void 171 | add_array_noise(rand_ctx *rng, float *array, int len, float deviation){ 172 | for (int i = 0; i < len; i++){ 173 | float noise = cheap_gaussian_noise(rng) * deviation; 174 | array[i] += noise; 175 | } 176 | } 177 | 178 | #define MAYBE_ADD_ARRAY_NOISE(rng, array, len, dev) do { \ 179 | if (dev) \ 180 | add_array_noise(rng, array, len, dev); \ 181 | } while(0) 182 | 183 | 184 | #endif 185 | -------------------------------------------------------------------------------- /rescale.h: -------------------------------------------------------------------------------- 1 | #ifndef HAVE_RESCALE_H 2 | #define HAVE_RESCALE_H 3 | 4 | #include "recur-common.h" 5 | 6 | struct _Image { 7 | uint width; 8 | uint height; 9 | u8 *data; 10 | }; 11 | 12 | typedef struct _Image Image; 13 | 14 | __attribute__((malloc)) Image * 15 | recur_load_pgm_file(const char *filename); 16 | 17 | void 18 | recur_exact_downscale(const u8 *restrict src, const int s_width, 19 | const int s_height, const int s_stride, 20 | u8 *restrict dest, const int d_width, 21 | const int d_height, const int d_stride); 22 | 23 | 24 | void 25 | recur_skipping_downscale(const u8 *restrict src, const int s_width, 26 | const int s_height, const int s_stride, 27 | u8 *restrict dest, const int d_width, 28 | const int d_height, const int d_stride); 29 | 30 | void 31 | recur_adaptive_downscale(const u8 *src, const int s_width, 32 | const int s_height, const int s_stride, 33 | u8 *dest, const int d_width, 34 | const int d_height, const int d_stride); 35 | 36 | void 37 | recur_float_downscale(const float *restrict src, const int s_width, 38 | const int s_height, const int s_stride, 39 | float *restrict dest, const int d_width, 40 | const int d_height, const int d_stride); 41 | 42 | 43 | /*recur_integer_downscale_to_float scales u8 planes down to smaller [0,1) 44 | float planes, using nearest-neighbour search (for now) */ 45 | 46 | static inline void 47 | recur_integer_downscale_to_float(const u8 *im, float *dest, int stride, 48 | int left, int top, int w, int h, int scale){ 49 | const u8 *plane = im + top * stride + left; 50 | int y, x; 51 | /*w, h are target width and height. width is assumed to be stride. */ 52 | for (y = 0; y < h; y++){ 53 | for (int y2 = 0; y2 < scale; y2++){ 54 | for (x = 0; x < w; x++){ 55 | for (int x2 = 0; x2 < scale; x2++){ 56 | dest[y * w + x] += plane[(y * scale + y2) * stride + x * scale + x2]; 57 | } 58 | } 59 | } 60 | } 61 | for (int i = 0; i < w * h; i++){ 62 | dest[i] /= (scale * scale * 256.0); 63 | } 64 | } 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /scripts/colour-gen: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys 3 | import argparse 4 | 5 | def get_spectrum_24(): 6 | reds = sum( 7 | [ 8 | [0], 9 | range(0, 100, 14), 10 | range(90, 80, -14), 11 | range(75, 0, -35), 12 | [0] * 10, 13 | range(12, 145, 42), 14 | range(145, 205, 15), 15 | range(205, 255, 5), 16 | [255] * 12, 17 | ], []) 18 | greens = sum( 19 | [ 20 | [0] * 12, 21 | range(0, 255, 23), 22 | [255] * 1, 23 | range(255, 0, -15), 24 | [0] * 5, 25 | range(0, 255, 31), 26 | ], []) 27 | blues = sum( 28 | [ 29 | [0], 30 | range(0, 80, 10), 31 | range(80, 160, 25), 32 | range(150, 0, -15), 33 | [0] * 17, 34 | range(0, 150, 15), 35 | range(150, 250, 43), 36 | ], []) 37 | 38 | rgb = zip(reds, greens, blues) 39 | print >> sys.stderr, "reds %d greens %d blues %d" % (len(reds), 40 | len(greens), 41 | len(blues)) 42 | comp = [(16 if (r * .3 + g * .6 + b * .1) > 128 else 15) 43 | for r, g, b in rgb] 44 | 45 | return rgb, comp 46 | 47 | def print_spectrum_24(): 48 | spectrum, comp = get_spectrum_24() 49 | for rgb, c in zip(spectrum, comp): 50 | r, g, b = rgb 51 | print "[48;2;%s;%s;%sm[38;5;%sm #%02x%02x%02x:%s%s" % (r, g, b, c, 52 | r, g, b, c, " " * 72) 53 | 54 | print "" 55 | 56 | def write_spectrum_24(): 57 | spectrum, comp = get_spectrum_24() 58 | n = len(spectrum) 59 | print '#define N_COLOURS_24 %s' % n 60 | print 61 | print "static const char *COLOURS_24[%s] = {" % (n + 1) 62 | for bg, c in zip(spectrum, comp): 63 | r, g, b = bg 64 | print ' "\\x1B[48;2;%s;%s;%sm\\x1B[38;5;%sm",' % (r, g, b, c) 65 | print ' NULL' 66 | print '};' 67 | 68 | def get_spectrum(): 69 | spectrum = [x for sublist in 70 | [ 71 | [0] * 5, 72 | [232, 232, 233, 233], 73 | range(17, 20), 74 | [24, 29], 75 | range(28, 46, 6), 76 | [76, 112, 118, 190], 77 | range(226, 190, -6), 78 | range(197, 202), 79 | range(207, 228, 5), 80 | range(155, 136, -5), 81 | ] 82 | for x in sublist] 83 | 84 | comp = [(15 if i > 15 else 253) for i, x in enumerate(spectrum)] 85 | return spectrum, comp 86 | #\x1B) 87 | 88 | def print_spectrum(): 89 | spectrum, comp = get_spectrum() 90 | for bg, c in zip(spectrum, comp): 91 | print "[48;5;%sm[38;5;%sm %s:%s.%s" % (bg, c, bg, c, " " * 72) 92 | 93 | print "" 94 | 95 | def write_spectrum(): 96 | spectrum, comp = get_spectrum() 97 | n = len(spectrum) 98 | print '#define N_COLOURS_256 %s' % n 99 | print 100 | print "static const char *COLOURS_256[%s] = {" % (n + 1) 101 | for bg, c in zip(spectrum, comp): 102 | print ' "\\x1B[48;5;%sm\\x1B[38;5;%sm",' % (bg, c) 103 | print ' NULL' 104 | print '};' 105 | 106 | 107 | def main(): 108 | parser = argparse.ArgumentParser() 109 | parser.add_argument('-p', '--print-spectrum', action='store_true', 110 | help="print colours rather than code") 111 | 112 | args = parser.parse_args() 113 | 114 | if args.print_spectrum: 115 | print_spectrum() 116 | print 117 | print_spectrum_24() 118 | else: 119 | print "#ifndef __GENERATED_COLOUR_SPECTRUM__" 120 | print "#define __GENERATED_COLOUR_SPECTRUM__" 121 | print "/* Generated by scripts/colour-gen, for ./text-cross-entropy.c */" 122 | print 123 | write_spectrum() 124 | print 125 | write_spectrum_24() 126 | print 127 | print "#endif" 128 | main() 129 | -------------------------------------------------------------------------------- /scripts/compare-nets: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import sys, os 4 | import argparse 5 | import subprocess 6 | import json 7 | import re 8 | import random 9 | # like this: 10 | # 11 | #for f in $dir/*.net; 12 | # do ./classify-test \ 13 | # -t ~/nocturnal-calls/morepork-test-with-intensities.txt \ 14 | # -d ~/nocturnal-calls/morepork-minutes -C 500 --summary\ 15 | # --ignore-start=1 -v0 --min-call-intensity=4.3 -f $f \ 16 | # >> morepork-scores-intensity-4.3-best.txt; \ 17 | #done 18 | # 19 | #but if $f has already been done, ignore it. 20 | 21 | 22 | def load_filelist(filename, strip=True, exclude=set()): 23 | files = set() 24 | if strip: 25 | bn = os.path.basename 26 | else: 27 | def bn(x): 28 | return x 29 | 30 | if filename is not None: 31 | f = open(filename) 32 | for line in f: 33 | if line[0] == '{': 34 | fn = bn(json.loads(line)['filename']) 35 | if fn not in exclude: 36 | files.add(fn) 37 | else: 38 | fns = [bn(x) for x in line.split() if x.endswith('.net')] 39 | files.update(x for x in fns if os.path.basename(x) not in exclude) 40 | files = set(x.encode('utf-8') for x in files) 41 | return files 42 | 43 | def main(): 44 | parser = argparse.ArgumentParser() 45 | parser.add_argument('-C', '--first-n', type=int, default=0, 46 | help="use this many audio files") 47 | parser.add_argument('-c', '--n-nets', type=int, default=0, 48 | help="test this many nets") 49 | parser.add_argument('-t', '--timings', action='append', 50 | help='read timings and intensities from here') 51 | parser.add_argument('-d', '--audio-directory', action='append', 52 | help='find audio in this directory') 53 | parser.add_argument('-D', '--net-directory', action='append', 54 | help='test all nets in this directory') 55 | parser.add_argument('-I', '--include-file', 56 | help='test nets named in this file') 57 | parser.add_argument('-x', '--exclude-file', 58 | help='exclude nets named in this file') 59 | parser.add_argument('-S', '--shuffle', action='store_true', 60 | help='process files in a more random order') 61 | parser.add_argument('-r', '--include-regex', 62 | help='included files must meet this regex') 63 | parser.add_argument('-i', '--min-call-intensity', type=float, default=0.0, 64 | help='only consider calls with this intensity or higher') 65 | parser.add_argument('--ignore-start', type=float, default=0.0, 66 | help="ignore this many seconds at start of each file") 67 | parser.add_argument('--presence-index', type=int, default=5, 68 | help="Nth highest value to use as presence indicator") 69 | args = parser.parse_args() 70 | #print >>sys.stderr, args 71 | #start from git root 72 | gitroot = os.path.abspath(os.path.dirname((os.path.dirname(__file__)))) 73 | os.chdir(gitroot) 74 | excluded_files = load_filelist(args.exclude_file) 75 | files = load_filelist(args.include_file, strip=False, exclude=excluded_files) 76 | if args.net_directory: 77 | for d in args.net_directory: 78 | files.update(os.path.join(d, x) for x in os.listdir(d) 79 | if x.endswith('.net') and 80 | os.path.basename(x) not in excluded_files) 81 | 82 | if args.include_regex: 83 | print >>sys.stderr, "restricting to regex /%s/" % args.include_regex 84 | r = re.compile(args.include_regex) 85 | files = set(x for x in files if r.search(x)) 86 | print >>sys.stderr, "looking at %d files" % len(files) 87 | cmd = ["./classify-test", "--summary", "-v0"] 88 | for arg, val in (("-C", args.first_n), 89 | ("--ignore-start", args.ignore_start), 90 | ("--presence-index", args.presence_index), 91 | ("--min-call-intensity", args.min_call_intensity)): 92 | if val is not None: 93 | cmd.append(arg) 94 | cmd.append(str(val)) 95 | 96 | for arg, vals in (("-t", args.timings), 97 | ("-d", args.audio_directory)): 98 | if vals: 99 | for v in vals: 100 | cmd.append(arg) 101 | cmd.append(v) 102 | 103 | files = list(files) 104 | if args.shuffle: 105 | random.shuffle(files) 106 | if args.n_nets: 107 | del files[args.n_nets:] 108 | 109 | for fn in files: 110 | #print >> sys.stderr, ' '.join(cmd) 111 | try: 112 | subprocess.check_call(cmd + ["-f", fn]) 113 | except subprocess.CalledProcessError, e: 114 | print >> sys.stderr, "could not process %s" % fn 115 | 116 | main() 117 | -------------------------------------------------------------------------------- /scripts/find-best-nets: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import json 4 | import sys 5 | from collections import defaultdict 6 | import argparse 7 | 8 | 9 | SMALLER_IS_BETTER = ['mean_dfb', 'min_dfb', 'cross_entropy', 'briar'] 10 | SMALLER_IS_BETTER += ['p.' + x for x in SMALLER_IS_BETTER] 11 | 12 | SCORE_ADJUST = { 13 | 'auc': 2.0, 14 | 'p.auc': 2.0, 15 | 'f1': 0.5, 16 | 'p.f1': 0.5, 17 | 'mcc': 0.5, 18 | 'p.mcc': 0.5, 19 | } 20 | 21 | PRESENCE_WEIGHT = 1.0 22 | 23 | def calc_score(k, v, best): 24 | v = v or 1e-10 25 | best = best or 1e-10 26 | score = best / v if k in SMALLER_IS_BETTER else v / best 27 | score *= SCORE_ADJUST.get(k, 1.0) 28 | if k.startswith('p.'): 29 | score *= PRESENCE_WEIGHT 30 | return score 31 | 32 | def print_summary(nets, n, cutoff): 33 | """print a semi-human-readable summary of the indicator metrics.""" 34 | n = min(n, len(nets)) 35 | summary = defaultdict(int) 36 | for k in sorted(nets[0].keys()): 37 | if k == 'filename': 38 | continue 39 | data = [(x[k], x['filename']) for x in nets] 40 | data.sort(reverse=(k not in SMALLER_IS_BETTER)) 41 | print 42 | print k 43 | print "=" * len(k) 44 | best = data[0][0] 45 | for i in range(n): 46 | v, fn = data[i] 47 | summary[fn] += calc_score(k, v, best) 48 | print "%2d: %#.4g %s" % (i + 1, v, fn) 49 | 50 | best = [(v, k) for k, v in summary.iteritems()] 51 | best.sort(reverse=True) 52 | print "\nsummary\n========" 53 | oldv = None 54 | for i, x in enumerate(best): 55 | v, fn = x 56 | if v < cutoff: 57 | break 58 | if v != oldv: 59 | rank = "%3d:" % (i + 1) 60 | else: 61 | rank = " " 62 | oldv = v 63 | print "%s %#.3g %s" % (rank, v, fn) 64 | 65 | 66 | def print_list(nets, n, cutoff): 67 | """print only filenames of nets that appear in the top n of at least 68 | cutoff indicators.""" 69 | n = min(n, len(nets)) 70 | summary = defaultdict(int) 71 | for k in nets[0].keys(): 72 | if k == 'filename': 73 | continue 74 | data = [(x[k], x['filename']) for x in nets] 75 | data.sort(reverse=(k not in SMALLER_IS_BETTER)) 76 | best = data[0][0] 77 | for v, fn in data[:n]: 78 | summary[fn] += calc_score(k, v, best) 79 | 80 | best = [(v, k) for k, v in summary.iteritems()] 81 | best.sort(reverse=True) 82 | for i, x in enumerate(best): 83 | v, fn = x 84 | if v < cutoff: 85 | break 86 | print fn 87 | 88 | 89 | def summarise_nets(fn, n=10, list_only=False, cutoff=2): 90 | nets = [] 91 | f = open(fn) 92 | for line in f: 93 | stats = json.loads(line) 94 | nets.append(stats) 95 | f.close() 96 | try: 97 | if list_only: 98 | print_list(nets, n, cutoff) 99 | else: 100 | print_summary(nets, n, cutoff) 101 | except IOError, e: 102 | #don't print traceback when piping to e.g. head 103 | if 'Broken pipe' not in str(e): 104 | raise 105 | 106 | 107 | 108 | def main(): 109 | parser = argparse.ArgumentParser() 110 | parser.add_argument('-n', '--top-n', type=int, default=10, 111 | help="list this many top nets per category") 112 | parser.add_argument('-l', '--list-only', action="store_true", 113 | help="only list the top filenames") 114 | parser.add_argument('-c', '--cutoff', type=float, default=1e-20, 115 | help='how many top-n listings a net needs' 116 | ' to be in summary') 117 | parser.add_argument('-p', '--presence-adjust', type=float, default=1.0, 118 | help="the relative importance of presence parameters") 119 | 120 | parser.add_argument('file', help='JSON file to use') 121 | args = parser.parse_args() 122 | 123 | global PRESENCE_WEIGHT 124 | PRESENCE_WEIGHT = args.presence_adjust 125 | 126 | summarise_nets(args.file, n=args.top_n, list_only=args.list_only, 127 | cutoff=args.cutoff) 128 | 129 | main() 130 | -------------------------------------------------------------------------------- /scripts/find-character-set: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | from collections import Counter 4 | import sys 5 | import argparse 6 | import math 7 | import re 8 | 9 | def main(): 10 | parser = argparse.ArgumentParser() 11 | parser.add_argument('-q', '--short', action="store_true", 12 | help="one-line view") 13 | parser.add_argument('-c', '--case-sensitive', action="store_true", 14 | help="treat upper and lower case separately") 15 | parser.add_argument('-t', '--threshold', type=float, default=1e-4, 16 | help='threshold of alphabet inclusion') 17 | parser.add_argument('-u', '--utf8', action="store_true", 18 | help='parse as UTF-8') 19 | parser.add_argument('-p', '--preserve-whitespace', action="store_true", 20 | help="don't collapse whitespace to single space") 21 | parser.add_argument('files', help='process these files', nargs='+') 22 | args = parser.parse_args() 23 | 24 | chars = Counter() 25 | for fn in args.files: 26 | if args.utf8: 27 | f = open(fn, "r") 28 | s = f.read() 29 | else: 30 | f = open(fn, "rb") 31 | #stupid Python 3. mac_roman seems to be defined for all bytes, 32 | #unlike latin_1 and cp1252. 33 | s = f.read().decode('mac_roman') 34 | 35 | if not args.preserve_whitespace: 36 | s = ' '.join(s.split()) 37 | if not args.case_sensitive: 38 | s = s.lower() 39 | f.close() 40 | chars.update(s) 41 | 42 | threshold = math.ceil(args.threshold * len(s)) 43 | if args.short: 44 | s = ('%-30s: {{%s}} {{%s}}' % (' '.join(args.files), 45 | '' .join(c for c, n in chars.most_common() 46 | if n >= threshold), 47 | '' .join(c for c, n in chars.most_common() 48 | if n < threshold))) 49 | print(str(s.encode('mac_roman'))[2: -1]) 50 | else: 51 | i = 1 52 | print(len(s), len(chars)) 53 | for c, n in chars.most_common(): 54 | print("%2d '%s' \\x%02x %10d %10.3g" % (i, c, ord(c.encode('mac_roman')), 55 | n, float(n) / len(s))) 56 | i += 1 57 | 58 | s = (''.join(c for c, n in chars.most_common())) 59 | print(c_formatted_string(s)) 60 | s = '' .join(c for c, n in chars.most_common() 61 | if n >= threshold) 62 | print('.alphabet = "%s",' % c_formatted_string(s)) 63 | s = '' .join(c for c, n in chars.most_common() 64 | if n < threshold) 65 | print('.collapse = "%s",' % c_formatted_string(s)) 66 | 67 | 68 | 69 | def c_formatted_string(s): 70 | s = str(s.encode('mac_roman'))[2: -1] 71 | s = s.replace('"', '\\"') 72 | return re.sub(r'(\\x[0-9a-f][0-9a-f])([0-9a-f])', 73 | r'\1""\2', s, flags=re.IGNORECASE) 74 | 75 | 76 | main() 77 | -------------------------------------------------------------------------------- /scripts/find-feature-means: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys 3 | import numpy as np 4 | import argparse 5 | 6 | def load(fn): 7 | # Find the number of fields in order to skip the first column 8 | f = open(fn) 9 | line = f.next() 10 | f.close() 11 | fields = line.count(',') + 1 12 | usecols = range(1, fields) 13 | 14 | return np.loadtxt(fn, delimiter=',', usecols=usecols) 15 | 16 | def bounded_identity(bound): 17 | def big_or_zero(x): 18 | if abs(x) >= bound: 19 | return x 20 | return 0.0 21 | return big_or_zero 22 | 23 | def bounded_recip(bound): 24 | def recip_or_bound(x): 25 | if abs(x) >= bound: 26 | return 1.0 / abs(x) 27 | return 1.0 / bound 28 | return recip_or_bound 29 | 30 | def main(): 31 | parser = argparse.ArgumentParser() 32 | parser.add_argument("csv_file", 33 | help="where to find the raw numbers") 34 | parser.add_argument("-F", "--feature-offsets", action="store_true", 35 | help="output in style needed by classify plugin") 36 | parser.add_argument("-b", "--bound", type=float, default=1e-4, 37 | help="disregard values smaller than this") 38 | args = parser.parse_args() 39 | data = load(args.csv_file) 40 | if args.feature_offsets: 41 | for name, raw_seq, f in ( 42 | ('median', np.median(data, axis=0), bounded_identity(args.bound)), 43 | ('mean', np.mean(data, axis=0), bounded_identity(args.bound)), 44 | ('std', np.std(data, axis=0), bounded_recip(args.bound)) 45 | ): 46 | print name, 47 | seq = [] 48 | for x in raw_seq: 49 | seq.append(f(x)) 50 | i = len(seq) - 1 51 | while i and not seq[i]: 52 | i -= 1 53 | 54 | print ':'.join('%.3g' % x for x in seq[:i + 1]) 55 | else: 56 | print "median" 57 | print np.median(data, axis=0) 58 | print "mean" 59 | print np.mean(data, axis=0) 60 | print "standard deviation" 61 | print np.std(data, axis=0) 62 | 63 | main() 64 | -------------------------------------------------------------------------------- /scripts/reduce-video.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ORIG=$1 4 | DEST=$2 5 | FPS=$3 6 | WIDTH=$4 7 | HEIGHT=$5 8 | DEINT=$6 9 | 10 | if [[ -n "$DEINT" ]]; then 11 | VF="-vf mcdeint" 12 | fi 13 | # -an for silent 14 | ffmpeg -r 20 -i "$ORIG" $VF -vcodec mpeg4 -s "${WIDTH}x${HEIGHT}" $DEST 15 | -------------------------------------------------------------------------------- /scripts/rnn_describe: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #Copyright 2014 Douglas Bagnall LGPL 3 | 4 | import sys 5 | import struct, array 6 | import pycdb 7 | import argparse 8 | 9 | def read_int(n, key, unsigned=False): 10 | v = n[key] 11 | format = ' bh i q'[len(v)] 12 | if format == ' ': 13 | raise ValueError("%r has invalid size %d" % (key, len(v))) 14 | if unsigned: 15 | format = format.upper() 16 | return struct.unpack('<' + format, v)[0] 17 | 18 | def read_uint(*args): 19 | return read_int(*args, unsigned=True) 20 | 21 | def read_flags(n, key): 22 | v = n[key] 23 | bitlen = len(v) * 8 24 | i = read_int(n, key, True) 25 | bits = [] 26 | for x in range(bitlen): 27 | if x and not x & 7: 28 | bits.append('|') 29 | bits.append('.X'[i & 1]) 30 | i >>= 1 31 | return ''.join(bits[::-1]) 32 | 33 | def read_float(n, key): 34 | v = n[key] 35 | format = ' f d'[len(v)] 36 | if format == ' ': 37 | raise ValueError("%r has invalid size %d" % (key, len(v))) 38 | return struct.unpack('<' + format, v)[0] 39 | 40 | def read_float_array(n, key): 41 | pass 42 | 43 | def read_string(n, key): 44 | return '\n' + n[key] 45 | 46 | def read_string_urlencoded(n, key): 47 | from urllib import unquote 48 | return '\n' + unquote(n[key]) 49 | 50 | def read_rng(*args): 51 | pass 52 | 53 | KEYS = {'net.i_size': (read_int, True), 54 | 'net.h_size': (read_int, True), 55 | 'net.o_size': (read_int, True), 56 | 'net.input_size': (read_int, True), 57 | 'net.hidden_size': (read_int, True), 58 | 'net.output_size': (read_int, True), 59 | 'net.ih_size': (read_int, True), 60 | 'net.ho_size': (read_int, True), 61 | 'net.bias': (read_int, True), #obsolete 62 | 'net.flags': (read_flags, True), 63 | 'net.ih_weights': (read_float_array, False), 64 | 'net.ho_weights': (read_float_array, False), 65 | 'bottom_layer.input_size': (read_int, True), 66 | 'bottom_layer.output_size': (read_int, True), 67 | 'bottom_layer.i_size': (read_int, True), 68 | 'bottom_layer.o_size': (read_int, True), 69 | 'bottom_layer.overlap': (read_int, True), 70 | 'bottom_layer.weights': (read_float_array, False), 71 | 'net.generation': (read_uint, True), 72 | 'bptt.depth': (read_int, True), 73 | 'bptt.index': (read_int, True), 74 | 'bptt.ih_scale': (read_float, True), 75 | 'bptt.ho_scale': (read_float, True), 76 | 'bptt.momentum': (read_float, True), 77 | 'bptt.momentum_weight': (read_float, True), 78 | 'bptt.learn_rate': (read_float, True), 79 | 'bptt.min_error_factor': (read_float, True), 80 | 'net.rng': (read_rng, False), 81 | 82 | 'save_format_version': (read_int, True), 83 | 'net.metadata': (read_string, True), 84 | 'net.presynaptic_noise': (read_float, True), 85 | 'net.activation': (read_int, True), 86 | 87 | #older versions 88 | 'bias': (read_int, True), 89 | 'depth': (read_int, True), 90 | 'flags': (read_int, True), 91 | 'h_size': (read_int, True), 92 | 'hidden_size': (read_int, True), 93 | 'ho_size': (read_int, True), 94 | 'ho_weights': (read_float_array, False), 95 | 'i_size': (read_int, True), 96 | 'ih_size': (read_int, True), 97 | 'ih_weights': (read_float_array, False), 98 | 'input_size': (read_int, True), 99 | 'o_size': (read_int, True), 100 | 'output_size': (read_int, True), 101 | 'generation': (read_int, True), 102 | } 103 | 104 | 105 | def read(fn, *keys): 106 | f = open(fn) 107 | n = pycdb.Reader(f.read()) 108 | f.close() 109 | 110 | if not keys: 111 | keys = sorted(n.keys()) 112 | for k in keys: 113 | try: 114 | fn, show = KEYS[k] 115 | if show: 116 | print "%24s: %s" % (k, fn(n, k)) 117 | except KeyError: 118 | print "unknown key '%s'" % k 119 | 120 | def main(): 121 | parser = argparse.ArgumentParser() 122 | parser.add_argument('-u', '--url-decode', action='store_true', 123 | help='treat metadata as url-encoded text') 124 | parser.add_argument('file', 125 | help='net file to look at') 126 | parser.add_argument('keys', nargs='*', 127 | help='keys to describe (default: all)') 128 | 129 | args = parser.parse_args() 130 | if args.url_decode: 131 | KEYS['net.metadata'] = (read_string_urlencoded, True) 132 | 133 | read(args.file, *args.keys) 134 | 135 | main() 136 | -------------------------------------------------------------------------------- /scripts/rnnca-train.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | HERE=$(git rev-parse --show-toplevel) 4 | cd "$HERE" 5 | 6 | URI=file://$HERE/test-video/lagos-288-192-20.avi 7 | RECORD_URI=file://$HERE/test-video/lagos-288-192-20-short.avi 8 | #URI=file://$HERE/test-video/rochester-288-192.avi 9 | 10 | MOMENTUM=0.95 11 | MSS=4000 12 | LR=1e-6 13 | HS=59 14 | BASENAME=$(basename $URI) 15 | PATTERN=Y00011102C000111 16 | 17 | for i in {1..20}; do 18 | if [[ $i < 3 ]]; then 19 | LR=1e-5 20 | elif [[ $i < 6 ]]; then 21 | LR=3e-6 22 | elif [[ $i < 10 ]]; then 23 | LR=1e-6 24 | else 25 | LR=3e-7 26 | fi 27 | for j in {1..5}; do 28 | echo training $i.$j LR $LR 29 | time gst-launch-1.0 --gst-plugin-path=plugins \ 30 | uridecodebin name=src uri=$URI \ 31 | ! videoscale method=nearest-neighbour ! videoconvert \ 32 | ! video/x-raw, format=I420, width=288, height=192, framerate=20/1 \ 33 | ! rnnca momentum-soft-start=$MSS momentum=$MOMENTUM learn-rate=$LR \ 34 | hidden-size=$HS log-file=rnnca.log training=1 playing=0 offsets=$PATTERN \ 35 | ! fakesink 36 | done 37 | 38 | echo video $i 39 | time gst-launch-1.0 --gst-plugin-path=plugins \ 40 | avimux name=mux ! \ 41 | filesink location=examples/rnnca-$BASENAME-$PATTERN-$HS-$i.avi \ 42 | uridecodebin name=src uri=$RECORD_URI \ 43 | ! videoscale method=nearest-neighbour ! videoconvert \ 44 | ! video/x-raw, format=I420, width=288, height=192, framerate=20/1 \ 45 | ! rnnca momentum-soft-start=$MSS momentum=$MOMENTUM learn-rate=$LR \ 46 | hidden-size=$HS log-file=rnnca.log training=1 playing=1 offsets=$PATTERN \ 47 | ! videoconvert ! x264enc bitrate=512 ! mux. 48 | done 49 | -------------------------------------------------------------------------------- /scripts/shuffler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import random 4 | import os, sys 5 | import re 6 | 7 | MIN_PARA_SIZE = 300 8 | 9 | def shuffle(text, min_para_size): 10 | paragraphs = re.split("\r?\n\s*\n\s*", text) 11 | snippets = [] 12 | p2 = None 13 | for p in paragraphs: 14 | if p2 is not None: 15 | p2 += "\n" + p 16 | if len(p2) > min_para_size: 17 | snippets.append(p2) 18 | p2 = None 19 | elif len(p) > min_para_size: 20 | snippets.append(p) 21 | else: 22 | p2 = p 23 | 24 | random.shuffle(snippets) 25 | return '\n\n'.join(snippets) 26 | 27 | 28 | def main(infile, outfile, min_para_size=MIN_PARA_SIZE): 29 | f = open(infile) 30 | s = shuffle(f.read(), int(min_para_size)) 31 | f.close() 32 | f = open(outfile, 'w') 33 | f.write(s) 34 | f.close() 35 | 36 | main(*sys.argv[1:]) 37 | -------------------------------------------------------------------------------- /scripts/switch-blas.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sudo update-alternatives --config libblas.so 3 | sudo update-alternatives --config libblas.so.3 4 | sudo update-alternatives --config liblapack.so 5 | sudo update-alternatives --config liblapack.so.3 6 | -------------------------------------------------------------------------------- /setup-charmodel.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | from distutils.extension import Extension 3 | 4 | setup( 5 | name = "charmodel", 6 | ext_modules = [ 7 | Extension( 8 | name="charmodel", 9 | library_dirs=['.'], 10 | libraries=['cdb'], 11 | sources=["py-recur-text.c", "charmodel-multi-predict.c", 12 | "charmodel-predict.c", "charmodel-init.c", "recur-nn.c", 13 | "recur-nn-io.c", "recur-nn-init.c"], 14 | depends=["charmodel.h", "pgm_dump.h", "badmaths.h", 15 | "charmodel-helpers.h", "path.h", "py-recur-text.h", 16 | "recur-common.h", "recur-nn.h", "colour.h"], # ..., etc. 17 | include_dirs = ["."], 18 | define_macros = [('_GNU_SOURCE', None), 19 | ('VECTOR', None)], 20 | extra_compile_args=['-march=native', '-ggdb', '-std=gnu11', '-I.', 21 | '-Wall', '-O3', '-ffast-math', '-fno-inline', 22 | '-DVECTOR', 23 | '-DPIC', 24 | ], 25 | language="c" 26 | ) 27 | ] 28 | ) 29 | -------------------------------------------------------------------------------- /setup-rnnumpy.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | from distutils.extension import Extension 3 | import numpy as np 4 | 5 | setup( 6 | name = "rnnumpy", 7 | ext_modules = [ 8 | Extension( 9 | name="rnnumpy", 10 | library_dirs=['.'], 11 | libraries=['cdb'], 12 | sources=["py-recur-numpy.c", "recur-nn.c", "colour.c", 13 | "recur-nn-io.c", "recur-nn-init.c"], 14 | depends=["pgm_dump.h", "badmaths.h", "colour.h", 15 | "path.h", #"py-recur-numpy.h", 16 | "recur-common.h", "recur-nn.h"], 17 | include_dirs = [".", np.get_include()], 18 | define_macros = [('_GNU_SOURCE', None), 19 | ('VECTOR', None)], 20 | extra_compile_args = ['-march=native', '-ggdb', '-std=gnu11', 21 | '-I.', '-Wall', '-O3', '-ffast-math', 22 | '-fno-inline', '-DVECTOR', '-DPIC', 23 | ], 24 | language="c" 25 | ) 26 | ] 27 | ) 28 | -------------------------------------------------------------------------------- /startup/README: -------------------------------------------------------------------------------- 1 | The files here help run Recur video plugins in kiosk/gallery mode. 2 | Do this from the repository root: 3 | 4 | make startup/rnnca.desktop 5 | # or 6 | make startup/recur.desktop 7 | 8 | and copy the desktop file to ~/.config/autostart on a Linux desktop. 9 | The next time you log in, the thing will start up automatically. but 10 | be warned that it will start up in full screen and *REALLY* ANNOYING 11 | if you do it on your work machine. It is for exhibition machines only. 12 | -------------------------------------------------------------------------------- /startup/recur-startup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sleep 5 4 | 5 | #move the mouse pointer to the side 6 | xdotool mousemove 2000 300 7 | 8 | mkdir -p nets 9 | mkdir -p images 10 | 11 | [[ "$DISPLAY" ]] || export DISPLAY=:0 12 | 13 | while true; do 14 | for x in *.net; do 15 | cp "$x" nets/$(date +%Y-%m-%d-%H-%M-%S)-$x 16 | done 17 | ./gtk-recur -f 2>> exhibition.log 18 | ./gtk-recur -f 2>> exhibition.log 19 | ./gtk-recur -f 2>> exhibition.log 20 | done 21 | -------------------------------------------------------------------------------- /startup/recur.desktop.template: -------------------------------------------------------------------------------- 1 | [Desktop Entry] 2 | Name=Recur Launcher 3 | Comment="Put this in ~/.config/autostart to launch Recur automatically" 4 | Exec="{recur-root}/startup/recur-startup.sh" 5 | Path="{recur-root}" 6 | Terminal=false 7 | Type=Application 8 | Categories= 9 | -------------------------------------------------------------------------------- /startup/rnnca-startup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sleep 5 4 | 5 | #move the mouse pointer to the side, just in case 6 | xdotool mousemove 2000 300 7 | 8 | mkdir -p nets 9 | mkdir -p images 10 | 11 | [[ "$DISPLAY" ]] || export DISPLAY=:0 12 | 13 | while true; do 14 | for x in rnnca*.net; do 15 | cp "$x" nets/$(date +%Y-%m-%d-%H-%M-%S)-$x 16 | done 17 | ./rnnca-player -f 2>> rnnca-exhibition.log 18 | done 19 | -------------------------------------------------------------------------------- /startup/rnnca.desktop.template: -------------------------------------------------------------------------------- 1 | [Desktop Entry] 2 | Name=RNNCA Launcher 3 | Comment="Put this in ~/.config/autostart to launch RNNCA automatically" 4 | Exec="{recur-root}/startup/rnnca-startup.sh" 5 | Path="{recur-root}" 6 | Terminal=false 7 | Type=Application 8 | Categories= 9 | -------------------------------------------------------------------------------- /test/charmodel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import sys, os 4 | 5 | HERE = os.path.dirname(os.path.dirname(__file__)) 6 | sys.path.insert(0, HERE) 7 | TEST_DIR = os.path.join(HERE, 'test') 8 | 9 | import charmodel 10 | from json import loads 11 | from functools import partial 12 | 13 | 14 | def parse_json_utf8(s): 15 | d = loads(s) 16 | d2 = {} 17 | for k, v in d.items(): 18 | k = k.encode('utf8') 19 | if isinstance(v, unicode): 20 | v = v.encode('utf8') 21 | d2[k] = v 22 | return d2 23 | 24 | 25 | def print_net(n): 26 | print "Net %s" % n 27 | for k in dir(n): 28 | print "%20s: %s" % (k, getattr(n, k)) 29 | 30 | print "Alphabet %s" % n.alphabet 31 | for k in dir(n.alphabet): 32 | print "%20s: %s" % (k, getattr(n.alphabet, k)) 33 | 34 | 35 | def main(): 36 | fn = TEST_DIR + '/multi-text-6c34c563i73-h99-o3650.net' 37 | n = charmodel.Net.load(fn, parse_json_utf8) 38 | print_net(n) 39 | 40 | main() 41 | -------------------------------------------------------------------------------- /test/multi-text-6c34c563i73-h99-o3650.net: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/douglasbagnall/recur/fc1eb38e14be07a17f2fd765e9925873556d2546/test/multi-text-6c34c563i73-h99-o3650.net -------------------------------------------------------------------------------- /test/rnnumpy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import sys, os 4 | import time 5 | import numpy as np 6 | 7 | HERE = os.path.dirname(os.path.dirname(__file__)) 8 | sys.path.insert(0, HERE) 9 | TEST_DIR = os.path.join(HERE, 'test') 10 | 11 | import rnnumpy 12 | 13 | def pair_123(n): 14 | # we go: 15 | # 1 0 0 1 0 16 | # 0 1 0 1 0 17 | # 0 0 1 1 0 18 | # 0 0 0 0 1 19 | # 20 | # After a diagonal stripe, the answer is 1. 21 | # There can be randomly arranged 1s in non-stripes. 22 | 23 | inputs = np.zeros((n, 3), dtype=np.float32) 24 | targets = np.zeros((n, 2), dtype=np.float32) 25 | choices = np.random.randint(10, size=(n)) 26 | 27 | # blank out the first 3 rows, to prevent edge errors 28 | choices[:3] = 9 29 | 30 | inputs[choices < 3, 0] = 1.0 31 | inputs[1:, 1] = inputs[:-1, 0] 32 | inputs[2:, 2] = inputs[:-2, 0] 33 | 34 | for i in range(0, 2): 35 | inputs[choices == 3 + i, i] = 1.0 36 | 37 | 38 | targets[3:, 1] = inputs[:-3, 0] * inputs[1:-2, 1] * inputs[2:-1, 2] 39 | targets[:, 0] = 1.0 - targets[:, 1] 40 | return inputs, targets 41 | 42 | 43 | def test_123(): 44 | inputs, targets = pair_123(20000) 45 | test, answers = pair_123(5000) 46 | 47 | for h in 7, 11, 19, 39, 79: 48 | print '-' * 77 49 | print "size", h 50 | t1 = time.time() 51 | net = rnnumpy.Net(inputs.shape[1], 52 | h, 53 | targets.shape[1], 54 | learn_rate=0.1, 55 | log_file="rnnnumpy.log", 56 | bptt_depth=5) 57 | net.train(inputs, targets, 1) 58 | 59 | t2 = time.time() 60 | print "training took %.4f" % (t2 - t1) 61 | 62 | results = net.classify(test) 63 | 64 | t1 = time.time() 65 | print "classifying took %.4f" % (t1 - t2) 66 | 67 | 68 | diff = np.abs(answers - results) 69 | 70 | mse = (diff ** 2).mean() 71 | 72 | ones_col = np.ones((diff.shape[0]), dtype=np.float32) 73 | 74 | stuck1 = np.zeros(diff.shape, dtype=np.float32) 75 | stuck2 = np.zeros(diff.shape, dtype=np.float32) 76 | stuck1[:, 0] = ones_col 77 | stuck2[:, 1] = ones_col 78 | 79 | wrong = np.sum(diff > 0.5) / 2 80 | 81 | if wrong and True: 82 | where_wrong = np.where(diff > 0.5) 83 | print where_wrong 84 | for y, x in zip(*where_wrong): 85 | if x: 86 | print test[y - 4:y], results[y, x], answers[y, x] 87 | 88 | 89 | rand = np.random.random(diff.shape) 90 | 91 | 92 | print "number wrong:" , wrong 93 | 94 | print "squared error", mse 95 | print "%10s %10s %10s" % ("pattern", "vs results", "vs answers") 96 | print "%10s %10.2f %10.2f" % ("[1 0]", 97 | ((results - stuck1) ** 2).mean(), 98 | ((answers - stuck1) ** 2).mean()) 99 | print "%10s %10.2f %10.2f" % ("[0 1]", 100 | ((results - stuck2) ** 2).mean(), 101 | ((answers - stuck2) ** 2).mean()) 102 | print "%10s %10.2f %10.2f" % ("random", 103 | ((results - rand) ** 2).mean(), 104 | ((answers - rand) ** 2).mean()) 105 | 106 | 107 | def main(): 108 | test_123() 109 | 110 | main() 111 | -------------------------------------------------------------------------------- /test/test_dct.c: -------------------------------------------------------------------------------- 1 | #include "mfcc.h" 2 | #include "pgm_dump.h" 3 | 4 | #define CYCLES 1000000 5 | 6 | int 7 | main(void){ 8 | #define len 13 9 | float input[len] = {3, 1, 4, 1, 5, 9, 2, 6, 5, 4}; 10 | float output[len]; 11 | float recycle[len]; 12 | float sum; 13 | START_TIMER(naive); 14 | sum = 0; 15 | for (int i = 0; i < CYCLES; i++){ 16 | recur_dct(input, output, len); 17 | sum += output[3]; 18 | } 19 | DEBUG("sum %g", sum); 20 | DEBUG_TIMER(naive); 21 | START_TIMER(cache); 22 | sum = 0; 23 | for (int i = 0; i < CYCLES; i++){ 24 | recur_dct_cached(input, output, len); 25 | sum += output[3]; 26 | } 27 | DEBUG("sum %g", sum); 28 | DEBUG_TIMER(cache); 29 | 30 | for (int i = 0; i < 10; i++){ 31 | recur_dct(input, output, len); 32 | //recur_dct_cached(input, output, len); 33 | recur_idct(output, recycle, len); 34 | } 35 | for (int i = 0; i < len; i++){ 36 | printf("in %6.6f out %6.6f rec %6.6f\n", 37 | input[i], output[i], recycle[i]); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /test/test_fb_backprop.c: -------------------------------------------------------------------------------- 1 | /*Copyright 2014 Douglas Bagnall LGPL 2 | 3 | This tests a toy classifier on the "FizzBuzz" task. 4 | 5 | The FB1 and FB2 constants below set the period, with 3 and 5 being standard. 6 | */ 7 | 8 | #define PERIODIC_PGM_DUMP 1 9 | #include "recur-nn.h" 10 | #include "pgm_dump.h" 11 | #include 12 | #include "path.h" 13 | #include "badmaths.h" 14 | #include 15 | #include 16 | #include 17 | 18 | #define NET_LOG_FILE "bptt.log" 19 | #define BPTT_DEPTH 30 20 | #define CONFAB_SIZE 70 21 | #define LEARN_RATE 0.002 22 | #define HIDDEN_SIZE 39 23 | #define INPUT_SIZE 2 24 | #define BATCH_SIZE 1 25 | 26 | #define MOMENTUM 0.95 27 | #define MOMENTUM_WEIGHT RNN_MOMENTUM_WEIGHT 28 | #define FB1 5 29 | #define FB2 3 30 | #define K_STOP 2000 31 | 32 | static const char CONFAB_LUT[] = " |S$"; 33 | //static const char CONFAB_LUT[] = " /\\X"; 34 | 35 | static inline void 36 | load_char_input(RecurNN *net, int c){ 37 | net->input_layer[0] = 1.0f; 38 | net->real_inputs[0] = (c & 1); 39 | net->real_inputs[1] = (c & 2) >> 1; 40 | } 41 | 42 | static inline float 43 | net_error_bptt(RecurNN *net, float *error, int c, int next){ 44 | load_char_input(net, c); 45 | float *answer = rnn_opinion(net, NULL, net->presynaptic_noise); 46 | error[0] = (next & 1) - (answer[0] > 0); 47 | error[1] = (!!(next & 2)) - (answer[1] > 0); 48 | return (fabsf(error[0]) + fabsf(error[1])) * 0.5; 49 | } 50 | 51 | static float 52 | sgd_one(RecurNN *net, const int current, const int next, uint batch_size){ 53 | RecurNNBPTT *bptt = net->bptt; 54 | rnn_bptt_advance(net); 55 | float sum = net_error_bptt(net, bptt->o_error, current, next); 56 | rnn_bptt_calculate(net, batch_size); 57 | return sum; 58 | } 59 | 60 | static inline int 61 | char_opinion(RecurNN *net, int c){ 62 | load_char_input(net, c); 63 | float * answer = rnn_opinion(net, NULL, 0); 64 | int a = ((answer[1] > 0) << 1) | (answer[0] > 0); 65 | return a; 66 | } 67 | 68 | static int 69 | confabulate(RecurNN *net, char *text, int len, uint c){ 70 | int i; 71 | c = MAX(c, 3); 72 | for (i = 0; i < len; i++){ 73 | c = char_opinion(net, c); 74 | text[i] = CONFAB_LUT[c]; 75 | } 76 | return text[i]; 77 | } 78 | 79 | #define FIZZBUZZ(x, a, b) ((((x) % (a) == 0) << 1) + ((x) % (b) == 0)); 80 | 81 | void 82 | epoch(RecurNN *net, const int len){ 83 | int i; 84 | char confab[CONFAB_SIZE + 1]; 85 | confab[CONFAB_SIZE] = 0; 86 | int current; 87 | int next = FIZZBUZZ(0, FB1, FB2); 88 | for(i = 1; i < len; i++){ 89 | current = next; 90 | next = FIZZBUZZ(i, FB1, FB2); 91 | float error = sgd_one(net, current, next, BATCH_SIZE); 92 | net->bptt->learn_rate *= 0.999999; 93 | if ((i & 1023) == 0){ 94 | int k = i >> 10; 95 | confabulate(net, confab, CONFAB_SIZE, next); 96 | DEBUG("%3dk %.2f %.4f confab: '%s'", k, error, net->bptt->learn_rate, confab); 97 | if (PERIODIC_PGM_DUMP){ 98 | RecurNNBPTT *bptt = net->bptt; 99 | dump_colour_weights_autoname(net->ih_weights, net->h_size, net->i_size, 100 | "ih-k", net->generation); 101 | dump_colour_weights_autoname(net->ih_weights, net->h_size, net->hidden_size, 102 | "hh-k", net->generation); 103 | dump_colour_weights_autoname(net->ho_weights, net->o_size, net->h_size, 104 | "ho-k", net->generation); 105 | dump_colour_weights_autoname(bptt->ih_momentum, net->h_size, net->i_size, 106 | "ih-momentum-k", net->generation); 107 | dump_colour_weights_autoname(bptt->ih_delta, net->h_size, net->i_size, 108 | "ih-tmp-k", net->generation); 109 | dump_colour_weights_autoname(bptt->ho_momentum, net->o_size, net->h_size, 110 | "ho-momentum-k", net->generation); 111 | } 112 | if (K_STOP && k > K_STOP) 113 | exit(0); 114 | } 115 | } 116 | } 117 | 118 | void dump_parameters(void){ 119 | DEBUG( 120 | "HIDDEN_SIZE %d\n" 121 | "LEARN_RATE %f\n" 122 | "MOMENTUM %f\n" 123 | , 124 | HIDDEN_SIZE, 125 | LEARN_RATE, 126 | MOMENTUM); 127 | } 128 | 129 | int 130 | main(void){ 131 | dump_parameters(); 132 | feenableexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW); 133 | RecurNN *net = rnn_new(INPUT_SIZE, HIDDEN_SIZE, 134 | INPUT_SIZE, RNN_NET_FLAG_STANDARD, 135 | 1, NET_LOG_FILE, BPTT_DEPTH, LEARN_RATE, 0, MOMENTUM, 136 | RNN_RELU); 137 | rnn_randomise_weights_auto(net); 138 | START_TIMER(epoch); 139 | epoch(net, 5000000); 140 | DEBUG_TIMER(epoch); 141 | rnn_delete_net(net); 142 | } 143 | -------------------------------------------------------------------------------- /test/test_mdct.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Douglas Bagnall LGPL */ 2 | #include "recur-common.h" 3 | #include 4 | #include "badmaths.h" 5 | #include 6 | #include 7 | #include 8 | #include "mdct.h" 9 | #include "window.h" 10 | 11 | //#define TEST_AUDIO_FILE "test-audio/494-slower-huge.wav" 12 | #define TEST_AUDIO_FILE "test-audio/371-slower-huge.wav" 13 | #define DEST_AUDIO_FILE "mdct-out.wav" 14 | 15 | #define WINDOW_BITS 9 16 | #define WINDOW_SIZE (1 << WINDOW_BITS) 17 | #define WINDOW_NO (WINDOW_BITS - 6) 18 | 19 | 20 | static inline void 21 | apply_window(float *restrict samples, const float *restrict window){ 22 | int half_window = WINDOW_SIZE / 2; 23 | int i; 24 | for (i = 0; i < half_window; i++){ 25 | samples[i] *= window[i]; 26 | } 27 | for (i = 0; i < half_window; i++){ 28 | samples[half_window + i] *= window[half_window - 1 - i]; 29 | } 30 | } 31 | 32 | 33 | int 34 | main(void){ 35 | int i; 36 | const int half_window = WINDOW_SIZE / 2; 37 | s16 buffer[half_window]; 38 | s16 buffer2[half_window]; 39 | float *pcm1 = calloc(WINDOW_SIZE, sizeof(float)); 40 | float *pcm2 = calloc(WINDOW_SIZE, sizeof(float)); 41 | //float *pcm3 = calloc(WINDOW_SIZE, sizeof(float)); 42 | 43 | mdct_lookup m_look; 44 | mdct_init(&m_look, WINDOW_SIZE); 45 | const float *window = _vorbis_window_get(WINDOW_NO); 46 | 47 | FILE *inf = fopen_or_abort(TEST_AUDIO_FILE, "r"); 48 | FILE *outf = fopen_or_abort(DEST_AUDIO_FILE, "w"); 49 | 50 | /*wav header is 44 bytes. reuse it as-is.*/ 51 | int len = fread(buffer, 1, 44, inf); 52 | fwrite(buffer, 1, 44, outf); 53 | 54 | for (;;){ 55 | len = fread(buffer, sizeof(s16), half_window, inf); 56 | DEBUG("read %d bytes", len); 57 | if (len == 0) 58 | break; 59 | if(len < half_window){ 60 | memset(buffer + len, 0, 61 | (half_window - len) * sizeof(s16)); 62 | } 63 | for(i = 0; i < half_window; i++){ 64 | float s = buffer[i] / 32768.0f; 65 | pcm1[i] = s * window[i]; 66 | pcm2[half_window + i] = s * window[half_window - 1 - i]; 67 | } 68 | mdct_forward(&m_look, pcm2, pcm2); 69 | mdct_backward(&m_look, pcm2, pcm2); 70 | 71 | for(i = 0; i < half_window; i++){ 72 | float s = (pcm1[half_window + i] * window[half_window - 1 - i] + 73 | pcm2[i] * window[i]); 74 | //float s = pcm1[half_window + i] + pcm2[i]; 75 | buffer2[i] = s * 32767.0f; 76 | } 77 | len = fwrite(buffer2, sizeof(s16), half_window, outf); 78 | DEBUG("wrote %d bytes buffer[0] %d buffer2[0] %d", len, buffer[0], buffer2[0]); 79 | float *tmp = pcm1; 80 | pcm1 = pcm2; 81 | pcm2 = tmp; 82 | } 83 | free(pcm1); 84 | free(pcm2); 85 | } 86 | -------------------------------------------------------------------------------- /test/test_mfcc_bins.c: -------------------------------------------------------------------------------- 1 | #include "recur-common.h" 2 | #include "mfcc.h" 3 | 4 | #define N_FFT_BINS 32 5 | #define MIN_FREQ 0 6 | #define MAX_FREQ (AUDIO_RATE * 0.499) 7 | #define KNEE_FREQ 700 8 | #define FOCUS_FREQ 600 9 | #define AUDIO_RATE 8000 10 | #define WINDOW_SIZE 512 11 | 12 | int 13 | main(void){ 14 | /*This draws the images of the audio bins */ 15 | recur_audio_binner_new(WINDOW_SIZE, RECUR_WINDOW_HANN, 16 | N_FFT_BINS, MIN_FREQ, MAX_FREQ, KNEE_FREQ, FOCUS_FREQ, 17 | AUDIO_RATE, 1.0, 2); 18 | } 19 | -------------------------------------------------------------------------------- /test/test_mfcc_table.c: -------------------------------------------------------------------------------- 1 | #include "recur-common.h" 2 | #include "mfcc.h" 3 | #include "pgm_dump.h" 4 | #include 5 | #include "path.h" 6 | 7 | #define RECUR_N_FFT_BINS 40 8 | #define RECUR_MFCC_MIN_FREQ 20 9 | #define RECUR_MFCC_MAX_FREQ (RECUR_AUDIO_RATE * 0.499) 10 | #define RECUR_MFCC_KNEE_FREQ 700 11 | #define RECUR_AUDIO_RATE 16000 12 | 13 | 14 | int 15 | main(void){ 16 | int fps_d = 1; 17 | int fps_n = 25; 18 | int expected_samples = RECUR_AUDIO_RATE * fps_d / fps_n; 19 | int min_window_size = ROUND_UP_4(expected_samples * 3 / 2); 20 | int window_size = gst_fft_next_fast_length (min_window_size); 21 | 22 | RecurAudioBinSlope * bins = recur_bin_slopes_new(RECUR_N_FFT_BINS, 23 | window_size / 2, 24 | RECUR_MFCC_MIN_FREQ, 25 | RECUR_MFCC_MAX_FREQ, 26 | 700, 0, 27 | RECUR_AUDIO_RATE 28 | ); 29 | 30 | printf("window size %d bins %d\n", window_size, 31 | RECUR_N_FFT_BINS); 32 | 33 | u8 *img = malloc_aligned_or_die(RECUR_N_FFT_BINS * window_size / 2); 34 | 35 | int i, j; 36 | float mul = 0.0; 37 | RecurAudioBinSlope *bin = &bins[0]; 38 | for (j = bin->left; j < bin->right; j++){ 39 | mul += bin->slope; 40 | img[j] = mul * 255; 41 | } 42 | printf("%2d. left %3d right %3d slope %f mul at end %f\n", 43 | 0, bin->left, bin->right, bin->slope, mul); 44 | 45 | for (i = 1; i < RECUR_N_FFT_BINS; i++){ 46 | bin = &bins[i]; 47 | mul = 0.0; 48 | for (j = bin->left; j < bin->right; j++){ 49 | mul += bin->slope; 50 | img[i * window_size / 2 + j] = mul * 255; 51 | img[(i - 1) * window_size / 2 + j] = (1.0 - mul) * 255; 52 | } 53 | printf("%2d. left %3d right %3d slope %f mul at end %f\n", 54 | i, bin->left, bin->right, bin->slope, mul); 55 | } 56 | bin = &bins[i]; 57 | mul = 0.0; 58 | for (j = bin->left; j < bin->right; j++){ 59 | mul += bin->slope; 60 | //img[i * window_size / 2 + j] = mul * 255; 61 | img[(i - 1) * window_size / 2 + j] = (1.0 - mul) * 255; 62 | } 63 | printf("%2d. left %3d right %3d slope %f mul at end %f\n", 64 | i, bin->left, bin->right, bin->slope, mul); 65 | pgm_dump(img, window_size / 2, RECUR_N_FFT_BINS, DEBUG_IMAGE_DIR "/mfcc4.pgm"); 66 | } 67 | -------------------------------------------------------------------------------- /test/test_poisson.c: -------------------------------------------------------------------------------- 1 | //#include "recur-nn.h" 2 | #include "recur-rng.h" 3 | #include "pgm_dump.h" 4 | #include 5 | #include 6 | #include 7 | 8 | 9 | static inline void 10 | bernoulli_jumps(rand_ctx *rng, int *jumps, int len, double p){ 11 | int i; 12 | double recip_lognp = 1.0 / log(1.0 - p); 13 | for (i = 0; i < len; i++){ 14 | jumps[i] = ceil(log(1.0 - rand_double(rng)) * recip_lognp); 15 | } 16 | } 17 | 18 | static inline void 19 | bernoulli_jumps_f(rand_ctx *rng, int *jumps, int len, float p){ 20 | int i; 21 | float recip_lognp = 1.0f / logf(1.0f - p); 22 | float r[len]; 23 | randomise_float_array(rng, r, len); 24 | for (i = 0; i < len; i++){ 25 | jumps[i] = ceilf(logf(1.0f - r[i]) * recip_lognp); 26 | } 27 | } 28 | 29 | 30 | 31 | #define WIDTH 800 32 | #define HEIGHT 600 33 | 34 | #define N_BINS 100 35 | 36 | int 37 | main(void){ 38 | rand_ctx rng; 39 | init_rand64(&rng, 2); 40 | u8 * im = malloc_aligned_or_die(WIDTH * HEIGHT); 41 | memset(im, 0, WIDTH * HEIGHT); 42 | int jumps[WIDTH]; 43 | double rate = 0.25; 44 | for (int y = 0; y < HEIGHT; y++){ 45 | for (int i = 0; i < 80 / rate; i++){ 46 | bernoulli_jumps_f(&rng, jumps, WIDTH, rate); 47 | for (int x = *jumps - 1, j = 1; x < WIDTH && j < WIDTH; j++){ 48 | if (im[y * WIDTH + x] < 254) 49 | im[y * WIDTH + x] += 2; 50 | else 51 | im[y * WIDTH + x] = 255; 52 | x += jumps[j]; 53 | } 54 | } 55 | rate *= 0.99; 56 | } 57 | int bins[N_BINS]; 58 | memset(bins, 0, sizeof(bins)); 59 | for (int i = 0; i < WIDTH; i++){ 60 | int j = jumps[i]; 61 | //fprintf(stderr, "%i ", j); 62 | if (j < N_BINS) 63 | bins[j]++; 64 | } 65 | int scale = 1; 66 | int max = 0; 67 | for (int i = 0; i < N_BINS; i++){ 68 | scale = MAX(scale, bins[i] / 90); 69 | if (bins[i]) 70 | max = i; 71 | } 72 | for (int i = 0; i <= max; i++){ 73 | fprintf(stderr, "\n%3d ", i); 74 | for (int j = 0; j < bins[i]; j+= scale){ 75 | fputc('+', stderr); 76 | } 77 | } 78 | fputc('\n', stderr); 79 | pgm_dump(im, WIDTH, HEIGHT, "poisson.pgm"); 80 | } 81 | -------------------------------------------------------------------------------- /test/test_random.c: -------------------------------------------------------------------------------- 1 | #include "recur-rng.h" 2 | #include "pgm_dump.h" 3 | #include 4 | #include 5 | 6 | 7 | #define PLOT_RESOLUTION 100 8 | #define OVERSAMPLE_BITS 12 9 | #define GAUSSIAN_SAMPLES (50000 << OVERSAMPLE_BITS) 10 | 11 | static inline int 12 | scale_and_confine(float f, float scale, int centre, int min, int max){ 13 | int i; 14 | f *= scale; 15 | f += centre + 0.5; 16 | i = (int)f; 17 | if (i < min) 18 | i = min; 19 | if (i > max) 20 | i = max; 21 | return i; 22 | } 23 | 24 | #define ITERATIONS 100000000 25 | 26 | void time_fn(rand_ctx *rng, void (*fn)(rand_ctx *, float *, float *, float), 27 | const char *name){ 28 | float var_a = 0; 29 | float var_b = 0; 30 | float mean_a = 0; 31 | float mean_b = 0; 32 | float a, b, d; 33 | START_TIMER(fn); 34 | for (int i = 1; i <= ITERATIONS; i++){ 35 | fn(rng, &a, &b, 1e-7f); 36 | d = a - mean_a; 37 | mean_a += d / i; 38 | var_a += d * (a - mean_a); 39 | 40 | d = b - mean_b; 41 | mean_b += d / i; 42 | var_b += d * (b - mean_a); 43 | } 44 | var_a /= ITERATIONS; 45 | var_b /= ITERATIONS; 46 | DEBUG("%s %d iterations", name, ITERATIONS); 47 | DEBUG_TIMER(fn); 48 | DEBUG("mean a %4e b %4e", mean_a, mean_b); 49 | DEBUG("stddev a %4e b %4e", sqrtf(var_a), sqrtf(var_b)); 50 | //DEBUG("sum a %4e b %4e", sum_a, sum_b); 51 | } 52 | 53 | static void 54 | singlecheap(rand_ctx *rng, float *restrict a, 55 | float *restrict b, const float deviation){ 56 | *a = cheap_gaussian_noise(rng) * deviation; 57 | *b = cheap_gaussian_noise(rng) * deviation; 58 | } 59 | 60 | void test_gauss2(rand_ctx *rng){ 61 | time_fn(rng, doublecheap_gaussian_noise_f, "doublecheap"); 62 | time_fn(rng, doublecheap2_gaussian_noise_f, "doublecheap2"); 63 | time_fn(rng, singlecheap, "singlecheap"); 64 | } 65 | 66 | 67 | static void 68 | test_gauss(rand_ctx *rng, int res, int samples, int bits, 69 | void (*fn)(rand_ctx *, float *, float *, float), 70 | char *filename){ 71 | int i, x, y; 72 | float a, b; 73 | int len = res * 8; 74 | float zero = res * 4; 75 | int bins[len]; 76 | memset(bins, 0, sizeof(bins)); 77 | for (i = 0; i < samples / 2; i++){ 78 | fn(rng, &a, &b, 0.8f); 79 | bins[scale_and_confine(a, res, zero, 0, len - 1)]++; 80 | bins[scale_and_confine(b, res, zero, 0, len - 1)]++; 81 | } 82 | 83 | int max = 0; 84 | for (i = 0; i < len; i++){ 85 | if (bins[i] > max) 86 | max = bins[i]; 87 | } 88 | int height = (max >> bits) + 1; 89 | u8 *plot = malloc((height + 1) * len); 90 | memset (plot, 0, (height + 1) * len); 91 | i = 0; 92 | for (y = height; y >= 0; y--){ 93 | for (x = 0; x < len; x++, i++){ 94 | int h = bins[x] >> bits; 95 | if (h == y){ 96 | plot[i] = (bins[x] >> (bits - 8)) & 255; 97 | } 98 | else if (h > y){ 99 | plot[i] = 255; 100 | } 101 | } 102 | } 103 | #if 0 104 | for (x = 0; x < len; x++, i++){ 105 | DEBUG("%i %i", x, bins[x]); 106 | } 107 | #endif 108 | 109 | pgm_dump(plot, len, height, filename); 110 | free(plot); 111 | } 112 | 113 | 114 | int 115 | main(void){ 116 | rand_ctx rng; 117 | init_rand64(&rng, 12345); 118 | //test_gauss2(&rng); 119 | test_gauss(&rng, PLOT_RESOLUTION, GAUSSIAN_SAMPLES, OVERSAMPLE_BITS, 120 | doublecheap_gaussian_noise_f, 121 | "doublecheap.pgm"); 122 | test_gauss(&rng, PLOT_RESOLUTION, GAUSSIAN_SAMPLES, OVERSAMPLE_BITS, 123 | doublecheap2_gaussian_noise_f, 124 | "doublecheap2.pgm"); 125 | test_gauss(&rng, PLOT_RESOLUTION, GAUSSIAN_SAMPLES, OVERSAMPLE_BITS, 126 | singlecheap, "singlecheap.pgm"); 127 | } 128 | -------------------------------------------------------------------------------- /test/test_rescale.c: -------------------------------------------------------------------------------- 1 | #include "rescale.h" 2 | #include "pgm_dump.h" 3 | #include 4 | #include "path.h" 5 | 6 | #define IMG_DIR TEST_DATA_DIR 7 | 8 | static const int sizes[] = { 9 | 512, 384, 10 | 399, 399, 11 | 1024, 768, 12 | 999, 750, 13 | 64, 48, 14 | 30, 20, 15 | 128, 96 16 | }; 17 | 18 | static const char *names[] = { 19 | "zoo-800-600", 20 | "zoo-400-300", 21 | "zoo-odd-size", 22 | "butterfly" 23 | }; 24 | 25 | 26 | void 27 | compare_to_truth(const u8 *im, const int w, const int h, 28 | const char *im_name, const char *truth_name, const char *method_name){ 29 | char name[100]; 30 | snprintf(name, sizeof(name), "%s/generated/%s-%dx%d-%s.pgm", 31 | IMG_DIR, im_name, w, h, truth_name); 32 | 33 | Image * im2 = recur_load_pgm_file(name); 34 | if (im2){ /*in most cases, the file will be missing */ 35 | double delta = 0; 36 | double delta2 = 0; 37 | for(int k = 0; k < w * h; k++){ 38 | int d = im[k] - im2->data[k]; 39 | delta2 += d * d; 40 | delta += abs(d); 41 | } 42 | printf("%11s vs %5s delta %.2f rms %.2f\n", 43 | truth_name, method_name, 44 | delta / (w * h), sqrt(delta2 / (w * h))); 45 | free(im2); 46 | } 47 | } 48 | 49 | 50 | static void 51 | report_delta(u8 *mem, u8 *mem2, int w, int h, 52 | const char *name, const char *msg){ 53 | double delta2 = 0.0; 54 | double delta = 0.0; 55 | for(int k = 0; k < w * h; k++){ 56 | int d = mem[k] - mem2[k]; 57 | delta2 += d * d; 58 | delta += abs(d); 59 | } 60 | printf("%12s %4dx%-4d %30s %.2f rms %.2f\n", 61 | name, w, h, msg, delta / (w * h), sqrt(delta2 / (w * h))); 62 | } 63 | 64 | int 65 | main(void){ 66 | uint i, j; 67 | char name[100]; 68 | for (j = 0; j < sizeof(names) / sizeof(names[0]); j++){ 69 | snprintf(name, sizeof(name), "%s/%s.pgm", IMG_DIR, names[j]); 70 | Image * im = recur_load_pgm_file(name); 71 | snprintf(name, sizeof(name), "/tmp/%s-dup.pgm", names[j]); 72 | pgm_dump(im->data, im->width, im->height, name); 73 | for (i = 0; i < sizeof(sizes) / sizeof(sizes[0]); i += 2){ 74 | int w = sizes[i]; 75 | int h = sizes[i + 1]; 76 | u8 *mem = malloc(w * h); 77 | 78 | snprintf(name, sizeof(name), IMG_DIR "/output/%s-%dx%d.pgm", names[j], w, h); 79 | recur_adaptive_downscale(im->data, im->width, im->height, im->width, 80 | mem, w, h, w); 81 | pgm_dump(mem, w, h, name); 82 | 83 | snprintf(name, sizeof(name), IMG_DIR "/output/%s-%dx%d-exact.pgm", 84 | names[j], w, h); 85 | START_TIMER(exact); 86 | recur_exact_downscale(im->data, im->width, im->height, im->width, 87 | mem, w, h, w); 88 | DEBUG_TIMER(exact); 89 | pgm_dump(mem, w, h, name); 90 | 91 | u8* mem2 = malloc(w * h); 92 | snprintf(name, sizeof(name), IMG_DIR "/output/%s-%dx%d-skipping.pgm", 93 | names[j], w, h); 94 | START_TIMER(skipping); 95 | recur_skipping_downscale(im->data, im->width, im->height, im->width, 96 | mem2, w, h, w); 97 | DEBUG_TIMER(skipping); 98 | 99 | pgm_dump(mem2, w, h, name); 100 | 101 | /*float -- convert to and from u8 */ 102 | float* float_mem = malloc(w * h * sizeof(float)); 103 | u8* mem3 = malloc(w * h); 104 | float* float_im = malloc(im->width * im->height * sizeof(float)); 105 | for (uint k = 0; k < im->width * im->height; k++){ 106 | float_im[k] = im->data[k]; 107 | } 108 | snprintf(name, sizeof(name), IMG_DIR "/output/%s-%dx%d-float.pgm", 109 | names[j], w, h); 110 | START_TIMER(float_); 111 | recur_float_downscale(float_im, im->width, im->height, im->width, 112 | float_mem, w, h, w); 113 | DEBUG_TIMER(float_); 114 | for (int k = 0; k < w * h; k++){ 115 | mem3[k] = float_mem[k]; 116 | } 117 | pgm_dump(mem3, w, h, name); 118 | free(float_mem); 119 | free(float_im); 120 | 121 | 122 | 123 | report_delta(mem, mem2, w, h, names[j], "skipping vs exact delta"); 124 | report_delta(mem, mem3, w, h, names[j], "float vs exact delta"); 125 | 126 | compare_to_truth(mem, w, h, names[j], "Magick", "exact"); 127 | compare_to_truth(mem, w, h, names[j], "PIL", "exact"); 128 | compare_to_truth(mem3, w, h, names[j], "Magick", "float"); 129 | compare_to_truth(mem3, w, h, names[j], "PIL", "float"); 130 | 131 | free(mem3); 132 | free(mem2); 133 | free(mem); 134 | } 135 | free(im); 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /test/test_simple_rescale.c: -------------------------------------------------------------------------------- 1 | #include "rescale.h" 2 | #include "pgm_dump.h" 3 | #include 4 | #include "path.h" 5 | 6 | static const char *names[] = { 7 | "cake", 8 | "zoo-odd-size", 9 | "butterfly" 10 | }; 11 | 12 | const int RECUR_INPUT_WIDTH = 300; 13 | const int RECUR_INPUT_HEIGHT = 200; 14 | 15 | 16 | static void 17 | save_scaled_image(u8 *im, int stride, int x, int y, int scale, const char *name){ 18 | float *dest = calloc(1, RECUR_INPUT_WIDTH * RECUR_INPUT_HEIGHT * sizeof(float)); 19 | 20 | recur_integer_downscale_to_float(im, dest, stride, x, y, 21 | RECUR_INPUT_WIDTH, RECUR_INPUT_HEIGHT, scale); 22 | 23 | pgm_dump_normalised_float(dest, RECUR_INPUT_WIDTH, RECUR_INPUT_HEIGHT, name); 24 | free(dest); 25 | } 26 | 27 | int 28 | main(void){ 29 | uint i, j; 30 | char name[100]; 31 | for (j = 0; j < sizeof(names) / sizeof(names[0]); j++){ 32 | snprintf(name, sizeof(name), "%s/%s.pgm", TEST_DATA_DIR, names[j]); 33 | Image * im = recur_load_pgm_file(name); 34 | for (i = 1; i < 6; i++){ 35 | snprintf(name, sizeof(name), "%s/generated/simple-%s-%d.pgm", TEST_DATA_DIR, names[j], i); 36 | save_scaled_image(im->data, im->width, 100, 100, i, name); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /test/test_utf8.c: -------------------------------------------------------------------------------- 1 | /* Copyright (C) 2014 Douglas Bagnall LGPL 2 | -*- coding: utf-8 -*- 3 | 4 | Tests SOME of the functions in ../utf8.h 5 | 6 | This file contains string literals encoded in utf-8. 7 | */ 8 | #include "../utf8.h" 9 | #include "../colour.h" 10 | 11 | static inline void 12 | dump_int_array(const int* a, int len){ 13 | for (int i = 0; i < len - 1; i++){ 14 | fprintf(stderr, "%d, ", a[i]); 15 | } 16 | if (len){ 17 | fprintf(stderr, "%d\n", a[len - 1]); 18 | } 19 | } 20 | 21 | static inline void 22 | dump_byte_array(const char* a, int len){ 23 | for (int i = 0; i < len - 1; i++){ 24 | fprintf(stderr, "%u, ", (u8)a[i]); 25 | } 26 | if (len){ 27 | fprintf(stderr, "%u\n", (u8)a[len - 1]); 28 | } 29 | } 30 | 31 | static const char *round_trip_strings[] = { 32 | "hello", 33 | "hēllo", 34 | "ā\n\t¬à⅜£ə⋄þÞøåơị̂·∴äċ ē¶ð…»“”⌷`wsΩąá-Āāēōī←", 35 | "顧己及人，請勿隨地大小二便", /*from http://languagelog.ldc.upenn.edu/nll/?p=13140*/ 36 | 37 | NULL 38 | }; 39 | 40 | static const char *bad_utf8_strings[] = { 41 | "should fail, bare continuation \x83", 42 | "incomplete character \xF3\x82 mid-string", 43 | "incomplete character at end \xF7\x9F", 44 | "\x89\x89\x89\x98\xa8\x89\x89\xc8\xd8\xd9 garbage at start", 45 | "\x80", "\x88", "\xee", "\xa0", /*random solitary high bytes*/ 46 | NULL 47 | }; 48 | 49 | const int MAXLEN = 1000; 50 | 51 | int 52 | test_utf8_should_fail() 53 | { 54 | int i; 55 | int errors = 0; 56 | int codepoints[MAXLEN + 1]; 57 | for (i = 0; ; i++){ 58 | const char *s = bad_utf8_strings[i]; 59 | if (!s){ 60 | break; 61 | } 62 | DEBUG("\nlooking at \"%s\"", s); 63 | int n_points = fill_codepoints_from_utf8(codepoints, MAXLEN, s); 64 | dump_int_array(codepoints, n_points + 1); 65 | const char *s2 = new_utf8_from_codepoints(codepoints, n_points); 66 | int diff = strcmp(s, s2); 67 | if (! diff){ 68 | errors++; 69 | DEBUG(C_RED "reconstruction works. it should fail" C_NORMAL); 70 | } 71 | else { 72 | DEBUG(C_GREEN "reconstruction rightly fails: " C_NORMAL 73 | "\"%s\"\noriginal len %lu, reconstructed %lu", 74 | s2, strlen(s), strlen(s2)); 75 | } 76 | 77 | free((void*)s2); 78 | } 79 | return errors; 80 | } 81 | 82 | 83 | 84 | int 85 | test_codepoint_round_trip(int(str2cp)(int*, int, const char*), 86 | char *(cp2new_str)(const int*, int)) 87 | { 88 | int i; 89 | int errors = 0; 90 | int codepoints[MAXLEN + 1]; 91 | for (i = 0; ; i++){ 92 | const char *s = round_trip_strings[i]; 93 | if (!s){ 94 | break; 95 | } 96 | int n_chars = strlen(s); 97 | DEBUG("\noriginal: %s", s); 98 | int n_points = str2cp(codepoints, MAXLEN, s); 99 | dump_int_array(codepoints, n_points); 100 | const char *s2 = cp2new_str(codepoints, n_points); 101 | int diff = strcmp(s, s2); 102 | if (diff){ 103 | errors++; 104 | int n_chars2 = strlen(s2); 105 | DEBUG(C_RED "reconstruction differs. len is %d" C_NORMAL, n_chars2); 106 | DEBUG("reconstructed: %s", s2); 107 | int n_points2 = str2cp(codepoints, MAXLEN, s2); 108 | dump_int_array(codepoints, n_points2); 109 | DEBUG("orignal bytes"); 110 | dump_byte_array(s, n_chars); 111 | DEBUG("reconstructed bytes"); 112 | dump_byte_array(s2, n_chars2); 113 | } 114 | DEBUG("points: %d, bytes: %d, %s diff: %d" C_NORMAL, n_points, n_chars, 115 | diff ? C_RED : C_GREEN, diff); 116 | free((void*)s2); 117 | } 118 | return errors; 119 | } 120 | 121 | int 122 | main(void){ 123 | int r = 0; 124 | 125 | DEBUG("\n" C_YELLOW "utf-8 reconstruction cycle" C_NORMAL); 126 | r += test_codepoint_round_trip(fill_codepoints_from_utf8, 127 | new_utf8_from_codepoints); 128 | 129 | DEBUG("\n" C_YELLOW "8 bit reconstruction" C_NORMAL); 130 | r += test_codepoint_round_trip(fill_codepoints_from_bytes, 131 | new_bytes_from_codepoints); 132 | 133 | DEBUG("\n" C_YELLOW "known bad strings where reconstruction should fail" C_NORMAL); 134 | r += test_utf8_should_fail(); 135 | 136 | if (r){ 137 | DEBUG(C_DARK_RED "\nTERRIBLE NEWS! " C_MAGENTA "%d " 138 | C_RED "failing tests" C_NORMAL, r); 139 | } 140 | else{ 141 | DEBUG("\n" C_CYAN "All tests pass. Perhaps there need to be more." C_NORMAL); 142 | } 143 | return r; 144 | } 145 | -------------------------------------------------------------------------------- /test/test_window.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Douglas Bagnall LGPL */ 2 | #include "recur-common.h" 3 | #include 4 | #include "badmaths.h" 5 | #include 6 | #include 7 | #include 8 | #include "mdct.h" 9 | #include "mfcc.h" 10 | #include "window.h" 11 | 12 | #define WINDOW_BITS 9 13 | #define WINDOW_SIZE (1 << WINDOW_BITS) 14 | #define WINDOW_NO (WINDOW_BITS - 6) 15 | 16 | 17 | static inline void 18 | compare_vorbis_window(int window_bits){ 19 | int i; 20 | int window_size = 1 << window_bits; 21 | int window_no = window_bits - 6; 22 | int half_window = window_size / 2; 23 | 24 | float *recur_window = malloc_aligned_or_die((window_size + 2) * sizeof(float)); 25 | recur_window_init(recur_window, window_size, RECUR_WINDOW_VORBIS, 1.0f); 26 | 27 | const float *vorbis_window = _vorbis_window_get(window_no); 28 | 29 | printf("window %d bits %d\n", window_bits, window_size); 30 | float worst_r = 0; 31 | float worst_rr = 0; 32 | for (i = 0; i < half_window; i++){ 33 | float v = vorbis_window[i]; 34 | float r = recur_window[i]; 35 | float rr = recur_window[window_size - 1 - i]; 36 | //printf("%3d vorbis %.3g mfcc %.3g %.3g diff %.3g %.3g\n", 37 | // i, v, r, rr, v - r, v - rr); 38 | worst_r = MAX(fabsf(v - r), worst_r); 39 | worst_rr = MAX(fabsf(v - rr), worst_rr); 40 | } 41 | printf("worst diff: left %g right %g\n\n", worst_r, worst_rr); 42 | free(recur_window); 43 | } 44 | 45 | 46 | 47 | int 48 | main(void){ 49 | int i; 50 | for (i = 6; i < 12; i++){ 51 | compare_vorbis_window(i); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /test/test_window_functions.c: -------------------------------------------------------------------------------- 1 | #include "mfcc.h" 2 | #include "pgm_dump.h" 3 | 4 | 5 | 6 | typedef struct { 7 | int window; 8 | char *name; 9 | } window_lut; 10 | 11 | #define HEIGHT 400 12 | #define WIDTH 300 13 | 14 | 15 | int 16 | main(void){ 17 | uint i; 18 | window_lut windows[] = { 19 | {RECUR_WINDOW_NONE, "none"}, 20 | {RECUR_WINDOW_HANN, "hann"}, 21 | {RECUR_WINDOW_VORBIS, "vorbis"}, 22 | {RECUR_WINDOW_MP3, "mp3"} 23 | }; 24 | float *mask = malloc(HEIGHT * sizeof(float)); 25 | u8 *pgm = malloc(HEIGHT * WIDTH); 26 | char name[200]; 27 | 28 | for (i = 0; i < sizeof(windows) / sizeof(windows[0]); i++){ 29 | window_lut *w = &windows[i]; 30 | recur_window_init(mask, HEIGHT, w->window, WIDTH - 1); 31 | memset(pgm, 0, HEIGHT * WIDTH); 32 | for (int y = 0; y < HEIGHT; y++){ 33 | for (int x = 0; x < mask[y]; x++){ 34 | pgm[y * WIDTH + x] = 100; 35 | } 36 | pgm[y * WIDTH + (int)mask[y]] = 255; 37 | } 38 | snprintf(name, sizeof(name), "%s/window-%s.pgm", IMAGE_DIR, w->name); 39 | 40 | pgm_dump(pgm, WIDTH, HEIGHT, name); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /text-classify-results.c: -------------------------------------------------------------------------------- 1 | /* Copyright (C) 2014 Douglas Bagnall GPL2+ 2 | 3 | Emit class probabilities for a set of files, based on a net trained by 4 | text-classify. 5 | 6 | Unlike most of the Recur repository, this file is licensed under the GNU 7 | General Public License, version 2 or greater. That is because it is linked to 8 | ccan/opt which is also GPL2+. 9 | 10 | Because of ccan/opt, --help will tell you something. 11 | */ 12 | 13 | #include "recur-nn.h" 14 | #include "recur-nn-helpers.h" 15 | #include "path.h" 16 | #include "badmaths.h" 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include "charmodel.h" 22 | #include "charmodel-helpers.h" 23 | #include "utf8.h" 24 | #include "colour.h" 25 | #include "opt-helpers.h" 26 | 27 | #define UNCLASSIFIED "*unclassified*" 28 | 29 | static int opt_ignore_start = 0; 30 | static int opt_min_length = 0; 31 | static char *opt_filename = NULL; 32 | static char *opt_test_dir = NULL; 33 | 34 | 35 | static struct opt_table options[] = { 36 | OPT_WITH_ARG("-m|--min-length=", opt_set_intval, opt_show_intval, &opt_min_length, 37 | "ignore texts shorter than this"), 38 | OPT_WITH_ARG("-i|--ignore-start", opt_set_intval, opt_show_intval, 39 | &opt_ignore_start, "don't classify this many first characters per block"), 40 | OPT_WITH_ARG("-f|--filename=", opt_set_charp, opt_show_charp, 41 | &opt_filename, "load/save net here"), 42 | OPT_WITH_ARG("-t|--test-dir=", opt_set_charp, opt_show_charp, 43 | &opt_test_dir, "emit scores for files in this directory"), 44 | 45 | OPT_WITHOUT_ARG("-h|--help", opt_usage_and_exit, 46 | "-f NET TEXTFILE [TEXTFILE...] \n" 47 | "Print classification probabilities of documents", 48 | "Print this message."), 49 | 50 | OPT_ENDTABLE 51 | }; 52 | 53 | 54 | int 55 | main(int argc, char *argv[]){ 56 | opt_register_table(options, NULL); 57 | if (!opt_parse(&argc, argv, opt_log_stderr)){ 58 | exit(1); 59 | } 60 | if (argc == 1){ 61 | DEBUG("No text files to evaluate!"); 62 | opt_usage_and_exit(argv[0]); 63 | } 64 | DEBUG("given %d arguments", argc - 1); 65 | 66 | RecurNN *net = rnn_load_net(opt_filename); 67 | RnnCharAlphabet *alphabet = rnn_char_new_alphabet_from_net(net); 68 | init_rand64_maybe_randomly(&net->rng, -1); 69 | 70 | int len = 0; 71 | int count = 0; 72 | 73 | if (opt_min_length <= opt_ignore_start){ 74 | DEBUG("hey! --min-length=%d <= --ignore-start=%d! Fixing.. now its %d.", 75 | opt_min_length, opt_ignore_start, opt_ignore_start + 1); 76 | opt_min_length = opt_ignore_start + 1; 77 | } 78 | float sum[net->output_size]; 79 | float sumsq[net->output_size]; 80 | float mean[net->output_size]; 81 | float stddev[net->output_size]; 82 | 83 | for (int i = 1; i < argc; i++){ 84 | const char *filename = argv[i]; 85 | u8* text = rnn_char_load_new_encoded_text(filename, alphabet, &len, 3); 86 | 87 | 88 | if (len >= opt_min_length){ 89 | memset(sum, 0, net->output_size * sizeof(float)); 90 | memset(sumsq, 0, net->output_size * sizeof(float)); 91 | int j, k; 92 | for (j = 0; j < opt_ignore_start; j++){ 93 | one_hot_opinion(net, text[j], 0); 94 | } 95 | for (j = opt_ignore_start; j < len; j++){ 96 | float *raw = one_hot_opinion(net, text[j], 0); 97 | float *answer = mean; 98 | softmax(answer, raw, net->output_size); 99 | for (k = 0; k < net->output_size; k++){ 100 | float a = answer[k]; 101 | sum[k] += a; 102 | sumsq[k] += a * a; 103 | } 104 | } 105 | for (k = 0; k < net->output_size; k++){ 106 | float m = sum[k] / (len - opt_ignore_start); 107 | stddev[k] = sqrtf(sumsq[k] / (len - opt_ignore_start) - m * m); 108 | mean[k] = m; 109 | } 110 | 111 | printf("%s mean: ", filename); 112 | for (k = 0; k < net->output_size; k++){ 113 | printf("%.3e ", mean[k]); 114 | } 115 | printf(" stddev: "); 116 | for (k = 0; k < net->output_size; k++){ 117 | printf("%.3e ", stddev[k]); 118 | } 119 | puts("\n"); 120 | } 121 | free(text); 122 | } 123 | DEBUG("processed %d texts", count); 124 | return 0; 125 | } 126 | -------------------------------------------------------------------------------- /text-confabulate.c: -------------------------------------------------------------------------------- 1 | /* Copyright (C) 2014 Douglas Bagnall GPL2+ 2 | 3 | This generates text using an RNN trained by text-predict. 4 | 5 | Unlike most of the Recur repository, this file is licensed under the GNU 6 | General Public License, version 2 or greater. That is because it is linked to 7 | ccan/opt which is also GPL2+. 8 | 9 | Because of ccan/opt, --help will tell you something. 10 | */ 11 | 12 | #include "recur-nn.h" 13 | #include "recur-nn-helpers.h" 14 | #include "charmodel.h" 15 | #include "colour.h" 16 | #include "opt-helpers.h" 17 | 18 | static char * opt_filename = NULL; 19 | static float opt_bias = 0; 20 | static uint opt_chars = 72; 21 | static char *opt_prefix = NULL; 22 | static bool opt_show_prefix = false; 23 | static char *opt_until = NULL; 24 | static char *opt_wait_for = NULL; 25 | static s64 opt_rng_seed = -1; 26 | 27 | static struct opt_table options[] = { 28 | OPT_WITH_ARG("-f|--filename=", opt_set_charp, opt_show_charp, &opt_filename, 29 | "load net from this file"), 30 | OPT_WITH_ARG("-B|--bias", opt_set_floatval, opt_show_floatval, 31 | &opt_bias, "bias toward probable characters " 32 | "(100 == deterministic)"), 33 | OPT_WITH_ARG("-n|--length=", opt_set_uintval, opt_show_uintval, 34 | &opt_chars, "confabulate this many characters"), 35 | OPT_WITH_ARG("-p|--prefix=", opt_set_charp, opt_show_charp, &opt_prefix, 36 | "seed the confabulator with this"), 37 | OPT_WITHOUT_ARG("--show-prefix", opt_set_bool, &opt_show_prefix, 38 | "print the prefix (if any)"), 39 | OPT_WITH_ARG("-u|--until=", opt_set_charp, opt_show_charp, &opt_until, 40 | "stop when this charactor appears"), 41 | OPT_WITH_ARG("--wait-for=", opt_set_charp, opt_show_charp, &opt_wait_for, 42 | "don't start until this charactor appears"), 43 | OPT_WITH_ARG("-r|--rng-seed=", opt_set_longval_bi, opt_show_longval_bi, 44 | &opt_rng_seed, "RNG seed (default: -1 for auto)"), 45 | 46 | OPT_WITHOUT_ARG("-h|--help", opt_usage_and_exit, 47 | ": Confabulate text using previously trained RNN", 48 | "Print this message."), 49 | OPT_ENDTABLE 50 | }; 51 | 52 | 53 | int 54 | main(int argc, char *argv[]){ 55 | opt_register_table(options, NULL); 56 | if (!opt_parse(&argc, argv, opt_log_stderr)){ 57 | exit(1); 58 | } 59 | if (argc > 1){ 60 | DEBUG("unused arguments:"); 61 | for (int i = 1; i < argc; i++){ 62 | DEBUG(" '%s'", argv[i]); 63 | } 64 | opt_usage(argv[0], NULL); 65 | } 66 | RecurNN *net = rnn_load_net(opt_filename); 67 | RnnCharAlphabet *alphabet = rnn_char_new_alphabet_from_net(net); 68 | 69 | init_rand64_maybe_randomly(&net->rng, opt_rng_seed); 70 | 71 | int prev_char = 0; 72 | 73 | if (opt_prefix){ 74 | int prefix_len; 75 | u8 *prefix_text = rnn_char_alloc_encoded_text(alphabet, 76 | opt_prefix, strlen(opt_prefix), &prefix_len, NULL, false); 77 | prev_char = rnn_char_prime(net, alphabet, prefix_text, prefix_len); 78 | if (opt_show_prefix){ 79 | printf(C_CYAN "%s" C_NORMAL, opt_prefix); 80 | } 81 | } 82 | 83 | /*XXX this could be done in small chunks */ 84 | int byte_len = opt_chars * 4 + 5; 85 | char *t = malloc(byte_len); 86 | 87 | int stop_point = -1; 88 | int start_point = -1; 89 | if (opt_until){ 90 | stop_point = rnn_char_get_codepoint(alphabet, opt_until); 91 | } 92 | if (opt_wait_for){ 93 | start_point = rnn_char_get_codepoint(alphabet, opt_wait_for); 94 | } 95 | 96 | rnn_char_confabulate(net, t, opt_chars, byte_len, 97 | alphabet, opt_bias, &prev_char, start_point, stop_point); 98 | fputs(t, stdout); 99 | fputs("\n", stdout); 100 | free(t); 101 | } 102 | --------------------------------------------------------------------------------