├── shell ├── .gitignore ├── rebar.config ├── src ├── b64fast.app.src └── b64fast.erl ├── LICENSE ├── README.md ├── test └── b64fast_tests.erl └── c_src ├── naive.h └── b64fast.c /shell: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ERL_LIBS=".:$ERL_LIBS" exec erl 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.so 3 | *.beam 4 | 5 | .eunit/ 6 | ebin/ 7 | _build/ 8 | c_src/b64fast.d 9 | rebar.lock 10 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {erl_opts, [debug_info]}. 2 | {deps, []}. 3 | 4 | {port_env, [{".*", "CFLAGS", "$CFLAGS -Wall -O2"}, {"ERL_LDFLAGS", " -L$ERL_EI_LIBDIR -lei"}]}. 5 | {port_specs, [{"priv/b64fast.so", ["c_src/*.c"]}]}. 6 | 7 | {plugins, [pc]}. 8 | 9 | {provider_hooks, 10 | [{pre, [ 11 | {compile, {pc, compile}}, 12 | {clean, {pc, clean}} 13 | ]} 14 | ]}. 15 | -------------------------------------------------------------------------------- /src/b64fast.app.src: -------------------------------------------------------------------------------- 1 | {application, b64fast, [ 2 | {description, "This NIF complements Erlang with a fast and optimized way of encoding and decoding bulk Base64 data. Compliant with RFC4648 - The Base16, Base32, and Base64 Data Encodings."}, 3 | {vsn, "0.2.3"}, 4 | {modules, []}, 5 | {registered, []}, 6 | {applications, [kernel, stdlib]}, 7 | {env, []}, 8 | {licenses, ["MIT"]}, 9 | {links, [{"GitHub", "https://github.com/zuckschwerdt/b64fast"}]}, 10 | {files, ["src", "c_src/*.[ch]", "test", "rebar.config", "README.md", "LICENSE"]} 11 | ]}. 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 Christian Zuckschwerdt 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /src/b64fast.erl: -------------------------------------------------------------------------------- 1 | -module(b64fast). 2 | -export([encode64/1, decode64/1]). 3 | -on_load(init/0). 4 | 5 | % The name of the application we're writing. This is the name 6 | % used for the Erlang .app file. 7 | 8 | -define(APPNAME, ?MODULE). 9 | 10 | % The name of the shared library we're going to load the NIF 11 | % code from. Defined in rebar.config as so_name. 12 | 13 | -define(LIBNAME, ?MODULE). 14 | 15 | % We could fall back to pure Erlang Base64 functions here. 16 | % Don't hide platform load errors for now, though. 17 | 18 | encode64(Bin) when is_binary(Bin) -> 19 | not_loaded(?LINE). 20 | 21 | decode64(Bin) when is_binary(Bin) -> 22 | not_loaded(?LINE). 23 | 24 | % Since we used init/0 in our -on_load() preprocessor directive, this 25 | % function will get called as the module is loaded. This is the perfect 26 | % place to load up our NIF shared library. Handily, the response of 27 | % erlang:load_nif/2 matches the return specification for -on_load() 28 | % functions. 29 | 30 | init() -> 31 | SoName = case code:priv_dir(?APPNAME) of 32 | {error, bad_name} -> 33 | case filelib:is_dir(filename:join(["..", priv])) of 34 | true -> 35 | filename:join(["..", priv, ?LIBNAME]); 36 | _ -> 37 | filename:join([priv, ?LIBNAME]) 38 | end; 39 | Dir -> 40 | filename:join(Dir, ?LIBNAME) 41 | end, 42 | erlang:load_nif(SoName, 0). 43 | 44 | % This is just a simple place holder. It mostly shouldn't ever be called 45 | % unless there was an unexpected error loading the NIF shared library. 46 | 47 | not_loaded(Line) -> 48 | erlang:nif_error({not_loaded, [{module, ?MODULE}, {line, Line}]}). 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Fast Base64 encoding/decoding NIF for Erlang 2 | ============================================ 3 | 4 | This NIF complements Erlang with a fast and optimized way of encoding and 5 | decoding bulk Base64 data. 6 | Compliant with RFC4648 - The Base16, Base32, and Base64 Data Encodings. 7 | 8 | Erlang is not well suited for fast sequential processing (s.a. Type B in 9 | http://jlouisramblings.blogspot.de/2013/07/problematic-traits-in-erlang.html). 10 | 11 | The NIF is well behaved and won't block the Erlang scheduler. It also doesn't 12 | need a dirty scheduler by breaking large amounts of data into chunks to limit 13 | the processing time to short timeslices. Big thanks to Steve Vinoski for making 14 | this easy (see https://github.com/vinoski/bitwise). 15 | 16 | Please note that the decoding doesn't handle whitespace, yet. 17 | 18 | Use an erl shell to quickly measure Base64 speed in native Erlang. 19 | 20 | ``` 21 | Data = binary:copy(<<"0123456789">>, 100000). % Create 1 MiB of data 22 | 23 | {Elapsed, Enc} = timer:tc(base64, encode, [Data]). 24 | Throughput_Encode = byte_size(Data) / Elapsed. % Throughput in MiB/s 25 | 26 | {Elapsed2, Dec} = timer:tc(base64, decode, [Enc]). 27 | Throughput_Decode = byte_size(Enc) / Elapsed2. % Throughput in MiB/s 28 | ``` 29 | 30 | And now try the (naive) NIF variant. 31 | 32 | ``` 33 | Data = binary:copy(<<"0123456789">>, 1000000). % 10 MiB of data 34 | 35 | {Elapsed, Enc} = timer:tc(b64fast, encode64, [Data]). 36 | Throughput_Encode = byte_size(Data) / Elapsed. % Throughput in MiB/s 37 | 38 | {Elapsed2, Dec} = timer:tc(b64fast, decode64, [Enc]). 39 | Throughput_Decode = byte_size(Enc) / Elapsed2. % Throughput in MiB/s 40 | ``` 41 | 42 | Quick comparison gives a speedup of x24 on encode and x23 on decode. 43 | -------------------------------------------------------------------------------- /test/b64fast_tests.erl: -------------------------------------------------------------------------------- 1 | -module(b64fast_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | 5 | bad_encode_test() -> 6 | ?assertException(error, badarg, b64fast:encode64( 42 )), 7 | ?assertException(error, badarg, b64fast:encode64( foo )), 8 | ?assertException(error, badarg, b64fast:encode64( "foo" )), 9 | ?assertException(error, badarg, b64fast:encode64( {foo} )). 10 | 11 | bad_decode_test() -> 12 | ?assertException(error, badarg, b64fast:decode64( 42 )), 13 | ?assertException(error, badarg, b64fast:decode64( foo )), 14 | ?assertException(error, badarg, b64fast:decode64( "foo" )), 15 | ?assertException(error, badarg, b64fast:decode64( {foo} )). 16 | 17 | encode_test() -> 18 | ?assert(b64fast:encode64(<< "zany" >>) =:= << "emFueQ==" >>), 19 | ?assert(b64fast:encode64(<< "zan" >>) =:= << "emFu" >>), 20 | ?assert(b64fast:encode64(<< "za" >>) =:= << "emE=" >>), 21 | ?assert(b64fast:encode64(<< "z" >>) =:= << "eg==" >>), 22 | ?assert(b64fast:encode64(<< >>) =:= << >>). 23 | 24 | decode_test() -> 25 | ?assert(b64fast:decode64(<< "emFueQ==" >>) =:= << "zany" >>), 26 | ?assert(b64fast:decode64(<< "emFu" >>) =:= << "zan" >>), 27 | ?assert(b64fast:decode64(<< "emE=" >>) =:= << "za" >>), 28 | ?assert(b64fast:decode64(<< "eg==" >>) =:= << "z" >>), 29 | ?assert(b64fast:decode64(<< >>) =:= << >>). 30 | 31 | % TODO: skip whitespace 32 | %padded_decode_test() -> 33 | % ?assert(b64fast:decode64(<< " emFu" >>) =:= << "zan" >>), 34 | % ?assert(b64fast:decode64(<< "em Fu" >>) =:= << "zan" >>), 35 | % ?assert(b64fast:decode64(<< "emFu " >>) =:= << "zan" >>), 36 | % ?assert(b64fast:decode64(<< " " >>) =:= << >>), 37 | % ?assert(b64fast:decode64(<< " =" >>) =:= << >>), 38 | % ?assert(b64fast:decode64(<< " ==" >>) =:= << >>), 39 | % ?assert(b64fast:decode64(<< "= " >>) =:= << >>), 40 | % ?assert(b64fast:decode64(<< "== " >>) =:= << >>). 41 | 42 | truncated_decode_test() -> 43 | ?assert(b64fast:decode64(<< "AAAA" >>) =:= << 0,0,0 >>), 44 | ?assert(b64fast:decode64(<< "AAA=" >>) =:= << 0,0 >>), 45 | ?assert(b64fast:decode64(<< "AAA" >>) =:= << 0,0 >>), 46 | ?assert(b64fast:decode64(<< "AA==" >>) =:= << 0 >>), 47 | ?assert(b64fast:decode64(<< "AA=" >>) =:= << 0 >>), 48 | ?assert(b64fast:decode64(<< "AA" >>) =:= << 0 >>), 49 | ?assert(b64fast:decode64(<< "A==" >>) =:= << >>), 50 | ?assert(b64fast:decode64(<< "A=" >>) =:= << >>), 51 | ?assert(b64fast:decode64(<< "A" >>) =:= << >>), 52 | ?assert(b64fast:decode64(<< "==" >>) =:= << >>), 53 | ?assert(b64fast:decode64(<< "=" >>) =:= << >>), 54 | ?assert(b64fast:decode64(<< >>) =:= << >>). 55 | 56 | backtoback_encode_test() -> 57 | Data = binary:copy(<<"0123456789">>, 100000), % 1 MiB of data 58 | ?assert(base64:encode(Data) =:= b64fast:encode64(Data)). 59 | 60 | backtoback_decode_test() -> 61 | Data = binary:copy(<<"0123456789">>, 100000), % 1 MiB of data 62 | % Enc = b64fast:encode64(Data), 63 | Enc = base64:encode(Data), 64 | ?assert(base64:decode(Enc) =:= b64fast:decode64(Enc)). 65 | 66 | speed_test() -> 67 | Data = binary:copy(<<"0123456789">>, 100000), % 1 MiB of data 68 | 69 | {Elapsed1, Enc1} = timer:tc(base64, encode, [Data]), 70 | io:fwrite(standard_error, "erlang encode ~B us ~f MiB/s~n", 71 | [Elapsed1, byte_size(Data) / Elapsed1]), 72 | 73 | {Elapsed2, _Dec1} = timer:tc(base64, decode, [Enc1]), 74 | io:fwrite(standard_error, "erlang decode ~B us ~f MiB/s~n", 75 | [Elapsed2, byte_size(Enc1) / Elapsed2]), 76 | 77 | {Elapsed3, Enc2} = timer:tc(b64fast, encode64, [Data]), 78 | io:fwrite(standard_error, "NIF encode ~B us ~f MiB/s~n", 79 | [Elapsed3, byte_size(Data) / Elapsed3]), 80 | 81 | {Elapsed4, _Dec2} = timer:tc(b64fast, decode64, [Enc2]), 82 | io:fwrite(standard_error, "NIF decode ~B us ~f MiB/s~n", 83 | [Elapsed4, byte_size(Enc2) / Elapsed4]). 84 | 85 | speed10_test() -> 86 | Data = binary:copy(<<"0123456789">>, 1000000), % 10 MiB of data 87 | 88 | {Elapsed3, Enc2} = timer:tc(b64fast, encode64, [Data]), 89 | io:fwrite(standard_error, "NIF encode ~B us ~f MiB/s~n", 90 | [Elapsed3, byte_size(Data) / Elapsed3]), 91 | 92 | {Elapsed4, _Dec2} = timer:tc(b64fast, decode64, [Enc2]), 93 | io:fwrite(standard_error, "NIF decode ~B us ~f MiB/s~n", 94 | [Elapsed4, byte_size(Enc2) / Elapsed4]). 95 | 96 | speed100_test() -> 97 | Data = binary:copy(<<"0123456789">>, 10000000), % 100 MiB of data 98 | 99 | {Elapsed3, Enc2} = timer:tc(b64fast, encode64, [Data]), 100 | io:fwrite(standard_error, "NIF encode ~B us ~f MiB/s~n", 101 | [Elapsed3, byte_size(Data) / Elapsed3]), 102 | 103 | {Elapsed4, _Dec2} = timer:tc(b64fast, decode64, [Enc2]), 104 | io:fwrite(standard_error, "NIF decode ~B us ~f MiB/s~n", 105 | [Elapsed4, byte_size(Enc2) / Elapsed4]). 106 | -------------------------------------------------------------------------------- /c_src/naive.h: -------------------------------------------------------------------------------- 1 | /* this is a naive byte-wise implementation of base64. */ 2 | /* use https://github.com/aklomp/base64 someday */ 3 | 4 | const unsigned char 5 | base64_table_enc[] = 6 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 7 | "abcdefghijklmnopqrstuvwxyz" 8 | "0123456789+/"; 9 | 10 | /* In the lookup table below, note that the value for '=' (character 61) is 11 | * 254, not 255. This character is used for in-band signaling of the end of 12 | * the datastream, and we will use that later. The characters A-Z, a-z, 0-9 13 | * and + / are mapped to their "decoded" values. The other bytes all map to 14 | * the value 255, which flags them as "invalid input". */ 15 | 16 | const unsigned char 17 | base64_table_dec[] = 18 | { 19 | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, /* 0..15 */ 20 | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, /* 16..31 */ 21 | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, 255, 63, /* 32..47 */ 22 | 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, 255, 254, 255, 255, /* 48..63 */ 23 | 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, /* 64..79 */ 24 | 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, 255, /* 80..95 */ 25 | 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, /* 96..111 */ 26 | 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255, 255, /* 112..127 */ 27 | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, /* 128..143 */ 28 | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 29 | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 30 | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 31 | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 32 | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 33 | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 34 | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 35 | }; 36 | 37 | static unsigned int 38 | base64_size(unsigned int bin_size) 39 | { 40 | unsigned int modulusLen = bin_size % 3; 41 | unsigned int pad = ((modulusLen&1) << 1) + ((modulusLen&2) >> 1); // 2 gives 1 and 1 gives 2, but 0 gives 0. 42 | return 4 * (bin_size + pad) / 3; 43 | } 44 | 45 | static unsigned int 46 | base64(const unsigned char* bin, unsigned int bin_size, unsigned char* res, unsigned int res_size) 47 | { 48 | if (!res || !bin) 49 | { 50 | return 0; 51 | } 52 | 53 | unsigned int modulusLen = bin_size % 3; 54 | unsigned int pad = ((modulusLen&1) << 1) + ((modulusLen&2) >> 1); // 2 gives 1 and 1 gives 2, but 0 gives 0. 55 | unsigned int outlen = 4 * (bin_size + pad) / 3; 56 | if (res_size < outlen) 57 | { 58 | return 0; 59 | } 60 | 61 | int byteNo; // I need this after the loop 62 | for (byteNo = 0; bin_size >= 3 && byteNo <= bin_size-3; byteNo += 3) 63 | { 64 | unsigned char b0 = bin[byteNo]; 65 | unsigned char b1 = bin[byteNo+1]; 66 | unsigned char b2 = bin[byteNo+2]; 67 | *res++ = base64_table_enc[ b0 >> 2 ]; 68 | *res++ = base64_table_enc[ ((0x03 & b0) << 4) + (b1 >> 4) ]; 69 | *res++ = base64_table_enc[ ((0x0f & b1) << 2) + (b2 >> 6) ]; 70 | *res++ = base64_table_enc[ 0x3f & b2 ]; 71 | } 72 | 73 | if (pad == 2) 74 | { 75 | *res++ = base64_table_enc[ bin[byteNo] >> 2 ]; 76 | *res++ = base64_table_enc[ (0x3 & bin[byteNo]) << 4 ]; 77 | *res++ = '='; 78 | *res++ = '='; 79 | } 80 | else if (pad == 1) 81 | { 82 | *res++ = base64_table_enc[ bin[byteNo] >> 2 ]; 83 | *res++ = base64_table_enc[ ((0x3 & bin[byteNo]) << 4) + (bin[byteNo+1] >> 4) ]; 84 | *res++ = base64_table_enc[ (0x0f & bin[byteNo+1]) << 2 ]; 85 | *res++ = '='; 86 | } 87 | 88 | return outlen; 89 | } 90 | 91 | static unsigned int 92 | unbase64_size(const unsigned char* ascii, unsigned int len) 93 | { 94 | if (!ascii) 95 | { 96 | return 0; 97 | } 98 | 99 | int pad = 0; 100 | 101 | if (len < 2) // 2 accesses below would be OOB. 102 | { 103 | return 0; 104 | } 105 | if (ascii[len - 1] == '=') ++pad; 106 | if (ascii[len - 2] == '=') ++pad; 107 | 108 | if (len - pad < 2) // truncated, less than a byte 109 | { 110 | return 0; 111 | } 112 | 113 | return (3 * len / 4 - pad); 114 | } 115 | 116 | static unsigned int 117 | unbase64(const unsigned char* ascii, unsigned int len, unsigned char *bin, unsigned int bin_size) 118 | { 119 | if (!ascii || !bin) 120 | { 121 | return 0; 122 | } 123 | 124 | int pad = 0; 125 | 126 | if (len < 2) // 2 accesses below would be OOB. 127 | { 128 | return 0; 129 | } 130 | if (ascii[len - 1] == '=') ++pad; 131 | if (ascii[len - 2] == '=') ++pad; 132 | 133 | if (len - pad < 2) // truncated, less than a byte 134 | { 135 | return 0; 136 | } 137 | 138 | unsigned int outlen = 3 * len / 4 - pad; 139 | if (bin_size < outlen) 140 | { 141 | return 0; 142 | } 143 | 144 | int charNo; 145 | for (charNo = 0; len >= 4 + pad && charNo <= len - 4 - pad; charNo += 4) 146 | { 147 | int A = base64_table_dec[ascii[charNo]]; 148 | int B = base64_table_dec[ascii[charNo+1]]; 149 | int C = base64_table_dec[ascii[charNo+2]]; 150 | int D = base64_table_dec[ascii[charNo+3]]; 151 | 152 | *bin++ = (A<<2) | (B>>4); 153 | *bin++ = (B<<4) | (C>>2); 154 | *bin++ = (C<<6) | (D); 155 | } 156 | 157 | if (pad <= 1 && charNo <= len - 3) // single padding or truncated and at least 3 chars left 158 | { 159 | int A = base64_table_dec[ascii[charNo]]; 160 | int B = base64_table_dec[ascii[charNo+1]]; 161 | int C = base64_table_dec[ascii[charNo+2]]; 162 | 163 | *bin++ = (A<<2) | (B>>4); 164 | *bin++ = (B<<4) | (C>>2); 165 | } 166 | else if (charNo <= len - 2) // double padding or truncated and at least 2 chars left 167 | { 168 | int A = base64_table_dec[ascii[charNo]]; 169 | int B = base64_table_dec[ascii[charNo+1]]; 170 | 171 | *bin++ = (A<<2) | (B>>4); 172 | } 173 | 174 | return outlen; 175 | } 176 | -------------------------------------------------------------------------------- /c_src/b64fast.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "erl_nif.h" 3 | 4 | #include "naive.h" 5 | 6 | /* 7 | * decode64_chunk is an "internal NIF" scheduled by decode64 below. It takes 8 | * the binary argument, same as the other functions here, but also 9 | * takes a count of the max number of bytes to process per timeslice, the 10 | * offset into the binary at which to start processing, the resource type 11 | * holding the resulting data, and it's size. 12 | */ 13 | static ERL_NIF_TERM 14 | decode64_chunk(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) 15 | { 16 | ErlNifResourceType* res_type = (ErlNifResourceType*)enif_priv_data(env); 17 | unsigned long offset, i, end, max_per_slice, res_size; 18 | struct timeval start, stop, slice; 19 | int pct, total = 0; 20 | ERL_NIF_TERM newargv[5]; 21 | ErlNifBinary bin; 22 | void* res; 23 | 24 | if (argc != 5 || !enif_inspect_binary(env, argv[0], &bin) || 25 | !enif_get_ulong(env, argv[1], &max_per_slice) || 26 | !enif_get_ulong(env, argv[2], &offset) || 27 | !enif_get_resource(env, argv[3], res_type, &res) || 28 | !enif_get_ulong(env, argv[4], &res_size)) 29 | return enif_make_badarg(env); 30 | end = offset + max_per_slice; 31 | if (end > bin.size) end = bin.size; 32 | i = offset; 33 | while (i < bin.size) { 34 | gettimeofday(&start, NULL); 35 | unbase64(bin.data+i, end-i, res+i*3/4, res_size-i*3/4); 36 | i = end; 37 | if (i == bin.size) break; 38 | gettimeofday(&stop, NULL); 39 | /* determine how much of the timeslice was used */ 40 | timersub(&stop, &start, &slice); 41 | pct = (int)((slice.tv_sec*1000000+slice.tv_usec)/10); 42 | total += pct; 43 | if (pct > 100) pct = 100; 44 | else if (pct == 0) pct = 1; 45 | if (enif_consume_timeslice(env, pct)) { 46 | /* the timeslice has been used up, so adjust our max_per_slice byte count based on 47 | * the processing we've done, then reschedule to run again */ 48 | max_per_slice = i - offset; 49 | if (total > 100) { 50 | int m = (int)(total/100); 51 | if (m == 1) 52 | max_per_slice -= (unsigned long)(max_per_slice*(total-100)/100); 53 | else 54 | max_per_slice = (unsigned long)(max_per_slice/m); 55 | } 56 | max_per_slice = max_per_slice / 4; 57 | max_per_slice = max_per_slice * 4; 58 | newargv[0] = argv[0]; 59 | newargv[1] = enif_make_ulong(env, max_per_slice); 60 | newargv[2] = enif_make_ulong(env, i); 61 | newargv[3] = argv[3]; 62 | newargv[4] = argv[4]; 63 | return enif_schedule_nif(env, "decode64_chunk", 0, decode64_chunk, argc, newargv); 64 | } 65 | end += max_per_slice; 66 | if (end > bin.size) end = bin.size; 67 | } 68 | return enif_make_resource_binary(env, res, res, res_size); 69 | } 70 | 71 | /* 72 | * decode64 just schedules decode64_chunk for execution, providing an initial 73 | * guess of 30KB for the max number of bytes to process before yielding the 74 | * scheduler thread. 75 | */ 76 | static ERL_NIF_TERM 77 | decode64(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) 78 | { 79 | ErlNifResourceType* res_type = (ErlNifResourceType*)enif_priv_data(env); 80 | ERL_NIF_TERM newargv[5]; 81 | ErlNifBinary bin; 82 | unsigned res_size; 83 | void* res; 84 | 85 | if (argc != 1 || !enif_inspect_binary(env, argv[0], &bin)) 86 | return enif_make_badarg(env); 87 | if (bin.size == 0) 88 | return argv[0]; 89 | res_size = unbase64_size(bin.data, bin.size); 90 | newargv[0] = argv[0]; 91 | newargv[1] = enif_make_ulong(env, 30720); // MOD4 92 | newargv[2] = enif_make_ulong(env, 0); 93 | res = enif_alloc_resource(res_type, res_size); 94 | newargv[3] = enif_make_resource(env, res); 95 | newargv[4] = enif_make_ulong(env, res_size); 96 | enif_release_resource(res); 97 | return enif_schedule_nif(env, "decode64_chunk", 0, decode64_chunk, 5, newargv); 98 | } 99 | 100 | /* 101 | * encode64_chunk is an "internal NIF" scheduled by encode64 below. It takes 102 | * the binary argument, same as the other functions here, but also 103 | * takes a count of the max number of bytes to process per timeslice, the 104 | * offset into the binary at which to start processing, the resource type 105 | * holding the resulting data, and it's size. 106 | */ 107 | static ERL_NIF_TERM 108 | encode64_chunk(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) 109 | { 110 | ErlNifResourceType* res_type = (ErlNifResourceType*)enif_priv_data(env); 111 | unsigned long offset, i, end, max_per_slice, res_size; 112 | struct timeval start, stop, slice; 113 | int pct, total = 0; 114 | ERL_NIF_TERM newargv[5]; 115 | ErlNifBinary bin; 116 | void* res; 117 | 118 | if (argc != 5 || !enif_inspect_binary(env, argv[0], &bin) || 119 | !enif_get_ulong(env, argv[1], &max_per_slice) || 120 | !enif_get_ulong(env, argv[2], &offset) || 121 | !enif_get_resource(env, argv[3], res_type, &res) || 122 | !enif_get_ulong(env, argv[4], &res_size)) 123 | return enif_make_badarg(env); 124 | end = offset + max_per_slice; 125 | if (end > bin.size) end = bin.size; 126 | i = offset; 127 | while (i < bin.size) { 128 | gettimeofday(&start, NULL); 129 | base64(bin.data+i, end-i, res+i*4/3, res_size-i*4/3); 130 | i = end; 131 | if (i == bin.size) break; 132 | gettimeofday(&stop, NULL); 133 | /* determine how much of the timeslice was used */ 134 | timersub(&stop, &start, &slice); 135 | pct = (int)((slice.tv_sec*1000000+slice.tv_usec)/10); 136 | total += pct; 137 | if (pct > 100) pct = 100; 138 | else if (pct == 0) pct = 1; 139 | if (enif_consume_timeslice(env, pct)) { 140 | /* the timeslice has been used up, so adjust our max_per_slice byte count based on 141 | * the processing we've done, then reschedule to run again */ 142 | max_per_slice = i - offset; 143 | if (total > 100) { 144 | int m = (int)(total/100); 145 | if (m == 1) 146 | max_per_slice -= (unsigned long)(max_per_slice*(total-100)/100); 147 | else 148 | max_per_slice = (unsigned long)(max_per_slice/m); 149 | } 150 | max_per_slice = max_per_slice / 3; 151 | max_per_slice = max_per_slice * 3; 152 | newargv[0] = argv[0]; 153 | newargv[1] = enif_make_ulong(env, max_per_slice); 154 | newargv[2] = enif_make_ulong(env, i); 155 | newargv[3] = argv[3]; 156 | newargv[4] = argv[4]; 157 | return enif_schedule_nif(env, "encode64_chunk", 0, encode64_chunk, argc, newargv); 158 | } 159 | end += max_per_slice; 160 | if (end > bin.size) end = bin.size; 161 | } 162 | return enif_make_resource_binary(env, res, res, res_size); 163 | } 164 | 165 | /* 166 | * encode64 just schedules encode64_chunk for execution, providing an initial 167 | * guess of 30KB for the max number of bytes to process before yielding the 168 | * scheduler thread. 169 | */ 170 | static ERL_NIF_TERM 171 | encode64(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) 172 | { 173 | ErlNifResourceType* res_type = (ErlNifResourceType*)enif_priv_data(env); 174 | ERL_NIF_TERM newargv[5]; 175 | ErlNifBinary bin; 176 | unsigned res_size; 177 | void* res; 178 | 179 | if (argc != 1 || !enif_inspect_binary(env, argv[0], &bin)) 180 | return enif_make_badarg(env); 181 | if (bin.size == 0) 182 | return argv[0]; 183 | res_size = base64_size(bin.size); 184 | newargv[0] = argv[0]; 185 | newargv[1] = enif_make_ulong(env, 30720); // MOD3 186 | newargv[2] = enif_make_ulong(env, 0); 187 | res = enif_alloc_resource(res_type, res_size); 188 | newargv[3] = enif_make_resource(env, res); 189 | newargv[4] = enif_make_ulong(env, res_size); 190 | enif_release_resource(res); 191 | return enif_schedule_nif(env, "encode64_chunk", 0, encode64_chunk, 5, newargv); 192 | } 193 | 194 | static int 195 | nifload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) 196 | { 197 | *priv_data = enif_open_resource_type(env, 198 | NULL, 199 | "b64fast", 200 | NULL, 201 | ERL_NIF_RT_CREATE|ERL_NIF_RT_TAKEOVER, 202 | NULL); 203 | return *priv_data == NULL; 204 | } 205 | 206 | static int 207 | nifupgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) 208 | { 209 | *priv_data = enif_open_resource_type(env, 210 | NULL, 211 | "b64fast", 212 | NULL, 213 | ERL_NIF_RT_TAKEOVER, 214 | NULL); 215 | return *priv_data == NULL; 216 | } 217 | 218 | static ErlNifFunc funcs[] = { 219 | {"encode64", 1, encode64}, 220 | {"decode64", 1, decode64}, 221 | }; 222 | ERL_NIF_INIT(b64fast, funcs, nifload, NULL, nifupgrade, NULL); 223 | --------------------------------------------------------------------------------