345 | 
| tinyendian | A minimal library providing functionality for changing the endianness of data. 30 | 31 | |
| T | Item type. Must be either 2 or 4 bytes long. 40 | 41 | |
| array | Buffer with values to fix byte order of. 43 | 44 | |
public @system pure nothrow @nogc auto fixUTFByteOrder(ubyte[] array)
48 | Convert byte order of an array encoded in UTF(8/16/32) to system endianness in place. 50 |
51 | 52 | 53 |Uses the UTF byte-order-mark (BOM) to determine UTF encoding. If there is no BOM 55 | at the beginning of array, UTF-8 is assumed (this is compatible with ASCII). The 56 | BOM, if any, will be removed from the buffer. 57 |
58 |If the encoding is determined to be UTF-16 or UTF-32 and there aren't enough bytes 59 | for the last code unit (i.e. if array.length is odd for UTF-16 or not divisible by 60 | 4 for UTF-32), the extra bytes (1 for UTF-16, 1-3 for UTF-32) are stripped. 61 |
62 |Note that this function does not check if the array is a valid UTF string. It 63 | only works with the BOM and 1,2 or 4-byte items. 64 |
65 | 66 | 67 |Parameters
69 || array | The array with UTF-data. 71 | 72 | |
Returns
76 |A struct with the following members: 77 |
78 | ubyte[] array A slice of the input array containing data in correct
79 | byte order, without BOM and in case of UTF-16/UTF-32,
80 | without stripped bytes, if any.
81 | UTFEncoding encoding Encoding of the result (UTF-8, UTF-16 or UTF-32)
82 | std.system.Endian endian Endianness of the original array.
83 | uint bytesStripped Number of bytes stripped from a UTF-16/UTF-32 array, if
84 | any. This is non-zero only if array.length was not
85 | divisible by 2 or 4 for UTF-16 and UTF-32, respectively.
86 |
Complexity
91 |(BIGOH array.length) 92 |
93 | 94 | 95 |A minimal library providing functionality for changing the endianness of data. 97 |
98 | 99 | 100 |Enums
| UTFEncoding | Unicode UTF encodings. 102 | 103 | |
Functions
| swapByteOrder | Swap byte order of items in an array in place. 104 | 105 | | |
| autofixUTFByteOrder | Convert byte order of an array encoded in UTF(8/16/32) to system endianness in place. 106 | 107 | |
14 |
19 |
28 |
45 |
46 |
47 |
--------------------------------------------------------------------------------
/dub.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "tinyendian",
3 | "description": "Lightweight endianness handling library",
4 | "authors": [ "Ferdinand Majerech" ],
5 | "importPaths": ["source"],
6 | "license": "Boost 1.0",
7 | "homepage": "https://github.com/kiith-sa/tinyendian",
8 | "copyright": "Copyright © 2014, Ferdinand Majerech",
9 |
10 | "buildTypes":
11 | {
12 | "debug": { "buildOptions": ["debugMode", "debugInfoC"] },
13 | "release": { "buildOptions": ["releaseMode", "optimize", "inline", "noBoundsCheck"] },
14 | "profile": { "buildOptions": ["releaseMode", "optimize", "noBoundsCheck", "debugInfoC"] },
15 | },
16 | }
17 |
--------------------------------------------------------------------------------
/hmod.cfg:
--------------------------------------------------------------------------------
1 | # This file contains configuration options for harbored-mod (hmod).
2 | #
3 | # By default, hmod loads configuration from file 'hmod.cfg' in the directory from where
4 | # hmod is running, if such file exists. These configuration options can also be passed
5 | # as command-line options for hmod, overriding contents of the config file, if any,
6 | # with the exception of options that allow multiple values (such as 'exclude' or
7 | # 'macros') where the values specified as command-line options are *added* to the values
8 | # in config file.
9 |
10 |
11 |
12 | # Source code files or directories to document. Specify more than once to document more
13 | # files/directories, e.g:
14 | #
15 | # source = ./source
16 | # source = ./thirdparty
17 | #
18 | # This will document both the source code in the ./source/ and ./thirdparty/ directories.
19 | #
20 | # For DUB (http://code.dlang.org) projects, './source' is usually a good setting here.
21 | source = ./source
22 |
23 |
24 | # Name of the project. Used on the main page and possibly other places to help identify
25 | # the project.
26 | project-name = tinyendian
27 |
28 |
29 | # Version of the project. Used on the main page and possibly other places to help
30 | # identify the project.
31 | project-version =
32 |
33 |
34 | # Set to true to disable Markdown support, so that only DDoc is used.
35 | no-markdown = false
36 |
37 |
38 | # Directory where the generated documentation will be written.
39 | output-directory = ./doc
40 |
41 | # Output format. Supported values:
42 | # html-simple
43 | # Every function, variable, etc. will be documented in a separate
44 | # HTML file. A documentation page of an aggregate (e.g. class) will
45 | # link to these files. Generates a large number of files; not
46 | # recommended for large projects.
47 | # html-aggregated
48 | # Members of an aggregate will be documented in the same file as the
49 | # aggregate, unless they are aggregates themselves. That is, member
50 | # functions or variables will be documented in the same file as the
51 | # class that contains them, but a subclass will be documented in a
52 | # separate file.
53 | format = html-aggregated
54 |
55 |
56 | # Modules or packages to exclude from generated documentation. Specify more than once to
57 | # exclude more modules/packages, e.g:
58 | #
59 | # exclude = tharsis.util
60 | # exclude = tharsis.entity.gamestate
61 | #
62 | # This will exclude both the package (or module) tharsis.util and module (or package)
63 | # tharsis.entity.gamestate .
64 |
65 | exclude =
66 |
67 |
68 | # DDoc+markdown source of the main page of your documentation. Currently the main page is
69 | # blank by default; this can be used to fill it with something useful.
70 |
71 | index =
72 |
73 |
74 | # Filename of DDoc+markdown source of additional content to add to the table of contents
75 | # sidebar. Useful e.g. to add links to tutorials. Can be used more than once to add
76 | # content from more files.
77 |
78 | toc-additional =
79 |
80 |
81 | # CSS file to use for styling. Can be used to replace the default style.
82 | # To create a new style, you can start by generating the default style file with
83 | # 'hmod --generate-css CSS_OUT_FILE' (CSS_OUT_FILE is name the generated file will have)
84 | # and then modifying the CSS to get the desired style.
85 |
86 | css =
87 |
88 |
89 | # File to load DDoc macros from. Can be used to override builtin macros or add new ones.
90 | # Can be specified more than once to use multiple macro files, e.g.:
91 | #
92 | # macros = macros.ddoc
93 | # macros = moremacros.ddoc
94 |
95 | macros =
96 |
97 |
98 | # Maximum file size of D modules harbored-mod will process, in kiB. Any file
99 | # bigger than specified size will be ignored. Useful in rare cases when modules
100 | # are huge enough that the parser spends too much memory.
101 | # By default, we ignore modules bigger than 16MiB.
102 |
103 | max-file-size = 16384
104 |
105 |
106 | # Additional config file to load, if needed. Configuration options in specified file will
107 | # override or add to any options specified before this line, and will be overridden by
108 | # any options after this line. Think of it as including the config file in this file.
109 |
110 | config =
111 |
112 |
113 |
114 | #---------------------------------------------------------------------------
115 | # Configuration options **only** useful for harbored-mod testing
116 | #---------------------------------------------------------------------------
117 | # Uncommenting these will result in printing help information; only useful for testing.
118 | #
119 | # # Print help message.
120 | #
121 | # help = true
122 | #
123 | #
124 | # # Generate default CSS file and write it to specified file.
125 | # generate-css = hmod-style.css
126 | #
127 | #
128 | # # Generate default config file and write it to 'hmod.cfg'.
129 | #
130 | # generate-cfg = true
131 |
--------------------------------------------------------------------------------
/meson.build:
--------------------------------------------------------------------------------
1 | # -*- mode: python; -*-
2 |
3 | project(
4 | 'tinyendian',
5 | 'd',
6 | meson_version: '>=0.41',
7 | version: '0.2.0',
8 | default_options: ['buildtype=release'],
9 | )
10 |
11 | src_dir = include_directories('source/')
12 | pkgc = import('pkgconfig')
13 |
14 | tinyendian_src = [
15 | 'source/tinyendian.d'
16 | ]
17 |
18 | install_headers(tinyendian_src, subdir: 'd/')
19 |
20 | tinyendian_lib = library(
21 | meson.project_name(),
22 | [tinyendian_src],
23 | include_directories: [src_dir],
24 | version: meson.project_version(),
25 | pic: true,
26 | install: true,
27 | )
28 |
29 | pkgc.generate(
30 | name: meson.project_name(),
31 | libraries: tinyendian_lib,
32 | subdirs: 'd/',
33 | version: meson.project_version(),
34 | description: 'Lightweight endianness library for D.'
35 | )
36 |
37 | # Make Tinyendian easy to use as subproject
38 | tinyendian_dep = declare_dependency(
39 | link_with: tinyendian_lib,
40 | include_directories: [src_dir]
41 | )
42 |
--------------------------------------------------------------------------------
/source/tinyendian.d:
--------------------------------------------------------------------------------
1 | // Copyright Ferdinand Majerech 2014.
2 | // Distributed under the Boost Software License, Version 1.0.
3 | // (See accompanying file LICENSE_1_0.txt or copy at
4 | // http://www.boost.org/LICENSE_1_0.txt)
5 |
6 | /// A minimal library providing functionality for changing the endianness of data.
7 | module tinyendian;
8 |
9 | import std.system : Endian, endian;
10 |
11 | /// Unicode UTF encodings.
12 | enum UTFEncoding : ubyte
13 | {
14 | UTF_8,
15 | UTF_16,
16 | UTF_32
17 | }
18 | ///
19 | @safe unittest
20 | {
21 | const ints = [314, -101];
22 | int[2] intsSwapBuffer = ints;
23 | swapByteOrder(intsSwapBuffer[]);
24 | swapByteOrder(intsSwapBuffer[]);
25 | assert(ints == intsSwapBuffer, "Lost information when swapping byte order");
26 |
27 | const floats = [3.14f, 10.1f];
28 | float[2] floatsSwapBuffer = floats;
29 | swapByteOrder(floatsSwapBuffer[]);
30 | swapByteOrder(floatsSwapBuffer[]);
31 | assert(floats == floatsSwapBuffer, "Lost information when swapping byte order");
32 | }
33 |
34 | /** Swap byte order of items in an array in place.
35 | *
36 | * Params:
37 | *
38 | * T = Item type. Must be either 2 or 4 bytes long.
39 | * array = Buffer with values to fix byte order of.
40 | */
41 | void swapByteOrder(T)(T[] array) @trusted @nogc pure nothrow
42 | if (T.sizeof == 2 || T.sizeof == 4)
43 | {
44 | // Swap the byte order of all read characters.
45 | foreach (ref item; array)
46 | {
47 | static if (T.sizeof == 2)
48 | {
49 | import std.algorithm.mutation : swap;
50 | swap(*cast(ubyte*)&item, *(cast(ubyte*)&item + 1));
51 | }
52 | else static if (T.sizeof == 4)
53 | {
54 | import core.bitop : bswap;
55 | const swapped = bswap(*cast(uint*)&item);
56 | item = *cast(const(T)*)&swapped;
57 | }
58 | else static assert(false, "Unsupported T: " ~ T.stringof);
59 | }
60 | }
61 |
62 | /// See fixUTFByteOrder.
63 | struct FixUTFByteOrderResult
64 | {
65 | ubyte[] array;
66 | UTFEncoding encoding;
67 | Endian endian;
68 | uint bytesStripped = 0;
69 | }
70 |
71 | /** Convert byte order of an array encoded in UTF(8/16/32) to system endianness in place.
72 | *
73 | * Uses the UTF byte-order-mark (BOM) to determine UTF encoding. If there is no BOM
74 | * at the beginning of array, UTF-8 is assumed (this is compatible with ASCII). The
75 | * BOM, if any, will be removed from the buffer.
76 | *
77 | * If the encoding is determined to be UTF-16 or UTF-32 and there aren't enough bytes
78 | * for the last code unit (i.e. if array.length is odd for UTF-16 or not divisible by
79 | * 4 for UTF-32), the extra bytes (1 for UTF-16, 1-3 for UTF-32) are stripped.
80 | *
81 | * Note that this function does $(B not) check if the array is a valid UTF string. It
82 | * only works with the BOM and 1,2 or 4-byte items.
83 | *
84 | * Params:
85 | *
86 | * array = The array with UTF-data.
87 | *
88 | * Returns:
89 | *
90 | * A struct with the following members:
91 | *
92 | * $(D ubyte[] array) A slice of the input array containing data in correct
93 | * byte order, without BOM and in case of UTF-16/UTF-32,
94 | * without stripped bytes, if any.
95 | * $(D UTFEncoding encoding) Encoding of the result (UTF-8, UTF-16 or UTF-32)
96 | * $(D std.system.Endian endian) Endianness of the original array.
97 | * $(D uint bytesStripped) Number of bytes stripped from a UTF-16/UTF-32 array, if
98 | * any. This is non-zero only if array.length was not
99 | * divisible by 2 or 4 for UTF-16 and UTF-32, respectively.
100 | *
101 | * Complexity: (BIGOH array.length)
102 | */
103 | auto fixUTFByteOrder(ubyte[] array) @safe @nogc pure nothrow
104 | {
105 | // Enumerates UTF BOMs, matching indices to byteOrderMarks/bomEndian.
106 | enum BOM: ubyte
107 | {
108 | UTF_8 = 0,
109 | UTF_16_LE = 1,
110 | UTF_16_BE = 2,
111 | UTF_32_LE = 3,
112 | UTF_32_BE = 4,
113 | None = ubyte.max
114 | }
115 |
116 | // These 2 are from std.stream
117 | static immutable ubyte[][5] byteOrderMarks = [ [0xEF, 0xBB, 0xBF],
118 | [0xFF, 0xFE],
119 | [0xFE, 0xFF],
120 | [0xFF, 0xFE, 0x00, 0x00],
121 | [0x00, 0x00, 0xFE, 0xFF] ];
122 | static immutable Endian[5] bomEndian = [ endian,
123 | Endian.littleEndian,
124 | Endian.bigEndian,
125 | Endian.littleEndian,
126 | Endian.bigEndian ];
127 |
128 | // Documented in function ddoc.
129 |
130 | FixUTFByteOrderResult result;
131 |
132 | // Detect BOM, if any, in the bytes we've read. -1 means no BOM.
133 | // Need the last match: First 2 bytes of UTF-32LE BOM match the UTF-16LE BOM. If we
134 | // used the first match, UTF-16LE would be detected when we have a UTF-32LE BOM.
135 | import std.algorithm.searching : startsWith;
136 | BOM bomId = BOM.None;
137 | foreach (i, bom; byteOrderMarks)
138 | if (array.startsWith(bom))
139 | bomId = cast(BOM)i;
140 |
141 | result.endian = (bomId != BOM.None) ? bomEndian[bomId] : Endian.init;
142 |
143 | // Start of UTF data (after BOM, if any)
144 | size_t start = 0;
145 | // If we've read more than just the BOM, put the rest into the array.
146 | with(BOM) final switch(bomId)
147 | {
148 | case None: result.encoding = UTFEncoding.UTF_8; break;
149 | case UTF_8:
150 | start = 3;
151 | result.encoding = UTFEncoding.UTF_8;
152 | break;
153 | case UTF_16_LE, UTF_16_BE:
154 | result.bytesStripped = array.length % 2;
155 | start = 2;
156 | result.encoding = UTFEncoding.UTF_16;
157 | break;
158 | case UTF_32_LE, UTF_32_BE:
159 | result.bytesStripped = array.length % 4;
160 | start = 4;
161 | result.encoding = UTFEncoding.UTF_32;
162 | break;
163 | }
164 |
165 | // If there's a BOM, we need to move data back to ensure it starts at array[0]
166 | if (start != 0)
167 | {
168 | array = array[start .. $ - result.bytesStripped];
169 | }
170 |
171 | // We enforce above that array.length is divisible by 2/4 for UTF-16/32
172 | if (endian != result.endian)
173 | {
174 | if (result.encoding == UTFEncoding.UTF_16)
175 | swapByteOrder(cast(wchar[])array);
176 | else if (result.encoding == UTFEncoding.UTF_32)
177 | swapByteOrder(cast(dchar[])array);
178 | }
179 |
180 | result.array = array;
181 | return result;
182 | }
183 | ///
184 | @safe unittest
185 | {
186 | {
187 | ubyte[] s = [0xEF, 0xBB, 0xBF, 'a'];
188 | FixUTFByteOrderResult r = fixUTFByteOrder(s);
189 | assert(r.encoding == UTFEncoding.UTF_8);
190 | assert(r.array.length == 1);
191 | assert(r.array == ['a']);
192 | assert(r.endian == Endian.littleEndian);
193 | }
194 |
195 | {
196 | ubyte[] s = ['a'];
197 | FixUTFByteOrderResult r = fixUTFByteOrder(s);
198 | assert(r.encoding == UTFEncoding.UTF_8);
199 | assert(r.array.length == 1);
200 | assert(r.array == ['a']);
201 | assert(r.endian == Endian.bigEndian);
202 | }
203 |
204 | {
205 | // strip 'a' b/c not complete unit
206 | ubyte[] s = [0xFE, 0xFF, 'a'];
207 | FixUTFByteOrderResult r = fixUTFByteOrder(s);
208 | assert(r.encoding == UTFEncoding.UTF_16);
209 | assert(r.array.length == 0);
210 | assert(r.endian == Endian.bigEndian);
211 | }
212 |
213 | }
214 |
--------------------------------------------------------------------------------
29 |
36 |
37 |
38 |
39 |
40 |
41 |
44 | public enum UTFEncoding : ubyte
30 | Unicode UTF encodings. 32 |
33 | 34 | 35 |Values
| UTF_8 | ||
| UTF_16 | ||
| UTF_32 |