├── ex ├── dcu03-input-u8.bin ├── dcu04-input-u16.bin ├── dcu06-output-sd.bin ├── dcu13-input-u8.bin ├── dcu14-input-u8.bin ├── odu07-output-sd.bin ├── odu12-output-identity-u32.bin ├── dcu01-output-u32.bin ├── ipu02-output-sd.bin ├── ipu03-output-sd.bin ├── odu09-output-0-sd.bin ├── odu09-output-1-sd.bin ├── odu09-output-2-sd.bin ├── odu10-output-1-u32.bin ├── odu10-output-3-u32.bin ├── odu13-output-0-sd.bin ├── odu13-output-1-sd.bin ├── odu13-output-2-sd.bin ├── odu16-output-u8.bin ├── odu20-input-u32.bin ├── odu20-output-u32.bin ├── dcu03-output-4-sd.bin ├── dcu04-output-u32.bin ├── dcu10-output-u32.bin ├── dcu23-output-u64.bin ├── odu01-input-u32.bin ├── odu02-input-sd.bin ├── odu02-output-sd.bin ├── odu14-input-u32.bin ├── odu14-output-u32.bin ├── 90b07-input-sd.bin ├── dcu01-input-u64.bin ├── dcu05-input-u64.bin ├── dcu11-input-u32.bin ├── dcu12-input-u32.bin ├── dcu19-output-u32.bin ├── odu18-output-u32.bin ├── 90b02-output-1-sd.bin ├── dcu03-output-2-sd.bin ├── dcu06-input-u32.bin ├── dcu09-input-u32.bin ├── dcu14-output-u32.bin ├── tdpu02-output-u8.bin ├── dcu17-output-d-u32.bin ├── dcu21-input-u64.bin ├── dcu21-output-u64.bin ├── dcu22-input-u64.bin ├── dcu22-output-u64.bin ├── odu06-output-b2-8-sd.bin ├── dcu08-input-u64.bin ├── odu06-input-sd.bin ├── odu06-output-b1-16-sd.bin ├── odu07-input-u32.bin ├── odu12-input-u32.bin ├── odu13-input-u32.bin ├── odu18-input-u32.bin ├── odu06-output-b1-10-sd.bin ├── odu10-input-u128.bin ├── odu12-output-swap-u32.bin ├── odu17-output-3-u32.bin ├── odu12-output-identity-r-u32.bin ├── odu12-output-swapRemoveEven-u32.bin ├── odu17-input-u32.bin ├── dcu11-output-u64.bin ├── 90b02-input-sd.bin ├── 90b02-output-0-sd.bin ├── dcu03-output-1-sd.bin ├── dcu23-input.txt ├── ipu04-output-u32.bin ├── dcu10-input.txt ├── dcu02-input-u64.bin ├── dcu02-output-u64.bin ├── dcu07-input-u16.bin ├── dcu08-output-u64.bin ├── dcu17-input-u32.bin ├── dcu17-output-u32.bin ├── dcu18-input-u32.bin ├── dcu18-output-u32.bin ├── dcu19-input-u32.bin ├── dcu20-input-u32.bin ├── dcu20-output-u32.bin ├── ipu01-input-u32.bin ├── ipu02-input-u32.bin ├── ipu03-input-sd.bin ├── ipu04-input-u32.bin ├── odu04-input.txt ├── odu11-input-u32.bin ├── odu11-output-u32.bin ├── odu15-input-u32.bin ├── odu15-output-u32.bin ├── odu16-input-u32.bin ├── odu19-input-u32.bin ├── odu19-output-u32.bin ├── odu21-input-u16.bin ├── odu22-input-u32.bin ├── riu01-input.txt ├── riu02-input.txt ├── riu03-input.txt ├── dcu04-output-d-u32.bin ├── dcu15-input.txt ├── odu05-input-double.bin ├── tdpu01-output-u32.bin ├── dcu07-output-sd.bin ├── odu03-input1-double.bin ├── odu03-input2-double.bin ├── odu03-output-double.bin ├── odu04-input-0-double.bin ├── odu05-output-double.bin ├── riu01-input-0-double.bin ├── dcu07-output-b-sd.bin ├── dcu07-output-l-sd.bin ├── dcu16-input-3byteblocks.bin ├── dcu15-output-u32.bin ├── dcu16-output-sd.bin ├── dcu16-output-l-sd.bin ├── odu23-output-b1-16-u32.bin ├── odu23-output-b2-8-u32.bin ├── odu09-input-u128.bin ├── odu23-output-b1-10-u32.bin ├── odu23-input-u32.bin ├── ipu01-input-u32.bin.select-3 ├── ipu01-input-u32.bin.select-3c ├── ipu01-input-u32.bin.select-8c └── hpu02-input.txt ├── docs └── images │ ├── ro-model.png │ └── ro-model-zoom.png ├── src ├── cephes.h ├── translate.h ├── hashmodulus.h ├── sa.h ├── Makefile.ec2-aarch64 ├── globals.h ├── assessments.h ├── globals-inst.h ├── sd-to-dec.c ├── poolalloc.h ├── enttypes.h ├── Makefile ├── Makefile.ec2 ├── Makefile.ec2-debug ├── u64-change-endianness.c ├── u64-to-ascii.c ├── binio.h ├── binutil.h ├── u32-to-ascii.c ├── sd-to-hex.c ├── dec-to-u64.c ├── hex-to-u32.c ├── u8-to-u32.c ├── incbeta.h ├── dec-to-u32.c ├── hweight.c ├── u32-xor-diff.c ├── bootstrap.h ├── dictionaryTree.h ├── sigfigs.c ├── u64-counter-endian.c ├── Makefile.mem ├── u32-to-u64.c ├── Makefile.asan ├── Makefile.thread ├── Makefile.debug ├── u32-xor.c ├── bits-in-use.c ├── u32-anddata.c ├── mementsource.c ├── u16-to-u32.c ├── health-tests.h ├── u64-to-u32.c ├── u32-bit-select.c ├── u32-to-sd.c ├── shannon.c ├── u64-counter-raw.c ├── u32-counter-raw.c ├── u16-to-sdbin.c ├── extractbits.c ├── u32-selectrange.c ├── u16-mcv.c ├── u64-scale-break.c ├── u32-discard-fixed-bits.c ├── double-sort.c ├── u64-randomsample.c ├── u32-randomsample.c ├── randomsample.c ├── u128-bit-select.c ├── double-merge.c ├── generate-precision.c ├── double-minmaxdelta.c ├── translate-data.c ├── discard-fixed-bits.c ├── markov.c ├── u32-counter-endian.c ├── u32-delta.c ├── randlib.h ├── rbt_misc.h ├── interleave-data.c ├── u32-counter-bitwidth.c ├── symbol-block-dist.c ├── u8-to-sd.c ├── u64-jent-to-delta.c ├── u32-manbin.c ├── u32-regress-to-mean.c ├── restart-transpose.c ├── u128-discard-fixed-bits.c ├── u32-downsample.c ├── downsample.c ├── u32-mcv.c ├── rbt_misc.c └── u32-gcd.c └── .gitignore /ex/dcu03-input-u8.bin: -------------------------------------------------------------------------------- 1 | AB01ab89 -------------------------------------------------------------------------------- /ex/dcu04-input-u16.bin: -------------------------------------------------------------------------------- 1 | AB01ab890 -------------------------------------------------------------------------------- /ex/dcu06-output-sd.bin: -------------------------------------------------------------------------------- 1 | AB01ab89 -------------------------------------------------------------------------------- /ex/dcu13-input-u8.bin: -------------------------------------------------------------------------------- 1 | AB01ab89 -------------------------------------------------------------------------------- /ex/dcu14-input-u8.bin: -------------------------------------------------------------------------------- 1 | AB01ab89 -------------------------------------------------------------------------------- /ex/odu07-output-sd.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/odu12-output-identity-u32.bin: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ex/dcu01-output-u32.bin: -------------------------------------------------------------------------------- 1 | ABCD0123abcd -------------------------------------------------------------------------------- /ex/ipu02-output-sd.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/ipu03-output-sd.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/odu09-output-0-sd.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/odu09-output-1-sd.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/odu09-output-2-sd.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/odu10-output-1-u32.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/odu10-output-3-u32.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/odu13-output-0-sd.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/odu13-output-1-sd.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/odu13-output-2-sd.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/odu16-output-u8.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/odu20-input-u32.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/odu20-output-u32.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/dcu03-output-4-sd.bin: -------------------------------------------------------------------------------- 1 |   -------------------------------------------------------------------------------- /ex/dcu04-output-u32.bin: -------------------------------------------------------------------------------- 1 | AB01ab89 -------------------------------------------------------------------------------- /ex/dcu10-output-u32.bin: -------------------------------------------------------------------------------- 1 | ABCDEFGH01234567abcdefgh -------------------------------------------------------------------------------- /ex/dcu23-output-u64.bin: -------------------------------------------------------------------------------- 1 | ABCDEFGH01234567abcdefgh -------------------------------------------------------------------------------- /ex/odu01-input-u32.bin: -------------------------------------------------------------------------------- 1 | ABCDEFGH01234567abcdefgh -------------------------------------------------------------------------------- /ex/odu02-input-sd.bin: -------------------------------------------------------------------------------- 1 | ABCDEFGH01234567abcdefgh -------------------------------------------------------------------------------- /ex/odu02-output-sd.bin: -------------------------------------------------------------------------------- 1 | !"#$%&'(12345678 -------------------------------------------------------------------------------- /ex/odu14-input-u32.bin: -------------------------------------------------------------------------------- 1 | ABCDEFGH01234567abcdefgh -------------------------------------------------------------------------------- /ex/odu14-output-u32.bin: -------------------------------------------------------------------------------- 1 |   -------------------------------------------------------------------------------- /ex/90b07-input-sd.bin: -------------------------------------------------------------------------------- 1 | !"#$%&'(12345678!"#$%&'( -------------------------------------------------------------------------------- /ex/dcu01-input-u64.bin: -------------------------------------------------------------------------------- 1 | ABCDEFGH01234567abcdefgh890 -------------------------------------------------------------------------------- /ex/dcu05-input-u64.bin: -------------------------------------------------------------------------------- 1 | ABCDEFGH01234567abcdefgh890 -------------------------------------------------------------------------------- /ex/dcu11-input-u32.bin: -------------------------------------------------------------------------------- 1 |   BR -------------------------------------------------------------------------------- /ex/dcu12-input-u32.bin: -------------------------------------------------------------------------------- 1 | ABCDEFGH01234567abcdefgh890 -------------------------------------------------------------------------------- /ex/dcu19-output-u32.bin: -------------------------------------------------------------------------------- 1 |   BR -------------------------------------------------------------------------------- /ex/odu18-output-u32.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/90b02-output-1-sd.bin: -------------------------------------------------------------------------------- 1 | 2468@BDFHPRTVX`b3579ACEGIQSUWYac -------------------------------------------------------------------------------- /ex/dcu03-output-2-sd.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/dcu06-input-u32.bin: -------------------------------------------------------------------------------- 1 | AB01ab890 -------------------------------------------------------------------------------- /ex/dcu09-input-u32.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/dcu14-output-u32.bin: -------------------------------------------------------------------------------- 1 | AB01ab89 -------------------------------------------------------------------------------- /ex/tdpu02-output-u8.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/dcu17-output-d-u32.bin: -------------------------------------------------------------------------------- 1 | # DD " -------------------------------------------------------------------------------- /ex/dcu21-input-u64.bin: -------------------------------------------------------------------------------- 1 | !DPf -------------------------------------------------------------------------------- /ex/dcu21-output-u64.bin: -------------------------------------------------------------------------------- 1 | !DPf -------------------------------------------------------------------------------- /ex/dcu22-input-u64.bin: -------------------------------------------------------------------------------- 1 | !DPf -------------------------------------------------------------------------------- /ex/dcu22-output-u64.bin: -------------------------------------------------------------------------------- 1 | !DPf -------------------------------------------------------------------------------- /ex/odu06-output-b2-8-sd.bin: -------------------------------------------------------------------------------- 1 |   2 |     -------------------------------------------------------------------------------- /ex/dcu08-input-u64.bin: -------------------------------------------------------------------------------- 1 | ʚ;ʚ; -------------------------------------------------------------------------------- /ex/odu06-input-sd.bin: -------------------------------------------------------------------------------- 1 |  2 |  !"#$%&'()* -------------------------------------------------------------------------------- /ex/odu06-output-b1-16-sd.bin: -------------------------------------------------------------------------------- 1 |   2 |     -------------------------------------------------------------------------------- /ex/odu07-input-u32.bin: -------------------------------------------------------------------------------- 1 |  2 | -------------------------------------------------------------------------------- /ex/odu12-input-u32.bin: -------------------------------------------------------------------------------- 1 |  2 | -------------------------------------------------------------------------------- /ex/odu13-input-u32.bin: -------------------------------------------------------------------------------- 1 |  2 | -------------------------------------------------------------------------------- /ex/odu18-input-u32.bin: -------------------------------------------------------------------------------- 1 |  2 | -------------------------------------------------------------------------------- /ex/odu06-output-b1-10-sd.bin: -------------------------------------------------------------------------------- 1 | 2 |     !"#$%& ' -------------------------------------------------------------------------------- /ex/odu10-input-u128.bin: -------------------------------------------------------------------------------- 1 | ABCDEFGH01234567abcdefghABCDEFGH01234567abcdefgh -------------------------------------------------------------------------------- /ex/odu12-output-swap-u32.bin: -------------------------------------------------------------------------------- 1 |  2 | -------------------------------------------------------------------------------- /ex/odu17-output-3-u32.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/odu12-output-identity-r-u32.bin: -------------------------------------------------------------------------------- 1 |  2 | -------------------------------------------------------------------------------- /ex/odu12-output-swapRemoveEven-u32.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/odu17-input-u32.bin: -------------------------------------------------------------------------------- 1 |  2 | -------------------------------------------------------------------------------- /ex/dcu11-output-u64.bin: -------------------------------------------------------------------------------- 1 |   BR -------------------------------------------------------------------------------- /ex/90b02-input-sd.bin: -------------------------------------------------------------------------------- 1 |   !"#$%&'()0123456789@ABCDEFGHIPQRSTUVWXY`abc -------------------------------------------------------------------------------- /ex/90b02-output-0-sd.bin: -------------------------------------------------------------------------------- 1 |  $(26@DHRV` !%)37AEISWa"&048BFPTXb#'159CGQUYc -------------------------------------------------------------------------------- /ex/dcu03-output-1-sd.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/dcu23-input.txt: -------------------------------------------------------------------------------- 1 | 5208208757389214273 2 | 3978425819141910832 3 | 7523094288207667809 -------------------------------------------------------------------------------- /ex/ipu04-output-u32.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/dcu10-input.txt: -------------------------------------------------------------------------------- 1 | 1145258561 2 | 1212630597 3 | 858927408 4 | 926299444 5 | 1684234849 6 | 1751606885 -------------------------------------------------------------------------------- /ex/dcu02-input-u64.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/dcu02-input-u64.bin -------------------------------------------------------------------------------- /ex/dcu02-output-u64.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/dcu02-output-u64.bin -------------------------------------------------------------------------------- /ex/dcu07-input-u16.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/dcu07-input-u16.bin -------------------------------------------------------------------------------- /ex/dcu08-output-u64.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/dcu08-output-u64.bin -------------------------------------------------------------------------------- /ex/dcu17-input-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/dcu17-input-u32.bin -------------------------------------------------------------------------------- /ex/dcu17-output-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/dcu17-output-u32.bin -------------------------------------------------------------------------------- /ex/dcu18-input-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/dcu18-input-u32.bin -------------------------------------------------------------------------------- /ex/dcu18-output-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/dcu18-output-u32.bin -------------------------------------------------------------------------------- /ex/dcu19-input-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/dcu19-input-u32.bin -------------------------------------------------------------------------------- /ex/dcu20-input-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/dcu20-input-u32.bin -------------------------------------------------------------------------------- /ex/dcu20-output-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/dcu20-output-u32.bin -------------------------------------------------------------------------------- /ex/ipu01-input-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/ipu01-input-u32.bin -------------------------------------------------------------------------------- /ex/ipu02-input-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/ipu02-input-u32.bin -------------------------------------------------------------------------------- /ex/ipu03-input-sd.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/ipu03-input-sd.bin -------------------------------------------------------------------------------- /ex/ipu04-input-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/ipu04-input-u32.bin -------------------------------------------------------------------------------- /ex/odu04-input.txt: -------------------------------------------------------------------------------- 1 | 0.111111 2 | 2.074924 3 | 2.145488 4 | 2.196152 5 | 2.029292 6 | 2.784276 7 | 3.000001 -------------------------------------------------------------------------------- /ex/odu11-input-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/odu11-input-u32.bin -------------------------------------------------------------------------------- /ex/odu11-output-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/odu11-output-u32.bin -------------------------------------------------------------------------------- /ex/odu15-input-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/odu15-input-u32.bin -------------------------------------------------------------------------------- /ex/odu15-output-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/odu15-output-u32.bin -------------------------------------------------------------------------------- /ex/odu16-input-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/odu16-input-u32.bin -------------------------------------------------------------------------------- /ex/odu19-input-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/odu19-input-u32.bin -------------------------------------------------------------------------------- /ex/odu19-output-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/odu19-output-u32.bin -------------------------------------------------------------------------------- /ex/odu21-input-u16.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/odu21-input-u16.bin -------------------------------------------------------------------------------- /ex/odu22-input-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/odu22-input-u32.bin -------------------------------------------------------------------------------- /ex/riu01-input.txt: -------------------------------------------------------------------------------- 1 | 0.111111 2 | 2.074924 3 | 2.145488 4 | 2.196152 5 | 2.029292 6 | 2.784276 7 | 3.000001 -------------------------------------------------------------------------------- /ex/riu02-input.txt: -------------------------------------------------------------------------------- 1 | 0.111111 2 | 2.074924 3 | 2.145488 4 | 2.196152 5 | 2.029292 6 | 2.784276 7 | 3.000001 -------------------------------------------------------------------------------- /ex/riu03-input.txt: -------------------------------------------------------------------------------- 1 | 0.111111 2 | 2.074924 3 | 2.145488 4 | 2.196152 5 | 2.029292 6 | 2.784276 7 | 3.000001 -------------------------------------------------------------------------------- /docs/images/ro-model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/docs/images/ro-model.png -------------------------------------------------------------------------------- /ex/dcu04-output-d-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/dcu04-output-d-u32.bin -------------------------------------------------------------------------------- /ex/dcu15-input.txt: -------------------------------------------------------------------------------- 1 | 41 42 43 44 45 46 47 48 2 | 30 31 32 33 34 35 36 37 3 | 61 62 63 64 65 66 67 68 4 | 38 39 30 -------------------------------------------------------------------------------- /ex/odu05-input-double.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/odu05-input-double.bin -------------------------------------------------------------------------------- /ex/tdpu01-output-u32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/tdpu01-output-u32.bin -------------------------------------------------------------------------------- /ex/dcu07-output-sd.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/odu03-input1-double.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/odu03-input1-double.bin -------------------------------------------------------------------------------- /ex/odu03-input2-double.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/odu03-input2-double.bin -------------------------------------------------------------------------------- /ex/odu03-output-double.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/odu03-output-double.bin -------------------------------------------------------------------------------- /ex/odu04-input-0-double.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/odu04-input-0-double.bin -------------------------------------------------------------------------------- /ex/odu05-output-double.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/odu05-output-double.bin -------------------------------------------------------------------------------- /ex/riu01-input-0-double.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/riu01-input-0-double.bin -------------------------------------------------------------------------------- /docs/images/ro-model-zoom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/docs/images/ro-model-zoom.png -------------------------------------------------------------------------------- /ex/dcu07-output-b-sd.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/dcu07-output-l-sd.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/dcu16-input-3byteblocks.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KeyPair-Consulting/Theseus/HEAD/ex/dcu16-input-3byteblocks.bin -------------------------------------------------------------------------------- /ex/dcu15-output-u32.bin: -------------------------------------------------------------------------------- 1 | ABCDEFGH01234567abcdefgh890 -------------------------------------------------------------------------------- /ex/dcu16-output-sd.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/dcu16-output-l-sd.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /ex/odu23-output-b1-16-u32.bin: -------------------------------------------------------------------------------- 1 |   2 |     -------------------------------------------------------------------------------- /ex/odu23-output-b2-8-u32.bin: -------------------------------------------------------------------------------- 1 |   2 |     -------------------------------------------------------------------------------- /ex/odu09-input-u128.bin: -------------------------------------------------------------------------------- 1 |  2 | -------------------------------------------------------------------------------- /ex/odu23-output-b1-10-u32.bin: -------------------------------------------------------------------------------- 1 | 2 |     !"#$%& ' -------------------------------------------------------------------------------- /ex/odu23-input-u32.bin: -------------------------------------------------------------------------------- 1 |  2 |  !"#$%&'()* -------------------------------------------------------------------------------- /ex/ipu01-input-u32.bin.select-3: -------------------------------------------------------------------------------- 1 | Best entropy up to weight 1: 0.58687744211050441 (mask: 0x10000000) 2 | Best entropy up to weight 2: 0.68058696835394372 (mask: 0x00000003) 3 | Best entropy up to weight 3: 0.68058696835394372 (mask: 0x00000003) 4 | Final Best Masks: 5 | 0x10000000 (weight 1): 0.58687744211050441 6 | 0x00000003 (weight 2): 0.68058696835394372 7 | test: 0x10000000 0x00000003 8 | -------------------------------------------------------------------------------- /ex/ipu01-input-u32.bin.select-3c: -------------------------------------------------------------------------------- 1 | Best entropy up to weight 1: 0 (mask: 0x00000001) 2 | Best entropy up to weight 2: 0.00088217340301532075 (mask: 0x10000002) 3 | Best entropy up to weight 3: 0.0028483489593084709 (mask: 0x50000002) 4 | Final Best Masks: 5 | 0x00000001 (weight 1): 0 6 | 0x10000002 (weight 2): 0.00088217340301532075 7 | 0x50000002 (weight 3): 0.0028483489593084709 8 | test: 0x00000001 0x10000002 0x50000002 9 | -------------------------------------------------------------------------------- /ex/ipu01-input-u32.bin.select-8c: -------------------------------------------------------------------------------- 1 | Best entropy up to weight 1: 0 (mask: 0x00000001) 2 | Best entropy up to weight 2: 0.00088217340301532075 (mask: 0x20000001) 3 | Best entropy up to weight 3: 0.0028483489593084709 (mask: 0x60000001) 4 | Best entropy up to weight 4: 0.0028483489593084709 (mask: 0x60000001) 5 | Final Best Masks: 6 | 0x00000001 (weight 1): 0 7 | 0x20000001 (weight 2): 0.00088217340301532075 8 | 0x60000001 (weight 3): 0.0028483489593084709 9 | test: 0x00000001 0x20000001 0x60000001 10 | -------------------------------------------------------------------------------- /src/cephes.h: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | 11 | #ifndef CEPHES_H_ 12 | #define CEPHES_H_ 13 | 14 | double cephes_igamc(double a, double x); 15 | double cephes_ndtri(double y0); 16 | 17 | #endif /* CEPHES_H_ */ 18 | -------------------------------------------------------------------------------- /src/translate.h: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #ifndef TRANSLATE_H 11 | #define TRANSLATE_H 12 | #include 13 | #include 14 | #include "entlib.h" 15 | bool translate(statData_t *S, size_t L, size_t *k, double *translatedMedian); 16 | #endif 17 | -------------------------------------------------------------------------------- /src/hashmodulus.h: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #ifndef HASHMODULUS_H 11 | #define HASHMODULUS_H 12 | 13 | #include "enttypes.h" 14 | 15 | #define MODULUSCOUNT 8 16 | // These are prime values, roughly doubling in size for each step 17 | static const signedStatData_t hashModulus[MODULUSCOUNT] = {1, 2, 5, 11, 31, 67, 127, 0}; 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Object files 5 | *.o 6 | *.ko 7 | *.obj 8 | *.elf 9 | 10 | # Linker output 11 | *.ilk 12 | *.map 13 | *.exp 14 | 15 | # Precompiled Headers 16 | *.gch 17 | *.pch 18 | 19 | # Libraries 20 | *.lib 21 | *.a 22 | *.la 23 | *.lo 24 | 25 | # Shared objects (inc. Windows DLLs) 26 | *.dll 27 | *.so 28 | *.so.* 29 | *.dylib 30 | 31 | # Executables 32 | *.exe 33 | *.out 34 | *.app 35 | *.i*86 36 | *.x86_64 37 | *.hex 38 | 39 | # Debug files 40 | *.dSYM/ 41 | *.su 42 | *.idb 43 | *.pdb 44 | 45 | # Kernel Module Compile Results 46 | *.mod* 47 | *.cmd 48 | .tmp_versions/ 49 | modules.order 50 | Module.symvers 51 | Mkfile.old 52 | dkms.conf 53 | -------------------------------------------------------------------------------- /src/sa.h: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #ifndef SA_H 11 | #define SA_H 12 | 13 | #include 14 | #include 15 | 16 | #include "entlib.h" 17 | 18 | #define SAIDX_MAX INT32_MAX 19 | #define SAIDX64_MAX INT64_MAX 20 | 21 | void calcSALCP(const statData_t *inData, size_t n, size_t k, saidx_t *SA, saidx_t *LCP); 22 | void calcSALCP64(const statData_t *inData, size_t n, size_t k, saidx64_t *SA, saidx64_t *LCP); 23 | #endif 24 | -------------------------------------------------------------------------------- /src/Makefile.ec2-aarch64: -------------------------------------------------------------------------------- 1 | #ec2 gcc aarch64 2 | CC=gcc 3 | 4 | MUTE=-Wno-conversion 5 | CFLAGS=-std=gnu11 -O2 -Wall -march=native -Wextra -fstack-protector-all -Wstack-protector --param ssp-buffer-size=4 -Wsign-conversion -Wformat-security -Werror -pie -fPIE -ftrapv -fstrict-overflow -fwrapv -fno-delete-null-pointer-checks -Wshadow -Wcast-qual -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wdeclaration-after-statement -Wformat=2 -Wswitch-default -Wswitch-enum -Wcast-align -Wpointer-arith -Wstrict-overflow=5 -Wundef -Wnested-externs -Wunreachable-code -Wlogical-op -Wfloat-equal -Wstrict-aliasing -Wredundant-decls -Wold-style-definition -Wtrampolines -ggdb3 -fno-omit-frame-pointer -ffloat-store -fno-common -fPIC -fPIE -fstrict-aliasing $(MUTE) -DUADDL_OVERFLOW 6 | 7 | include Makefile.rules 8 | -------------------------------------------------------------------------------- /src/globals.h: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #ifndef GLOBALS_H 11 | #define GLOBALS_H 12 | 13 | #define ERRORSLOTS 16 14 | #define LABELLEN 64 15 | 16 | #include 17 | #include 18 | 19 | extern int configVerbose; 20 | extern bool configBootstrapParams; 21 | extern size_t configThreadCount; 22 | extern double globalErrors[ERRORSLOTS]; 23 | extern char errorLabels[ERRORSLOTS][LABELLEN]; 24 | 25 | extern double configBootstrapConfidence; 26 | extern size_t configBootstrapRounds; 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /src/assessments.h: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #ifndef ASSESSMENTS_h 11 | #define ASSESSMENTS_h 12 | 13 | #include 14 | #include 15 | 16 | #include "entlib.h" 17 | #include "randlib.h" 18 | 19 | double bootstrapAssessments(struct entropyTestingResult *result, size_t count, size_t bitWidth, double *IIDminent, struct randstate *rstate); 20 | double bootstrapParameters(struct entropyTestingResult *result, size_t count, size_t bitWidth, double *IIDminent, struct randstate *rstate); 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /src/globals-inst.h: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #ifndef GLOBALSINST_H 11 | #define GLOBALSINST_H 12 | #include 13 | #include 14 | #include 15 | 16 | #include "entlib.h" 17 | #include "globals.h" 18 | 19 | int configVerbose = 0; 20 | bool configBootstrapParams = false; 21 | size_t configThreadCount = 0; 22 | double globalErrors[ERRORSLOTS] = {-1.0}; 23 | char errorLabels[ERRORSLOTS][LABELLEN] = {0}; 24 | 25 | double configBootstrapConfidence = 0.99; 26 | size_t configBootstrapRounds = 15000; 27 | #endif 28 | -------------------------------------------------------------------------------- /src/sd-to-dec.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "entlib.h" 22 | 23 | int main(void) { 24 | statData_t indata; 25 | 26 | while (feof(stdin) == 0) { 27 | if (fread(&indata, sizeof(statData_t), 1, stdin) == 1) { 28 | printf("%u\n", indata); 29 | } 30 | } 31 | 32 | return (0); 33 | } 34 | -------------------------------------------------------------------------------- /src/poolalloc.h: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #ifndef POOLALLOC_H 11 | #define POOLALLOC_H 12 | 13 | #include 14 | 15 | #define SEGMENTSIZEBOUND 134217728 // 128MB 16 | 17 | struct memSegment { 18 | char *segmentStart; 19 | void *nextFree; 20 | struct memSegment *nextSegment; 21 | size_t blockSize; 22 | size_t blockCount; 23 | }; 24 | 25 | struct memSegment *initPool(size_t bsize, size_t bcount); 26 | size_t delPool(struct memSegment *pool); 27 | void blockFree(void *discard, struct memSegment *pool); 28 | void *blockAlloc(struct memSegment *pool); 29 | #endif 30 | -------------------------------------------------------------------------------- /src/enttypes.h: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020-2024 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #ifndef ENTTYPES_H 11 | #define ENTTYPES_H 12 | 13 | #include 14 | 15 | #ifndef DBL_INFINITY 16 | #define DBL_INFINITY __builtin_inf() 17 | #endif 18 | 19 | #ifdef U32STATDATA 20 | #define statData_t uint32_t 21 | #define signedStatData_t int64_t 22 | #define STATDATA_MAX UINT32_MAX 23 | #define STATDATA_BITS 32U 24 | #define STATDATA_STRING "uint32_t" 25 | #else 26 | #define statData_t uint8_t 27 | #define signedStatData_t int16_t 28 | #define STATDATA_MAX UINT8_MAX 29 | #define STATDATA_BITS 8U 30 | #define STATDATA_STRING "uint8_t" 31 | #endif 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /src/Makefile: -------------------------------------------------------------------------------- 1 | #ec2 gcc is now gcc-11 2 | CC=gcc 3 | RDRND=-mrdrnd 4 | 5 | #gcc now explains how strncpy works. I know how it works. 6 | MUTE=-Wno-stringop-truncation -Wno-format-nonliteral 7 | CFLAGS=-std=gnu11 -O2 -Wall -m64 -march=native -Wextra -fstack-protector-all -Wstack-protector --param ssp-buffer-size=4 -Wsign-conversion -Wformat-security -Werror -pie -fPIE -ftrapv -fstrict-overflow -fwrapv -fno-delete-null-pointer-checks -Wshadow -Wcast-qual -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wdeclaration-after-statement -Wformat=2 -Wswitch-default -Wswitch-enum -Wcast-align -Wpointer-arith -Wstrict-overflow=5 -Wundef -Wnested-externs -Wunreachable-code -Wlogical-op -Wfloat-equal -Wstrict-aliasing -Wredundant-decls -Wold-style-definition -Wtrampolines -ggdb3 -fno-omit-frame-pointer -ffloat-store -fno-common -fPIC -fPIE -fstrict-aliasing -mbmi2 -mbmi -msse4.2 $(RDRND) -mfpmath=sse -msse2 $(MUTE) -DUADDL_OVERFLOW 8 | 9 | include Makefile.rules 10 | -------------------------------------------------------------------------------- /src/Makefile.ec2: -------------------------------------------------------------------------------- 1 | #ec2 gcc is now gcc-11 2 | CC=gcc 3 | RDRND=-mrdrnd 4 | 5 | #gcc now explains how strncpy works. I know how it works. 6 | MUTE=-Wno-stringop-truncation -Wno-format-nonliteral 7 | CFLAGS=-std=gnu11 -O2 -Wall -m64 -march=native -Wextra -fstack-protector-all -Wstack-protector --param ssp-buffer-size=4 -Wsign-conversion -Wformat-security -Werror -pie -fPIE -ftrapv -fstrict-overflow -fwrapv -fno-delete-null-pointer-checks -Wshadow -Wcast-qual -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wdeclaration-after-statement -Wformat=2 -Wswitch-default -Wswitch-enum -Wcast-align -Wpointer-arith -Wstrict-overflow=5 -Wundef -Wnested-externs -Wunreachable-code -Wlogical-op -Wfloat-equal -Wstrict-aliasing -Wredundant-decls -Wold-style-definition -Wtrampolines -ggdb3 -fno-omit-frame-pointer -ffloat-store -fno-common -fPIC -fPIE -fstrict-aliasing -mbmi2 -mbmi -msse4.2 $(RDRND) -mfpmath=sse -msse2 $(MUTE) -DUADDL_OVERFLOW 8 | 9 | include Makefile.rules 10 | -------------------------------------------------------------------------------- /src/Makefile.ec2-debug: -------------------------------------------------------------------------------- 1 | #ec2 gcc 2 | CC=gcc 3 | RDRND=-mrdrnd 4 | 5 | MUTE=-Wno-conversion 6 | 7 | CFLAGS=-std=gnu11 -Wall -m64 -march=native -Wextra -fstack-protector-all -Wstack-protector --param ssp-buffer-size=4 -Wsign-conversion -Wformat-security -Werror -pie -fPIE -ftrapv -fstrict-overflow -fwrapv -fno-delete-null-pointer-checks -Wshadow -Wcast-qual -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wdeclaration-after-statement -Wformat=2 -Wswitch-default -Wswitch-enum -Wcast-align -Wpointer-arith -Wstrict-overflow=5 -Wundef -Wnested-externs -Wunreachable-code -Wlogical-op -Wfloat-equal -Wstrict-aliasing -Wredundant-decls -Wold-style-definition -Wtrampolines -ggdb3 -fno-omit-frame-pointer -ffloat-store -fno-common -fPIC -fPIE -fstrict-aliasing -mbmi2 -mbmi -msse4.2 $(RDRND) -mfpmath=sse -msse2 $(MUTE) -DUADDL_OVERFLOW -O1 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all 8 | LDFLAGS=-lasan -lubsan 9 | 10 | include Makefile.rules 11 | -------------------------------------------------------------------------------- /src/u64-change-endianness.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2021 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, KeyPair Consulting, Inc. 8 | */ 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "binio.h" 16 | #include "binutil.h" 17 | #include "globals-inst.h" 18 | 19 | int main(void) { 20 | uint64_t *input = NULL; 21 | size_t datalen; 22 | 23 | if ((datalen = readuint64file(stdin, &input)) < 1) { 24 | perror("No data read."); 25 | exit(EX_DATAERR); 26 | } 27 | 28 | for (size_t i = 0; i < datalen; i++) { 29 | input[i] = reverse64(input[i]); 30 | } 31 | 32 | if (fwrite(input, sizeof(uint64_t), datalen, stdout) != datalen) { 33 | perror("Can't write out data"); 34 | } 35 | 36 | free(input); 37 | } 38 | -------------------------------------------------------------------------------- /src/u64-to-ascii.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include "precision.h" 17 | 18 | /*noreturn static void useageExit(void) 19 | { 20 | fprintf(stderr, "Usage:\n"); 21 | fprintf(stderr, "u64-to-ascii\n"); 22 | fprintf(stderr, "The values are expected to be provided via stdin.\n"); 23 | exit(EX_USAGE); 24 | }*/ 25 | 26 | int main(void) { 27 | uint64_t data; 28 | 29 | assert(PRECISION(UINT_MAX) == 32); 30 | 31 | while (feof(stdin) == 0) { 32 | if (fread(&data, sizeof(uint64_t), 1, stdin) == 1) { 33 | printf("%zu\n", data); 34 | } 35 | } 36 | 37 | return (0); 38 | } 39 | -------------------------------------------------------------------------------- /src/binio.h: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020-2024 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #ifndef BINIO_H 11 | #define BINIO_H 12 | 13 | #include 14 | #include "entlib.h" 15 | 16 | size_t readuint32file(FILE *input, uint32_t **buffer); 17 | size_t readuint64file(FILE *input, uint64_t **buffer); 18 | size_t readuintfile(FILE *input, statData_t **buffer); 19 | size_t readuintfileloc(FILE *input, statData_t **buffer, size_t subsetIndex, size_t subsetSize); 20 | size_t readdoublefile(FILE *input, double **buffer); 21 | size_t readasciidoubles(FILE *input, double **buffer); 22 | void mergeSortedLists(const double *in1, size_t len1, const double *in2, size_t len2, double *out); 23 | size_t readasciidoublepoints(FILE *input, double **buffer); 24 | size_t readasciiuint64s(FILE *input, uint64_t **buffer); 25 | #endif 26 | -------------------------------------------------------------------------------- /src/binutil.h: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #ifndef BINUTIL_H 11 | #define BINUTIL_H 12 | 13 | #include 14 | #include "entlib.h" 15 | 16 | uint32_t extractbits(const uint32_t input, const uint32_t bitMask); 17 | void extractbitsArray(const uint32_t *input, statData_t *output, const size_t datalen, const uint32_t bitMask); 18 | uint32_t expandBits(const uint32_t input, const uint32_t bitMask); 19 | uint32_t getActiveBits(const uint32_t *data, size_t datalen); 20 | statData_t getActiveBitsSD(const statData_t *data, size_t datalen); 21 | statData_t highBit(statData_t in); 22 | statData_t lowBit(statData_t in); 23 | uint32_t u32highBit(uint32_t in); 24 | uint32_t u32lowBit(uint32_t in); 25 | uint32_t reverse32(uint32_t input); 26 | uint64_t reverse64(uint64_t input); 27 | void reverse128(uint64_t *input); 28 | size_t serialXOR(statData_t *data, size_t datalen, size_t compression); 29 | #endif 30 | -------------------------------------------------------------------------------- /src/u32-to-ascii.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include "precision.h" 17 | 18 | /*noreturn static void useageExit(void) 19 | { 20 | fprintf(stderr, "Usage:\n"); 21 | fprintf(stderr, "u32-to-ascii\n"); 22 | fprintf(stderr, "Prints the provided binary data as human-readable decimal values.\n"); 23 | fprintf(stderr, "The values are expected to be provided via stdin.\n"); 24 | fprintf(stderr, "The values are output by stdout.\n"); 25 | exit(EX_USAGE); 26 | }*/ 27 | 28 | int main(void) { 29 | uint32_t data; 30 | 31 | assert(PRECISION(UINT_MAX) == 32); 32 | 33 | while (feof(stdin) == 0) { 34 | if (fread(&data, sizeof(uint32_t), 1, stdin) == 1) { 35 | printf("%u\n", data); 36 | } 37 | } 38 | 39 | return (0); 40 | } 41 | -------------------------------------------------------------------------------- /src/sd-to-hex.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "entlib.h" 22 | /* 23 | noreturn static void useageExit(void) { 24 | fprintf(stderr, "Usage:\n"); 25 | fprintf(stderr, "sd-to-hex\n"); 26 | fprintf(stderr, "Output the bytes in hex format.\n"); 27 | fprintf(stderr, "The values are expected to be provided via stdin.\n"); 28 | fprintf(stderr, "The output values are sent to stdout.\n"); 29 | exit(EX_USAGE); 30 | } 31 | */ 32 | 33 | int main(void) { 34 | statData_t indata; 35 | 36 | while (feof(stdin) == 0) { 37 | if (fread(&indata, sizeof(statData_t), 1, stdin) == 1) { 38 | printf("%X\n", indata); 39 | } 40 | } 41 | 42 | return (0); 43 | } 44 | -------------------------------------------------------------------------------- /src/dec-to-u64.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include "precision.h" 21 | 22 | /* 23 | noreturn static void useageExit(void) { 24 | fprintf(stderr, "Usage:\n"); 25 | fprintf(stderr, "dec-to-u32\n"); 26 | fprintf(stderr, "The values are expected to be provided via stdin, one per line.\n"); 27 | exit(EX_USAGE); 28 | } 29 | */ 30 | 31 | int main(void) { 32 | int res; 33 | uint64_t indata; 34 | 35 | while (feof(stdin) == 0) { 36 | res = scanf("%" SCNu64 "\n", &indata); 37 | if (res == 1) { 38 | if (fwrite(&indata, sizeof(uint64_t), 1, stdout) != 1) { 39 | perror("Can't write to stdout"); 40 | exit(EX_OSERR); 41 | } 42 | } 43 | } 44 | 45 | return (0); 46 | } 47 | -------------------------------------------------------------------------------- /src/hex-to-u32.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "precision.h" 21 | /* 22 | noreturn static void useageExit(void) { 23 | fprintf(stderr, "Usage:\n"); 24 | fprintf(stderr, "hex-to-u32\n"); 25 | fprintf(stderr, "The values are expected to be provided via stdin, one per line.\n"); 26 | exit(EX_USAGE); 27 | } 28 | */ 29 | 30 | int main(void) { 31 | uint32_t data; 32 | int res; 33 | uint32_t indata; 34 | 35 | while (feof(stdin) == 0) { 36 | res = scanf("%x\n", &indata); 37 | data = (uint32_t)indata; 38 | if (res == 1) { 39 | if (fwrite(&data, sizeof(uint32_t), 1, stdout) != 1) { 40 | perror("Can't write to stdout"); 41 | exit(EX_OSERR); 42 | } 43 | } 44 | } 45 | 46 | return (0); 47 | } 48 | -------------------------------------------------------------------------------- /src/u8-to-u32.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include "precision.h" 19 | 20 | /*noreturn static void useageExit(void) 21 | { 22 | fprintf(stderr, "Usage:\n"); 23 | fprintf(stderr, "u8-to-u32\n"); 24 | fprintf(stderr, "The values are expected to be provided via stdin.\n"); 25 | exit(EX_USAGE); 26 | } 27 | */ 28 | int main(void) { 29 | uint32_t outdata; 30 | uint8_t indata; 31 | 32 | assert(PRECISION(UINT_MAX) == 32); 33 | 34 | while (feof(stdin) == 0) { 35 | if (fread(&indata, sizeof(uint8_t), 1, stdin) == 1) { 36 | outdata = (uint8_t)indata; 37 | if (fwrite(&outdata, sizeof(uint32_t), 1, stdout) != 1) { 38 | perror("Can't write to output"); 39 | exit(EX_OSERR); 40 | } 41 | } 42 | } 43 | 44 | return (0); 45 | } 46 | -------------------------------------------------------------------------------- /src/incbeta.h: -------------------------------------------------------------------------------- 1 | /* 2 | * zlib License 3 | * 4 | * Regularized Incomplete Beta Function 5 | * 6 | * Copyright (c) 2016, 2017 Lewis Van Winkle 7 | * http://CodePlea.com 8 | * 9 | * This software is provided 'as-is', without any express or implied 10 | * warranty. In no event will the authors be held liable for any damages 11 | * arising from the use of this software. 12 | * 13 | * Permission is granted to anyone to use this software for any purpose, 14 | * including commercial applications, and to alter it and redistribute it 15 | * freely, subject to the following restrictions: 16 | * 17 | * 1. The origin of this software must not be misrepresented; you must not 18 | * claim that you wrote the original software. If you use this software 19 | * in a product, an acknowledgement in the product documentation would be 20 | * appreciated but is not required. 21 | * 2. Altered source versions must be plainly marked as such, and must not be 22 | * misrepresented as being the original software. 23 | * 3. This notice may not be removed or altered from any source distribution. 24 | */ 25 | 26 | /* This file is included as part of the Theseus distribution. 27 | */ 28 | 29 | #ifndef INCBETA_H 30 | #define INCBETA_H 31 | 32 | double incbeta(double a, double b, double x); 33 | 34 | #endif /*INCBETA_H*/ 35 | -------------------------------------------------------------------------------- /src/dec-to-u32.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include "precision.h" 20 | 21 | /* 22 | noreturn static void useageExit(void) { 23 | fprintf(stderr, "Usage:\n"); 24 | fprintf(stderr, "dec-to-u32\n"); 25 | fprintf(stderr, "The values are expected to be provided via stdin, one per line.\n"); 26 | exit(EX_USAGE); 27 | } 28 | */ 29 | 30 | int main(void) { 31 | uint32_t data; 32 | int res; 33 | uint32_t indata; 34 | 35 | assert(PRECISION(UINT_MAX) == 32); 36 | 37 | while (feof(stdin) == 0) { 38 | res = scanf("%u\n", &indata); 39 | data = (uint32_t)indata; 40 | if (res == 1) { 41 | if (fwrite(&data, sizeof(uint32_t), 1, stdout) != 1) { 42 | perror("Can't write to stdout"); 43 | exit(EX_OSERR); 44 | } 45 | } 46 | } 47 | 48 | return (0); 49 | } 50 | -------------------------------------------------------------------------------- /src/hweight.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include "precision.h" 20 | 21 | noreturn static void useageExit(void) { 22 | fprintf(stderr, "Usage:\n"); 23 | fprintf(stderr, "hweight bitmask\n"); 24 | fprintf(stderr, "output is the hamming weight of the bitmask\n"); 25 | exit(EX_USAGE); 26 | } 27 | 28 | int main(int argc, char *argv[]) { 29 | long long int inint; 30 | uint32_t bitMask; 31 | uint32_t outputBits; 32 | 33 | assert(PRECISION(LLONG_MAX) > 32); 34 | 35 | if (argc != 2) { 36 | useageExit(); 37 | } 38 | 39 | inint = strtoll(argv[1], NULL, 0); 40 | if ((inint < 0) || (inint > UINT_MAX)) { 41 | useageExit(); 42 | } 43 | 44 | bitMask = (uint32_t)inint; 45 | outputBits = (uint32_t)__builtin_popcount(bitMask); 46 | 47 | if ((outputBits == 0) || (outputBits > 32)) { 48 | useageExit(); 49 | } 50 | 51 | printf("%u\n", outputBits); 52 | 53 | return EX_OK; 54 | } 55 | -------------------------------------------------------------------------------- /src/u32-xor-diff.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | /*noreturn static void useageExit(void) 18 | { 19 | fprintf(stderr, "Usage:\n"); 20 | fprintf(stderr, "u32-xor-diff\n"); 21 | fprintf(stderr, "Output the running XOR of adjacent values.\n"); 22 | fprintf(stderr, "The values are expected to be provided via stdin.\n"); 23 | exit(EX_USAGE); 24 | }*/ 25 | 26 | int main(void) { 27 | uint32_t currentData; 28 | uint32_t priorData; 29 | uint32_t rawData; 30 | 31 | // Read in the first value 32 | if (fread(&priorData, sizeof(uint32_t), 1, stdin) != 1) { 33 | perror("Can't read initial value"); 34 | exit(EX_OSERR); 35 | } 36 | 37 | while (feof(stdin) == 0) { 38 | if (fread(¤tData, sizeof(uint32_t), 1, stdin) == 1) { 39 | rawData = priorData ^ currentData; 40 | if (fwrite(&rawData, sizeof(uint32_t), 1, stdout) != 1) { 41 | perror("Can't write to output"); 42 | exit(EX_OSERR); 43 | } 44 | priorData = currentData; 45 | } 46 | } 47 | 48 | return (0); 49 | } 50 | -------------------------------------------------------------------------------- /src/bootstrap.h: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #ifndef BOOTSTRAP_H 11 | #define BOOTSTRAP_H 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #include "fancymath.h" 26 | #include "randlib.h" 27 | 28 | double calculatePercentile(double p, double *data, size_t datalen, double validMin, double validMax); 29 | double BCaBootstrapPercentile(double p, double *data, size_t datalen, double validMin, double validMax, double *confidenceInterval, size_t rounds, double alpha, struct randstate *rstate); 30 | double calculateMean(double *data, size_t datalen); 31 | double BCaBootstrapMean(double *data, size_t datalen, double validMin, double validMax, double *confidenceInterval, size_t rounds, double alpha, struct randstate *rstate); 32 | size_t aboveValue(double value, const double *data, size_t datalen); 33 | size_t belowValue(double value, const double *data, size_t datalen); 34 | size_t trimDoubleRange(double **dataptr, size_t datalen, double validMin, double validMax); 35 | double processedCalculatePercentile(double p, double *data, size_t datalen, bool sorted, int extraVerbose); 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /ex/hpu02-input.txt: -------------------------------------------------------------------------------- 1 | 0.0, 0.0 2 | 0.010000007399197262,0.097507479880620529 3 | 0.020000014798394524,0.13931019590278582 4 | 0.030000022197591791,0.15463122944266899 5 | 0.040000029596789048,0.16202776346734171 6 | 0.050000036995986319,0.17151353943342887 7 | 0.060000044395183583,0.18751052556718961 8 | 0.070000051794380833,0.20658938499691396 9 | 0.080000059193578096,0.22852197721055623 10 | 0.09000006659277536,0.25382356204949263 11 | 0.10000007399197264,0.28106592079303278 12 | 0.1100000813911699,0.30987423659567748 13 | 0.12000008879036717,0.34030862469565337 14 | 0.1300000961895644,0.37224420006914999 15 | 0.14000010358876167,0.40566061267991393 16 | 0.15000011098795893,0.44015948707913793 17 | 0.16000011838715619,0.47563046057714325 18 | 0.17000012578635348,0.51173183601398409 19 | 0.18000013318555072,0.54800746544817724 20 | 0.19000014058474798,0.58412213577372629 21 | 0.20000014798394528,0.61982885293353562 22 | 0.21000015538314251,0.65397675949125578 23 | 0.2200001627823398,0.68684282381127126 24 | 0.23000017018153704,0.71729234714854395 25 | 0.24000017758073433,0.7447852090794298 26 | 0.25000018497993159,0.76901445368437227 27 | 0.2600001923791288,0.78988924236601388 28 | 0.27000019977832612,0.80676481650229426 29 | 0.28000020717752333,0.81968394103212716 30 | 0.29000021457672065,0.82956412972112203 31 | 0.30050022234587775,0.8369629439616838 32 | 0.31050022974507502,0.84160214308407733 33 | 0.32050023714427228,0.84489377742745797 34 | 0.3335002467632287,0.84721274265511493 35 | 0.34150025268258649,0.84800016591045169 36 | 0.35000025897190418,0.84873552601543445 37 | 0.36000026637110144,0.84907465091678369 38 | 0.37000027377029876,0.84934557928626186 -------------------------------------------------------------------------------- /src/dictionaryTree.h: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #ifndef DICTTREE_H 11 | #define DICTTREE_H 12 | 13 | #include 14 | #include 15 | #include "enttypes.h" 16 | #include "poolalloc.h" 17 | 18 | // These contain the actual hash table (dictionaryEntry) 19 | // 16 bytes (ish) each for 1 byte statData 20 | struct dictionaryPage { 21 | signedStatData_t curModulus; // We need to be able to represent k 22 | bool prefixFound; // Has the prefix ending at this node been encountered? 23 | statData_t maxEntry; // What is the max postfix? 24 | // The table containing entries. The postfix is present iff count>0. 25 | struct dictionaryEntry *entries; 26 | }; 27 | 28 | // Each node in the hash table must identify the symbol 29 | // 24 bytes (ish) each 30 | struct dictionaryEntry { 31 | statData_t symbol; 32 | size_t count; 33 | struct dictionaryPage *branch; 34 | }; 35 | 36 | bool treeIncrementDict(const statData_t *prior, size_t pLen, struct dictionaryPage *head, size_t k, bool createEntry, bool leafCounts, struct dictionaryPage *prefixLoc, struct memSegment **mempools); 37 | size_t treePredictDict(const statData_t *prior, size_t pLen, statData_t *next, struct dictionaryPage *head, struct dictionaryPage **prefixLoc); 38 | struct dictionaryPage *newDictionaryPage(struct memSegment **mempools); 39 | size_t delDictionary(struct dictionaryPage *head, size_t k, size_t *modulusCount, size_t *occupiedCount, struct memSegment **mempools); 40 | #endif 41 | -------------------------------------------------------------------------------- /src/sigfigs.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2024 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, KeyPair Consulting, Inc. 8 | */ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | int main(int argc, char *argv[]) { 17 | char *endptr; 18 | unsigned long long int inint; 19 | double indouble; 20 | unsigned int sigfigs; 21 | char fmt[256]; 22 | 23 | if(argc != 3) { 24 | fprintf(stderr, "sigfigs \n"); 25 | fprintf(stderr, "Got %d parameters.\n", argc); 26 | exit(EX_NOINPUT); 27 | } 28 | 29 | endptr = NULL; 30 | indouble = strtod(argv[1], &endptr); 31 | 32 | #pragma GCC diagnostic push 33 | #pragma GCC diagnostic ignored "-Wfloat-equal" 34 | //These double values are flags set by "strtod", so exact comparison is correct. 35 | if ((endptr == NULL) || (*endptr != '\0') || ((indouble == 0.0) && (endptr == argv[1])) || (((indouble == HUGE_VAL) || (indouble == -HUGE_VAL)) && errno == ERANGE)) { 36 | fprintf(stderr, "Can't interpret target double value\n"); 37 | exit(EX_DATAERR); 38 | } 39 | #pragma GCC diagnostic pop 40 | 41 | endptr = NULL; 42 | inint = strtoull(argv[2], &endptr, 0); 43 | 44 | if((inint == 0) || (inint > 22) || (endptr == NULL) || (*endptr != '\0')) { 45 | fprintf(stderr, "Invalid number of significant figures.\n"); 46 | exit(EX_DATAERR); 47 | } 48 | 49 | sigfigs = (unsigned int) inint; 50 | 51 | sprintf(fmt, "%%.0%ug", sigfigs); 52 | printf(fmt, indouble); 53 | printf("\n"); 54 | return 0; 55 | } 56 | -------------------------------------------------------------------------------- /src/u64-counter-endian.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "binio.h" 17 | #include "binutil.h" 18 | #include "globals-inst.h" 19 | 20 | int main(void) { 21 | uint64_t *input = NULL; 22 | uint64_t *swappedData = NULL; 23 | uint64_t *outData = NULL; 24 | size_t datalen; 25 | size_t nativeSmallerCount = 0; 26 | 27 | if ((datalen = readuint64file(stdin, &input)) < 1) { 28 | perror("No data read."); 29 | exit(EX_DATAERR); 30 | } 31 | 32 | swappedData = malloc(sizeof(uint64_t) * datalen); 33 | 34 | if (swappedData == NULL) { 35 | perror("Can't allocate memory."); 36 | exit(EX_OSERR); 37 | } 38 | 39 | for (size_t i = 0; i < datalen; i++) { 40 | swappedData[i] = reverse64(input[i]); 41 | if (input[i] <= swappedData[i]) nativeSmallerCount++; 42 | } 43 | 44 | if (nativeSmallerCount >= (datalen - nativeSmallerCount)) { 45 | fprintf(stderr, "Native byte order seems better (%g)\n", (double)nativeSmallerCount / (double)datalen); 46 | outData = input; 47 | } else { 48 | fprintf(stderr, "Swapped byte order seems better (%g)\n", (double)(datalen - nativeSmallerCount) / (double)datalen); 49 | outData = swappedData; 50 | } 51 | 52 | if (fwrite(outData, sizeof(uint64_t), datalen, stdout) != datalen) { 53 | perror("Can't write out data"); 54 | } 55 | 56 | free(swappedData); 57 | free(input); 58 | } 59 | -------------------------------------------------------------------------------- /src/Makefile.mem: -------------------------------------------------------------------------------- 1 | #for static analysis, use 2 | #scan-build-20 --use-cc=/usr/bin/clang-20 make 3 | #and setup the compiler as clang 4 | 5 | RDRND=-mrdrnd 6 | 7 | #debugging 8 | CC=clang-20 9 | #Silence selected warnings: 10 | #-Wno-padded 11 | #It is not clear why I should care about padding the compiler is adding. 12 | # 13 | #-Wno-bad-function-cast 14 | #I don't see a problem with the construction (int)floor(1.3) and this is used here. 15 | # 16 | #-Wno-gnu-statement-expression 17 | #GCC MACRO extensions are used (clang supports these) 18 | # 19 | #-Wno-unsafe-buffer-usage 20 | #This seems to exist to push users toward a C++ feature, but this is in C. 21 | # 22 | #-Wno-pre-c11-compat 23 | #This codebase is fundamentally c99 targeted (it uses some of the floating point fanciness introduced in C99, 24 | #and declares variables in for loop initializers). _Noreturn is used, which is a c11 feature. 25 | #Such warnings about such uses aren't useful. 26 | MUTE=-Wno-padded -Wno-bad-function-cast -Wno-gnu-statement-expression -Wno-unsafe-buffer-usage -Wno-pre-c11-compat -Wno-format-nonliteral 27 | 28 | #For general debugging (e.g., with static analyzer) 29 | #standard debug 30 | #DEBUGCFLAGS=-O2 31 | #DEBUGLDFLAGS=-lprofiler 32 | #undefined flag and ASAN 33 | #DEBUGCFLAGS=-O1 -fsanitize=address -fsanitize=undefined 34 | #DEBUGLDFLAGS=-fsanitize=address -fsanitize=undefined 35 | #For sanitize memory 36 | DEBUGCFLAGS=-O1 -fsanitize-memory-track-origins -fsanitize=memory -fno-optimize-sibling-calls -g 37 | DEBUGLDFLAGS=-fsanitize=memory -g 38 | #for thread san 39 | #DEBUGCFLAGS=-O1 -fsanitize=thread 40 | #DEBUGLDFLAGS=-fsanitize=thread 41 | 42 | CFLAGS=-Wall -Wextra -Weverything -march=native -mfpmath=sse -msse2 -msse4.2 -fno-omit-frame-pointer -Wformat -Wformat-security -Werror=format-security -Werror $(RDRND) $(MUTE) -DUADDL_OVERFLOW -fdenormal-fp-math=ieee $(DEBUGCFLAGS) 43 | LDFLAGS=$(DEBUGLDFLAGS) 44 | 45 | include Makefile.rules 46 | -------------------------------------------------------------------------------- /src/u32-to-u64.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2025 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include // for getopt, optind 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "binutil.h" 21 | #include "globals-inst.h" 22 | #include "precision.h" 23 | 24 | noreturn static void useageExit(void) { 25 | fprintf(stderr, "Usage:\n"); 26 | fprintf(stderr, "u32-to-u64 [-r] [-t]\n"); 27 | fprintf(stderr, "-r\tSwitch endianness of input values.\n"); 28 | fprintf(stderr, "The values are expected to be provided via stdin and the output via stdout.\n"); 29 | exit(EX_USAGE); 30 | } 31 | 32 | int main(int argc, char *argv[]) { 33 | size_t res; 34 | uint32_t inData; 35 | uint64_t out; 36 | bool configReverse; 37 | int opt; 38 | 39 | configReverse = false; 40 | 41 | while ((opt = getopt(argc, argv, "r")) != -1) { 42 | switch (opt) { 43 | case 'r': 44 | configReverse = true; 45 | break; 46 | default: /* ? */ 47 | useageExit(); 48 | } 49 | } 50 | 51 | if (argc != optind) { 52 | useageExit(); 53 | } 54 | 55 | while (feof(stdin) == 0) { 56 | res = fread(&inData, sizeof(uint32_t), 1, stdin); 57 | 58 | if (res == 1) { 59 | out = (uint64_t)inData; 60 | if (configReverse) { 61 | out = reverse64(out); 62 | } 63 | 64 | if (fwrite(&out, sizeof(uint64_t), 1, stdout) != 1) { 65 | perror("Can't write to stdout"); 66 | exit(EX_OSERR); 67 | } 68 | } 69 | } 70 | 71 | return (0); 72 | } 73 | -------------------------------------------------------------------------------- /src/Makefile.asan: -------------------------------------------------------------------------------- 1 | #for static analysis, use 2 | #scan-build-20 --use-cc=/usr/bin/clang-20 make 3 | #and setup the compiler as clang 4 | 5 | RDRND=-mrdrnd 6 | 7 | #debugging 8 | CC=clang-20 9 | #Silence selected warnings: 10 | #-Wno-padded 11 | #It is not clear why I should care about padding the compiler is adding. 12 | # 13 | #-Wno-bad-function-cast 14 | #I don't see a problem with the construction (int)floor(1.3) and this is used here. 15 | # 16 | #-Wno-gnu-statement-expression 17 | #GCC MACRO extensions are used (clang supports these) 18 | # 19 | #-Wno-unsafe-buffer-usage 20 | #This seems to exist to push users toward a C++ feature, but this is in C. 21 | # 22 | #-Wno-pre-c11-compat 23 | #This codebase is fundamentally c99 targeted (it uses some of the floating point fanciness introduced in C99, 24 | #and declares variables in for loop initializers). _Noreturn is used, which is a c11 feature. 25 | #Such warnings about such uses aren't useful. 26 | MUTE=-Wno-padded -Wno-bad-function-cast -Wno-gnu-statement-expression -Wno-unsafe-buffer-usage -Wno-pre-c11-compat -Wno-format-nonliteral 27 | 28 | #For general debugging (e.g., with static analyzer) 29 | #standard debug 30 | #DEBUGCFLAGS=-O2 31 | #DEBUGLDFLAGS=-lprofiler 32 | #undefined flag and ASAN 33 | DEBUGCFLAGS=-O1 -fsanitize=address -fsanitize=leak -fsanitize=undefined -fno-optimize-sibling-calls -g 34 | DEBUGLDFLAGS=-fsanitize=address -fsanitize=leak -fsanitize=undefined -g 35 | #For sanitize memory 36 | #DEBUGCFLAGS=-O1 -fsanitize-memory-track-origins -fsanitize=memory 37 | #DEBUGLDFLAGS=-fsanitize=memory 38 | #for thread san 39 | #DEBUGCFLAGS=-O1 -fsanitize=thread 40 | #DEBUGLDFLAGS=-fsanitize=thread 41 | 42 | CFLAGS=-Wall -Wextra -Weverything -march=native -mfpmath=sse -msse2 -msse4.2 -fno-omit-frame-pointer -Wformat -Wformat-security -Werror=format-security -Werror $(RDRND) $(MUTE) -DUADDL_OVERFLOW -fdenormal-fp-math=ieee $(DEBUGCFLAGS) 43 | LDFLAGS=$(DEBUGLDFLAGS) 44 | 45 | include Makefile.rules 46 | -------------------------------------------------------------------------------- /src/Makefile.thread: -------------------------------------------------------------------------------- 1 | #for static analysis, use 2 | #scan-build-20 --use-cc=/usr/bin/clang-20 make 3 | #and setup the compiler as clang 4 | 5 | RDRND=-mrdrnd 6 | 7 | #debugging 8 | CC=clang-20 9 | #Silence selected warnings: 10 | #-Wno-padded 11 | #It is not clear why I should care about padding the compiler is adding. 12 | # 13 | #-Wno-bad-function-cast 14 | #I don't see a problem with the construction (int)floor(1.3) and this is used here. 15 | # 16 | #-Wno-gnu-statement-expression 17 | #GCC MACRO extensions are used (clang supports these) 18 | # 19 | #-Wno-unsafe-buffer-usage 20 | #This seems to exist to push users toward a C++ feature, but this is in C. 21 | # 22 | #-Wno-pre-c11-compat 23 | #This codebase is fundamentally c99 targeted (it uses some of the floating point fanciness introduced in C99, 24 | #and declares variables in for loop initializers). _Noreturn is used, which is a c11 feature. 25 | #Such warnings about such uses aren't useful. 26 | MUTE=-Wno-padded -Wno-bad-function-cast -Wno-gnu-statement-expression -Wno-unsafe-buffer-usage -Wno-pre-c11-compat -Wno-format-nonliteral 27 | 28 | #For general debugging (e.g., with static analyzer) 29 | #standard debug 30 | #DEBUGCFLAGS=-O2 31 | #DEBUGLDFLAGS=-lprofiler 32 | #undefined flag and ASAN 33 | #DEBUGCFLAGS=-O1 -fsanitize=address -fsanitize=undefined 34 | #DEBUGLDFLAGS=-fsanitize=address -fsanitize=undefined 35 | #For sanitize memory 36 | #DEBUGCFLAGS=-O1 -fsanitize-memory-track-origins -fsanitize=memory 37 | #DEBUGLDFLAGS=-fsanitize=memory 38 | #for thread san 39 | DEBUGCFLAGS=-O1 -fsanitize=thread -g -fno-optimize-sibling-calls 40 | DEBUGLDFLAGS=-fsanitize=thread -g 41 | 42 | CFLAGS=-Wall -Wextra -Weverything -march=native -mfpmath=sse -msse2 -msse4.2 -O2 -g -fno-omit-frame-pointer -Wformat -Wformat-security -Werror=format-security -fno-optimize-sibling-calls -Werror $(RDRND) $(MUTE) -DUADDL_OVERFLOW -fdenormal-fp-math=ieee $(DEBUGCFLAGS) 43 | LDFLAGS=$(DEBUGLDFLAGS) 44 | 45 | include Makefile.rules 46 | -------------------------------------------------------------------------------- /src/Makefile.debug: -------------------------------------------------------------------------------- 1 | #for static analysis, use 2 | #scan-build-20 --use-cc=/usr/bin/clang-20 make 3 | #and setup the compiler as clang 4 | 5 | RDRND=-mrdrnd 6 | 7 | #debugging 8 | CC=clang-20 9 | #Silence selected warnings: 10 | #-Wno-padded 11 | #It is not clear why I should care about padding the compiler is adding. 12 | # 13 | #-Wno-bad-function-cast 14 | #I don't see a problem with the construction (int)floor(1.3) and this is used here. 15 | # 16 | #-Wno-gnu-statement-expression 17 | #GCC MACRO extensions are used (clang supports these) 18 | # 19 | #-Wno-unsafe-buffer-usage 20 | #This seems to exist to push users toward a C++ feature, but this is in C. 21 | # 22 | #-Wno-pre-c11-compat 23 | #This codebase is fundamentally c99 targeted (it uses some of the floating point fanciness introduced in C99, 24 | #and declares variables in for loop initializers). _Noreturn is used, which is a c11 feature. 25 | #Such warnings about such uses aren't useful. 26 | MUTE=-Wno-padded -Wno-bad-function-cast -Wno-gnu-statement-expression -Wno-unsafe-buffer-usage -Wno-pre-c11-compat -Wno-format-nonliteral 27 | 28 | #For general debugging (e.g., with static analyzer) 29 | #standard debug 30 | DEBUGCFLAGS=-O2 31 | #DEBUGLDFLAGS=-flto -lprofiler 32 | DEBUGLDFLAGS= 33 | #undefined flag and ASAN 34 | #DEBUGCFLAGS=-O1 -fsanitize=address -fsanitize=undefined 35 | #DEBUGLDFLAGS=-fsanitize=address -fsanitize=undefined 36 | #For sanitize memory 37 | #DEBUGCFLAGS=-O1 -fsanitize-memory-track-origins -fsanitize=memory 38 | #DEBUGLDFLAGS=-fsanitize=memory 39 | #for thread san 40 | #DEBUGCFLAGS=-O1 -fsanitize=thread 41 | #DEBUGLDFLAGS=-fsanitize=thread 42 | 43 | CFLAGS=-Wall -Wextra -Weverything -march=native -mfpmath=sse -msse2 -msse4.2 -g -fno-omit-frame-pointer -Wformat -Wformat-security -Werror=format-security -Werror $(RDRND) $(MUTE) -DUADDL_OVERFLOW -fdenormal-fp-math=ieee $(DEBUGCFLAGS) -flto -I/usr/lib/llvm-20/lib/clang/20/include 44 | LDFLAGS=-flto -fuse-ld=lld $(DEBUGLDFLAGS) 45 | 46 | include Makefile.rules 47 | -------------------------------------------------------------------------------- /src/u32-xor.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020-2024 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | noreturn static void useageExit(void) { 18 | fprintf(stderr, "Usage:\n"); 19 | fprintf(stderr, "u32-xor \n"); 20 | fprintf(stderr, "Outputs the value of XORed with \n"); 21 | fprintf(stderr, "The output values are send to stdout.\n"); 22 | exit(EX_USAGE); 23 | } 24 | 25 | int main(int argc, char *argv[]) { 26 | uint32_t currentData1, currentData2; 27 | uint32_t outData; 28 | FILE *fp1, *fp2; 29 | bool ready1, ready2; 30 | 31 | if (argc != 3) useageExit(); 32 | 33 | if ((fp1 = fopen(argv[1], "rb")) == NULL) { 34 | perror("Can't open first file\n"); 35 | useageExit(); 36 | } 37 | if ((fp2 = fopen(argv[2], "rb")) == NULL) { 38 | perror("Can't open second file.\n"); 39 | useageExit(); 40 | } 41 | 42 | currentData1 = currentData2 = 0; 43 | ready1 = ready2 = false; 44 | 45 | while ((feof(fp1) == 0) && (feof(fp2) == 0) && (ferror(fp1) == 0) && (ferror(fp2) == 0)) { 46 | if (!ready1 && (fread(¤tData1, sizeof(uint32_t), 1, fp1) == 1)) { 47 | ready1 = true; 48 | } 49 | if (!ready2 && (fread(¤tData2, sizeof(uint32_t), 1, fp2) == 1)) { 50 | ready2 = true; 51 | } 52 | 53 | if (ready1 && ready2) { 54 | outData = currentData1 ^ currentData2; 55 | if (fwrite(&outData, sizeof(uint32_t), 1, stdout) != 1) { 56 | perror("Can't write to output"); 57 | exit(EX_OSERR); 58 | } 59 | ready1 = false; 60 | ready2 = false; 61 | } 62 | } 63 | 64 | fclose(fp1); 65 | fclose(fp2); 66 | 67 | return 0; 68 | } 69 | -------------------------------------------------------------------------------- /src/bits-in-use.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "binio.h" 22 | #include "binutil.h" 23 | #include "globals-inst.h" 24 | #include "precision.h" 25 | 26 | noreturn static void useageExit(void) { 27 | fprintf(stderr, "Usage:\n"); 28 | fprintf(stderr, "bits-in-use \n"); 29 | fprintf(stderr, "inputfile is assumed to consist of uint32_ts\n"); 30 | fprintf(stderr, "outputs the number of bits required to represent data after removing stuck and superfluous bits.\n"); 31 | exit(EX_USAGE); 32 | } 33 | 34 | int main(int argc, char *argv[]) { 35 | FILE *infp; 36 | size_t datalen; 37 | uint32_t *data = NULL; 38 | uint32_t bitmask; 39 | uint32_t outputBits; 40 | 41 | configVerbose = 2; 42 | 43 | if (argc != 2) { 44 | useageExit(); 45 | } 46 | 47 | if ((infp = fopen(argv[1], "rb")) == NULL) { 48 | perror("Can't open file"); 49 | exit(EX_NOINPUT); 50 | } 51 | 52 | if ((datalen = readuint32file(infp, &data)) < 1) { 53 | perror("File contains no data"); 54 | exit(EX_DATAERR); 55 | } 56 | assert(data != NULL); 57 | 58 | fprintf(stderr, "Read in %zu uint32_ts\n", datalen); 59 | if (fclose(infp) != 0) { 60 | perror("Can't close input file"); 61 | exit(EX_OSERR); 62 | } 63 | 64 | bitmask = getActiveBits(data, datalen); 65 | 66 | fprintf(stderr, "Found bitmask 0x%08x\n", bitmask); 67 | 68 | outputBits = (uint32_t)(__builtin_popcountl(bitmask)); 69 | 70 | printf("%u\n", outputBits); 71 | 72 | free(data); 73 | 74 | return EX_OK; 75 | } 76 | -------------------------------------------------------------------------------- /src/u32-anddata.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020-2024 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include "precision.h" 20 | 21 | noreturn static void useageExit(void) { 22 | fprintf(stderr, "Usage:\n"); 23 | fprintf(stderr, "u32-anddata [inputfile] \n"); 24 | fprintf(stderr, "Takes the uint32_t symbols in (machine format) and bitwise ANDs each symbol with .\n"); 25 | fprintf(stderr, "inputfile is assumed to be a stream of uint32_ts\n"); 26 | fprintf(stderr, "Outputs uint32_ts to stdout\n"); 27 | fprintf(stderr, "The result is the data bitwise anded with the provided bitmask output to stdout\n"); 28 | exit(EX_USAGE); 29 | } 30 | 31 | int main(int argc, char *argv[]) { 32 | FILE *infp; 33 | uint32_t data; 34 | uint32_t andMask; 35 | long long int inll; 36 | 37 | if (argc == 3) { 38 | if ((infp = fopen(argv[1], "rb")) == NULL) { 39 | perror("Can't open file"); 40 | exit(EX_NOINPUT); 41 | } 42 | argv++; 43 | } else if (argc == 2) { 44 | infp = stdin; 45 | } else { 46 | useageExit(); 47 | } 48 | 49 | inll = strtoll(argv[1], NULL, 0); 50 | if ((inll > 0) && (inll < (int long long)UINT32_MAX)) { 51 | andMask = (uint32_t)inll; 52 | } else { 53 | useageExit(); 54 | } 55 | 56 | fprintf(stderr, "Andmask: 0x%08x\n", andMask); 57 | 58 | fprintf(stderr, "Outputting data\n"); 59 | 60 | while (feof(infp) == 0) { 61 | if (fread(&data, sizeof(uint32_t), 1, infp) == 1) { 62 | data &= andMask; 63 | 64 | if (fwrite(&data, sizeof(uint32_t), 1, stdout) != 1) { 65 | perror("Can't write output to stdout"); 66 | exit(EX_OSERR); 67 | } 68 | } else { 69 | perror("Can't read input from file"); 70 | exit(EX_OSERR); 71 | } 72 | } 73 | 74 | fclose(infp); 75 | 76 | return EX_OK; 77 | } 78 | -------------------------------------------------------------------------------- /src/mementsource.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "globals-inst.h" 18 | #include "randlib.h" 19 | 20 | // Make sure to lock this to a single core when testing. 21 | // make the entire memory block 2^24 bytes 22 | #define BLOCKMASKLEN 24U 23 | #define BLOCKLEN (1U << BLOCKMASKLEN) 24 | #define BLOCKMASK (BLOCKLEN - 1U) 25 | 26 | #define OUTPUTSIZE 1000001 27 | 28 | __attribute__ ((optnone)) int main(void) { 29 | FILE *fp; 30 | uint32_t curoffset = 0; 31 | uint32_t *data; 32 | uint32_t *outputtag; 33 | uint64_t *outputtsc; 34 | struct randstate rstate; 35 | 36 | initGenerator(&rstate); 37 | 38 | seedGenerator(&rstate); 39 | 40 | // Allocate the block 41 | data = malloc(sizeof(uint32_t) * BLOCKLEN); 42 | assert(data != NULL); 43 | 44 | outputtsc = malloc(sizeof(uint64_t) * OUTPUTSIZE); 45 | assert(outputtsc != NULL); 46 | 47 | outputtag = malloc(sizeof(uint32_t) * OUTPUTSIZE); 48 | assert(outputtag != NULL); 49 | 50 | // Populate the block 51 | for (size_t i = 0; i < BLOCKLEN; i++) data[i] = randomu32(&rstate); 52 | 53 | fp = fopen("tagout-u32.bin", "w"); 54 | assert(fp != NULL); 55 | 56 | while (1) { 57 | uint32_t rawloc; 58 | size_t res; 59 | 60 | for (uint32_t j = 0; j < OUTPUTSIZE; j++) { 61 | unsigned int dummy; 62 | uint32_t newData = randomu32(&rstate); 63 | uint64_t start; 64 | uint64_t end; 65 | 66 | start = __builtin_ia32_rdtscp(&dummy); 67 | rawloc = data[curoffset]; 68 | data[curoffset] = newData; 69 | end = __builtin_ia32_rdtscp(&dummy); 70 | 71 | outputtag[j] = (rawloc & (~BLOCKMASK)) >> BLOCKMASKLEN; 72 | outputtsc[j] = end - start; 73 | curoffset = rawloc & BLOCKMASK; 74 | } 75 | 76 | fwrite(outputtag, sizeof(uint32_t), OUTPUTSIZE, fp); 77 | res = fwrite(outputtsc, sizeof(uint64_t), OUTPUTSIZE, stdout); 78 | assert(res == OUTPUTSIZE); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/u16-to-u32.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "precision.h" 22 | 23 | noreturn static void useageExit(void) { 24 | fprintf(stderr, "Usage:\n"); 25 | fprintf(stderr, "u16-to-u32 [-d]\n"); 26 | fprintf(stderr, "-d output differences between adjacent values.\n"); 27 | fprintf(stderr, "The values are expected to be provided via stdin.\n"); 28 | exit(EX_USAGE); 29 | } 30 | 31 | int main(int argc, char *argv[]) { 32 | size_t res; 33 | bool configDiffMode; 34 | uint16_t lastSymbol; 35 | int opt; 36 | 37 | assert(PRECISION(UINT_MAX) == 32); 38 | 39 | configDiffMode = false; 40 | 41 | while ((opt = getopt(argc, argv, "d")) != -1) { 42 | switch (opt) { 43 | case 'd': 44 | configDiffMode = true; 45 | break; 46 | default: /* ? */ 47 | useageExit(); 48 | } 49 | } 50 | 51 | if (argc != optind) { 52 | useageExit(); 53 | } 54 | 55 | if (configDiffMode) { 56 | res = fread(&lastSymbol, sizeof(uint16_t), 1, stdin); 57 | if (res != 1) { 58 | perror("Can't read initial symbol"); 59 | exit(EX_OSERR); 60 | } 61 | } else { 62 | lastSymbol = 0; 63 | } 64 | 65 | while (feof(stdin) == 0) { 66 | uint32_t outdata; 67 | uint16_t indata; 68 | 69 | res = fread(&indata, sizeof(uint16_t), 1, stdin); 70 | if (res == 1) { 71 | if (configDiffMode) { 72 | uint16_t curdelta; 73 | curdelta = (uint16_t)(indata - lastSymbol); 74 | outdata = (uint32_t)curdelta; 75 | lastSymbol = indata; 76 | } else { 77 | outdata = (uint32_t)indata; 78 | } 79 | 80 | if (fwrite(&outdata, sizeof(uint32_t), 1, stdout) != 1) { 81 | perror("Can't write to stdout"); 82 | exit(EX_OSERR); 83 | } 84 | } 85 | } 86 | 87 | return (0); 88 | } 89 | -------------------------------------------------------------------------------- /src/health-tests.h: -------------------------------------------------------------------------------- 1 | #ifndef HEALTH_TESTS_H 2 | #define HEALTH_TESTS_H 3 | 4 | #define CROSS_RCT_SHIFT_BYTES 5U 5 | #define CROSS_RCT_TESTS 36 6 | 7 | // CrossRCT state. 8 | // The meaning of A, B, and C are as in SP 800-90B, Section 4.4.1. 9 | // A is the reference symbol being compared against (in this case, a vector of 40 single-bit symbols, comprised of 8 literal values and 32 crosswise values) 10 | // B is the number of times that the current symbol has been encountered consecutively (i.e., we have encountered a length-B run of the relevant symbol) 11 | // C is the cutoff (the test is judged a failure if a run of C or more symbols is found) 12 | struct crossRCTstate { 13 | size_t RCT_Input; 14 | size_t RCT_C[CROSS_RCT_TESTS]; 15 | uint64_t RCT_A; 16 | size_t RCT_B[CROSS_RCT_TESTS]; 17 | size_t RCT_Failures[CROSS_RCT_TESTS]; 18 | size_t *runCounts[CROSS_RCT_TESTS]; 19 | size_t runCountLengths[CROSS_RCT_TESTS]; 20 | uint32_t maxLiteralFailures; 21 | uint32_t maxCrossFailures; 22 | }; 23 | 24 | // Set the initial cross RCT state 25 | void initCrossRCT(size_t baseRCTcutoff, size_t crossRCTcutoff, struct crossRCTstate *state); 26 | bool crossRCT(uint8_t in, struct crossRCTstate *state); 27 | 28 | // RCT state. 29 | // The meaning of A, B, and C are as in SP 800-90B, Section 4.4.1. 30 | // A is the reference symbol being compared against 31 | // B is the number of times that the current symbol has been encountered consecutively (i.e., we have encountered a length-B run of the relevant symbol) 32 | // C is the cutoff (the test is judged a failure if a run of C or more symbols is found) 33 | struct RCTstate { 34 | size_t RCT_Input; 35 | size_t RCT_C; 36 | uint64_t RCT_A; 37 | size_t RCT_B; 38 | size_t RCT_Failures; 39 | size_t *runCounts; 40 | size_t runCountsLength; 41 | }; 42 | 43 | void initRCT(size_t RCTcutoff, struct RCTstate *state); 44 | //X is the symbol being input to the health test. 45 | bool RCT(uint64_t X, struct RCTstate *state); 46 | 47 | //APT state 48 | struct APTstate { 49 | size_t APT_Input; 50 | size_t APT_Window_Count; 51 | size_t APT_W; 52 | size_t APT_C; 53 | size_t APT_i; 54 | uint64_t APT_A; 55 | size_t APT_B; 56 | size_t APT_Failures; 57 | size_t *APTcounts; 58 | }; 59 | 60 | void initAPT(size_t APT_cutoff, size_t APT_window, struct APTstate *state); 61 | bool APT(uint64_t in, struct APTstate *state); 62 | #endif 63 | -------------------------------------------------------------------------------- /src/u64-to-u32.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2024 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, KeyPair Consulting, Inc. 8 | */ 9 | #include 10 | #include // for getopt, optind 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include "binutil.h" 20 | #include "globals-inst.h" 21 | #include "precision.h" 22 | 23 | noreturn static void useageExit(void) { 24 | fprintf(stderr, "Usage:\n"); 25 | fprintf(stderr, "u64-to-u32 [-r] [-t]\n"); 26 | fprintf(stderr, "-r\tSwitch endianness of input values.\n"); 27 | fprintf(stderr, "-t\tTruncate input values.\n"); 28 | fprintf(stderr, "The values are expected to be provided via stdin and the output via stdout.\n"); 29 | exit(EX_USAGE); 30 | } 31 | 32 | int main(int argc, char *argv[]) { 33 | size_t res; 34 | uint64_t inData; 35 | uint32_t out; 36 | bool configReverse; 37 | bool configTruncate; 38 | int opt; 39 | 40 | assert(PRECISION(UINT_MAX) == 32); 41 | 42 | configReverse = false; 43 | configTruncate = false; 44 | 45 | while ((opt = getopt(argc, argv, "rt")) != -1) { 46 | switch (opt) { 47 | case 'r': 48 | configReverse = true; 49 | break; 50 | case 't': 51 | configTruncate = true; 52 | break; 53 | default: /* ? */ 54 | useageExit(); 55 | } 56 | } 57 | 58 | if (argc != optind) { 59 | useageExit(); 60 | } 61 | 62 | while (feof(stdin) == 0) { 63 | res = fread(&inData, sizeof(uint64_t), 1, stdin); 64 | if (res == 1) { 65 | if (configReverse) { 66 | inData = reverse64(inData); 67 | } 68 | 69 | if (configTruncate) inData = inData & 0xFFFFFFFF; 70 | 71 | if (inData > UINT32_MAX) { 72 | fprintf(stderr, "raw input out of range: %016" PRIx64 "\n", inData); 73 | exit(EX_DATAERR); 74 | } else { 75 | out = (uint32_t)inData; 76 | } 77 | 78 | if (fwrite(&out, sizeof(uint32_t), 1, stdout) != 1) { 79 | perror("Can't write to stdout"); 80 | exit(EX_OSERR); 81 | } 82 | } 83 | } 84 | 85 | return (0); 86 | } 87 | -------------------------------------------------------------------------------- /src/u32-bit-select.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include "binutil.h" 20 | #include "entlib.h" 21 | #include "globals-inst.h" 22 | #include "precision.h" 23 | 24 | noreturn static void useageExit(void) { 25 | fprintf(stderr, "Usage:\n"); 26 | fprintf(stderr, "u32-bit-select [-r] (bit)\n"); 27 | fprintf(stderr, "-r\tReverse the endianness of the u32 inputs before selecting the bit.\n"); 28 | fprintf(stderr, "Outputs only the selected bit (0 is the lsb, 31 is msb).\n"); 29 | fprintf(stderr, "The 32-bit values are expected to be provided via stdin.\n"); 30 | fprintf(stderr, "output is " STATDATA_STRING " via stdout.\n"); 31 | exit(EX_USAGE); 32 | } 33 | 34 | int main(int argc, char *argv[]) { 35 | uint32_t data; 36 | uint32_t curbitmask; 37 | int bitpos; 38 | int opt; 39 | bool configReverse; 40 | 41 | configReverse = false; 42 | 43 | while ((opt = getopt(argc, argv, "r")) != -1) { 44 | switch (opt) { 45 | case 'r': 46 | configReverse = true; 47 | break; 48 | default: /* ? */ 49 | useageExit(); 50 | } 51 | } 52 | 53 | argc -= optind; 54 | argv += optind; 55 | 56 | if (argc != 1) { 57 | useageExit(); 58 | } else { 59 | bitpos = atoi(argv[0]); 60 | } 61 | 62 | if ((bitpos < 0) || (bitpos > 31)) { 63 | useageExit(); 64 | } 65 | 66 | curbitmask = 1U << bitpos; 67 | 68 | if (configReverse) { 69 | curbitmask = reverse32(curbitmask); 70 | } 71 | 72 | while (feof(stdin) == 0) { 73 | size_t res; 74 | statData_t outdata; 75 | 76 | res = fread(&data, sizeof(uint32_t), 1, stdin); 77 | if (res == 1) { 78 | outdata = (statData_t)(((curbitmask & data) == 0) ? 0U : 1U); 79 | if (fwrite(&outdata, sizeof(statData_t), 1, stdout) != 1) { 80 | perror("Can't write to stdout"); 81 | exit(EX_OSERR); 82 | } 83 | } 84 | } 85 | 86 | return (0); 87 | } 88 | -------------------------------------------------------------------------------- /src/u32-to-sd.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include "binio.h" 20 | #include "entlib.h" 21 | #include "globals-inst.h" 22 | #include "precision.h" 23 | 24 | noreturn static void useageExit(void) { 25 | fprintf(stderr, "Usage:\n"); 26 | fprintf(stderr, "u32-to-sd \n"); 27 | fprintf(stderr, "inputfile is assumed to be a stream of uint32_ts\n"); 28 | fprintf(stderr, "output sent to stdout is a stream of " STATDATA_STRING " integers\n"); 29 | exit(EX_USAGE); 30 | } 31 | 32 | int main(int argc, char *argv[]) { 33 | FILE *infp; 34 | size_t datalen; 35 | uint32_t *data = NULL; 36 | statData_t *dataout = NULL; 37 | size_t i; 38 | 39 | if (argc != 2) { 40 | useageExit(); 41 | } 42 | 43 | if ((infp = fopen(argv[1], "rb")) == NULL) { 44 | perror("Can't open file"); 45 | exit(EX_NOINPUT); 46 | } 47 | 48 | if ((datalen = readuint32file(infp, &data)) < 1) { 49 | perror("Data file is empty"); 50 | exit(EX_DATAERR); 51 | } 52 | assert(data != NULL); 53 | 54 | if ((dataout = malloc(sizeof(statData_t) * datalen)) == NULL) { 55 | perror("Can't allocate output array"); 56 | exit(EX_OSERR); 57 | } 58 | 59 | fprintf(stderr, "Read in %zu uint32_ts\n", datalen); 60 | if (fclose(infp) != 0) { 61 | perror("Can't close intput file"); 62 | exit(EX_OSERR); 63 | } 64 | 65 | fprintf(stderr, "Outputting data\n"); 66 | for (i = 0; i < datalen; i++) { 67 | if (data[i] > STATDATA_MAX) { 68 | fprintf(stderr, "Value %u at index %zu is out of range\n", data[i], i); 69 | free(data); 70 | free(dataout); 71 | exit(EX_DATAERR); 72 | } else { 73 | dataout[i] = (statData_t)data[i]; 74 | } 75 | } 76 | 77 | if (fwrite(dataout, sizeof(statData_t), datalen, stdout) != datalen) { 78 | perror("Can't write output to stdout"); 79 | exit(EX_OSERR); 80 | } 81 | 82 | free(data); 83 | free(dataout); 84 | 85 | return EX_OK; 86 | } 87 | -------------------------------------------------------------------------------- /src/shannon.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "binio.h" 21 | #include "entlib.h" 22 | #include "globals-inst.h" 23 | #include "precision.h" 24 | #include "translate.h" 25 | 26 | noreturn static void useageExit(void) { 27 | fprintf(stderr, "Usage:\n"); 28 | fprintf(stderr, "shannon [-v] \n"); 29 | fprintf(stderr, "Calculate the shannon entropy for a data set.\n"); 30 | fprintf(stderr, "inputfile is presumed to be a stream of " STATDATA_STRING "s\n"); 31 | fprintf(stderr, "-v\tVerbose mode (can be used several times for increased verbosity).\n"); 32 | exit(EX_USAGE); 33 | } 34 | 35 | int main(int argc, char *argv[]) { 36 | FILE *infp; 37 | size_t datalen; 38 | statData_t *data = NULL; 39 | double shannonEnt; 40 | size_t k; 41 | double median; 42 | int opt; 43 | 44 | configVerbose = 0; 45 | 46 | while ((opt = getopt(argc, argv, "v")) != -1) { 47 | switch (opt) { 48 | case 'v': 49 | configVerbose++; 50 | break; 51 | default: /* ? */ 52 | useageExit(); 53 | } 54 | } 55 | 56 | argc -= optind; 57 | argv += optind; 58 | 59 | if (argc != 1) { 60 | useageExit(); 61 | } 62 | 63 | if ((infp = fopen(argv[0], "rb")) == NULL) { 64 | perror("Can't open file"); 65 | exit(EX_NOINPUT); 66 | } 67 | 68 | datalen = readuintfile(infp, &data); 69 | assert(data != NULL); 70 | if (configVerbose > 0) printf("Read in %zu integers\n", datalen); 71 | 72 | if (fclose(infp) != 0) { 73 | perror("Couldn't close input data file"); 74 | exit(EX_OSERR); 75 | } 76 | 77 | assert(datalen > 0); 78 | 79 | if (configVerbose > 0) fprintf(stderr, "Estimating Shannon entropy.\n"); 80 | 81 | translate(data, datalen, &k, &median); 82 | shannonEnt = shannonEntropyEstimate(data, datalen, k); 83 | 84 | printf("Assessed Shannon entropy = %.17g\n", shannonEnt); 85 | 86 | free(data); 87 | 88 | return EX_OK; 89 | } 90 | -------------------------------------------------------------------------------- /src/u64-counter-raw.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2024 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "binio.h" 21 | #include "globals-inst.h" 22 | #include "precision.h" 23 | 24 | noreturn static void useageExit(void) { 25 | fprintf(stderr, "Usage:\n"); 26 | fprintf(stderr, "u64-counter-raw \n"); 27 | fprintf(stderr, "Extract deltas treated as 64-bit unsigned counters (that roll may roll over).\n"); 28 | exit(EX_USAGE); 29 | } 30 | 31 | int main(int argc, char *argv[]) { 32 | FILE *infp; 33 | size_t datalen; 34 | uint64_t *data = NULL; 35 | size_t i; 36 | uint64_t delta; 37 | uint64_t mindelta = UINT64_MAX; 38 | uint64_t maxdelta = 0; 39 | 40 | if (argc != 2) { 41 | useageExit(); 42 | } 43 | 44 | if ((infp = fopen(argv[1], "rb")) == NULL) { 45 | perror("Can't open file"); 46 | exit(EX_NOINPUT); 47 | } 48 | 49 | datalen = readuint64file(infp, &data); 50 | assert(data != NULL); 51 | 52 | fprintf(stderr, "Read in %zu uint64_ts\n", datalen); 53 | if (fclose(infp) != 0) { 54 | perror("Can't close intput file"); 55 | exit(EX_OSERR); 56 | } 57 | 58 | if (datalen < 2) { 59 | fprintf(stderr, "Too little data\n"); 60 | exit(EX_DATAERR); 61 | } 62 | 63 | for (i = 1; i < datalen; i++) { 64 | delta = data[i] - data[i - 1]; 65 | if (delta < mindelta) mindelta = delta; 66 | if (delta > maxdelta) maxdelta = delta; 67 | data[i - 1] = delta; 68 | } 69 | if (mindelta != 0) { 70 | fprintf(stderr, "Shifting data down by %" PRIu64 ". Maximum value now %" PRIu64 "\n", mindelta, maxdelta - mindelta); 71 | 72 | for (i = 0; i < datalen - 1; i++) { 73 | data[i] -= mindelta; 74 | } 75 | } 76 | 77 | if (fwrite(data, sizeof(uint64_t), datalen - 1, stdout) != datalen - 1) { 78 | if (data != NULL) free(data); 79 | data = NULL; 80 | perror("Can't write to stdout"); 81 | exit(EX_OSERR); 82 | } 83 | 84 | if (data != NULL) free(data); 85 | data = NULL; 86 | return (0); 87 | } 88 | -------------------------------------------------------------------------------- /src/u32-counter-raw.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020-2024 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "binio.h" 22 | #include "globals-inst.h" 23 | #include "precision.h" 24 | 25 | noreturn static void useageExit(void) { 26 | fprintf(stderr, "Usage:\n"); 27 | fprintf(stderr, "u32-counter-raw \n"); 28 | fprintf(stderr, "Extract deltas treated as 32-bit unsigned counters (that roll may roll over).\n"); 29 | exit(EX_USAGE); 30 | } 31 | 32 | int main(int argc, char *argv[]) { 33 | FILE *infp; 34 | size_t datalen; 35 | uint32_t *data = NULL; 36 | size_t i; 37 | uint32_t delta; 38 | uint32_t mindelta = UINT32_MAX; 39 | uint32_t maxdelta = 0; 40 | 41 | if (argc != 2) { 42 | useageExit(); 43 | } 44 | 45 | if ((infp = fopen(argv[1], "rb")) == NULL) { 46 | perror("Can't open file"); 47 | exit(EX_NOINPUT); 48 | } 49 | 50 | datalen = readuint32file(infp, &data); 51 | assert(data != NULL); 52 | 53 | fprintf(stderr, "Read in %zu uint32_ts\n", datalen); 54 | if (fclose(infp) != 0) { 55 | perror("Can't close intput file"); 56 | exit(EX_OSERR); 57 | } 58 | 59 | if (datalen < 2) { 60 | fprintf(stderr, "Too little data\n"); 61 | exit(EX_DATAERR); 62 | } 63 | 64 | for (i = 1; i < datalen; i++) { 65 | delta = data[i] - data[i - 1]; 66 | if (delta < mindelta) mindelta = delta; 67 | if (delta > maxdelta) maxdelta = delta; 68 | data[i - 1] = delta; 69 | } 70 | if (mindelta != 0) { 71 | fprintf(stderr, "Shifting data down by %u. Maximum value now %u\n", mindelta, maxdelta - mindelta); 72 | 73 | for (i = 0; i < datalen - 1; i++) { 74 | data[i] -= mindelta; 75 | } 76 | } 77 | 78 | if (fwrite(data, sizeof(uint32_t), datalen - 1, stdout) != datalen - 1) { 79 | if (data != NULL) free(data); 80 | data = NULL; 81 | perror("Can't write to stdout"); 82 | exit(EX_OSERR); 83 | } 84 | 85 | if (data != NULL) free(data); 86 | data = NULL; 87 | return (0); 88 | } 89 | -------------------------------------------------------------------------------- /src/u16-to-sdbin.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "entlib.h" 19 | 20 | noreturn static void useageExit(void) { 21 | fprintf(stderr, "Usage:\n"); 22 | fprintf(stderr, "u16-to-sdbin [-l] [-b]\n"); 23 | fprintf(stderr, "Expand packed bits that are stored in u16 values.\n"); 24 | fprintf(stderr, "-l\t extract bits from low bit to high bit\n"); 25 | fprintf(stderr, "-b\t 16 bit values are in big endian format.\n"); 26 | fprintf(stderr, "The values are expected to be provided via stdin.\n"); 27 | exit(EX_USAGE); 28 | } 29 | 30 | int main(int argc, char *argv[]) { 31 | statData_t outdata; 32 | uint16_t indata; 33 | uint16_t bitmask; 34 | size_t j; 35 | bool configLTH; 36 | bool configBigEndian; 37 | int opt; 38 | 39 | configLTH = false; 40 | configBigEndian = false; 41 | 42 | while ((opt = getopt(argc, argv, "lb")) != -1) { 43 | switch (opt) { 44 | case 'l': 45 | configLTH = true; 46 | break; 47 | case 'b': 48 | configBigEndian = true; 49 | break; 50 | default: /* ? */ 51 | useageExit(); 52 | } 53 | } 54 | 55 | if (argc != optind) { 56 | useageExit(); 57 | } 58 | 59 | while (feof(stdin) == 0) { 60 | if (fread(&indata, sizeof(uint16_t), 1, stdin) == 1) { 61 | if (configBigEndian) { 62 | indata = (uint16_t)(((indata >> 8) & 0x00FF) | ((indata << 8) & 0xFF00)); 63 | } 64 | if (configLTH) { 65 | bitmask = (uint16_t)0x0001; 66 | } else { 67 | bitmask = (uint16_t)0x8000; 68 | } 69 | 70 | for (j = 0; j < 16; j++) { 71 | outdata = (statData_t)(((indata & bitmask) == 0) ? 0U : 1U); 72 | if (fwrite(&outdata, sizeof(statData_t), 1, stdout) != 1) { 73 | perror("Can't write to output"); 74 | exit(EX_OSERR); 75 | } 76 | 77 | if (configLTH) { 78 | bitmask = (uint16_t)((bitmask << 1) & 0xFFFF); 79 | } else { 80 | bitmask = bitmask >> 1; 81 | } 82 | } 83 | } 84 | } 85 | 86 | return (0); 87 | } 88 | -------------------------------------------------------------------------------- /src/extractbits.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "binio.h" 21 | #include "binutil.h" 22 | #include "entlib.h" 23 | #include "globals-inst.h" 24 | #include "precision.h" 25 | 26 | noreturn static void useageExit(void) { 27 | fprintf(stderr, "Usage:\n"); 28 | fprintf(stderr, "extractbits \n"); 29 | fprintf(stderr, "inputfile is assumed to be a stream of uint32_ts\n"); 30 | fprintf(stderr, "output is sent to stdout is in " STATDATA_STRING " format\n"); 31 | exit(EX_USAGE); 32 | } 33 | 34 | int main(int argc, char *argv[]) { 35 | FILE *infp; 36 | size_t datalen; 37 | statData_t outdata; 38 | uint32_t *data = NULL; 39 | size_t i; 40 | uint32_t outputBits; 41 | uint32_t bitmask; 42 | 43 | if (argc != 3) { 44 | fprintf(stderr, "Wrong number of arguments: argc=%d\n", argc); 45 | useageExit(); 46 | } 47 | 48 | if ((infp = fopen(argv[1], "rb")) == NULL) { 49 | perror("Can't open file"); 50 | exit(EX_NOINPUT); 51 | } 52 | 53 | bitmask = (uint32_t)strtoll(argv[2], NULL, 0); 54 | outputBits = (uint32_t)__builtin_popcount(bitmask); 55 | 56 | if ((outputBits == 0) || (outputBits > STATDATA_BITS)) { 57 | fprintf(stderr, "The number of outputBits is out of range: 0 > %u > %u\n", outputBits, STATDATA_BITS); 58 | useageExit(); 59 | } 60 | 61 | fprintf(stderr, "Input bitmask 0x%X (Hamming weight: %u)\n", bitmask, outputBits); 62 | 63 | datalen = readuint32file(infp, &data); 64 | assert(data != NULL); 65 | assert(datalen > 0); 66 | fprintf(stderr, "Read in %zu uint32_ts\n", datalen); 67 | if (fclose(infp) != 0) { 68 | perror("Can't close input file"); 69 | exit(EX_OSERR); 70 | } 71 | 72 | fprintf(stderr, "Outputting data\n"); 73 | for (i = 0; i < datalen; i++) { 74 | outdata = (statData_t)extractbits(data[i], bitmask); 75 | if (fwrite(&outdata, sizeof(statData_t), 1, stdout) != 1) { 76 | perror("Can't write output to stdout"); 77 | exit(EX_OSERR); 78 | } 79 | } 80 | 81 | free(data); 82 | 83 | return EX_OK; 84 | } 85 | -------------------------------------------------------------------------------- /src/u32-selectrange.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include "binio.h" 20 | #include "globals-inst.h" 21 | #include "precision.h" 22 | 23 | noreturn static void useageExit(void) { 24 | fprintf(stderr, "Usage:\n"); 25 | fprintf(stderr, "u32-selectrange inputfile low high\n"); 26 | fprintf(stderr, "Extracts all values between low and high (inclusive).\n"); 27 | fprintf(stderr, "inputfile is assumed to be a stream of uint32_ts\n"); 28 | fprintf(stderr, "output is to stdout, and is u32 ints\n"); 29 | exit(EX_USAGE); 30 | } 31 | 32 | int main(int argc, char *argv[]) { 33 | FILE *infp; 34 | uint32_t *data = NULL; 35 | size_t i; 36 | size_t datalen; 37 | uint32_t lowValue, highValue; 38 | long long int inll; 39 | 40 | if (argc != 4) { 41 | useageExit(); 42 | } 43 | 44 | if ((infp = fopen(argv[1], "rb")) == NULL) { 45 | perror("Can't open file"); 46 | exit(EX_NOINPUT); 47 | } 48 | 49 | inll = strtoll(argv[2], NULL, 0); 50 | if ((inll < 0) || (inll > UINT_MAX)) { 51 | useageExit(); 52 | } else { 53 | lowValue = (uint32_t)inll; 54 | } 55 | 56 | inll = strtoll(argv[3], NULL, 0); 57 | if ((inll < 0) || (inll > UINT_MAX) || (inll < lowValue)) { 58 | useageExit(); 59 | } else { 60 | highValue = (uint32_t)inll; 61 | } 62 | 63 | datalen = readuint32file(infp, &data); 64 | if (datalen < 1) { 65 | useageExit(); 66 | } 67 | assert(data != NULL); 68 | 69 | fprintf(stderr, "Read in %zu samples\n", datalen); 70 | if (fclose(infp) != 0) { 71 | perror("Can't close input file"); 72 | exit(EX_OSERR); 73 | } 74 | 75 | fprintf(stderr, "Outputting data in the interval [%u, %u]\n", lowValue, highValue); 76 | 77 | fprintf(stderr, "Outputting the data...\n"); 78 | for (i = 0; i < datalen; i++) { 79 | if ((data[i] >= lowValue) && (data[i] <= highValue)) { 80 | if (fwrite(&(data[i]), sizeof(uint32_t), 1, stdout) != 1) { 81 | perror("Can't write output to stdout"); 82 | exit(EX_OSERR); 83 | } 84 | } 85 | } 86 | 87 | free(data); 88 | return EX_OK; 89 | } 90 | -------------------------------------------------------------------------------- /src/u16-mcv.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "precision.h" 23 | #include "fancymath.h" 24 | 25 | int main(void) { 26 | size_t res; 27 | size_t symbolCount[UINT16_MAX+1] = {0}; 28 | size_t maxSymbolCount = 0; 29 | size_t symbols = 0; 30 | uint16_t maxSymbol = 0; 31 | double pu; 32 | double phat; 33 | 34 | 35 | while (feof(stdin) == 0) { 36 | uint16_t indata; 37 | 38 | res = fread(&indata, sizeof(uint16_t), 1, stdin); 39 | if (res == 1) { 40 | symbolCount[indata]++; 41 | symbols++; 42 | } 43 | } 44 | 45 | for(uint32_t j=0; j<=UINT16_MAX; j++) { 46 | if(symbolCount[j] > maxSymbolCount) { 47 | maxSymbolCount = symbolCount[j]; 48 | maxSymbol = (uint16_t)j; 49 | } 50 | } 51 | fprintf(stderr, "Most common symbol is 0x%4X (count %zu)\n", maxSymbol, maxSymbolCount); 52 | 53 | phat=((double)maxSymbolCount) / ((double)symbols); 54 | // Note, this is the raw value. A higher value maps to a more conservative estimate, so we then look at upper confidence interval bound. 55 | 56 | // Note, this is a local guess at a confidence interval, under the assumption that this most probable symbol proportion is distributed 57 | // as per the binomial distribution (this CI estimate is made under a normal approximation of the binomial distribution); 58 | // this assumption isn't reasonable, as the most probable symbol can never have less than ceil(L/k) symbols, 59 | // so the entire low end of the binomial distribution isn't in the support for the actual distribution. 60 | // If we (a priori) knew what the MLS was, we could make this estimate, but we don't here. 61 | // This actual distribution here is that of the maximum bin of a multinomial distribution, but this is complicated. 62 | pu = phat + ZALPHA * sqrt(phat * (1.0 - phat) / ((double)symbols - 1.0)); 63 | if (pu > 1.0) { 64 | pu = 1.0; 65 | } 66 | fprintf(stderr, "p_hat = %.17g\n", phat); 67 | fprintf(stderr, "p_u = %.17g\n", pu); 68 | fprintf(stderr, "minentropy = %.17g\n", -log2(pu)); 69 | 70 | return 0; 71 | } 72 | -------------------------------------------------------------------------------- /src/u64-scale-break.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include // for getopt, optind 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "binutil.h" 22 | #include "globals-inst.h" 23 | #include "precision.h" 24 | 25 | noreturn static void useageExit(void) { 26 | fprintf(stderr, "Usage:\n"); 27 | fprintf(stderr, "u64-scale-break \n"); 28 | fprintf(stderr, "\tNumber of bits in low order.\n"); 29 | fprintf(stderr, "\tFactor to multiply high value.\n"); 30 | fprintf(stderr, "\tFactor to multiply low value.\n"); 31 | fprintf(stderr, "The values are expected to be provided via stdin and the output via stdout.\n"); 32 | exit(EX_USAGE); 33 | } 34 | 35 | int main(int argc, char *argv[]) { 36 | uint16_t configLowBits=10; 37 | uint64_t configScaleHigh; 38 | uint64_t configScaleLow; 39 | char *endptr; 40 | unsigned long int intval; 41 | uint64_t bitmask; 42 | 43 | if (argc != 4) { 44 | useageExit(); 45 | } 46 | 47 | intval = strtoul(argv[1], &endptr, 0); 48 | if((*endptr != '\0') || (intval > 63) || (intval == 0)) 49 | useageExit(); 50 | configLowBits = (uint16_t)intval; 51 | 52 | intval = strtoul(argv[2], &endptr, 0); 53 | if((*endptr != '\0') || (intval == ULONG_MAX)) 54 | useageExit(); 55 | configScaleHigh = intval; 56 | 57 | intval = strtoul(argv[3], &endptr, 0); 58 | if((*endptr != '\0') || (intval == ULONG_MAX)) 59 | useageExit(); 60 | configScaleLow = intval; 61 | 62 | bitmask = (1ULL << configLowBits) - 1ULL; 63 | fprintf(stderr, "width: %u, scaleHigh: %zu, scaleLow: %zu, bitmask: 0x%016lX\n", configLowBits, configScaleHigh, configScaleLow, bitmask); 64 | 65 | while (feof(stdin) == 0) { 66 | uint64_t inData; 67 | size_t res; 68 | uint64_t curVal; 69 | res = fread(&inData, sizeof(uint64_t), 1, stdin); 70 | if (res == 1) { 71 | curVal = ((inData & bitmask) * configScaleLow) + ((inData >> configLowBits) * configScaleHigh); 72 | 73 | if (fwrite(&curVal, sizeof(uint64_t), 1, stdout) != 1) { 74 | perror("Can't write to stdout"); 75 | exit(EX_OSERR); 76 | } 77 | } 78 | } 79 | return (0); 80 | } 81 | -------------------------------------------------------------------------------- /src/u32-discard-fixed-bits.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "binio.h" 21 | #include "binutil.h" 22 | #include "globals-inst.h" 23 | #include "precision.h" 24 | 25 | noreturn static void useageExit(void) { 26 | fprintf(stderr, "Usage:\n"); 27 | fprintf(stderr, "u32-discard-fixed-bits \n"); 28 | fprintf(stderr, "inputfile is assumed to be a stream of uint32_ts\n"); 29 | fprintf(stderr, "output sent to stdout are uint32_tegers, with all fixed bits removed and the non-fixed bits moved to the LSB of the output.\n"); 30 | exit(EX_USAGE); 31 | } 32 | 33 | int main(int argc, char *argv[]) { 34 | FILE *infp; 35 | size_t datalen; 36 | uint32_t *data = NULL; 37 | size_t i; 38 | uint32_t bitmask; 39 | uint32_t maxval, minval; 40 | uint32_t bits; 41 | double doubleBits; 42 | 43 | if (argc != 2) { 44 | useageExit(); 45 | } 46 | 47 | if ((infp = fopen(argv[1], "rb")) == NULL) { 48 | perror("Can't open file"); 49 | exit(EX_NOINPUT); 50 | } 51 | 52 | datalen = readuint32file(infp, &data); 53 | assert(data != NULL); 54 | assert(datalen > 0); 55 | 56 | fprintf(stderr, "Read in %zu uint32_ts\n", datalen); 57 | if (fclose(infp) != 0) { 58 | perror("Can't close intput file"); 59 | exit(EX_OSERR); 60 | } 61 | 62 | maxval = 0; 63 | minval = UINT_MAX; 64 | 65 | for (i = 0; i < datalen; i++) { 66 | if (data[i] > maxval) { 67 | maxval = data[i]; 68 | } 69 | if (data[i] < minval) { 70 | minval = data[i]; 71 | } 72 | } 73 | 74 | doubleBits = ceil(log2((double)maxval + 1.0)); 75 | assert(doubleBits >= 0.0); 76 | bits = (uint32_t)doubleBits; 77 | 78 | bitmask = getActiveBits(data, datalen); 79 | 80 | fprintf(stderr, "Symbols in the range [%u, %u], %u bit, bitmask: 0x%08X\n", minval, maxval, bits, bitmask); 81 | 82 | fprintf(stderr, "Outputting data\n"); 83 | for (i = 0; i < datalen; i++) { 84 | data[i] = extractbits(data[i], bitmask); 85 | } 86 | 87 | if (fwrite(data, sizeof(uint32_t), datalen, stdout) != datalen) { 88 | perror("Can't write output to stdout"); 89 | exit(EX_OSERR); 90 | } 91 | 92 | free(data); 93 | 94 | return EX_OK; 95 | } 96 | -------------------------------------------------------------------------------- /src/double-sort.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "binio.h" 22 | #include "globals-inst.h" 23 | #include "precision.h" 24 | 25 | /*Takes doubles and sorts them, outputting them to stdout.*/ 26 | noreturn static void useageExit(void) { 27 | fprintf(stderr, "Usage:\n"); 28 | fprintf(stderr, "double-sort \n"); 29 | fprintf(stderr, "Takes doubles from the file and sorts them.\n"); 30 | exit(EX_USAGE); 31 | } 32 | 33 | // No fancyness is needed here. 34 | static int doublecompare(const void *in1, const void *in2) { 35 | double left, right; 36 | 37 | assert(in1 != NULL); 38 | assert(in2 != NULL); 39 | 40 | left = *((const double *)in1); 41 | right = *((const double *)in2); 42 | 43 | assert(!isnan(left) && !isnan(right)); 44 | 45 | if (left < right) { 46 | return (-1); 47 | } else if (left > right) { 48 | return (1); 49 | } else { 50 | return (0); 51 | } 52 | } 53 | 54 | int main(int argc, char *argv[]) { 55 | size_t datalen; 56 | double *data; 57 | FILE *fp; 58 | 59 | configVerbose = 0; 60 | data = NULL; 61 | 62 | if (argc != 2) { 63 | useageExit(); 64 | } 65 | 66 | if ((fp = fopen(argv[1], "rb")) == NULL) { 67 | perror("Can't open file for reading"); 68 | exit(EX_OSERR); 69 | } 70 | 71 | fprintf(stderr, "Reading the data.\n"); 72 | if ((datalen = readdoublefile(fp, &data)) < 1) { 73 | perror("File is empty"); 74 | exit(EX_DATAERR); 75 | } 76 | 77 | fclose(fp); 78 | 79 | if (datalen == 0) { 80 | useageExit(); 81 | } 82 | 83 | assert(data != NULL); 84 | 85 | // Sort the data 86 | fprintf(stderr, "Sorting the data.\n"); 87 | qsort(data, datalen, sizeof(double), doublecompare); 88 | 89 | fprintf(stderr, "Writing the data.\n"); 90 | while ((datalen > 0) && (ferror(stdout) != 0)) { 91 | size_t written; 92 | 93 | written = fwrite(data, sizeof(double), datalen, stdout); 94 | data += written; 95 | datalen -= written; 96 | } 97 | 98 | if (ferror(stdout) != 0) { 99 | perror("Can't write sorted data"); 100 | exit(EX_DATAERR); 101 | } 102 | 103 | free(data); 104 | return (EX_OK); 105 | } 106 | -------------------------------------------------------------------------------- /src/u64-randomsample.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2024 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, KeyPair Consulting, Inc. 8 | */ 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "binio.h" 22 | #include "randlib.h" 23 | #include "globals-inst.h" 24 | 25 | noreturn static void useageExit(void) { 26 | fprintf(stderr, "Usage:\n"); 27 | fprintf(stderr, "u64-randomsample inputfile outputSamples\n"); 28 | fprintf(stderr, "Produce a random sample of the input file of size outputSamples.\n"); 29 | fprintf(stderr, "inputfile is assumed to be a stream of uint64_ts\n"); 30 | fprintf(stderr, "output is to stdout, and is u64 ints\n"); 31 | exit(EX_USAGE); 32 | } 33 | 34 | int main(int argc, char *argv[]) { 35 | FILE *infp; 36 | uint64_t *data = NULL; 37 | uint64_t *outputData = NULL; 38 | size_t datalen; 39 | size_t outputDataLen; 40 | struct randstate rstate; 41 | unsigned long long inint; 42 | 43 | initGenerator(&rstate); 44 | seedGenerator(&rstate); 45 | 46 | if (argc != 3) { 47 | useageExit(); 48 | } 49 | 50 | if ((infp = fopen(argv[1], "rb")) == NULL) { 51 | perror("Can't open file"); 52 | exit(EX_NOINPUT); 53 | } 54 | 55 | inint = strtoull(argv[2], NULL, 0); 56 | if (((inint == ULLONG_MAX) && errno == ERANGE) || (errno == EINVAL) || (inint > SIZE_MAX) || (inint == 0)) { 57 | useageExit(); 58 | } 59 | outputDataLen = (size_t)inint; 60 | 61 | datalen = readuint64file(infp, &data); 62 | if (datalen < 1) { 63 | useageExit(); 64 | } 65 | 66 | assert(data != NULL); 67 | 68 | fprintf(stderr, "Read in %zu samples\n", datalen); 69 | if (fclose(infp) != 0) { 70 | perror("Can't close input file"); 71 | exit(EX_OSERR); 72 | } 73 | 74 | if ((outputData = malloc(outputDataLen * sizeof(uint64_t))) == NULL) { 75 | perror("Can't allocate array for output data"); 76 | exit(EX_OSERR); 77 | } 78 | 79 | fprintf(stderr, "Randomly selecting data\n"); 80 | for(size_t i = 0; i < outputDataLen; i++) { 81 | outputData[i] = data[randomRange64(datalen-1, &rstate)]; 82 | } 83 | free(data); 84 | 85 | 86 | fprintf(stderr, "Outputting the data...\n"); 87 | if (fwrite(outputData, sizeof(uint64_t), outputDataLen, stdout) != outputDataLen) { 88 | perror("Can't write output to stdout"); 89 | exit(EX_OSERR); 90 | } 91 | 92 | free(outputData); 93 | return EX_OK; 94 | } 95 | -------------------------------------------------------------------------------- /src/u32-randomsample.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2024 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, KeyPair Consulting, Inc. 8 | */ 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "binio.h" 22 | #include "randlib.h" 23 | #include "globals-inst.h" 24 | 25 | noreturn static void useageExit(void) { 26 | fprintf(stderr, "Usage:\n"); 27 | fprintf(stderr, "u32-randomsample inputfile outputSamples\n"); 28 | fprintf(stderr, "Produce a random sample of the input file of size outputSamples.\n"); 29 | fprintf(stderr, "inputfile is assumed to be a stream of uint32_ts\n"); 30 | fprintf(stderr, "output is to stdout, and is u32 ints\n"); 31 | exit(EX_USAGE); 32 | } 33 | 34 | int main(int argc, char *argv[]) { 35 | FILE *infp; 36 | uint32_t *data = NULL; 37 | uint32_t *outputData = NULL; 38 | size_t datalen; 39 | size_t outputDataLen; 40 | struct randstate rstate; 41 | unsigned long long inint; 42 | 43 | initGenerator(&rstate); 44 | seedGenerator(&rstate); 45 | 46 | if (argc != 3) { 47 | useageExit(); 48 | } 49 | 50 | if ((infp = fopen(argv[1], "rb")) == NULL) { 51 | perror("Can't open file"); 52 | exit(EX_NOINPUT); 53 | } 54 | 55 | inint = strtoull(argv[2], NULL, 0); 56 | if (((inint == ULLONG_MAX) && errno == ERANGE) || (errno == EINVAL) || (inint > SIZE_MAX) || (inint == 0)) { 57 | useageExit(); 58 | } 59 | outputDataLen = (size_t)inint; 60 | 61 | datalen = readuint32file(infp, &data); 62 | if (datalen < 1) { 63 | useageExit(); 64 | } 65 | 66 | assert(data != NULL); 67 | 68 | fprintf(stderr, "Read in %zu samples\n", datalen); 69 | if (fclose(infp) != 0) { 70 | perror("Can't close input file"); 71 | exit(EX_OSERR); 72 | } 73 | 74 | if ((outputData = malloc(outputDataLen * sizeof(uint32_t))) == NULL) { 75 | perror("Can't allocate array for output data"); 76 | exit(EX_OSERR); 77 | } 78 | 79 | fprintf(stderr, "Randomly selecting data\n"); 80 | for(size_t i = 0; i < outputDataLen; i++) { 81 | outputData[i] = data[randomRange64(datalen-1, &rstate)]; 82 | } 83 | free(data); 84 | 85 | 86 | fprintf(stderr, "Outputting the data...\n"); 87 | if (fwrite(outputData, sizeof(uint32_t), outputDataLen, stdout) != outputDataLen) { 88 | perror("Can't write output to stdout"); 89 | exit(EX_OSERR); 90 | } 91 | 92 | free(outputData); 93 | 94 | return EX_OK; 95 | } 96 | -------------------------------------------------------------------------------- /src/randomsample.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2024 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, KeyPair Consulting, Inc. 8 | */ 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "binio.h" 22 | #include "randlib.h" 23 | #include "globals-inst.h" 24 | 25 | noreturn static void useageExit(void) { 26 | fprintf(stderr, "Usage:\n"); 27 | fprintf(stderr, "randomsample inputfile outputSamples\n"); 28 | fprintf(stderr, "Produce a random sample of the input file of size outputSamples.\n"); 29 | fprintf(stderr, "inputfile is assumed to be a stream of uint64_ts in decimal format, one per line.\n"); 30 | fprintf(stderr, "output is to stdout, and is u64 ints\n"); 31 | exit(EX_USAGE); 32 | } 33 | 34 | int main(int argc, char *argv[]) { 35 | FILE *infp; 36 | uint64_t *data = NULL; 37 | uint64_t *outputData = NULL; 38 | size_t datalen; 39 | size_t outputDataLen; 40 | struct randstate rstate; 41 | unsigned long long inint; 42 | 43 | initGenerator(&rstate); 44 | seedGenerator(&rstate); 45 | 46 | if (argc != 3) { 47 | useageExit(); 48 | } 49 | 50 | if ((infp = fopen(argv[1], "rb")) == NULL) { 51 | perror("Can't open file"); 52 | exit(EX_NOINPUT); 53 | } 54 | 55 | inint = strtoull(argv[2], NULL, 0); 56 | if (((inint == ULLONG_MAX) && errno == ERANGE) || (errno == EINVAL) || (inint > SIZE_MAX) || (inint == 0)) { 57 | useageExit(); 58 | } 59 | outputDataLen = (size_t)inint; 60 | 61 | datalen = readasciiuint64s(infp, &data); 62 | if (datalen < 1) { 63 | useageExit(); 64 | } 65 | 66 | assert(data != NULL); 67 | 68 | fprintf(stderr, "Read in %zu samples\n", datalen); 69 | if (fclose(infp) != 0) { 70 | perror("Can't close input file"); 71 | exit(EX_OSERR); 72 | } 73 | 74 | if ((outputData = malloc(outputDataLen * sizeof(uint64_t))) == NULL) { 75 | perror("Can't allocate array for output data"); 76 | exit(EX_OSERR); 77 | } 78 | 79 | fprintf(stderr, "Randomly selecting data\n"); 80 | for(size_t i = 0; i < outputDataLen; i++) { 81 | outputData[i] = data[randomRange64(datalen-1, &rstate)]; 82 | } 83 | 84 | free(data); 85 | 86 | fprintf(stderr, "Outputting the data...\n"); 87 | if (fwrite(outputData, sizeof(uint64_t), outputDataLen, stdout) != outputDataLen) { 88 | perror("Can't write output to stdout"); 89 | exit(EX_OSERR); 90 | } 91 | 92 | free(outputData); 93 | 94 | return EX_OK; 95 | } 96 | -------------------------------------------------------------------------------- /src/u128-bit-select.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include "binutil.h" 20 | #include "entlib.h" 21 | #include "globals-inst.h" 22 | #include "precision.h" 23 | 24 | noreturn static void useageExit(void) { 25 | fprintf(stderr, "Usage:\n"); 26 | fprintf(stderr, "bit-select-u128 (bit)\n"); 27 | fprintf(stderr, "The 128-bit values are expected to be provided via stdin.\n"); 28 | fprintf(stderr, "output is " STATDATA_STRING " via stdout.\n"); 29 | exit(EX_USAGE); 30 | } 31 | 32 | /*0 - 63 in the first uint64 33 | 64 - 127 in the second uint64 34 | (assume little endian)*/ 35 | static void bitmask(uint64_t *input, int bitpos) { 36 | assert(input != NULL); 37 | assert((bitpos >= 0) && (bitpos <= 127)); 38 | 39 | if (bitpos >= 64) { 40 | input[0] = 0; 41 | input[1] = 0x01ULL << (bitpos - 64); 42 | } else { 43 | input[0] = 0x01ULL << (bitpos); 44 | input[1] = 0; 45 | } 46 | 47 | assert((__builtin_popcountll(input[0]) + __builtin_popcountll(input[1])) == 1); 48 | } 49 | 50 | int main(int argc, char *argv[]) { 51 | uint64_t data[2]; 52 | statData_t outdata; 53 | uint64_t curbitmask[2]; 54 | size_t res; 55 | int bitpos; 56 | int opt; 57 | bool configReverse; 58 | 59 | configReverse = false; 60 | 61 | while ((opt = getopt(argc, argv, "r")) != -1) { 62 | switch (opt) { 63 | case 'r': 64 | configReverse = true; 65 | break; 66 | default: /* ? */ 67 | useageExit(); 68 | } 69 | } 70 | 71 | argc -= optind; 72 | argv += optind; 73 | 74 | if (argc != 1) { 75 | useageExit(); 76 | } else { 77 | bitpos = atoi(argv[0]); 78 | } 79 | 80 | if ((bitpos < 0) || (bitpos > 127)) { 81 | useageExit(); 82 | } 83 | 84 | bitmask(curbitmask, bitpos); 85 | if (configReverse) { 86 | reverse128(curbitmask); 87 | } 88 | 89 | while (feof(stdin) == 0) { 90 | res = fread(data, sizeof(uint64_t), 2, stdin); 91 | if (res == 2) { 92 | outdata = (((curbitmask[0] & data[0]) | (curbitmask[1] & data[1])) == 0) ? 0 : 1; 93 | if (fwrite(&outdata, sizeof(statData_t), 1, stdout) != 1) { 94 | perror("Can't write to stdout"); 95 | exit(EX_OSERR); 96 | } 97 | } 98 | } 99 | 100 | return (0); 101 | } 102 | -------------------------------------------------------------------------------- /src/double-merge.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "binio.h" 21 | #include "globals-inst.h" 22 | #include "precision.h" 23 | 24 | /*Takes doubles from stdin (1 per line) and gives the pth percentile*/ 25 | noreturn static void useageExit(void) { 26 | fprintf(stderr, "Usage:\n"); 27 | fprintf(stderr, "double-merge \n"); 28 | fprintf(stderr, "Merges two sorted lists into a single merged list.\n"); 29 | exit(EX_USAGE); 30 | } 31 | 32 | static bool verifySorted(double *data, size_t datalen) { 33 | double last; 34 | bool result = true; 35 | 36 | assert(datalen >= 1); 37 | assert(data != NULL); 38 | 39 | last = *data; 40 | data++; 41 | 42 | for (size_t i = 1; i < datalen; i++) { 43 | if (*data < last) { 44 | result = false; 45 | break; 46 | } 47 | 48 | last = *data; 49 | 50 | data++; 51 | } 52 | 53 | return result; 54 | } 55 | 56 | int main(int argc, char *argv[]) { 57 | size_t datalen1, datalen2; 58 | double *data1, *data2, *outData; 59 | FILE *fp; 60 | 61 | configVerbose = 0; 62 | data1 = NULL; 63 | data2 = NULL; 64 | 65 | if (argc != 4) { 66 | useageExit(); 67 | } 68 | 69 | if ((fp = fopen(argv[1], "rb")) == NULL) { 70 | perror("Can't open file1 for reading"); 71 | exit(EX_OSERR); 72 | } 73 | fprintf(stderr, "Reading data set 1.\n"); 74 | datalen1 = readdoublefile(fp, &data1); 75 | fclose(fp); 76 | 77 | if ((fp = fopen(argv[2], "rb")) == NULL) { 78 | perror("Can't open file2 for reading"); 79 | exit(EX_OSERR); 80 | } 81 | fprintf(stderr, "Reading data set 2.\n"); 82 | datalen2 = readdoublefile(fp, &data2); 83 | fclose(fp); 84 | 85 | outData = malloc(sizeof(double) * (datalen1 + datalen2)); 86 | assert(outData); 87 | 88 | // Merge the data 89 | fprintf(stderr, "Merging the data.\n"); 90 | mergeSortedLists(data1, datalen1, data2, datalen2, outData); 91 | assert(verifySorted(outData, datalen1 + datalen2)); 92 | 93 | if ((fp = fopen(argv[3], "wb")) == NULL) { 94 | perror("Can't open output file for writing"); 95 | exit(EX_OSERR); 96 | } 97 | 98 | fprintf(stderr, "Writing the data.\n"); 99 | fwrite(outData, sizeof(double), datalen1 + datalen2, fp); 100 | 101 | fclose(fp); 102 | 103 | free(outData); 104 | free(data1); 105 | free(data2); 106 | return (EX_OK); 107 | } 108 | -------------------------------------------------------------------------------- /src/generate-precision.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #if defined(__x86_64) || defined(__x86_64__) 16 | #include 17 | #ifndef bit_SSE4_2 18 | #define bit_SSE4_2 (1 << 20) 19 | #endif 20 | 21 | #ifndef bit_BMI2 22 | #define bit_BMI2 (1 << 20) 23 | #endif 24 | #endif 25 | 26 | int main(void) { 27 | #if defined(__x86_64) || defined(__x86_64__) 28 | uint32_t eax = 0; 29 | uint32_t ebx = 0; 30 | uint32_t ecx = 0; 31 | uint32_t edx = 0; 32 | uint32_t ids = 0; 33 | #endif 34 | 35 | printf("#ifndef PRECISION_H\n"); 36 | printf("#define PRECISION_H\n"); 37 | #if __GNUC__ > 4 38 | printf("#define PRECISION(value) _Generic(value, \\\n"); 39 | printf(" unsigned char : %d, \\\n", __builtin_popcountll((uintmax_t)UCHAR_MAX)); 40 | printf(" unsigned short : %d, \\\n", __builtin_popcountll((uintmax_t)USHRT_MAX)); 41 | printf(" unsigned int : %d, \\\n", __builtin_popcountll((uintmax_t)UINT_MAX)); 42 | printf(" unsigned long : %d, \\\n", __builtin_popcountll((uintmax_t)ULONG_MAX)); 43 | printf(" unsigned long long : %d, \\\n", __builtin_popcountll((uintmax_t)ULLONG_MAX)); 44 | printf(" signed char : %d, \\\n", __builtin_popcountll((uintmax_t)SCHAR_MAX)); 45 | printf(" signed short : %d, \\\n", __builtin_popcountll((uintmax_t)SHRT_MAX)); 46 | printf(" signed int : %d, \\\n", __builtin_popcountll((uintmax_t)INT_MAX)); 47 | printf(" signed long : %d, \\\n", __builtin_popcountll((uintmax_t)LONG_MAX)); 48 | printf(" signed long long : %d)\n", __builtin_popcountll((uintmax_t)LLONG_MAX)); 49 | #else 50 | printf("#define PRECISION(umax_value) __builtin_popcountll(umax_value)\n"); 51 | #endif 52 | 53 | #if defined(__x86_64) || defined(__x86_64__) 54 | __get_cpuid(0, &eax, &ebx, &ecx, &edx); 55 | ids = eax; 56 | 57 | if (ids >= 1) { 58 | __get_cpuid(1, &eax, &ebx, &ecx, &edx); 59 | if (ecx & bit_SSE4_2) { 60 | printf("#define SSE42\n"); 61 | } 62 | if (ecx & bit_SSE4_1) { 63 | printf("#define SSE41\n"); 64 | } 65 | if (ecx & bit_SSE3) { 66 | printf("#define SSE3\n"); 67 | } 68 | if (ecx & bit_RDRND) { 69 | printf("#define RDRND\n"); 70 | } 71 | if (ecx & bit_POPCNT) { 72 | printf("#define POPCNT\n"); 73 | } 74 | if (edx & bit_SSE) { 75 | printf("#define SSE\n"); 76 | } 77 | if (edx & bit_SSE2) { 78 | printf("#define SSE2\n"); 79 | } 80 | } 81 | 82 | if (ids >= 7) { 83 | __get_cpuid(7, &eax, &ebx, &ecx, &edx); 84 | if (ebx & bit_BMI2) { 85 | printf("#define BMI2\n"); 86 | } 87 | } 88 | #endif 89 | 90 | printf("\n"); 91 | printf("#endif\n"); 92 | 93 | return 0; 94 | } 95 | -------------------------------------------------------------------------------- /src/double-minmaxdelta.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "binio.h" 22 | #include "bootstrap.h" 23 | #include "fancymath.h" 24 | #include "globals-inst.h" 25 | #include "precision.h" 26 | #include "randlib.h" 27 | 28 | /*Takes doubles from stdin and gives the mean*/ 29 | noreturn static void useageExit(void) { 30 | fprintf(stderr, "Usage:\n"); 31 | fprintf(stderr, "double-minmaxdelta [-v] [-0] [filename]\n"); 32 | fprintf(stderr, "Takes doubles from the file (if provided) or stdin (by default) (1 per line in ascii mode) and gives the mean.\n"); 33 | fprintf(stderr, "-v\tVerbose mode (can be used up to 3 times for increased verbosity).\n"); 34 | fprintf(stderr, "-0\tRead in doubles in machine-specific format.\n"); 35 | exit(EX_USAGE); 36 | } 37 | 38 | int main(int argc, char *argv[]) { 39 | size_t datalen; 40 | double *data; 41 | int opt; 42 | bool configBinary; 43 | FILE *fp; 44 | double minValue = DBL_INFINITY, maxValue = -DBL_INFINITY; 45 | 46 | configVerbose = 0; 47 | data = NULL; 48 | configBinary = false; 49 | 50 | assert(PRECISION(UINT_MAX) >= 32); 51 | 52 | while ((opt = getopt(argc, argv, "0v")) != -1) { 53 | switch (opt) { 54 | case 'v': 55 | configVerbose++; 56 | break; 57 | case '0': 58 | configBinary = true; 59 | break; 60 | default: /* ? */ 61 | useageExit(); 62 | } 63 | } 64 | 65 | argc -= optind; 66 | argv += optind; 67 | 68 | if ((argc != 0) && (argc != 1)) { 69 | useageExit(); 70 | } 71 | 72 | if (argc == 1) { 73 | if ((fp = fopen(argv[0], "rb")) == NULL) { 74 | perror("Can't open file"); 75 | exit(EX_OSERR); 76 | } 77 | } else { 78 | fp = stdin; 79 | } 80 | 81 | if (configBinary) { 82 | datalen = readdoublefile(fp, &data); 83 | } else { 84 | datalen = readasciidoubles(fp, &data); 85 | } 86 | 87 | if (datalen == 0) { 88 | useageExit(); 89 | } 90 | 91 | assert(data != NULL); 92 | 93 | for (size_t j = 0; j < datalen; j++) { 94 | if (data[j] > maxValue) maxValue = data[j]; 95 | if (data[j] < minValue) minValue = data[j]; 96 | } 97 | 98 | if (configVerbose > 0) { 99 | fprintf(stderr, "Max: %.17g\n", maxValue); 100 | fprintf(stderr, "Min: %.17g\n", minValue); 101 | } 102 | 103 | printf("%.17g\n", maxValue - minValue); 104 | 105 | free(data); 106 | return (EX_OK); 107 | } 108 | -------------------------------------------------------------------------------- /src/translate-data.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include // for getopt, optind 12 | #include 13 | #include 14 | #include // for uint32_t 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "binio.h" 22 | #include "entlib.h" 23 | #include "globals-inst.h" 24 | #include "precision.h" 25 | #include "translate.h" 26 | 27 | noreturn static void useageExit(void) { 28 | fprintf(stderr, "Usage:\n"); 29 | fprintf(stderr, "translate-data [-v] \n"); 30 | fprintf(stderr, "Perform an order-preserving map to arrange the input symbols to (0, ..., k-1)\n"); 31 | fprintf(stderr, "inputfile is presumed to consist of " STATDATA_STRING " in machine format\n"); 32 | fprintf(stderr, "output is sent to stdout, and is " STATDATA_STRING " in machine format\n"); 33 | fprintf(stderr, "-v\tVerbose mode (can be used up to 3 times for increased verbosity).\n"); 34 | fprintf(stderr, "-n\tNo data output. Report number of symbols on stdout.\n"); 35 | exit(EX_USAGE); 36 | } 37 | 38 | int main(int argc, char *argv[]) { 39 | FILE *infp; 40 | size_t datalen; 41 | statData_t *data; 42 | size_t k; 43 | int opt; 44 | bool configSymbolsOnly; 45 | double median; 46 | 47 | configVerbose = 0; 48 | configSymbolsOnly = false; 49 | data = NULL; 50 | 51 | assert(PRECISION(UINT_MAX) >= 32); 52 | 53 | while ((opt = getopt(argc, argv, "vn")) != -1) { 54 | switch (opt) { 55 | case 'v': 56 | configVerbose++; 57 | break; 58 | case 'n': 59 | configSymbolsOnly = true; 60 | break; 61 | default: /* ? */ 62 | useageExit(); 63 | } 64 | } 65 | 66 | argc -= optind; 67 | argv += optind; 68 | 69 | if (argc != 1) { 70 | useageExit(); 71 | } 72 | 73 | if ((infp = fopen(argv[0], "rb")) == NULL) { 74 | perror("Can't open file"); 75 | exit(EX_NOINPUT); 76 | } 77 | 78 | datalen = readuintfile(infp, &data); 79 | assert(data != NULL); 80 | 81 | if (configVerbose > 0) { 82 | fprintf(stderr, "Read in %zu integers\n", datalen); 83 | } 84 | if (fclose(infp) != 0) { 85 | perror("Couldn't close input data file"); 86 | exit(EX_OSERR); 87 | } 88 | 89 | assert(datalen > 0); 90 | 91 | translate(data, datalen, &k, &median); 92 | if (configVerbose > 0) { 93 | fprintf(stderr, "Found %zu symbols\n", k); 94 | } 95 | 96 | if (configSymbolsOnly) { 97 | printf("%zu\n", k); 98 | } else { 99 | if (fwrite(data, sizeof(statData_t), datalen, stdout) != datalen) { 100 | perror("Can't write to output"); 101 | exit(EX_OSERR); 102 | } 103 | } 104 | 105 | free(data); 106 | return EX_OK; 107 | } 108 | -------------------------------------------------------------------------------- /src/discard-fixed-bits.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "binio.h" 21 | #include "binutil.h" 22 | #include "globals-inst.h" 23 | #include "precision.h" 24 | 25 | noreturn static void useageExit(void) { 26 | fprintf(stderr, "Usage:\n"); 27 | fprintf(stderr, "discard-fixed-bits inputfile\n"); 28 | fprintf(stderr, "inputfile is assumed to be a stream of " STATDATA_STRING " \n"); 29 | fprintf(stderr, "output sent to stdout are " STATDATA_STRING ", with all fixed bits removed and the non-fixed bits moved to the lsb of the output.\n"); 30 | exit(EX_USAGE); 31 | } 32 | 33 | int main(int argc, char *argv[]) { 34 | FILE *infp; 35 | size_t datalen; 36 | statData_t *data = NULL; 37 | uint32_t *u32data = NULL; 38 | uint32_t bitmask; 39 | statData_t maxval, minval; 40 | uint32_t bits; 41 | double doubleBits; 42 | 43 | configVerbose = 2; 44 | 45 | if (argc != 2) { 46 | useageExit(); 47 | } 48 | 49 | if ((infp = fopen(argv[1], "rb")) == NULL) { 50 | perror("Can't open file"); 51 | exit(EX_NOINPUT); 52 | } 53 | 54 | datalen = readuintfile(infp, &data); 55 | assert(data != NULL); 56 | assert(datalen > 0); 57 | 58 | if ((u32data = malloc(sizeof(uint32_t) * datalen)) == NULL) { 59 | perror("Can't allocate extra memory"); 60 | exit(EX_OSERR); 61 | } 62 | 63 | fprintf(stderr, "Read in %zu " STATDATA_STRING "s\n", datalen); 64 | if (fclose(infp) != 0) { 65 | perror("Can't close intput file"); 66 | exit(EX_OSERR); 67 | } 68 | 69 | maxval = 0; 70 | minval = STATDATA_MAX; 71 | 72 | for (size_t i = 0; i < datalen; i++) { 73 | if (data[i] > maxval) { 74 | maxval = data[i]; 75 | } 76 | if (data[i] < minval) { 77 | minval = data[i]; 78 | } 79 | u32data[i] = data[i]; 80 | } 81 | 82 | doubleBits = ceil(log2((double)maxval + 1.0)); 83 | assert(doubleBits >= 0.0); 84 | bits = (statData_t)doubleBits; 85 | 86 | bitmask = getActiveBits(u32data, datalen); 87 | 88 | fprintf(stderr, "Symbols in the range [%u, %u], %u bit, bitmask: 0x%08X\n", minval, maxval, bits, bitmask); 89 | 90 | fprintf(stderr, "Outputting data\n"); 91 | for (size_t i = 0; i < datalen; i++) { 92 | uint32_t out; 93 | out = extractbits(u32data[i], bitmask); 94 | assert(out <= STATDATA_MAX); 95 | data[i] = (statData_t)out; 96 | } 97 | 98 | if (fwrite(data, sizeof(statData_t), datalen, stdout) != datalen) { 99 | perror("Can't write output to stdout"); 100 | exit(EX_OSERR); 101 | } 102 | 103 | free(data); 104 | free(u32data); 105 | 106 | return EX_OK; 107 | } 108 | -------------------------------------------------------------------------------- /src/markov.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "binio.h" 21 | #include "entlib.h" 22 | #include "globals-inst.h" 23 | #include "precision.h" 24 | #include "translate.h" 25 | 26 | noreturn static void useageExit(void) { 27 | fprintf(stderr, "Usage:\n"); 28 | fprintf(stderr, "markov [-v] [-c] [-p ] \n"); 29 | fprintf(stderr, "Run some variant of the SP 800-90B 2016 Markov test.\n"); 30 | fprintf(stderr, "inputfile is presumed to be a stream of " STATDATA_STRING "s\n"); 31 | fprintf(stderr, "-v\tVerbose mode (can be used several times for increased verbosity).\n"); 32 | fprintf(stderr, "-p \tThe lowest acceptable probablility for a symbol to be relevant.\n"); 33 | fprintf(stderr, "-c\tDisable the creation of confidence intervals.\n"); 34 | exit(EX_USAGE); 35 | } 36 | 37 | int main(int argc, char *argv[]) { 38 | FILE *infp; 39 | size_t datalen; 40 | statData_t *data = NULL; 41 | double minent; 42 | size_t k; 43 | double median; 44 | int opt; 45 | bool configConfidenceIntervals; 46 | double configProbCutoff = 0.0; 47 | double indouble; 48 | 49 | configVerbose = 0; 50 | configConfidenceIntervals = true; 51 | configProbCutoff = 0.0; 52 | 53 | while ((opt = getopt(argc, argv, "vcp:")) != -1) { 54 | switch (opt) { 55 | case 'v': 56 | configVerbose++; 57 | break; 58 | case 'c': 59 | configConfidenceIntervals = false; 60 | break; 61 | case 'p': 62 | indouble = strtod(optarg, NULL); 63 | if ((indouble > 1.0) || (indouble < 0.0)) { 64 | useageExit(); 65 | } 66 | 67 | configProbCutoff = indouble; 68 | break; 69 | default: /* ? */ 70 | useageExit(); 71 | } 72 | } 73 | 74 | argc -= optind; 75 | argv += optind; 76 | 77 | if (argc != 1) { 78 | useageExit(); 79 | } 80 | 81 | if ((infp = fopen(argv[0], "rb")) == NULL) { 82 | perror("Can't open file"); 83 | exit(EX_NOINPUT); 84 | } 85 | 86 | datalen = readuintfile(infp, &data); 87 | assert(data != NULL); 88 | printf("Read in %zu integers\n", datalen); 89 | if (fclose(infp) != 0) { 90 | perror("Couldn't close input data file"); 91 | exit(EX_OSERR); 92 | } 93 | 94 | assert(datalen > 0); 95 | 96 | fprintf(stderr, "Running markov test.\n"); 97 | 98 | translate(data, datalen, &k, &median); 99 | minent = NSAMarkovEstimate(data, datalen, k, "Literal", configConfidenceIntervals, configProbCutoff); 100 | 101 | printf("Assessed min entropy = %.17g\n", minent); 102 | 103 | free(data); 104 | 105 | return EX_OK; 106 | } 107 | -------------------------------------------------------------------------------- /src/u32-counter-endian.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "binio.h" 22 | #include "binutil.h" 23 | #include "globals-inst.h" 24 | #include "precision.h" 25 | 26 | noreturn static void useageExit(void) { 27 | fprintf(stderr, "Usage:\n"); 28 | fprintf(stderr, "u32-counter-endian [-d] \n"); 29 | fprintf(stderr, "Trys to guess counter endianness, and translates to the local platform.\n"); 30 | fprintf(stderr, "-d output differences between adjacent values (when viewing the data as a 32-bit unsigned counter with rollover).\n"); 31 | exit(EX_USAGE); 32 | } 33 | 34 | int main(int argc, char *argv[]) { 35 | FILE *infp; 36 | size_t datalen; 37 | uint32_t *data = NULL; 38 | size_t i; 39 | size_t incCountNative, incCountReversed; 40 | bool configDiffMode; 41 | int opt; 42 | 43 | configDiffMode = false; 44 | incCountNative = 0; 45 | incCountReversed = 0; 46 | 47 | while ((opt = getopt(argc, argv, "d")) != -1) { 48 | switch (opt) { 49 | case 'd': 50 | configDiffMode = true; 51 | break; 52 | default: /* ? */ 53 | useageExit(); 54 | } 55 | } 56 | 57 | argc -= optind; 58 | argv += optind; 59 | 60 | if (argc != 1) { 61 | useageExit(); 62 | } 63 | 64 | if ((infp = fopen(argv[0], "rb")) == NULL) { 65 | perror("Can't open file"); 66 | exit(EX_NOINPUT); 67 | } 68 | 69 | datalen = readuint32file(infp, &data); 70 | assert(data != NULL); 71 | 72 | fprintf(stderr, "Read in %zu uint32_ts\n", datalen); 73 | 74 | if (fclose(infp) != 0) { 75 | perror("Can't close intput file"); 76 | exit(EX_OSERR); 77 | } 78 | 79 | for (i = 1; i < datalen; i++) { 80 | if (data[i - 1] <= data[i]) incCountNative++; 81 | if (reverse32(data[i - 1]) <= reverse32(data[i])) incCountReversed++; 82 | } 83 | 84 | if (incCountNative >= incCountReversed) { 85 | fprintf(stderr, "Native format detected (%.17g vs %.17g)\n", ((double)incCountNative) / ((double)datalen), ((double)incCountReversed) / ((double)datalen)); 86 | } else { 87 | fprintf(stderr, "Reversed format detected (%.17g vs %.17g)\n", ((double)incCountReversed) / ((double)datalen), ((double)incCountNative) / ((double)datalen)); 88 | for (i = 0; i < datalen; i++) { 89 | data[i] = reverse32(data[i]); 90 | } 91 | } 92 | 93 | if (configDiffMode) { 94 | for (i = 1; i < datalen; i++) { 95 | data[i - 1] = data[i] - data[i - 1]; 96 | } 97 | datalen--; 98 | } 99 | 100 | if (fwrite(data, sizeof(uint32_t), datalen, stdout) != datalen) { 101 | perror("Can't write to stdout"); 102 | exit(EX_OSERR); 103 | } 104 | 105 | free(data); 106 | 107 | return (0); 108 | } 109 | -------------------------------------------------------------------------------- /src/u32-delta.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020-2024 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "binio.h" 21 | #include "globals-inst.h" 22 | #include "precision.h" 23 | 24 | noreturn static void useageExit(void) { 25 | fprintf(stderr, "Usage:\n"); 26 | fprintf(stderr, "u32-delta \n"); 27 | fprintf(stderr, "Extract deltas and then translate the result to a positive value.\n"); 28 | fprintf(stderr, "The values are output via stdout.\n"); 29 | exit(EX_USAGE); 30 | } 31 | 32 | int main(int argc, char *argv[]) { 33 | FILE *infp; 34 | size_t datalen; 35 | uint32_t *data = NULL; 36 | size_t i; 37 | int64_t *delta = NULL; 38 | int64_t min; 39 | int64_t max; 40 | int64_t newmax; 41 | int64_t curdelta; 42 | 43 | if (argc != 2) { 44 | useageExit(); 45 | } 46 | 47 | if ((infp = fopen(argv[1], "rb")) == NULL) { 48 | perror("Can't open file"); 49 | exit(EX_NOINPUT); 50 | } 51 | 52 | datalen = readuint32file(infp, &data); 53 | assert(data != NULL); 54 | 55 | fprintf(stderr, "Read in %zu uint32_ts\n", datalen); 56 | if (fclose(infp) != 0) { 57 | perror("Can't close intput file"); 58 | exit(EX_OSERR); 59 | } 60 | 61 | if ((delta = malloc(datalen * sizeof(int64_t))) == NULL) { 62 | perror("Can't allocate extra memory"); 63 | exit(EX_OSERR); 64 | } 65 | 66 | max = INT64_MIN; 67 | min = INT64_MAX; 68 | 69 | if (datalen < 2) { 70 | fprintf(stderr, "Too little data\n"); 71 | exit(EX_DATAERR); 72 | } 73 | 74 | for (i = 1; i < datalen; i++) { 75 | curdelta = (int64_t)data[i] - (int64_t)data[i - 1]; 76 | 77 | delta[i - 1] = curdelta; 78 | if (curdelta < min) min = curdelta; 79 | if (curdelta > max) max = curdelta; 80 | } 81 | 82 | newmax = max - min; 83 | 84 | fprintf(stderr, "min diff: %" PRId64 ", max diff: %" PRId64 "\n", min, max); 85 | if (newmax > UINT32_MAX) { 86 | if (delta != NULL) free(delta); 87 | delta = NULL; 88 | if (data != NULL) free(data); 89 | data = NULL; 90 | fprintf(stderr, "Can't map this to the appropriate range\n"); 91 | exit(EX_DATAERR); 92 | } 93 | 94 | for (i = 0; i < datalen - 1; i++) { 95 | assert(delta[i] >= min); 96 | assert(delta[i] - min <= UINT32_MAX); 97 | data[i] = (uint32_t)(delta[i] - min); 98 | } 99 | 100 | if (fwrite(data, sizeof(uint32_t), datalen - 1, stdout) != datalen - 1) { 101 | if (delta != NULL) free(delta); 102 | delta = NULL; 103 | if (data != NULL) free(data); 104 | data = NULL; 105 | perror("Can't write to stdout"); 106 | exit(EX_OSERR); 107 | } 108 | 109 | if (delta != NULL) free(delta); 110 | delta = NULL; 111 | if (data != NULL) free(data); 112 | data = NULL; 113 | return (0); 114 | } 115 | -------------------------------------------------------------------------------- /src/randlib.h: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #ifndef RANDLIB_H 11 | #define RANDLIB_H 12 | #include 13 | #include 14 | #include "SFMT.h" 15 | #include "enttypes.h" 16 | 17 | struct randstate { 18 | bool seeded; 19 | bool deterministic; 20 | bool buffered32Avail; 21 | uint32_t buffered32; 22 | sfmt_t sfmt; 23 | uint64_t xoshiro256starstarState[4]; 24 | }; 25 | 26 | struct SUMSstate { 27 | double bias; 28 | double serialCoefficient; 29 | double stepNoiseMean; 30 | double stepNoiseStdDev; 31 | double leftStepSize; 32 | double rightStepSize; 33 | uint32_t lastOutput; 34 | double lastEntropy; 35 | }; 36 | 37 | struct MBMstate { 38 | double A; 39 | double R; 40 | double sigma; 41 | double limit; 42 | double V; 43 | uint32_t lastOutput; 44 | double lastEntropy; 45 | size_t noiseFlips; 46 | }; 47 | 48 | struct processedMBMstate { 49 | bool initilized; 50 | size_t aCurIndex; 51 | uint32_t aOutputs[3]; 52 | double aEntropies[3]; 53 | uint32_t lastOutput; 54 | double lastEntropy; 55 | }; 56 | 57 | struct sinBiasState { 58 | double magnitude; 59 | double period; 60 | double currentPhase; 61 | double lastEntropy; 62 | }; 63 | 64 | void initGenerator(struct randstate *rstate); 65 | void seedGenerator(struct randstate *rstate); 66 | uint64_t randomu64(struct randstate *rstate); 67 | uint32_t randomu32(struct randstate *rstate); 68 | uint32_t randomRange(uint32_t high, struct randstate *rstate); 69 | uint64_t randomRange64(uint64_t high, struct randstate *rstate); 70 | uint32_t genRandBiasedBit(double bias, struct randstate *rstate); 71 | unsigned char *genRandBitBytes(double bias, size_t datalen, struct randstate *rstate); 72 | void genRandInts(statData_t *data, size_t datalen, uint32_t k, struct randstate *rstate); 73 | double randomSignedUnit(struct randstate *rstate); 74 | double randomUnit(struct randstate *rstate); 75 | void normalVariate(double mean, double stdDev, double *out1, double *out2, struct randstate *rstate); 76 | double truncatedNormalminEnt(double mean, double stddev, uint32_t bits); 77 | uint32_t genNormalInt(double mean, double stddev, uint32_t bits, struct sinBiasState *bstate, struct randstate *rstate); 78 | uint32_t SUMSgenerate(struct SUMSstate *sstate, struct randstate *rstate); 79 | uint32_t genRandCorrelatedBit(double c, uint32_t lastbit, struct randstate *rstate); 80 | uint32_t genRandBiasedInt(uint32_t bits, double p, struct randstate *rstate); 81 | uint32_t LFSRFilter(uint32_t input, uint32_t bits, uint64_t *state, uint64_t polynomial); 82 | void FYShuffle(double *indata, size_t datalen, struct randstate *rstate); 83 | statData_t ringOscillatorNextDeterministicSample(double oscFreq, double oscPhase, double sampleFreq, double samplePhase); 84 | statData_t ringOscillatorNextNonDeterministicSample(double oscFreq, double oscJitter, double *relOscPhase, double sampleFreq, double *relSamplePhase, struct randstate *rstate); 85 | void randomBits(statData_t *data, size_t datalen, struct randstate *rstate); 86 | uint8_t genRandBit(double p, struct randstate *rstate); 87 | #endif 88 | -------------------------------------------------------------------------------- /src/rbt_misc.h: -------------------------------------------------------------------------------- 1 | /*************************************************************************** 2 | * Red-Black Tree routines, implements balanced tree data storage. 3 | * 4 | * The code base was originally written by Emin Marinian: 5 | * http://www.csua.berkeley.edu/~emin/index.html 6 | * http://alum.mit.edu/www/emin 7 | * License historically available here: 8 | * http://web.mit.edu/~emin/www.old/source_code/red_black_tree/LICENSE 9 | * This is available using the Wayback machine: 10 | * https://web.archive.org/web/20180112031521/http://web.mit.edu/~emin/www.old/source_code/red_black_tree/LICENSE 11 | * This is include below: 12 | * 13 | * Redistribution and use in source and binary forms, with or without 14 | * modification, are permitted provided that neither the name of Emin 15 | * Martinian nor the names of any contributors are be used to endorse or 16 | * promote products derived from this software without specific prior 17 | * written permission. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | * 31 | ***************************************************************************/ 32 | 33 | /* This file is included as part of the Theseus distribution. 34 | * 35 | * Author(s) 36 | * Emin Marinian 37 | * Joshua E. Hill, UL VS LLC. 38 | * Joshua E. Hill, KeyPair Consulting, Inc. 39 | */ 40 | 41 | #ifndef INC_E_MISC_ 42 | #define INC_E_MISC_ 43 | 44 | #include 45 | 46 | /* CONVENTIONS: All data structures for red-black trees have the prefix */ 47 | /* "rb_" to prevent name conflicts. */ 48 | /* */ 49 | /* Function names: Each word in a function name begins with */ 50 | /* a capital letter. An example funcntion name is */ 51 | /* CreateRedTree(a,b,c). Furthermore, each function name */ 52 | /* should begin with a capital letter to easily distinguish */ 53 | /* them from variables. */ 54 | /* */ 55 | /* Variable names: Each word in a variable name begins with */ 56 | /* a capital letter EXCEPT the first letter of the variable */ 57 | /* name. For example, int newLongInt. Global variables have */ 58 | /* names beginning with "g". An example of a global */ 59 | /* variable name is gNewtonsConstant. */ 60 | 61 | void Assert(int assertion, char* error); 62 | void* SafeMalloc(size_t size); 63 | void NullFunction(void* junk); 64 | 65 | #endif 66 | -------------------------------------------------------------------------------- /src/interleave-data.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2023 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include // for getopt, optind 12 | #include 13 | #include 14 | #include // for uint32_t 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "binio.h" 22 | #include "entlib.h" 23 | #include "globals-inst.h" 24 | #include "precision.h" 25 | #include "translate.h" 26 | 27 | noreturn static void useageExit(void) { 28 | fprintf(stderr, "Usage:\n"); 29 | fprintf(stderr, "interleave-data \n"); 30 | fprintf(stderr, "Interleave the data from inputfile1 and inputfile2\n"); 31 | fprintf(stderr, "files are presumed to consist of " STATDATA_STRING " in machine format\n"); 32 | fprintf(stderr, "output is sent to stdout, and is " STATDATA_STRING " in machine format\n"); 33 | fprintf(stderr, "-v\tVerbose mode (can be used up to 3 times for increased verbosity).\n"); 34 | exit(EX_USAGE); 35 | } 36 | 37 | int main(int argc, char *argv[]) { 38 | FILE *infp; 39 | size_t datalen1, datalen2; 40 | statData_t *data1; 41 | statData_t *data2; 42 | statData_t *combinedData; 43 | size_t symbolSets; 44 | int opt; 45 | 46 | configVerbose = 0; 47 | data1 = NULL; 48 | data2 = NULL; 49 | 50 | assert(PRECISION(UINT_MAX) >= 32); 51 | 52 | while ((opt = getopt(argc, argv, "vn")) != -1) { 53 | switch (opt) { 54 | case 'v': 55 | configVerbose++; 56 | break; 57 | default: /* ? */ 58 | useageExit(); 59 | } 60 | } 61 | 62 | argc -= optind; 63 | argv += optind; 64 | 65 | if (argc != 2) { 66 | useageExit(); 67 | } 68 | 69 | if ((infp = fopen(argv[0], "rb")) == NULL) { 70 | perror("Can't open inputfile1"); 71 | exit(EX_NOINPUT); 72 | } 73 | 74 | datalen1 = readuintfile(infp, &data1); 75 | assert(data1 != NULL); 76 | 77 | if (configVerbose > 0) { 78 | fprintf(stderr, "Read in %zu integers from inputfile1\n", datalen1); 79 | } 80 | if (fclose(infp) != 0) { 81 | perror("Couldn't close inputfile1"); 82 | exit(EX_OSERR); 83 | } 84 | 85 | assert(datalen1 > 0); 86 | 87 | if ((infp = fopen(argv[1], "rb")) == NULL) { 88 | perror("Can't open inputfile2"); 89 | exit(EX_NOINPUT); 90 | } 91 | 92 | datalen2 = readuintfile(infp, &data2); 93 | assert(data2 != NULL); 94 | 95 | if (configVerbose > 0) { 96 | fprintf(stderr, "Read in %zu integers from inputfile2\n", datalen2); 97 | } 98 | if (fclose(infp) != 0) { 99 | perror("Couldn't close inputfile2"); 100 | exit(EX_OSERR); 101 | } 102 | 103 | assert(datalen2 > 0); 104 | 105 | symbolSets = (datalen1 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "binio.h" 21 | #include "globals-inst.h" 22 | #include "precision.h" 23 | 24 | noreturn static void useageExit(void) { 25 | fprintf(stderr, "Usage:\n"); 26 | fprintf(stderr, "u32-counter-bitwidth \n"); 27 | fprintf(stderr, "Extract deltas under the assumption that the data is an increasing counter of some inferred bitwidth.\n"); 28 | exit(EX_USAGE); 29 | } 30 | 31 | int main(int argc, char *argv[]) { 32 | FILE *infp; 33 | size_t datalen; 34 | uint32_t *data = NULL; 35 | uint32_t *delta = NULL; 36 | uint32_t rawmax; 37 | uint64_t nextpower; 38 | uint32_t min; 39 | uint32_t max; 40 | 41 | if (argc != 2) { 42 | useageExit(); 43 | } 44 | 45 | if ((infp = fopen(argv[1], "rb")) == NULL) { 46 | perror("Can't open file"); 47 | exit(EX_NOINPUT); 48 | } 49 | 50 | if ((datalen = readuint32file(infp, &data)) < 1) { 51 | useageExit(); 52 | } 53 | 54 | assert(data != NULL); 55 | 56 | fprintf(stderr, "Read in %zu uint32_ts\n", datalen); 57 | if (fclose(infp) != 0) { 58 | perror("Can't close intput file"); 59 | exit(EX_OSERR); 60 | } 61 | 62 | rawmax = 0; 63 | for (size_t i = 0; i < datalen; i++) 64 | if (data[i] > rawmax) rawmax = data[i]; 65 | 66 | rawmax |= rawmax >> 1; 67 | rawmax |= rawmax >> 2; 68 | rawmax |= rawmax >> 4; 69 | rawmax |= rawmax >> 8; 70 | rawmax |= rawmax >> 16; 71 | nextpower = rawmax + 1; 72 | 73 | fprintf(stderr, "Next binary power: %" PRIu64 " (assuming a %i bit counter)\n", nextpower, __builtin_popcount(rawmax)); 74 | 75 | if ((delta = malloc(datalen * sizeof(uint32_t))) == NULL) { 76 | perror("Can't allocate extra memory"); 77 | exit(EX_OSERR); 78 | } 79 | 80 | max = 0; 81 | min = UINT32_MAX; 82 | 83 | if (datalen < 2) { 84 | fprintf(stderr, "Too little data\n"); 85 | exit(EX_DATAERR); 86 | } 87 | 88 | for (size_t i = 1; i < datalen; i++) { 89 | uint64_t deltamin; 90 | 91 | if (data[i] >= data[i - 1]) { 92 | deltamin = data[i] - data[i - 1]; 93 | } else { 94 | deltamin = ((uint64_t)data[i] + nextpower) - (uint64_t)data[i - 1]; 95 | } 96 | 97 | assert(deltamin <= UINT32_MAX); 98 | 99 | delta[i - 1] = (uint32_t)deltamin; 100 | if (deltamin < min) min = (uint32_t)deltamin; 101 | if (deltamin > max) max = (uint32_t)deltamin; 102 | } 103 | 104 | assert(max >= min); 105 | fprintf(stderr, "min diff: %u, max diff: %u\n", min, max); 106 | fprintf(stderr, "Translating min diff to 0 (new max diff: %u)\n", max - min); 107 | 108 | for (size_t i = 0; i < datalen - 1; i++) { 109 | assert(delta[i] >= min); 110 | data[i] = delta[i] - min; 111 | } 112 | 113 | if (fwrite(data, sizeof(uint32_t), datalen - 1, stdout) != datalen - 1) { 114 | if (delta != NULL) free(delta); 115 | delta = NULL; 116 | if (data != NULL) free(data); 117 | data = NULL; 118 | perror("Can't write to stdout"); 119 | exit(EX_OSERR); 120 | } 121 | 122 | if (delta != NULL) free(delta); 123 | delta = NULL; 124 | if (data != NULL) free(data); 125 | data = NULL; 126 | return (0); 127 | } 128 | -------------------------------------------------------------------------------- /src/symbol-block-dist.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2025 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, KeyPair Consulting, Inc. 8 | */ 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "binio.h" 23 | #include "entlib.h" 24 | #include "globals-inst.h" 25 | #include "precision.h" 26 | #include "translate.h" 27 | 28 | noreturn static void useageExit(void) { 29 | fprintf(stderr, "Usage:\n"); 30 | fprintf(stderr, "symbol-block-dist [-v] \n"); 31 | fprintf(stderr, "Create provide the distribution of the count of in (non-overlapping) blocks in the provided file.\n"); 32 | fprintf(stderr, "inputfile is presumed to be a stream of " STATDATA_STRING "s\n"); 33 | fprintf(stderr, "-v\tVerbose mode (can be used several times for increased verbosity).\n"); 34 | exit(EX_USAGE); 35 | } 36 | 37 | int main(int argc, char *argv[]) { 38 | FILE *infp; 39 | size_t datalen; 40 | statData_t *data = NULL; 41 | int opt; 42 | statData_t referenceSymbol; 43 | size_t blockSize; 44 | char *endptr; 45 | unsigned long long int inullint; 46 | size_t *symbolCount; 47 | 48 | configVerbose = 0; 49 | 50 | while ((opt = getopt(argc, argv, "v")) != -1) { 51 | switch (opt) { 52 | case 'v': 53 | configVerbose++; 54 | break; 55 | default: /* ? */ 56 | useageExit(); 57 | } 58 | } 59 | 60 | argc -= optind; 61 | argv += optind; 62 | 63 | if (argc != 3) { 64 | useageExit(); 65 | } 66 | 67 | // Establish the block size 68 | endptr=NULL; 69 | inullint = strtoull(argv[1], &endptr, 0); 70 | if (((inullint == ULLONG_MAX) && (errno == ERANGE)) || (inullint == 0)) { 71 | fprintf(stderr, "Can't interpret block size value\n"); 72 | useageExit(); 73 | } 74 | blockSize = (size_t)inullint; 75 | 76 | // Establish the reference symbol 77 | endptr=NULL; 78 | inullint = strtoull(argv[0], &endptr, 0); 79 | if ((inullint >= 256) || ((inullint == 0) && (endptr == argv[0]))) { 80 | fprintf(stderr, "Can't interpret reference symbol value\n"); 81 | useageExit(); 82 | } 83 | referenceSymbol = (statData_t)inullint; 84 | 85 | if ((infp = fopen(argv[2], "rb")) == NULL) { 86 | perror("Can't open file"); 87 | exit(EX_NOINPUT); 88 | } 89 | 90 | datalen = readuintfile(infp, &data); 91 | assert(data != NULL); 92 | printf("Read in %zu integers\n", datalen); 93 | if (fclose(infp) != 0) { 94 | perror("Couldn't close input data file"); 95 | exit(EX_OSERR); 96 | } 97 | 98 | assert(datalen > 0); 99 | 100 | assert(datalen >= blockSize); 101 | 102 | symbolCount = malloc((blockSize+1)*sizeof(size_t)); 103 | 104 | fprintf(stderr, "Counting the number of symbol %u in %zu-symbol blocks.\n", referenceSymbol, blockSize); 105 | memset(symbolCount, 0, (blockSize+1)*sizeof(size_t)); 106 | 107 | for(size_t i = 0; i < datalen; i += blockSize) { 108 | statData_t *curBase = data+i; 109 | size_t curBlockCount = 0; 110 | //New block processing 111 | for(size_t j = 0; j < blockSize; j++) { 112 | if(curBase[j] == referenceSymbol) curBlockCount++; 113 | } 114 | symbolCount[curBlockCount]++; 115 | } 116 | 117 | free(data); 118 | 119 | for(size_t j = 0; j <= blockSize; j++) 120 | if(symbolCount[j]>0) printf("%zu: %zu\n", j, symbolCount[j]); 121 | 122 | free(symbolCount); 123 | return EX_OK; 124 | } 125 | -------------------------------------------------------------------------------- /src/u8-to-sd.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "entlib.h" 22 | 23 | noreturn static void useageExit(void) { 24 | fprintf(stderr, "Usage:\n"); 25 | fprintf(stderr, "u8-to-sd [-l] [-v] \n"); 26 | fprintf(stderr, "-l\tBytes should be output low bits to high bits.\n"); 27 | fprintf(stderr, "-v\tIncrease verbosity. Can be used multiple times.\n"); 28 | fprintf(stderr, "The values are expected to be provided via stdin.\n"); 29 | fprintf(stderr, "The output values are sent to stdout in the form of " STATDATA_STRING " integers.\n"); 30 | exit(EX_USAGE); 31 | } 32 | 33 | int main(int argc, char *argv[]) { 34 | statData_t outdata; 35 | uint8_t indata; 36 | uint8_t bitmask; 37 | uint8_t configBitsPerSample; 38 | uint8_t j; 39 | bool configLTH; 40 | int configVerbose; 41 | int opt; 42 | unsigned long inint; 43 | 44 | configLTH = false; 45 | configVerbose = 0; 46 | 47 | while ((opt = getopt(argc, argv, "vl")) != -1) { 48 | switch (opt) { 49 | case 'v': 50 | configVerbose++; 51 | break; 52 | case 'l': 53 | configLTH = true; 54 | break; 55 | default: /* ? */ 56 | useageExit(); 57 | } 58 | } 59 | 60 | argc -= optind; 61 | argv += optind; 62 | 63 | if (argc != 1) { 64 | useageExit(); 65 | } 66 | 67 | inint = strtoul(argv[0], NULL, 0); 68 | if ((inint == ULONG_MAX) || (errno == EINVAL) || (inint > 4)) { 69 | useageExit(); 70 | } 71 | 72 | configBitsPerSample = (uint8_t)inint; 73 | if ((configBitsPerSample != 1) && (configBitsPerSample != 2) && (configBitsPerSample != 4)) { 74 | if (configVerbose > 0) fprintf(stderr, "Bits per sample: %u\n", configBitsPerSample); 75 | useageExit(); 76 | } 77 | 78 | while (feof(stdin) == 0) { 79 | if (fread(&indata, sizeof(uint8_t), 1, stdin) == 1) { 80 | if (configVerbose > 1) fprintf(stderr, "indata: 0x%X\n", indata); 81 | 82 | // Set the low order configBitsPerSample bits in the bitmask 83 | bitmask = (uint8_t)((1U << configBitsPerSample) - 1U); 84 | if (!configLTH) { 85 | // Shift the masked bits to the high order 86 | bitmask = (uint8_t)(bitmask << (8U - configBitsPerSample)); 87 | } 88 | 89 | for (j = 0; j < 8; j = (uint8_t)(j + configBitsPerSample)) { 90 | if (configVerbose > 1) fprintf(stderr, "Bitmask: 0x%X\n", bitmask); 91 | assert(bitmask != 0); 92 | outdata = (statData_t)(indata & bitmask); 93 | if (configVerbose > 1) fprintf(stderr, "pre-outdata: 0x%X\n", outdata); 94 | if (configLTH) { 95 | outdata = (uint8_t)(outdata >> j); 96 | } else { 97 | outdata = (uint8_t)(outdata >> (8U - configBitsPerSample - j)); 98 | } 99 | 100 | if (configVerbose > 1) fprintf(stderr, "outdata: 0x%X\n", outdata); 101 | if (fwrite(&outdata, sizeof(statData_t), 1, stdout) != 1) { 102 | perror("Can't write to output"); 103 | exit(EX_OSERR); 104 | } 105 | 106 | if (configLTH) { 107 | bitmask = (uint8_t)(bitmask << configBitsPerSample); 108 | } else { 109 | bitmask = (uint8_t)(bitmask >> configBitsPerSample); 110 | } 111 | } 112 | } 113 | } 114 | 115 | return (0); 116 | } 117 | -------------------------------------------------------------------------------- /src/u64-jent-to-delta.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "binio.h" 18 | #include "binutil.h" 19 | #include "globals-inst.h" 20 | 21 | noreturn static void useageExit(void) { 22 | fprintf(stderr, "Usage:\n"); 23 | fprintf(stderr, "u64-jent-to-delta\n"); 24 | fprintf(stderr, "input comes from stdin are in uint64_t, in the default jent delta format.\n"); 25 | fprintf(stderr, "output sent to stdout is the number of nanoseconds represented by the delta.\n"); 26 | exit(EX_USAGE); 27 | } 28 | 29 | static uint64_t mullerDeltaToNanosecondDelta(uint64_t delta) { 30 | // The upper 32 bits of the delta contains the number of seconds since the epoch. We do not (in general) expect this to roll over. 31 | // Note: if the time is adjusted between samples, then it may go negative, but these should be discarded, not treated as integer rollovers. 32 | const uint64_t secondsPlace = (delta >> 32) * 1000000000UL; 33 | const uint64_t nanosecondsPlace = delta & 0xFFFFFFFFUL; 34 | 35 | // Process the lower 32bits (the nanosecond count). 36 | // Did the _seconds_ place get borrowed from? 37 | // The nanoseconds place should normally be in the range [0, 999999999], 38 | // but in the instance where the second count is borrowed from, the calculation then 2^32 was added (by borrowing from seconds place). 39 | // As a consequence, bit 31 (the high order bit in the 32-bit word) signals if borrowing occurred. 40 | if ((nanosecondsPlace & ((uint64_t)0x80000000UL)) == 0) { 41 | // No borrowing 42 | // Convert to a reasonable counter by just shift/multiply and add 43 | return secondsPlace + nanosecondsPlace; 44 | } else { 45 | // The second count was borrowed from; this is correct, so the shift/multiply for the seconds place is the same. 46 | // The lower bits, however, need to have the (wrong) borrow reversed (the original borrow was 2^32), 47 | // and the correct one applied (10^9). 48 | const uint64_t correctedNanosecondsPlace = 1000000000UL + nanosecondsPlace - ((uint64_t)0x0100000000UL); 49 | return secondsPlace + correctedNanosecondsPlace; 50 | } 51 | } 52 | 53 | int main(void) { 54 | uint64_t *input = NULL; 55 | uint64_t *nativeData = NULL; 56 | uint64_t *swappedData = NULL; 57 | uint64_t *outData = NULL; 58 | size_t datalen; 59 | size_t nativeSmallerCount = 0; 60 | 61 | if ((datalen = readuint64file(stdin, &input)) < 1) { 62 | useageExit(); 63 | } 64 | 65 | nativeData = malloc(sizeof(uint64_t) * datalen); 66 | swappedData = malloc(sizeof(uint64_t) * datalen); 67 | 68 | if ((nativeData == NULL) || (swappedData == NULL)) { 69 | perror("Can't allocate memory."); 70 | exit(EX_OSERR); 71 | } 72 | 73 | for (size_t i = 0; i < datalen; i++) { 74 | nativeData[i] = mullerDeltaToNanosecondDelta(input[i]); 75 | swappedData[i] = mullerDeltaToNanosecondDelta(reverse64(input[i])); 76 | if (nativeData[i] <= swappedData[i]) nativeSmallerCount++; 77 | } 78 | 79 | if (nativeSmallerCount >= (datalen - nativeSmallerCount)) { 80 | fprintf(stderr, "Native byte order seems better (%g)\n", (double)nativeSmallerCount / (double)datalen); 81 | outData = nativeData; 82 | } else { 83 | fprintf(stderr, "Swapped byte order seems better (%g)\n", (double)(datalen - nativeSmallerCount) / (double)datalen); 84 | outData = swappedData; 85 | } 86 | 87 | if (fwrite(outData, sizeof(uint64_t), datalen, stdout) != datalen) { 88 | perror("Can't write out data"); 89 | } 90 | 91 | free(nativeData); 92 | free(swappedData); 93 | free(input); 94 | } 95 | -------------------------------------------------------------------------------- /src/u32-manbin.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020-2025 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "binio.h" 22 | #include "globals-inst.h" 23 | #include "precision.h" 24 | #include "rbt_misc.h" 25 | #include "red_black_tree.h" 26 | 27 | noreturn static void useageExit(void) { 28 | fprintf(stderr, "Usage:\n"); 29 | fprintf(stderr, "u32-manbin inputfile cutoff_1 ... cutoff_{n-1}\n"); 30 | fprintf(stderr, "convert data to one of the n bin numbers (0, ..., n-1).\n"); 31 | fprintf(stderr, "The cutoffs specify the first value in the next bin (so the first bin is [0, cutoff_1), the second bin is [cutoff_1, cutoff_2), the last bin is [cutoff_{n-1}, UINT32_MAX ] etc.)\n"); 32 | fprintf(stderr, "inputfile is assumed to be a stream of uint32_ts\n"); 33 | fprintf(stderr, "output is to stdout, and is uint8_t values\n"); 34 | exit(EX_USAGE); 35 | } 36 | 37 | int main(int argc, char *argv[]) { 38 | FILE *infp; 39 | uint32_t *data = NULL; 40 | size_t i; 41 | int curarg; 42 | size_t bounds; 43 | size_t datalen; 44 | int64_t cutoffs[255]; 45 | int64_t lowbound; 46 | size_t symbolCount[256] = {0}; 47 | 48 | for (i = 0; i < 255; i++) cutoffs[i] = -1; 49 | // The total number of bins must <= 256, so 50 | // bounds + 1 <= 256 51 | // (argc-2) + 1 <= 257 52 | // argc <= 258 53 | if ((argc < 3) || (argc > 258)) { 54 | useageExit(); 55 | } 56 | 57 | if ((infp = fopen(argv[1], "rb")) == NULL) { 58 | perror("Can't open file"); 59 | exit(EX_NOINPUT); 60 | } 61 | 62 | for (curarg = 2; curarg < argc; curarg++) { 63 | unsigned long lout = strtoul(argv[curarg], NULL, 0); 64 | if ((lout > UINT32_MAX) || (errno != 0)) useageExit(); 65 | cutoffs[curarg - 2] = (int64_t)lout; 66 | } 67 | 68 | bounds = (size_t)argc - 2; 69 | fprintf(stderr, "A total of %zu output bins.\n", bounds + 1); 70 | assert(bounds <= UINT8_MAX); 71 | if (bounds < 1) useageExit(); 72 | if (cutoffs[0] == 0) useageExit(); 73 | if (cutoffs[bounds - 1] > UINT32_MAX) useageExit(); 74 | 75 | lowbound = 0; 76 | 77 | for (i = 0; i < bounds; i++) { 78 | if (lowbound >= cutoffs[i]) { 79 | useageExit(); 80 | } else { 81 | fprintf(stderr, "[ %ld, %ld ), ", lowbound, cutoffs[i]); 82 | lowbound = cutoffs[i]; 83 | } 84 | } 85 | fprintf(stderr, "[ %ld, %ld ]\n", cutoffs[bounds - 1], (int64_t)UINT32_MAX); 86 | 87 | datalen = readuint32file(infp, &data); 88 | if (datalen < 2) { 89 | useageExit(); 90 | } 91 | 92 | assert(data != NULL); 93 | 94 | fprintf(stderr, "Read in %zu samples\n", datalen); 95 | if (fclose(infp) != 0) { 96 | perror("Can't close input file"); 97 | exit(EX_OSERR); 98 | } 99 | 100 | fprintf(stderr, "Outputting the data...\n"); 101 | for (i = 0; i < datalen; i++) { 102 | // set this to the last valid symbol 103 | uint8_t symbol = (uint8_t)bounds; 104 | for (uint8_t j = 0; j < bounds; j++) { 105 | if (cutoffs[j] > data[i]) { 106 | symbol = j; 107 | break; 108 | } 109 | } 110 | 111 | assert(symbol <= bounds); 112 | symbolCount[symbol] ++; 113 | 114 | if (fwrite(&symbol, sizeof(uint8_t), 1, stdout) != 1) { 115 | perror("Can't write output to stdout"); 116 | exit(EX_OSERR); 117 | } 118 | } 119 | 120 | for(size_t j = 0; j < 256; j++) { 121 | if(symbolCount[j] > 0) fprintf(stderr, "%zu: %zu\n", j, symbolCount[j]); 122 | } 123 | 124 | free(data); 125 | return EX_OK; 126 | } 127 | -------------------------------------------------------------------------------- /src/u32-regress-to-mean.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020-2024 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "binio.h" 23 | #include "globals-inst.h" 24 | #include "precision.h" 25 | 26 | noreturn static void useageExit(void) { 27 | fprintf(stderr, "Usage:\n"); 28 | fprintf(stderr, "u32-regress-to-mean \n"); 29 | fprintf(stderr, "Calculate the mean, and then force each k-block to have this mean, and then round the resulting values.\n"); 30 | exit(EX_USAGE); 31 | } 32 | 33 | int main(int argc, char *argv[]) { 34 | FILE *infp; 35 | size_t datalen; 36 | uint32_t *data = NULL; 37 | long double globalMean = 0.0L; 38 | char *nextc; 39 | size_t k; 40 | size_t blockCount; 41 | 42 | if (argc != 3) { 43 | useageExit(); 44 | } 45 | 46 | if ((infp = fopen(argv[1], "rb")) == NULL) { 47 | perror("Can't open file"); 48 | exit(EX_NOINPUT); 49 | } 50 | 51 | if ((datalen = readuint32file(infp, &data)) < 1) { 52 | useageExit(); 53 | } 54 | 55 | assert(data != NULL); 56 | k = strtoul(argv[2], &nextc, 0); 57 | if (*nextc != '\0') { 58 | useageExit(); 59 | } 60 | 61 | fprintf(stderr, "Read in %zu uint32_ts\n", datalen); 62 | if (fclose(infp) != 0) { 63 | perror("Can't close intput file"); 64 | exit(EX_OSERR); 65 | } 66 | 67 | // Make k divide datalen. (discard other data) 68 | blockCount = datalen / k; 69 | datalen -= (datalen % k); 70 | 71 | fprintf(stderr, "Processing %zu blocks of %zu samples each.\n", blockCount, k); 72 | 73 | // Calculate the global mean 74 | for (size_t i = 0; i < datalen; i++) { 75 | globalMean += (long double)data[i]; 76 | } 77 | globalMean /= (long double)datalen; // 0 <= globalMean <= UINT32_MAX 78 | fprintf(stderr, "Global mean is %.22Lg.\n", globalMean); 79 | 80 | for (size_t i = 0; i < blockCount; i++) { 81 | long double localMean = 0.0L; 82 | long double localDelta; 83 | 84 | // Calculate the local mean and the necessary delta to fixup the local data. 85 | for (size_t j = 0; j < k; j++) { 86 | localMean += (long double)data[i * k + j]; 87 | } 88 | localMean /= (long double)k; // 0 <= localMean <= UINT32_MAX 89 | 90 | localDelta = globalMean - localMean; 91 | 92 | if (roundl(fabsl(localDelta)) >= 1.0L) { 93 | long double newLocalMean = 0.0L; 94 | int64_t integerAdjust = (int64_t)roundl(localDelta); 95 | 96 | // Rewrite the data block. 97 | for (size_t j = 0; j < k; j++) { 98 | int64_t newdata = (int64_t)data[i * k + j] + integerAdjust; 99 | assert(newdata >= 0); 100 | assert(newdata <= UINT32_MAX); 101 | data[i * k + j] = (uint32_t)newdata; 102 | newLocalMean += (long double)newdata; 103 | } 104 | newLocalMean /= (long double)k; // 0 <= newLocalMean <= UINT32_MAX 105 | 106 | fprintf(stderr, "Adjusting block %zu by %ld: Block mean: %.22Lg (delta: %.22Lg) -> %.22Lg (delta: %.22Lg)\n", i, integerAdjust, localMean, fabsl(localDelta), newLocalMean, fabsl(globalMean - newLocalMean)); 107 | 108 | assert(fabsl(globalMean - newLocalMean) <= fabsl(localDelta)); 109 | } 110 | } 111 | 112 | if (fwrite(data, sizeof(uint32_t), datalen, stdout) != datalen) { 113 | if (data != NULL) free(data); 114 | data = NULL; 115 | perror("Can't write to stdout"); 116 | exit(EX_OSERR); 117 | } 118 | 119 | if (data != NULL) free(data); 120 | data = NULL; 121 | return (0); 122 | } 123 | -------------------------------------------------------------------------------- /src/restart-transpose.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include // for getopt, optarg, optind 21 | #include // for uint32_t 22 | 23 | #include "binio.h" 24 | #include "entlib.h" 25 | #include "globals-inst.h" 26 | #include "precision.h" 27 | 28 | noreturn static void useageExit(void) { 29 | fprintf(stderr, "Usage:\n"); 30 | fprintf(stderr, "restart-transpose [-v] [-l ] [-d ,] \n"); 31 | fprintf(stderr, "inputfile is assumed to be a sequence of " STATDATA_STRING " integers\n"); 32 | fprintf(stderr, "output is sent to stdout\n"); 33 | fprintf(stderr, "-v \t verbose mode.\n"); 34 | fprintf(stderr, "-l \tRead the substring of length . (default index = 0)\n"); 35 | fprintf(stderr, "-d ,\tPerform the restart testing using the described matrix dimensions. (default is 1000x1000)\n"); 36 | exit(EX_USAGE); 37 | } 38 | 39 | int main(int argc, char *argv[]) { 40 | FILE *infp; 41 | int opt; 42 | statData_t *data = NULL; 43 | size_t datalen; 44 | unsigned long long int inint; 45 | char *nextOption; 46 | size_t i, j; 47 | 48 | size_t configSubsetIndex; 49 | size_t configRestarts; 50 | size_t configSamplesPerRestart; 51 | 52 | configSubsetIndex = 0; 53 | configRestarts = 1000; 54 | configSamplesPerRestart = 1000; 55 | 56 | assert(PRECISION(UINT_MAX) >= 32); 57 | assert(PRECISION((unsigned char)UCHAR_MAX) == 8); 58 | 59 | while ((opt = getopt(argc, argv, "vl:d:")) != -1) { 60 | switch (opt) { 61 | case 'v': 62 | configVerbose++; 63 | break; 64 | case 'd': 65 | inint = strtoull(optarg, &nextOption, 0); 66 | if ((inint == ULLONG_MAX) || (errno == EINVAL) || (nextOption == NULL) || (*nextOption != ',')) { 67 | useageExit(); 68 | } 69 | configRestarts = inint; 70 | 71 | nextOption++; 72 | 73 | inint = strtoull(nextOption, NULL, 0); 74 | if ((inint == ULLONG_MAX) || (errno == EINVAL)) { 75 | useageExit(); 76 | } 77 | configSamplesPerRestart = inint; 78 | break; 79 | case 'l': 80 | inint = strtoull(optarg, NULL, 0); 81 | if ((inint == ULLONG_MAX) || (errno == EINVAL)) { 82 | useageExit(); 83 | } 84 | configSubsetIndex = inint; 85 | break; 86 | default: /* ? */ 87 | useageExit(); 88 | } 89 | } 90 | 91 | argc -= optind; 92 | argv += optind; 93 | 94 | if (argc != 1) { 95 | useageExit(); 96 | } 97 | 98 | if ((infp = fopen(argv[0], "rb")) == NULL) { 99 | perror("Can't open file"); 100 | exit(EX_NOINPUT); 101 | } 102 | 103 | assert(configRestarts * configSamplesPerRestart > 0); 104 | 105 | datalen = readuintfileloc(infp, &data, configSubsetIndex, configRestarts * configSamplesPerRestart); 106 | assert(data != NULL); 107 | assert(datalen == configRestarts * configSamplesPerRestart); 108 | 109 | if (fclose(infp) != 0) { 110 | perror("Couldn't close input data file"); 111 | exit(EX_OSERR); 112 | } 113 | 114 | if (configVerbose > 0) { 115 | fprintf(stderr, "Transpose %zu samples.\n", datalen); 116 | } 117 | 118 | for (i = 0; i < configSamplesPerRestart; i++) { 119 | for (j = 0; j < configRestarts; j++) { 120 | if (fwrite(data + configSamplesPerRestart * j + i, sizeof(statData_t), 1, stdout) != 1) { 121 | perror("Can't write output to stdout"); 122 | exit(EX_OSERR); 123 | } 124 | } 125 | } 126 | 127 | free(data); 128 | 129 | return EX_OK; 130 | } 131 | -------------------------------------------------------------------------------- /src/u128-discard-fixed-bits.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "binio.h" 21 | #include "binutil.h" 22 | #include "globals-inst.h" 23 | #include "precision.h" 24 | 25 | noreturn static void useageExit(void) { 26 | fprintf(stderr, "Usage:\n"); 27 | fprintf(stderr, "discard-fixed-bits-u128 inputfile outputgroup\n"); 28 | fprintf(stderr, "inputfile is assumed to be a stream of uint128_t\n"); 29 | fprintf(stderr, "outputgroup sent to stdout are uint32_tegers, with all fixed bits removed and the non-fixed bits moved to the LSB of the output.\n"); 30 | exit(EX_USAGE); 31 | } 32 | 33 | int main(int argc, char *argv[]) { 34 | FILE *infp; 35 | size_t datalen; 36 | uint32_t *data = NULL; 37 | uint32_t *dataPartition[4]; 38 | size_t i, j; 39 | uint32_t bitmask[4]; 40 | uint32_t maxval, minval; 41 | uint32_t bits; 42 | double doubleBits; 43 | int outputGroup; 44 | 45 | assert(PRECISION(UINT_MAX) >= 32); 46 | assert(PRECISION(SIZE_MAX) > 32); 47 | 48 | if (argc != 3) { 49 | useageExit(); 50 | } 51 | 52 | outputGroup = atoi(argv[2]); 53 | 54 | if ((infp = fopen(argv[1], "rb")) == NULL) { 55 | perror("Can't open file"); 56 | exit(EX_NOINPUT); 57 | } 58 | 59 | datalen = readuint32file(infp, &data); 60 | assert(data != NULL); 61 | assert(datalen > 0); 62 | assert((datalen % 4) == 0); 63 | 64 | dataPartition[0] = calloc(datalen / 4, sizeof(uint32_t)); 65 | dataPartition[1] = calloc(datalen / 4, sizeof(uint32_t)); 66 | dataPartition[2] = calloc(datalen / 4, sizeof(uint32_t)); 67 | dataPartition[3] = calloc(datalen / 4, sizeof(uint32_t)); 68 | 69 | if ((dataPartition[0] == NULL) || (dataPartition[1] == NULL) || (dataPartition[2] == NULL) || (dataPartition[3] == NULL)) { 70 | perror("Can't allocate partitions"); 71 | exit(EX_OSERR); 72 | } 73 | 74 | for (j = 0; j < datalen; j++) { 75 | (dataPartition[j & 0x3])[j >> 2] = data[j]; 76 | } 77 | 78 | fprintf(stderr, "Read in %zu uint32_ts\n", datalen); 79 | if (fclose(infp) != 0) { 80 | perror("Can't close intput file"); 81 | exit(EX_OSERR); 82 | } 83 | 84 | for (j = 0; j < 4; j++) { 85 | maxval = 0; 86 | minval = UINT_MAX; 87 | 88 | for (i = 0; i < datalen / 4; i++) { 89 | if ((dataPartition[j])[i] > maxval) { 90 | maxval = (dataPartition[j])[i]; 91 | } 92 | if ((dataPartition[j])[i] < minval) { 93 | minval = (dataPartition[j])[i]; 94 | } 95 | } 96 | doubleBits = ceil(log2((double)maxval + 1.0)); 97 | assert(doubleBits >= 0.0); 98 | bits = (uint32_t)doubleBits; 99 | 100 | bitmask[j] = getActiveBits(dataPartition[j], datalen / 4); 101 | 102 | fprintf(stderr, "Symbols in the range [%u, %u] (%u bit: bitmask 0x%08X)\n", minval, maxval, bits, bitmask[j]); 103 | 104 | for (i = 0; i < datalen / 4; i++) { 105 | (dataPartition[j])[i] = extractbits((dataPartition[j])[i], bitmask[j]); 106 | } 107 | } 108 | 109 | fprintf(stderr, "%d bits total\n", __builtin_popcount(bitmask[0]) + __builtin_popcount(bitmask[1]) + __builtin_popcount(bitmask[2]) + __builtin_popcount(bitmask[3])); 110 | 111 | fprintf(stderr, "Outputting group %d\n", outputGroup); 112 | 113 | for (i = 0; i < datalen / 4; i++) { 114 | if (fwrite(&((dataPartition[outputGroup])[i]), sizeof(uint32_t), 1, stdout) != 1) { 115 | perror("Can't write output to stdout"); 116 | exit(EX_OSERR); 117 | } 118 | } 119 | 120 | free(data); 121 | free(dataPartition[0]); 122 | free(dataPartition[1]); 123 | free(dataPartition[2]); 124 | free(dataPartition[3]); 125 | 126 | return EX_OK; 127 | } 128 | -------------------------------------------------------------------------------- /src/u32-downsample.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2022 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "binio.h" 22 | #include "entlib.h" 23 | #include "globals-inst.h" 24 | 25 | noreturn static void useageExit(void) { 26 | fprintf(stderr, "Usage:\n"); 27 | fprintf(stderr, "u32-downsample [-b ] \n"); 28 | fprintf(stderr, "Groups data by index into modular classes mod evenly into the block size.\n"); 29 | fprintf(stderr, "\tNumber of input samples per output samples\n"); 30 | fprintf(stderr, "-b\tSamples per output block (default 1000000)\n"); 31 | fprintf(stderr, "The u32 values are output via stdout.\n"); 32 | exit(EX_USAGE); 33 | } 34 | 35 | int main(int argc, char *argv[]) { 36 | size_t trimLen; 37 | uint32_t configRate; 38 | size_t configBlockSize; 39 | long int inparam; 40 | int opt; 41 | size_t datalen; 42 | uint32_t *data; 43 | uint32_t *outputBuffer; 44 | unsigned long long inint; 45 | char *nextOption; 46 | FILE *infp; 47 | size_t conjClass; 48 | size_t conjClassSubIndex; 49 | size_t conjClassPartitionSize; 50 | size_t j; 51 | 52 | configVerbose = 0; 53 | configBlockSize = 1000000; 54 | data = NULL; 55 | outputBuffer = NULL; 56 | 57 | while ((opt = getopt(argc, argv, "vb:")) != -1) { 58 | switch (opt) { 59 | case 'v': 60 | configVerbose++; 61 | break; 62 | case 'b': 63 | inint = strtoull(optarg, &nextOption, 0); 64 | if ((inint == ULLONG_MAX) || (errno == EINVAL) || (nextOption == NULL)) { 65 | useageExit(); 66 | } 67 | configBlockSize = inint; 68 | break; 69 | default: /* ? */ 70 | useageExit(); 71 | } 72 | } 73 | 74 | argc -= optind; 75 | argv += optind; 76 | 77 | if (argc != 2) { 78 | useageExit(); 79 | } 80 | 81 | inparam = strtol(argv[0], NULL, 10); 82 | if ((inparam <= 0) || (inparam > UINT32_MAX)) { 83 | useageExit(); 84 | } else { 85 | configRate = (uint32_t)inparam; 86 | } 87 | 88 | if ((infp = fopen(argv[1], "rb")) == NULL) { 89 | perror("Can't open file"); 90 | exit(EX_NOINPUT); 91 | } 92 | 93 | datalen = readuint32file(infp, &data); 94 | assert(data != NULL); 95 | 96 | if (configVerbose > 0) { 97 | fprintf(stderr, "Read in %zu uint32_t integers\n", datalen); 98 | } 99 | // Only deal with data than can be evenly partitioned into a multiple of configRate blocks, each of size configBlockSize 100 | // remove data 101 | 102 | trimLen = datalen % (configRate * configBlockSize); 103 | fprintf(stderr, "Trimming %zu samples\n", trimLen); 104 | datalen = datalen - trimLen; 105 | 106 | if (fclose(infp) != 0) { 107 | perror("Couldn't close input data file"); 108 | exit(EX_OSERR); 109 | } 110 | 111 | assert(datalen > 0); 112 | if ((outputBuffer = malloc(datalen * sizeof(uint32_t))) == NULL) { 113 | perror("Can't allocate output buffer"); 114 | exit(EX_OSERR); 115 | } 116 | 117 | conjClass = 0; 118 | conjClassSubIndex = 0; 119 | assert((datalen % configRate) == 0); 120 | conjClassPartitionSize = datalen / configRate; 121 | 122 | for (j = 0; j < datalen; j++) { 123 | outputBuffer[conjClass * conjClassPartitionSize + conjClassSubIndex] = data[j]; 124 | conjClass++; 125 | if (conjClass == configRate) { 126 | conjClass = 0; 127 | conjClassSubIndex++; 128 | } 129 | } 130 | 131 | assert(conjClass == 0); 132 | assert((conjClassSubIndex % configBlockSize) == 0); 133 | 134 | free(data); 135 | 136 | if (fwrite(outputBuffer, sizeof(uint32_t), datalen, stdout) != datalen) { 137 | perror("Can't write output to stdout"); 138 | exit(EX_OSERR); 139 | } 140 | 141 | free(outputBuffer); 142 | 143 | return (0); 144 | } 145 | -------------------------------------------------------------------------------- /src/downsample.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "binio.h" 22 | #include "entlib.h" 23 | #include "globals-inst.h" 24 | 25 | noreturn static void useageExit(void) { 26 | fprintf(stderr, "Usage:\n"); 27 | fprintf(stderr, "downsample [-b ] \n"); 28 | fprintf(stderr, "Groups data by index into modular classes mod evenly into the block size.\n"); 29 | fprintf(stderr, "\tNumber of input samples per output samples\n"); 30 | fprintf(stderr, "-b\tSamples per output block (default 1000000)\n"); 31 | fprintf(stderr, "The " STATDATA_STRING " values are output via stdout.\n"); 32 | exit(EX_USAGE); 33 | } 34 | 35 | int main(int argc, char *argv[]) { 36 | size_t trimLen; 37 | uint32_t configRate; 38 | size_t configBlockSize; 39 | long int inparam; 40 | int opt; 41 | size_t datalen; 42 | statData_t *data; 43 | statData_t *outputBuffer; 44 | unsigned long long inint; 45 | char *nextOption; 46 | FILE *infp; 47 | size_t conjClass; 48 | size_t conjClassSubIndex; 49 | size_t conjClassPartitionSize; 50 | size_t j; 51 | 52 | configVerbose = 0; 53 | configBlockSize = 1000000; 54 | data = NULL; 55 | outputBuffer = NULL; 56 | 57 | while ((opt = getopt(argc, argv, "vb:")) != -1) { 58 | switch (opt) { 59 | case 'v': 60 | configVerbose++; 61 | break; 62 | case 'b': 63 | inint = strtoull(optarg, &nextOption, 0); 64 | if ((inint == ULLONG_MAX) || (errno == EINVAL) || (nextOption == NULL)) { 65 | useageExit(); 66 | } 67 | configBlockSize = inint; 68 | break; 69 | default: /* ? */ 70 | useageExit(); 71 | } 72 | } 73 | 74 | argc -= optind; 75 | argv += optind; 76 | 77 | if (argc != 2) { 78 | useageExit(); 79 | } 80 | 81 | inparam = strtol(argv[0], NULL, 10); 82 | if ((inparam <= 0) || (inparam > UINT32_MAX)) { 83 | useageExit(); 84 | } else { 85 | configRate = (uint32_t)inparam; 86 | } 87 | 88 | if ((infp = fopen(argv[1], "rb")) == NULL) { 89 | perror("Can't open file"); 90 | exit(EX_NOINPUT); 91 | } 92 | 93 | datalen = readuintfile(infp, &data); 94 | assert(data != NULL); 95 | 96 | if (configVerbose > 0) { 97 | fprintf(stderr, "Read in %zu integers\n", datalen); 98 | } 99 | // Only deal with data than can be evenly partitioned into a multiple of configRate blocks, each of size configBlockSize 100 | // remove data 101 | 102 | trimLen = datalen % (configRate * configBlockSize); 103 | fprintf(stderr, "Trimming %zu samples\n", trimLen); 104 | datalen = datalen - trimLen; 105 | 106 | if (fclose(infp) != 0) { 107 | perror("Couldn't close input data file"); 108 | exit(EX_OSERR); 109 | } 110 | 111 | assert(datalen > 0); 112 | if ((outputBuffer = malloc(datalen * sizeof(statData_t))) == NULL) { 113 | perror("Can't allocate output buffer"); 114 | exit(EX_OSERR); 115 | } 116 | 117 | conjClass = 0; 118 | conjClassSubIndex = 0; 119 | assert((datalen % configRate) == 0); 120 | conjClassPartitionSize = datalen / configRate; 121 | 122 | for (j = 0; j < datalen; j++) { 123 | outputBuffer[conjClass * conjClassPartitionSize + conjClassSubIndex] = data[j]; 124 | conjClass++; 125 | if (conjClass == configRate) { 126 | conjClass = 0; 127 | conjClassSubIndex++; 128 | } 129 | } 130 | 131 | assert(conjClass == 0); 132 | assert((conjClassSubIndex % configBlockSize) == 0); 133 | 134 | free(data); 135 | 136 | if (fwrite(outputBuffer, sizeof(statData_t), datalen, stdout) != datalen) { 137 | perror("Can't write output to stdout"); 138 | exit(EX_OSERR); 139 | } 140 | 141 | free(outputBuffer); 142 | 143 | return (0); 144 | } 145 | -------------------------------------------------------------------------------- /src/u32-mcv.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "binio.h" 21 | #include "entlib.h" 22 | #include "globals-inst.h" 23 | #include "precision.h" 24 | #include "fancymath.h" 25 | 26 | static int uintcompare(const void *in1, const void *in2) { 27 | const uint32_t *left; 28 | const uint32_t *right; 29 | 30 | left = in1; 31 | right = in2; 32 | 33 | if (*left < *right) { 34 | return (-1); 35 | } else if (*left > *right) { 36 | return (1); 37 | } else { 38 | return (0); 39 | } 40 | } 41 | 42 | noreturn static void useageExit(void) { 43 | fprintf(stderr, "Usage:\n"); 44 | fprintf(stderr, "u32-mcv \n"); 45 | fprintf(stderr, "inputfile is assumed to be a stream of uint32_ts\n"); 46 | exit(EX_USAGE); 47 | } 48 | 49 | int main(int argc, char *argv[]) { 50 | FILE *infp; 51 | size_t datalen; 52 | uint32_t *data = NULL; 53 | size_t maxSymbolCount = 0; 54 | uint32_t MLS = 0; 55 | size_t curSymbolCount = 0; 56 | size_t k = 0; 57 | uint32_t curSymbol; 58 | double pu; 59 | double phat; 60 | 61 | if (argc != 2) { 62 | useageExit(); 63 | } 64 | 65 | if ((infp = fopen(argv[1], "rb")) == NULL) { 66 | perror("Can't open file"); 67 | exit(EX_NOINPUT); 68 | } 69 | 70 | if ((datalen = readuint32file(infp, &data)) < 1) { 71 | perror("Data file is empty"); 72 | exit(EX_DATAERR); 73 | } 74 | 75 | assert(data != NULL); 76 | 77 | fprintf(stderr, "Read in %zu uint32_ts\n", datalen); 78 | assert(datalen > 0); 79 | 80 | if (fclose(infp) != 0) { 81 | perror("Can't close intput file"); 82 | exit(EX_OSERR); 83 | } 84 | 85 | //Sort the data 86 | qsort(data, datalen, sizeof(uint32_t), uintcompare); 87 | 88 | MLS = curSymbol = data[0]; 89 | curSymbolCount = 1; 90 | 91 | for(size_t j=1; j maxSymbolCount) { 98 | maxSymbolCount = curSymbolCount; 99 | MLS = curSymbol; 100 | } 101 | 102 | curSymbolCount = 1; 103 | curSymbol = data[j]; 104 | } 105 | } 106 | //This is a transition. 107 | k++; 108 | if(curSymbolCount > maxSymbolCount) { 109 | maxSymbolCount = curSymbolCount; 110 | MLS = curSymbol; 111 | } 112 | 113 | fprintf(stderr, "Encountered %zu distinct symbols.\n", k); 114 | fprintf(stderr, "Most common symbol is 0x%08X (count %zu)\n", MLS, maxSymbolCount); 115 | 116 | phat=((double)maxSymbolCount) / ((double)datalen); 117 | // Note, this is the raw value. A larger value maps to a more conservative estimate, so we then look at upper confidence interval bound. 118 | 119 | // Note, this is a local guess at a confidence interval, under the assumption that this most probable symbol proportion is distributed 120 | // as per the binomial distribution (this CI estimate is made under a normal approximation of the binomial distribution); 121 | // this assumption isn't reasonable, as the most probable symbol can never have less than ceil(L/k) symbols, 122 | // so the entire low end of the binomial distribution isn't in the support for the actual distribution. 123 | // If we (a priori) knew what the MLS was, we could make this estimate, but we don't here. 124 | // This actual distribution here is that of the maximum bin of a multinomial distribution, but this is complicated. 125 | pu = phat + ZALPHA * sqrt(phat * (1.0 - phat) / ((double)datalen - 1.0)); 126 | if (pu > 1.0) { 127 | pu = 1.0; 128 | } 129 | fprintf(stderr, "p_hat = %.17g\n", phat); 130 | fprintf(stderr, "p_u = %.17g\n", pu); 131 | fprintf(stderr, "minentropy = %.17g\n", -log2(pu)); 132 | free(data); 133 | 134 | return EX_OK; 135 | } 136 | -------------------------------------------------------------------------------- /src/rbt_misc.c: -------------------------------------------------------------------------------- 1 | /*************************************************************************** 2 | * Red-Black Tree routines, implements balanced tree data storage. 3 | * 4 | * The code base was originally written by Emin Marinian: 5 | * http://www.csua.berkeley.edu/~emin/index.html 6 | * http://alum.mit.edu/www/emin 7 | * License historically available here: 8 | * http://web.mit.edu/~emin/www.old/source_code/red_black_tree/LICENSE 9 | * This is available using the Wayback machine: 10 | * https://web.archive.org/web/20180112031521/http://web.mit.edu/~emin/www.old/source_code/red_black_tree/LICENSE 11 | * This is include below: 12 | * 13 | * Redistribution and use in source and binary forms, with or without 14 | * modification, are permitted provided that neither the name of Emin 15 | * Martinian nor the names of any contributors are be used to endorse or 16 | * promote products derived from this software without specific prior 17 | * written permission. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | * 31 | ***************************************************************************/ 32 | 33 | /* This file is included as part of the Theseus distribution. 34 | * 35 | * Author(s) 36 | * Emin Marinian 37 | * Joshua E. Hill, UL VS LLC. 38 | * Joshua E. Hill, KeyPair Consulting, Inc. 39 | */ 40 | 41 | #include "rbt_misc.h" 42 | #include 43 | #include // for stderr, NULL, fprintf 44 | #include // for exit 45 | 46 | /***********************************************************************/ 47 | /* FUNCTION: void Assert(int assertion, char* error) */ 48 | /**/ 49 | /* INPUTS: assertion should be a predicated that the programmer */ 50 | /* assumes to be true. If this assumption is not true the message */ 51 | /* error is printed and the program exits. */ 52 | /**/ 53 | /* OUTPUT: None. */ 54 | /**/ 55 | /* Modifies input: none */ 56 | /**/ 57 | /* Note: If DEBUG_ASSERT is not defined then assertions should not */ 58 | /* be in use as they will slow down the code. Therefore the */ 59 | /* compiler will complain if an assertion is used when */ 60 | /* DEBUG_ASSERT is undefined. */ 61 | /***********************************************************************/ 62 | 63 | void Assert(int assertion, char* error) { 64 | if (assertion == 0) { 65 | fprintf(stderr, "Assertion Failed: %s\n", error); 66 | exit(-1); 67 | } 68 | } 69 | 70 | /***********************************************************************/ 71 | /* FUNCTION: SafeMalloc */ 72 | /**/ 73 | /* INPUTS: size is the size to malloc */ 74 | /**/ 75 | /* OUTPUT: returns pointer to allocated memory if succesful */ 76 | /**/ 77 | /* EFFECT: mallocs new memory. If malloc fails, prints error message */ 78 | /* and terminates program. */ 79 | /**/ 80 | /* Modifies Input: none */ 81 | /**/ 82 | /***********************************************************************/ 83 | 84 | void* SafeMalloc(size_t size) { 85 | void* result; 86 | 87 | if ((result = malloc(size)) != NULL) { /* assignment intentional */ 88 | return (result); 89 | } else { 90 | fprintf(stderr, "memory overflow: malloc failed in SafeMalloc."); 91 | fprintf(stderr, " Exiting Program.\n"); 92 | exit(-1); 93 | } 94 | } 95 | /* NullFunction does nothing it is included so that it can be passed */ 96 | /* as a function to RBTreeCreate when no other suitable function has */ 97 | /* been defined */ 98 | 99 | void NullFunction(void* junk) { 100 | assert(junk != NULL); 101 | } 102 | -------------------------------------------------------------------------------- /src/u32-gcd.c: -------------------------------------------------------------------------------- 1 | /* This file is part of the Theseus distribution. 2 | * Copyright 2020-2024 Joshua E. Hill 3 | * 4 | * Licensed under the 3-clause BSD license. For details, see the LICENSE file. 5 | * 6 | * Author(s) 7 | * Joshua E. Hill, UL VS LLC. 8 | * Joshua E. Hill, KeyPair Consulting, Inc. 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "binio.h" 22 | #include "globals-inst.h" 23 | #include "precision.h" 24 | 25 | noreturn static void useageExit(void) { 26 | fprintf(stderr, "Usage:\n"); 27 | fprintf(stderr, "u32-gcd \n"); 28 | fprintf(stderr, "Find common divisors, and remove these factors.\n"); 29 | fprintf(stderr, "The values are output to stdout.\n"); 30 | exit(EX_USAGE); 31 | } 32 | 33 | static int uintcompare(const void *in1, const void *in2) { 34 | const uint32_t *left; 35 | const uint32_t *right; 36 | 37 | left = in1; 38 | right = in2; 39 | 40 | if (*left < *right) { 41 | return (-1); 42 | } else if (*left > *right) { 43 | return (1); 44 | } else { 45 | return (0); 46 | } 47 | } 48 | 49 | static uint32_t gcd(uint32_t a, uint32_t b) { 50 | // Make a greater than or equal. 51 | if (a < b) { 52 | uint32_t c = a; 53 | a = b; 54 | b = c; 55 | } 56 | 57 | while (b != 0) { 58 | uint32_t r; 59 | 60 | r = a % b; 61 | 62 | a = b; 63 | b = r; 64 | } 65 | 66 | return a; 67 | } 68 | 69 | int main(int argc, char *argv[]) { 70 | FILE *infp; 71 | size_t datalen; 72 | uint32_t *data = NULL; 73 | uint32_t *gcds = NULL; 74 | size_t i; 75 | uint32_t curfactor; 76 | 77 | if (argc != 2) { 78 | useageExit(); 79 | } 80 | 81 | if ((infp = fopen(argv[1], "rb")) == NULL) { 82 | perror("Can't open file"); 83 | exit(EX_NOINPUT); 84 | } 85 | 86 | datalen = readuint32file(infp, &data); 87 | assert(data != NULL); 88 | 89 | fprintf(stderr, "Read in %zu uint32_ts\n", datalen); 90 | if (fclose(infp) != 0) { 91 | perror("Can't close input file"); 92 | exit(EX_OSERR); 93 | } 94 | 95 | if ((gcds = malloc(sizeof(uint32_t) * datalen)) == NULL) { 96 | perror("Can't allocate gcd array"); 97 | exit(EX_OSERR); 98 | } 99 | 100 | if (datalen < 2) { 101 | fprintf(stderr, "Too little data\n"); 102 | exit(EX_DATAERR); 103 | } 104 | 105 | curfactor = data[0]; 106 | for (i = 1; i < datalen; i++) { 107 | gcds[i] = gcd(data[i - 1], data[i]); 108 | if (curfactor != 1) curfactor = gcd(data[i], curfactor); 109 | } 110 | 111 | if (curfactor > 1) { 112 | fprintf(stderr, "Found common divisor %u\n", curfactor); 113 | for (i = 0; i < datalen; i++) { 114 | data[i] /= curfactor; 115 | } 116 | } else { 117 | // Look for an almost common factor 118 | uint32_t mostCommonGCD = 0; 119 | uint32_t curGCD = 0; 120 | size_t curcount = 0; 121 | size_t maxcount = 0; 122 | double mostCommonGCDrate; 123 | 124 | qsort(gcds, datalen - 1, sizeof(uint32_t), uintcompare); 125 | for (i = 0; i < datalen - 1; i++) { 126 | if (gcds[i] == curGCD) { 127 | curcount++; 128 | } else { 129 | curcount = 1; 130 | curGCD = gcds[i]; 131 | } 132 | 133 | if (curcount >= maxcount) { 134 | mostCommonGCD = curGCD; 135 | maxcount = curcount; 136 | } 137 | } 138 | 139 | mostCommonGCDrate = ((double)maxcount) / ((double)(datalen - 1)); 140 | 141 | fprintf(stderr, "Most common gcd is %u (%g)\n", mostCommonGCD, mostCommonGCDrate); 142 | if ((mostCommonGCDrate > 0.99) && (mostCommonGCD > 1)) { 143 | fprintf(stderr, "Ignoring fuzz\n"); 144 | for (i = 0; i < datalen; i++) { 145 | data[i] /= mostCommonGCD; 146 | } 147 | } 148 | } 149 | 150 | if (fwrite(data, sizeof(uint32_t), datalen, stdout) != datalen) { 151 | if (data != NULL) { 152 | free(data); 153 | data = NULL; 154 | } 155 | if (gcds != NULL) { 156 | free(gcds); 157 | gcds = NULL; 158 | } 159 | 160 | perror("Can't write to stdout"); 161 | exit(EX_OSERR); 162 | } 163 | 164 | if (data != NULL) { 165 | free(data); 166 | data = NULL; 167 | } 168 | if (gcds != NULL) { 169 | free(gcds); 170 | gcds = NULL; 171 | } 172 | 173 | return (0); 174 | } 175 | --------------------------------------------------------------------------------