├── .clang-format ├── .github └── workflows │ ├── deb-build.yaml │ └── rpm-build.yaml ├── .gitignore ├── AUTHORS ├── COPYING ├── COPYING.LESSER ├── ChangeLog ├── INSTALL ├── Makefile.am ├── README ├── bootstrap.sh ├── configure.ac ├── debian ├── changelog ├── control ├── copyright ├── libwandio1-dev.dirs ├── libwandio1-dev.install ├── libwandio1.dirs ├── libwandio1.install ├── rules ├── source │ └── format ├── wandio1-tools.dirs ├── wandio1-tools.install └── wandio1-tools.manpages ├── debpkg-build.sh ├── debpkg-setup.sh ├── lib ├── Makefile.am ├── curl-helper.c ├── curl-helper.h ├── ior-bzip.c ├── ior-http.c ├── ior-lzma.c ├── ior-peek.c ├── ior-qat.c ├── ior-stdio.c ├── ior-swift.c ├── ior-thread.c ├── ior-zlib.c ├── ior-zstd-lz4.c ├── iow-bzip.c ├── iow-lz4.c ├── iow-lzma.c ├── iow-lzo.c ├── iow-qat.c ├── iow-stdio.c ├── iow-thread.c ├── iow-zlib.c ├── iow-zstd.c ├── swift-support │ ├── .dirstamp │ ├── jsmn.c │ ├── jsmn.h │ ├── jsmn_utils.c │ ├── jsmn_utils.h │ ├── keystone.c │ └── keystone.h ├── wandio.c ├── wandio.h └── wandio_internal.h ├── m4 ├── attributes.m4 └── visibility.m4 ├── rpm └── libwandio1.spec ├── rpmpkg-build.sh ├── rpmpkg-setup.sh ├── run-tests-rpm.sh ├── run-tests.sh ├── test ├── do-basic-tests.sh └── files │ ├── big.multistream.txt.bz2 │ ├── big.txt │ ├── big.txt.bz2 │ ├── big.txt.gz │ ├── big.txt.lz4 │ ├── big.txt.xz │ └── big.txt.zst └── tools └── wandiocat ├── Makefile.am ├── wandiocat.1 └── wcat.c /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | # BasedOnStyle: LLVM 4 | AccessModifierOffset: -2 5 | AlignAfterOpenBracket: Align 6 | AlignConsecutiveAssignments: false 7 | AlignConsecutiveDeclarations: false 8 | AlignEscapedNewlinesLeft: false 9 | AlignOperands: true 10 | AlignTrailingComments: true 11 | AllowAllParametersOfDeclarationOnNextLine: true 12 | AllowShortBlocksOnASingleLine: false 13 | AllowShortCaseLabelsOnASingleLine: false 14 | AllowShortFunctionsOnASingleLine: None 15 | AllowShortIfStatementsOnASingleLine: false 16 | AllowShortLoopsOnASingleLine: false 17 | AlwaysBreakAfterDefinitionReturnType: None 18 | AlwaysBreakAfterReturnType: None 19 | AlwaysBreakBeforeMultilineStrings: false 20 | AlwaysBreakTemplateDeclarations: false 21 | BinPackArguments: true 22 | BinPackParameters: true 23 | BraceWrapping: 24 | AfterClass: false 25 | AfterControlStatement: false 26 | AfterEnum: false 27 | AfterFunction: false 28 | AfterNamespace: false 29 | AfterObjCDeclaration: false 30 | AfterStruct: false 31 | AfterUnion: false 32 | BeforeCatch: false 33 | BeforeElse: false 34 | IndentBraces: false 35 | BreakBeforeBinaryOperators: None 36 | BreakBeforeBraces: Custom 37 | BreakBeforeTernaryOperators: true 38 | BreakConstructorInitializersBeforeComma: false 39 | BreakAfterJavaFieldAnnotations: false 40 | BreakStringLiterals: true 41 | ColumnLimit: 80 42 | CommentPragmas: '^ IWYU pragma:' 43 | ConstructorInitializerAllOnOneLineOrOnePerLine: false 44 | ConstructorInitializerIndentWidth: 4 45 | ContinuationIndentWidth: 4 46 | Cpp11BracedListStyle: true 47 | DerivePointerAlignment: false 48 | DisableFormat: false 49 | ExperimentalAutoDetectBinPacking: false 50 | ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] 51 | IncludeCategories: 52 | - Regex: '^"config' 53 | Priority: -1 54 | - Regex: '^<' 55 | Priority: 4 56 | - Regex: '^"' 57 | Priority: 5 58 | IndentCaseLabels: false 59 | IndentWidth: 8 60 | IndentWrappedFunctionNames: false 61 | KeepEmptyLinesAtTheStartOfBlocks: true 62 | MacroBlockBegin: '' 63 | MacroBlockEnd: '' 64 | MaxEmptyLinesToKeep: 1 65 | NamespaceIndentation: None 66 | ObjCBlockIndentWidth: 2 67 | ObjCSpaceAfterProperty: false 68 | ObjCSpaceBeforeProtocolList: true 69 | PenaltyBreakBeforeFirstCallParameter: 19 70 | PenaltyBreakComment: 300 71 | PenaltyBreakFirstLessLess: 120 72 | PenaltyBreakString: 1000 73 | PenaltyExcessCharacter: 1000000 74 | PenaltyReturnTypeOnItsOwnLine: 60 75 | PointerAlignment: Right 76 | ReflowComments: true 77 | SortIncludes: true 78 | SpaceAfterCStyleCast: false 79 | SpaceBeforeAssignmentOperators: true 80 | SpaceBeforeParens: ControlStatements 81 | SpaceInEmptyParentheses: false 82 | SpacesBeforeTrailingComments: 2 83 | SpacesInAngles: false 84 | SpacesInContainerLiterals: true 85 | SpacesInCStyleCastParentheses: false 86 | SpacesInParentheses: false 87 | SpacesInSquareBrackets: false 88 | Standard: Auto 89 | TabWidth: 8 90 | UseTab: Never 91 | JavaScriptQuotes: Leave 92 | ... 93 | -------------------------------------------------------------------------------- /.github/workflows/deb-build.yaml: -------------------------------------------------------------------------------- 1 | name: Packaging for Debian and Ubuntu 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | container: 12 | image: ${{ matrix.target }} 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | arch: 17 | - amd64 18 | target: 19 | - "debian:buster" 20 | - "debian:bookworm" 21 | - "debian:bullseye" 22 | - "ubuntu:jammy" 23 | - "ubuntu:noble" 24 | - "ubuntu:focal" 25 | 26 | steps: 27 | - name: Checkout repo 28 | uses: actions/checkout@v4 29 | - name: Install prereq packages 30 | run: ./debpkg-setup.sh 31 | - name: Build packages 32 | run: ./debpkg-build.sh 33 | - name: Store packages 34 | run: | 35 | export DIRNAME=`echo ${{ matrix.target }} | tr ':' '_'` 36 | mkdir -p packages/${DIRNAME} 37 | cp ../*.deb packages/${DIRNAME}/ 38 | - name: Run tests 39 | run: | 40 | export DIRNAME=`echo ${{ matrix.target }} | tr ':' '_'` 41 | ./run-tests.sh ${DIRNAME} 42 | - name: Publish packages to cloudsmith 43 | uses: wandnz/action-cloudsmith-upload-packages@v1.8 44 | with: 45 | path: packages/ 46 | repo: ${{ secrets.CLOUDSMITH_OWNER }}/libwandio 47 | username: salcock 48 | api_key: ${{ secrets.CLOUDSMITH_API_KEY }} 49 | -------------------------------------------------------------------------------- /.github/workflows/rpm-build.yaml: -------------------------------------------------------------------------------- 1 | name: Packaging for Centos and Fedora 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | container: 12 | image: ${{ matrix.target }} 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | arch: 17 | - amd64 18 | target: 19 | - "fedora:39" 20 | - "fedora:40" 21 | - "rockylinux:8" 22 | - "rockylinux:9" 23 | 24 | 25 | steps: 26 | - name: Checkout repo 27 | uses: actions/checkout@v4 28 | - name: Install prereq packages 29 | run: ./rpmpkg-setup.sh ${{ matrix.target }} 30 | - name: Build packages 31 | run: ./rpmpkg-build.sh ${{ matrix.target }} 32 | - name: Store packages 33 | run: | 34 | export DIRNAME=`echo ${{ matrix.target }} | tr ':' '_'` 35 | mkdir -p packages/${DIRNAME} 36 | cp ~/rpmbuild/RPMS/x86_64/*.rpm packages/${DIRNAME}/ 37 | - name: Run tests 38 | run: | 39 | export DIRNAME=`echo ${{ matrix.target }} | tr ':' '_'` 40 | ./run-tests-rpm.sh ${DIRNAME} 41 | - name: Publish packages to cloudsmith 42 | uses: salcock/action-cloudsmith-upload-packages@v1.8 43 | with: 44 | path: packages/ 45 | repo: ${{ secrets.CLOUDSMITH_OWNER }}/libwandio 46 | username: salcock 47 | api_key: ${{ secrets.CLOUDSMITH_API_KEY }} 48 | 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.la 2 | *.o 3 | *.lo 4 | *Makefile.in 5 | *Makefile 6 | *.deps 7 | *.libs 8 | *.swp 9 | config\.* 10 | configure 11 | compile 12 | autom4te.cache/ 13 | depcomp 14 | install-sh 15 | aclocal.m4 16 | libtool 17 | ltmain.sh 18 | missing 19 | stamp-h1 20 | m4/*.m4 21 | tools/wandiocat/wandiocat 22 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | The original authors of libwandio were: 2 | * Perry Lorier 3 | * Shane Alcock 4 | 5 | Libwandio is currently maintained by Shane Alcock . 6 | 7 | Thanks to Alistair King for numerous contributions (including the entirety 8 | of the HTTP reader), bug reports and being kind enough to take on the job of 9 | separating libwandio from libtrace. 10 | 11 | Thanks to Robert Zeh for adding support for zstd. 12 | 13 | Thanks to Sergey Cherepanov for adding support for lz4. 14 | 15 | Thanks to Mingwei Zhang for adding ability to resume HTTP reads and fixing 16 | a ton of lazy error checking issues. 17 | 18 | Thanks to Derrick Lyndon Pallas for fixing an uninitialised memory error 19 | when closing a wandio writer. 20 | 21 | Thanks to Brad Cowie for packaging libwandio for Debian. 22 | -------------------------------------------------------------------------------- /COPYING.LESSER: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | 9 | This version of the GNU Lesser General Public License incorporates 10 | the terms and conditions of version 3 of the GNU General Public 11 | License, supplemented by the additional permissions listed below. 12 | 13 | 0. Additional Definitions. 14 | 15 | As used herein, "this License" refers to version 3 of the GNU Lesser 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 17 | General Public License. 18 | 19 | "The Library" refers to a covered work governed by this License, 20 | other than an Application or a Combined Work as defined below. 21 | 22 | An "Application" is any work that makes use of an interface provided 23 | by the Library, but which is not otherwise based on the Library. 24 | Defining a subclass of a class defined by the Library is deemed a mode 25 | of using an interface provided by the Library. 26 | 27 | A "Combined Work" is a work produced by combining or linking an 28 | Application with the Library. The particular version of the Library 29 | with which the Combined Work was made is also called the "Linked 30 | Version". 31 | 32 | The "Minimal Corresponding Source" for a Combined Work means the 33 | Corresponding Source for the Combined Work, excluding any source code 34 | for portions of the Combined Work that, considered in isolation, are 35 | based on the Application, and not on the Linked Version. 36 | 37 | The "Corresponding Application Code" for a Combined Work means the 38 | object code and/or source code for the Application, including any data 39 | and utility programs needed for reproducing the Combined Work from the 40 | Application, but excluding the System Libraries of the Combined Work. 41 | 42 | 1. Exception to Section 3 of the GNU GPL. 43 | 44 | You may convey a covered work under sections 3 and 4 of this License 45 | without being bound by section 3 of the GNU GPL. 46 | 47 | 2. Conveying Modified Versions. 48 | 49 | If you modify a copy of the Library, and, in your modifications, a 50 | facility refers to a function or data to be supplied by an Application 51 | that uses the facility (other than as an argument passed when the 52 | facility is invoked), then you may convey a copy of the modified 53 | version: 54 | 55 | a) under this License, provided that you make a good faith effort to 56 | ensure that, in the event an Application does not supply the 57 | function or data, the facility still operates, and performs 58 | whatever part of its purpose remains meaningful, or 59 | 60 | b) under the GNU GPL, with none of the additional permissions of 61 | this License applicable to that copy. 62 | 63 | 3. Object Code Incorporating Material from Library Header Files. 64 | 65 | The object code form of an Application may incorporate material from 66 | a header file that is part of the Library. You may convey such object 67 | code under terms of your choice, provided that, if the incorporated 68 | material is not limited to numerical parameters, data structure 69 | layouts and accessors, or small macros, inline functions and templates 70 | (ten or fewer lines in length), you do both of the following: 71 | 72 | a) Give prominent notice with each copy of the object code that the 73 | Library is used in it and that the Library and its use are 74 | covered by this License. 75 | 76 | b) Accompany the object code with a copy of the GNU GPL and this license 77 | document. 78 | 79 | 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, 82 | taken together, effectively do not restrict modification of the 83 | portions of the Library contained in the Combined Work and reverse 84 | engineering for debugging such modifications, if you also do each of 85 | the following: 86 | 87 | a) Give prominent notice with each copy of the Combined Work that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 92 | document. 93 | 94 | c) For a Combined Work that displays copyright notices during 95 | execution, include the copyright notice for the Library among 96 | these notices, as well as a reference directing the user to the 97 | copies of the GNU GPL and this license document. 98 | 99 | d) Do one of the following: 100 | 101 | 0) Convey the Minimal Corresponding Source under the terms of this 102 | License, and the Corresponding Application Code in a form 103 | suitable for, and under terms that permit, the user to 104 | recombine or relink the Application with a modified version of 105 | the Linked Version to produce a modified Combined Work, in the 106 | manner specified by section 6 of the GNU GPL for conveying 107 | Corresponding Source. 108 | 109 | 1) Use a suitable shared library mechanism for linking with the 110 | Library. A suitable mechanism is one that (a) uses at run time 111 | a copy of the Library already present on the user's computer 112 | system, and (b) will operate properly with a modified version 113 | of the Library that is interface-compatible with the Linked 114 | Version. 115 | 116 | e) Provide Installation Information, but only if you would otherwise 117 | be required to provide such information under section 6 of the 118 | GNU GPL, and only to the extent that such information is 119 | necessary to install and execute a modified version of the 120 | Combined Work produced by recombining or relinking the 121 | Application with a modified version of the Linked Version. (If 122 | you use option 4d0, the Installation Information must accompany 123 | the Minimal Corresponding Source and Corresponding Application 124 | Code. If you use option 4d1, you must provide the Installation 125 | Information in the manner specified by section 6 of the GNU GPL 126 | for conveying Corresponding Source.) 127 | 128 | 5. Combined Libraries. 129 | 130 | You may place library facilities that are a work based on the 131 | Library side by side in a single library together with other library 132 | facilities that are not Applications and are not covered by this 133 | License, and convey such a combined library under terms of your 134 | choice, if you do both of the following: 135 | 136 | a) Accompany the combined library with a copy of the same work based 137 | on the Library, uncombined with any other library facilities, 138 | conveyed under the terms of this License. 139 | 140 | b) Give prominent notice with the combined library that part of it 141 | is a work based on the Library, and explaining where to find the 142 | accompanying uncombined form of the same work. 143 | 144 | 6. Revised Versions of the GNU Lesser General Public License. 145 | 146 | The Free Software Foundation may publish revised and/or new versions 147 | of the GNU Lesser General Public License from time to time. Such new 148 | versions will be similar in spirit to the present version, but may 149 | differ in detail to address new problems or concerns. 150 | 151 | Each version is given a distinguishing version number. If the 152 | Library as you received it specifies that a certain numbered version 153 | of the GNU Lesser General Public License "or any later version" 154 | applies to it, you have the option of following the terms and 155 | conditions either of that published version or of any later version 156 | published by the Free Software Foundation. If the Library as you 157 | received it does not specify a version number of the GNU Lesser 158 | General Public License, you may choose any version of the GNU Lesser 159 | General Public License ever published by the Free Software Foundation. 160 | 161 | If the Library as you received it specifies that a proxy can decide 162 | whether future versions of the GNU Lesser General Public License shall 163 | apply, that proxy's public statement of acceptance of any version is 164 | permanent authorization for you to choose that version for the 165 | Library. 166 | 167 | -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- 1 | Version 4.2.6 2 | ============= 3 | * Fix truncation bug when reading multi-stream bzip2 files. 4 | 5 | 6 | Version 4.2.5 7 | ============= 8 | * HTTP: disconnect readers that are idle (or very slow) for 9 | 60 seconds to avoid hanging waiting for data that is unlikely to 10 | ever arrive. 11 | 12 | Version 4.2.4 13 | ============= 14 | * Debian packaging has been modernised (thanks to Perry Lorier) 15 | * Fix curl handle leaking when re-attempting an HTTP request 16 | * Fix bug in compression method detection based on the suffix of 17 | an output filename 18 | * Replace various outdated macros in configure scripts 19 | 20 | Version 4.2.3 21 | ============= 22 | Thanks to Mingwei Zhang for some of the fixes in this release 23 | 24 | * Fix segfault when wandio_wflush() is called with a NULL IO handle 25 | * HTTP: return approriate error code when curl fails an HTTP request 26 | * HTTP: use TCP keep-alives to prevent long-running sessions from hanging. 27 | 28 | Version 4.2.2 29 | ============= 30 | Thanks to Alistair King and Mingwei Zhang for the fixes in this release 31 | 32 | * Fixed incorrect error bug in HTTP reader when using libcurl < 7.20.0 33 | * Fixed bad scoping of Swift Keystone auth requests 34 | * Allow users to override Swift storage URLs via 'OS_STORAGE_URL' 35 | 36 | Version 4.2.1 37 | ============= 38 | * Fixed crashes / wacky behaviour when writing lz4 compressed output. 39 | 40 | Version 4.2.0 41 | ============= 42 | * Include wandio version number in the HTTP user-agent 43 | * Improved error detection and handling when reading HTTP 44 | * Replaced 'off_t' parameters and return values with 'int64_t' in 45 | recently added API functions. 46 | * Fixed potential uninitialised memory error when closing a wandio writer. 47 | * Export symbols for all format-specific open functions. 48 | 49 | Version 4.1.2 50 | ============= 51 | * Fix buffer overflow bug in the swift reading code (thanks Alistair). 52 | 53 | Version 4.1.1 54 | ============= 55 | * Fix bad version numbering of installed libwandder.so 56 | 57 | Version 4.1.0 58 | ============= 59 | Thanks to Sergey Cherepanov and Alistair King for contributing to this release 60 | 61 | * Added support for both reading and writing lz4 compressed files. 62 | * Added support for using Intel QuickAssist (if present) for reading 63 | and writing gzip compressed files. 64 | * Added new functionality for interacting with wandio files 65 | (wandio_fgets(), wandio_vprintf(), wandio_printf()) 66 | * Added API function for determining correct compression method 67 | based on output filename extension (wandio_detect_compression_type()) 68 | * Some potential performance improvements by aligning buffers sensibly. 69 | * Fixed "inline" errors when building with LLVM. 70 | 71 | Version 4.0.0 72 | ============= 73 | Thanks to Alistair King, Mingwei Zhang and Robert Zeh for their contributions 74 | 75 | * Added support for reading and writing zstd. 76 | * HTTP reads can now be resumed from where they left off. 77 | * Added wandio_wflush API for flushing writer output without closing. 78 | - implemented for zlib, thread and stdio 79 | * wandiocat: fixed crash when the compression method is invalid 80 | * Fixed various documentation errors and bad error checks. 81 | * Re-versioned to avoid versioning issues with old packages from our libtrace 82 | days. 83 | 84 | 85 | Version 1.0.4 86 | ============= 87 | 88 | Full credit to Alistair King for the bug fixes in this release 89 | 90 | * Switched from GPL to LGPL license 91 | * Fixed nasty segfaults when running on a 32 bit system (hopefully for good!) 92 | * Fixed another memory error when calling read after an error. 93 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | Requirements: 2 | * >=automake-1.9 3 | 4 | 5 | Strongly recommended: 6 | * libpthread 7 | * zlib-dev 8 | * libbz2-dev 9 | * liblzma-dev 10 | * liblzo2-dev 11 | * liblz4-dev 12 | * libzstd-dev 13 | 14 | ---------------------------------- 15 | 16 | Installing WANDIO: 17 | 18 | ./bootstrap.sh (only if you've cloned the source from GitHub) 19 | ./configure 20 | make 21 | make install 22 | 23 | The above series of commands will install wandio into /usr/local/lib. If 24 | you wish to install to a non-standard location, append the 25 | --prefix=DIR option to the ./configure command. ./configure also takes 26 | a number of other options - run ./configure --help to view a comprehensive 27 | list. 28 | 29 | You may need to add the library location (e.g. /usr/local/lib) to your 30 | /etc/ld.so.conf and run 'ldconfig' as root. 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = lib tools/wandiocat 2 | 3 | ACLOCAL_AMFLAGS = -I m4 4 | AUTOMAKE_OPTIONS = 1.9 foreign 5 | 6 | .PHONY: docs 7 | 8 | bootstrap-clean: 9 | $(RM) -f Makefile.in aclocal.m4 compile config.* \ 10 | configure depcomp install-sh ltmain.sh missing \ 11 | mkinstalldirs stamp-h.in \ 12 | lib/Makefile.in replace/libreplace.la replace/Makefile.in 13 | $(RM) -rf autom4te.cache/ 14 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | WANDIO 4.2.6 2 | 3 | --------------------------------------------------------------------------- 4 | Copyright (c) 2007-2022 The University of Waikato, Hamilton, New Zealand. 5 | All rights reserved. 6 | 7 | This code was originally developed by the University of Waikato WAND 8 | research group. For further information please see http://www.wand.net.nz/. 9 | --------------------------------------------------------------------------- 10 | 11 | See INSTALL for instructions on how to install WANDIO. 12 | 13 | This directory contains source code for WANDIO, a library for reading from, and 14 | writing to, files. Depending on libraries available at compile time, WANDIO 15 | provides transparent compression/decompression for the following formats: 16 | - zlib (gzip) 17 | - bzip2 18 | - lzo (write-only) 19 | - lzma 20 | - zstd 21 | - lz4 22 | - Intel QAT (write-only) 23 | - http (read-only) 24 | 25 | WANDIO also improves IO performance by performing compression/decompression in a 26 | separate thread (if pthreads are available). 27 | 28 | Documentation for WANDIO and its included tools can be found at 29 | https://github.com/LibtraceTeam/wandio/wiki 30 | 31 | Bugs should be reported by either emailing contact@wand.net.nz or filing 32 | an issue at https://github.com/LibtraceTeam/wandio 33 | 34 | It is licensed under the Lesser GNU General Public License (LGPL) version 3. 35 | Please see the included files COPYING and COPYING.LESSER for details of this 36 | license. 37 | -------------------------------------------------------------------------------- /bootstrap.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | set +x 4 | # Prefer aclocal 1.9 if we can find it 5 | (which aclocal-1.11 &> /dev/null && aclocal-1.11 -I m4) || 6 | (which aclocal-1.9 &>/dev/null && aclocal-1.9 -I m4) || 7 | aclocal -I m4 8 | 9 | # Darwin bizarrely uses glibtoolize 10 | (which libtoolize &>/dev/null && libtoolize --force --copy) || 11 | glibtoolize --force --copy 12 | 13 | (which autoheader2.50 &>/dev/null && autoheader2.50) || 14 | autoheader 15 | 16 | # Prefer automake-1.9 if we can find it 17 | (which automake-1.11 &>/dev/null && automake-1.11 --add-missing --copy --foreign) || 18 | (which automake-1.10 &>/dev/null && automake-1.10 --add-missing --copy --foreign) || 19 | (which automake-1.9 &>/dev/null && automake-1.9 --add-missing --copy --foreign) || 20 | automake --add-missing --copy --foreign 21 | 22 | (which autoconf2.50 &>/dev/null && autoconf2.50) || 23 | autoconf 24 | -------------------------------------------------------------------------------- /debian/changelog: -------------------------------------------------------------------------------- 1 | libwandio1 (4.2.6-1) unstable; urgency=medium 2 | 3 | * Fix truncation bug when reading multi-stream bzip2 files. 4 | 5 | -- Shane Alcock Tue, 19 Sep 2023 11:44:19 +1200 6 | 7 | libwandio1 (4.2.5-1) unstable; urgency=medium 8 | 9 | * HTTP: disconnect the reader from server and return an error 10 | if (next to) no data is received for a minute. 11 | 12 | -- Shane Alcock Tue, 13 Jun 2023 16:19:40 +1300 13 | 14 | libwandio1 (4.2.4-1) unstable; urgency=medium 15 | 16 | * writing: fix bugs with compression method detection based on file 17 | suffix. 18 | * fix leaking of curl handles when an HTTP download is reset by the 19 | remote peer. 20 | * Trim trailing whitespace. 21 | * debian/rules: Use dh_prep rather than "dh_clean -k". 22 | * Use secure URI in Homepage field. 23 | * Bump debhelper from deprecated 9 to 12. 24 | * Set debhelper-compat version in Build-Depends. 25 | * Remove Section on wandio1-tools, Section on libwandio1 that duplicate 26 | source. 27 | 28 | -- Shane Alcock Fri, 21 Oct 2022 09:50:18 +1300 29 | 30 | libwandio1 (4.2.3-1) unstable; urgency=medium 31 | 32 | * Fix segfault when wandio_wflush() is called with a NULL IO handle 33 | * HTTP: return approriate error code when curl fails an HTTP request 34 | * HTTP: use TCP keep-alives to prevent long-running sessions from 35 | hanging. 36 | 37 | -- Shane Alcock Thu, 14 May 2020 17:31:17 +1200 38 | 39 | libwandio1 (4.2.2-1) unstable; urgency=medium 40 | 41 | * Fixed incorrect error bug in HTTP reader when using libcurl < 7.20.0 42 | * Fixed bad scoping of Swift Keystone auth requests 43 | * Allow users to override Swift storage URLs via 'OS_STORAGE_URL' 44 | 45 | -- Shane Alcock Mon, 22 Jul 2019 11:07:50 +1200 46 | 47 | libwandio1 (4.2.1-1) unstable; urgency=medium 48 | 49 | * Fixed bug in lz4 compression that was causing crashes and other 50 | unpredictable behaviour. 51 | 52 | -- Shane Alcock Thu, 16 May 2019 16:57:06 +1200 53 | 54 | libwandio1 (4.2.0-1) unstable; urgency=medium 55 | 56 | * Include wandio version number in the HTTP user-agent 57 | * Improved error detection and handling when reading HTTP 58 | * Replaced 'off_t' parameters and return values with 'int64_t' in 59 | recently added API functions. 60 | * Fixed potential uninitialised memory error when closing a wandio writer. 61 | * Export symbols for all format-specific 'open' functions. 62 | 63 | -- Shane Alcock Fri, 10 May 2019 13:31:49 +1200 64 | 65 | libwandio1 (4.1.2-1) unstable; urgency=medium 66 | 67 | * Fix swift buffer overflow bug 68 | 69 | -- Shane Alcock Wed, 06 Mar 2019 13:07:02 +1300 70 | 71 | libwandio1 (4.1.1-1) unstable; urgency=medium 72 | 73 | * Fix bad version number for installed .so 74 | 75 | -- Shane Alcock Fri, 01 Mar 2019 11:53:31 +1300 76 | 77 | libwandio1 (4.1.0-1) unstable; urgency=medium 78 | 79 | * Added support for reading and writing lz4 80 | * Added support for using QuickAssist to do gzip (de)compression 81 | * Added new functionality for interacting with wandio files 82 | (wandio_fgets(), wandio_vprintf(), wandio_printf()) 83 | * Added API function for determining correct compression method 84 | based on output filename extension 85 | 86 | -- Shane Alcock Wed, 13 Feb 2019 15:11:15 +1300 87 | 88 | libwandio1 (4.0.0-1) unstable; urgency=medium 89 | 90 | * New upstream release 91 | 92 | -- Brad Cowie Tue, 06 Sep 2016 13:50:40 +1300 93 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: libwandio1 2 | Priority: optional 3 | Maintainer: Shane Alcock 4 | Build-Depends: debhelper-compat (= 12), 5 | dh-autoreconf, 6 | zlib1g-dev, liblzma-dev, libbz2-dev, 7 | liblzo2-dev, libcurl4-openssl-dev, liblz4-dev (>= 0.0~r131), 8 | libzstd1-dev | libzstd-dev (>= 1) 9 | Standards-Version: 4.1.3 10 | Section: libs 11 | Homepage: https://github.com/LibtraceTeam/wandio 12 | 13 | Package: wandio1-tools 14 | Architecture: any 15 | Depends: ${shlibs:Depends}, ${misc:Depends} 16 | Provides: wandio-tools 17 | Conflicts: wandio-tools 18 | Description: example tools for the libwandio library 19 | This package contains some example tools to demonstrate the libwandio library. 20 | . 21 | libwandio is a file I/O library that will read and write both compressed and 22 | uncompressed files. All compression-related operations are performed in a 23 | separate thread where possible resulting in significant performance gains for 24 | tasks where I/O is the limiting factor (most simple trace analysis tasks are 25 | I/O-limited). 26 | . 27 | libwandio was originally developed by the WAND Network Research Group at 28 | Waikato University, New Zealand. 29 | 30 | Package: libwandio1 31 | Architecture: any 32 | Multi-Arch: same 33 | Depends: ${shlibs:Depends}, ${misc:Depends} 34 | Description: multi-threaded file compression and decompression library 35 | File I/O library that will read and write both compressed and uncompressed 36 | files. All compression-related operations are performed in a separate thread 37 | where possible resulting in significant performance gains for tasks where I/O 38 | is the limiting factor (most simple trace analysis tasks are I/O-limited). 39 | . 40 | libwandio was originally developed by the WAND Network Research Group at 41 | Waikato University, New Zealand. 42 | 43 | Package: libwandio1-dev 44 | Section: libdevel 45 | Architecture: any 46 | Depends: libwandio1 (= ${binary:Version}), ${misc:Depends}, 47 | ${shlibs:Depends} 48 | Provides: libwandio-dev 49 | Conflicts: libwandio-dev 50 | Description: development headers for the libwandio library 51 | This package contains development headers and other ancillary files for 52 | the libwandio library. 53 | . 54 | libwandio is a file I/O library that will read and write both compressed and 55 | uncompressed files. All compression-related operations are performed in a 56 | separate thread where possible resulting in significant performance gains for 57 | tasks where I/O is the limiting factor (most simple trace analysis tasks are 58 | I/O-limited). 59 | . 60 | libwandio was originally developed by the WAND Network Research Group at 61 | Waikato University, New Zealand. 62 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: Libwandio 3 | Source: http://research.wand.net.nz/software/libwandio.php 4 | 5 | Files: * 6 | Copyright: 2018 The University of Waikato, Hamilton, New Zealand. 7 | License: LGPL-3 8 | This package is free software; you can redistribute it and/or 9 | modify it under the terms of the GNU Lesser General Public 10 | License as published by the Free Software Foundation; either 11 | version 3 of the License, or (at your option) 12 | any later version. 13 | . 14 | This package is distributed in the hope that it will be useful, 15 | but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 17 | Lesser General Public License for more details. 18 | . 19 | You should have received a copy of the GNU Lesser General Public 20 | License along with this package; if not, write to the Free Software 21 | Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 22 | . 23 | On Debian systems, the complete text of the GNU Lesser General 24 | Public License can be found in `/usr/share/common-licenses/LGPL-3'. 25 | -------------------------------------------------------------------------------- /debian/libwandio1-dev.dirs: -------------------------------------------------------------------------------- 1 | usr/lib 2 | usr/include 3 | -------------------------------------------------------------------------------- /debian/libwandio1-dev.install: -------------------------------------------------------------------------------- 1 | usr/include/wandio.h 2 | usr/lib/*/libwandio.so 3 | -------------------------------------------------------------------------------- /debian/libwandio1.dirs: -------------------------------------------------------------------------------- 1 | usr/lib 2 | -------------------------------------------------------------------------------- /debian/libwandio1.install: -------------------------------------------------------------------------------- 1 | usr/lib/*/lib*.so.* 2 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | # Uncomment this to turn on verbose mode. 4 | #export DH_VERBOSE=1 5 | 6 | # Enable all hardening features, since traces are untrusted input. 7 | export DEB_BUILD_MAINT_OPTIONS = hardening=+all 8 | DPKG_EXPORT_BUILDFLAGS = 1 9 | include /usr/share/dpkg/buildflags.mk 10 | 11 | DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH) 12 | 13 | # These are used for cross-compiling and for saving the configure script 14 | # from having to guess our platform (since we know it already) 15 | DEB_HOST_GNU_TYPE ?= $(shell dpkg-architecture -qDEB_HOST_GNU_TYPE) 16 | DEB_BUILD_GNU_TYPE ?= $(shell dpkg-architecture -qDEB_BUILD_GNU_TYPE) 17 | CONFFLAGS = 18 | ifeq ($(DEB_BUILD_GNU_TYPE), $(DEB_HOST_GNU_TYPE)) 19 | CONFFLAGS += --build $(DEB_HOST_GNU_TYPE) 20 | else 21 | CONFFLAGS += --build $(DEB_BUILD_GNU_TYPE) --host $(DEB_HOST_GNU_TYPE) 22 | endif 23 | 24 | 25 | configure: 26 | 27 | configure-stamp: configure 28 | dh_testdir 29 | 30 | dh_autoreconf 31 | 32 | ./configure $(CONFFLAGS) \ 33 | --prefix=/usr \ 34 | --mandir=\$${prefix}/share/man \ 35 | --infodir=\$${prefix}/share/info \ 36 | --libdir=\$${prefix}/lib/${DEB_HOST_MULTIARCH} 37 | 38 | touch configure-stamp 39 | 40 | build: build-arch build-indep 41 | build-arch: build-stamp 42 | build-indep: build-stamp 43 | build-stamp: configure-stamp 44 | dh_testdir 45 | 46 | # Add here commands to compile the package. 47 | $(MAKE) 48 | 49 | touch build-stamp 50 | 51 | clean: 52 | dh_testdir 53 | dh_testroot 54 | rm -f build-stamp configure-stamp 55 | 56 | [ ! -f Makefile ] || $(MAKE) clean 57 | [ ! -f Makefile ] || $(MAKE) distclean 58 | 59 | dh_autoreconf_clean 60 | dh_clean 61 | 62 | install: build 63 | dh_testdir 64 | dh_testroot 65 | dh_prep 66 | dh_installdirs 67 | 68 | # Add here commands to install the package into debian/tmp 69 | $(MAKE) install DESTDIR=$(CURDIR)/debian/tmp 70 | 71 | 72 | # Build architecture-independent files here. 73 | binary-indep: build install 74 | # We have nothing to do by default. 75 | 76 | # Build architecture-dependent files here. 77 | binary-arch: build install 78 | dh_testdir 79 | dh_testroot 80 | #dh_installchangelogs ChangeLog 81 | dh_installchangelogs 82 | dh_installdocs 83 | dh_installexamples 84 | dh_installman 85 | dh_install --sourcedir=debian/tmp 86 | dh_link 87 | dh_strip 88 | dh_compress 89 | dh_fixperms 90 | dh_makeshlibs 91 | dh_installdeb 92 | dh_shlibdeps 93 | dh_gencontrol 94 | dh_md5sums 95 | dh_builddeb 96 | 97 | binary: binary-indep binary-arch 98 | .PHONY: build clean binary-indep binary-arch binary install 99 | -------------------------------------------------------------------------------- /debian/source/format: -------------------------------------------------------------------------------- 1 | 3.0 (quilt) 2 | -------------------------------------------------------------------------------- /debian/wandio1-tools.dirs: -------------------------------------------------------------------------------- 1 | usr/bin 2 | -------------------------------------------------------------------------------- /debian/wandio1-tools.install: -------------------------------------------------------------------------------- 1 | usr/bin/wandiocat 2 | -------------------------------------------------------------------------------- /debian/wandio1-tools.manpages: -------------------------------------------------------------------------------- 1 | tools/wandiocat/wandiocat.1 2 | -------------------------------------------------------------------------------- /debpkg-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x -e -o pipefail 4 | 5 | export DEBEMAIL='packaging@wand.net.nz' 6 | export DEBFULLNAME='WAND Packaging' 7 | export DEBIAN_FRONTEND=noninteractive 8 | 9 | export SOURCENAME=`echo ${GITHUB_REF##*/} | cut -d '-' -f 1` 10 | 11 | mk-build-deps -i -r -t 'apt-get -f -y --force-yes' 12 | dpkg-buildpackage -b -us -uc -rfakeroot -j4 13 | -------------------------------------------------------------------------------- /debpkg-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x -e -o pipefail 4 | 5 | export DEBEMAIL='packaging@wand.net.nz' 6 | export DEBFULLNAME='WAND Packaging' 7 | export DEBIAN_FRONTEND=noninteractive 8 | 9 | export SOURCENAME=`echo ${GITHUB_REF##*/} | cut -d '-' -f 1` 10 | 11 | 12 | apt-get update 13 | apt-get install -y equivs devscripts dpkg-dev quilt curl apt-transport-https \ 14 | apt-utils ssl-cert ca-certificates gnupg lsb-release debhelper git \ 15 | pkg-config 16 | 17 | curl -1sLf 'https://dl.cloudsmith.io/public/wand/libwandio/cfg/setup/bash.deb.sh' | bash 18 | 19 | apt-get update 20 | apt-get upgrade -y 21 | 22 | LSB=`lsb_release -cs` 23 | if [ "${LSB}" == "bionic" ]; then 24 | apt install -y debhelper/${LSB}-backports 25 | fi 26 | 27 | 28 | -------------------------------------------------------------------------------- /lib/Makefile.am: -------------------------------------------------------------------------------- 1 | lib_LTLIBRARIES=libwandio.la 2 | 3 | include_HEADERS=wandio.h 4 | 5 | AM_CFLAGS=@LIBCFLAGS@ @CFLAG_VISIBILITY@ 6 | AM_CXXFLAGS=@LIBCXXFLAGS@ @CFLAG_VISIBILITY@ 7 | 8 | if HAVE_ZLIB 9 | LIBTRACEIO_ZLIB=ior-zlib.c iow-zlib.c 10 | else 11 | LIBTRACEIO_ZLIB= 12 | endif 13 | 14 | if HAVE_BZLIB 15 | LIBTRACEIO_BZLIB=ior-bzip.c iow-bzip.c 16 | else 17 | LIBTRACEIO_BZLIB= 18 | endif 19 | 20 | if HAVE_LZO 21 | LIBTRACEIO_LZO=iow-lzo.c 22 | else 23 | LIBTRACEIO_LZO= 24 | endif 25 | 26 | if HAVE_LZMA 27 | LIBTRACEIO_LZMA=ior-lzma.c iow-lzma.c 28 | else 29 | LIBTRACEIO_LZMA= 30 | endif 31 | 32 | if HAVE_ZSTD 33 | LIBTRACEIO_ZSTD=iow-zstd.c 34 | else 35 | LIBTRACEIO_ZSTD= 36 | endif 37 | 38 | if HAVE_LZ4 39 | LIBTRACEIO_LZ4=iow-lz4.c 40 | else 41 | LIBTRACEIO_LZ4= 42 | endif 43 | 44 | if HAVE_ZSTD_OR_LZ4 45 | LIBTRACEIO_ZSTD_LZ4=ior-zstd-lz4.c 46 | else 47 | LIBTRACEIO_ZSTD_LZ4= 48 | endif 49 | 50 | # if we have HTTP support, we also build Swift support 51 | if HAVE_HTTP 52 | LIBTRACEIO_HTTP=ior-http.c curl-helper.c curl-helper.h \ 53 | ior-swift.c \ 54 | swift-support/keystone.c swift-support/keystone.h \ 55 | swift-support/jsmn.c swift-support/jsmn.h \ 56 | swift-support/jsmn_utils.c swift-support/jsmn_utils.h 57 | else 58 | LIBTRACE_HTTP= 59 | endif 60 | 61 | if HAVE_QATZIP 62 | LIBTRACEIO_QATZIP=ior-qat.c iow-qat.c 63 | else 64 | LIBTRACEIO_QATZIP= 65 | endif 66 | 67 | libwandio_la_SOURCES=wandio.c ior-peek.c ior-stdio.c ior-thread.c \ 68 | iow-stdio.c iow-thread.c wandio.h wandio_internal.h \ 69 | $(LIBTRACEIO_ZLIB) $(LIBTRACEIO_BZLIB) $(LIBTRACEIO_LZO) \ 70 | $(LIBTRACEIO_LZMA) $(LIBTRACEIO_HTTP) $(LIBTRACEIO_ZSTD) \ 71 | $(LIBTRACEIO_LZ4) $(LIBTRACEIO_ZSTD_LZ4) \ 72 | $(LIBTRACEIO_QATZIP) 73 | 74 | AM_CPPFLAGS = @ADD_INCLS@ 75 | libwandio_la_LIBADD = @LIBWANDIO_LIBS@ 76 | libwandio_la_LDFLAGS=-version-info 6:5:0 @ADD_LDFLAGS@ 77 | 78 | -------------------------------------------------------------------------------- /lib/curl-helper.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2016 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | /* Author: Alistair King */ 28 | 29 | #include "curl-helper.h" 30 | #include 31 | #include 32 | #include 33 | 34 | /* we lock calls to curl_global_init because it does non-thread-safe things, but 35 | this is still a little sketchy because apparently it calls a bunch of 36 | non-curl functions that are also not thread safe 37 | (http://curl.haxx.se/mail/lib-2008-02/0126.html) and so users of libwandio 38 | could be calling those when we call curl_global_init :( */ 39 | static pthread_mutex_t cg_lock = PTHREAD_MUTEX_INITIALIZER; 40 | static int cg_init_cnt = 0; 41 | 42 | void curl_helper_safe_global_init(void) { 43 | /* set up global curl structures (see note above) */ 44 | pthread_mutex_lock(&cg_lock); 45 | if (!cg_init_cnt) { 46 | curl_global_init(CURL_GLOBAL_DEFAULT); 47 | } 48 | cg_init_cnt++; 49 | pthread_mutex_unlock(&cg_lock); 50 | } 51 | 52 | void curl_helper_safe_global_cleanup(void) { 53 | /* clean up global curl structures (see note above) */ 54 | pthread_mutex_lock(&cg_lock); 55 | assert(cg_init_cnt); 56 | cg_init_cnt--; 57 | if (!cg_init_cnt) 58 | curl_global_cleanup(); 59 | pthread_mutex_unlock(&cg_lock); 60 | } 61 | -------------------------------------------------------------------------------- /lib/curl-helper.h: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2016 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | /* Author: Alistair King */ 28 | 29 | #ifndef CURL_HELPER_H 30 | #define CURL_HELPER_H 1 /**< Guard Define */ 31 | 32 | /* Helper for code that uses libcurl (HTTP, Swift, Keystone) */ 33 | 34 | void curl_helper_safe_global_init(void); 35 | void curl_helper_safe_global_cleanup(void); 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /lib/ior-bzip.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2019 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | #include "config.h" 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include "wandio.h" 36 | 37 | /* Libwandio IO module implementing a bzip reader */ 38 | 39 | enum err_t { ERR_OK = 1, ERR_EOF = 0, ERR_ERROR = -1 }; 40 | 41 | struct bz_t { 42 | bz_stream strm; 43 | char inbuff[WANDIO_BUFFER_SIZE]; 44 | int outoffset; 45 | io_t *parent; 46 | enum err_t err; 47 | uint8_t streamopen; 48 | }; 49 | 50 | extern io_source_t bz_source; 51 | 52 | #define DATA(io) ((struct bz_t *)((io)->data)) 53 | #define min(a, b) ((a) < (b) ? (a) : (b)) 54 | 55 | DLLEXPORT io_t *bz_open(io_t *parent) { 56 | io_t *io; 57 | if (!parent) 58 | return NULL; 59 | io = malloc(sizeof(io_t)); 60 | io->source = &bz_source; 61 | io->data = malloc(sizeof(struct bz_t)); 62 | 63 | DATA(io)->parent = parent; 64 | 65 | DATA(io)->strm.next_in = NULL; 66 | DATA(io)->strm.avail_in = 0; 67 | DATA(io)->strm.next_out = NULL; 68 | DATA(io)->strm.avail_out = 0; 69 | DATA(io)->strm.bzalloc = NULL; 70 | DATA(io)->strm.bzfree = NULL; 71 | DATA(io)->strm.opaque = NULL; 72 | DATA(io)->err = ERR_OK; 73 | 74 | BZ2_bzDecompressInit(&DATA(io)->strm, 0, /* Verbosity */ 75 | 0); /* small */ 76 | DATA(io)->streamopen = 1; 77 | return io; 78 | } 79 | 80 | static int64_t bz_read(io_t *io, void *buffer, int64_t len) { 81 | if (DATA(io)->err == ERR_EOF) 82 | return 0; /* EOF */ 83 | if (DATA(io)->err == ERR_ERROR) { 84 | errno = EIO; 85 | return -1; /* ERROR! */ 86 | } 87 | 88 | DATA(io)->strm.avail_out = len; 89 | DATA(io)->strm.next_out = buffer; 90 | 91 | while (DATA(io)->err == ERR_OK && DATA(io)->strm.avail_out > 0) { 92 | while (DATA(io)->strm.avail_in <= 0) { 93 | int bytes_read = 94 | wandio_read(DATA(io)->parent, DATA(io)->inbuff, 95 | sizeof(DATA(io)->inbuff)); 96 | if (bytes_read == 0) /* EOF */ 97 | return len - DATA(io)->strm.avail_out; 98 | if (bytes_read < 0) { /* Error */ 99 | /* Errno should already be set */ 100 | DATA(io)->err = ERR_ERROR; 101 | /* Return how much data we managed to read ok */ 102 | if (DATA(io)->strm.avail_out != (uint32_t)len) { 103 | return len - DATA(io)->strm.avail_out; 104 | } 105 | /* Now return error */ 106 | return -1; 107 | } 108 | DATA(io)->strm.next_in = DATA(io)->inbuff; 109 | DATA(io)->strm.avail_in = bytes_read; 110 | } 111 | /* Decompress some data into the output buffer */ 112 | int err = BZ2_bzDecompress(&DATA(io)->strm); 113 | switch (err) { 114 | case BZ_OK: 115 | DATA(io)->err = ERR_OK; 116 | break; 117 | case BZ_STREAM_END: 118 | /* Stream is over, but there could be more stream to 119 | * follow (e.g. if the file was compressed by pbzip2) 120 | */ 121 | BZ2_bzDecompressEnd(&DATA(io)->strm); 122 | BZ2_bzDecompressInit(&DATA(io)->strm, 0, 0); 123 | break; 124 | default: 125 | errno = EIO; 126 | DATA(io)->err = ERR_ERROR; 127 | } 128 | } 129 | /* Return the number of bytes decompressed */ 130 | return len - DATA(io)->strm.avail_out; 131 | } 132 | 133 | static void bz_close(io_t *io) { 134 | if (DATA(io)->streamopen) { 135 | BZ2_bzDecompressEnd(&DATA(io)->strm); 136 | } 137 | wandio_destroy(DATA(io)->parent); 138 | free(io->data); 139 | free(io); 140 | } 141 | 142 | io_source_t bz_source = {"bzip", bz_read, NULL, /* peek */ 143 | NULL, /* tell */ 144 | NULL, /* seek */ 145 | bz_close}; 146 | -------------------------------------------------------------------------------- /lib/ior-lzma.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2019 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | #include "config.h" 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include "wandio.h" 36 | 37 | /* Libwandio IO module implementing an lzma reader */ 38 | 39 | enum err_t { ERR_OK = 1, ERR_EOF = 0, ERR_ERROR = -1 }; 40 | 41 | struct lzma_t { 42 | uint8_t inbuff[WANDIO_BUFFER_SIZE]; 43 | lzma_stream strm; 44 | io_t *parent; 45 | int outoffset; 46 | enum err_t err; 47 | }; 48 | 49 | extern io_source_t lzma_source; 50 | 51 | #define DATA(io) ((struct lzma_t *)((io)->data)) 52 | #define min(a, b) ((a) < (b) ? (a) : (b)) 53 | 54 | DLLEXPORT io_t *lzma_open(io_t *parent) { 55 | io_t *io; 56 | if (!parent) 57 | return NULL; 58 | io = malloc(sizeof(io_t)); 59 | io->source = &lzma_source; 60 | io->data = malloc(sizeof(struct lzma_t)); 61 | 62 | DATA(io)->parent = parent; 63 | 64 | memset(&DATA(io)->strm, 0, sizeof(DATA(io)->strm)); 65 | DATA(io)->err = ERR_OK; 66 | 67 | if (lzma_auto_decoder(&DATA(io)->strm, UINT64_MAX, 0) != LZMA_OK) { 68 | free(io->data); 69 | free(io); 70 | fprintf(stderr, "auto decoder failed\n"); 71 | return NULL; 72 | } 73 | 74 | return io; 75 | } 76 | 77 | static int64_t lzma_read(io_t *io, void *buffer, int64_t len) { 78 | if (DATA(io)->err == ERR_EOF) 79 | return 0; /* EOF */ 80 | if (DATA(io)->err == ERR_ERROR) { 81 | errno = EIO; 82 | return -1; /* ERROR! */ 83 | } 84 | 85 | DATA(io)->strm.avail_out = len; 86 | DATA(io)->strm.next_out = buffer; 87 | 88 | while (DATA(io)->err == ERR_OK && DATA(io)->strm.avail_out > 0) { 89 | while (DATA(io)->strm.avail_in <= 0) { 90 | int bytes_read = wandio_read(DATA(io)->parent, 91 | (char *)DATA(io)->inbuff, 92 | sizeof(DATA(io)->inbuff)); 93 | if (bytes_read == 0) { 94 | /* EOF */ 95 | if (DATA(io)->strm.avail_out == (uint32_t)len) { 96 | DATA(io)->err = ERR_EOF; 97 | return 0; 98 | } 99 | /* Return how much data we've managed to read 100 | * so far. */ 101 | return len - DATA(io)->strm.avail_out; 102 | } 103 | if (bytes_read < 0) { /* Error */ 104 | /* errno should be set */ 105 | DATA(io)->err = ERR_ERROR; 106 | /* Return how much data we managed to read ok */ 107 | if (DATA(io)->strm.avail_out != (uint32_t)len) { 108 | return len - DATA(io)->strm.avail_out; 109 | } 110 | /* Now return error */ 111 | return -1; 112 | } 113 | DATA(io)->strm.next_in = DATA(io)->inbuff; 114 | DATA(io)->strm.avail_in = bytes_read; 115 | } 116 | /* Decompress some data into the output buffer */ 117 | lzma_ret err = lzma_code(&DATA(io)->strm, LZMA_RUN); 118 | switch (err) { 119 | case LZMA_OK: 120 | DATA(io)->err = ERR_OK; 121 | break; 122 | case LZMA_STREAM_END: 123 | DATA(io)->err = ERR_EOF; 124 | break; 125 | default: 126 | errno = EIO; 127 | DATA(io)->err = ERR_ERROR; 128 | } 129 | } 130 | /* Return the number of bytes decompressed */ 131 | return len - DATA(io)->strm.avail_out; 132 | } 133 | 134 | static void lzma_close(io_t *io) { 135 | lzma_end(&DATA(io)->strm); 136 | wandio_destroy(DATA(io)->parent); 137 | free(io->data); 138 | free(io); 139 | } 140 | 141 | io_source_t lzma_source = {"lzma", lzma_read, NULL, /* peek */ 142 | NULL, /* tell */ 143 | NULL, /* seek */ 144 | lzma_close}; 145 | -------------------------------------------------------------------------------- /lib/ior-peek.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2019 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | #include "config.h" 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include "wandio.h" 38 | 39 | /* Libwandio IO module implementing a peeking reader. 40 | * 41 | * Assuming my understanding of Perry's code is correct, this module provides 42 | * generic support for "peeking" that can be used in concert with any other 43 | * implemented IO reader. 44 | * 45 | * The other IO reader is a "child" to the peeking reader and is used to read 46 | * the data into a buffer managed by the peeking reader. Any actual "peeks" 47 | * are serviced from the managed buffer, which means that we do not have to 48 | * manipulate the read offsets directly in zlib or bzip, for instance. 49 | */ 50 | 51 | /* for O_DIRECT we have to read in multiples of this */ 52 | #define MIN_READ_SIZE 4096 53 | /* Round reads for peeks into the buffer up to this size */ 54 | #define PEEK_SIZE (WANDIO_BUFFER_SIZE) 55 | 56 | struct peek_t { 57 | io_t *child; 58 | char *buffer; 59 | int64_t length; /* Length of buffer */ 60 | int64_t offset; /* Offset into buffer */ 61 | }; 62 | 63 | extern io_source_t peek_source; 64 | 65 | #define DATA(io) ((struct peek_t *)((io)->data)) 66 | #define MIN(a, b) ((a) < (b) ? (a) : (b)) 67 | 68 | DLLEXPORT io_t *peek_open(io_t *child) { 69 | io_t *io; 70 | if (!child) 71 | return NULL; 72 | io = malloc(sizeof(io_t)); 73 | io->data = malloc(sizeof(struct peek_t)); 74 | io->source = &peek_source; 75 | 76 | /* Wrap the peeking reader around the "child" */ 77 | DATA(io)->child = child; 78 | DATA(io)->buffer = NULL; 79 | DATA(io)->length = 0; 80 | DATA(io)->offset = 0; 81 | 82 | return io; 83 | } 84 | 85 | /* Read at least "len" bytes from the child io into the internal buffer, and 86 | return how many bytes was actually read. 87 | */ 88 | static int64_t refill_buffer(io_t *io, int64_t len) { 89 | int64_t bytes_read; 90 | assert(DATA(io)->length - DATA(io)->offset == 0); 91 | /* Select the largest of "len", PEEK_SIZE and the current peek buffer 92 | * size and then round up to the nearest multiple of MIN_READ_SIZE 93 | */ 94 | bytes_read = len < PEEK_SIZE ? PEEK_SIZE : len; 95 | bytes_read = 96 | bytes_read < DATA(io)->length ? DATA(io)->length : bytes_read; 97 | bytes_read += MIN_READ_SIZE - (bytes_read % MIN_READ_SIZE); 98 | /* Is the current buffer big enough? */ 99 | if (DATA(io)->length < bytes_read) { 100 | int res = 0; 101 | void *buf_ptr = (void *)(DATA(io)->buffer); 102 | 103 | if (buf_ptr) 104 | free(buf_ptr); 105 | DATA(io)->length = bytes_read; 106 | DATA(io)->offset = 0; 107 | #if _POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600 108 | /* We need to do this as read() of O_DIRECT might happen into 109 | * this buffer. The docs suggest 512 bytes is all we need to 110 | * align to, but I'm suspicious. I expect disks with 4k blocks 111 | * will arrive soon, and thus 4k is the minimum I'm willing to 112 | * live with. 113 | */ 114 | res = posix_memalign(&buf_ptr, 4096, DATA(io)->length); 115 | if (res != 0) { 116 | fprintf(stderr, "Error aligning IO buffer: %d\n", res); 117 | errno = res; 118 | return -1; 119 | } 120 | DATA(io)->buffer = buf_ptr; 121 | #else 122 | res = 0; /* << Silly warning */ 123 | (void)res; /* << Another silly warning */ 124 | DATA(io)->buffer = malloc(DATA(io)->length); 125 | #endif 126 | } else 127 | DATA(io)->length = bytes_read; 128 | 129 | assert(DATA(io)->buffer); 130 | 131 | /* Now actually attempt to read that many bytes */ 132 | bytes_read = DATA(io)->child->source->read( 133 | DATA(io)->child, DATA(io)->buffer, bytes_read); 134 | 135 | DATA(io)->offset = 0; 136 | DATA(io)->length = bytes_read; 137 | 138 | /* Error? */ 139 | if (bytes_read < 1) 140 | return bytes_read; 141 | 142 | return bytes_read; 143 | } 144 | 145 | static int64_t peek_read(io_t *io, void *buffer, int64_t len) { 146 | int64_t ret = 0; 147 | 148 | /* Have we previously encountered an error? */ 149 | if (DATA(io)->length < 0) { 150 | return DATA(io)->length; 151 | } 152 | 153 | /* Is some of this data in the buffer? */ 154 | if (DATA(io)->buffer && DATA(io)->length) { 155 | ret = MIN(len, DATA(io)->length - DATA(io)->offset); 156 | 157 | /* Copy anything we've got into their buffer, and shift our 158 | * offset so that we don't peek at the data we've read again */ 159 | memcpy(buffer, DATA(io)->buffer + DATA(io)->offset, ret); 160 | buffer += ret; 161 | DATA(io)->offset += ret; 162 | len -= ret; 163 | } 164 | 165 | /* Use the child reader to get the rest of the required data */ 166 | if (len > 0) { 167 | /* To get here, the buffer must be empty */ 168 | assert(DATA(io)->length - DATA(io)->offset == 0); 169 | int64_t bytes_read; 170 | /* If they're reading exactly a block size, just use that, no 171 | * point in malloc'ing and memcpy()ing needlessly. However, if 172 | * the buffer isn't aligned, we need to pass on an aligning 173 | * buffer, skip this and do it into our own aligned buffer. 174 | */ 175 | if ((len % MIN_READ_SIZE == 0) && 176 | ((ptrdiff_t)buffer % 4096) == 0) { 177 | assert(((ptrdiff_t)buffer % 4096) == 0); 178 | bytes_read = DATA(io)->child->source->read( 179 | DATA(io)->child, buffer, len); 180 | /* Error? */ 181 | if (bytes_read < 1) { 182 | /* Return if we have managed to get some data ok 183 | */ 184 | if (ret > 0) 185 | return ret; 186 | /* Return the error upstream */ 187 | return bytes_read; 188 | } 189 | } else { 190 | bytes_read = refill_buffer(io, len); 191 | if (bytes_read < 1) { 192 | /* Return if we have managed to get some data ok 193 | */ 194 | if (ret > 0) 195 | return ret; 196 | /* Return the error upstream */ 197 | return bytes_read; 198 | } 199 | /* Now grab the number of bytes asked for. */ 200 | len = len < bytes_read ? len : bytes_read; 201 | memcpy(buffer, DATA(io)->buffer, len); 202 | 203 | DATA(io)->offset = len; 204 | bytes_read = len; 205 | } 206 | ret += bytes_read; 207 | } 208 | 209 | /* Have we read past the end of the buffer? */ 210 | if (DATA(io)->buffer && DATA(io)->offset >= DATA(io)->length) { 211 | /* If so, free the memory it used */ 212 | free(DATA(io)->buffer); 213 | DATA(io)->buffer = NULL; 214 | DATA(io)->offset = 0; 215 | DATA(io)->length = 0; 216 | } 217 | 218 | return ret; 219 | } 220 | 221 | static void *alignedrealloc(void *old, size_t oldsize, size_t size, int *res) { 222 | #if _POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600 223 | void *new; 224 | /* Shortcut resizing */ 225 | if (size < oldsize) 226 | return old; 227 | *res = posix_memalign(&new, 4096, size); 228 | if (*res != 0) { 229 | fprintf(stderr, "Error aligning IO buffer: %d\n", *res); 230 | 231 | return NULL; 232 | } 233 | assert(oldsize < size); 234 | memcpy(new, old, oldsize); 235 | free(old); 236 | return new; 237 | #else 238 | /* These no-ops are to stop the compiler whinging about unused 239 | * parameters */ 240 | (void)oldsize; 241 | (void)res; 242 | return realloc(old, size); 243 | #endif 244 | } 245 | 246 | static int64_t peek_peek(io_t *io, void *buffer, int64_t len) { 247 | int64_t ret = 0; 248 | int res = 0; 249 | 250 | /* Is there enough data in the buffer to serve this request? */ 251 | if (DATA(io)->length - DATA(io)->offset < len) { 252 | /* No, we need to extend the buffer. */ 253 | int64_t read_amount = 254 | len - (DATA(io)->length - DATA(io)->offset); 255 | /* Round the read_amount up to the nearest MB */ 256 | read_amount += 257 | PEEK_SIZE - ((DATA(io)->length + read_amount) % PEEK_SIZE); 258 | DATA(io)->buffer = 259 | alignedrealloc(DATA(io)->buffer, DATA(io)->length, 260 | DATA(io)->length + read_amount, &res); 261 | 262 | if (DATA(io)->buffer == NULL) { 263 | return res; 264 | } 265 | 266 | /* Use the child reader to read more data into our managed 267 | * buffer */ 268 | read_amount = wandio_read(DATA(io)->child, 269 | DATA(io)->buffer + DATA(io)->length, 270 | read_amount); 271 | 272 | /* Pass errors up */ 273 | if (read_amount < 0) { 274 | return read_amount; 275 | } 276 | 277 | DATA(io)->length += read_amount; 278 | } 279 | 280 | /* Right, now return data from the buffer (that now should be large 281 | * enough, but might not be if we hit EOF) */ 282 | ret = MIN(len, DATA(io)->length - DATA(io)->offset); 283 | memcpy(buffer, DATA(io)->buffer + DATA(io)->offset, ret); 284 | return ret; 285 | } 286 | 287 | static int64_t peek_tell(io_t *io) { 288 | /* We don't actually maintain a read offset as such, so we want to 289 | * return the child's read offset */ 290 | return wandio_tell(DATA(io)->child); 291 | } 292 | 293 | static int64_t peek_seek(io_t *io, int64_t offset, int whence) { 294 | /* Again, we don't have a genuine read offset so we need to pass this 295 | * one on to the child */ 296 | return wandio_seek(DATA(io)->child, offset, whence); 297 | } 298 | 299 | static void peek_close(io_t *io) { 300 | /* Make sure we close the child that is doing the actual reading! */ 301 | wandio_destroy(DATA(io)->child); 302 | if (DATA(io)->buffer) 303 | free(DATA(io)->buffer); 304 | free(io->data); 305 | free(io); 306 | } 307 | 308 | io_source_t peek_source = {"peek", peek_read, peek_peek, 309 | peek_tell, peek_seek, peek_close}; 310 | -------------------------------------------------------------------------------- /lib/ior-qat.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2019 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | #include "config.h" 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include "wandio.h" 34 | 35 | #define DATA(io) ((struct qat_t *)((io)->data)) 36 | enum err_t { ERR_OK = 1, ERR_EOF = 0, ERR_ERROR = -1 }; 37 | 38 | extern io_source_t qat_source; 39 | 40 | static void qat_perror(int errcode) { 41 | if (errcode >= 0) { 42 | return; 43 | } 44 | 45 | switch (errcode) { 46 | case QZ_FAIL: 47 | fprintf(stderr, "QATzip failed for some unspecified reason.\n"); 48 | break; 49 | case QZ_PARAMS: 50 | fprintf(stderr, "Invalid parameters provided to QATzip.\n"); 51 | break; 52 | case QZ_BUF_ERROR: 53 | fprintf(stderr, "QATzip buffer was too small.\n"); 54 | break; 55 | case QZ_DATA_ERROR: 56 | fprintf(stderr, "QATzip input data was corrupted.\n"); 57 | break; 58 | case QZ_NOSW_NO_HW: 59 | fprintf(stderr, "QAT HW not detected.\n"); 60 | break; 61 | case QZ_NOSW_NO_MDRV: 62 | fprintf(stderr, "QAT memory driver not detected.\n"); 63 | break; 64 | case QZ_NOSW_NO_INST_ATTACH: 65 | fprintf(stderr, "Unable to attach to QAT instance.\n"); 66 | break; 67 | case QZ_NOSW_LOW_MEM: 68 | fprintf(stderr, "Insufficient pinned memory for QAT.\n"); 69 | break; 70 | } 71 | } 72 | 73 | struct qat_t { 74 | QzSession_T sess; 75 | io_t *parent; 76 | unsigned char inbuff[WANDIO_BUFFER_SIZE * 10]; 77 | int64_t inoffset; 78 | int64_t indecomp; 79 | int64_t insize; 80 | enum err_t err; 81 | }; 82 | 83 | DLLEXPORT io_t *qat_open(io_t *parent) { 84 | 85 | int x; 86 | io_t *io; 87 | QzSessionParams_T params; 88 | 89 | io = (io_t *)malloc(sizeof(io_t)); 90 | io->source = &qat_source; 91 | io->data = malloc(sizeof(struct qat_t)); 92 | 93 | DATA(io)->parent = parent; 94 | DATA(io)->inoffset = 0; 95 | DATA(io)->indecomp = 0; 96 | DATA(io)->err = ERR_OK; 97 | DATA(io)->insize = sizeof(DATA(io)->inbuff); 98 | 99 | if ((x = qzInit(&(DATA(io)->sess), 0)) != QZ_OK) { 100 | qat_perror(x); 101 | free(io->data); 102 | free(io); 103 | return NULL; 104 | } 105 | 106 | params.huffman_hdr = QZ_DYNAMIC_HDR; 107 | params.direction = QZ_DIR_DECOMPRESS; 108 | params.comp_lvl = 1; 109 | params.comp_algorithm = QZ_DEFLATE; 110 | params.data_fmt = QZ_DATA_FORMAT_DEFAULT; 111 | params.poll_sleep = QZ_POLL_SLEEP_DEFAULT; 112 | params.max_forks = QZ_MAX_FORK_DEFAULT; 113 | params.sw_backup = 0; 114 | params.hw_buff_sz = QZ_HW_BUFF_SZ; 115 | params.strm_buff_sz = QZ_STRM_BUFF_SZ_DEFAULT; 116 | params.input_sz_thrshold = QZ_COMP_THRESHOLD_DEFAULT; 117 | params.req_cnt_thrshold = 4; 118 | params.wait_cnt_thrshold = QZ_WAIT_CNT_THRESHOLD_DEFAULT; 119 | 120 | if ((x = qzSetupSession(&(DATA(io)->sess), ¶ms)) != QZ_OK) { 121 | qat_perror(x); 122 | free(io->data); 123 | free(io); 124 | return NULL; 125 | } 126 | 127 | return io; 128 | } 129 | 130 | static inline int64_t _read_from_parent(io_t *io) { 131 | int bytes_read; 132 | int64_t unread; 133 | 134 | unread = DATA(io)->inoffset - DATA(io)->indecomp; 135 | 136 | if (unread < 256 * 1024) { 137 | if (unread == 0) { 138 | DATA(io)->inoffset = 0; 139 | DATA(io)->indecomp = 0; 140 | } else if (DATA(io)->insize - DATA(io)->inoffset < 256 * 1024) { 141 | /* compact buffer */ 142 | memmove(DATA(io)->inbuff, 143 | DATA(io)->inbuff + DATA(io)->indecomp, unread); 144 | DATA(io)->inoffset = unread; 145 | DATA(io)->indecomp = 0; 146 | } 147 | } 148 | 149 | while (1) { 150 | bytes_read = wandio_read(DATA(io)->parent, 151 | DATA(io)->inbuff + DATA(io)->inoffset, 152 | DATA(io)->insize - DATA(io)->inoffset); 153 | 154 | if (bytes_read < 0) { 155 | DATA(io)->err = ERR_ERROR; 156 | return -1; 157 | } 158 | 159 | if (bytes_read == 0 && 160 | DATA(io)->inoffset == DATA(io)->indecomp) { 161 | DATA(io)->err = ERR_EOF; 162 | return 0; 163 | } 164 | 165 | DATA(io)->inoffset += bytes_read; 166 | if (bytes_read == 0 || DATA(io)->inoffset >= DATA(io)->insize) { 167 | break; 168 | } 169 | } 170 | return DATA(io)->inoffset - DATA(io)->indecomp; 171 | } 172 | 173 | static int64_t qat_read(io_t *io, void *buffer, int64_t len) { 174 | int rc; 175 | unsigned int dst_len, src_len; 176 | int64_t readsofar = 0; 177 | 178 | if (DATA(io)->err == ERR_EOF) { 179 | return 0; 180 | } 181 | if (DATA(io)->err == ERR_ERROR) { 182 | errno = EIO; 183 | return -1; 184 | } 185 | 186 | while (DATA(io)->err == ERR_OK && readsofar < len) { 187 | 188 | rc = _read_from_parent(io); 189 | if (rc <= 0) { 190 | if (readsofar > 0) { 191 | break; 192 | } 193 | return rc; 194 | } 195 | 196 | src_len = (unsigned int)DATA(io)->inoffset - DATA(io)->indecomp; 197 | dst_len = (unsigned int)len; 198 | rc = qzDecompress(&(DATA(io)->sess), 199 | DATA(io)->inbuff + DATA(io)->indecomp, 200 | &src_len, buffer + readsofar, &dst_len); 201 | 202 | if (rc != QZ_OK && rc != QZ_BUF_ERROR && rc != QZ_DATA_ERROR) { 203 | DATA(io)->err = ERR_ERROR; 204 | qat_perror(rc); 205 | return -1; 206 | } else { 207 | DATA(io)->err = ERR_OK; 208 | } 209 | 210 | DATA(io)->indecomp += src_len; 211 | readsofar += dst_len; 212 | } 213 | 214 | return readsofar; 215 | } 216 | 217 | static void qat_close(io_t *io) { 218 | qzTeardownSession(&(DATA(io)->sess)); 219 | qzClose(&(DATA(io)->sess)); 220 | wandio_destroy(DATA(io)->parent); 221 | free(io->data); 222 | free(io); 223 | } 224 | 225 | io_source_t qat_source = {"qatr", qat_read, NULL, /* peek */ 226 | NULL, /* tell */ 227 | NULL, /* seek */ 228 | qat_close}; 229 | -------------------------------------------------------------------------------- /lib/ior-stdio.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2019 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | #define _GNU_SOURCE 1 28 | #include "config.h" 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include "wandio.h" 36 | #include "wandio_internal.h" 37 | 38 | /* Libwandio IO module implementing a standard IO reader, i.e. no decompression 39 | */ 40 | 41 | struct stdio_t { 42 | int fd; 43 | }; 44 | 45 | extern io_source_t stdio_source; 46 | 47 | #define DATA(io) ((struct stdio_t *)((io)->data)) 48 | 49 | DLLEXPORT io_t *stdio_open(const char *filename) { 50 | io_t *io = malloc(sizeof(io_t)); 51 | io->data = malloc(sizeof(struct stdio_t)); 52 | 53 | if (strcmp(filename, "-") == 0) 54 | DATA(io)->fd = 0; /* STDIN */ 55 | else 56 | DATA(io)->fd = 57 | open(filename, O_RDONLY 58 | #ifdef O_DIRECT 59 | | (force_directio_read ? O_DIRECT : 0) 60 | #endif 61 | ); 62 | io->source = &stdio_source; 63 | 64 | if (DATA(io)->fd == -1) { 65 | free(io); 66 | return NULL; 67 | } 68 | 69 | return io; 70 | } 71 | 72 | static int64_t stdio_read(io_t *io, void *buffer, int64_t len) { 73 | return read(DATA(io)->fd, buffer, len); 74 | } 75 | 76 | static int64_t stdio_tell(io_t *io) { 77 | return lseek(DATA(io)->fd, 0, SEEK_CUR); 78 | } 79 | 80 | static int64_t stdio_seek(io_t *io, int64_t offset, int whence) { 81 | return lseek(DATA(io)->fd, offset, whence); 82 | } 83 | 84 | static void stdio_close(io_t *io) { 85 | close(DATA(io)->fd); 86 | free(io->data); 87 | free(io); 88 | } 89 | 90 | io_source_t stdio_source = {"stdio", stdio_read, NULL, 91 | stdio_tell, stdio_seek, stdio_close}; 92 | -------------------------------------------------------------------------------- /lib/ior-swift.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2016 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | /* Author: Alistair King 28 | * 29 | * This code has been adapted from kurl: 30 | * https://github.com/attractivechaos/klib 31 | * (released under the MIT/X11 license) 32 | */ 33 | 34 | #include "config.h" 35 | #include 36 | #include 37 | #include "swift-support/keystone.h" 38 | #include "wandio.h" 39 | 40 | /* Libwandio IO module implementing an OpenStack Swift reader */ 41 | 42 | struct swift_t { 43 | // Name of the Swift container to read 44 | char *container; 45 | 46 | // Name of the Swift object to read 47 | char *object; 48 | 49 | // Authentication credentials to use with Keystone 50 | keystone_auth_creds_t creds; 51 | 52 | // Keystone stoken and Swift storage URL (either returned by Keystone or 53 | // extracted from the environment) 54 | keystone_auth_token_t token; 55 | 56 | // Child reader that does the actual data download 57 | io_t *http_reader; 58 | }; 59 | 60 | extern io_source_t swift_source; 61 | 62 | #define DATA(io) ((struct swift_t *)((io)->data)) 63 | 64 | #define SWIFT_PFX "swift://" 65 | #define SWIFT_PFX_LEN 8 66 | #define SWIFT_AUTH_TOKEN_HDR "X-Auth-Token: " 67 | 68 | DLLEXPORT io_t *swift_open(const char *filename); 69 | static int64_t swift_read(io_t *io, void *buffer, int64_t len); 70 | static int64_t swift_tell(io_t *io); 71 | static int64_t swift_seek(io_t *io, int64_t offset, int whence); 72 | static void swift_close(io_t *io); 73 | 74 | static int parse_swifturl(const char *swifturl, char **container, 75 | char **object) { 76 | const char *ctmp; 77 | const char *otmp; 78 | size_t clen, olen; 79 | // parse string like swift://CONTAINER/OBJECT 80 | if (!swifturl || strlen(swifturl) < SWIFT_PFX_LEN + 1 || 81 | strncmp(swifturl, SWIFT_PFX, SWIFT_PFX_LEN) != 0) { 82 | // malformed URL 83 | return -1; 84 | } 85 | // now, skip over the prefix and assume this is the container 86 | ctmp = swifturl + SWIFT_PFX_LEN; 87 | // and then look for the next '/' 88 | if ((otmp = strchr(ctmp, '/')) == NULL) { 89 | // malformed URL (no object?) 90 | return -1; 91 | } 92 | otmp++; // skip over the slash 93 | // now we know how long things are, so allocate some memory 94 | clen = otmp - ctmp - 1; 95 | olen = strlen(otmp); 96 | if ((*container = malloc(clen + 1)) == NULL) { 97 | return -1; 98 | } 99 | // and copy the string in 100 | memcpy(*container, ctmp, clen); 101 | (*container)[clen] = '\0'; 102 | 103 | // now the object name 104 | if ((*object = malloc(olen + 1)) == NULL) { 105 | free(*container); 106 | return -1; 107 | } 108 | memcpy(*object, otmp, olen); 109 | (*object)[olen] = '\0'; 110 | 111 | return 0; 112 | } 113 | 114 | static char *build_auth_token_hdr(char *token) { 115 | char *hdr; 116 | 117 | if ((hdr = malloc(strlen(SWIFT_AUTH_TOKEN_HDR) + strlen(token) + 1)) == 118 | NULL) { 119 | return NULL; 120 | } 121 | 122 | strcpy(hdr, SWIFT_AUTH_TOKEN_HDR); 123 | strcat(hdr, token); 124 | 125 | return hdr; 126 | } 127 | 128 | static char *build_http_url(struct swift_t *s) { 129 | // STORAGE_URL/CONTAINER/OBJECT 130 | char *url; 131 | 132 | if ((url = malloc(strlen(s->token.storage_url) + 1 + 133 | strlen(s->container) + 1 + strlen(s->object) + 1)) == 134 | NULL) { 135 | return NULL; 136 | } 137 | 138 | strcpy(url, s->token.storage_url); 139 | strcat(url, "/"); 140 | strcat(url, s->container); 141 | strcat(url, "/"); 142 | strcat(url, s->object); 143 | 144 | return url; 145 | } 146 | 147 | static int get_token(struct swift_t *s) { 148 | // first see if the token is explicitly loaded into the env 149 | if (keystone_env_parse_token(&s->token) == 1) { 150 | // then let's use it 151 | return 0; 152 | } 153 | 154 | // ok, so let's see if there are auth credentials available 155 | if (keystone_env_parse_creds(&s->creds) != 1) { 156 | // no credentials available, nothing we can do 157 | fprintf(stderr, 158 | "ERROR: Could not find either Keystone v3 " 159 | "authentication environment variables\n" 160 | " (OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, etc.), " 161 | "or auth token variables \n" 162 | " (OS_AUTH_TOKEN, OS_STORAGE_URL).\n"); 163 | return -1; 164 | } 165 | 166 | // alright, we have credentials, ask keystone helper to get a token 167 | if (keystone_authenticate(&s->creds, &s->token) != 1) { 168 | fprintf(stderr, 169 | "ERROR: Swift (Keystone v3) authentication failed. " 170 | "Check credentials and retry\n"); 171 | return -1; 172 | } 173 | 174 | // we now have a token that we can at least try and use 175 | return 0; 176 | } 177 | 178 | io_t *swift_open(const char *filename) { 179 | io_t *io = malloc(sizeof(io_t)); 180 | char *auth_hdr = NULL; 181 | char *http_url = NULL; 182 | 183 | if (!io) 184 | return NULL; 185 | io->data = malloc(sizeof(struct swift_t)); 186 | if (!io->data) { 187 | free(io); 188 | return NULL; 189 | } 190 | memset(io->data, 0, sizeof(struct swift_t)); 191 | 192 | io->source = &swift_source; 193 | 194 | // parse the filename in to container and object 195 | if (parse_swifturl(filename, &DATA(io)->container, &DATA(io)->object) != 196 | 0) { 197 | swift_close(io); 198 | return NULL; 199 | } 200 | 201 | if (get_token(DATA(io)) != 0) { 202 | goto err; 203 | } 204 | 205 | // DEBUG: 206 | // keystone_auth_token_dump(&DATA(io)->token); 207 | 208 | // by here we are sure that we have an auth token and storage url, so we 209 | // need to do two more things: build the header to pass to curl, and 210 | // build the full http object path for the GET request 211 | if ((auth_hdr = build_auth_token_hdr(DATA(io)->token.token)) == NULL) { 212 | goto err; 213 | } 214 | 215 | // build the full HTTP URL 216 | if ((http_url = build_http_url(DATA(io))) == NULL) { 217 | goto err; 218 | } 219 | 220 | // open the child reader! 221 | if ((DATA(io)->http_reader = http_open_hdrs(http_url, &auth_hdr, 1)) == 222 | NULL) { 223 | goto err; 224 | } 225 | 226 | return io; 227 | 228 | err: 229 | free(auth_hdr); 230 | free(http_url); 231 | swift_close(io); 232 | return NULL; 233 | } 234 | 235 | static int64_t swift_read(io_t *io, void *buffer, int64_t len) { 236 | if (!DATA(io)->http_reader) 237 | return 0; // end-of-file? 238 | return wandio_read(DATA(io)->http_reader, buffer, len); 239 | } 240 | 241 | static int64_t swift_tell(io_t *io) { 242 | if (!DATA(io)->http_reader) 243 | return -1; 244 | return wandio_tell(DATA(io)->http_reader); 245 | } 246 | 247 | static int64_t swift_seek(io_t *io, int64_t offset, int whence) { 248 | if (!DATA(io)->http_reader) 249 | return -1; 250 | return wandio_seek(DATA(io)->http_reader, offset, whence); 251 | } 252 | 253 | static void swift_close(io_t *io) { 254 | free(DATA(io)->container); 255 | free(DATA(io)->object); 256 | keystone_auth_creds_destroy(&DATA(io)->creds); 257 | keystone_auth_token_destroy(&DATA(io)->token); 258 | 259 | if (DATA(io)->http_reader != NULL) { 260 | wandio_destroy(DATA(io)->http_reader); 261 | } 262 | 263 | free(io->data); 264 | free(io); 265 | } 266 | 267 | io_source_t swift_source = {"swift", swift_read, NULL, 268 | swift_tell, swift_seek, swift_close}; 269 | -------------------------------------------------------------------------------- /lib/ior-thread.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2019 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | #include "config.h" 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include "wandio.h" 38 | #include "wandio_internal.h" 39 | #ifdef HAVE_SYS_PRCTL_H 40 | #include 41 | #endif 42 | 43 | /* Libwandio IO module implementing a threaded reader. 44 | * 45 | * This module enables another IO reader, called the "parent", to perform its 46 | * reading using a separate thread. The reading thread reads data into a 47 | * series of 1MB buffers. Once all the buffers are full, it waits for the 48 | * main thread to free up some of the buffers by consuming data from them. The 49 | * reading thread also uses a pthread condition to indicate to the main thread 50 | * that there is data available in the buffers. 51 | */ 52 | 53 | /* 1MB Buffer */ 54 | extern io_source_t thread_source; 55 | 56 | /* This structure defines a single buffer or "slice" */ 57 | struct buffer_t { 58 | char *space; /* The buffer itself */ 59 | int len; /* The size of the buffer */ 60 | enum { EMPTY = 0, FULL = 1 } state; /* Is the buffer in use? */ 61 | }; 62 | 63 | struct state_t { 64 | /* The collection of buffers (or slices) */ 65 | struct buffer_t *buffer; 66 | /* The index of the buffer to read into next */ 67 | int in_buffer; 68 | /* The read offset into the current buffer */ 69 | int64_t offset; 70 | /* The reading thread */ 71 | pthread_t producer; 72 | /* Indicates that there is a free buffer to read into */ 73 | pthread_cond_t space_avail; 74 | /* Indicates that there is data in one of the buffers */ 75 | pthread_cond_t data_ready; 76 | /* The mutex for the read buffers */ 77 | pthread_mutex_t mutex; 78 | /* The parent reader */ 79 | io_t *io; 80 | /* Indicates whether the main thread is concluding */ 81 | bool closing; 82 | }; 83 | 84 | #define DATA(x) ((struct state_t *)((x)->data)) 85 | #define INBUFFER(x) (DATA(x)->buffer[DATA(x)->in_buffer]) 86 | #define min(a, b) ((a) < (b) ? (a) : (b)) 87 | 88 | /* The reading thread */ 89 | static void *thread_producer(void *userdata) { 90 | io_t *state = (io_t *)userdata; 91 | int buffer = 0; 92 | bool running = true; 93 | 94 | #ifdef PR_SET_NAME 95 | char namebuf[17]; 96 | if (prctl(PR_GET_NAME, namebuf, 0, 0, 0) == 0) { 97 | namebuf[16] = '\0'; /* Make sure it's NUL terminated */ 98 | /* If the filename is too long, overwrite the last few bytes */ 99 | if (strlen(namebuf) > 9) { 100 | strcpy(namebuf + 10, "[ior]"); 101 | } else { 102 | strncat(namebuf, " [ior]", 16); 103 | } 104 | prctl(PR_SET_NAME, namebuf, 0, 0, 0); 105 | } 106 | #endif 107 | 108 | pthread_mutex_lock(&DATA(state)->mutex); 109 | do { 110 | /* If all the buffers are full, we need to wait for one to 111 | * become free otherwise we have nowhere to write to! */ 112 | while (DATA(state)->buffer[buffer].state == FULL) { 113 | if (DATA(state)->closing) 114 | break; 115 | pthread_cond_wait(&DATA(state)->space_avail, 116 | &DATA(state)->mutex); 117 | } 118 | 119 | /* Don't bother reading any more data if we are shutting up 120 | * shop */ 121 | if (DATA(state)->closing) { 122 | break; 123 | } 124 | pthread_mutex_unlock(&DATA(state)->mutex); 125 | 126 | /* Get the parent reader to fill the buffer */ 127 | DATA(state)->buffer[buffer].len = wandio_read( 128 | DATA(state)->io, DATA(state)->buffer[buffer].space, 129 | WANDIO_BUFFER_SIZE); 130 | 131 | pthread_mutex_lock(&DATA(state)->mutex); 132 | 133 | DATA(state)->buffer[buffer].state = FULL; 134 | 135 | /* If we've not reached the end of the file keep going */ 136 | running = (DATA(state)->buffer[buffer].len > 0); 137 | 138 | /* Signal that there is data available for the main thread */ 139 | pthread_cond_signal(&DATA(state)->data_ready); 140 | 141 | /* Move on to the next buffer */ 142 | buffer = (buffer + 1) % max_buffers; 143 | 144 | } while (running); 145 | 146 | /* If we reach here, it's all over so start tidying up */ 147 | wandio_destroy(DATA(state)->io); 148 | 149 | pthread_cond_signal(&DATA(state)->data_ready); 150 | pthread_mutex_unlock(&DATA(state)->mutex); 151 | 152 | return NULL; 153 | } 154 | 155 | static void thread_close(io_t *io) { 156 | unsigned int i; 157 | pthread_mutex_lock(&DATA(io)->mutex); 158 | DATA(io)->closing = true; 159 | pthread_cond_signal(&DATA(io)->space_avail); 160 | pthread_mutex_unlock(&DATA(io)->mutex); 161 | 162 | /* Wait for the thread to exit */ 163 | if (DATA(io)->producer != 0) { 164 | pthread_join(DATA(io)->producer, NULL); 165 | } 166 | 167 | pthread_mutex_destroy(&DATA(io)->mutex); 168 | pthread_cond_destroy(&DATA(io)->space_avail); 169 | pthread_cond_destroy(&DATA(io)->data_ready); 170 | 171 | for (i = 0; i < max_buffers; i++) { 172 | if (DATA(io)->buffer[i].space) { 173 | free(DATA(io)->buffer[i].space); 174 | } 175 | } 176 | free(DATA(io)->buffer); 177 | free(DATA(io)); 178 | free(io); 179 | } 180 | 181 | DLLEXPORT io_t *thread_open(io_t *parent) { 182 | io_t *state; 183 | sigset_t set; 184 | int s; 185 | unsigned int i; 186 | 187 | if (!parent) { 188 | return NULL; 189 | } 190 | 191 | sigfillset(&set); 192 | state = malloc(sizeof(io_t)); 193 | state->data = calloc(1, sizeof(struct state_t)); 194 | state->source = &thread_source; 195 | 196 | pthread_mutex_init(&DATA(state)->mutex, NULL); 197 | pthread_cond_init(&DATA(state)->data_ready, NULL); 198 | pthread_cond_init(&DATA(state)->space_avail, NULL); 199 | 200 | DATA(state)->producer = 0; 201 | DATA(state)->buffer = 202 | (struct buffer_t *)malloc(sizeof(struct buffer_t) * max_buffers); 203 | memset(DATA(state)->buffer, 0, sizeof(struct buffer_t) * max_buffers); 204 | 205 | for (i = 0; i < max_buffers; i++) { 206 | #if _POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600 207 | if (posix_memalign((void **)&(DATA(state)->buffer[i].space), 208 | 4096, WANDIO_BUFFER_SIZE) != 0) { 209 | thread_close(state); 210 | return NULL; 211 | } 212 | #else 213 | DATA(state)->buffer[i].space = calloc(1, WANDIO_BUFFER_SIZE); 214 | #endif 215 | } 216 | DATA(state)->in_buffer = 0; 217 | DATA(state)->offset = 0; 218 | 219 | DATA(state)->io = parent; 220 | DATA(state)->closing = false; 221 | 222 | /* Create the reading thread */ 223 | s = pthread_sigmask(SIG_SETMASK, &set, NULL); 224 | if (s != 0) { 225 | return NULL; 226 | } 227 | pthread_create(&DATA(state)->producer, NULL, thread_producer, state); 228 | sigemptyset(&set); 229 | s = pthread_sigmask(SIG_SETMASK, &set, NULL); 230 | if (s != 0) { 231 | return NULL; 232 | } 233 | 234 | return state; 235 | } 236 | 237 | static int64_t thread_read(io_t *state, void *buffer, int64_t len) { 238 | int slice; 239 | int copied = 0; 240 | int newbuffer; 241 | 242 | while (len > 0) { 243 | pthread_mutex_lock(&DATA(state)->mutex); 244 | 245 | /* Wait for the reader thread to provide us with some data */ 246 | while (INBUFFER(state).state == EMPTY) { 247 | ++read_waits; 248 | pthread_cond_wait(&DATA(state)->data_ready, 249 | &DATA(state)->mutex); 250 | } 251 | 252 | /* Check for errors and EOF */ 253 | if (INBUFFER(state).len < 1) { 254 | 255 | if (copied < 1) { 256 | errno = EIO; /* FIXME: Preserve the errno from 257 | the other thread */ 258 | copied = INBUFFER(state).len; 259 | } 260 | 261 | pthread_mutex_unlock(&DATA(state)->mutex); 262 | return copied; 263 | } 264 | 265 | /* Copy the next available slice into the main buffer */ 266 | slice = min(INBUFFER(state).len - DATA(state)->offset, len); 267 | 268 | pthread_mutex_unlock(&DATA(state)->mutex); 269 | 270 | memcpy(buffer, INBUFFER(state).space + DATA(state)->offset, 271 | slice); 272 | 273 | buffer += slice; 274 | len -= slice; 275 | copied += slice; 276 | 277 | pthread_mutex_lock(&DATA(state)->mutex); 278 | DATA(state)->offset += slice; 279 | newbuffer = DATA(state)->in_buffer; 280 | 281 | /* If we've read everything from the current slice, let the 282 | * read thread know that there is now more space available 283 | * and start reading from the next slice */ 284 | if (DATA(state)->offset >= INBUFFER(state).len) { 285 | INBUFFER(state).state = EMPTY; 286 | pthread_cond_signal(&DATA(state)->space_avail); 287 | newbuffer = (newbuffer + 1) % max_buffers; 288 | DATA(state)->offset = 0; 289 | } 290 | 291 | pthread_mutex_unlock(&DATA(state)->mutex); 292 | 293 | DATA(state)->in_buffer = newbuffer; 294 | } 295 | return copied; 296 | } 297 | 298 | io_source_t thread_source = {"thread", thread_read, NULL, /* peek */ 299 | NULL, /* tell */ 300 | NULL, /* seek */ 301 | thread_close}; 302 | -------------------------------------------------------------------------------- /lib/ior-zlib.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2019 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | #include "config.h" 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include "wandio.h" 36 | 37 | /* Libwandio IO module implementing a zlib reader */ 38 | 39 | enum err_t { ERR_OK = 1, ERR_EOF = 0, ERR_ERROR = -1 }; 40 | 41 | struct zlib_t { 42 | /* bytef is what zlib uses for buffer pointers */ 43 | Bytef inbuff[WANDIO_BUFFER_SIZE]; 44 | z_stream strm; 45 | io_t *parent; 46 | int outoffset; 47 | enum err_t err; 48 | size_t sincelastend; 49 | }; 50 | 51 | extern io_source_t zlib_source; 52 | 53 | #define DATA(io) ((struct zlib_t *)((io)->data)) 54 | #define min(a, b) ((a) < (b) ? (a) : (b)) 55 | 56 | DLLEXPORT io_t *zlib_open(io_t *parent) { 57 | io_t *io; 58 | if (!parent) 59 | return NULL; 60 | io = malloc(sizeof(io_t)); 61 | io->source = &zlib_source; 62 | io->data = malloc(sizeof(struct zlib_t)); 63 | 64 | DATA(io)->parent = parent; 65 | 66 | DATA(io)->strm.next_in = NULL; 67 | DATA(io)->strm.avail_in = 0; 68 | DATA(io)->strm.next_out = NULL; 69 | DATA(io)->strm.avail_out = 0; 70 | DATA(io)->strm.zalloc = Z_NULL; 71 | DATA(io)->strm.zfree = Z_NULL; 72 | DATA(io)->strm.opaque = NULL; 73 | DATA(io)->err = ERR_OK; 74 | DATA(io)->sincelastend = 1; 75 | 76 | inflateInit2(&DATA(io)->strm, 15 | 32); 77 | 78 | return io; 79 | } 80 | 81 | static int64_t zlib_read(io_t *io, void *buffer, int64_t len) { 82 | if (DATA(io)->err == ERR_EOF) 83 | return 0; /* EOF */ 84 | if (DATA(io)->err == ERR_ERROR) { 85 | errno = EIO; 86 | return -1; /* ERROR! */ 87 | } 88 | 89 | DATA(io)->strm.avail_out = len; 90 | DATA(io)->strm.next_out = (Bytef *)buffer; 91 | 92 | while (DATA(io)->err == ERR_OK && DATA(io)->strm.avail_out > 0) { 93 | while (DATA(io)->strm.avail_in <= 0) { 94 | int bytes_read = wandio_read(DATA(io)->parent, 95 | (char *)DATA(io)->inbuff, 96 | sizeof(DATA(io)->inbuff)); 97 | if (bytes_read == 0) { 98 | /* If we get EOF immediately after a 99 | * Z_STREAM_END, then we assume we've reached 100 | * the end of the file. If there was data 101 | * between the Z_STREAM_END and the EOF, the 102 | * file has more likely been truncated. 103 | */ 104 | if (DATA(io)->sincelastend > 0) { 105 | fprintf(stderr, 106 | "Unexpected EOF while reading " 107 | "compressed file -- file is " 108 | "probably incomplete\n"); 109 | errno = EIO; 110 | DATA(io)->err = ERR_ERROR; 111 | return -1; 112 | } 113 | 114 | /* EOF */ 115 | if (DATA(io)->strm.avail_out == (uint32_t)len) { 116 | DATA(io)->err = ERR_EOF; 117 | return 0; 118 | } 119 | /* Return how much data we've managed to read so 120 | * far. */ 121 | return len - DATA(io)->strm.avail_out; 122 | } 123 | if (bytes_read < 0) { /* Error */ 124 | /* errno should be set */ 125 | DATA(io)->err = ERR_ERROR; 126 | /* Return how much data we managed to read ok */ 127 | if (DATA(io)->strm.avail_out != (uint32_t)len) { 128 | return len - DATA(io)->strm.avail_out; 129 | } 130 | /* Now return error */ 131 | return -1; 132 | } 133 | DATA(io)->strm.next_in = DATA(io)->inbuff; 134 | DATA(io)->strm.avail_in = bytes_read; 135 | DATA(io)->sincelastend += bytes_read; 136 | } 137 | /* Decompress some data into the output buffer */ 138 | int err = inflate(&DATA(io)->strm, 0); 139 | switch (err) { 140 | case Z_OK: 141 | DATA(io)->err = ERR_OK; 142 | break; 143 | case Z_STREAM_END: 144 | /* You would think that an "EOF" on the stream would 145 | * mean we'd want to pass on an EOF? Nope. Some tools 146 | * (*cough* corel *cough*) annoyingly close and reopen 147 | * the gzip stream leaving Z_STREAM_END mines for us to 148 | * find. 149 | */ 150 | inflateEnd(&DATA(io)->strm); 151 | inflateInit2(&DATA(io)->strm, 15 | 32); 152 | DATA(io)->err = ERR_OK; 153 | DATA(io)->sincelastend = 0; 154 | break; 155 | default: 156 | errno = EIO; 157 | DATA(io)->err = ERR_ERROR; 158 | } 159 | } 160 | /* Return the number of bytes decompressed */ 161 | return len - DATA(io)->strm.avail_out; 162 | } 163 | 164 | static void zlib_close(io_t *io) { 165 | inflateEnd(&DATA(io)->strm); 166 | wandio_destroy(DATA(io)->parent); 167 | free(io->data); 168 | free(io); 169 | } 170 | 171 | io_source_t zlib_source = {"zlib", zlib_read, NULL, /* peek */ 172 | NULL, /* tell */ 173 | NULL, /* seek */ 174 | zlib_close}; 175 | -------------------------------------------------------------------------------- /lib/iow-bzip.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2019 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | #include "config.h" 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include "wandio.h" 35 | 36 | /* Libwandio IO module implement a bzip writer */ 37 | 38 | enum err_t { ERR_OK = 1, ERR_EOF = 0, ERR_ERROR = -1 }; 39 | 40 | struct bzw_t { 41 | bz_stream strm; 42 | char outbuff[WANDIO_BUFFER_SIZE]; 43 | int inoffset; 44 | iow_t *child; 45 | enum err_t err; 46 | }; 47 | 48 | extern iow_source_t bz_wsource; 49 | 50 | #define DATA(iow) ((struct bzw_t *)((iow)->data)) 51 | #define min(a, b) ((a) < (b) ? (a) : (b)) 52 | 53 | DLLEXPORT iow_t *bz_wopen(iow_t *child, int compress_level) { 54 | iow_t *iow; 55 | if (!child) 56 | return NULL; 57 | iow = malloc(sizeof(iow_t)); 58 | iow->source = &bz_wsource; 59 | iow->data = malloc(sizeof(struct bzw_t)); 60 | 61 | DATA(iow)->child = child; 62 | 63 | DATA(iow)->strm.next_in = NULL; 64 | DATA(iow)->strm.avail_in = 0; 65 | DATA(iow)->strm.next_out = DATA(iow)->outbuff; 66 | DATA(iow)->strm.avail_out = sizeof(DATA(iow)->outbuff); 67 | DATA(iow)->strm.bzalloc = NULL; 68 | DATA(iow)->strm.bzfree = NULL; 69 | DATA(iow)->strm.opaque = NULL; 70 | DATA(iow)->err = ERR_OK; 71 | 72 | BZ2_bzCompressInit(&DATA(iow)->strm, compress_level, /* Block size */ 73 | 0, /* Verbosity */ 74 | 30); /* Work factor */ 75 | 76 | return iow; 77 | } 78 | 79 | static int64_t bz_wwrite(iow_t *iow, const char *buffer, int64_t len) { 80 | if (DATA(iow)->err == ERR_EOF) { 81 | return 0; /* EOF */ 82 | } 83 | if (DATA(iow)->err == ERR_ERROR) { 84 | return -1; /* ERROR! */ 85 | } 86 | 87 | DATA(iow)->strm.next_in = (char *)buffer; 88 | DATA(iow)->strm.avail_in = len; 89 | 90 | while (DATA(iow)->err == ERR_OK && DATA(iow)->strm.avail_in > 0) { 91 | while (DATA(iow)->strm.avail_out <= 0) { 92 | int bytes_written = 93 | wandio_wwrite(DATA(iow)->child, DATA(iow)->outbuff, 94 | sizeof(DATA(iow)->outbuff)); 95 | if (bytes_written <= 0) { /* Error */ 96 | DATA(iow)->err = ERR_ERROR; 97 | /* Return how much data we managed to write ok 98 | */ 99 | if (DATA(iow)->strm.avail_in != (uint32_t)len) { 100 | return len - DATA(iow)->strm.avail_in; 101 | } 102 | /* Now return error */ 103 | return -1; 104 | } 105 | DATA(iow)->strm.next_out = DATA(iow)->outbuff; 106 | DATA(iow)->strm.avail_out = sizeof(DATA(iow)->outbuff); 107 | } 108 | /* Decompress some data into the output buffer */ 109 | int err = BZ2_bzCompress(&DATA(iow)->strm, 0); 110 | switch (err) { 111 | case BZ_RUN_OK: 112 | case BZ_OK: 113 | DATA(iow)->err = ERR_OK; 114 | break; 115 | default: 116 | DATA(iow)->err = ERR_ERROR; 117 | break; 118 | } 119 | } 120 | /* Return the number of bytes compressed */ 121 | return len - DATA(iow)->strm.avail_in; 122 | } 123 | 124 | static int bz_wflush(iow_t *iow) { 125 | /* TODO implement this */ 126 | (void)iow; // silence compiler warning 127 | return -1; 128 | } 129 | 130 | static void bz_wclose(iow_t *iow) { 131 | while (BZ2_bzCompress(&DATA(iow)->strm, BZ_FINISH) == BZ_OK) { 132 | /* Need to flush the output buffer */ 133 | wandio_wwrite(DATA(iow)->child, DATA(iow)->outbuff, 134 | sizeof(DATA(iow)->outbuff) - 135 | DATA(iow)->strm.avail_out); 136 | DATA(iow)->strm.next_out = DATA(iow)->outbuff; 137 | DATA(iow)->strm.avail_out = sizeof(DATA(iow)->outbuff); 138 | } 139 | BZ2_bzCompressEnd(&DATA(iow)->strm); 140 | wandio_wwrite(DATA(iow)->child, DATA(iow)->outbuff, 141 | sizeof(DATA(iow)->outbuff) - DATA(iow)->strm.avail_out); 142 | wandio_wdestroy(DATA(iow)->child); 143 | free(iow->data); 144 | free(iow); 145 | } 146 | 147 | iow_source_t bz_wsource = {"bzw", bz_wwrite, bz_wflush, bz_wclose}; 148 | -------------------------------------------------------------------------------- /lib/iow-lz4.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of libwandio 3 | * 4 | * Copyright (c) 2007-2019 The University of Waikato, Hamilton, 5 | * New Zealand. 6 | * 7 | * Authors: Sergey Cherepanov 8 | * 9 | * All rights reserved. 10 | * 11 | * This code has been developed by the University of Waikato WAND 12 | * research group. For further information please see http://www.wand.net.nz/ 13 | * 14 | * libwandio is free software; you can redistribute it and/or modify 15 | * it under the terms of the GNU General Public License as published by 16 | * the Free Software Foundation; either version 2 of the License, or 17 | * (at your option) any later version. 18 | * 19 | * libwandio is distributed in the hope that it will be useful, 20 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 21 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 | * GNU General Public License for more details. 23 | * 24 | * You should have received a copy of the GNU General Public License 25 | * along with libwandio; if not, write to the Free Software 26 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 27 | * 28 | */ 29 | 30 | #include "config.h" 31 | #include 32 | #include 33 | #if HAVE_LIBLZ4F 34 | #include 35 | #endif 36 | #include 37 | #include 38 | #include "wandio.h" 39 | 40 | enum err_t { ERR_OK = 1, ERR_EOF = 0, ERR_ERROR = -1 }; 41 | 42 | struct lz4w_t { 43 | iow_t *child; 44 | enum err_t err; 45 | #if HAVE_LIBLZ4F 46 | LZ4F_compressionContext_t cctx; 47 | LZ4F_preferences_t prefs; 48 | #endif 49 | char outbuf[1024 * 1024 * 2]; 50 | int outbuf_size_max; 51 | int outbuf_index; 52 | }; 53 | 54 | #define DATA(iow) ((struct lz4w_t *)((iow)->data)) 55 | extern iow_source_t lz4_wsource; 56 | 57 | DLLEXPORT iow_t *lz4_wopen(iow_t *child, int compress_level) { 58 | iow_t *iow; 59 | if (!child) { 60 | return NULL; 61 | } 62 | iow = malloc(sizeof(iow_t)); 63 | /* flush rely on sufficiently large output buffer and not making loops 64 | */ 65 | assert(sizeof(DATA(iow)->outbuf) >= 1024 * 1024); 66 | iow->source = &lz4_wsource; 67 | iow->data = malloc(sizeof(struct lz4w_t)); 68 | memset(DATA(iow), 0, sizeof(struct lz4w_t)); 69 | DATA(iow)->child = child; 70 | DATA(iow)->err = ERR_OK; 71 | DATA(iow)->outbuf_size_max = sizeof(DATA(iow)->outbuf) / 2; 72 | DATA(iow)->outbuf_index = 0; 73 | 74 | #if HAVE_LIBLZ4F 75 | memset(&(DATA(iow)->prefs), 0, sizeof(LZ4F_preferences_t)); 76 | DATA(iow)->prefs.compressionLevel = compress_level; 77 | LZ4F_errorCode_t result = 78 | LZ4F_createCompressionContext(&DATA(iow)->cctx, LZ4F_VERSION); 79 | if (LZ4F_isError(result)) { 80 | free(iow->data); 81 | free(iow); 82 | fprintf(stderr, "lz4 write open failed %s\n", 83 | LZ4F_getErrorName(result)); 84 | return NULL; 85 | } 86 | 87 | result = 88 | LZ4F_compressBegin(DATA(iow)->cctx, DATA(iow)->outbuf, 89 | sizeof(DATA(iow)->outbuf), &(DATA(iow)->prefs)); 90 | if (LZ4F_isError(result)) { 91 | LZ4F_freeCompressionContext(DATA(iow)->cctx); 92 | free(iow->data); 93 | free(iow); 94 | fprintf(stderr, "lz4 write open failed %s\n", 95 | LZ4F_getErrorName(result)); 96 | return NULL; 97 | } 98 | DATA(iow)->outbuf_index = result; 99 | #endif 100 | return iow; 101 | } 102 | 103 | static int64_t lz4_wwrite(iow_t *iow, const char *buffer, int64_t len) { 104 | if (DATA(iow)->err == ERR_EOF) { 105 | return 0; /* EOF */ 106 | } 107 | if (DATA(iow)->err == ERR_ERROR) { 108 | return -1; /* ERROR! */ 109 | } 110 | 111 | if (len <= 0) { 112 | return 0; 113 | } 114 | /* Lz4 does not have a streaming comrpession */ 115 | /* We need to handle arbitrarily large input buffer, by limiting to 1MB 116 | * pieces */ 117 | int inbuf_len; /* piece len */ 118 | int inbuf_index = 119 | 0; /* index is from begin of buffer, can spawn several pieces */ 120 | 121 | while (true) { 122 | if (len - inbuf_index >= DATA(iow)->outbuf_size_max) { 123 | inbuf_len = DATA(iow)->outbuf_size_max; 124 | } else { 125 | inbuf_len = len - inbuf_index; 126 | } 127 | 128 | size_t upper_bound = 0, result = 0; 129 | #if HAVE_LIBLZ4F 130 | upper_bound = 131 | LZ4F_compressBound(inbuf_len, &(DATA(iow)->prefs)); 132 | #endif 133 | if ((size_t)upper_bound > 134 | sizeof(DATA(iow)->outbuf) - DATA(iow)->outbuf_index) { 135 | int bytes_written = 136 | wandio_wwrite(DATA(iow)->child, DATA(iow)->outbuf, 137 | DATA(iow)->outbuf_index); 138 | if (bytes_written <= 0) { 139 | DATA(iow)->err = ERR_ERROR; 140 | return -1; 141 | } 142 | assert(bytes_written == DATA(iow)->outbuf_index); 143 | DATA(iow)->outbuf_index = 0; 144 | } 145 | 146 | if (upper_bound > (int64_t)sizeof(DATA(iow)->outbuf)) { 147 | fprintf(stderr, "invalid upper bound calculated by lz4 library: %zu\n", upper_bound); 148 | errno = EINVAL; 149 | return -1; 150 | } 151 | 152 | #if HAVE_LIBLZ4F 153 | result = LZ4F_compressUpdate( 154 | DATA(iow)->cctx, 155 | DATA(iow)->outbuf + DATA(iow)->outbuf_index, 156 | sizeof(DATA(iow)->outbuf) - DATA(iow)->outbuf_index, 157 | buffer + inbuf_index, inbuf_len, NULL); 158 | if (LZ4F_isError(result)) { 159 | fprintf(stderr, "lz4 compress error %ld %s\n", result, 160 | LZ4F_getErrorName(result)); 161 | errno = EIO; 162 | return -1; 163 | } 164 | #endif 165 | DATA(iow)->outbuf_index += result; 166 | inbuf_index += inbuf_len; 167 | if (inbuf_index >= len) { 168 | return inbuf_len; 169 | } 170 | } 171 | } 172 | 173 | static int lz4_wflush(iow_t *iow) { 174 | size_t result = 0; 175 | int64_t bytes_written = wandio_wwrite( 176 | DATA(iow)->child, DATA(iow)->outbuf, DATA(iow)->outbuf_index); 177 | if (bytes_written < 0) { 178 | fprintf(stderr, "lz4 compress flush error\n"); 179 | DATA(iow)->err = ERR_ERROR; 180 | return -1; 181 | } 182 | assert(bytes_written == DATA(iow)->outbuf_index); 183 | DATA(iow)->outbuf_index = 0; 184 | #if HAVE_LIBLZ4F 185 | result = LZ4F_flush(DATA(iow)->cctx, DATA(iow)->outbuf, 186 | sizeof(DATA(iow)->outbuf), NULL); 187 | if (LZ4F_isError(result)) { 188 | fprintf(stderr, "lz4 compress flush error %ld %s\n", result, 189 | LZ4F_getErrorName(result)); 190 | errno = EIO; 191 | return -1; 192 | } 193 | DATA(iow)->outbuf_index = 0; 194 | #endif 195 | if (result > 0) { 196 | bytes_written = 197 | wandio_wwrite(DATA(iow)->child, DATA(iow)->outbuf, result); 198 | if (bytes_written <= 0) { 199 | fprintf(stderr, "lz4 compress flush error\n"); 200 | DATA(iow)->err = ERR_ERROR; 201 | return -1; 202 | } 203 | assert(bytes_written == (int64_t)result); 204 | } 205 | int64_t res = wandio_wflush(DATA(iow)->child); 206 | if (res < 0) { 207 | DATA(iow)->err = ERR_ERROR; 208 | errno = EIO; 209 | return res; 210 | } 211 | return 0; 212 | } 213 | 214 | static void lz4_wclose(iow_t *iow) { 215 | lz4_wflush(iow); 216 | 217 | #if HAVE_LIBLZ4F 218 | size_t result = 0; 219 | result = LZ4F_compressEnd(DATA(iow)->cctx, DATA(iow)->outbuf, 220 | sizeof(DATA(iow)->outbuf), NULL); 221 | if (LZ4F_isError(result)) { 222 | fprintf(stderr, "lz4 compress close error %ld %s\n", result, 223 | LZ4F_getErrorName(result)); 224 | errno = EIO; 225 | } 226 | int64_t bytes_written = 227 | wandio_wwrite(DATA(iow)->child, DATA(iow)->outbuf, result); 228 | if (bytes_written <= 0) { 229 | fprintf(stderr, "lz4 compress close write error\n"); 230 | errno = EIO; 231 | } 232 | #endif 233 | wandio_wdestroy(DATA(iow)->child); 234 | #if HAVE_LIBLZ4F 235 | LZ4F_freeCompressionContext(DATA(iow)->cctx); 236 | #endif 237 | free(iow->data); 238 | free(iow); 239 | } 240 | 241 | iow_source_t lz4_wsource = {"lz4w", lz4_wwrite, lz4_wflush, lz4_wclose}; 242 | -------------------------------------------------------------------------------- /lib/iow-lzma.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2019 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | #include "config.h" 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include "wandio.h" 36 | 37 | /* Libwandio IO module implementing an lzma writer */ 38 | 39 | enum err_t { ERR_OK = 1, ERR_EOF = 0, ERR_ERROR = -1 }; 40 | 41 | struct lzmaw_t { 42 | lzma_stream strm; 43 | uint8_t outbuff[WANDIO_BUFFER_SIZE]; 44 | iow_t *child; 45 | enum err_t err; 46 | int inoffset; 47 | }; 48 | 49 | extern iow_source_t lzma_wsource; 50 | 51 | #define DATA(iow) ((struct lzmaw_t *)((iow)->data)) 52 | #define min(a, b) ((a) < (b) ? (a) : (b)) 53 | 54 | DLLEXPORT iow_t *lzma_wopen(iow_t *child, int compress_level) { 55 | iow_t *iow; 56 | if (!child) 57 | return NULL; 58 | iow = malloc(sizeof(iow_t)); 59 | iow->source = &lzma_wsource; 60 | iow->data = malloc(sizeof(struct lzmaw_t)); 61 | 62 | DATA(iow)->child = child; 63 | 64 | memset(&DATA(iow)->strm, 0, sizeof(DATA(iow)->strm)); 65 | DATA(iow)->strm.next_out = DATA(iow)->outbuff; 66 | DATA(iow)->strm.avail_out = sizeof(DATA(iow)->outbuff); 67 | DATA(iow)->err = ERR_OK; 68 | 69 | if (lzma_easy_encoder(&DATA(iow)->strm, compress_level, 70 | LZMA_CHECK_CRC64) != LZMA_OK) { 71 | free(iow->data); 72 | free(iow); 73 | return NULL; 74 | } 75 | 76 | return iow; 77 | } 78 | 79 | static int64_t lzma_wwrite(iow_t *iow, const char *buffer, int64_t len) { 80 | if (DATA(iow)->err == ERR_EOF) { 81 | return 0; /* EOF */ 82 | } 83 | if (DATA(iow)->err == ERR_ERROR) { 84 | return -1; /* ERROR! */ 85 | } 86 | 87 | DATA(iow)->strm.next_in = (const uint8_t *)buffer; 88 | DATA(iow)->strm.avail_in = len; 89 | 90 | while (DATA(iow)->err == ERR_OK && DATA(iow)->strm.avail_in > 0) { 91 | /* Flush output data. */ 92 | while (DATA(iow)->strm.avail_out <= 0) { 93 | int bytes_written = 94 | wandio_wwrite(DATA(iow)->child, DATA(iow)->outbuff, 95 | sizeof(DATA(iow)->outbuff)); 96 | if (bytes_written <= 0) { /* Error */ 97 | DATA(iow)->err = ERR_ERROR; 98 | /* Return how much data we managed to write */ 99 | if (DATA(iow)->strm.avail_in != (uint32_t)len) { 100 | return len - DATA(iow)->strm.avail_in; 101 | } 102 | /* Now return error */ 103 | return -1; 104 | } 105 | DATA(iow)->strm.next_out = DATA(iow)->outbuff; 106 | DATA(iow)->strm.avail_out = sizeof(DATA(iow)->outbuff); 107 | } 108 | /* Decompress some data into the output buffer */ 109 | lzma_ret err = lzma_code(&DATA(iow)->strm, LZMA_RUN); 110 | switch (err) { 111 | case LZMA_OK: 112 | DATA(iow)->err = ERR_OK; 113 | break; 114 | default: 115 | DATA(iow)->err = ERR_ERROR; 116 | } 117 | } 118 | /* Return the number of bytes decompressed */ 119 | return len - DATA(iow)->strm.avail_in; 120 | } 121 | 122 | static int lzma_wflush(iow_t *iow) { 123 | /* TODO implement this */ 124 | (void)iow; // silence compiler warning 125 | return 0; 126 | } 127 | 128 | static void lzma_wclose(iow_t *iow) { 129 | lzma_ret res; 130 | while (1) { 131 | res = lzma_code(&DATA(iow)->strm, LZMA_FINISH); 132 | 133 | if (res == LZMA_STREAM_END) 134 | break; 135 | if (res != LZMA_OK) { 136 | fprintf(stderr, 137 | "Z_STREAM_ERROR while closing output\n"); 138 | break; 139 | } 140 | 141 | wandio_wwrite(DATA(iow)->child, (char *)DATA(iow)->outbuff, 142 | sizeof(DATA(iow)->outbuff) - 143 | DATA(iow)->strm.avail_out); 144 | DATA(iow)->strm.next_out = DATA(iow)->outbuff; 145 | DATA(iow)->strm.avail_out = sizeof(DATA(iow)->outbuff); 146 | } 147 | 148 | wandio_wwrite(DATA(iow)->child, (char *)DATA(iow)->outbuff, 149 | sizeof(DATA(iow)->outbuff) - DATA(iow)->strm.avail_out); 150 | lzma_end(&DATA(iow)->strm); 151 | wandio_wdestroy(DATA(iow)->child); 152 | free(iow->data); 153 | free(iow); 154 | } 155 | 156 | iow_source_t lzma_wsource = {"xz", lzma_wwrite, lzma_wflush, lzma_wclose}; 157 | -------------------------------------------------------------------------------- /lib/iow-qat.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2019 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | #include "config.h" 28 | #include "wandio.h" 29 | 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | #define DATA(iow) ((struct qatw_t *)((iow)->data)) 36 | 37 | extern iow_source_t qat_wsource; 38 | 39 | enum err_t { ERR_OK = 1, ERR_EOF = 0, ERR_ERROR = -1 }; 40 | 41 | struct qatw_t { 42 | QzSession_T sess; 43 | iow_t *child; 44 | unsigned char outbuff[WANDIO_BUFFER_SIZE * 10]; 45 | int64_t outused; 46 | enum err_t err; 47 | }; 48 | 49 | static void qat_perror(int errcode) { 50 | if (errcode >= 0) { 51 | return; 52 | } 53 | 54 | switch (errcode) { 55 | case QZ_FAIL: 56 | fprintf(stderr, "QATzip failed for some unspecified reason.\n"); 57 | break; 58 | case QZ_PARAMS: 59 | fprintf(stderr, "Invalid parameters provided to QATzip.\n"); 60 | break; 61 | case QZ_BUF_ERROR: 62 | fprintf(stderr, "QATzip buffer was too small.\n"); 63 | break; 64 | case QZ_DATA_ERROR: 65 | fprintf(stderr, "QATzip input data was corrupted.\n"); 66 | break; 67 | case QZ_NOSW_NO_HW: 68 | fprintf(stderr, "QAT HW not detected.\n"); 69 | break; 70 | case QZ_NOSW_NO_MDRV: 71 | fprintf(stderr, "QAT memory driver not detected.\n"); 72 | break; 73 | case QZ_NOSW_NO_INST_ATTACH: 74 | fprintf(stderr, "Unable to attach to QAT instance.\n"); 75 | break; 76 | case QZ_NOSW_LOW_MEM: 77 | fprintf(stderr, "Insufficient pinned memory for QAT.\n"); 78 | break; 79 | } 80 | } 81 | 82 | DLLEXPORT iow_t *qat_wopen(iow_t *child, int compress_level) { 83 | 84 | iow_t *iow; 85 | QzSessionParams_T params; 86 | int x; 87 | 88 | if (!child) { 89 | return NULL; 90 | } 91 | 92 | iow = (iow_t *)malloc(sizeof(iow_t)); 93 | 94 | iow->source = &(qat_wsource); 95 | iow->data = (struct qatw_t *)calloc(1, sizeof(struct qatw_t)); 96 | DATA(iow)->outused = 0; 97 | DATA(iow)->child = child; 98 | DATA(iow)->err = ERR_OK; 99 | 100 | if ((x = qzInit(&(DATA(iow)->sess), 0)) != QZ_OK) { 101 | qat_perror(x); 102 | free(iow->data); 103 | free(iow); 104 | return NULL; 105 | } 106 | 107 | params.huffman_hdr = QZ_DYNAMIC_HDR; 108 | params.direction = QZ_DIR_COMPRESS; 109 | params.data_fmt = QZ_DATA_FORMAT_DEFAULT; 110 | params.comp_lvl = compress_level; 111 | params.comp_algorithm = QZ_DEFLATE; 112 | params.poll_sleep = QZ_POLL_SLEEP_DEFAULT; 113 | params.max_forks = QZ_MAX_FORK_DEFAULT; 114 | params.sw_backup = 0; 115 | params.hw_buff_sz = QZ_HW_BUFF_SZ; 116 | params.strm_buff_sz = QZ_STRM_BUFF_SZ_DEFAULT; 117 | params.input_sz_thrshold = QZ_COMP_THRESHOLD_DEFAULT; 118 | params.req_cnt_thrshold = 4; 119 | params.wait_cnt_thrshold = QZ_WAIT_CNT_THRESHOLD_DEFAULT; 120 | 121 | if ((x = qzSetupSession(&(DATA(iow)->sess), ¶ms)) != QZ_OK) { 122 | qat_perror(x); 123 | free(iow->data); 124 | free(iow); 125 | return NULL; 126 | } 127 | 128 | return iow; 129 | } 130 | 131 | static inline int64_t _qat_wwrite(iow_t *iow, const char *buffer, int64_t len, 132 | unsigned int last) { 133 | 134 | int64_t consumed = 0; 135 | int64_t spaceleft; 136 | unsigned int dst_len, src_len; 137 | unsigned char dummy[128]; 138 | int rc; 139 | 140 | while (DATA(iow)->err == ERR_OK && consumed < len) { 141 | 142 | spaceleft = sizeof(DATA(iow)->outbuff) - DATA(iow)->outused; 143 | src_len = (unsigned int)len; 144 | 145 | if (spaceleft < qzMaxCompressedLength(src_len)) { 146 | int written = 147 | wandio_wwrite(DATA(iow)->child, DATA(iow)->outbuff, 148 | DATA(iow)->outused); 149 | if (written <= 0) { 150 | DATA(iow)->err = ERR_ERROR; 151 | return -1; 152 | } 153 | DATA(iow)->outused = 0; 154 | spaceleft = sizeof(DATA(iow)->outbuff); 155 | } 156 | 157 | dst_len = (unsigned int)spaceleft; 158 | rc = qzCompress( 159 | &(DATA(iow)->sess), 160 | buffer != NULL ? (unsigned char *)buffer : dummy, &src_len, 161 | DATA(iow)->outbuff + DATA(iow)->outused, &dst_len, last); 162 | 163 | if (rc < 0) { 164 | DATA(iow)->err = ERR_ERROR; 165 | qat_perror(rc); 166 | return -1; 167 | } else { 168 | DATA(iow)->err = ERR_OK; 169 | } 170 | 171 | consumed += src_len; 172 | DATA(iow)->outused += dst_len; 173 | } 174 | 175 | return len - consumed; 176 | } 177 | 178 | static int64_t qat_wwrite(iow_t *iow, const char *buffer, int64_t len) { 179 | return _qat_wwrite(iow, buffer, len, 0); 180 | } 181 | 182 | static int qat_wflush(iow_t *iow) { 183 | 184 | int rc; 185 | 186 | rc = _qat_wwrite(iow, NULL, 0, 1); 187 | if (DATA(iow)->err == ERR_ERROR) { 188 | return -1; 189 | } 190 | 191 | rc = wandio_wwrite(DATA(iow)->child, DATA(iow)->outbuff, 192 | DATA(iow)->outused); 193 | if (rc < 0) { 194 | DATA(iow)->err = ERR_ERROR; 195 | return rc; 196 | } 197 | 198 | if ((rc = wandio_wflush(DATA(iow)->child)) < 0) { 199 | DATA(iow)->err = ERR_ERROR; 200 | return rc; 201 | } 202 | 203 | DATA(iow)->outused = 0; 204 | 205 | return ERR_OK; 206 | } 207 | 208 | static void qat_wclose(iow_t *iow) { 209 | 210 | int rc; 211 | 212 | rc = qat_wflush(iow); 213 | rc = qzTeardownSession(&(DATA(iow)->sess)); 214 | rc = qzClose(&(DATA(iow)->sess)); 215 | 216 | wandio_wdestroy(DATA(iow)->child); 217 | free(iow->data); 218 | free(iow); 219 | } 220 | 221 | iow_source_t qat_wsource = {"qatw", qat_wwrite, qat_wflush, qat_wclose}; 222 | -------------------------------------------------------------------------------- /lib/iow-stdio.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2019 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | #define _GNU_SOURCE 1 28 | #include "config.h" 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include "wandio.h" 38 | #include "wandio_internal.h" 39 | 40 | /* Libwandio IO module implementing a standard IO writer, i.e. no decompression 41 | */ 42 | 43 | enum { MIN_WRITE_SIZE = 4096 }; 44 | 45 | struct stdiow_t { 46 | char buffer[MIN_WRITE_SIZE]; 47 | int offset; 48 | int fd; 49 | }; 50 | 51 | extern iow_source_t stdio_wsource; 52 | 53 | #define DATA(iow) ((struct stdiow_t *)((iow)->data)) 54 | 55 | static int safe_open(const char *filename, int flags) { 56 | int fd = -1; 57 | uid_t userid = 0; 58 | gid_t groupid = 0; 59 | char *sudoenv = NULL; 60 | 61 | /* Try opening with O_DIRECT */ 62 | #ifdef O_DIRECT 63 | fd = open(filename, 64 | flags | O_WRONLY | O_CREAT | O_TRUNC | 65 | (force_directio_write ? O_DIRECT : 0), 66 | 0666); 67 | #endif 68 | /* If that failed (or we don't support O_DIRECT) try opening without */ 69 | if (fd == -1) { 70 | fd = open(filename, flags | O_WRONLY | O_CREAT | O_TRUNC, 0666); 71 | } 72 | 73 | if (fd == -1) 74 | return fd; 75 | 76 | /* If we're running via sudo, we want to write files owned by the 77 | * original user rather than root. 78 | * 79 | * TODO: make this some sort of config option */ 80 | if (getuid() == 0) { 81 | sudoenv = getenv("SUDO_UID"); 82 | if (sudoenv != NULL) { 83 | userid = strtol(sudoenv, NULL, 10); 84 | } 85 | sudoenv = getenv("SUDO_GID"); 86 | if (sudoenv != NULL) { 87 | groupid = strtol(sudoenv, NULL, 10); 88 | } 89 | 90 | if (userid != 0 && fchown(fd, userid, groupid) == -1) { 91 | perror("fchown"); 92 | return -1; 93 | } 94 | } 95 | 96 | return fd; 97 | } 98 | 99 | DLLEXPORT iow_t *stdio_wopen(const char *filename, int flags) { 100 | iow_t *iow = malloc(sizeof(iow_t)); 101 | iow->source = &stdio_wsource; 102 | iow->data = malloc(sizeof(struct stdiow_t)); 103 | 104 | if (strcmp(filename, "-") == 0) 105 | DATA(iow)->fd = 1; /* STDOUT */ 106 | else { 107 | DATA(iow)->fd = safe_open(filename, flags); 108 | } 109 | 110 | if (DATA(iow)->fd == -1) { 111 | free(iow); 112 | return NULL; 113 | } 114 | 115 | DATA(iow)->offset = 0; 116 | 117 | return iow; 118 | } 119 | 120 | #define min(a, b) ((a) < (b) ? (a) : (b)) 121 | #define max(a, b) ((a) > (b) ? (a) : (b)) 122 | /* Round A Down to the nearest multiple of B */ 123 | #define rounddown(a, b) ((a)-((a)%b)) 124 | 125 | /* When doing directio (O_DIRECT) we need to make sure that we write multiples 126 | * of MIN_WRITE_SIZE. So we accumulate data into DATA(iow)->buffer, and write it 127 | * out when we get at least MIN_WRITE_SIZE. 128 | * 129 | * Since most writes are likely to be larger than MIN_WRITE_SIZE optimise for 130 | * that case. 131 | */ 132 | static int64_t stdio_wwrite(iow_t *iow, const char *buffer, int64_t len) { 133 | int towrite = len; 134 | /* Round down size to the nearest multiple of MIN_WRITE_SIZE */ 135 | 136 | assert(towrite >= 0); 137 | 138 | while (DATA(iow)->offset + towrite >= MIN_WRITE_SIZE) { 139 | int err; 140 | struct iovec iov[2]; 141 | int total = (DATA(iow)->offset + towrite); 142 | int amount; 143 | int count = 0; 144 | /* Round down to the nearest multiple */ 145 | total = total - (total % MIN_WRITE_SIZE); 146 | amount = total; 147 | if (DATA(iow)->offset) { 148 | iov[count].iov_base = DATA(iow)->buffer; 149 | iov[count].iov_len = min(DATA(iow)->offset, amount); 150 | amount -= iov[count].iov_len; 151 | ++count; 152 | } 153 | /* How much to write from this buffer? */ 154 | if (towrite) { 155 | iov[count].iov_base = (void *) 156 | buffer; /* cast away constness, which is safe 157 | * here 158 | */ 159 | iov[count].iov_len = amount; 160 | amount -= iov[count].iov_len; 161 | ++count; 162 | } 163 | assert(amount == 0); 164 | err = writev(DATA(iow)->fd, iov, count); 165 | if (err == -1) 166 | return -1; 167 | 168 | /* Drop off "err" bytes from the beginning of the buffers */ 169 | amount = min(DATA(iow)->offset, 170 | err); /* How much we took out of the buffer */ 171 | memmove(DATA(iow)->buffer, DATA(iow)->buffer + amount, 172 | DATA(iow)->offset - amount); 173 | DATA(iow)->offset -= amount; 174 | 175 | err -= amount; /* How much was written */ 176 | 177 | assert(err <= towrite); 178 | 179 | buffer += err; 180 | towrite -= err; 181 | 182 | assert(DATA(iow)->offset == 0); 183 | } 184 | 185 | /* Make sure we're not going to overflow the buffer. The above writev 186 | * should assure that this is true 187 | */ 188 | assert(DATA(iow)->offset + towrite <= MIN_WRITE_SIZE); 189 | assert(towrite >= 0); 190 | 191 | if (towrite > 0) { 192 | /* Copy the remainder into the buffer to write next time. */ 193 | memcpy(DATA(iow)->buffer + DATA(iow)->offset, buffer, towrite); 194 | DATA(iow)->offset += towrite; 195 | } 196 | 197 | return len; 198 | } 199 | 200 | static int stdio_wflush(iow_t *iow) { 201 | 202 | int err; 203 | /* Now, there might be some non multiple of the direct filesize left 204 | * over, if so turn off O_DIRECT and write the final chunk. 205 | */ 206 | #ifdef O_DIRECT 207 | err = fcntl(DATA(iow)->fd, F_GETFL); 208 | if (err != -1 && (err & O_DIRECT) != 0) { 209 | fcntl(DATA(iow)->fd, F_SETFL, err & ~O_DIRECT); 210 | } else if (err < 0) { 211 | return err; 212 | } 213 | #endif 214 | err = write(DATA(iow)->fd, DATA(iow)->buffer, DATA(iow)->offset); 215 | DATA(iow)->offset = 0; 216 | return err; 217 | } 218 | 219 | static void stdio_wclose(iow_t *iow) { 220 | stdio_wflush(iow); 221 | close(DATA(iow)->fd); 222 | free(iow->data); 223 | free(iow); 224 | } 225 | 226 | iow_source_t stdio_wsource = {"stdiow", stdio_wwrite, stdio_wflush, 227 | stdio_wclose}; 228 | -------------------------------------------------------------------------------- /lib/iow-thread.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2019 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | #include "config.h" 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include "wandio.h" 36 | #include "wandio_internal.h" 37 | #ifdef HAVE_SYS_PRCTL_H 38 | #include 39 | #endif 40 | 41 | /* Libwandio IO module implementing a threaded writer. 42 | * 43 | * This module enables another IO writer, called the "child", to perform its 44 | * writing using a separate thread. The main thread writes data into a series 45 | * of 1MB buffers. Meanwhile, the writing thread writes out of these buffers 46 | * using the callback for the child reader. pthread conditions are used to 47 | * communicate between the two threads, e.g. when there are buffers available 48 | * for the main thread to copy data into or when there is data available for 49 | * the write thread to write. 50 | */ 51 | 52 | #define BUFFERS 5 53 | 54 | extern iow_source_t thread_wsource; 55 | 56 | /* This structure defines a single buffer or "slice" */ 57 | struct buffer_t { 58 | char buffer[WANDIO_BUFFER_SIZE]; /* The buffer itself */ 59 | int len; /* The size of the buffer */ 60 | enum { EMPTY = 0, FULL = 1 } state; /* Is the buffer in use? */ 61 | bool flush; 62 | }; 63 | 64 | struct state_t { 65 | /* The collection of buffers (or slices) */ 66 | struct buffer_t buffer[BUFFERS]; 67 | /* The write offset into the current buffer */ 68 | int64_t offset; 69 | /* The writing thread */ 70 | pthread_t consumer; 71 | /* The child writer */ 72 | iow_t *iow; 73 | /* Indicates that there is data in one of the buffers */ 74 | pthread_cond_t data_ready; 75 | /* Indicates that there is a free buffer to write into */ 76 | pthread_cond_t space_avail; 77 | /* The mutex for the write buffers */ 78 | pthread_mutex_t mutex; 79 | /* The index of the buffer to write into next */ 80 | int out_buffer; 81 | /* Indicates whether the main thread is concluding */ 82 | bool closing; 83 | }; 84 | 85 | #define DATA(x) ((struct state_t *)((x)->data)) 86 | #define OUTBUFFER(x) (DATA(x)->buffer[DATA(x)->out_buffer]) 87 | #define min(a, b) ((a) < (b) ? (a) : (b)) 88 | 89 | /* The writing thread */ 90 | static void *thread_consumer(void *userdata) { 91 | int buffer = 0; 92 | bool running = true; 93 | iow_t *state = (iow_t *)userdata; 94 | 95 | #ifdef PR_SET_NAME 96 | char namebuf[17]; 97 | if (prctl(PR_GET_NAME, namebuf, 0, 0, 0) == 0) { 98 | namebuf[16] = '\0'; /* Make sure it's NUL terminated */ 99 | /* If the filename is too long, overwrite the last few bytes */ 100 | if (strlen(namebuf) > 9) { 101 | strcpy(namebuf + 10, "[iow]"); 102 | } else { 103 | strncat(namebuf, " [iow]", 16); 104 | } 105 | prctl(PR_SET_NAME, namebuf, 0, 0, 0); 106 | } 107 | #endif 108 | 109 | pthread_mutex_lock(&DATA(state)->mutex); 110 | do { 111 | /* Wait for data that we can write */ 112 | while (DATA(state)->buffer[buffer].state == EMPTY) { 113 | /* Unless, of course, the program is over! */ 114 | if (DATA(state)->closing) 115 | break; 116 | pthread_cond_wait(&DATA(state)->data_ready, 117 | &DATA(state)->mutex); 118 | } 119 | /* Empty the buffer using the child writer */ 120 | pthread_mutex_unlock(&DATA(state)->mutex); 121 | if (DATA(state)->buffer[buffer].len > 0) { 122 | wandio_wwrite(DATA(state)->iow, 123 | DATA(state)->buffer[buffer].buffer, 124 | DATA(state)->buffer[buffer].len); 125 | } 126 | if (DATA(state)->buffer[buffer].flush) { 127 | wandio_wflush(DATA(state)->iow); 128 | } 129 | pthread_mutex_lock(&DATA(state)->mutex); 130 | 131 | /* If we've not reached the end of the file keep going */ 132 | running = (DATA(state)->buffer[buffer].len > 0); 133 | DATA(state)->buffer[buffer].len = 0; 134 | DATA(state)->buffer[buffer].state = EMPTY; 135 | DATA(state)->buffer[buffer].flush = false; 136 | 137 | /* Signal that we've freed up another buffer for the main 138 | * thread to copy data into */ 139 | pthread_cond_signal(&DATA(state)->space_avail); 140 | 141 | /* Move on to the next buffer */ 142 | buffer = (buffer + 1) % BUFFERS; 143 | 144 | } while (running); 145 | 146 | /* If we reach here, it's all over so start tidying up */ 147 | wandio_wdestroy(DATA(state)->iow); 148 | 149 | pthread_mutex_unlock(&DATA(state)->mutex); 150 | return NULL; 151 | } 152 | 153 | DLLEXPORT iow_t *thread_wopen(iow_t *child) { 154 | iow_t *state; 155 | 156 | if (!child) { 157 | return NULL; 158 | } 159 | 160 | state = malloc(sizeof(iow_t)); 161 | state->data = calloc(1, sizeof(struct state_t)); 162 | state->source = &thread_wsource; 163 | 164 | DATA(state)->out_buffer = 0; 165 | DATA(state)->offset = 0; 166 | pthread_mutex_init(&DATA(state)->mutex, NULL); 167 | pthread_cond_init(&DATA(state)->data_ready, NULL); 168 | pthread_cond_init(&DATA(state)->space_avail, NULL); 169 | 170 | DATA(state)->iow = child; 171 | DATA(state)->closing = false; 172 | 173 | /* Start the writer thread */ 174 | pthread_create(&DATA(state)->consumer, NULL, thread_consumer, state); 175 | 176 | return state; 177 | } 178 | 179 | static int64_t thread_wwrite(iow_t *state, const char *buffer, int64_t len) { 180 | int slice; 181 | int copied = 0; 182 | int newbuffer; 183 | 184 | pthread_mutex_lock(&DATA(state)->mutex); 185 | while (len > 0) { 186 | 187 | /* Wait for there to be space available for us to write into */ 188 | while (OUTBUFFER(state).state == FULL) { 189 | write_waits++; 190 | pthread_cond_wait(&DATA(state)->space_avail, 191 | &DATA(state)->mutex); 192 | } 193 | 194 | /* Copy out of our main buffer into the next available slice */ 195 | slice = min((int64_t)sizeof(OUTBUFFER(state).buffer) - 196 | DATA(state)->offset, 197 | len); 198 | 199 | pthread_mutex_unlock(&DATA(state)->mutex); 200 | memcpy(OUTBUFFER(state).buffer + DATA(state)->offset, buffer, 201 | slice); 202 | pthread_mutex_lock(&DATA(state)->mutex); 203 | 204 | DATA(state)->offset += slice; 205 | OUTBUFFER(state).len += slice; 206 | 207 | buffer += slice; 208 | len -= slice; 209 | copied += slice; 210 | newbuffer = DATA(state)->out_buffer; 211 | 212 | /* If we've filled a buffer, move on to the next one and 213 | * signal to the write thread that there is something for it 214 | * to do */ 215 | if (DATA(state)->offset >= 216 | (int64_t)sizeof(OUTBUFFER(state).buffer)) { 217 | OUTBUFFER(state).state = FULL; 218 | OUTBUFFER(state).flush = false; 219 | pthread_cond_signal(&DATA(state)->data_ready); 220 | DATA(state)->offset = 0; 221 | newbuffer = (newbuffer + 1) % BUFFERS; 222 | } 223 | 224 | DATA(state)->out_buffer = newbuffer; 225 | } 226 | 227 | pthread_mutex_unlock(&DATA(state)->mutex); 228 | return copied; 229 | } 230 | 231 | static int thread_wflush(iow_t *iow) { 232 | int64_t flushed = 0; 233 | pthread_mutex_lock(&DATA(iow)->mutex); 234 | if (DATA(iow)->offset > 0) { 235 | flushed = DATA(iow)->offset; 236 | OUTBUFFER(iow).state = FULL; 237 | OUTBUFFER(iow).flush = true; 238 | pthread_cond_signal(&DATA(iow)->data_ready); 239 | DATA(iow)->offset = 0; 240 | DATA(iow)->out_buffer = (DATA(iow)->out_buffer + 1) % BUFFERS; 241 | } 242 | 243 | pthread_mutex_unlock(&DATA(iow)->mutex); 244 | return (int)flushed; 245 | } 246 | 247 | static void thread_wclose(iow_t *iow) { 248 | pthread_mutex_lock(&DATA(iow)->mutex); 249 | DATA(iow)->closing = true; 250 | pthread_cond_signal(&DATA(iow)->data_ready); 251 | pthread_mutex_unlock(&DATA(iow)->mutex); 252 | pthread_join(DATA(iow)->consumer, NULL); 253 | 254 | pthread_mutex_destroy(&DATA(iow)->mutex); 255 | pthread_cond_destroy(&DATA(iow)->data_ready); 256 | pthread_cond_destroy(&DATA(iow)->space_avail); 257 | 258 | free(iow->data); 259 | free(iow); 260 | } 261 | 262 | iow_source_t thread_wsource = {"threadw", thread_wwrite, thread_wflush, 263 | thread_wclose}; 264 | -------------------------------------------------------------------------------- /lib/iow-zlib.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2019 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | #include "config.h" 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include "wandio.h" 36 | 37 | /* Libwandio IO module implementing a zlib writer */ 38 | 39 | enum err_t { ERR_OK = 1, ERR_EOF = 0, ERR_ERROR = -1 }; 40 | 41 | struct zlibw_t { 42 | z_stream strm; 43 | Bytef outbuff[WANDIO_BUFFER_SIZE]; 44 | iow_t *child; 45 | enum err_t err; 46 | int inoffset; 47 | }; 48 | 49 | extern iow_source_t zlib_wsource; 50 | 51 | #define DATA(iow) ((struct zlibw_t *)((iow)->data)) 52 | #define min(a, b) ((a) < (b) ? (a) : (b)) 53 | 54 | DLLEXPORT iow_t *zlib_wopen(iow_t *child, int compress_level) { 55 | iow_t *iow; 56 | if (!child) 57 | return NULL; 58 | iow = malloc(sizeof(iow_t)); 59 | iow->source = &zlib_wsource; 60 | iow->data = malloc(sizeof(struct zlibw_t)); 61 | 62 | DATA(iow)->child = child; 63 | 64 | DATA(iow)->strm.next_in = NULL; 65 | DATA(iow)->strm.avail_in = 0; 66 | DATA(iow)->strm.next_out = DATA(iow)->outbuff; 67 | DATA(iow)->strm.avail_out = sizeof(DATA(iow)->outbuff); 68 | DATA(iow)->strm.zalloc = Z_NULL; 69 | DATA(iow)->strm.zfree = Z_NULL; 70 | DATA(iow)->strm.opaque = NULL; 71 | DATA(iow)->err = ERR_OK; 72 | 73 | deflateInit2(&DATA(iow)->strm, compress_level, /* Level */ 74 | Z_DEFLATED, /* Method */ 75 | 15 | 16, /* 15 bits of windowsize, 16 == use gzip header */ 76 | 9, /* Use maximum (fastest) amount of memory usage */ 77 | Z_DEFAULT_STRATEGY); 78 | 79 | return iow; 80 | } 81 | 82 | static int64_t zlib_wwrite(iow_t *iow, const char *buffer, int64_t len) { 83 | if (DATA(iow)->err == ERR_EOF) { 84 | return 0; /* EOF */ 85 | } 86 | if (DATA(iow)->err == ERR_ERROR) { 87 | return -1; /* ERROR! */ 88 | } 89 | 90 | DATA(iow)->strm.next_in = 91 | (Bytef *)buffer; /* This casts away const, but it's really const 92 | * anyway 93 | */ 94 | DATA(iow)->strm.avail_in = len; 95 | 96 | while (DATA(iow)->err == ERR_OK && DATA(iow)->strm.avail_in > 0) { 97 | while (DATA(iow)->strm.avail_out <= 0) { 98 | int bytes_written = wandio_wwrite( 99 | DATA(iow)->child, (char *)DATA(iow)->outbuff, 100 | sizeof(DATA(iow)->outbuff)); 101 | if (bytes_written <= 0) { /* Error */ 102 | DATA(iow)->err = ERR_ERROR; 103 | /* Return how much data we managed to write ok 104 | */ 105 | if (DATA(iow)->strm.avail_in != (uint32_t)len) { 106 | return len - DATA(iow)->strm.avail_in; 107 | } 108 | /* Now return error */ 109 | return -1; 110 | } 111 | DATA(iow)->strm.next_out = DATA(iow)->outbuff; 112 | DATA(iow)->strm.avail_out = sizeof(DATA(iow)->outbuff); 113 | } 114 | /* Decompress some data into the output buffer */ 115 | int err = deflate(&DATA(iow)->strm, Z_NO_FLUSH); 116 | switch (err) { 117 | case Z_OK: 118 | DATA(iow)->err = ERR_OK; 119 | break; 120 | default: 121 | DATA(iow)->err = ERR_ERROR; 122 | } 123 | } 124 | /* Return the number of bytes compressed */ 125 | return len - DATA(iow)->strm.avail_in; 126 | } 127 | 128 | static int zlib_wflush(iow_t *iow) { 129 | 130 | int res; 131 | 132 | res = deflate(&DATA(iow)->strm, Z_SYNC_FLUSH); 133 | if (res == Z_STREAM_ERROR) { 134 | fprintf(stderr, "Z_STREAM_ERROR while flushing output\n"); 135 | DATA(iow)->err = ERR_ERROR; 136 | return -1; 137 | } 138 | 139 | res = wandio_wwrite(DATA(iow)->child, (char *)DATA(iow)->outbuff, 140 | sizeof(DATA(iow)->outbuff) - 141 | DATA(iow)->strm.avail_out); 142 | if (res < 0) { 143 | DATA(iow)->err = ERR_ERROR; 144 | return res; 145 | } 146 | if ((res = wandio_wflush(DATA(iow)->child)) < 0) { 147 | DATA(iow)->err = ERR_ERROR; 148 | return res; 149 | } 150 | 151 | DATA(iow)->strm.next_out = DATA(iow)->outbuff; 152 | DATA(iow)->strm.avail_out = sizeof(DATA(iow)->outbuff); 153 | return res; 154 | } 155 | 156 | static void zlib_wclose(iow_t *iow) { 157 | int res; 158 | 159 | while (1) { 160 | res = deflate(&DATA(iow)->strm, Z_FINISH); 161 | 162 | if (res == Z_STREAM_END) 163 | break; 164 | if (res == Z_STREAM_ERROR) { 165 | fprintf(stderr, 166 | "Z_STREAM_ERROR while closing output\n"); 167 | break; 168 | } 169 | 170 | wandio_wwrite(DATA(iow)->child, (char *)DATA(iow)->outbuff, 171 | sizeof(DATA(iow)->outbuff) - 172 | DATA(iow)->strm.avail_out); 173 | DATA(iow)->strm.next_out = DATA(iow)->outbuff; 174 | DATA(iow)->strm.avail_out = sizeof(DATA(iow)->outbuff); 175 | } 176 | 177 | deflateEnd(&DATA(iow)->strm); 178 | wandio_wwrite(DATA(iow)->child, (char *)DATA(iow)->outbuff, 179 | sizeof(DATA(iow)->outbuff) - DATA(iow)->strm.avail_out); 180 | wandio_wdestroy(DATA(iow)->child); 181 | free(iow->data); 182 | free(iow); 183 | } 184 | 185 | iow_source_t zlib_wsource = {"zlibw", zlib_wwrite, zlib_wflush, zlib_wclose}; 186 | -------------------------------------------------------------------------------- /lib/iow-zstd.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of libwandio 3 | * 4 | * Copyright (c) 2007-2019 The University of Waikato, Hamilton, 5 | * New Zealand. 6 | * 7 | * Authors: Robert Zeh, Sergey Cherepanov 8 | * 9 | * All rights reserved. 10 | * 11 | * This code has been developed by the University of Waikato WAND 12 | * research group. For further information please see http://www.wand.net.nz/ 13 | * 14 | * libwandio is free software; you can redistribute it and/or modify 15 | * it under the terms of the GNU General Public License as published by 16 | * the Free Software Foundation; either version 2 of the License, or 17 | * (at your option) any later version. 18 | * 19 | * libwandio is distributed in the hope that it will be useful, 20 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 21 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 | * GNU General Public License for more details. 23 | * 24 | * You should have received a copy of the GNU General Public License 25 | * along with libwandio; if not, write to the Free Software 26 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 27 | * 28 | */ 29 | 30 | #include "config.h" 31 | #include 32 | #include 33 | #include "wandio.h" 34 | 35 | enum err_t { ERR_OK = 1, ERR_EOF = 0, ERR_ERROR = -1 }; 36 | 37 | struct zstdw_t { 38 | iow_t *child; 39 | enum err_t err; 40 | ZSTD_CStream *stream; 41 | ZSTD_outBuffer output_buffer; 42 | ZSTD_inBuffer input_buffer; 43 | char outbuff[WANDIO_BUFFER_SIZE]; 44 | }; 45 | 46 | #define DATA(iow) ((struct zstdw_t *)((iow)->data)) 47 | extern iow_source_t zstd_wsource; 48 | 49 | DLLEXPORT iow_t *zstd_wopen(iow_t *child, int compress_level) { 50 | iow_t *iow; 51 | if (!child) 52 | return NULL; 53 | iow = malloc(sizeof(iow_t)); 54 | iow->source = &zstd_wsource; 55 | iow->data = malloc(sizeof(struct zstdw_t)); 56 | DATA(iow)->child = child; 57 | DATA(iow)->err = ERR_OK; 58 | DATA(iow)->stream = ZSTD_createCStream(); 59 | ZSTD_initCStream(DATA(iow)->stream, compress_level); 60 | return iow; 61 | } 62 | 63 | static int64_t zstd_wwrite(iow_t *iow, const char *buffer, int64_t len) { 64 | if (DATA(iow)->err == ERR_EOF) { 65 | return 0; /* EOF */ 66 | } 67 | if (DATA(iow)->err == ERR_ERROR) { 68 | return -1; /* ERROR! */ 69 | } 70 | 71 | if (len <= 0) { 72 | return 0; 73 | } 74 | 75 | DATA(iow)->input_buffer.src = buffer; 76 | DATA(iow)->input_buffer.size = len; 77 | DATA(iow)->input_buffer.pos = 0; 78 | 79 | while (DATA(iow)->input_buffer.pos < (size_t)len) { 80 | DATA(iow)->output_buffer.dst = DATA(iow)->outbuff; 81 | DATA(iow)->output_buffer.pos = 0; 82 | DATA(iow)->output_buffer.size = sizeof(DATA(iow)->outbuff); 83 | 84 | size_t return_code = ZSTD_compressStream( 85 | DATA(iow)->stream, &DATA(iow)->output_buffer, 86 | &DATA(iow)->input_buffer); 87 | if (ZSTD_isError(return_code)) { 88 | fprintf(stderr, "Problem compressing stream: %s\n", 89 | ZSTD_getErrorName(return_code)); 90 | DATA(iow)->err = ERR_ERROR; 91 | return -1; 92 | } 93 | int bytes_written = 94 | wandio_wwrite(DATA(iow)->child, DATA(iow)->outbuff, 95 | DATA(iow)->output_buffer.pos); 96 | if (bytes_written <= 0) { 97 | DATA(iow)->err = ERR_ERROR; 98 | return -1; 99 | } 100 | } 101 | return DATA(iow)->input_buffer.pos; 102 | } 103 | 104 | static int zstd_wflush(iow_t *iow) { 105 | /* TODO implement this */ 106 | (void)(iow); 107 | return 0; 108 | } 109 | 110 | static void zstd_wclose(iow_t *iow) { 111 | size_t result = 1; 112 | /* I'm not sure if this loop is exactly the right thing to do, 113 | but it is what happens in zstd's zstd/programs/fileio.c. */ 114 | while (result != 0) { 115 | DATA(iow)->output_buffer.pos = 0; 116 | result = ZSTD_endStream(DATA(iow)->stream, 117 | &DATA(iow)->output_buffer); 118 | 119 | if (ZSTD_isError(result)) { 120 | fprintf(stderr, "ZSTD error while closing output: %s\n", 121 | ZSTD_getErrorName(result)); 122 | return; 123 | } 124 | wandio_wwrite(DATA(iow)->child, DATA(iow)->outbuff, 125 | DATA(iow)->output_buffer.pos); 126 | } 127 | wandio_wdestroy(DATA(iow)->child); 128 | ZSTD_freeCStream(DATA(iow)->stream); 129 | free(iow->data); 130 | free(iow); 131 | } 132 | 133 | iow_source_t zstd_wsource = {"zstdw", zstd_wwrite, zstd_wflush, zstd_wclose}; 134 | -------------------------------------------------------------------------------- /lib/swift-support/.dirstamp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LibtraceTeam/wandio/529105dbedb369659c72e9f1c99ce2c1f969c683/lib/swift-support/.dirstamp -------------------------------------------------------------------------------- /lib/swift-support/jsmn.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Serge A. Zaitsev 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to deal 6 | * in the Software without restriction, including without limitation the rights 7 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | * copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | * THE SOFTWARE. 21 | */ 22 | 23 | #include "jsmn.h" 24 | 25 | /** 26 | * Allocates a fresh unused token from the token pull. 27 | */ 28 | static jsmntok_t *jsmn_alloc_token(jsmn_parser *parser, 29 | jsmntok_t *tokens, size_t num_tokens) { 30 | jsmntok_t *tok; 31 | if (parser->toknext >= num_tokens) { 32 | return NULL; 33 | } 34 | tok = &tokens[parser->toknext++]; 35 | tok->start = tok->end = -1; 36 | tok->size = 0; 37 | #ifdef JSMN_PARENT_LINKS 38 | tok->parent = -1; 39 | #endif 40 | return tok; 41 | } 42 | 43 | /** 44 | * Fills token type and boundaries. 45 | */ 46 | static void jsmn_fill_token(jsmntok_t *token, jsmntype_t type, 47 | int start, int end) { 48 | token->type = type; 49 | token->start = start; 50 | token->end = end; 51 | token->size = 0; 52 | } 53 | 54 | /** 55 | * Fills next available token with JSON primitive. 56 | */ 57 | static int jsmn_parse_primitive(jsmn_parser *parser, const char *js, 58 | size_t len, jsmntok_t *tokens, size_t num_tokens) { 59 | jsmntok_t *token; 60 | int start; 61 | 62 | start = parser->pos; 63 | 64 | for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { 65 | switch (js[parser->pos]) { 66 | #ifndef JSMN_STRICT 67 | /* In strict mode primitive must be followed by "," or "}" or "]" */ 68 | case ':': 69 | #endif 70 | case '\t' : case '\r' : case '\n' : case ' ' : 71 | case ',' : case ']' : case '}' : 72 | goto found; 73 | } 74 | if (js[parser->pos] < 32 || js[parser->pos] >= 127) { 75 | parser->pos = start; 76 | return JSMN_ERROR_INVAL; 77 | } 78 | } 79 | #ifdef JSMN_STRICT 80 | /* In strict mode primitive must be followed by a comma/object/array */ 81 | parser->pos = start; 82 | return JSMN_ERROR_PART; 83 | #endif 84 | 85 | found: 86 | if (tokens == NULL) { 87 | parser->pos--; 88 | return 0; 89 | } 90 | token = jsmn_alloc_token(parser, tokens, num_tokens); 91 | if (token == NULL) { 92 | parser->pos = start; 93 | return JSMN_ERROR_NOMEM; 94 | } 95 | jsmn_fill_token(token, JSMN_PRIMITIVE, start, parser->pos); 96 | #ifdef JSMN_PARENT_LINKS 97 | token->parent = parser->toksuper; 98 | #endif 99 | parser->pos--; 100 | return 0; 101 | } 102 | 103 | /** 104 | * Fills next token with JSON string. 105 | */ 106 | static int jsmn_parse_string(jsmn_parser *parser, const char *js, 107 | size_t len, jsmntok_t *tokens, size_t num_tokens) { 108 | jsmntok_t *token; 109 | 110 | int start = parser->pos; 111 | 112 | parser->pos++; 113 | 114 | /* Skip starting quote */ 115 | for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { 116 | char c = js[parser->pos]; 117 | 118 | /* Quote: end of string */ 119 | if (c == '\"') { 120 | if (tokens == NULL) { 121 | return 0; 122 | } 123 | token = jsmn_alloc_token(parser, tokens, num_tokens); 124 | if (token == NULL) { 125 | parser->pos = start; 126 | return JSMN_ERROR_NOMEM; 127 | } 128 | jsmn_fill_token(token, JSMN_STRING, start+1, parser->pos); 129 | #ifdef JSMN_PARENT_LINKS 130 | token->parent = parser->toksuper; 131 | #endif 132 | return 0; 133 | } 134 | 135 | /* Backslash: Quoted symbol expected */ 136 | if (c == '\\' && parser->pos + 1 < len) { 137 | int i; 138 | parser->pos++; 139 | switch (js[parser->pos]) { 140 | /* Allowed escaped symbols */ 141 | case '\"': case '/' : case '\\' : case 'b' : 142 | case 'f' : case 'r' : case 'n' : case 't' : 143 | break; 144 | /* Allows escaped symbol \uXXXX */ 145 | case 'u': 146 | parser->pos++; 147 | for(i = 0; i < 4 && parser->pos < len && js[parser->pos] != '\0'; i++) { 148 | /* If it isn't a hex character we have an error */ 149 | if(!((js[parser->pos] >= 48 && js[parser->pos] <= 57) || /* 0-9 */ 150 | (js[parser->pos] >= 65 && js[parser->pos] <= 70) || /* A-F */ 151 | (js[parser->pos] >= 97 && js[parser->pos] <= 102))) { /* a-f */ 152 | parser->pos = start; 153 | return JSMN_ERROR_INVAL; 154 | } 155 | parser->pos++; 156 | } 157 | parser->pos--; 158 | break; 159 | /* Unexpected symbol */ 160 | default: 161 | parser->pos = start; 162 | return JSMN_ERROR_INVAL; 163 | } 164 | } 165 | } 166 | parser->pos = start; 167 | return JSMN_ERROR_PART; 168 | } 169 | 170 | /** 171 | * Parse JSON string and fill tokens. 172 | */ 173 | int jsmn_parse(jsmn_parser *parser, const char *js, size_t len, 174 | jsmntok_t *tokens, unsigned int num_tokens) { 175 | int r; 176 | int i; 177 | jsmntok_t *token; 178 | int count = parser->toknext; 179 | 180 | for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { 181 | char c; 182 | jsmntype_t type; 183 | 184 | c = js[parser->pos]; 185 | switch (c) { 186 | case '{': case '[': 187 | count++; 188 | if (tokens == NULL) { 189 | break; 190 | } 191 | token = jsmn_alloc_token(parser, tokens, num_tokens); 192 | if (token == NULL) 193 | return JSMN_ERROR_NOMEM; 194 | if (parser->toksuper != -1) { 195 | tokens[parser->toksuper].size++; 196 | #ifdef JSMN_PARENT_LINKS 197 | token->parent = parser->toksuper; 198 | #endif 199 | } 200 | token->type = (c == '{' ? JSMN_OBJECT : JSMN_ARRAY); 201 | token->start = parser->pos; 202 | parser->toksuper = parser->toknext - 1; 203 | break; 204 | case '}': case ']': 205 | if (tokens == NULL) 206 | break; 207 | type = (c == '}' ? JSMN_OBJECT : JSMN_ARRAY); 208 | #ifdef JSMN_PARENT_LINKS 209 | if (parser->toknext < 1) { 210 | return JSMN_ERROR_INVAL; 211 | } 212 | token = &tokens[parser->toknext - 1]; 213 | for (;;) { 214 | if (token->start != -1 && token->end == -1) { 215 | if (token->type != type) { 216 | return JSMN_ERROR_INVAL; 217 | } 218 | token->end = parser->pos + 1; 219 | parser->toksuper = token->parent; 220 | break; 221 | } 222 | if (token->parent == -1) { 223 | if(token->type != type || parser->toksuper == -1) { 224 | return JSMN_ERROR_INVAL; 225 | } 226 | break; 227 | } 228 | token = &tokens[token->parent]; 229 | } 230 | #else 231 | for (i = parser->toknext - 1; i >= 0; i--) { 232 | token = &tokens[i]; 233 | if (token->start != -1 && token->end == -1) { 234 | if (token->type != type) { 235 | return JSMN_ERROR_INVAL; 236 | } 237 | parser->toksuper = -1; 238 | token->end = parser->pos + 1; 239 | break; 240 | } 241 | } 242 | /* Error if unmatched closing bracket */ 243 | if (i == -1) return JSMN_ERROR_INVAL; 244 | for (; i >= 0; i--) { 245 | token = &tokens[i]; 246 | if (token->start != -1 && token->end == -1) { 247 | parser->toksuper = i; 248 | break; 249 | } 250 | } 251 | #endif 252 | break; 253 | case '\"': 254 | r = jsmn_parse_string(parser, js, len, tokens, num_tokens); 255 | if (r < 0) return r; 256 | count++; 257 | if (parser->toksuper != -1 && tokens != NULL) 258 | tokens[parser->toksuper].size++; 259 | break; 260 | case '\t' : case '\r' : case '\n' : case ' ': 261 | break; 262 | case ':': 263 | parser->toksuper = parser->toknext - 1; 264 | break; 265 | case ',': 266 | if (tokens != NULL && parser->toksuper != -1 && 267 | tokens[parser->toksuper].type != JSMN_ARRAY && 268 | tokens[parser->toksuper].type != JSMN_OBJECT) { 269 | #ifdef JSMN_PARENT_LINKS 270 | parser->toksuper = tokens[parser->toksuper].parent; 271 | #else 272 | for (i = parser->toknext - 1; i >= 0; i--) { 273 | if (tokens[i].type == JSMN_ARRAY || tokens[i].type == JSMN_OBJECT) { 274 | if (tokens[i].start != -1 && tokens[i].end == -1) { 275 | parser->toksuper = i; 276 | break; 277 | } 278 | } 279 | } 280 | #endif 281 | } 282 | break; 283 | #ifdef JSMN_STRICT 284 | /* In strict mode primitives are: numbers and booleans */ 285 | case '-': case '0': case '1' : case '2': case '3' : case '4': 286 | case '5': case '6': case '7' : case '8': case '9': 287 | case 't': case 'f': case 'n' : 288 | /* And they must not be keys of the object */ 289 | if (tokens != NULL && parser->toksuper != -1) { 290 | jsmntok_t *t = &tokens[parser->toksuper]; 291 | if (t->type == JSMN_OBJECT || 292 | (t->type == JSMN_STRING && t->size != 0)) { 293 | return JSMN_ERROR_INVAL; 294 | } 295 | } 296 | #else 297 | /* In non-strict mode every unquoted value is a primitive */ 298 | default: 299 | #endif 300 | r = jsmn_parse_primitive(parser, js, len, tokens, num_tokens); 301 | if (r < 0) return r; 302 | count++; 303 | if (parser->toksuper != -1 && tokens != NULL) 304 | tokens[parser->toksuper].size++; 305 | break; 306 | 307 | #ifdef JSMN_STRICT 308 | /* Unexpected char in strict mode */ 309 | default: 310 | return JSMN_ERROR_INVAL; 311 | #endif 312 | } 313 | } 314 | 315 | if (tokens != NULL) { 316 | for (i = parser->toknext - 1; i >= 0; i--) { 317 | /* Unmatched opened object or array */ 318 | if (tokens[i].start != -1 && tokens[i].end == -1) { 319 | return JSMN_ERROR_PART; 320 | } 321 | } 322 | } 323 | 324 | return count; 325 | } 326 | 327 | /** 328 | * Creates a new parser based over a given buffer with an array of tokens 329 | * available. 330 | */ 331 | void jsmn_init(jsmn_parser *parser) { 332 | parser->pos = 0; 333 | parser->toknext = 0; 334 | parser->toksuper = -1; 335 | } 336 | 337 | -------------------------------------------------------------------------------- /lib/swift-support/jsmn.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Serge A. Zaitsev 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to deal 6 | * in the Software without restriction, including without limitation the rights 7 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | * copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | * THE SOFTWARE. 21 | */ 22 | 23 | #ifndef __JSMN_H_ 24 | #define __JSMN_H_ 25 | 26 | #include 27 | 28 | #ifdef __cplusplus 29 | extern "C" { 30 | #endif 31 | 32 | /** 33 | * JSON type identifier. Basic types are: 34 | * o Object 35 | * o Array 36 | * o String 37 | * o Other primitive: number, boolean (true/false) or null 38 | */ 39 | typedef enum { 40 | JSMN_UNDEFINED = 0, 41 | JSMN_OBJECT = 1, 42 | JSMN_ARRAY = 2, 43 | JSMN_STRING = 3, 44 | JSMN_PRIMITIVE = 4 45 | } jsmntype_t; 46 | 47 | enum jsmnerr { 48 | /* Not enough tokens were provided */ 49 | JSMN_ERROR_NOMEM = -1, 50 | /* Invalid character inside JSON string */ 51 | JSMN_ERROR_INVAL = -2, 52 | /* The string is not a full JSON packet, more bytes expected */ 53 | JSMN_ERROR_PART = -3 54 | }; 55 | 56 | /** 57 | * JSON token description. 58 | * type type (object, array, string etc.) 59 | * start start position in JSON data string 60 | * end end position in JSON data string 61 | */ 62 | typedef struct { 63 | jsmntype_t type; 64 | int start; 65 | int end; 66 | int size; 67 | #ifdef JSMN_PARENT_LINKS 68 | int parent; 69 | #endif 70 | } jsmntok_t; 71 | 72 | /** 73 | * JSON parser. Contains an array of token blocks available. Also stores 74 | * the string being parsed now and current position in that string 75 | */ 76 | typedef struct { 77 | unsigned int pos; /* offset in the JSON string */ 78 | unsigned int toknext; /* next token to allocate */ 79 | int toksuper; /* superior token node, e.g parent object or array */ 80 | } jsmn_parser; 81 | 82 | /** 83 | * Create JSON parser over an array of tokens 84 | */ 85 | void jsmn_init(jsmn_parser *parser); 86 | 87 | /** 88 | * Run JSON parser. It parses a JSON data string into and array of tokens, each describing 89 | * a single JSON object. 90 | */ 91 | int jsmn_parse(jsmn_parser *parser, const char *js, size_t len, 92 | jsmntok_t *tokens, unsigned int num_tokens); 93 | 94 | #ifdef __cplusplus 95 | } 96 | #endif 97 | 98 | #endif /* __JSMN_H_ */ 99 | -------------------------------------------------------------------------------- /lib/swift-support/jsmn_utils.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2012 The Regents of the University of California. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * 1. Redistributions of source code must retain the above copyright notice, 8 | * this list of conditions and the following disclaimer. 9 | * 10 | * 2. Redistributions in binary form must reproduce the above copyright notice, 11 | * this list of conditions and the following disclaimer in the documentation 12 | * and/or other materials provided with the distribution. 13 | * 14 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 18 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 19 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 20 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 21 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 22 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 23 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 24 | * POSSIBILITY OF SUCH DAMAGE. 25 | */ 26 | 27 | #include "config.h" 28 | 29 | #include 30 | #include 31 | #include 32 | 33 | #include "jsmn_utils.h" 34 | 35 | int jsmn_isnull(const char *json, jsmntok_t *tok) 36 | { 37 | if (tok->type == JSMN_PRIMITIVE && 38 | strncmp("null", json + tok->start, tok->end - tok->start) == 0) { 39 | return 1; 40 | } 41 | return 0; 42 | } 43 | 44 | int jsmn_streq(const char *json, jsmntok_t *tok, const char *s) 45 | { 46 | if (tok->type == JSMN_STRING && 47 | (int) strlen(s) == tok->end - tok->start && 48 | strncmp(json + tok->start, s, tok->end - tok->start) == 0) { 49 | return 1; 50 | } 51 | return 0; 52 | } 53 | 54 | jsmntok_t *jsmn_skip(jsmntok_t *tok) 55 | { 56 | int i; 57 | jsmntok_t *s; 58 | switch(tok->type) 59 | { 60 | case JSMN_PRIMITIVE: 61 | case JSMN_STRING: 62 | JSMN_NEXT(tok); 63 | break; 64 | case JSMN_OBJECT: 65 | case JSMN_ARRAY: 66 | s = tok; 67 | JSMN_NEXT(tok); // move onto first key 68 | for(i=0; isize; i++) { 69 | tok = jsmn_skip(tok); 70 | if (s->type == JSMN_OBJECT) { 71 | tok = jsmn_skip(tok); 72 | } 73 | } 74 | break; 75 | default: 76 | assert(0); 77 | } 78 | 79 | return tok; 80 | } 81 | 82 | void jsmn_strcpy(char *dest, jsmntok_t *tok, const char *json) 83 | { 84 | memcpy(dest, json + tok->start, tok->end - tok->start); 85 | dest[tok->end - tok->start] = '\0'; 86 | } 87 | 88 | int jsmn_strtoul(unsigned long *dest, const char *json, jsmntok_t *tok) 89 | { 90 | char intbuf[20]; 91 | char *endptr = NULL; 92 | assert(tok->end - tok->start < 20); 93 | jsmn_strcpy(intbuf, tok, json); 94 | *dest = strtoul(intbuf, &endptr, 10); 95 | if (*endptr != '\0') { 96 | return -1; 97 | } 98 | return 0; 99 | } 100 | 101 | int jsmn_strtod(double *dest, const char *json, jsmntok_t *tok) 102 | { 103 | char intbuf[128]; 104 | char *endptr = NULL; 105 | assert(tok->end - tok->start < 128); 106 | jsmn_strcpy(intbuf, tok, json); 107 | *dest = strtod(intbuf, &endptr); 108 | if (*endptr != '\0') { 109 | return -1; 110 | } 111 | return 0; 112 | } 113 | -------------------------------------------------------------------------------- /lib/swift-support/jsmn_utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2012 The Regents of the University of California. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * 1. Redistributions of source code must retain the above copyright notice, 8 | * this list of conditions and the following disclaimer. 9 | * 10 | * 2. Redistributions in binary form must reproduce the above copyright notice, 11 | * this list of conditions and the following disclaimer in the documentation 12 | * and/or other materials provided with the distribution. 13 | * 14 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 18 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 19 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 20 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 21 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 22 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 23 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 24 | * POSSIBILITY OF SUCH DAMAGE. 25 | */ 26 | 27 | #ifndef __JSMN_UTILS_H 28 | #define __JSMN_UTILS_H 29 | 30 | #include "jsmn.h" 31 | 32 | #define JSMN_NEXT(tok) (tok)++ 33 | 34 | /** Helper macro to check for an assumed string value 35 | * 36 | * @param json JSON string 37 | * @param tok pointer to the current token 38 | * @param str expected token string value 39 | * 40 | * Note: the err label must be defined in the calling code 41 | */ 42 | #define jsmn_str_assert(json, tok, str) \ 43 | do { \ 44 | if (json_strcmp(json, tok, str) != 0) { \ 45 | goto err; \ 46 | } \ 47 | } while(0) 48 | 49 | /** Helper macro to check for an assumed string value 50 | * 51 | * @param tok pointer to the current token 52 | * @param exptype expected token type 53 | * 54 | * Note: the err label must be defined in the calling code 55 | */ 56 | #define jsmn_type_assert(tok, exptype) \ 57 | do { \ 58 | if (tok->type != exptype) { \ 59 | goto err; \ 60 | } \ 61 | } while(0) 62 | 63 | /** Check if the given token is a "null" 64 | * 65 | * @param json JSON string 66 | * @param tok pointer to the current token 67 | * @return 0 if the token is NOT null, 1 otherwise 68 | */ 69 | int jsmn_isnull(const char *json, jsmntok_t *tok); 70 | 71 | /** Check if the given token is a string and the value matches the given string 72 | * 73 | * @param json JSON string 74 | * @param tok pointer to the current token 75 | * @param s string to compare 76 | * @return 0 if the strings do not match, 1 otherwise 77 | */ 78 | int jsmn_streq(const char *json, jsmntok_t *tok, const char *s); 79 | 80 | /** Advance the token over the current value 81 | * 82 | * @param tok token to advance 83 | * @return pointer to the token after skipping the current value 84 | * 85 | * This function is useful for entirely skipping complex values (e.g. arrays, 86 | * objects), that do not need to be parsed. 87 | */ 88 | jsmntok_t *jsmn_skip(jsmntok_t *tok); 89 | 90 | /** Copy the given string value to the given string buffer 91 | * 92 | * @param dest string buffer to copy into 93 | * @param tok pointer to the current token 94 | * @param json JSON string 95 | * 96 | * The dest buffer must be large enough to hold (tok->end - tok->start)+1 97 | * characters. 98 | */ 99 | void jsmn_strcpy(char *dest, jsmntok_t *tok, const char *json); 100 | 101 | /** Convert the given json primitive value into a unsigned long 102 | * 103 | * @param dest pointer to an unsigned long 104 | * @param json JSON string 105 | * @param tok pointer to the current token 106 | * @return 0 if the value was parsed successfully, -1 otherwise 107 | */ 108 | int jsmn_strtoul(unsigned long *dest, const char *json, jsmntok_t *tok); 109 | 110 | /** Convert the given json primitive value into a double 111 | * 112 | * @param dest pointer to a double 113 | * @param json JSON string 114 | * @param tok pointer to the current token 115 | * @return 0 if the value was parsed successfully, -1 otherwise 116 | */ 117 | int jsmn_strtod(double *dest, const char *json, jsmntok_t *tok); 118 | 119 | #endif /* __JSMN_UTILS_H */ 120 | -------------------------------------------------------------------------------- /lib/swift-support/keystone.h: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2016 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | /* Author: Alistair King */ 28 | 29 | #include "config.h" 30 | 31 | /* Helper for Swift module that does Swift Keystone V3 Auth */ 32 | 33 | #ifndef SWIFT_KEYSTONE_H 34 | #define SWIFT_KEYSTONE_H 1 /**< Guard Define */ 35 | 36 | /** Represents Keystone password authentication credentials */ 37 | typedef struct { 38 | 39 | /** Auth URL */ 40 | char *auth_url; 41 | 42 | /** Username */ 43 | char *username; 44 | 45 | /** Password */ 46 | char *password; 47 | 48 | /** Project */ 49 | char *project; 50 | 51 | /** Domain ID */ 52 | char *domain_id; 53 | 54 | } keystone_auth_creds_t; 55 | 56 | /* Represents the result of a Keystone authentication request */ 57 | typedef struct { 58 | 59 | /** Token */ 60 | char *token; 61 | 62 | /** Swift Storage URL */ 63 | char *storage_url; 64 | 65 | } keystone_auth_token_t; 66 | 67 | /** 68 | * Attempt to extract authentication credentials from the environment 69 | * 70 | * @param creds[out] Pointer to auth credentials structure to fill 71 | * @return 1 if credentials were found, 0 if not, -1 if an error occurred 72 | * 73 | * This function will return 0 unless ALL required credentials were found (the 74 | * only optional credential is the domain ID). It is possible that some 75 | * credentials may have been successfully parsed, so it is still important to 76 | * pass the credential structure to keystone_auth_creds_free. 77 | * 78 | * Environment variables searched are as follows: 79 | * - auth_url: OS_AUTH_URL [required] 80 | * - username: OS_USERNAME [required] 81 | * - password: OS_PASSWORD [required] 82 | * - project: OS_PROJECT [required] 83 | * - domain_id: OS_PROJECT_DOMAIN_ID [optional] 84 | */ 85 | int keystone_env_parse_creds(keystone_auth_creds_t *creds); 86 | 87 | /** 88 | * Attempt to extract an authentication token from the environment 89 | * 90 | * @param token[out] Pointer to an auth result structure to fill 91 | * @return 1 if a token and URL was found, 0 if not, -1 if an error occurred 92 | */ 93 | int keystone_env_parse_token(keystone_auth_token_t *token); 94 | 95 | /** 96 | * Free the memory used by the given credentials 97 | * 98 | * @param creds Pointer to auth credentials structure 99 | * 100 | * This function **does not** free the structure itself. 101 | */ 102 | void keystone_auth_creds_destroy(keystone_auth_creds_t *creds); 103 | 104 | /** 105 | * Free the memory used by the given auth token 106 | * 107 | * @param token Pointer to auth token structure 108 | * 109 | * This function **does not** free the structure itself. 110 | */ 111 | void keystone_auth_token_destroy(keystone_auth_token_t *token); 112 | 113 | /** 114 | * Perform Keystone V3 authentication 115 | * 116 | * @param creds Authentication credentials 117 | * @param token[out] Pointer to auth token result structure to fill 118 | * @return 1 if authentication was successful, 0 if it failed, -1 if an error 119 | * occurred. 120 | * 121 | * @TODO: consider adding an error message to the result struct? 122 | */ 123 | int keystone_authenticate(keystone_auth_creds_t *creds, 124 | keystone_auth_token_t *token); 125 | 126 | /** 127 | * Dump the given auth token to stdout. 128 | * Uses the same format as the `swift auth` CLI command. 129 | * 130 | * @param auth pointer to a valid authentication result structure. 131 | */ 132 | void keystone_auth_token_dump(keystone_auth_token_t *token); 133 | 134 | /** Free a token result */ 135 | 136 | #endif 137 | -------------------------------------------------------------------------------- /lib/wandio_internal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2016 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | #ifndef WANDIO_INTERNAL_H 28 | #define WANDIO_INTERNAL_H 1 /**< Guard Define */ 29 | #include "config.h" 30 | #include 31 | #include 32 | #include 33 | 34 | /** @name libwandioio options 35 | * @{ */ 36 | extern int force_directio_read; 37 | extern int force_directio_write; 38 | extern uint64_t write_waits; 39 | extern uint64_t read_waits; 40 | extern unsigned int use_threads; 41 | extern unsigned int max_buffers; 42 | extern int loghttpservererrors; 43 | /* @} */ 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /m4/attributes.m4: -------------------------------------------------------------------------------- 1 | dnl Macros for checking if various gcc compiler attributes are present 2 | dnl 3 | dnl Written by Shane Alcock , but some credit 4 | dnl should be given to Diego Pettenò whose 5 | dnl macros were very useful in helping me figure out how to write my 6 | dnl own. 7 | dnl 8 | dnl 9 | 10 | AC_DEFUN([check_WERROR], 11 | [ 12 | AC_REQUIRE([AC_PROG_CC]) 13 | AC_CACHE_CHECK( 14 | [if -Werror flag is supported by compiler], 15 | [lt_cv_werror_flag], 16 | [saved="$CFLAGS" 17 | CFLAGS="$CFLAGS -Werror" 18 | AC_COMPILE_IFELSE([AC_LANG_SOURCE([int a;])], 19 | [eval lt_cv_werror_flag='yes'], 20 | [eval lt_cv_werror_flag='no']) 21 | CFLAGS="$saved" 22 | 23 | ]) 24 | ]) 25 | 26 | AC_DEFUN([gcc_PACKED], 27 | [ 28 | AC_REQUIRE([check_WERROR]) 29 | HAVE_ATTRIBUTE_PACKED=0 30 | if test -n "$CC"; then 31 | AS_IF([eval test x$lt_cv_werror_flag = xyes], [errflag=-Werror], []) 32 | AC_CACHE_CHECK([if compiler supports __attribute__((packed))], 33 | [lt_cv_attribute_packed], 34 | [saved="$CFLAGS" 35 | CFLAGS="$CFLAGS $errflag" 36 | AC_COMPILE_IFELSE([AC_LANG_SOURCE( 37 | [struct s { char a; char b; int val; long val2; void *ptr;} __attribute__((packed));])], 38 | [lt_cv_attribute_packed=yes], 39 | [lt_cv_attribute_packed=no] 40 | ) 41 | CFLAGS="$saved" 42 | ]) 43 | if test x$lt_cv_attribute_packed = xyes; then 44 | HAVE_ATTRIBUTE_PACKED=1 45 | fi 46 | fi 47 | AC_SUBST([HAVE_ATTRIBUTE_PACKED]) 48 | AC_DEFINE_UNQUOTED([HAVE_ATTRIBUTE_PACKED], [$HAVE_ATTRIBUTE_PACKED], 49 | [Define to 1 or 0, depending on whether the compiler supports the gcc packed attribute.]) 50 | ]) 51 | 52 | AC_DEFUN([gcc_UNUSED], 53 | [ 54 | AC_REQUIRE([check_WERROR]) 55 | HAVE_ATTRIBUTE_UNUSED=0 56 | if test -n "$CC"; then 57 | AS_IF([eval test x$lt_cv_werror_flag = xyes], [errflag=-Werror], []) 58 | AC_CACHE_CHECK([if compiler supports __attribute__((unused))], 59 | [lt_cv_attribute_unused], 60 | [saved="$CFLAGS" 61 | CFLAGS="$CFLAGS $errflag" 62 | AC_COMPILE_IFELSE([AC_LANG_SOURCE( 63 | [void func(int a, __attribute__((unused)) int b);])], 64 | [lt_cv_attribute_unused=yes], 65 | [lt_cv_attribute_unused=no] 66 | ) 67 | CFLAGS="$saved" 68 | ]) 69 | if test x$lt_cv_attribute_unused = xyes; then 70 | HAVE_ATTRIBUTE_UNUSED=1 71 | fi 72 | fi 73 | AC_SUBST([HAVE_ATTRIBUTE_UNUSED]) 74 | AC_DEFINE_UNQUOTED([HAVE_ATTRIBUTE_UNUSED], [$HAVE_ATTRIBUTE_UNUSED], 75 | [Define to 1 or 0, depending on whether the compiler supports the gcc unused attribute.]) 76 | ]) 77 | 78 | AC_DEFUN([gcc_DEPRECATED], 79 | [ 80 | AC_REQUIRE([check_WERROR]) 81 | HAVE_ATTRIBUTE_DEPRECATED=0 82 | if test -n "$CC"; then 83 | AS_IF([eval test x$lt_cv_werror_flag = xyes], [errflag=-Werror], []) 84 | AC_CACHE_CHECK([if compiler supports __attribute__((deprecated))], 85 | [lt_cv_attribute_deprecated], 86 | [saved="$CFLAGS" 87 | CFLAGS="$CFLAGS $errflag" 88 | AC_COMPILE_IFELSE([AC_LANG_SOURCE( 89 | [void func(int a, int b) __attribute__((deprecated));])], 90 | [lt_cv_attribute_deprecated=yes], 91 | [lt_cv_attribute_deprecated=no] 92 | ) 93 | CFLAGS="$saved" 94 | ]) 95 | if test x$lt_cv_attribute_deprecated = xyes; then 96 | HAVE_ATTRIBUTE_DEPRECATED=1 97 | fi 98 | fi 99 | AC_SUBST([HAVE_ATTRIBUTE_DEPRECATED]) 100 | AC_DEFINE_UNQUOTED([HAVE_ATTRIBUTE_DEPRECATED], [$HAVE_ATTRIBUTE_DEPRECATED], 101 | [Define to 1 or 0, depending on whether the compiler supports the gcc deprecated attribute.]) 102 | ]) 103 | 104 | AC_DEFUN([gcc_FORMAT], 105 | [ 106 | AC_REQUIRE([check_WERROR]) 107 | HAVE_ATTRIBUTE_FORMAT=0 108 | if test -n "$CC"; then 109 | AS_IF([eval test x$lt_cv_werror_flag = xyes], [errflag=-Werror], []) 110 | AC_CACHE_CHECK([if compiler supports __attribute__((format(printf)))], 111 | [lt_cv_attribute_format], 112 | [saved="$CFLAGS" 113 | CFLAGS="$CFLAGS $errflag" 114 | AC_COMPILE_IFELSE([AC_LANG_SOURCE( 115 | [void __attribute__((format(printf, 1, 2))) foo(const char *fmt, ...);])], 116 | [lt_cv_attribute_format=yes], 117 | [lt_cv_attribute_format=no] 118 | ) 119 | CFLAGS="$saved" 120 | ]) 121 | if test x$lt_cv_attribute_format = xyes; then 122 | HAVE_ATTRIBUTE_FORMAT=1 123 | fi 124 | fi 125 | AC_SUBST([HAVE_ATTRIBUTE_FORMAT]) 126 | AC_DEFINE_UNQUOTED([HAVE_ATTRIBUTE_FORMAT], [$HAVE_ATTRIBUTE_FORMAT], 127 | [Define to 1 or 0, depending on whether the compiler supports the format(printf) attribute.]) 128 | ]) 129 | 130 | AC_DEFUN([gcc_PURE], 131 | [ 132 | AC_REQUIRE([check_WERROR]) 133 | HAVE_ATTRIBUTE_PURE=0 134 | if test -n "$CC"; then 135 | AS_IF([eval test x$lt_cv_werror_flag = xyes], [errflag=-Werror], []) 136 | AC_CACHE_CHECK([if compiler supports __attribute__((pure))], 137 | [lt_cv_attribute_pure], 138 | [saved="$CFLAGS" 139 | CFLAGS="$CFLAGS $errflag" 140 | AC_COMPILE_IFELSE([AC_LANG_SOURCE( 141 | [void func(int a, int b) __attribute__((pure));])], 142 | [lt_cv_attribute_pure=yes], 143 | [lt_cv_attribute_pure=no] 144 | ) 145 | CFLAGS="$saved" 146 | ]) 147 | if test x$lt_cv_attribute_pure = xyes; then 148 | HAVE_ATTRIBUTE_PURE=1 149 | fi 150 | fi 151 | AC_SUBST([HAVE_ATTRIBUTE_PURE]) 152 | AC_DEFINE_UNQUOTED([HAVE_ATTRIBUTE_PURE], [$HAVE_ATTRIBUTE_PURE], 153 | [Define to 1 or 0, depending on whether the compiler supports the pure attribute.]) 154 | ]) 155 | 156 | 157 | -------------------------------------------------------------------------------- /m4/visibility.m4: -------------------------------------------------------------------------------- 1 | # visibility.m4 serial 1 (gettext-0.15) 2 | dnl Copyright (C) 2005 Free Software Foundation, Inc. 3 | dnl This file is free software; the Free Software Foundation 4 | dnl gives unlimited permission to copy and/or distribute it, 5 | dnl with or without modifications, as long as this notice is preserved. 6 | 7 | dnl From Bruno Haible. 8 | 9 | dnl Tests whether the compiler supports the command-line option 10 | dnl -fvisibility=hidden and the function and variable attributes 11 | dnl __attribute__((__visibility__("hidden"))) and 12 | dnl __attribute__((__visibility__("default"))). 13 | dnl Does *not* test for __visibility__("protected") - which has tricky 14 | dnl semantics (see the 'vismain' test in glibc) and does not exist e.g. on 15 | dnl MacOS X. 16 | dnl Does *not* test for __visibility__("internal") - which has processor 17 | dnl dependent semantics. 18 | dnl Does *not* test for #pragma GCC visibility push(hidden) - which is 19 | dnl "really only recommended for legacy code". 20 | dnl Set the variable CFLAG_VISIBILITY. 21 | dnl Defines and sets the variable HAVE_VISIBILITY. 22 | 23 | AC_DEFUN([gl_VISIBILITY], 24 | [ 25 | AC_REQUIRE([AC_PROG_CC]) 26 | CFLAG_VISIBILITY= 27 | HAVE_VISIBILITY=0 28 | if test -n "$GCC"; then 29 | AC_MSG_CHECKING([for simple visibility declarations]) 30 | AC_CACHE_VAL(gl_cv_cc_visibility, [ 31 | gl_save_CFLAGS="$CFLAGS" 32 | CFLAGS="$CFLAGS -fvisibility=hidden" 33 | AC_COMPILE_IFELSE( 34 | [AC_LANG_PROGRAM([], 35 | [[extern __attribute__((__visibility__("hidden"))) int hiddenvar; 36 | extern __attribute__((__visibility__("default"))) int exportedvar; 37 | extern __attribute__((__visibility__("hidden"))) int hiddenfunc (void); 38 | extern __attribute__((__visibility__("default"))) int exportedfunc (void);]])], 39 | gl_cv_cc_visibility=yes, 40 | gl_cv_cc_visibility=no) 41 | CFLAGS="$gl_save_CFLAGS"]) 42 | AC_MSG_RESULT([$gl_cv_cc_visibility]) 43 | if test $gl_cv_cc_visibility = yes; then 44 | CFLAG_VISIBILITY="-fvisibility=hidden" 45 | HAVE_VISIBILITY=1 46 | fi 47 | fi 48 | AC_SUBST([CFLAG_VISIBILITY]) 49 | AC_SUBST([HAVE_VISIBILITY]) 50 | AC_DEFINE_UNQUOTED([HAVE_VISIBILITY], [$HAVE_VISIBILITY], 51 | [Define to 1 or 0, depending whether the compiler supports simple visibility declarations.]) 52 | ]) 53 | 54 | -------------------------------------------------------------------------------- /rpm/libwandio1.spec: -------------------------------------------------------------------------------- 1 | Name: libwandio1 2 | Version: 4.2.6 3 | Release: 1%{?dist} 4 | Summary: C Multi-Threaded File Compression and Decompression Library 5 | 6 | License: LGPLv3 7 | URL: https://github.com/LibtraceTeam/wandio 8 | Source0: https://github.com/LibtraceTeam/wandio/archive/%{version}.tar.gz 9 | 10 | BuildRequires: gcc 11 | BuildRequires: gcc-c++ 12 | BuildRequires: make 13 | BuildRequires: zlib-devel 14 | BuildRequires: lzo-devel 15 | BuildRequires: bzip2-devel 16 | BuildRequires: lz4-devel 17 | BuildRequires: xz-devel 18 | BuildRequires: libzstd-devel 19 | BuildRequires: libcurl-devel 20 | 21 | %description 22 | File I/O library that will read and write both compressed and uncompressed 23 | files. All compression-related operations are performed in a separate thread 24 | where possible resulting in significant performance gains for tasks where I/O 25 | is the limiting factor (most simple trace analysis tasks are I/O-limited). 26 | 27 | libwandio was originally developed by the WAND Network Research Group at Waikato 28 | University, New Zealand. 29 | 30 | %package devel 31 | Summary: Development files for %{name} 32 | Requires: %{name}%{?_isa} = %{version}-%{release} 33 | 34 | %package tools 35 | Summary: Example tools for the %{name} library 36 | Requires: %{name}%{?_isa} = %{version}-%{release} 37 | 38 | %description devel 39 | The %{name}-devel package contains libraries and header files for 40 | developing applications that use %{name}. 41 | 42 | %description tools 43 | The %{name}-tools package contains some example tools to demonstrate the 44 | %{name} library. 45 | 46 | %prep 47 | %setup -q -n wandio-%{version} 48 | 49 | %build 50 | %configure --disable-static --mandir=%{_mandir} 51 | make %{?_smp_mflags} 52 | 53 | 54 | %install 55 | rm -rf $RPM_BUILD_ROOT 56 | %make_install 57 | find $RPM_BUILD_ROOT -name '*.la' -exec rm -f {} ';' 58 | 59 | %post -p /sbin/ldconfig 60 | 61 | %postun -p /sbin/ldconfig 62 | 63 | %files 64 | %license COPYING 65 | %{_libdir}/*.so.* 66 | 67 | %files devel 68 | %{_includedir}/* 69 | %{_libdir}/*.so 70 | 71 | %files tools 72 | %doc %{_mandir}/man1/wandiocat.1.gz 73 | %{_bindir}/wandiocat 74 | 75 | %changelog 76 | * Tue Sep 19 2023 Shane Alcock - 4.2.6-1 77 | - New upstream release (4.2.6) 78 | 79 | * Tue Jun 13 2023 Shane Alcock - 4.2.5-1 80 | - New upstream release (4.2.5) 81 | 82 | * Fri Oct 21 2022 Shane Alcock - 4.2.4-1 83 | - New upstream release (4.2.4) 84 | 85 | * Thu May 14 2020 Shane Alcock - 4.2.3-1 86 | - New upstream release (4.2.3) 87 | 88 | * Mon Jul 22 2019 Shane Alcock - 4.2.2-1 89 | - New upstream release (4.2.2) 90 | 91 | * Thu May 16 2019 Shane Alcock - 4.2.1-1 92 | - New upstream release (4.2.1) 93 | 94 | * Fri May 10 2019 Shane Alcock - 4.2.0-1 95 | - New upstream release (4.2.0) 96 | 97 | * Thu Feb 21 2019 Shane Alcock - 4.1.0-1 98 | - First libwandio package 99 | 100 | 101 | -------------------------------------------------------------------------------- /rpmpkg-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x -e -o pipefail 4 | 5 | export QA_RPATHS=$[ 0x0001 ] 6 | SOURCENAME=`echo ${GITHUB_REF##*/} | cut -d '-' -f 1` 7 | 8 | ./bootstrap.sh && ./configure && make dist 9 | cp wandio-*.tar.gz ~/rpmbuild/SOURCES/${SOURCENAME}.tar.gz 10 | cp rpm/libwandio1.spec ~/rpmbuild/SPECS/ 11 | 12 | cd ~/rpmbuild && rpmbuild -bb --define "debug_package %{nil}" SPECS/libwandio1.spec 13 | 14 | -------------------------------------------------------------------------------- /rpmpkg-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x -e -o pipefail 3 | 4 | 5 | mkdir -p /run/user/${UID} 6 | chmod 0700 /run/user/${UID} 7 | yum install -y wget make gcc 8 | 9 | curl -1sLf \ 10 | 'https://dl.cloudsmith.io/public/wand/libwandio/cfg/setup/bash.rpm.sh' \ 11 | | bash 12 | 13 | yum update -y 14 | 15 | if [ "$1" =~ "rocky*" ]; then 16 | dnf install dnf-plugins-core epel-release || true 17 | dnf config-manager --set-enabled PowerTools || true 18 | fi 19 | 20 | if [ "$1" =~ "alma*" ]; then 21 | dnf install dnf-plugins-core epel-release || true 22 | dnf config-manager --set-enabled PowerTools || true 23 | fi 24 | 25 | 26 | if [ "$1" = "centos:8" ]; then 27 | yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm || true 28 | dnf install -y 'dnf-command(config-manager)' || true 29 | yum config-manager --set-enabled PowerTools || true 30 | 31 | fi 32 | 33 | if [ "$1" = "centos:7" ]; then 34 | yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm || true 35 | fi 36 | 37 | if [[ "$1" =~ fedora* ]]; then 38 | dnf install -y rpm-build rpmdevtools 'dnf-command(builddep)' which 39 | dnf group install -y "C Development Tools and Libraries" 40 | dnf builddep -y rpm/libwandio1.spec 41 | else 42 | yum install -y rpm-build yum-utils rpmdevtools which 43 | yum groupinstall -y 'Development Tools' 44 | yum-builddep -y rpm/libwandio1.spec 45 | fi 46 | 47 | rpmdev-setuptree 48 | -------------------------------------------------------------------------------- /run-tests-rpm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x -e -o pipefail 4 | SOURCENAME=`echo ${GITHUB_REF##*/} | cut -d '-' -f 1` 5 | 6 | ZSTDREQ=zstd 7 | 8 | if [ "$1" = "centos_8" ]; then 9 | yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm 10 | 11 | dnf install -y 'dnf-command(config-manager)' || true 12 | yum config-manager --set-enabled PowerTools || true 13 | 14 | # XXX Temporary, until Centos updates to 8.2 where libzstd is 15 | # included in the base OS 16 | # ref: https://lists.fedoraproject.org/archives/list/epel-devel@lists.fedoraproject.org/thread/MFZCRQCULJALRIJJFSSAETSDZ4RL6GCU/ 17 | 18 | wget -N https://archives.fedoraproject.org/pub/archive/epel/8.1/Everything/x86_64/Packages/z/zstd-1.4.4-1.el8.x86_64.rpm && rpm -U zstd-1.4.4-1.el8.x86_64.rpm 19 | 20 | ZSTDREQ= 21 | fi 22 | 23 | if [ "$1" = "centos_7" ]; then 24 | yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm || true 25 | fi 26 | 27 | yum install -y xz ${ZSTDREQ} gzip bzip2 lzop lz4 diffutils 28 | 29 | yum install -y packages/$1/libwandio1-${SOURCENAME}-*.rpm 30 | yum install -y packages/$1/libwandio1-devel-${SOURCENAME}-*.rpm 31 | yum install -y packages/$1/libwandio1-tools-${SOURCENAME}-*.rpm 32 | 33 | cd test && ./do-basic-tests.sh 34 | 35 | -------------------------------------------------------------------------------- /run-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x -e -o pipefail 4 | 5 | export DEBIAN_FRONTEND=noninteractive 6 | 7 | apt-get update 8 | 9 | apt-get install -y curl apt-transport-https \ 10 | ssl-cert ca-certificates gnupg lsb-release git 11 | apt-get upgrade -y 12 | 13 | curl -1sLf 'https://dl.cloudsmith.io/public/wand/libwandio/cfg/setup/bash.deb.sh' | bash 14 | 15 | apt-get update 16 | 17 | apt-get install -y devscripts equivs 18 | apt-get install -y zstd lzop xz-utils liblz4-tool 19 | 20 | mk-build-deps --install --remove --tool "apt-get -o Debug::pkgProblemResolver=yes -y --no-install-recommends" 21 | 22 | deb=`find packages/$1/libwandio1-dev* -maxdepth 1 -type f` 23 | pkg_filename=$(basename "${deb}") 24 | IFS=_ read pkg_name pkg_version pkg_arch <<< $(basename -s ".deb" "${pkg_filename}") 25 | 26 | dpkg -i packages/$1/libwandio1_${pkg_version}_${pkg_arch}.deb 27 | dpkg -i packages/$1/libwandio1-dev_${pkg_version}_${pkg_arch}.deb 28 | dpkg -i packages/$1/wandio1-tools_${pkg_version}_${pkg_arch}.deb 29 | 30 | cd test && ./do-basic-tests.sh 31 | -------------------------------------------------------------------------------- /test/do-basic-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export PATH="../tools/wandiocat:$PATH" 4 | 5 | OK=0 6 | FAIL="" 7 | 8 | do_test() { 9 | 10 | if $@; then 11 | OK=$[ OK + 1 ] 12 | else 13 | FAIL="$FAIL 14 | $@" 15 | fi 16 | } 17 | 18 | 19 | do_read_test() { 20 | wandiocat $2 | md5sum | cut -d " " -f 1 > /tmp/wandiotest.md5 21 | 22 | diff -q /tmp/wandiotest.md5 /tmp/wandiobase.md5 > /dev/null 23 | 24 | if [ $? -ne 0 ]; then 25 | FAIL="$FAIL 26 | reading $1 test file" 27 | echo " fail" 28 | else 29 | OK=$[ OK + 1 ] 30 | echo " pass" 31 | fi 32 | } 33 | 34 | do_write_test() { 35 | 36 | if [ $1 = "text" ]; then 37 | wandiocat -o /tmp/wandiowrite.out files/big.txt 38 | else 39 | wandiocat -z 1 -Z $1 -o /tmp/wandiowrite.out files/big.txt 40 | fi 41 | 42 | 43 | case $1 in 44 | text) 45 | TOOL="cat" 46 | ;; 47 | gzip) 48 | TOOL="gzip -d -c" 49 | ;; 50 | bzip2) 51 | TOOL="bzip2 -d -c" 52 | ;; 53 | lzma) 54 | TOOL="xz -d -c" 55 | ;; 56 | lz4) 57 | TOOL="lz4 -d -c" 58 | ;; 59 | zstd) 60 | TOOL="zstd -q -d -c" 61 | ;; 62 | lzo) 63 | TOOL="lzop -q -d -c" 64 | ;; 65 | *) 66 | echo " fail (unrecognised format?)" 67 | FAIL="$FAIL 68 | writing $1 test file" 69 | return 70 | esac 71 | 72 | $TOOL /tmp/wandiowrite.out | md5sum | cut -d " " -f 1 > /tmp/wandiotest2.md5 73 | 74 | if [ $1 != "lzo" ]; then 75 | wandiocat /tmp/wandiowrite.out | md5sum | cut -d " " -f 1 > /tmp/wandiotest.md5 76 | diff -q /tmp/wandiotest.md5 /tmp/wandiobase.md5 > /dev/null 77 | 78 | if [ $? -ne 0 ]; then 79 | FAIL="$FAIL 80 | writing $1 test file" 81 | echo " fail (read with wandiocat)" 82 | return 83 | fi 84 | fi 85 | 86 | 87 | diff -q /tmp/wandiotest2.md5 /tmp/wandiobase.md5 > /dev/null 88 | if [ $? -ne 0 ]; then 89 | FAIL="$FAIL 90 | writing $1 test file" 91 | echo " fail (read with standard tool)" 92 | else 93 | OK=$[ OK + 1 ] 94 | echo " pass" 95 | fi 96 | } 97 | 98 | REQBINARIES=( gzip bzip2 xz lz4 zstd lzop ) 99 | 100 | for bin in ${REQBINARIES[*]}; do 101 | command -v $bin > /dev/null || echo "$bin not detected on system -- $bin tests may fail" 102 | done 103 | 104 | cat files/big.txt | md5sum | cut -d " " -f 1 > /tmp/wandiobase.md5 105 | 106 | echo -n \* Reading text... 107 | do_read_test text files/big.txt 108 | 109 | echo -n \* Reading gzip... 110 | do_read_test gzip files/big.txt.gz 111 | 112 | echo -n \* Reading bzip2... 113 | do_read_test bzip2 files/big.txt.bz2 114 | 115 | echo -n \* Reading multi-stream bzip2... 116 | do_read_test bzip2 files/big.multistream.txt.bz2 117 | 118 | echo -n \* Reading lzma... 119 | do_read_test lzma files/big.txt.xz 120 | 121 | echo -n \* Reading lz4... 122 | do_read_test lz4 files/big.txt.lz4 123 | 124 | echo -n \* Reading zstd... 125 | do_read_test zstd files/big.txt.zst 126 | 127 | echo -n \* Writing text... 128 | do_write_test text 129 | 130 | echo -n \* Writing gzip... 131 | do_write_test gzip 132 | 133 | echo -n \* Writing bzip2... 134 | do_write_test bzip2 135 | 136 | echo -n \* Writing lzma... 137 | do_write_test lzma 138 | 139 | echo -n \* Writing lz4... 140 | do_write_test lz4 141 | 142 | echo -n \* Writing zstd... 143 | do_write_test zstd 144 | 145 | echo -n \* Writing lzo... 146 | do_write_test lzo 147 | 148 | echo 149 | echo "Tests passed: $OK" 150 | echo "Tests failed: $FAIL" 151 | 152 | if [ -z "$FAIL" ]; then 153 | exit 0 154 | else 155 | exit 1 156 | fi 157 | -------------------------------------------------------------------------------- /test/files/big.multistream.txt.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LibtraceTeam/wandio/529105dbedb369659c72e9f1c99ce2c1f969c683/test/files/big.multistream.txt.bz2 -------------------------------------------------------------------------------- /test/files/big.txt.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LibtraceTeam/wandio/529105dbedb369659c72e9f1c99ce2c1f969c683/test/files/big.txt.bz2 -------------------------------------------------------------------------------- /test/files/big.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LibtraceTeam/wandio/529105dbedb369659c72e9f1c99ce2c1f969c683/test/files/big.txt.gz -------------------------------------------------------------------------------- /test/files/big.txt.lz4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LibtraceTeam/wandio/529105dbedb369659c72e9f1c99ce2c1f969c683/test/files/big.txt.lz4 -------------------------------------------------------------------------------- /test/files/big.txt.xz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LibtraceTeam/wandio/529105dbedb369659c72e9f1c99ce2c1f969c683/test/files/big.txt.xz -------------------------------------------------------------------------------- /test/files/big.txt.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LibtraceTeam/wandio/529105dbedb369659c72e9f1c99ce2c1f969c683/test/files/big.txt.zst -------------------------------------------------------------------------------- /tools/wandiocat/Makefile.am: -------------------------------------------------------------------------------- 1 | bin_PROGRAMS = wandiocat 2 | wandiocat_SOURCES = wcat.c 3 | wandiocat_CFLAGS = -I"$(top_srcdir)/lib" 4 | wandiocat_CXXFLAGS = -I"$(top_srcdir)/lib" 5 | wandiocat_LDFLAGS = -L"$(top_srcdir)/lib" -lwandio 6 | dist_man_MANS = wandiocat.1 7 | -------------------------------------------------------------------------------- /tools/wandiocat/wandiocat.1: -------------------------------------------------------------------------------- 1 | .TH WANDIOCAT 1 "2019-02-12" "wandiocat" "libwandio" 2 | 3 | .SH NAME 4 | wandiocat \- read and write files using libwandio routines 5 | 6 | .SH SYNOPSIS 7 | \fBwandiocat\fR [\fB-z\fR \fIlevel\fR] [\fB-Z\fR \fImethod\fR] 8 | [\fB-o\fR \fIoutputfilename\fR] \fBinputfile\fR [\fBinputfile\fR ...] 9 | 10 | .SH DESCRIPTION 11 | \fBwandiocat\fR is a simple program designed to demonstrate how libwandio can 12 | be used to easily read and write files using any of the supported compression 13 | methods. Each input file specified will be concatenated into a single 14 | output file (or written to standard output if no output file is specified) 15 | which will be compressed according to the compression method and level 16 | supplied by the user. 17 | 18 | wandiocat will automatically detect the compression method required to 19 | read the input files, so they do not need to be decompressed first. 20 | 21 | .SH OPTIONS 22 | .TP 23 | \fB-h\fR 24 | Show summary of supported command-line options. 25 | 26 | .TP 27 | \fB-Z \fImethod\fR 28 | Specifies which compression method should be used to write output. 29 | Must be one of 'gzip', 'bzip2', 'lzma', 'lzo', 'zstd', or 'lz4'. If 30 | not specified, the output is written uncompressed. 31 | 32 | .TP 33 | \fB-z \fIlevel\fR 34 | Specifies the compression level to be used when writing output. The level 35 | must be an integer value between 0 (uncompressed) and 9 (maximum compression) 36 | inclusive. Usually compression level 1 is sufficient for most purposes. 37 | Defaults to 0. 38 | 39 | .TP 40 | \fB-o \fIoutputfilename\fR 41 | Sets the name of the output file. If not specified, output will be written 42 | to standard output instead. 43 | 44 | .SH SECURITY 45 | \fBwandiocat\fR should usually be run unprivileged. The only exception would 46 | be when the user wants to use Intel QuickAssist hardware to perform gzip 47 | compression (and has configured and installed libwandio with QuickAssist 48 | support enabled) -- in this case, \fBwandiocat\fR may need to be run as root 49 | to be able to access the hardware. 50 | 51 | .SH AUTHOR 52 | wandiocat was originally written by Perry Lorier, while working for the 53 | University of Waikato WAND Network Research Group . 54 | 55 | .PP 56 | This manual page was written by Shane Alcock 57 | -------------------------------------------------------------------------------- /tools/wandiocat/wcat.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (c) 2007-2019 The University of Waikato, Hamilton, New Zealand. 4 | * All rights reserved. 5 | * 6 | * This file is part of libwandio. 7 | * 8 | * This code has been developed by the University of Waikato WAND 9 | * research group. For further information please see http://www.wand.net.nz/ 10 | * 11 | * libwandio is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU Lesser General Public License as published by 13 | * the Free Software Foundation; either version 3 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * libwandio is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public License 22 | * along with this program. If not, see . 23 | * 24 | * 25 | */ 26 | 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include "wandio.h" 34 | 35 | static void printhelp() { 36 | printf("wandiocat: concatenate files into a single compressed file\n"); 37 | printf("\n"); 38 | printf("Available options:\n\n"); 39 | printf(" -z \n"); 40 | printf(" Sets a compression level for the output file, must be \n"); 41 | printf(" between 0 (uncompressed) and 9 (max compression)\n"); 42 | printf(" Default is 0.\n"); 43 | printf(" -Z \n"); 44 | printf(" Set the compression method. Must be one of 'gzip', \n"); 45 | printf(" 'bzip2', 'lzo', 'lzma', 'zstd' or 'lz4'. If not specified, " 46 | "no\n"); 47 | printf(" compression is performed.\n"); 48 | printf(" -o \n"); 49 | printf(" The name of the output file. If not specified, output\n"); 50 | printf(" is written to standard output.\n"); 51 | } 52 | 53 | int main(int argc, char *argv[]) { 54 | int compress_level = 0; 55 | int compress_type = WANDIO_COMPRESS_NONE; 56 | char *output = "-"; 57 | int c; 58 | char *buffer = NULL; 59 | while ((c = getopt(argc, argv, "Z:z:o:h")) != -1) { 60 | switch (c) { 61 | case 'Z': { 62 | struct wandio_compression_type *compression_type = 63 | wandio_lookup_compression_type(optarg); 64 | if (compression_type == 0) { 65 | fprintf( 66 | stderr, 67 | "Unable to lookup compression type: '%s'\n", 68 | optarg); 69 | return -1; 70 | } 71 | compress_type = compression_type->compress_type; 72 | 73 | } break; 74 | case 'z': 75 | compress_level = atoi(optarg); 76 | break; 77 | case 'o': 78 | output = optarg; 79 | break; 80 | case 'h': 81 | printhelp(); 82 | return 0; 83 | case '?': 84 | if (optopt == 'Z' || optopt == 'z' || optopt == 'o') 85 | fprintf(stderr, 86 | "Option -%c requires an argument.\n", 87 | optopt); 88 | else if (isprint(optopt)) 89 | fprintf(stderr, "Unknown option `-%c'.\n", 90 | optopt); 91 | else 92 | fprintf(stderr, 93 | "Unknown option character `\\x%x'.\n", 94 | optopt); 95 | return 1; 96 | default: 97 | abort(); 98 | } 99 | } 100 | 101 | iow_t *iow = wandio_wcreate(output, compress_type, compress_level, 0); 102 | /* stdout */ 103 | int i; 104 | int rc = 0; 105 | 106 | #if _POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600 107 | if (posix_memalign((void **)&buffer, 4096, WANDIO_BUFFER_SIZE) != 0) { 108 | fprintf(stderr, 109 | "Unable to allocate aligned buffer for wandiocat: %s\n", 110 | strerror(errno)); 111 | abort(); 112 | } 113 | #else 114 | buffer = malloc(WANDIO_BUFFER_SIZE); 115 | #endif 116 | 117 | for (i = optind; i < argc; ++i) { 118 | io_t *ior = wandio_create(argv[i]); 119 | if (!ior) { 120 | fprintf(stderr, "Failed to open %s\n", argv[i]); 121 | rc++; 122 | continue; 123 | } 124 | 125 | int64_t len; 126 | do { 127 | len = wandio_read(ior, buffer, WANDIO_BUFFER_SIZE); 128 | if (len > 0) 129 | wandio_wwrite(iow, buffer, len); 130 | } while (len > 0); 131 | 132 | wandio_destroy(ior); 133 | } 134 | free(buffer); 135 | wandio_wdestroy(iow); 136 | return rc; 137 | } 138 | --------------------------------------------------------------------------------