├── .clang-format ├── .gitignore ├── .gitmodules ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── SECURITY.md ├── SOFTWARE_ONLY_CONFIG.md ├── jmh └── qat-java-bench │ ├── README.md │ ├── pom.xml │ └── src │ └── main │ └── java │ └── com │ └── intel │ └── qat │ └── jmh │ ├── BenchmarkDriver.java │ ├── JavaUtilZipBench.java │ ├── JavaUtilZipStreamBench.java │ ├── QatJavaBench.java │ └── QatJavaStreamBench.java ├── pom.xml └── src ├── main ├── java │ ├── com │ │ └── intel │ │ │ └── qat │ │ │ ├── InternalJNI.java │ │ │ ├── Native.java │ │ │ ├── QatCompressorOutputStream.java │ │ │ ├── QatDecompressorInputStream.java │ │ │ ├── QatZipper.java │ │ │ ├── QatZstdSequenceProducer.java │ │ │ ├── examples │ │ │ ├── ByteArrayExample.java │ │ │ └── InputOutputStreamExample.java │ │ │ └── package-info.java │ └── module-info.java └── jni │ ├── CMakeLists.txt │ ├── com_intel_qat_InternalJNI.c │ ├── com_intel_qat_InternalJNI.h │ ├── util.c │ └── util.h └── test ├── java └── com │ └── intel │ └── qat │ ├── NativeTest.java │ ├── QatCompressorOutputStreamTests.java │ ├── QatDecompressorInputStreamTests.java │ ├── QatTestSuite.java │ ├── QatZipperTests.java │ └── fuzzing │ └── FuzzerTest.java └── resources └── sample.txt /.clang-format: -------------------------------------------------------------------------------- 1 | Language: Cpp 2 | BasedOnStyle: Google 3 | AlignAfterOpenBracket: Align 4 | BinPackParameters: false 5 | AllowAllParametersOfDeclarationOnNextLine: false 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Based on https://github.com/github/gitignore/blob/main/Java.gitignore and https://github.com/github/gitignore/blob/main/Maven.gitignore 2 | 3 | # Compiled class file 4 | *.class 5 | 6 | # Log file 7 | *.log 8 | 9 | # BlueJ files 10 | *.ctxt 11 | 12 | # Mobile Tools for Java (J2ME) 13 | .mtj.tmp/ 14 | 15 | # Package Files # 16 | *.jar 17 | *.war 18 | *.nar 19 | *.ear 20 | *.zip 21 | *.tar.gz 22 | *.rar 23 | 24 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 25 | hs_err_pid* 26 | replay_pid* 27 | 28 | target/ 29 | pom.xml.tag 30 | pom.xml.releaseBackup 31 | pom.xml.versionsBackup 32 | pom.xml.next 33 | release.properties 34 | dependency-reduced-pom.xml 35 | buildNumber.properties 36 | .mvn/timing.properties 37 | # https://github.com/takari/maven-wrapper#usage-without-binary-jar 38 | .mvn/wrapper/maven-wrapper.jar 39 | 40 | # Eclipse m2e generated files 41 | # Eclipse Core 42 | .project 43 | # JDT-specific (Eclipse Java Development Tools) 44 | .classpath 45 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "src/main/jni/external/qat-zstd-plugin"] 2 | path = src/main/jni/external/qat-zstd-plugin 3 | url = https://github.com/intel/QAT-ZSTD-Plugin.git 4 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, caste, color, religion, or sexual 10 | identity and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the overall 26 | community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or advances of 31 | any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email address, 35 | without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | CommunityCodeOfConduct AT intel DOT com. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series of 86 | actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or permanent 93 | ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within the 113 | community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.1, available at 119 | [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. 120 | 121 | Community Impact Guidelines were inspired by 122 | [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. 123 | 124 | For answers to common questions about this code of conduct, see the FAQ at 125 | [https://www.contributor-covenant.org/faq][FAQ]. Translations are available at 126 | [https://www.contributor-covenant.org/translations][translations]. 127 | 128 | [homepage]: https://www.contributor-covenant.org 129 | [v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html 130 | [Mozilla CoC]: https://github.com/mozilla/diversity 131 | [FAQ]: https://www.contributor-covenant.org/faq -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## Issues 4 | 5 | Here you'll find instructions on how to contribute to the QAT-Java Library. 6 | 7 | Your contributions are most welcome! You'll find it is best to begin 8 | with a conversation about your changes, rather than just writing a bunch 9 | of code and contributing it out of the blue. 10 | A great way to suggest new features, offer to add a feature, 11 | or just begin a dialog about the QAT-Java Library is to open an issue in our [GitHub Issues Database](https://github.com/intel/qat-java/issues) 12 | 13 | 14 | ### Contribution Guide 15 | 16 | We accept contributions as pull requests on GitHub. Please follow these simple rules: 17 | 18 | * A PR should have a clear purpose, and do one thing only, and nothing more. This will enable us review your PR more quickly. 19 | * Each commit in PR should be a small, atomic change representing one step in development. 20 | * Please squash intermediate steps within PR for bugfixes, style cleanups, reversions, etc., so they would not appear in merged PR history. 21 | * Please explain anything non-obvious from the code in comments, commit messages, or the PR description, as appropriate. 22 | 23 | ### Code formatting and check 24 | 25 | Please check the formatting of the code by running: 26 | 27 | ``` 28 | mvn spotless:check 29 | ``` 30 | 31 | The code can be formatted with: 32 | ``` 33 | mvn spotless:apply 34 | ``` 35 | ### License 36 | 37 | Java* Native Interface binding for Intel® Quick Assist Technology is licensed under the terms in [BSD](https://github.com/intel/qat-java/blob/main/LICENSE). By contributing to the project, you agree to the license and copyright terms therein and release your contribution under these terms. 38 | 39 | ### Sign your work 40 | 41 | Please use the sign-off line at the end of the patch. Your signature certifies that you wrote the patch or otherwise have the right to pass it on as an open-source patch. The rules are pretty simple: if you can certify 42 | the below (from [developercertificate.org](http://developercertificate.org/)): 43 | 44 | ``` 45 | Developer Certificate of Origin 46 | Version 1.1 47 | 48 | Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 49 | 660 York Street, Suite 102, 50 | San Francisco, CA 94110 USA 51 | 52 | Everyone is permitted to copy and distribute verbatim copies of this 53 | license document, but changing it is not allowed. 54 | 55 | Developer's Certificate of Origin 1.1 56 | 57 | By making a contribution to this project, I certify that: 58 | 59 | (a) The contribution was created in whole or in part by me and I 60 | have the right to submit it under the open source license 61 | indicated in the file; or 62 | 63 | (b) The contribution is based upon previous work that, to the best 64 | of my knowledge, is covered under an appropriate open source 65 | license and I have the right under that license to submit that 66 | work with modifications, whether created in whole or in part 67 | by me, under the same open source license (unless I am 68 | permitted to submit under a different license), as indicated 69 | in the file; or 70 | 71 | (c) The contribution was provided directly to me by some other 72 | person who certified (a), (b) or (c) and I have not modified 73 | it. 74 | 75 | (d) I understand and agree that this project and the contribution 76 | are public and that a record of the contribution (including all 77 | personal information I submit with it, including my sign-off) is 78 | maintained indefinitely and may be redistributed consistent with 79 | this project or the open source license(s) involved. 80 | ``` 81 | 82 | Then you just add a line to every git commit message: 83 | 84 | Signed-off-by: Joe Smith 85 | 86 | Use your real name (sorry, no pseudonyms or anonymous contributions.) 87 | 88 | If you set your `user.name` and `user.email` git configs, you can sign your 89 | commit automatically with `git commit -s`. 90 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | ################################################################ 2 | # BSD LICENSE 3 | # 4 | # Copyright(c) 2007-2023 Intel Corporation. All rights reserved. 5 | # All rights reserved. 6 | # 7 | # Redistribution and use in source and binary forms, with or without 8 | # modification, are permitted provided that the following conditions 9 | # are met: 10 | # 11 | # * Redistributions of source code must retain the above copyright 12 | # notice, this list of conditions and the following disclaimer. 13 | # * Redistributions in binary form must reproduce the above copyright 14 | # notice, this list of conditions and the following disclaimer in 15 | # the documentation and/or other materials provided with the 16 | # distribution. 17 | # * Neither the name of Intel Corporation nor the names of its 18 | # contributors may be used to endorse or promote products derived 19 | # from this software without specific prior written permission. 20 | # 21 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | # 33 | ################################################################ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Java* Native Interface Binding for Intel® QuickAssist Technology 2 | 3 | **Qat-Java** is a library that accelerates data compression using Intel® [QuickAssist Technology (QAT)](https://www.intel.com/content/www/us/en/architecture-and-technology/intel-quick-assist-technology-overview.html). 4 | For more details on Intel® QAT and installation instructions, refer to the [QAT Documentation](https://intel.github.io/quickassist/index.html). 5 | 6 | Qat-Java currently supports the following compression algorithms: 7 | - **DEFLATE** 8 | - **LZ4** 9 | - **Zstandard (ZSTD)** 10 | 11 | ## Prerequisites 12 | 13 | This release was validated with the following tools and libraries: 14 | 15 | - [QATlib v24.09.0](https://github.com/intel/qatlib) 16 | - [QATzip v1.3.0](https://github.com/intel/QATzip/releases) and its dependencies 17 | - [Zstandard v1.5.4](https://github.com/facebook/zstd) 18 | - [Zstd-jni v1.5.6-1](https://github.com/luben/zstd-jni) or newer (if you're not building from source) 19 | - GCC 8.5 or newer 20 | - JDK 17 or newer 21 | - Clang (for fuzz testing) 22 | 23 | ## Build 24 | 25 | To build Qat-Java, run: 26 | 27 | ``` 28 | mvn clean package 29 | ``` 30 | 31 | ### Additional Maven Targets 32 | 33 | In addition to `mvn clean package`, the following Maven goals are available: 34 | 35 | - `clean` — Cleans the build directory. 36 | - `compile` — Compiles the source code. 37 | - `test` — Compiles and runs all unit tests. 38 | - `package` — Packages the compiled classes into JAR files (in the `target/` directory). 39 | - `javadoc:javadoc` — Generates Javadoc API documentation. 40 | - `spotless:check` — Checks that the source code is properly formatted. 41 | - `spotless:apply` — Automatically fixes code formatting issues. 42 | - `site` — Generates Surefire test reports in `target/site/`. 43 | 44 | ## Testing 45 | 46 | To run all unit tests: 47 | 48 | ``` 49 | mvn test 50 | ``` 51 | 52 | ## Fuzz Testing 53 | 54 | To enable fuzz testing, install the [Jazzer](https://github.com/CodeIntelligenceTesting/jazzer/blob/main/CONTRIBUTING.md) tool and run: 55 | 56 | ``` 57 | mvn test -Dfuzzing=true 58 | ``` 59 | 60 | ## Examples 61 | 62 | To run an example from the `com.intel.qat.examples` package, use the following command: 63 | 64 | ``` 65 | java -cp .:./target/classes/:path/to/zstd-jni-1.5.6-1.jar com.intel.qat.examples. 66 | ``` 67 | 68 | ## Authors 69 | 70 | - **Mulugeta Mammo** — [mulugeta.mammo@intel.com](mailto:mulugeta.mammo@intel.com) 71 | - **Olasoji Denloye** — [olasoji.denloye@intel.com](mailto:olasoji.denloye@intel.com) 72 | - **Praveen Nishchal** — [praveen.nishchal@intel.com](mailto:praveen.nishchal@intel.com) 73 | 74 | **Zstandard compression contributions** by: 75 | - Jacob Greenfield 76 | - Matthew West 77 | - Tommy Parisi 78 | 79 | ## Contributing 80 | 81 | Thank you for your interest in contributing! 82 | Please refer to the [CONTRIBUTING.md](CONTRIBUTING.md) document for details on how to get involved. 83 | 84 | ## Contact 85 | 86 | For questions or more information about this library, contact: 87 | - [mulugeta.mammo@intel.com](mailto:mulugeta.mammo@intel.com) 88 | - [olasoji.denloye@intel.com](mailto:olasoji.denloye@intel.com) 89 | 90 | * Java is a registered trademark of Oracle and/or its affiliates. 91 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | Intel is committed to rapidly addressing security vulnerabilities affecting our customers and providing clear guidance on the solution, impact, severity and mitigation. 3 | 4 | ## Reporting a Vulnerability 5 | Please report any security vulnerabilities in this project utilizing the guidelines [here](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html). 6 | -------------------------------------------------------------------------------- /SOFTWARE_ONLY_CONFIG.md: -------------------------------------------------------------------------------- 1 | ## Software-only Configuration 2 | When a system has no QAT hardware available, Qat-Java can still be made to run in a software-only mode. This mode is configured by leveraging the Linux distribution's built-in package manager. Please ensure the machine is using kernel version 6.0 or newer so that compatible library and firmware versions are applied. 3 | 4 | - Install QATzip and QATlib on RHEL 8, RHEL 9, CentOS Stream 8, or CentOS Stream 9. 5 | 6 | ``` 7 | sudo dnf install qatzip qatzip-devel qatzip-libs qatlib qatlib-devel qatlib-service qatlib-tests 8 | ``` 9 | 10 | - Install QATzip and QATlib on SLES 15 or openSUSE Leap 15. 11 | 12 | ``` 13 | sudo zypper install libqatzip3 qatzip qatzip-devel qatzip qatlib qatlib-devel 14 | ``` 15 | -------------------------------------------------------------------------------- /jmh/qat-java-bench/README.md: -------------------------------------------------------------------------------- 1 | ## JMH 2 | A set of JMH benchmarks. 3 | ``` 4 | Name | Algorithm 5 | -------------------------------------------- 6 | QatJavaBench | DEFLATE, LZ4, ZSTD 7 | QatJavaStreamBench | DEFLATE, LZ4, ZSTD 8 | ``` 9 | 10 | ## Build 11 | To build the benchmark, execute the below command: 12 | ``` 13 | mvn clean package 14 | ``` 15 | 16 | ## Run 17 | To run the benchmark, use the below command: 18 | 19 | ``` 20 | java -jar target/benchmarks.jar QatJavaBench -p file=/path/to/a/text-corpus -p algorithm=<"DEFLATE"|"LZ4"|"ZSTD"> -p level= -p blockSize= 21 | java -jar target/benchmarks.jar QatJavaStreamBench -p file=/path/to/a/text-corpus -p algorithm=<"DEFLATE"|"LZ4"|"ZSTD"> -p level= -p blockSize= 22 | ``` 23 | 24 | Examples: 25 | ``` 26 | java -jar target/benchmarks.jar QatJavaBench -p file=silesia/dickens -p algorithm="DEFLATE" -p level=6 -p blockSize=65536 -f 1 -wi 1 -i 2 -t 1 27 | java -jar target/benchmarks.jar QatJavaStreamBench -p file=silesia/dickens -p algorithm="LZ4" -p level=6 -p blockSize=65536 -f 1 -wi 1 -i 2 -t 1 28 | ``` 29 | 30 | You may get a text corpus for benchmarking from [Silesia compression corpus](https://sun.aei.polsl.pl//~sdeor/index.php?page=silesia). 31 | -------------------------------------------------------------------------------- /jmh/qat-java-bench/pom.xml: -------------------------------------------------------------------------------- 1 | 31 | 32 | 34 | 4.0.0 35 | 36 | com.intel.qat.jmh 37 | qat-java-bench 38 | 2.3.2 39 | jar 40 | 41 | JMH benchmark sample: Java 42 | 43 | 47 | 48 | 49 | 50 | org.openjdk.jmh 51 | jmh-core 52 | ${jmh.version} 53 | 54 | 55 | org.openjdk.jmh 56 | jmh-generator-annprocess 57 | ${jmh.version} 58 | provided 59 | 60 | 61 | com.intel.qat 62 | qat-java 63 | 2.3.2 64 | 65 | 66 | com.github.luben 67 | zstd-jni 68 | 1.5.6-1 69 | 70 | 71 | 72 | 73 | UTF-8 74 | 75 | 78 | 1.37 79 | 80 | 83 | 1.8 84 | 85 | 88 | benchmarks 89 | 90 | 91 | 92 | 93 | 94 | com.diffplug.spotless 95 | spotless-maven-plugin 96 | 2.41.1 97 | 98 | 99 | format 100 | process-sources 101 | 102 | check 103 | apply 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | *.java 112 | 113 | 114 | 115 | 116 | 117 | 1.18.1 118 | 119 | 120 | 121 | 122 | 123 | org.apache.maven.plugins 124 | maven-compiler-plugin 125 | 3.11.0 126 | 127 | ${javac.target} 128 | ${javac.target} 129 | ${javac.target} 130 | 131 | 132 | org.openjdk.jmh 133 | jmh-generator-annprocess 134 | ${jmh.version} 135 | 136 | 137 | 138 | 139 | 140 | org.apache.maven.plugins 141 | maven-shade-plugin 142 | 3.5.1 143 | 144 | 145 | package 146 | 147 | shade 148 | 149 | 150 | ${uberjar.name} 151 | 152 | 153 | com.intel.qat.jmh.BenchmarkDriver 154 | 155 | 156 | 157 | 158 | 159 | 163 | *:* 164 | 165 | META-INF/*.SF 166 | META-INF/*.DSA 167 | META-INF/*.RSA 168 | META-INF/MANIFEST.MF 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | maven-clean-plugin 181 | 3.3.2 182 | 183 | 184 | maven-deploy-plugin 185 | 3.1.1 186 | 187 | 188 | maven-install-plugin 189 | 3.1.1 190 | 191 | 192 | maven-jar-plugin 193 | 3.3.0 194 | 195 | 196 | maven-javadoc-plugin 197 | 3.6.3 198 | 199 | 200 | maven-resources-plugin 201 | 3.3.1 202 | 203 | 204 | maven-site-plugin 205 | 4.0.0-M12 206 | 207 | 208 | maven-source-plugin 209 | 3.3.0 210 | 211 | 212 | maven-surefire-plugin 213 | 3.2.2 214 | 215 | 216 | 217 | 218 | 219 | 220 | -------------------------------------------------------------------------------- /jmh/qat-java-bench/src/main/java/com/intel/qat/jmh/BenchmarkDriver.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | 7 | package com.intel.qat.jmh; 8 | 9 | import java.io.File; 10 | import java.util.Collection; 11 | import org.openjdk.jmh.results.Result; 12 | import org.openjdk.jmh.results.RunResult; 13 | import org.openjdk.jmh.runner.Runner; 14 | import org.openjdk.jmh.runner.RunnerException; 15 | import org.openjdk.jmh.runner.options.CommandLineOptionException; 16 | import org.openjdk.jmh.runner.options.CommandLineOptions; 17 | import org.openjdk.jmh.util.Optional; 18 | 19 | public class BenchmarkDriver { 20 | public static void main(String[] args) { 21 | try { 22 | CommandLineOptions cmdOpts = new CommandLineOptions(args); 23 | 24 | Optional> col = cmdOpts.getParameter("file"); 25 | if (!col.hasValue()) { 26 | throw new IllegalArgumentException("A file parameter is required."); 27 | } 28 | String file = col.get().iterator().next(); 29 | 30 | // Run benchmark 31 | Collection results = new Runner(cmdOpts).run(); 32 | 33 | // Print score in MB/sec (if benchmark mode is throughput) 34 | System.out.println(); 35 | long fileSize = new File(file).length(); 36 | for (RunResult rr : results) { 37 | Result r = rr.getPrimaryResult(); 38 | String algo = rr.getParams().getParam("algorithm"); 39 | String name = rr.getParams().getBenchmark() + (algo != null ? " (" + algo + ")" : ""); 40 | if (r.getScoreUnit().equals("ops/s")) { 41 | double speed = r.getScore() * fileSize / (1024 * 1024); 42 | System.out.printf("%-54s%.2f MB/sec%n", name, speed); 43 | } 44 | } 45 | } catch (RunnerException e) { 46 | System.err.printf("%s: %s", "Error", e.getMessage()); 47 | } catch (CommandLineOptionException e) { 48 | System.err.printf("%s: %s", "Error parsing command line", e.getMessage()); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /jmh/qat-java-bench/src/main/java/com/intel/qat/jmh/JavaUtilZipBench.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | 7 | package com.intel.qat.jmh; 8 | 9 | import java.nio.file.Files; 10 | import java.nio.file.Paths; 11 | import java.util.concurrent.atomic.AtomicBoolean; 12 | import java.util.zip.DataFormatException; 13 | import java.util.zip.Deflater; 14 | import java.util.zip.Inflater; 15 | import org.openjdk.jmh.annotations.Benchmark; 16 | import org.openjdk.jmh.annotations.Param; 17 | import org.openjdk.jmh.annotations.Scope; 18 | import org.openjdk.jmh.annotations.State; 19 | 20 | @State(Scope.Benchmark) 21 | public class JavaUtilZipBench { 22 | private static AtomicBoolean flag = new AtomicBoolean(false); 23 | 24 | @Param({""}) 25 | static String file; 26 | 27 | @Param({"6"}) 28 | static int level; 29 | 30 | @Param({"65536"}) 31 | static int chunkSize; 32 | 33 | @State(Scope.Thread) 34 | public static class ThreadState { 35 | byte[] buf; 36 | byte[] src; 37 | byte[] compressed; 38 | byte[] decompressed; 39 | int maxChunkSize; 40 | 41 | public ThreadState() { 42 | try { 43 | // Read input 44 | src = Files.readAllBytes(Paths.get(file)); 45 | int srclen = src.length; 46 | 47 | // Max chunk size is 25% more, just in case. 48 | maxChunkSize = (int) (chunkSize * 1.25); 49 | 50 | // Temporary buffer for compressed file 51 | buf = new byte[((srclen + chunkSize - 1) / chunkSize) * maxChunkSize]; 52 | 53 | // Compress 54 | Deflater deflater = new Deflater(level); 55 | int off = 0; 56 | int pos = 0; 57 | while (off < srclen) { 58 | int csize = Math.min(chunkSize, srclen - off); 59 | deflater.setInput(src, off, csize); 60 | deflater.finish(); 61 | int c = deflater.deflate(buf, pos, csize); 62 | pos += c; 63 | off += deflater.getBytesRead(); 64 | deflater.reset(); 65 | } 66 | deflater.end(); 67 | 68 | // Prepare compressed array of size EXACTLY compLen 69 | int clen = pos; 70 | compressed = new byte[clen]; 71 | System.arraycopy(buf, 0, compressed, 0, compressed.length); 72 | 73 | // Buffer for decompression 74 | decompressed = new byte[src.length]; 75 | 76 | // Decompress 77 | Inflater inflater = new Inflater(); 78 | off = 0; 79 | pos = 0; 80 | while (off < clen) { 81 | inflater.setInput(compressed, off, Math.min(chunkSize, clen - off)); 82 | int c = 83 | inflater.inflate(decompressed, pos, Math.min(chunkSize, decompressed.length - pos)); 84 | pos += c; 85 | off += inflater.getBytesRead(); 86 | inflater.reset(); 87 | } 88 | inflater.end(); 89 | 90 | if (flag.compareAndSet(false, true)) { 91 | System.out.println("\n------------------------"); 92 | System.out.printf("Compression ratio: %.2f%n", (double) src.length / clen); 93 | System.out.println("------------------------"); 94 | } 95 | } catch (Exception e) { 96 | e.printStackTrace(); 97 | } 98 | } 99 | } 100 | 101 | @Benchmark 102 | public void compress(ThreadState state) { 103 | Deflater deflater = new Deflater(level); 104 | int off = 0; 105 | int pos = 0; 106 | int srclen = state.src.length; 107 | while (off < srclen) { 108 | int csize = Math.min(chunkSize, srclen - off); 109 | deflater.setInput(state.src, off, csize); 110 | deflater.finish(); 111 | int c = deflater.deflate(state.buf, pos, csize); 112 | pos += c; 113 | off += deflater.getBytesRead(); 114 | deflater.reset(); 115 | } 116 | deflater.end(); 117 | } 118 | 119 | @Benchmark 120 | public void decompress(ThreadState state) throws DataFormatException { 121 | Inflater inflater = new Inflater(); 122 | int off = 0; 123 | int pos = 0; 124 | int clen = state.compressed.length; 125 | while (off < clen) { 126 | inflater.setInput(state.compressed, off, Math.min(chunkSize, clen - off)); 127 | int c = 128 | inflater.inflate( 129 | state.decompressed, pos, Math.min(chunkSize, state.decompressed.length - pos)); 130 | pos += c; 131 | off += inflater.getBytesRead(); 132 | inflater.reset(); 133 | } 134 | inflater.end(); 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /jmh/qat-java-bench/src/main/java/com/intel/qat/jmh/JavaUtilZipStreamBench.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | 7 | package com.intel.qat.jmh; 8 | 9 | import java.io.ByteArrayInputStream; 10 | import java.io.ByteArrayOutputStream; 11 | import java.io.IOException; 12 | import java.nio.file.Files; 13 | import java.nio.file.Paths; 14 | import java.util.concurrent.atomic.AtomicBoolean; 15 | import java.util.zip.Deflater; 16 | import java.util.zip.DeflaterOutputStream; 17 | import java.util.zip.Inflater; 18 | import java.util.zip.InflaterInputStream; 19 | import org.openjdk.jmh.annotations.Benchmark; 20 | import org.openjdk.jmh.annotations.Param; 21 | import org.openjdk.jmh.annotations.Scope; 22 | import org.openjdk.jmh.annotations.State; 23 | 24 | @State(Scope.Benchmark) 25 | public class JavaUtilZipStreamBench { 26 | private static final int BUFFER_SIZE = 1 << 16; // 64KB 27 | private static AtomicBoolean flag = new AtomicBoolean(false); 28 | 29 | @Param({""}) 30 | static String file; 31 | 32 | @Param({"6"}) 33 | static int level; 34 | 35 | @State(Scope.Thread) 36 | public static class ThreadState { 37 | byte[] src; 38 | byte[] compressed; 39 | 40 | public ThreadState() { 41 | try { 42 | // Read input 43 | src = Files.readAllBytes(Paths.get(file)); 44 | 45 | // Compress input using streams 46 | ByteArrayOutputStream compressedOutput = new ByteArrayOutputStream(); 47 | Deflater deflater = new Deflater(level); 48 | DeflaterOutputStream outputStream = 49 | new DeflaterOutputStream(compressedOutput, deflater, BUFFER_SIZE); 50 | outputStream.write(src, 0, src.length); 51 | outputStream.close(); 52 | 53 | compressed = compressedOutput.toByteArray(); 54 | 55 | if (flag.compareAndSet(false, true)) { 56 | System.out.println("\n------------------------"); 57 | System.out.printf("Compression ratio: %.2f%n", (double) src.length / compressed.length); 58 | System.out.println("------------------------"); 59 | } 60 | } catch (IOException e) { 61 | e.printStackTrace(); 62 | } 63 | } 64 | } 65 | 66 | @Benchmark 67 | public void compress(ThreadState state) throws IOException { 68 | ByteArrayOutputStream compressedOutput = new ByteArrayOutputStream(); 69 | DeflaterOutputStream outputStream = 70 | new DeflaterOutputStream(compressedOutput, new Deflater(level), BUFFER_SIZE); 71 | outputStream.write(state.src, 0, state.src.length); 72 | outputStream.close(); 73 | } 74 | 75 | @Benchmark 76 | public void decompress(ThreadState state) throws IOException { 77 | ByteArrayInputStream compressedInput = new ByteArrayInputStream(state.compressed); 78 | InflaterInputStream inputStream = 79 | new InflaterInputStream(compressedInput, new Inflater(), BUFFER_SIZE); 80 | 81 | int bytesRead = 0; 82 | byte[] buffer = new byte[BUFFER_SIZE]; 83 | while ((bytesRead = inputStream.read(buffer)) != -1) 84 | ; 85 | inputStream.close(); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /jmh/qat-java-bench/src/main/java/com/intel/qat/jmh/QatJavaBench.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | 7 | package com.intel.qat.jmh; 8 | 9 | import com.intel.qat.QatZipper; 10 | import com.intel.qat.QatZipper.Algorithm; 11 | import java.io.ByteArrayOutputStream; 12 | import java.io.IOException; 13 | import java.nio.ByteBuffer; 14 | import java.nio.file.Files; 15 | import java.nio.file.Paths; 16 | import java.util.concurrent.atomic.AtomicBoolean; 17 | import org.openjdk.jmh.annotations.Benchmark; 18 | import org.openjdk.jmh.annotations.Param; 19 | import org.openjdk.jmh.annotations.Scope; 20 | import org.openjdk.jmh.annotations.State; 21 | 22 | @State(Scope.Benchmark) 23 | public class QatJavaBench { 24 | private static AtomicBoolean flag = new AtomicBoolean(false); 25 | 26 | @Param({""}) 27 | static String file; 28 | 29 | @Param({"6"}) 30 | static int level; 31 | 32 | @Param({"65536"}) 33 | static int blockSize; 34 | 35 | @Param({"DEFLATE", "LZ4", "ZSTD"}) 36 | static String algorithm; 37 | 38 | @State(Scope.Thread) 39 | public static class ThreadState { 40 | byte[] src; 41 | byte[] compressed; 42 | byte[] decompressed; 43 | Algorithm algorithm; 44 | 45 | public ThreadState() { 46 | try { 47 | 48 | switch (QatJavaBench.algorithm) { 49 | case "DEFLATE": 50 | algorithm = Algorithm.DEFLATE; 51 | break; 52 | case "LZ4": 53 | algorithm = Algorithm.LZ4; 54 | break; 55 | case "ZSTD": 56 | algorithm = Algorithm.ZSTD; 57 | break; 58 | default: 59 | throw new IllegalArgumentException("Invalid algorithm. Supported are DEFLATE and LZ4."); 60 | } 61 | 62 | QatZipper qzip = new QatZipper.Builder().setAlgorithm(algorithm).setLevel(level).build(); 63 | 64 | // Read input 65 | src = Files.readAllBytes(Paths.get(file)); 66 | 67 | int maxCompressedSize = qzip.maxCompressedLength(blockSize); 68 | byte[] compressedBlock = new byte[maxCompressedSize]; 69 | ByteArrayOutputStream compressedOut = new ByteArrayOutputStream(); 70 | for (int offset = 0; offset < src.length; offset += blockSize) { 71 | int length = Math.min(blockSize, src.length - offset); 72 | 73 | int compressedSize = 74 | qzip.compress(src, offset, length, compressedBlock, 0, maxCompressedSize); 75 | 76 | // Write the size of the compressed block and then the block itself. 77 | compressedOut.write(intToBytes(compressedSize)); 78 | compressedOut.write(compressedBlock, 0, compressedSize); 79 | } 80 | compressed = compressedOut.toByteArray(); 81 | 82 | ByteArrayOutputStream decompressedOut = new ByteArrayOutputStream(); 83 | int compressedBlockSize = 0; 84 | for (int offset = 0; offset < compressed.length; offset += compressedBlockSize) { 85 | compressedBlockSize = bytesToInt(compressed, offset); 86 | offset += 4; 87 | 88 | byte[] decompressedBlock = new byte[blockSize]; 89 | qzip.decompress(compressed, offset, compressedBlockSize, decompressedBlock, 0, blockSize); 90 | decompressedOut.write(decompressedBlock); 91 | } 92 | decompressed = decompressedOut.toByteArray(); 93 | 94 | qzip.end(); 95 | 96 | if (flag.compareAndSet(false, true)) { 97 | System.out.println("\n------------------------"); 98 | System.out.printf("Compression ratio: %.2f%n", (double) src.length / compressed.length); 99 | System.out.println("------------------------"); 100 | } 101 | } catch (IOException e) { 102 | e.printStackTrace(); 103 | } 104 | } 105 | } 106 | 107 | @Benchmark 108 | public void compress(ThreadState state) throws IOException { 109 | QatZipper qzip = new QatZipper.Builder().setAlgorithm(state.algorithm).setLevel(level).build(); 110 | int maxCompressedSize = qzip.maxCompressedLength(blockSize); 111 | byte[] compressedBlock = new byte[maxCompressedSize]; 112 | ByteArrayOutputStream compressedOut = new ByteArrayOutputStream(); 113 | for (int offset = 0; offset < state.src.length; offset += blockSize) { 114 | int length = Math.min(blockSize, state.src.length - offset); 115 | 116 | int compressedSize = 117 | qzip.compress(state.src, offset, length, compressedBlock, 0, maxCompressedSize); 118 | 119 | // Write the size of the compressed block and then the block itself. 120 | compressedOut.write(intToBytes(compressedSize)); 121 | compressedOut.write(compressedBlock, 0, compressedSize); 122 | } 123 | state.compressed = compressedOut.toByteArray(); 124 | qzip.end(); 125 | } 126 | 127 | @Benchmark 128 | public static void decompress(ThreadState state) throws IOException { 129 | QatZipper qzip = new QatZipper.Builder().setAlgorithm(state.algorithm).setLevel(level).build(); 130 | ByteArrayOutputStream decompressedOut = new ByteArrayOutputStream(); 131 | int compressedBlockSize = 0; 132 | for (int offset = 0; offset < state.compressed.length; offset += compressedBlockSize) { 133 | compressedBlockSize = bytesToInt(state.compressed, offset); 134 | offset += 4; 135 | 136 | byte[] decompressedBlock = new byte[blockSize]; 137 | qzip.decompress( 138 | state.compressed, offset, compressedBlockSize, decompressedBlock, 0, blockSize); 139 | 140 | decompressedOut.write(decompressedBlock); 141 | } 142 | state.decompressed = decompressedOut.toByteArray(); 143 | qzip.end(); 144 | } 145 | 146 | private static byte[] intToBytes(int value) { 147 | return ByteBuffer.allocate(4).putInt(value).array(); 148 | } 149 | 150 | private static int bytesToInt(byte[] bytes, int offset) { 151 | return ByteBuffer.wrap(bytes, offset, 4).getInt(); 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /jmh/qat-java-bench/src/main/java/com/intel/qat/jmh/QatJavaStreamBench.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | 7 | package com.intel.qat.jmh; 8 | 9 | import static com.intel.qat.QatZipper.Algorithm; 10 | import static com.intel.qat.QatZipper.Builder; 11 | 12 | import com.intel.qat.QatCompressorOutputStream; 13 | import com.intel.qat.QatDecompressorInputStream; 14 | import com.intel.qat.QatZipper; 15 | import java.io.ByteArrayInputStream; 16 | import java.io.ByteArrayOutputStream; 17 | import java.io.IOException; 18 | import java.nio.file.Files; 19 | import java.nio.file.Paths; 20 | import java.util.concurrent.atomic.AtomicBoolean; 21 | import org.openjdk.jmh.annotations.Benchmark; 22 | import org.openjdk.jmh.annotations.Param; 23 | import org.openjdk.jmh.annotations.Scope; 24 | import org.openjdk.jmh.annotations.State; 25 | 26 | @State(Scope.Benchmark) 27 | public class QatJavaStreamBench { 28 | private static final int BUFFER_SIZE = 1 << 16; // 64KB 29 | private static AtomicBoolean flag = new AtomicBoolean(false); 30 | 31 | @Param({""}) 32 | static String file; 33 | 34 | @Param({"6"}) 35 | static int level; 36 | 37 | @Param({"DEFLATE", "LZ4", "ZSTD"}) 38 | static String algorithm; 39 | 40 | @State(Scope.Thread) 41 | public static class ThreadState { 42 | byte[] src; 43 | byte[] compressed; 44 | Algorithm algorithm; 45 | 46 | public ThreadState() { 47 | try { 48 | switch (QatJavaStreamBench.algorithm) { 49 | case "DEFLATE": 50 | algorithm = Algorithm.DEFLATE; 51 | break; 52 | case "LZ4": 53 | algorithm = Algorithm.LZ4; 54 | break; 55 | case "ZSTD": 56 | algorithm = Algorithm.ZSTD; 57 | break; 58 | default: 59 | throw new IllegalArgumentException("Invalid algorithm. Supported are DEFLATE and LZ4."); 60 | } 61 | 62 | // Read input 63 | src = Files.readAllBytes(Paths.get(file)); 64 | 65 | // Compress input using streams 66 | ByteArrayOutputStream compressedOutput = new ByteArrayOutputStream(); 67 | Builder builder = new QatZipper.Builder().setAlgorithm(algorithm).setLevel(level); 68 | QatCompressorOutputStream qatOutputStream = 69 | new QatCompressorOutputStream(compressedOutput, BUFFER_SIZE, builder); 70 | qatOutputStream.write(src); 71 | qatOutputStream.close(); 72 | 73 | // Get compressed data from stream 74 | compressed = compressedOutput.toByteArray(); 75 | 76 | if (flag.compareAndSet(false, true)) { 77 | System.out.println("\n------------------------"); 78 | System.out.printf("Compression ratio: %.2f%n", (double) src.length / compressed.length); 79 | System.out.println("------------------------"); 80 | } 81 | } catch (IOException e) { 82 | e.printStackTrace(); 83 | } 84 | } 85 | } 86 | 87 | @Benchmark 88 | public void compress(ThreadState state) throws IOException { 89 | ByteArrayOutputStream compressedOutput = new ByteArrayOutputStream(); 90 | Builder builder = new Builder().setAlgorithm(state.algorithm).setLevel(level); 91 | QatCompressorOutputStream qatOutputStream = 92 | new QatCompressorOutputStream(compressedOutput, BUFFER_SIZE, builder); 93 | qatOutputStream.write(state.src); 94 | qatOutputStream.close(); 95 | } 96 | 97 | @Benchmark 98 | public void decompress(ThreadState state) throws IOException { 99 | ByteArrayInputStream compressedInput = new ByteArrayInputStream(state.compressed); 100 | Builder builder = new Builder().setAlgorithm(state.algorithm).setLevel(level); 101 | QatDecompressorInputStream qatInputStream = 102 | new QatDecompressorInputStream(compressedInput, BUFFER_SIZE, builder); 103 | 104 | int bytesRead = 0; 105 | byte[] buffer = new byte[BUFFER_SIZE]; 106 | while ((bytesRead = qatInputStream.read(buffer)) != -1) 107 | ; 108 | qatInputStream.close(); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | com.intel.qat 7 | qat-java 8 | 2.3.2 9 | jar 10 | qat-java 11 | Qat-Java is a data compression library that uses Intel® QAT to speed up compression. 12 | https://github.com/intel/qat-java 13 | 14 | 15 | BSD License 16 | https://opensource.org/license/bsd-3-clause 17 | 18 | 19 | 20 | scm:git:git://github.com/intel/qat-java.git 21 | scm:git:git@github.com:intel/qat-java.git 22 | https://github.com/intel/qat-java 23 | 24 | 25 | 26 | Mammo, Mulugeta 27 | mulugeta.mammo@intel.com 28 | Intel 29 | 30 | 31 | 32 | Denloye, Olasoji 33 | olasoji.denloye@intel.com 34 | Intel 35 | 36 | 37 | 38 | 39 | UTF-8 40 | UTF-8 41 | ${fuzzing} 42 | -runs=10000 -max_total_time=300 -max_len=4194304 43 | 44 | 45 | 46 | 47 | org.apache.maven.plugins 48 | maven-compiler-plugin 49 | 3.11.0 50 | 51 | 11 52 | true 53 | 54 | 55 | 56 | org.apache.maven.plugins 57 | maven-enforcer-plugin 58 | 3.4.1 59 | 60 | 61 | enforce-java 62 | 63 | enforce 64 | 65 | 66 | 67 | 68 | [11,) 69 | 70 | 71 | true 72 | 73 | 74 | 75 | 76 | 77 | org.apache.maven.plugins 78 | maven-jar-plugin 79 | 3.3.0 80 | 81 | 82 | 83 | true 84 | 85 | 86 | 87 | com/intel/qat/examples/** 88 | 89 | 90 | 91 | 92 | org.apache.maven.plugins 93 | maven-surefire-plugin 94 | 3.2.2 95 | 96 | @{argLine} -Djava.library.path=${project.build.directory}/cbuild 97 | 98 | 99 | 100 | org.apache.maven.plugins 101 | maven-site-plugin 102 | 4.0.0-M12 103 | 104 | 105 | org.apache.maven.plugins 106 | maven-project-info-reports-plugin 107 | 3.5.0 108 | 109 | 110 | org.apache.maven.plugins 111 | maven-source-plugin 112 | 3.3.0 113 | 114 | 115 | attach-sources 116 | 117 | jar 118 | 119 | 120 | 121 | 122 | 123 | org.apache.maven.plugins 124 | maven-javadoc-plugin 125 | 3.6.3 126 | 127 | 8 128 | 23.0.2 129 | false 130 | com.intel.qat.examples 131 | 132 | 133 | 134 | attach-javadocs 135 | 136 | jar 137 | 138 | 139 | 140 | 141 | 142 | org.jacoco 143 | jacoco-maven-plugin 144 | 0.8.11 145 | 146 | ${basedir}/target/coverage-reports/jacoco-unit.exec 147 | ${basedir}/target/coverage-reports/jacoco-unit.exec 148 | 149 | 150 | 151 | jacoco-initialize 152 | 153 | prepare-agent 154 | 155 | 156 | 157 | jacoco-report 158 | test 159 | 160 | report 161 | 162 | 163 | 164 | 165 | 166 | com.diffplug.spotless 167 | spotless-maven-plugin 168 | 2.41.1 169 | 170 | 171 | format 172 | process-sources 173 | 174 | check 175 | apply 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | *.java 184 | 185 | 186 | 187 | 188 | 189 | 1.18.1 190 | 191 | 192 | 193 | 194 | 195 | org.apache.maven.plugins 196 | maven-antrun-plugin 197 | 3.1.0 198 | 199 | 200 | build-cpp 201 | generate-sources 202 | 203 | 204 | 206 | 207 | 208 | 209 | 210 | 211 | 213 | 214 | 215 | 217 | 218 | 219 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 232 | 233 | 234 | 235 | 236 | 237 | run 238 | 239 | 240 | 241 | clean-native 242 | clean 243 | 244 | run 245 | 246 | 247 | 248 | Builddir: ${project.build.directory} 249 | Basedir: ${basedir} 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | ant-contrib 258 | ant-contrib 259 | 1.0b3 260 | 261 | 262 | ant 263 | ant 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | org.apache.maven.plugins 275 | maven-surefire-report-plugin 276 | 3.2.2 277 | 278 | false 279 | 280 | 281 | 282 | 283 | 284 | 285 | com.github.luben 286 | zstd-jni 287 | 1.5.6-1 288 | 289 | 290 | org.junit.platform 291 | junit-platform-suite 292 | 1.10.1 293 | test 294 | 295 | 296 | org.junit.jupiter 297 | junit-jupiter 298 | 5.10.1 299 | test 300 | 301 | 302 | com.code-intelligence 303 | jazzer 304 | 0.22.1 305 | 306 | 307 | 308 | 309 | fuzz 310 | 311 | 312 | fuzzing 313 | true 314 | 315 | 316 | 317 | 318 | 319 | exec-maven-plugin 320 | 3.0.0 321 | org.codehaus.mojo 322 | 323 | 324 | FuzzerTest 325 | test 326 | 327 | exec 328 | 329 | 330 | test 331 | java 332 | 333 | -ea 334 | -cp 335 | ${project.build.directory}/qat-java-2.3.2.jar 336 | -cp 337 | 338 | com.code_intelligence.jazzer.Jazzer 339 | 340 | --target_class=com.intel.qat.fuzzing.FuzzerTest 341 | 342 | --asan 343 | --native 344 | ${fuzzParameters} 345 | 346 | 347 | 348 | 349 | 350 | 351 | 352 | 353 | 354 | 355 | -------------------------------------------------------------------------------- /src/main/java/com/intel/qat/InternalJNI.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | package com.intel.qat; 7 | 8 | import java.nio.ByteBuffer; 9 | 10 | /** JNI wrapper, for internal use. */ 11 | enum InternalJNI { 12 | ; 13 | 14 | static { 15 | Native.loadLibrary(); 16 | initFieldIDs(); 17 | } 18 | 19 | static native void initFieldIDs(); 20 | 21 | static native int setup( 22 | QatZipper qzip, int algo, int level, int mode, int pmode, int dataFormat, int hwBufferSize); 23 | 24 | static native int maxCompressedSize(int qzKey, long sourceSize); 25 | 26 | static native int compressByteArray( 27 | QatZipper qzip, 28 | int qzKey, 29 | byte[] src, 30 | int srcOff, 31 | int srcLen, 32 | byte[] dst, 33 | int dstOff, 34 | int dstLen, 35 | int retryCount); 36 | 37 | static native int decompressByteArray( 38 | QatZipper qzip, 39 | int qzKey, 40 | byte[] src, 41 | int srcOff, 42 | int srcLen, 43 | byte[] dst, 44 | int dstOff, 45 | int dstLen, 46 | int retryCount); 47 | 48 | static native int compressByteBuffer( 49 | int qzKey, 50 | ByteBuffer srcBuffer, 51 | byte[] src, 52 | int srcOff, 53 | int srcLen, 54 | byte[] dst, 55 | int dstOff, 56 | int dstLen, 57 | int retryCount); 58 | 59 | static native int decompressByteBuffer( 60 | int qzKey, 61 | ByteBuffer srcBuffer, 62 | byte[] src, 63 | int srcOff, 64 | int srcLen, 65 | byte[] dst, 66 | int dstOff, 67 | int dstLen, 68 | int retryCount); 69 | 70 | static native int compressDirectByteBuffer( 71 | int qzKey, 72 | ByteBuffer src, 73 | int srcOff, 74 | int srcLen, 75 | ByteBuffer dst, 76 | int dstOff, 77 | int dstLen, 78 | int retryCount); 79 | 80 | static native int decompressDirectByteBuffer( 81 | int qzKey, 82 | ByteBuffer src, 83 | int srcOff, 84 | int srcLen, 85 | ByteBuffer dst, 86 | int dstOff, 87 | int dstLen, 88 | int retryCount); 89 | 90 | static native int compressDirectByteBufferSrc( 91 | int qzKey, 92 | ByteBuffer src, 93 | int srcOff, 94 | int srcLen, 95 | byte[] dstArr, 96 | int dstOff, 97 | int dstLen, 98 | int retryCount); 99 | 100 | static native int decompressDirectByteBufferSrc( 101 | int qzKey, 102 | ByteBuffer src, 103 | int srcOff, 104 | int srcLen, 105 | byte[] dstArr, 106 | int dstOff, 107 | int dstLen, 108 | int retryCount); 109 | 110 | static native int compressDirectByteBufferDst( 111 | int qzKey, 112 | ByteBuffer src, 113 | byte[] srcArr, 114 | int srcOff, 115 | int srcLen, 116 | ByteBuffer dst, 117 | int dstOff, 118 | int dstLen, 119 | int retryCount); 120 | 121 | static native int decompressDirectByteBufferDst( 122 | int qzKey, 123 | ByteBuffer src, 124 | byte[] srcArr, 125 | int srcOff, 126 | int srcLen, 127 | ByteBuffer dst, 128 | int dstOff, 129 | int dstLen, 130 | int retryCount); 131 | 132 | static native long zstdGetSeqProdFunction(); 133 | 134 | static native long zstdCreateSeqProdState(); 135 | 136 | static native void zstdFreeSeqProdState(long sequenceProdState); 137 | 138 | static native int teardown(int qzKey); 139 | } 140 | -------------------------------------------------------------------------------- /src/main/java/com/intel/qat/Native.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | 7 | package com.intel.qat; 8 | 9 | import java.io.File; 10 | import java.io.FileOutputStream; 11 | import java.io.IOException; 12 | import java.io.InputStream; 13 | import java.nio.file.Files; 14 | 15 | /** Class for loading system library - libqat-java.so */ 16 | class Native { 17 | private static boolean loaded = false; 18 | private static String extension = ""; 19 | 20 | @SuppressWarnings({"deprecation", "removal"}) 21 | static boolean isLoaded() { 22 | if (loaded) return true; 23 | try { 24 | SecurityManager sm = System.getSecurityManager(); 25 | if (sm == null) { 26 | System.loadLibrary("qat-java"); 27 | } else { 28 | java.security.PrivilegedAction pa = 29 | () -> { 30 | System.loadLibrary("qat-java"); 31 | return null; 32 | }; 33 | java.security.AccessController.doPrivileged(pa); 34 | } 35 | loaded = true; 36 | } catch (UnsatisfiedLinkError e) { 37 | loaded = false; 38 | } 39 | return loaded; 40 | } 41 | 42 | static String getLibName() { 43 | return "/com/intel/qat/" + getOSName() + "/" + getOSArch() + "/" + "libqat-java" + extension; 44 | } 45 | 46 | static String getOSName() { 47 | String os = System.getProperty("os.name"); 48 | String ret; 49 | if (os.contains("Linux")) { 50 | ret = "linux"; 51 | extension = ".so"; 52 | } else throw new UnsupportedOperationException("Operating System is not supported"); 53 | return ret; 54 | } 55 | 56 | static String getOSArch() { 57 | return System.getProperty("os.arch"); 58 | } 59 | 60 | @SuppressWarnings({"deprecation", "removal"}) 61 | static synchronized void loadLibrary() { 62 | if (isLoaded()) return; 63 | String libName = getLibName(); 64 | File tempNativeLib = null; 65 | File tempNativeLibLock = null; 66 | try (InputStream in = Native.class.getResourceAsStream(libName)) { 67 | if (in == null) { 68 | throw new UnsupportedOperationException( 69 | "Unsupported OS/arch, cannot find " + libName + ". Please try building from source."); 70 | } 71 | // To avoid race condition with other concurrently running Java processes 72 | // using qat-java create the .lck file first. 73 | 74 | tempNativeLibLock = File.createTempFile("libqat-java", extension + ".lck"); 75 | tempNativeLib = new File(tempNativeLibLock.getAbsolutePath().replaceFirst(".lck$", "")); 76 | try (FileOutputStream out = new FileOutputStream(tempNativeLib)) { 77 | byte[] buf = new byte[4096]; 78 | int bytesRead; 79 | while (true) { 80 | bytesRead = in.read(buf); 81 | if (bytesRead == -1) break; 82 | out.write(buf, 0, bytesRead); 83 | } 84 | } 85 | boolean isSymbolicLink = Files.isSymbolicLink(tempNativeLib.toPath()); 86 | if (isSymbolicLink) { 87 | throw new IOException( 88 | "Failed to load native qat-java library." 89 | + tempNativeLib.toPath() 90 | + " is a symbolic link."); 91 | } 92 | File finalTempNativeLib = tempNativeLib; 93 | SecurityManager sm = System.getSecurityManager(); 94 | if (sm == null) { 95 | System.load(finalTempNativeLib.getAbsolutePath()); 96 | } else { 97 | java.security.PrivilegedAction pa = 98 | () -> { 99 | System.load(finalTempNativeLib.getAbsolutePath()); 100 | return null; 101 | }; 102 | java.security.AccessController.doPrivileged(pa); 103 | } 104 | loaded = true; 105 | } catch (IOException e) { 106 | throw new ExceptionInInitializerError( 107 | "Failed to load native qat-java library.\n" + e.getMessage()); 108 | } finally { 109 | if (tempNativeLib != null) tempNativeLib.deleteOnExit(); 110 | if (tempNativeLibLock != null) tempNativeLibLock.deleteOnExit(); 111 | } 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/main/java/com/intel/qat/QatCompressorOutputStream.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | 7 | package com.intel.qat; 8 | 9 | import static com.intel.qat.QatZipper.Algorithm; 10 | 11 | import com.intel.qat.QatZipper.Builder; 12 | import java.io.FilterOutputStream; 13 | import java.io.IOException; 14 | import java.io.OutputStream; 15 | import java.util.Objects; 16 | 17 | /** 18 | * This class implements an OutputStream that compresses data using Intel ® QuickAssist 19 | * Technology (QAT). 20 | */ 21 | public class QatCompressorOutputStream extends FilterOutputStream { 22 | private byte[] inputBuffer; 23 | private int inputPosition; 24 | private QatZipper qzip; 25 | private byte[] outputBuffer; 26 | private int outputPosition; 27 | private boolean closed; 28 | 29 | /** The default size in bytes of the output buffer (64KB). */ 30 | public static final int DEFAULT_BUFFER_SIZE = 1 << 16; 31 | 32 | /** 33 | * Creates a new output stream with the given parameters. 34 | * 35 | * @param out the output stream 36 | * @param bufferSize the output buffer size 37 | * @param builder pre configured QatZipper builder 38 | */ 39 | public QatCompressorOutputStream(OutputStream out, int bufferSize, Builder builder) { 40 | super(out); 41 | if (bufferSize <= 0) throw new IllegalArgumentException(); 42 | Objects.requireNonNull(out); 43 | qzip = builder.build(); 44 | inputBuffer = new byte[bufferSize]; 45 | outputBuffer = new byte[qzip.maxCompressedLength(bufferSize)]; 46 | closed = false; 47 | } 48 | 49 | /** 50 | * Creates a new output stream with the given parameters. 51 | * 52 | * @param out the output stream 53 | * @param algorithm the compression algorithm 54 | */ 55 | public QatCompressorOutputStream(OutputStream out, Algorithm algorithm) { 56 | this(out, DEFAULT_BUFFER_SIZE, algorithm); 57 | } 58 | 59 | /** 60 | * Creates a new output stream with the given parameters. 61 | * 62 | * @param out the output stream 63 | * @param bufferSize the output buffer size 64 | * @param algorithm the compression algorithm . 65 | */ 66 | public QatCompressorOutputStream(OutputStream out, int bufferSize, Algorithm algorithm) { 67 | this(out, bufferSize, new Builder().setAlgorithm(algorithm)); 68 | } 69 | 70 | /** 71 | * Writes a byte to the compressed output stream. 72 | * 73 | * @param b the data to be written 74 | * @throws IOException if this stream is closed 75 | */ 76 | @Override 77 | public void write(int b) throws IOException { 78 | if (closed) throw new IOException("Stream is closed"); 79 | if (inputPosition == inputBuffer.length) { 80 | flush(); 81 | } 82 | inputBuffer[inputPosition++] = (byte) b; 83 | } 84 | 85 | /** 86 | * Writes data from the given byte array to the compressed output stream. 87 | * 88 | * @param b the data to be written 89 | * @throws IOException if this stream is closed 90 | */ 91 | @Override 92 | public void write(byte[] b) throws IOException { 93 | write(b, 0, b.length); 94 | } 95 | 96 | /** 97 | * Writes data from the given byte array to the compressed output stream. 98 | * 99 | * @param b the data to be written 100 | * @param off the starting offset of the data 101 | * @param len the length of the data 102 | * @throws IOException if this stream is closed 103 | */ 104 | @Override 105 | public void write(byte[] b, int off, int len) throws IOException { 106 | if (closed) throw new IOException("Stream is closed"); 107 | Objects.requireNonNull(b); 108 | if (off < 0 || len < 0 || off + len > b.length) throw new IndexOutOfBoundsException(); 109 | 110 | int bytesToWrite = 0; 111 | while (len > (bytesToWrite = (inputBuffer.length - inputPosition))) { 112 | System.arraycopy(b, off, inputBuffer, inputPosition, bytesToWrite); 113 | inputPosition += bytesToWrite; 114 | len -= bytesToWrite; 115 | off += bytesToWrite; 116 | flush(); 117 | } 118 | System.arraycopy(b, off, inputBuffer, inputPosition, len); 119 | inputPosition += len; 120 | } 121 | 122 | /** 123 | * Flushes all buffered data to the compressed output stream. This method will compress and write 124 | * all buffered data to the output stream. 125 | * 126 | * @throws IOException if this stream is closed 127 | */ 128 | @Override 129 | public void flush() throws IOException { 130 | if (closed) throw new IOException("Stream is closed"); 131 | if (inputPosition == 0) return; 132 | int currentPosition = inputPosition; 133 | inputPosition = 0; 134 | int compressedBytes = 135 | qzip.compress( 136 | inputBuffer, 137 | inputPosition, 138 | currentPosition, 139 | outputBuffer, 140 | outputPosition, 141 | outputBuffer.length - outputPosition); 142 | out.write(outputBuffer, 0, compressedBytes); 143 | out.flush(); 144 | outputPosition = 0; 145 | } 146 | 147 | /** 148 | * Writes any remaining data to the compressed output stream and releases resources. This method 149 | * will close the underlying output stream. 150 | * 151 | * @throws IOException if an I/O error occurs 152 | */ 153 | @Override 154 | public void close() throws IOException { 155 | if (closed) return; 156 | flush(); 157 | qzip.end(); 158 | out.close(); 159 | inputBuffer = null; 160 | outputBuffer = null; 161 | closed = true; 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /src/main/java/com/intel/qat/QatDecompressorInputStream.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | 7 | package com.intel.qat; 8 | 9 | import static com.intel.qat.QatZipper.Algorithm; 10 | 11 | import com.github.luben.zstd.ZstdException; 12 | import java.io.FilterInputStream; 13 | import java.io.IOException; 14 | import java.io.InputStream; 15 | import java.nio.BufferOverflowException; 16 | import java.util.Objects; 17 | 18 | /** 19 | * This class implements an InputStream that decompresses data using Intel ® QuickAssist 20 | * Technology (QAT). 21 | */ 22 | public class QatDecompressorInputStream extends FilterInputStream { 23 | private byte[] inputBuffer; 24 | private byte[] outputBuffer; 25 | private int inputPosition; 26 | private int outputPosition; 27 | private int inputBufferLimit; 28 | private int outputBufferLimit; 29 | private QatZipper qzip; 30 | private boolean closed; 31 | private boolean eof; 32 | 33 | /** The default size in bytes of the input buffer (64KB). */ 34 | public static final int DEFAULT_BUFFER_SIZE = 1 << 16; 35 | 36 | /** The maximum buffer size in bytes of the input buffer (512KB). */ 37 | public static int MAX_BUFFER_SIZE = 512 * 1024; 38 | 39 | /** 40 | * Creates a new input stream with the given parameters. 41 | * 42 | * @param in the input stream 43 | * @param bufferSize the input buffer size 44 | * @param algorithm the compression algorithm. 45 | */ 46 | public QatDecompressorInputStream(InputStream in, int bufferSize, Algorithm algorithm) { 47 | this(in, bufferSize, new QatZipper.Builder().setAlgorithm(algorithm)); 48 | } 49 | 50 | /** 51 | * Creates a new input stream with the given parameters. 52 | * 53 | * @param in the input stream 54 | * @param algorithm the compression algorithm. 55 | */ 56 | public QatDecompressorInputStream(InputStream in, Algorithm algorithm) { 57 | this(in, DEFAULT_BUFFER_SIZE, algorithm); 58 | } 59 | 60 | /** 61 | * Creates a new input stream with the given parameters. 62 | * 63 | * @param in the input stream 64 | * @param bufferSize the input buffer size 65 | * @param builder pre configured QatZipper builder 66 | */ 67 | public QatDecompressorInputStream(InputStream in, int bufferSize, QatZipper.Builder builder) { 68 | super(in); 69 | if (bufferSize <= 0) throw new IllegalArgumentException(); 70 | Objects.requireNonNull(in); 71 | if (builder.getAlgorithm().equals(Algorithm.ZSTD)) MAX_BUFFER_SIZE = Integer.MAX_VALUE - 1; 72 | int inputBufferSize = 0; 73 | try { 74 | inputBufferSize = Math.max(bufferSize, in.available()); 75 | inputBufferSize = Math.min(MAX_BUFFER_SIZE, inputBufferSize); 76 | } catch (IOException ioe) { 77 | inputBufferSize = bufferSize; 78 | } 79 | inputBuffer = new byte[inputBufferSize]; 80 | outputBuffer = new byte[bufferSize]; 81 | outputPosition = outputBuffer.length; 82 | inputBufferLimit = inputBuffer.length; 83 | outputBufferLimit = outputBuffer.length; 84 | closed = false; 85 | eof = false; 86 | this.qzip = builder.build(); 87 | } 88 | 89 | /** 90 | * Reads the next byte of uncompressed data. 91 | * 92 | * @return the next byte of data or -1 if the end of the stream is reached. 93 | * @throws IOException if the stream is closed 94 | */ 95 | @Override 96 | public int read() throws IOException { 97 | if (closed) throw new IOException("Stream is closed"); 98 | if (eof && outputPosition == outputBufferLimit) return -1; 99 | if (outputPosition == outputBufferLimit) { 100 | fill(); 101 | } 102 | if (eof && outputPosition == outputBufferLimit) return -1; 103 | return Byte.toUnsignedInt(outputBuffer[outputPosition++]); 104 | } 105 | 106 | /** 107 | * Reads uncompressed data into the provided array. 108 | * 109 | * @param b the array into which the data is read 110 | * @return the number of bytes read 111 | * @throws IOException if the stream is closed 112 | */ 113 | @Override 114 | public int read(byte[] b) throws IOException { 115 | return read(b, 0, b.length); 116 | } 117 | 118 | /** 119 | * Reads uncompressed data into the provided array. 120 | * 121 | * @param b the array into which the data is read 122 | * @param off the starting offset in the array 123 | * @param len the maximum number of bytes to be read 124 | * @return the number of bytes read 125 | * @throws IOException if the stream is closed 126 | */ 127 | @Override 128 | public int read(byte[] b, int off, int len) throws IOException { 129 | if (closed) throw new IOException("Stream is closed"); 130 | Objects.requireNonNull(b); 131 | if (off < 0 || len < 0 || off + len > b.length) throw new IndexOutOfBoundsException(); 132 | if (eof && outputPosition == outputBufferLimit) return -1; 133 | int result = 0; 134 | int bytesToRead = 0; 135 | while (len > (bytesToRead = outputBufferLimit - outputPosition)) { 136 | System.arraycopy(outputBuffer, outputPosition, b, off, bytesToRead); 137 | outputPosition += bytesToRead; 138 | len -= bytesToRead; 139 | result += bytesToRead; 140 | off += bytesToRead; 141 | if (eof) { 142 | return result == 0 ? -1 : result; 143 | } 144 | fill(); 145 | } 146 | System.arraycopy(outputBuffer, outputPosition, b, off, len); 147 | outputPosition += len; 148 | result += len; 149 | return result; 150 | } 151 | 152 | /** 153 | * Returns an estimate of the number of uncompressed bytes that can be read. 154 | * 155 | * @return 0 if and only if the end of the stream is reached 156 | * @throws IOException if the stream is closed 157 | */ 158 | @Override 159 | public int available() throws IOException { 160 | if (closed) throw new IOException("Stream is closed"); 161 | if (outputPosition != outputBufferLimit) return (outputBufferLimit - outputPosition); 162 | if (eof) return 0; 163 | else return 1; 164 | } 165 | 166 | /** 167 | * Closes this input stream and releases resources. This method will close the underlying output 168 | * stream. 169 | * 170 | * @throws IOException if an I/O error occurs 171 | */ 172 | @Override 173 | public void close() throws IOException { 174 | if (closed) return; 175 | qzip.end(); 176 | in.close(); 177 | inputBuffer = null; 178 | outputBuffer = null; 179 | closed = true; 180 | } 181 | 182 | /** Marks the current position in this input stream. This method does nothing. */ 183 | @Override 184 | public void mark(int readLimit) {} 185 | 186 | /** 187 | * Repositions this stream to the position at the time the mark method was last called. This 188 | * method does nothing but throw an IOException. 189 | * 190 | * @throws IOException when invoked. 191 | */ 192 | @Override 193 | public void reset() throws IOException { 194 | throw new IOException("mark/reset not supported"); 195 | } 196 | 197 | /** 198 | * Tests if this input stream supports the mark and reset methods. This method unconditionally 199 | * returns false 200 | * 201 | * @return false 202 | */ 203 | @Override 204 | public boolean markSupported() { 205 | return false; 206 | } 207 | 208 | /** 209 | * Skips up to n bytes of compressed bytes. 210 | * 211 | * @param n the maximum number of bytes to skip 212 | * @return the number of bytes skipped or 0 if n is negative. 213 | */ 214 | @Override 215 | public long skip(long n) throws IOException { 216 | if (n < 0) return 0; 217 | return read(new byte[(int) n]); 218 | } 219 | 220 | private void growOutputBuffer() { 221 | int oldSize = outputBuffer.length; 222 | if (oldSize == MAX_BUFFER_SIZE) throw new BufferOverflowException(); 223 | int newSize = (oldSize > Integer.MAX_VALUE / 2) ? MAX_BUFFER_SIZE : oldSize * 2; 224 | outputBuffer = new byte[newSize]; 225 | outputPosition = 0; 226 | outputBufferLimit = outputBuffer.length; 227 | } 228 | 229 | private void fill() throws IOException { 230 | if (eof) return; 231 | 232 | // Read from inputStream 233 | int bytesRead = 0; 234 | while (inputPosition < inputBufferLimit && bytesRead >= 0) { 235 | bytesRead = in.read(inputBuffer, inputPosition, inputBufferLimit - inputPosition); 236 | inputPosition += Math.max(0, bytesRead); 237 | if (bytesRead < 0 && inputPosition == 0) { 238 | eof = true; 239 | return; 240 | } 241 | } 242 | 243 | inputBufferLimit = inputPosition; 244 | inputPosition = 0; 245 | 246 | int decompressed = 0; 247 | outputPosition = 0; 248 | outputBufferLimit = outputBuffer.length; 249 | for (; (decompressed == 0 && inputPosition == 0); ) { 250 | try { 251 | decompressed = 252 | qzip.decompress( 253 | inputBuffer, 254 | inputPosition, 255 | inputBufferLimit, 256 | outputBuffer, 257 | outputPosition, 258 | outputBufferLimit - outputPosition); 259 | inputPosition += qzip.getBytesRead(); 260 | outputPosition += decompressed; 261 | } catch (ZstdException zstde) { 262 | growOutputBuffer(); 263 | } 264 | if (decompressed == 0 && inputPosition == 0) growOutputBuffer(); 265 | } 266 | outputBufferLimit = outputPosition; 267 | outputPosition = 0; 268 | if (inputPosition != inputBufferLimit) { 269 | System.arraycopy( 270 | inputBuffer, inputPosition, inputBuffer, 0, inputBufferLimit - inputPosition); 271 | inputPosition = inputBufferLimit - inputPosition; 272 | inputBufferLimit = inputBuffer.length; 273 | } else if (bytesRead < 0 && inputPosition == inputBufferLimit) eof = true; 274 | else { 275 | inputPosition = 0; 276 | inputBufferLimit = inputBuffer.length; 277 | } 278 | if (decompressed == 0) { 279 | fill(); 280 | } 281 | } 282 | } 283 | -------------------------------------------------------------------------------- /src/main/java/com/intel/qat/QatZipper.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | 7 | package com.intel.qat; 8 | 9 | import com.github.luben.zstd.Zstd; 10 | import com.github.luben.zstd.ZstdCompressCtx; 11 | import com.github.luben.zstd.ZstdDecompressCtx; 12 | import java.nio.ByteBuffer; 13 | import java.nio.ReadOnlyBufferException; 14 | 15 | /** 16 | * This class provides methods that can be used to compress and decompress data using {@link 17 | * Algorithm#DEFLATE} or {@link Algorithm#LZ4}. 18 | * 19 | *

The following code snippet demonstrates how to use the class to compress and decompress a 20 | * string. 21 | * 22 | *

23 | * 24 | * 
{@code
 25 |  * try {
 26 |  *   byte[] input = "Hello, world!".getBytes("UTF-8");
 27 |  *
 28 |  *   QatZipper qzip = new QatZipper.Builder().build();
 29 |  *
 30 |  *   // Create a buffer with enough size for compression
 31 |  *   byte[] output = new byte[qzip.maxCompressedLength(input.length)];
 32 |  *
 33 |  *   // Compress the bytes
 34 |  *   int clen = qzip.compress(input, output);
 35 |  *
 36 |  *   // Decompress the bytes into a String
 37 |  *   byte[] result = new byte[input.length];
 38 |  *   qzip.decompress(output, 0, clen, result, 0, result.length);
 39 |  *
 40 |  *   // Release resources
 41 |  *   qzip.end();
 42 |  *
 43 |  *   // Convert the bytes into a String
 44 |  *   String outputStr = new String(result, "UTF-8");
 45 |  * } catch (java.io.UnsupportedEncodingException e) {
 46 |  * //
 47 |  * } catch (RuntimeException e) {
 48 |  * //
 49 |  * }
 50 |  * }
51 | * 52 | *
53 | * 54 | * To release QAT resources used by this QatZipper, the end() method 55 | * should be called explicitly. 56 | */ 57 | public class QatZipper { 58 | /** The default compression algorithm. */ 59 | public static final Algorithm DEFAULT_ALGORITHM = Algorithm.DEFLATE; 60 | 61 | /** The default compression level is 6. */ 62 | public static final int DEFAULT_COMPRESS_LEVEL = 6; 63 | 64 | /** The default execution mode. */ 65 | public static final Mode DEFAULT_MODE = Mode.AUTO; 66 | 67 | /** 68 | * The default number of times QatZipper attempts to acquire hardware resources is 0. 69 | */ 70 | public static final int DEFAULT_RETRY_COUNT = 0; 71 | 72 | /** The default polling mode. */ 73 | public static final PollingMode DEFAULT_POLLING_MODE = PollingMode.BUSY; 74 | 75 | /** The default data format. */ 76 | public static final DataFormat DEFAULT_DATA_FORMAT = DataFormat.DEFLATE_GZIP_EXT; 77 | 78 | /** The default hardware buffer size. */ 79 | public static final HardwareBufferSize DEFAULT_HW_BUFFER_SIZE = 80 | HardwareBufferSize.DEFAULT_BUFFER_SIZE; 81 | 82 | /** The current compression algorithm. */ 83 | private Algorithm algorithm = DEFAULT_ALGORITHM; 84 | 85 | /** The compression level. */ 86 | private int level = DEFAULT_COMPRESS_LEVEL; 87 | 88 | /** The mode of execution. */ 89 | private Mode mode = DEFAULT_MODE; 90 | 91 | /** 92 | * The number of retry counts for session creation before Qat-Java gives up and throws an error. 93 | */ 94 | private int retryCount = DEFAULT_RETRY_COUNT; 95 | 96 | /** The polling mode. */ 97 | private PollingMode pollingMode = DEFAULT_POLLING_MODE; 98 | 99 | /** The data format. */ 100 | private DataFormat dataFormat = DEFAULT_DATA_FORMAT; 101 | 102 | /** The buffer size for QAT device. */ 103 | private HardwareBufferSize hwBufferSize = DEFAULT_HW_BUFFER_SIZE; 104 | 105 | /** Indicates if a QAT session is valid or not. */ 106 | private boolean isValid; 107 | 108 | /** Number of bytes read from the source by the most recent call to a compress/decompress. */ 109 | private int bytesRead; 110 | 111 | /** 112 | * Number of bytes written to the destination by the most recent call to a compress/decompress. 113 | */ 114 | private int bytesWritten; 115 | 116 | /** Id of this object (set by the JNI code) based on algorithm, etc.. */ 117 | private int qzKey; 118 | 119 | /** Zstd compress context. */ 120 | private ZstdCompressCtx zstdCompressCtx; 121 | 122 | /** Zstd decompress context. */ 123 | private ZstdDecompressCtx zstdDecompressCtx; 124 | 125 | /** Checksum flag, currently valid for only ZSTD. */ 126 | private boolean checksumFlag; 127 | 128 | /** The compression algorithm to use. DEFLATE and LZ4 are supported. */ 129 | public static enum Algorithm { 130 | /** The deflate compression algorithm. */ 131 | DEFLATE, 132 | 133 | /** The LZ4 compression algorithm. */ 134 | LZ4, 135 | 136 | /** The Zstandard compression algorithm. */ 137 | ZSTD 138 | } 139 | 140 | /** The mode of execution for QAT. */ 141 | public static enum Mode { 142 | /** 143 | * A hardware-only execution mode. QatZipper would fail if hardware resources cannot be acquired 144 | * after finite retries. 145 | */ 146 | HARDWARE, 147 | 148 | /** 149 | * A hardware execution mode with a software fail over. QatZipper would fail over to software 150 | * execution mode if hardware resources cannot be acquired after finite retries. 151 | */ 152 | AUTO; 153 | } 154 | 155 | /** 156 | * Polling mode dictates how QAT processes compression/decompression requests and waits for a 157 | * response, directly affecting the performance of these operations. Two polling modes are 158 | * supported: BUSY and PERIODICAL. BUSY polling is the default polling mode.
159 | *
160 | * Use BUSY polling mode when: 161 | * 162 | *
    163 | *
  • Your CPUs are not fully saturated and have cycles to spare. 164 | *
  • Your workload is latency-sensitive. 165 | *
166 | * 167 | *
168 | * Use PERIODICAL polling mode when: 169 | * 170 | *
    171 | *
  • Your workload has very high CPU utilization. 172 | *
  • Your workload is throughput-sensitive. 173 | *
174 | */ 175 | public static enum PollingMode { 176 | /** Use this mode unless your workload is CPU-bound. */ 177 | BUSY, 178 | 179 | /** Use this mode when your workload is CPU-bound. */ 180 | PERIODICAL 181 | } 182 | 183 | /** 184 | * The data format to use. Qat-Java supports the following:
185 | * 186 | *
    187 | *
  • DEFLATE_4B -- raw DEFLATE format with 4 byte header. 188 | *
  • DEFLATE_GZIP -- DEFLATE wrapped by GZip header and footer. 189 | *
  • DEFLATE_GZIP_EXT -- DEFLATE wrapped by GZip extended header and footer. 190 | *
  • DEFLATE_RAW -- raw DEFLATE format. 191 | *
  • ZLIB -- ZLIB 192 | *
193 | */ 194 | public static enum DataFormat { 195 | /** Raw DEFLATE format with 4 byte header. */ 196 | DEFLATE_4B, 197 | 198 | /** DEFLATE wrapped by GZip header and footer. */ 199 | DEFLATE_GZIP, 200 | 201 | /** DEFLATE wrapped by GZip extended header and footer. */ 202 | DEFLATE_GZIP_EXT, 203 | 204 | /** Raw DEFLATE format. */ 205 | DEFLATE_RAW, 206 | 207 | /** ZLIB */ 208 | ZLIB 209 | } 210 | 211 | /** 212 | * Hardware buffer size the QAT uses internally.
213 | * 214 | *
    215 | *
  • DEFAULT_BUFFER_SIZE -- 64KB 216 | *
  • MAX_BUFFER_SIZE -- 512KB 217 | *
218 | */ 219 | public static enum HardwareBufferSize { 220 | /** The default buffer size for the QAT device (64KB). */ 221 | DEFAULT_BUFFER_SIZE(64 * 1024), 222 | 223 | /** The maximum buffer size allowed for the QAT device (512KB). */ 224 | MAX_BUFFER_SIZE(512 * 1024); 225 | 226 | private final int value; 227 | 228 | private HardwareBufferSize(int hwBufferSize) { 229 | value = hwBufferSize; 230 | } 231 | 232 | /** 233 | * Gets the value of the hw buffer size. 234 | * 235 | * @return the value of the buffer size in bytes. 236 | */ 237 | public int getValue() { 238 | return value; 239 | } 240 | } 241 | 242 | /** 243 | * Checks if QAT hardware is available. 244 | * 245 | * @return true if QAT is available, false otherwise. 246 | */ 247 | public static boolean isQatAvailable() { 248 | return QatAvailableHolder.IS_QAT_AVAILABLE; 249 | } 250 | 251 | /** Defers static initialization until {@link #isQatAvailable()} is invoked. */ 252 | static class QatAvailableHolder { 253 | static final boolean IS_QAT_AVAILABLE; 254 | 255 | static { 256 | boolean isQatAvailable; 257 | try { 258 | final QatZipper qzip = new QatZipper.Builder().setMode(Mode.HARDWARE).build(); 259 | qzip.end(); 260 | isQatAvailable = true; 261 | } catch (UnsatisfiedLinkError 262 | | ExceptionInInitializerError 263 | | NoClassDefFoundError 264 | | RuntimeException e) { 265 | isQatAvailable = false; 266 | } 267 | IS_QAT_AVAILABLE = isQatAvailable; 268 | } 269 | } 270 | 271 | /** 272 | * Builder is used to build instances of {@link QatZipper} from values configured by the setters. 273 | */ 274 | public static class Builder { 275 | private Algorithm algorithm = DEFAULT_ALGORITHM; 276 | private int level = DEFAULT_COMPRESS_LEVEL; 277 | private Mode mode = DEFAULT_MODE; 278 | private int retryCount = DEFAULT_RETRY_COUNT; 279 | private PollingMode pollingMode = DEFAULT_POLLING_MODE; 280 | private DataFormat dataFormat = DEFAULT_DATA_FORMAT; 281 | private HardwareBufferSize hwBufferSize = DEFAULT_HW_BUFFER_SIZE; 282 | 283 | /** Constructs a builder that has default values for QatZipper. */ 284 | public Builder() {} 285 | 286 | /** 287 | * Sets the compression {@link Algorithm}. 288 | * 289 | * @param algorithm the {@link Algorithm}. 290 | * @return This Builder. 291 | */ 292 | public Builder setAlgorithm(Algorithm algorithm) { 293 | this.algorithm = algorithm; 294 | return this; 295 | } 296 | 297 | /** 298 | * Gets the compression {@link Algorithm}. 299 | * 300 | * @return The Algorithm. 301 | */ 302 | Algorithm getAlgorithm() { 303 | return this.algorithm; 304 | } 305 | 306 | /** 307 | * Sets the compression level. 308 | * 309 | * @param level the compression level. 310 | * @return This Builder. 311 | */ 312 | public Builder setLevel(int level) { 313 | this.level = level; 314 | return this; 315 | } 316 | 317 | /** 318 | * Sets the mode of execution. 319 | * 320 | * @param mode the {@link Mode}. 321 | * @return This Builder. 322 | */ 323 | public Builder setMode(Mode mode) { 324 | this.mode = mode; 325 | return this; 326 | } 327 | 328 | /** 329 | * Sets the number of tries to acquire hardware resouces before giving up. 330 | * 331 | * @param retryCount the {@link PollingMode}. 332 | * @return This Builder. 333 | */ 334 | public Builder setRetryCount(int retryCount) { 335 | this.retryCount = retryCount; 336 | return this; 337 | } 338 | 339 | /** 340 | * Sets the {@link PollingMode}. 341 | * 342 | * @param pollingMode the {@link PollingMode}. 343 | * @return This Builder. 344 | */ 345 | public Builder setPollingMode(PollingMode pollingMode) { 346 | this.pollingMode = pollingMode; 347 | return this; 348 | } 349 | 350 | /** 351 | * Sets the {@link DataFormat}. 352 | * 353 | * @param dataFormat the {@link DataFormat}. 354 | * @return This Builder. 355 | */ 356 | public Builder setDataFormat(DataFormat dataFormat) { 357 | this.dataFormat = dataFormat; 358 | return this; 359 | } 360 | 361 | /** 362 | * Sets the {@link HardwareBufferSize} QAT uses. 363 | * 364 | * @param hwBufferSize the {@link HardwareBufferSize} for QAT. 365 | * @return This Builder. 366 | */ 367 | public Builder setHardwareBufferSize(HardwareBufferSize hwBufferSize) { 368 | this.hwBufferSize = hwBufferSize; 369 | return this; 370 | } 371 | 372 | /** 373 | * Returns an instance of {@link QatZipper} created from the fields set on this builder. 374 | * 375 | * @return A QatZipper. 376 | */ 377 | public QatZipper build() throws RuntimeException { 378 | return new QatZipper(this); 379 | } 380 | 381 | @Override 382 | public String toString() { 383 | return "QatZipper{algorithm=" 384 | + algorithm 385 | + ", level=" 386 | + level 387 | + ", mode=" 388 | + mode 389 | + ", retryCount=" 390 | + retryCount 391 | + ", pollingMode=" 392 | + pollingMode 393 | + ", dataFormat=" 394 | + dataFormat 395 | + ", hardwareBufferSize=" 396 | + hwBufferSize 397 | + "}"; 398 | } 399 | } 400 | 401 | private QatZipper(Builder builder) throws RuntimeException { 402 | algorithm = builder.algorithm; 403 | level = builder.level; 404 | mode = builder.mode; 405 | retryCount = builder.retryCount; 406 | pollingMode = builder.pollingMode; 407 | dataFormat = builder.dataFormat; 408 | hwBufferSize = builder.hwBufferSize; 409 | 410 | if (retryCount < 0) throw new IllegalArgumentException("Invalid value for retry count"); 411 | 412 | if (algorithm == Algorithm.ZSTD) { 413 | zstdCompressCtx = new ZstdCompressCtx(); 414 | zstdDecompressCtx = new ZstdDecompressCtx(); 415 | } 416 | 417 | int status = 418 | InternalJNI.setup( 419 | this, 420 | algorithm.ordinal(), 421 | level, 422 | mode.ordinal(), 423 | pollingMode.ordinal(), 424 | dataFormat.ordinal(), 425 | hwBufferSize.getValue()); 426 | 427 | if (algorithm == Algorithm.ZSTD) { 428 | final int QZ_OK = 0; // indicates that ZSTD can start QAT device 429 | if (mode == Mode.HARDWARE || (mode == Mode.AUTO && status == QZ_OK)) { 430 | zstdCompressCtx.registerSequenceProducer(new QatZstdSequenceProducer()); 431 | zstdCompressCtx.setSequenceProducerFallback(true); 432 | } 433 | zstdCompressCtx.setLevel(level); 434 | } 435 | 436 | isValid = true; 437 | } 438 | 439 | /** 440 | * Returns an instance of {@link QatZipper} that uses default values for all parameters. 441 | * 442 | *

return A QatZipper. 443 | */ 444 | public QatZipper() { 445 | this(new Builder()); 446 | } 447 | 448 | /** 449 | * Returns an instance of {@link QatZipper} that uses the provided algorithm. Default values are 450 | * used for all the other parameters. 451 | * 452 | * @param algorithm the {@link Algorithm}. 453 | *

return A QatZipper. 454 | */ 455 | public QatZipper(Algorithm algorithm) { 456 | this(new Builder().setAlgorithm(algorithm)); 457 | } 458 | 459 | /** 460 | * Returns an instance of {@link QatZipper} that uses the provided algorithm and compression 461 | * level. Default values are used for all the other parameters. 462 | * 463 | * @param algorithm the {@link Algorithm}. 464 | * @param level the compression level. 465 | *

return A QatZipper. 466 | */ 467 | public QatZipper(Algorithm algorithm, int level) { 468 | this(new Builder().setAlgorithm(algorithm).setLevel(level)); 469 | } 470 | 471 | /** 472 | * Returns the maximum compression length for the specified source length. Use this method to 473 | * estimate the size of a buffer for compression given the size of a source buffer. 474 | * 475 | * @param len the length of the source array or buffer. 476 | * @return the maximum compression length for the specified length. 477 | */ 478 | public int maxCompressedLength(long len) { 479 | if (!isValid) throw new IllegalStateException("QAT session has been closed"); 480 | 481 | if (algorithm != Algorithm.ZSTD) { 482 | return InternalJNI.maxCompressedSize(qzKey, len); 483 | } 484 | 485 | return (int) Zstd.compressBound(len); 486 | } 487 | 488 | /** 489 | * Compresses the source array and stores the result in the destination array. Returns the actual 490 | * number of bytes of the compressed data. 491 | * 492 | * @param src the source array holding the source data 493 | * @param dst the destination array for the compressed data 494 | * @return the size of the compressed data in bytes 495 | */ 496 | public int compress(byte[] src, byte[] dst) { 497 | return compress(src, 0, src.length, dst, 0, dst.length); 498 | } 499 | 500 | /** 501 | * Compresses the source array, starting at the specified offset, and stores the result in the 502 | * destination array starting at the specified destination offset. Returns the actual number of 503 | * bytes of data compressed. 504 | * 505 | * @param src the source array holding the source data 506 | * @param srcOffset the start offset of the source data 507 | * @param srcLen the length of source data to compress 508 | * @param dst the destination array for the compressed data 509 | * @param dstOffset the destination offset where to start storing the compressed data 510 | * @param dstLen the maximum length that can be written to the destination array 511 | * @return the size of the compressed data in bytes 512 | */ 513 | public int compress( 514 | byte[] src, int srcOffset, int srcLen, byte[] dst, int dstOffset, int dstLen) { 515 | if (!isValid) throw new IllegalStateException("QAT session has been closed"); 516 | 517 | if (src == null || dst == null || srcLen == 0 || dst.length == 0) 518 | throw new IllegalArgumentException( 519 | "Either source or destination array or both have size 0 or null value"); 520 | 521 | if (srcOffset < 0 || srcLen < 0 || srcOffset > src.length - srcLen) 522 | throw new ArrayIndexOutOfBoundsException("Source offset is out of bound"); 523 | 524 | if (dstOffset < 0 || dstLen < 0 || dstOffset > dst.length - dstLen) 525 | throw new ArrayIndexOutOfBoundsException("Destination offset is out of bound"); 526 | 527 | if (algorithm != Algorithm.ZSTD) { 528 | return compressByteArray(src, srcOffset, srcLen, dst, dstOffset, dstLen); 529 | } else { 530 | bytesRead = bytesWritten = 0; 531 | int compressedSize = 532 | zstdCompressCtx.compressByteArray(dst, dstOffset, dstLen, src, srcOffset, srcLen); 533 | bytesWritten = compressedSize; 534 | bytesRead = srcLen; 535 | return compressedSize; 536 | } 537 | } 538 | 539 | private int compressByteArray( 540 | byte[] src, int srcOffset, int srcLen, byte[] dst, int dstOffset, int dstLen) { 541 | bytesRead = bytesWritten = 0; 542 | int compressedSize = 543 | InternalJNI.compressByteArray( 544 | this, qzKey, src, srcOffset, srcLen, dst, dstOffset, dstLen, retryCount); 545 | 546 | // bytesRead is updated by compressByteArray. We only need to update bytesWritten. 547 | bytesWritten = compressedSize; 548 | return compressedSize; 549 | } 550 | 551 | /** 552 | * Compresses the source buffer and stores the result in the destination buffer. Returns actual 553 | * number of bytes of compressed data. 554 | * 555 | *

On Success, the positions of both the source and destinations buffers are advanced by the 556 | * number of bytes read from the source and the number of bytes of compressed data written to the 557 | * destination. 558 | * 559 | * @param src the source buffer holding the source data 560 | * @param dst the destination array that will store the compressed data 561 | * @return the size of the compressed data in bytes 562 | */ 563 | public int compress(ByteBuffer src, ByteBuffer dst) { 564 | if (!isValid) throw new IllegalStateException("QAT session has been closed"); 565 | 566 | if ((src == null || dst == null) 567 | || (src.position() == src.limit() || dst.position() == dst.limit())) 568 | throw new IllegalArgumentException(); 569 | 570 | if (dst.isReadOnly()) throw new ReadOnlyBufferException(); 571 | 572 | if (algorithm != Algorithm.ZSTD) { 573 | return compressByteBuffer(src, dst); 574 | } else { 575 | // ZSTD treats the first parameter as the destination and the second as the source. 576 | return zstdCompressCtx.compress(dst, src); 577 | } 578 | } 579 | 580 | private int compressByteBuffer(ByteBuffer src, ByteBuffer dst) { 581 | final int srcPos = src.position(); 582 | final int dstPos = dst.position(); 583 | 584 | bytesRead = bytesWritten = 0; 585 | 586 | int compressedSize = 0; 587 | if (src.hasArray() && dst.hasArray()) { 588 | compressedSize = 589 | InternalJNI.compressByteBuffer( 590 | qzKey, 591 | src, 592 | src.array(), 593 | srcPos, 594 | src.remaining(), 595 | dst.array(), 596 | dstPos, 597 | dst.remaining(), 598 | retryCount); 599 | dst.position(dstPos + compressedSize); 600 | } else if (src.isDirect() && dst.isDirect()) { 601 | compressedSize = 602 | InternalJNI.compressDirectByteBuffer( 603 | qzKey, src, srcPos, src.remaining(), dst, dstPos, dst.remaining(), retryCount); 604 | } else if (src.hasArray() && dst.isDirect()) { 605 | compressedSize = 606 | InternalJNI.compressDirectByteBufferDst( 607 | qzKey, 608 | src, 609 | src.array(), 610 | srcPos, 611 | src.remaining(), 612 | dst, 613 | dstPos, 614 | dst.remaining(), 615 | retryCount); 616 | } else if (src.isDirect() && dst.hasArray()) { 617 | compressedSize = 618 | InternalJNI.compressDirectByteBufferSrc( 619 | qzKey, 620 | src, 621 | srcPos, 622 | src.remaining(), 623 | dst.array(), 624 | dstPos, 625 | dst.remaining(), 626 | retryCount); 627 | dst.position(dstPos + compressedSize); 628 | } else { 629 | int srcLen = src.remaining(); 630 | int dstLen = dst.remaining(); 631 | 632 | byte[] srcArr = new byte[srcLen]; 633 | byte[] dstArr = new byte[dstLen]; 634 | 635 | src.get(srcArr); 636 | dst.get(dstArr); 637 | 638 | src.position(src.position() - srcLen); 639 | dst.position(dst.position() - dstLen); 640 | 641 | int pos = src.position(); 642 | compressedSize = 643 | InternalJNI.compressByteBuffer( 644 | qzKey, src, srcArr, 0, srcLen, dstArr, 0, dstLen, retryCount); 645 | src.position(pos + src.position()); 646 | dst.put(dstArr, 0, compressedSize); 647 | } 648 | 649 | bytesRead = src.position() - srcPos; 650 | bytesWritten = dst.position() - dstPos; 651 | 652 | return compressedSize; 653 | } 654 | 655 | /** 656 | * Decompresses the source array and stores the result in the destination array. Returns the 657 | * actual number of bytes of decompressed data. 658 | * 659 | * @param src the source array holding the compressed data 660 | * @param dst the destination array for the decompressed data 661 | * @return the size of the decompressed data in bytes 662 | */ 663 | public int decompress(byte[] src, byte[] dst) { 664 | return decompress(src, 0, src.length, dst, 0, dst.length); 665 | } 666 | 667 | /** 668 | * Decompresses the source array, starting at the specified offset, and stores the result in the 669 | * destination array starting at the specified destination offset. Returns the actual number of 670 | * bytes of data decompressed. 671 | * 672 | * @param src the source array holding the compressed data 673 | * @param srcOffset the start offset of the source 674 | * @param srcLen the length of source data to decompress 675 | * @param dst the destination array for the decompressed data 676 | * @param dstOffset the destination offset where to start storing the decompressed data 677 | * @param dstLen the maximum length that can be written to the destination array 678 | * @return the size of the decompressed data in bytes 679 | */ 680 | public int decompress( 681 | byte[] src, int srcOffset, int srcLen, byte[] dst, int dstOffset, int dstLen) { 682 | if (!isValid) throw new IllegalStateException("QAT session has been closed"); 683 | 684 | if (src == null || dst == null || srcLen == 0 || dst.length == 0) 685 | throw new IllegalArgumentException("Empty source or/and destination byte array(s)"); 686 | 687 | if (srcOffset < 0 || srcLen < 0 || srcOffset > src.length - srcLen) 688 | throw new ArrayIndexOutOfBoundsException("Source offset is out of bound"); 689 | 690 | if (dstOffset < 0 || dstLen < 0 || dstOffset > dst.length - dstLen) 691 | throw new ArrayIndexOutOfBoundsException("Destination offset is out of bound"); 692 | 693 | if (algorithm != Algorithm.ZSTD) { 694 | return decompressByteArray(src, srcOffset, srcLen, dst, dstOffset, dstLen); 695 | } else { 696 | bytesRead = bytesWritten = 0; 697 | int decompressedSize = 698 | zstdDecompressCtx.decompressByteArray(dst, dstOffset, dstLen, src, srcOffset, srcLen); 699 | bytesWritten = decompressedSize; 700 | bytesRead = srcLen; 701 | return decompressedSize; 702 | } 703 | } 704 | 705 | private int decompressByteArray( 706 | byte[] src, int srcOffset, int srcLen, byte[] dst, int dstOffset, int dstLen) { 707 | bytesRead = bytesWritten = 0; 708 | 709 | int decompressedSize = 710 | InternalJNI.decompressByteArray( 711 | this, qzKey, src, srcOffset, srcLen, dst, dstOffset, dstLen, retryCount); 712 | 713 | // bytesRead is updated by decompressedByteArray. We only need to update bytesWritten. 714 | bytesWritten = decompressedSize; 715 | 716 | return decompressedSize; 717 | } 718 | 719 | /** 720 | * Deompresses the source buffer and stores the result in the destination buffer. Returns actual 721 | * number of bytes of decompressed data. 722 | * 723 | *

On Success, the positions of both the source and destinations buffers are advanced by the 724 | * number of bytes of compressed data read from the source and the number of bytes of decompressed 725 | * data written to the destination. 726 | * 727 | * @param src the source buffer holding the compressed data 728 | * @param dst the destination array that will store the decompressed data 729 | * @return the size of the decompressed data in bytes 730 | */ 731 | public int decompress(ByteBuffer src, ByteBuffer dst) { 732 | if (!isValid) throw new IllegalStateException("QAT session has been closed"); 733 | 734 | if ((src == null || dst == null) 735 | || (src.position() == src.limit() || dst.position() == dst.limit())) 736 | throw new IllegalArgumentException(); 737 | 738 | if (dst.isReadOnly()) throw new ReadOnlyBufferException(); 739 | 740 | if (algorithm != Algorithm.ZSTD) { 741 | return decompressByteBuffer(src, dst); 742 | } else { 743 | // ZSTD treats the first parameter as the destination and the second as the source. 744 | if (!src.isDirect()) 745 | throw new IllegalArgumentException( 746 | "Zstd-jni requires source buffers to be direct byte buffers"); 747 | return zstdDecompressCtx.decompress(dst, src); 748 | } 749 | } 750 | 751 | private int decompressByteBuffer(ByteBuffer src, ByteBuffer dst) { 752 | final int srcPos = src.position(); 753 | final int dstPos = dst.position(); 754 | 755 | bytesRead = bytesWritten = 0; 756 | 757 | int decompressedSize = 0; 758 | if (src.hasArray() && dst.hasArray()) { 759 | decompressedSize = 760 | InternalJNI.decompressByteBuffer( 761 | qzKey, 762 | src, 763 | src.array(), 764 | srcPos, 765 | src.remaining(), 766 | dst.array(), 767 | dstPos, 768 | dst.remaining(), 769 | retryCount); 770 | dst.position(dstPos + decompressedSize); 771 | } else if (src.isDirect() && dst.isDirect()) { 772 | decompressedSize = 773 | InternalJNI.decompressDirectByteBuffer( 774 | qzKey, src, srcPos, src.remaining(), dst, dstPos, dst.remaining(), retryCount); 775 | } else if (src.hasArray() && dst.isDirect()) { 776 | decompressedSize = 777 | InternalJNI.decompressDirectByteBufferDst( 778 | qzKey, 779 | src, 780 | src.array(), 781 | srcPos, 782 | src.remaining(), 783 | dst, 784 | dstPos, 785 | dst.remaining(), 786 | retryCount); 787 | } else if (src.isDirect() && dst.hasArray()) { 788 | decompressedSize = 789 | InternalJNI.decompressDirectByteBufferSrc( 790 | qzKey, 791 | src, 792 | srcPos, 793 | src.remaining(), 794 | dst.array(), 795 | dstPos, 796 | dst.remaining(), 797 | retryCount); 798 | dst.position(dstPos + decompressedSize); 799 | } else { 800 | int srcLen = src.remaining(); 801 | int dstLen = dst.remaining(); 802 | 803 | byte[] srcArr = new byte[srcLen]; 804 | byte[] dstArr = new byte[dstLen]; 805 | 806 | src.get(srcArr); 807 | dst.get(dstArr); 808 | 809 | src.position(src.position() - srcLen); 810 | dst.position(dst.position() - dstLen); 811 | 812 | int pos = src.position(); 813 | decompressedSize = 814 | InternalJNI.decompressByteBuffer( 815 | qzKey, src, srcArr, 0, srcLen, dstArr, 0, dstLen, retryCount); 816 | src.position(pos + src.position()); 817 | dst.put(dstArr, 0, decompressedSize); 818 | } 819 | 820 | if (decompressedSize < 0) throw new RuntimeException("QAT: Compression failed"); 821 | 822 | bytesRead = src.position() - srcPos; 823 | bytesWritten = dst.position() - dstPos; 824 | 825 | return decompressedSize; 826 | } 827 | 828 | /** 829 | * Returns the number of bytes read from the source array or buffer by the most recent call to 830 | * compress/decompress. 831 | * 832 | * @return the number of bytes read from source. 833 | */ 834 | public int getBytesRead() { 835 | return bytesRead; 836 | } 837 | 838 | /** 839 | * Returns the number of bytes written to the destination array or buffer by the most recent call 840 | * to compress/decompress. 841 | * 842 | * @return the number of bytes written to destination. 843 | */ 844 | public int getBytesWritten() { 845 | return bytesWritten; 846 | } 847 | 848 | /** 849 | * Sets a checksum flag. Currently valid only for ZSTD. 850 | * 851 | * @param checksumFlag the checksum flag. 852 | * @throws UnsupportedOperationException if called for a compressor algorithm other than ZSTD. 853 | */ 854 | public void setChecksumFlag(boolean checksumFlag) { 855 | if (algorithm != Algorithm.ZSTD) 856 | throw new UnsupportedOperationException( 857 | "Setting a checksum flag is currently valid only for ZSTD compressor"); 858 | zstdCompressCtx.setChecksum(checksumFlag); 859 | } 860 | 861 | /** 862 | * Ends the current QAT session by freeing up resources. A new session must be used after a 863 | * successful call of this method. 864 | * 865 | * @throws RuntimeException if QAT session cannot be gracefully ended. 866 | */ 867 | public void end() throws RuntimeException { 868 | if (!isValid) throw new IllegalStateException("Invalid QAT session"); 869 | InternalJNI.teardown(qzKey); 870 | isValid = false; 871 | } 872 | } 873 | -------------------------------------------------------------------------------- /src/main/java/com/intel/qat/QatZstdSequenceProducer.java: -------------------------------------------------------------------------------- 1 | package com.intel.qat; 2 | 3 | import com.github.luben.zstd.SequenceProducer; 4 | 5 | /** 6 | * An implementation of an abstract sequence producer plugin. Pass an instance of this class to 7 | * {@link com.github.luben.zstd.ZstdCompressCtx#registerSequenceProducer(SequenceProducer)} to 8 | * enable QAT-accelerated compression with zstd. 9 | */ 10 | public class QatZstdSequenceProducer implements SequenceProducer { 11 | 12 | /** Constructs a new ZSTD sequence producer. */ 13 | public QatZstdSequenceProducer() {} 14 | 15 | @Override 16 | public long getFunctionPointer() { 17 | return InternalJNI.zstdGetSeqProdFunction(); 18 | } 19 | 20 | @Override 21 | public long createState() { 22 | return InternalJNI.zstdCreateSeqProdState(); 23 | } 24 | 25 | @Override 26 | public void freeState(long state) { 27 | InternalJNI.zstdFreeSeqProdState(state); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/intel/qat/examples/ByteArrayExample.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | 7 | package com.intel.qat.examples; 8 | 9 | import com.intel.qat.QatZipper; 10 | 11 | public class ByteArrayExample { 12 | 13 | public static void main(String[] args) { 14 | try { 15 | String inputStr = "The quick brown fox jumps over the lazy dog."; 16 | byte[] input = inputStr.getBytes("UTF-8"); 17 | 18 | QatZipper qzip = new QatZipper.Builder().build(); 19 | 20 | // Create a buffer with enough size for compression 21 | byte[] compressedData = new byte[qzip.maxCompressedLength(input.length)]; 22 | 23 | // Compress the bytes 24 | int clen = qzip.compress(input, compressedData); 25 | 26 | // Decompress the bytes into a String 27 | byte[] decompressedData = new byte[input.length]; 28 | int decompressedLength = 29 | qzip.decompress(compressedData, 0, clen, decompressedData, 0, decompressedData.length); 30 | 31 | // Release resources 32 | qzip.end(); 33 | 34 | // Convert the bytes into a String 35 | String outputStr = new String(decompressedData, 0, decompressedLength, "UTF-8"); 36 | System.out.println("Decompressed data: " + outputStr); 37 | } catch (java.io.UnsupportedEncodingException | RuntimeException e) { 38 | e.printStackTrace(); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/com/intel/qat/examples/InputOutputStreamExample.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | 7 | package com.intel.qat.examples; 8 | 9 | import static com.intel.qat.QatZipper.Algorithm; 10 | 11 | import com.intel.qat.QatCompressorOutputStream; 12 | import com.intel.qat.QatDecompressorInputStream; 13 | import java.io.ByteArrayInputStream; 14 | import java.io.ByteArrayOutputStream; 15 | import java.io.IOException; 16 | 17 | public class InputOutputStreamExample { 18 | 19 | public static void main(String[] args) { 20 | try { 21 | byte[] dataToCompress = "The quick brown fox jumps over the lazy dog.".getBytes("UTF-8"); 22 | 23 | ByteArrayOutputStream compressedOutput = new ByteArrayOutputStream(); 24 | QatCompressorOutputStream qatOutputStream = 25 | new QatCompressorOutputStream(compressedOutput, Algorithm.DEFLATE); 26 | qatOutputStream.write(dataToCompress); 27 | qatOutputStream.close(); 28 | 29 | byte[] compressedData = compressedOutput.toByteArray(); 30 | 31 | ByteArrayInputStream compressedInput = new ByteArrayInputStream(compressedData); 32 | QatDecompressorInputStream qatInputStream = 33 | new QatDecompressorInputStream(compressedInput, Algorithm.DEFLATE); 34 | ByteArrayOutputStream decompressedOutput = new ByteArrayOutputStream(); 35 | 36 | byte[] buffer = new byte[1024]; 37 | int bytesRead; 38 | while ((bytesRead = qatInputStream.read(buffer)) != -1) { 39 | decompressedOutput.write(buffer, 0, bytesRead); 40 | } 41 | qatInputStream.close(); 42 | 43 | byte[] decompressedData = decompressedOutput.toByteArray(); 44 | System.out.println("Decompressed data: " + new String(decompressedData, "UTF-8")); 45 | } catch (IOException e) { 46 | e.printStackTrace(); 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/intel/qat/package-info.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | 7 | /** 8 | * Provides classes needed to compress and decompress data using Intel® QuickAssist Technology. 9 | * 10 | * @since 1.0 11 | * @see com.intel.qat 12 | */ 13 | package com.intel.qat; 14 | -------------------------------------------------------------------------------- /src/main/java/module-info.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | 7 | /** 8 | * Defines APIs for data compression using Intel® QuickAssit Technology. 9 | * 10 | *

The implementation uses the QATZip library 11 | * through JNI bindings. 12 | */ 13 | module com.intel.qat { 14 | requires com.github.luben.zstd_jni; 15 | 16 | exports com.intel.qat; 17 | } 18 | -------------------------------------------------------------------------------- /src/main/jni/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # ****************************************************************************** 2 | # Copyright (C) 2023 Intel Corporation 3 | # 4 | # SPDX-License-Identifier: BSD 5 | # ****************************************************************************** 6 | 7 | cmake_minimum_required(VERSION 3.10) 8 | 9 | project(qat-java LANGUAGES C) 10 | 11 | # Set default build type early 12 | if(NOT CMAKE_BUILD_TYPE) 13 | set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) 14 | endif() 15 | 16 | # Define shared library name 17 | set(SHARED_LIBRARY_NAME qat-java) 18 | 19 | # Source files 20 | set(SOURCE_FILES 21 | com_intel_qat_InternalJNI.c 22 | util.c 23 | ) 24 | 25 | # Set C standard and compile flags 26 | set(CMAKE_C_STANDARD 11) 27 | set(CMAKE_C_STANDARD_REQUIRED ON) 28 | 29 | # Define common compile flags 30 | set(COMMON_FLAGS "") 31 | list(APPEND COMMON_FLAGS 32 | -O2 33 | -Wall 34 | -Wextra 35 | -Wpedantic 36 | -Werror 37 | -fstack-protector-strong 38 | -D_FORTIFY_SOURCE=2 39 | ) 40 | 41 | # Sanitizer option 42 | option(SANITIZE_MEMORY "Enable memory sanitizing" OFF) 43 | if(SANITIZE_MEMORY) 44 | if(CMAKE_C_COMPILER_ID MATCHES "Clang") 45 | list(APPEND COMMON_FLAGS -fsanitize=fuzzer-no-link,address) 46 | else() 47 | message(WARNING "Memory sanitizing requires Clang compiler") 48 | endif() 49 | endif() 50 | 51 | # Find JNI 52 | find_package(JNI REQUIRED) 53 | 54 | # Configure QAT-ZSTD submodule 55 | set(QAT_ZSTD_DIR "${CMAKE_SOURCE_DIR}/external/qat-zstd-plugin") 56 | set(QAT_ZSTD_LIB "${QAT_ZSTD_DIR}/src/libqatseqprod.a") 57 | 58 | # Ensure submodule is initialized 59 | find_package(Git QUIET) 60 | if(GIT_FOUND) 61 | execute_process( 62 | COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive 63 | WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} 64 | RESULT_VARIABLE GIT_SUBMODULE_RESULT 65 | ERROR_VARIABLE GIT_SUBMODULE_ERROR 66 | ) 67 | 68 | if(NOT GIT_SUBMODULE_RESULT EQUAL 0) 69 | message(FATAL_ERROR "Git submodule update failed: ${GIT_SUBMODULE_ERROR}") 70 | endif() 71 | 72 | # Checkout specific version/tag 73 | execute_process( 74 | COMMAND ${GIT_EXECUTABLE} fetch --tags 75 | WORKING_DIRECTORY ${QAT_ZSTD_DIR} 76 | ) 77 | execute_process( 78 | COMMAND ${GIT_EXECUTABLE} checkout tags/v0.2.0 79 | WORKING_DIRECTORY ${QAT_ZSTD_DIR} 80 | RESULT_VARIABLE GIT_CHECKOUT_RESULT 81 | ) 82 | 83 | if(NOT GIT_CHECKOUT_RESULT EQUAL 0) 84 | message(FATAL_ERROR "Failed to checkout QAT-ZSTD v0.2.0") 85 | endif() 86 | endif() 87 | 88 | # Build QAT-ZSTD statically 89 | add_custom_target(build_qat_zstd 90 | COMMAND ${CMAKE_MAKE_PROGRAM} 91 | WORKING_DIRECTORY ${QAT_ZSTD_DIR} 92 | COMMENT "Building QAT-ZSTD plugin" 93 | ) 94 | 95 | # Create shared library 96 | add_library(${SHARED_LIBRARY_NAME} SHARED ${SOURCE_FILES}) 97 | 98 | # Include directories 99 | target_include_directories(${SHARED_LIBRARY_NAME} PRIVATE 100 | ${JNI_INCLUDE_DIRS} 101 | ${QAT_ZSTD_DIR}/src 102 | ) 103 | 104 | # Compile options 105 | target_compile_options(${SHARED_LIBRARY_NAME} PRIVATE ${COMMON_FLAGS}) 106 | 107 | # Link libraries 108 | find_library(QATZIP_LIB qatzip) 109 | if(NOT QATZIP_LIB) 110 | message(FATAL_ERROR "Could not find system library: qatzip") 111 | endif() 112 | 113 | target_link_libraries(${SHARED_LIBRARY_NAME} PRIVATE 114 | ${QATZIP_LIB} 115 | ${QAT_ZSTD_LIB} 116 | pthread 117 | ) 118 | 119 | # Depend on the custom build target 120 | add_dependencies(${SHARED_LIBRARY_NAME} build_qat_zstd) 121 | 122 | # Properties 123 | set_target_properties(${SHARED_LIBRARY_NAME} PROPERTIES 124 | POSITION_INDEPENDENT_CODE ON 125 | INSTALL_RPATH "$ORIGIN" 126 | ) 127 | 128 | # Print configuration 129 | message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") 130 | message(STATUS "Memory sanitizing: ${SANITIZE_MEMORY}") 131 | message(STATUS "Compiler flags: ${COMMON_FLAGS}") 132 | message(STATUS "QAT-ZSTD directory: ${QAT_ZSTD_DIR}") 133 | 134 | -------------------------------------------------------------------------------- /src/main/jni/com_intel_qat_InternalJNI.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class com_intel_qat_InternalJNI */ 4 | 5 | #ifndef _Included_com_intel_qat_InternalJNI 6 | #define _Included_com_intel_qat_InternalJNI 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | /* 11 | * Class: com_intel_qat_InternalJNI 12 | * Method: initFieldIDs 13 | * Signature: ()V 14 | */ 15 | JNIEXPORT void JNICALL Java_com_intel_qat_InternalJNI_initFieldIDs(JNIEnv *, 16 | jclass); 17 | 18 | /* 19 | * Class: com_intel_qat_InternalJNI 20 | * Method: setup 21 | * Signature: (Lcom/intel/qat/QatZipper;IIIIII)I 22 | */ 23 | JNIEXPORT jint JNICALL Java_com_intel_qat_InternalJNI_setup(JNIEnv *, jclass, 24 | jobject, jint, jint, 25 | jint, jint, jint, 26 | jint); 27 | 28 | /* 29 | * Class: com_intel_qat_InternalJNI 30 | * Method: maxCompressedSize 31 | * Signature: (IJ)I 32 | */ 33 | JNIEXPORT jint JNICALL Java_com_intel_qat_InternalJNI_maxCompressedSize( 34 | JNIEnv *, jclass, jint, jlong); 35 | 36 | /* 37 | * Class: com_intel_qat_InternalJNI 38 | * Method: compressByteArray 39 | * Signature: (Lcom/intel/qat/QatZipper;I[BII[BIII)I 40 | */ 41 | JNIEXPORT jint JNICALL Java_com_intel_qat_InternalJNI_compressByteArray( 42 | JNIEnv *, jclass, jobject, jint, jbyteArray, jint, jint, jbyteArray, jint, 43 | jint, jint); 44 | 45 | /* 46 | * Class: com_intel_qat_InternalJNI 47 | * Method: decompressByteArray 48 | * Signature: (Lcom/intel/qat/QatZipper;I[BII[BIII)I 49 | */ 50 | JNIEXPORT jint JNICALL Java_com_intel_qat_InternalJNI_decompressByteArray( 51 | JNIEnv *, jclass, jobject, jint, jbyteArray, jint, jint, jbyteArray, jint, 52 | jint, jint); 53 | 54 | /* 55 | * Class: com_intel_qat_InternalJNI 56 | * Method: compressByteBuffer 57 | * Signature: (ILjava/nio/ByteBuffer;[BII[BIII)I 58 | */ 59 | JNIEXPORT jint JNICALL Java_com_intel_qat_InternalJNI_compressByteBuffer( 60 | JNIEnv *, jclass, jint, jobject, jbyteArray, jint, jint, jbyteArray, jint, 61 | jint, jint); 62 | 63 | /* 64 | * Class: com_intel_qat_InternalJNI 65 | * Method: decompressByteBuffer 66 | * Signature: (ILjava/nio/ByteBuffer;[BII[BIII)I 67 | */ 68 | JNIEXPORT jint JNICALL Java_com_intel_qat_InternalJNI_decompressByteBuffer( 69 | JNIEnv *, jclass, jint, jobject, jbyteArray, jint, jint, jbyteArray, jint, 70 | jint, jint); 71 | 72 | /* 73 | * Class: com_intel_qat_InternalJNI 74 | * Method: compressDirectByteBuffer 75 | * Signature: (ILjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;III)I 76 | */ 77 | JNIEXPORT jint JNICALL Java_com_intel_qat_InternalJNI_compressDirectByteBuffer( 78 | JNIEnv *, jclass, jint, jobject, jint, jint, jobject, jint, jint, jint); 79 | 80 | /* 81 | * Class: com_intel_qat_InternalJNI 82 | * Method: decompressDirectByteBuffer 83 | * Signature: (ILjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;III)I 84 | */ 85 | JNIEXPORT jint JNICALL 86 | Java_com_intel_qat_InternalJNI_decompressDirectByteBuffer(JNIEnv *, jclass, 87 | jint, jobject, jint, 88 | jint, jobject, jint, 89 | jint, jint); 90 | 91 | /* 92 | * Class: com_intel_qat_InternalJNI 93 | * Method: compressDirectByteBufferSrc 94 | * Signature: (ILjava/nio/ByteBuffer;II[BIII)I 95 | */ 96 | JNIEXPORT jint JNICALL 97 | Java_com_intel_qat_InternalJNI_compressDirectByteBufferSrc(JNIEnv *, jclass, 98 | jint, jobject, jint, 99 | jint, jbyteArray, 100 | jint, jint, jint); 101 | 102 | /* 103 | * Class: com_intel_qat_InternalJNI 104 | * Method: decompressDirectByteBufferSrc 105 | * Signature: (ILjava/nio/ByteBuffer;II[BIII)I 106 | */ 107 | JNIEXPORT jint JNICALL 108 | Java_com_intel_qat_InternalJNI_decompressDirectByteBufferSrc( 109 | JNIEnv *, jclass, jint, jobject, jint, jint, jbyteArray, jint, jint, jint); 110 | 111 | /* 112 | * Class: com_intel_qat_InternalJNI 113 | * Method: compressDirectByteBufferDst 114 | * Signature: (ILjava/nio/ByteBuffer;[BIILjava/nio/ByteBuffer;III)I 115 | */ 116 | JNIEXPORT jint JNICALL 117 | Java_com_intel_qat_InternalJNI_compressDirectByteBufferDst(JNIEnv *, jclass, 118 | jint, jobject, 119 | jbyteArray, jint, 120 | jint, jobject, jint, 121 | jint, jint); 122 | 123 | /* 124 | * Class: com_intel_qat_InternalJNI 125 | * Method: decompressDirectByteBufferDst 126 | * Signature: (ILjava/nio/ByteBuffer;[BIILjava/nio/ByteBuffer;III)I 127 | */ 128 | JNIEXPORT jint JNICALL 129 | Java_com_intel_qat_InternalJNI_decompressDirectByteBufferDst(JNIEnv *, jclass, 130 | jint, jobject, 131 | jbyteArray, jint, 132 | jint, jobject, 133 | jint, jint, jint); 134 | 135 | /* 136 | * Class: com_intel_qat_InternalJNI 137 | * Method: zstdGetSeqProdFunction 138 | * Signature: ()J 139 | */ 140 | JNIEXPORT jlong JNICALL 141 | Java_com_intel_qat_InternalJNI_zstdGetSeqProdFunction(JNIEnv *, jclass); 142 | 143 | /* 144 | * Class: com_intel_qat_InternalJNI 145 | * Method: zstdCreateSeqProdState 146 | * Signature: ()J 147 | */ 148 | JNIEXPORT jlong JNICALL 149 | Java_com_intel_qat_InternalJNI_zstdCreateSeqProdState(JNIEnv *, jclass); 150 | 151 | /* 152 | * Class: com_intel_qat_InternalJNI 153 | * Method: zstdFreeSeqProdState 154 | * Signature: (J)V 155 | */ 156 | JNIEXPORT void JNICALL 157 | Java_com_intel_qat_InternalJNI_zstdFreeSeqProdState(JNIEnv *, jclass, jlong); 158 | 159 | /* 160 | * Class: com_intel_qat_InternalJNI 161 | * Method: teardown 162 | * Signature: (I)I 163 | */ 164 | JNIEXPORT jint JNICALL Java_com_intel_qat_InternalJNI_teardown(JNIEnv *, jclass, jint); 165 | 166 | #ifdef __cplusplus 167 | } 168 | #endif 169 | #endif 170 | -------------------------------------------------------------------------------- /src/main/jni/util.c: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | #include "util.h" 7 | 8 | #include 9 | 10 | /** 11 | * Retrieves the QAT error message string for a given error code. 12 | * 13 | * @param err_code The QAT error code 14 | * @return A constant string describing the error code, or "QZ_UNKNOWN" for 15 | * unrecognized codes 16 | */ 17 | const char *get_err_str(int err_code) { 18 | // clang-format off 19 | switch (err_code) { 20 | case QZ_OK: return "QZ_OK: The operation was successful"; 21 | case QZ_DUPLICATE: return "QZ_DUPLICATE: Duplicate operation detected"; 22 | case QZ_FORCE_SW: return "QZ_FORCE_SW: Forced to use software implementation"; 23 | case QZ_PARAMS: return "QZ_PARAMS: Invalid or incorrect parameters provided"; 24 | case QZ_FAIL: return "QZ_FAIL: General operation failure"; 25 | case QZ_BUF_ERROR: return "QZ_BUF_ERROR: Buffer-related error occurred"; 26 | case QZ_DATA_ERROR: return "QZ_DATA_ERROR: Input data is corrupted or invalid"; 27 | case QZ_TIMEOUT: return "QZ_TIMEOUT: Operation timed out"; 28 | case QZ_INTEG: return "QZ_INTEG: Integrity check failed"; 29 | case QZ_NO_HW: return "QZ_NO_HW: No hardware acceleration available"; 30 | case QZ_NO_MDRV: return "QZ_NO_MDRV: Missing or incompatible driver"; 31 | case QZ_NO_INST_ATTACH: return "QZ_NO_INST_ATTACH: Failed to attach to instance"; 32 | case QZ_LOW_MEM: return "QZ_LOW_MEM: Insufficient memory available"; 33 | case QZ_LOW_DEST_MEM: return "QZ_LOW_DEST_MEM: Insufficient destination memory"; 34 | case QZ_UNSUPPORTED_FMT: return "QZ_UNSUPPORTED_FMT: Unsupported format detected"; 35 | case QZ_NONE: return "QZ_NONE: No error condition specified"; 36 | case QZ_NOSW_NO_HW: return "QZ_NOSW_NO_HW: No software fallback and hardware unavailable"; 37 | case QZ_NOSW_NO_MDRV: return "QZ_NOSW_NO_MDRV: No software fallback and missing driver"; 38 | case QZ_NOSW_NO_INST_ATTACH: return "QZ_NOSW_NO_INST_ATTACH: No software fallback and instance attachment failed"; 39 | case QZ_NOSW_LOW_MEM: return "QZ_NOSW_LOW_MEM: No software fallback and insufficient memory"; 40 | case QZ_NO_SW_AVAIL: return "QZ_NO_SW_AVAIL: No software implementation available"; 41 | case QZ_NOSW_UNSUPPORTED_FMT: return "QZ_NOSW_UNSUPPORTED_FMT: No software fallback and unsupported format"; 42 | case QZ_POST_PROCESS_ERROR: return "QZ_POST_PROCESS_ERROR: Error during post-processing"; 43 | case QZ_METADATA_OVERFLOW: return "QZ_METADATA_OVERFLOW: Metadata exceeds allocated space"; 44 | case QZ_OUT_OF_RANGE: return "QZ_OUT_OF_RANGE: Value outside acceptable range"; 45 | case QZ_NOT_SUPPORTED: return "QZ_NOT_SUPPORTED: Operation or feature not supported"; 46 | default: 47 | return "QZ_UNKNOWN: Unknown error code"; 48 | } 49 | // clang-format on 50 | } 51 | -------------------------------------------------------------------------------- /src/main/jni/util.h: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | 7 | #ifndef UTIL_H_ 8 | #define UTIL_H_ 9 | 10 | #include 11 | 12 | #ifdef __cplusplus 13 | extern "C" { 14 | #endif 15 | 16 | const char *get_err_str(int err_code); 17 | 18 | #ifdef __cplusplus 19 | } 20 | #endif 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /src/test/java/com/intel/qat/NativeTest.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | 7 | package com.intel.qat; 8 | 9 | import static org.junit.jupiter.api.Assertions.*; 10 | 11 | import java.lang.reflect.Field; 12 | import org.junit.jupiter.api.BeforeEach; 13 | import org.junit.jupiter.api.Test; 14 | 15 | public class NativeTest { 16 | @BeforeEach 17 | void setUp() { 18 | setField("loaded", false); 19 | } 20 | 21 | @Test 22 | void testLoadNativeLibraryAlreadyLoaded() { 23 | setField("loaded", true); 24 | 25 | assertDoesNotThrow(() -> Native.loadLibrary()); 26 | assertTrue(Native.isLoaded(), "Library should be marked as loaded"); 27 | } 28 | 29 | @Test 30 | void testLoadNativeLibraryNotPresent() { 31 | setField("loaded", false); 32 | try { 33 | Native.loadLibrary(); 34 | assertTrue(true); 35 | } catch (RuntimeException e) { 36 | } 37 | } 38 | 39 | private void setField(String fieldName, T value) { 40 | try { 41 | Field field = Native.class.getDeclaredField(fieldName); 42 | field.setAccessible(true); 43 | field.set(null, value); 44 | } catch (NoSuchFieldException | IllegalAccessException e) { 45 | throw new RuntimeException("Failed to set field: " + fieldName, e); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/test/java/com/intel/qat/QatCompressorOutputStreamTests.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | 7 | package com.intel.qat; 8 | 9 | import static com.intel.qat.QatZipper.Algorithm; 10 | import static com.intel.qat.QatZipper.Mode; 11 | import static org.junit.jupiter.api.Assertions.assertEquals; 12 | import static org.junit.jupiter.api.Assertions.assertFalse; 13 | import static org.junit.jupiter.api.Assertions.assertTrue; 14 | import static org.junit.jupiter.api.Assertions.fail; 15 | 16 | import java.io.ByteArrayOutputStream; 17 | import java.io.IOException; 18 | import java.nio.file.Files; 19 | import java.nio.file.Paths; 20 | import java.util.Arrays; 21 | import java.util.Random; 22 | import java.util.stream.Stream; 23 | import org.junit.jupiter.api.AfterEach; 24 | import org.junit.jupiter.api.BeforeAll; 25 | import org.junit.jupiter.api.Test; 26 | import org.junit.jupiter.params.ParameterizedTest; 27 | import org.junit.jupiter.params.provider.Arguments; 28 | import org.junit.jupiter.params.provider.EnumSource; 29 | import org.junit.jupiter.params.provider.MethodSource; 30 | 31 | public class QatCompressorOutputStreamTests { 32 | private static final String SAMPLE_CORPUS = "src/test/resources/sample.txt"; 33 | 34 | private QatZipper qzip; 35 | private static byte[] src; 36 | 37 | private Random rnd = new Random(); 38 | 39 | @BeforeAll 40 | public static void setup() throws IOException { 41 | src = Files.readAllBytes(Paths.get(SAMPLE_CORPUS)); 42 | } 43 | 44 | @AfterEach 45 | public void cleanupSession() { 46 | if (qzip != null) qzip.end(); 47 | } 48 | 49 | public static Stream provideModeAlgorithmParams() { 50 | if (QatZipper.isQatAvailable()) { 51 | return Stream.of( 52 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE), 53 | Arguments.of(Mode.AUTO, Algorithm.LZ4), 54 | Arguments.of(Mode.AUTO, Algorithm.ZSTD), 55 | Arguments.of(Mode.HARDWARE, Algorithm.DEFLATE), 56 | Arguments.of(Mode.HARDWARE, Algorithm.ZSTD), 57 | Arguments.of(Mode.HARDWARE, Algorithm.LZ4)); 58 | } else { 59 | return Stream.of( 60 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE), Arguments.of(Mode.AUTO, Algorithm.LZ4)); 61 | } 62 | } 63 | 64 | public static Stream provideModeAlgorithmLengthParams() { 65 | if (QatZipper.isQatAvailable()) { 66 | return Stream.of( 67 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 16384), 68 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 65536), 69 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 524288), 70 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 1048576), 71 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 16384), 72 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 65536), 73 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 524288), 74 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 1048576), 75 | Arguments.of(Mode.AUTO, Algorithm.ZSTD, 16384), 76 | Arguments.of(Mode.AUTO, Algorithm.ZSTD, 65536), 77 | Arguments.of(Mode.AUTO, Algorithm.ZSTD, 524288), 78 | Arguments.of(Mode.AUTO, Algorithm.ZSTD, 1048576), 79 | Arguments.of(Mode.HARDWARE, Algorithm.DEFLATE, 16384), 80 | Arguments.of(Mode.HARDWARE, Algorithm.DEFLATE, 65536), 81 | Arguments.of(Mode.HARDWARE, Algorithm.DEFLATE, 524288), 82 | Arguments.of(Mode.HARDWARE, Algorithm.DEFLATE, 1048576), 83 | Arguments.of(Mode.HARDWARE, Algorithm.LZ4, 16384), 84 | Arguments.of(Mode.HARDWARE, Algorithm.LZ4, 65536), 85 | Arguments.of(Mode.HARDWARE, Algorithm.LZ4, 524288), 86 | Arguments.of(Mode.HARDWARE, Algorithm.LZ4, 1048576), 87 | Arguments.of(Mode.HARDWARE, Algorithm.ZSTD, 16384), 88 | Arguments.of(Mode.HARDWARE, Algorithm.ZSTD, 65536), 89 | Arguments.of(Mode.HARDWARE, Algorithm.ZSTD, 524288), 90 | Arguments.of(Mode.HARDWARE, Algorithm.ZSTD, 1048576)); 91 | } else { 92 | return Stream.of( 93 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 16384), 94 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 65536), 95 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 524288), 96 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 1048576), 97 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 16384), 98 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 65536), 99 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 524288), 100 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 1048576), 101 | Arguments.of(Mode.AUTO, Algorithm.ZSTD, 16384), 102 | Arguments.of(Mode.AUTO, Algorithm.ZSTD, 65536), 103 | Arguments.of(Mode.AUTO, Algorithm.ZSTD, 524288), 104 | Arguments.of(Mode.AUTO, Algorithm.ZSTD, 1048576)); 105 | } 106 | } 107 | 108 | public static Stream provideAlgorithmLevelParams() { 109 | return Stream.of( 110 | Arguments.of(Algorithm.DEFLATE, 1), 111 | Arguments.of(Algorithm.DEFLATE, 2), 112 | Arguments.of(Algorithm.DEFLATE, 3), 113 | Arguments.of(Algorithm.DEFLATE, 4), 114 | Arguments.of(Algorithm.DEFLATE, 5), 115 | Arguments.of(Algorithm.DEFLATE, 6), 116 | Arguments.of(Algorithm.DEFLATE, 7), 117 | Arguments.of(Algorithm.DEFLATE, 8), 118 | Arguments.of(Algorithm.DEFLATE, 9), 119 | Arguments.of(Algorithm.ZSTD, 1), 120 | Arguments.of(Algorithm.ZSTD, 2), 121 | Arguments.of(Algorithm.ZSTD, 3), 122 | Arguments.of(Algorithm.ZSTD, 4), 123 | Arguments.of(Algorithm.ZSTD, 5), 124 | Arguments.of(Algorithm.ZSTD, 6), 125 | Arguments.of(Algorithm.ZSTD, 7), 126 | Arguments.of(Algorithm.ZSTD, 8), 127 | Arguments.of(Algorithm.ZSTD, 9), 128 | Arguments.of(Algorithm.LZ4, 1), 129 | Arguments.of(Algorithm.LZ4, 2), 130 | Arguments.of(Algorithm.LZ4, 3), 131 | Arguments.of(Algorithm.LZ4, 4), 132 | Arguments.of(Algorithm.LZ4, 5), 133 | Arguments.of(Algorithm.LZ4, 6), 134 | Arguments.of(Algorithm.LZ4, 7), 135 | Arguments.of(Algorithm.LZ4, 8), 136 | Arguments.of(Algorithm.LZ4, 9)); 137 | } 138 | 139 | @Test 140 | public void testOutputStreamNullStream() throws IOException { 141 | ByteArrayOutputStream outputStream = null; 142 | try { 143 | try (QatCompressorOutputStream compressedStream = 144 | new QatCompressorOutputStream(outputStream, Algorithm.DEFLATE)) {} 145 | fail("Failed to catch NullPointerException"); 146 | } catch (NullPointerException | IOException e) { 147 | assertTrue(true); 148 | } 149 | } 150 | 151 | @ParameterizedTest 152 | @EnumSource(Algorithm.class) 153 | public void testOutputStreamOneArgConstructor(Algorithm algo) throws IOException { 154 | if (!QatZipper.isQatAvailable()) { 155 | return; 156 | } 157 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 158 | try { 159 | try (QatCompressorOutputStream compressedStream = 160 | new QatCompressorOutputStream(outputStream, algo)) {} 161 | } catch (IOException | IllegalArgumentException e) { 162 | fail(e.getMessage()); 163 | } 164 | } 165 | 166 | @ParameterizedTest 167 | @EnumSource(Algorithm.class) 168 | public void testOutputStreamConstructor1(Algorithm algo) throws IOException { 169 | if (!QatZipper.isQatAvailable()) { 170 | return; 171 | } 172 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 173 | try { 174 | try (QatCompressorOutputStream compressedStream = 175 | new QatCompressorOutputStream(outputStream, 16 * 1024, algo)) {} 176 | } catch (IOException | IllegalArgumentException e) { 177 | fail(e.getMessage()); 178 | } 179 | } 180 | 181 | @ParameterizedTest 182 | @MethodSource("provideAlgorithmLevelParams") 183 | public void testOutputStreamConstructor2(Algorithm algo, int level) throws IOException { 184 | if (!QatZipper.isQatAvailable()) { 185 | return; 186 | } 187 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 188 | try { 189 | try (QatCompressorOutputStream compressedStream = 190 | new QatCompressorOutputStream( 191 | outputStream, 192 | 16 * 1024, 193 | new QatZipper.Builder().setAlgorithm(algo).setLevel(level))) {} 194 | } catch (IOException | IllegalArgumentException e) { 195 | fail(e.getMessage()); 196 | } 197 | } 198 | 199 | @ParameterizedTest 200 | @MethodSource("provideModeAlgorithmParams") 201 | public void testOutputStreamConstructor3(Mode mode, Algorithm algo) throws IOException { 202 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 203 | try { 204 | try (QatCompressorOutputStream compressedStream = 205 | new QatCompressorOutputStream( 206 | outputStream, 16 * 1024, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) {} 207 | } catch (IOException | IllegalArgumentException e) { 208 | fail(e.getMessage()); 209 | } 210 | } 211 | 212 | @Test 213 | public void testOutputStreamBadBufferSize() throws IOException { 214 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 215 | try { 216 | try (QatCompressorOutputStream compressedStream = 217 | new QatCompressorOutputStream(outputStream, 0, Algorithm.LZ4)) { 218 | fail("Failed to catch IllegalArgumentException"); 219 | } 220 | } catch (IllegalArgumentException e) { 221 | assertTrue(true); 222 | } 223 | } 224 | 225 | @ParameterizedTest 226 | @MethodSource("provideModeAlgorithmLengthParams") 227 | public void testOutputStreamWriteAll1(Mode mode, Algorithm algo, int size) throws IOException { 228 | qzip = new QatZipper.Builder().setAlgorithm(algo).setMode(mode).build(); 229 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 230 | try (QatCompressorOutputStream compressedStream = 231 | new QatCompressorOutputStream( 232 | outputStream, size, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 233 | compressedStream.write(src); 234 | } 235 | byte[] outputStreamBuf = outputStream.toByteArray(); 236 | byte[] result = new byte[src.length]; 237 | qzip.decompress(outputStreamBuf, 0, outputStreamBuf.length, result, 0, result.length); 238 | 239 | assertTrue(Arrays.equals(src, result)); 240 | } 241 | 242 | @ParameterizedTest 243 | @MethodSource("provideModeAlgorithmLengthParams") 244 | public void testOutputStreamWriteAll3(Mode mode, Algorithm algo, int size) throws IOException { 245 | qzip = new QatZipper.Builder().setAlgorithm(algo).setMode(mode).build(); 246 | byte[] src = Files.readAllBytes(Paths.get(SAMPLE_CORPUS)); 247 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 248 | try (QatCompressorOutputStream compressedStream = 249 | new QatCompressorOutputStream( 250 | outputStream, size, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 251 | int i; 252 | int len = 0; 253 | for (i = 0; i < src.length; i += len) { 254 | len = Math.min(rnd.nextInt(20 * 1024), src.length - i); 255 | compressedStream.write(src, i, len); 256 | } 257 | assertEquals(src.length, i); 258 | } 259 | byte[] outputStreamBuf = outputStream.toByteArray(); 260 | byte[] result = new byte[src.length]; 261 | qzip.decompress(outputStreamBuf, 0, outputStreamBuf.length, result, 0, result.length); 262 | 263 | assertTrue(Arrays.equals(src, result)); 264 | } 265 | 266 | @ParameterizedTest 267 | @MethodSource("provideModeAlgorithmLengthParams") 268 | public void testOutputStreamWriteByte(Mode mode, Algorithm algo, int size) throws IOException { 269 | qzip = new QatZipper.Builder().setAlgorithm(algo).setMode(mode).build(); 270 | byte[] src = Files.readAllBytes(Paths.get(SAMPLE_CORPUS)); 271 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 272 | try (QatCompressorOutputStream compressedStream = 273 | new QatCompressorOutputStream( 274 | outputStream, size, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 275 | int i; 276 | int len = 0; 277 | for (i = 0; i < src.length; i += len) { 278 | if (i % 10 == 0) { // doWriteByte 279 | len = 1; 280 | compressedStream.write((int) src[i]); 281 | } else { 282 | len = Math.min(rnd.nextInt(20 * 1024), src.length - i); 283 | compressedStream.write(src, i, len); 284 | } 285 | } 286 | assertEquals(src.length, i); 287 | } 288 | byte[] outputStreamBuf = outputStream.toByteArray(); 289 | byte[] result = new byte[src.length]; 290 | qzip.decompress(outputStreamBuf, 0, outputStreamBuf.length, result, 0, result.length); 291 | 292 | assertTrue(Arrays.equals(src, result)); 293 | } 294 | 295 | @ParameterizedTest 296 | @MethodSource("provideModeAlgorithmLengthParams") 297 | public void testOutputStreamWriteFlush(Mode mode, Algorithm algo, int size) throws IOException { 298 | qzip = new QatZipper.Builder().setAlgorithm(algo).setMode(mode).build(); 299 | byte[] src = Files.readAllBytes(Paths.get(SAMPLE_CORPUS)); 300 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 301 | try (QatCompressorOutputStream compressedStream = 302 | new QatCompressorOutputStream( 303 | outputStream, size, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 304 | int i; 305 | int len = 0; 306 | for (i = 0; i < src.length; i += len) { 307 | if (i > 0 && i % 10 == 0) { // doFlush 308 | compressedStream.flush(); 309 | } 310 | len = Math.min(rnd.nextInt(20 * 1024), src.length - i); 311 | compressedStream.write(src, i, len); 312 | } 313 | assertEquals(src.length, i); 314 | } 315 | byte[] outputStreamBuf = outputStream.toByteArray(); 316 | byte[] result = new byte[src.length]; 317 | qzip.decompress(outputStreamBuf, 0, outputStreamBuf.length, result, 0, result.length); 318 | 319 | assertTrue(Arrays.equals(src, result)); 320 | } 321 | 322 | @ParameterizedTest 323 | @MethodSource("provideModeAlgorithmLengthParams") 324 | public void testOutputStreamClose(Mode mode, Algorithm algo, int size) throws IOException { 325 | byte[] src = Files.readAllBytes(Paths.get(SAMPLE_CORPUS)); 326 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 327 | QatCompressorOutputStream compressedStream = 328 | new QatCompressorOutputStream( 329 | outputStream, size, new QatZipper.Builder().setAlgorithm(algo).setMode(mode)); 330 | compressedStream.close(); 331 | try { 332 | compressedStream.write(src); 333 | fail("Failed to catch IOException!"); 334 | } catch (IOException ioe) { 335 | assertTrue(true); 336 | } 337 | } 338 | 339 | @ParameterizedTest 340 | @MethodSource("provideModeAlgorithmLengthParams") 341 | public void testOutputStreamWriteAfterClose(Mode mode, Algorithm algo, int size) 342 | throws IOException { 343 | byte[] src = Files.readAllBytes(Paths.get(SAMPLE_CORPUS)); 344 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 345 | QatCompressorOutputStream compressedStream = 346 | new QatCompressorOutputStream( 347 | outputStream, size, new QatZipper.Builder().setAlgorithm(algo).setMode(mode)); 348 | compressedStream.close(); 349 | try { 350 | compressedStream.write(src[0]); 351 | fail("Failed to catch IOException!"); 352 | } catch (IOException ioe) { 353 | assertTrue(true); 354 | } 355 | } 356 | 357 | @ParameterizedTest 358 | @MethodSource("provideModeAlgorithmLengthParams") 359 | public void testOutputStreamFlushAfterClose(Mode mode, Algorithm algo, int size) 360 | throws IOException { 361 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 362 | QatCompressorOutputStream compressedStream = 363 | new QatCompressorOutputStream( 364 | outputStream, size, new QatZipper.Builder().setAlgorithm(algo).setMode(mode)); 365 | compressedStream.close(); 366 | try { 367 | compressedStream.flush(); 368 | fail("Failed to catch IOException!"); 369 | } catch (IOException ioe) { 370 | assertTrue(true); 371 | } 372 | } 373 | 374 | @ParameterizedTest 375 | @MethodSource("provideModeAlgorithmLengthParams") 376 | public void testOutputStreamDoubleClose(Mode mode, Algorithm algo, int size) throws IOException { 377 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 378 | QatCompressorOutputStream compressedStream = 379 | new QatCompressorOutputStream( 380 | outputStream, size, new QatZipper.Builder().setAlgorithm(algo).setMode(mode)); 381 | compressedStream.close(); 382 | compressedStream.close(); 383 | assertTrue(true); 384 | } 385 | 386 | @ParameterizedTest 387 | @MethodSource("provideModeAlgorithmLengthParams") 388 | public void testOutputStreamFlushOnClose(Mode mode, Algorithm algo, int size) throws IOException { 389 | QatZipper qzip = new QatZipper.Builder().setAlgorithm(algo).setMode(mode).build(); 390 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 391 | final byte[] preResult; 392 | try (QatCompressorOutputStream compressedStream = 393 | new QatCompressorOutputStream( 394 | outputStream, size, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 395 | compressedStream.write(src); 396 | preResult = outputStream.toByteArray(); 397 | } 398 | 399 | byte[] outputStreamBuf = outputStream.toByteArray(); 400 | assertFalse(Arrays.equals(outputStreamBuf, preResult)); 401 | byte[] result = new byte[src.length]; 402 | qzip.decompress(outputStreamBuf, 0, outputStreamBuf.length, result, 0, result.length); 403 | 404 | assertTrue(Arrays.equals(src, result)); 405 | } 406 | 407 | @ParameterizedTest 408 | @MethodSource("provideModeAlgorithmParams") 409 | public void testOutputStreamWriteNullArray(Mode mode, Algorithm algo) throws IOException { 410 | qzip = new QatZipper.Builder().setAlgorithm(algo).setMode(mode).build(); 411 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 412 | try (QatCompressorOutputStream compressedStream = 413 | new QatCompressorOutputStream( 414 | outputStream, 16 * 1024, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 415 | try { 416 | compressedStream.write(null, 33, 100); 417 | fail("Failed to catch NullPointerException"); 418 | } catch (NullPointerException npe) { 419 | assertTrue(true); 420 | } 421 | } 422 | } 423 | 424 | @ParameterizedTest 425 | @MethodSource("provideModeAlgorithmParams") 426 | public void testOutputStreamWriteBadOffset(Mode mode, Algorithm algo) throws IOException { 427 | qzip = new QatZipper.Builder().setAlgorithm(algo).setMode(mode).build(); 428 | byte[] src = Files.readAllBytes(Paths.get(SAMPLE_CORPUS)); 429 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 430 | try (QatCompressorOutputStream compressedStream = 431 | new QatCompressorOutputStream( 432 | outputStream, 16 * 1024, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 433 | try { 434 | compressedStream.write(src, -33, 100); 435 | fail("Failed to catch IndexOutOfBoundsException"); 436 | } catch (IndexOutOfBoundsException oob) { 437 | assertTrue(true); 438 | } 439 | } 440 | } 441 | 442 | @ParameterizedTest 443 | @MethodSource("provideModeAlgorithmParams") 444 | public void testOutputStreamWriteBadLength(Mode mode, Algorithm algo) throws IOException { 445 | qzip = new QatZipper.Builder().setAlgorithm(algo).setMode(mode).build(); 446 | byte[] src = Files.readAllBytes(Paths.get(SAMPLE_CORPUS)); 447 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 448 | try (QatCompressorOutputStream compressedStream = 449 | new QatCompressorOutputStream( 450 | outputStream, 16 * 1024, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 451 | try { 452 | compressedStream.write(src, src.length - 1, -100); 453 | fail("Failed to catch IndexOutOfBoundsException"); 454 | } catch (IndexOutOfBoundsException oob) { 455 | assertTrue(true); 456 | } 457 | } 458 | } 459 | 460 | @ParameterizedTest 461 | @MethodSource("provideModeAlgorithmParams") 462 | public void testOutputStreamWriteBadOffsetAndLength(Mode mode, Algorithm algo) 463 | throws IOException { 464 | qzip = new QatZipper.Builder().setAlgorithm(algo).setMode(mode).build(); 465 | byte[] src = Files.readAllBytes(Paths.get(SAMPLE_CORPUS)); 466 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 467 | try (QatCompressorOutputStream compressedStream = 468 | new QatCompressorOutputStream( 469 | outputStream, 16 * 1024, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 470 | try { 471 | compressedStream.write(src, src.length - 1, 100); 472 | fail("Failed to catch IndexOutOfBoundsException"); 473 | } catch (IndexOutOfBoundsException oob) { 474 | assertTrue(true); 475 | } 476 | } 477 | } 478 | } 479 | -------------------------------------------------------------------------------- /src/test/java/com/intel/qat/QatDecompressorInputStreamTests.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | 7 | package com.intel.qat; 8 | 9 | import static com.intel.qat.QatZipper.Algorithm; 10 | import static com.intel.qat.QatZipper.Mode; 11 | import static org.junit.jupiter.api.Assertions.assertEquals; 12 | import static org.junit.jupiter.api.Assertions.assertFalse; 13 | import static org.junit.jupiter.api.Assertions.assertTrue; 14 | import static org.junit.jupiter.api.Assertions.fail; 15 | 16 | import java.io.ByteArrayInputStream; 17 | import java.io.ByteArrayOutputStream; 18 | import java.io.IOException; 19 | import java.nio.file.Files; 20 | import java.nio.file.Paths; 21 | import java.util.Arrays; 22 | import java.util.Random; 23 | import java.util.stream.Stream; 24 | import org.junit.jupiter.api.BeforeAll; 25 | import org.junit.jupiter.api.Test; 26 | import org.junit.jupiter.params.ParameterizedTest; 27 | import org.junit.jupiter.params.provider.Arguments; 28 | import org.junit.jupiter.params.provider.EnumSource; 29 | import org.junit.jupiter.params.provider.MethodSource; 30 | 31 | public class QatDecompressorInputStreamTests { 32 | private static final String SAMPLE_CORPUS = "src/test/resources/sample.txt"; 33 | private static byte[] src; 34 | private static byte[] deflateBytes; 35 | private static byte[] lz4Bytes; 36 | private static byte[] zstdBytes; 37 | private Random RANDOM = new Random(); 38 | 39 | @BeforeAll 40 | public static void setup() throws IOException { 41 | src = Files.readAllBytes(Paths.get(SAMPLE_CORPUS)); 42 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 43 | try (QatCompressorOutputStream compressedStream = 44 | new QatCompressorOutputStream(outputStream, 16 * 1024, Algorithm.LZ4)) { 45 | compressedStream.write(src); 46 | } 47 | lz4Bytes = outputStream.toByteArray(); 48 | ByteArrayOutputStream outputStream2 = new ByteArrayOutputStream(); 49 | try (QatCompressorOutputStream compressedStream = 50 | new QatCompressorOutputStream(outputStream2, 16 * 1024, Algorithm.DEFLATE)) { 51 | compressedStream.write(src); 52 | } 53 | deflateBytes = outputStream2.toByteArray(); 54 | ByteArrayOutputStream outputStream3 = new ByteArrayOutputStream(); 55 | try (QatCompressorOutputStream compressedStream = 56 | new QatCompressorOutputStream(outputStream3, 16 * 1024, Algorithm.ZSTD)) { 57 | compressedStream.write(src); 58 | } 59 | zstdBytes = outputStream3.toByteArray(); 60 | } 61 | 62 | public static byte[] selectInputBytes(Algorithm algo) { 63 | switch (algo) { 64 | case DEFLATE: 65 | return deflateBytes; 66 | case LZ4: 67 | return lz4Bytes; 68 | case ZSTD: 69 | return zstdBytes; 70 | default: 71 | throw new RuntimeException(); 72 | } 73 | } 74 | 75 | public static Stream provideModeAlgorithmParams() { 76 | if (QatZipper.isQatAvailable()) { 77 | return Stream.of( 78 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE), 79 | Arguments.of(Mode.AUTO, Algorithm.LZ4), 80 | Arguments.of(Mode.AUTO, Algorithm.ZSTD), 81 | Arguments.of(Mode.HARDWARE, Algorithm.DEFLATE), 82 | Arguments.of(Mode.HARDWARE, Algorithm.LZ4), 83 | Arguments.of(Mode.HARDWARE, Algorithm.ZSTD)); 84 | } else { 85 | return Stream.of( 86 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE), 87 | Arguments.of(Mode.AUTO, Algorithm.LZ4), 88 | Arguments.of(Mode.AUTO, Algorithm.ZSTD)); 89 | } 90 | } 91 | 92 | public static Stream provideModeAlgorithmLengthParams() { 93 | if (QatZipper.isQatAvailable()) { 94 | return Stream.of( 95 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 16384), 96 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 65536), 97 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 524288), 98 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 1048576), 99 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 16384), 100 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 65536), 101 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 524288), 102 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 1048576), 103 | Arguments.of(Mode.HARDWARE, Algorithm.DEFLATE, 16384), 104 | Arguments.of(Mode.HARDWARE, Algorithm.DEFLATE, 65536), 105 | Arguments.of(Mode.HARDWARE, Algorithm.DEFLATE, 524288), 106 | Arguments.of(Mode.HARDWARE, Algorithm.DEFLATE, 1048576), 107 | Arguments.of(Mode.HARDWARE, Algorithm.LZ4, 16384), 108 | Arguments.of(Mode.HARDWARE, Algorithm.LZ4, 65536), 109 | Arguments.of(Mode.HARDWARE, Algorithm.LZ4, 524288), 110 | Arguments.of(Mode.HARDWARE, Algorithm.LZ4, 1048576)); 111 | } else { 112 | return Stream.of( 113 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 16384), 114 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 65536), 115 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 524288), 116 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 1048576), 117 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 16384), 118 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 65536), 119 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 524288), 120 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 1048576)); 121 | } 122 | } 123 | 124 | public static Stream provideModeAlgorithmSkipLengthParams() { 125 | if (QatZipper.isQatAvailable()) { 126 | return Stream.of( 127 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 0), 128 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 1024), 129 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 16384), 130 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 0), 131 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 1024), 132 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 16384), 133 | Arguments.of(Mode.HARDWARE, Algorithm.DEFLATE, 0), 134 | Arguments.of(Mode.HARDWARE, Algorithm.DEFLATE, 1024), 135 | Arguments.of(Mode.HARDWARE, Algorithm.DEFLATE, 16384), 136 | Arguments.of(Mode.HARDWARE, Algorithm.LZ4, 0), 137 | Arguments.of(Mode.HARDWARE, Algorithm.LZ4, 1024), 138 | Arguments.of(Mode.HARDWARE, Algorithm.LZ4, 16384)); 139 | } else { 140 | return Stream.of( 141 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 0), 142 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 1024), 143 | Arguments.of(Mode.AUTO, Algorithm.DEFLATE, 16384), 144 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 0), 145 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 1024), 146 | Arguments.of(Mode.AUTO, Algorithm.LZ4, 16384)); 147 | } 148 | } 149 | 150 | @Test 151 | public void testNullStream() { 152 | ByteArrayInputStream inputStream = null; 153 | try { 154 | try (QatDecompressorInputStream decompressedStream = 155 | new QatDecompressorInputStream(inputStream, Algorithm.ZSTD)) {} 156 | fail("Failed to catch NullPointerException"); 157 | } catch (NullPointerException | IOException e) { 158 | assertTrue(true); 159 | } 160 | } 161 | 162 | @Test 163 | public void testConstructor() { 164 | if (!QatZipper.isQatAvailable()) { 165 | return; 166 | } 167 | ByteArrayInputStream inputStream = new ByteArrayInputStream(deflateBytes); 168 | try { 169 | try (QatDecompressorInputStream decompressedStream = 170 | new QatDecompressorInputStream(inputStream, 16 * 1024, Algorithm.LZ4)) {} 171 | } catch (IOException | IllegalArgumentException e) { 172 | fail(e.getMessage()); 173 | } 174 | } 175 | 176 | @Test 177 | public void testTwoArgConstructor() { 178 | if (!QatZipper.isQatAvailable()) { 179 | return; 180 | } 181 | ByteArrayInputStream inputStream = new ByteArrayInputStream(deflateBytes); 182 | try { 183 | try (QatDecompressorInputStream decompressedStream = 184 | new QatDecompressorInputStream(inputStream, Algorithm.DEFLATE)) {} 185 | } catch (IOException | IllegalArgumentException e) { 186 | fail(e.getMessage()); 187 | } 188 | } 189 | 190 | @ParameterizedTest 191 | @EnumSource(Algorithm.class) 192 | public void testConstructor1(Algorithm algo) { 193 | if (!QatZipper.isQatAvailable()) { 194 | return; 195 | } 196 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 197 | try { 198 | try (QatDecompressorInputStream decompressedStream = 199 | new QatDecompressorInputStream(inputStream, 16 * 1024, algo)) {} 200 | } catch (IOException | IllegalArgumentException e) { 201 | fail(e.getMessage()); 202 | } 203 | } 204 | 205 | @ParameterizedTest 206 | @MethodSource("provideModeAlgorithmParams") 207 | public void testConstructor2(Mode mode, Algorithm algo) { 208 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 209 | try { 210 | try (QatDecompressorInputStream decompressedStream = 211 | new QatDecompressorInputStream( 212 | inputStream, 16 * 1024, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) {} 213 | } catch (IOException | IllegalArgumentException e) { 214 | fail(e.getMessage()); 215 | } 216 | } 217 | 218 | @ParameterizedTest 219 | @EnumSource(Algorithm.class) 220 | public void testReset(Algorithm algo) throws IOException { 221 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 222 | try (QatDecompressorInputStream decompressedStream = 223 | new QatDecompressorInputStream( 224 | inputStream, 225 | 16 * 1024, 226 | new QatZipper.Builder().setAlgorithm(algo).setMode(Mode.AUTO))) { 227 | try { 228 | decompressedStream.reset(); 229 | fail("Failed to catch IOException!"); 230 | } catch (IOException ioe) { 231 | assertTrue(true); 232 | } 233 | } 234 | } 235 | 236 | @ParameterizedTest 237 | @EnumSource(Algorithm.class) 238 | public void testMarkSupported(Algorithm algo) throws IOException { 239 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 240 | try (QatDecompressorInputStream decompressedStream = 241 | new QatDecompressorInputStream( 242 | inputStream, 243 | 16 * 1024, 244 | new QatZipper.Builder().setAlgorithm(algo).setMode(Mode.AUTO))) { 245 | assertFalse(decompressedStream.markSupported()); 246 | } 247 | } 248 | 249 | @ParameterizedTest 250 | @MethodSource("provideModeAlgorithmLengthParams") 251 | public void testInputStreamReadAll1(Mode mode, Algorithm algo, int bufferSize) 252 | throws IOException { 253 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 254 | byte[] result = new byte[src.length]; 255 | try (QatDecompressorInputStream decompressedStream = 256 | new QatDecompressorInputStream( 257 | inputStream, bufferSize, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 258 | int read = decompressedStream.read(result); 259 | assertEquals(result.length, read); 260 | } 261 | assertTrue(Arrays.equals(src, result)); 262 | } 263 | 264 | @ParameterizedTest 265 | @MethodSource("provideModeAlgorithmLengthParams") 266 | public void testInputStreamReadAll3(Mode mode, Algorithm algo, int bufferSize) 267 | throws IOException { 268 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 269 | byte[] result = new byte[src.length]; 270 | try (QatDecompressorInputStream decompressedStream = 271 | new QatDecompressorInputStream( 272 | inputStream, bufferSize, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 273 | int i; 274 | int len = 0; 275 | for (i = 0; i < result.length; i += len) { 276 | len = Math.min(RANDOM.nextInt(20 * 1024), result.length - i); 277 | int read = decompressedStream.read(result, i, len); 278 | assertEquals(len, read); 279 | } 280 | assertEquals(result.length, i); 281 | } 282 | assertTrue(Arrays.equals(src, result)); 283 | } 284 | 285 | @ParameterizedTest 286 | @MethodSource("provideModeAlgorithmLengthParams") 287 | public void testInputStreamReadShort(Mode mode, Algorithm algo, int bufferSize) 288 | throws IOException { 289 | byte[] newSrc = Arrays.copyOf(src, bufferSize / 2); 290 | ByteArrayOutputStream outStream = new ByteArrayOutputStream(); 291 | try (QatCompressorOutputStream outputStream = 292 | new QatCompressorOutputStream( 293 | outStream, bufferSize, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 294 | outputStream.write(newSrc); 295 | } 296 | byte[] compressedBytes = outStream.toByteArray(); 297 | ByteArrayInputStream inputStream = new ByteArrayInputStream(compressedBytes); 298 | byte[] result = new byte[bufferSize]; 299 | int read = 0; 300 | try (QatDecompressorInputStream decompressedStream = 301 | new QatDecompressorInputStream( 302 | inputStream, bufferSize, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 303 | read = decompressedStream.read(result); 304 | assertEquals(-1, decompressedStream.read()); 305 | assertEquals(0, decompressedStream.available()); 306 | } 307 | assertTrue(Arrays.equals(newSrc, Arrays.copyOf(result, read))); 308 | } 309 | 310 | @ParameterizedTest 311 | @MethodSource("provideModeAlgorithmLengthParams") 312 | public void testInputStreamReadShort2(Mode mode, Algorithm algo, int bufferSize) 313 | throws IOException { 314 | byte[] newSrc = Arrays.copyOf(src, 1024); 315 | ByteArrayOutputStream outStream = new ByteArrayOutputStream(); 316 | try (QatCompressorOutputStream outputStream = 317 | new QatCompressorOutputStream( 318 | outStream, bufferSize, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 319 | outputStream.write(newSrc); 320 | } 321 | byte[] compressedBytes = outStream.toByteArray(); 322 | ByteArrayInputStream inputStream = new ByteArrayInputStream(compressedBytes); 323 | byte[] result = new byte[bufferSize]; 324 | int read = 0; 325 | try (QatDecompressorInputStream decompressedStream = 326 | new QatDecompressorInputStream( 327 | inputStream, 4, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 328 | read = decompressedStream.read(result); 329 | assertEquals(-1, decompressedStream.read()); 330 | assertEquals(0, decompressedStream.available()); 331 | } 332 | assertTrue(Arrays.equals(newSrc, Arrays.copyOf(result, read))); 333 | } 334 | 335 | @ParameterizedTest 336 | @MethodSource("provideModeAlgorithmLengthParams") 337 | public void testInputStreamReadByte(Mode mode, Algorithm algo, int bufferSize) 338 | throws IOException { 339 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 340 | byte[] result = new byte[src.length]; 341 | try (QatDecompressorInputStream decompressedStream = 342 | new QatDecompressorInputStream( 343 | inputStream, bufferSize, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 344 | int i; 345 | int len = 0; 346 | for (i = 0; i < result.length; i += len) { 347 | if (i % 10 == 0) { // doReadByte 348 | len = 1; 349 | result[i] = (byte) decompressedStream.read(); 350 | } else { 351 | len = Math.min(RANDOM.nextInt(20 * 1024), result.length - i); 352 | int read = decompressedStream.read(result, i, len); 353 | assertEquals(len, read); 354 | } 355 | } 356 | assertEquals(result.length, i); 357 | assertEquals(-1, decompressedStream.read()); 358 | assertEquals(0, decompressedStream.available()); 359 | } 360 | assertTrue(Arrays.equals(src, result)); 361 | } 362 | 363 | @ParameterizedTest 364 | @MethodSource("provideModeAlgorithmLengthParams") 365 | public void testInputStreamReadAvailable(Mode mode, Algorithm algo, int bufferSize) 366 | throws IOException { 367 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 368 | byte[] result = new byte[src.length]; 369 | try (QatDecompressorInputStream decompressedStream = 370 | new QatDecompressorInputStream( 371 | inputStream, bufferSize, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 372 | int i; 373 | int len = 0; 374 | for (i = 0; i < result.length; i += len) { 375 | len = decompressedStream.available(); 376 | assertTrue(len > 0); 377 | int read = decompressedStream.read(result, i, len); 378 | assertEquals(len, read); 379 | } 380 | assertEquals(result.length, i); 381 | } 382 | assertTrue(Arrays.equals(src, result)); 383 | } 384 | 385 | @ParameterizedTest 386 | @MethodSource("provideModeAlgorithmLengthParams") 387 | public void testInputStreamClose(Mode mode, Algorithm algo, int bufferSize) throws IOException { 388 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 389 | byte[] result = new byte[src.length]; 390 | QatDecompressorInputStream decompressedStream = 391 | new QatDecompressorInputStream( 392 | inputStream, bufferSize, new QatZipper.Builder().setAlgorithm(algo).setMode(mode)); 393 | decompressedStream.close(); 394 | try { 395 | int r = decompressedStream.read(result); 396 | fail("Failed to catch IOException. Return was " + r); 397 | } catch (IOException ioe) { 398 | assertTrue(true); 399 | } 400 | } 401 | 402 | @ParameterizedTest 403 | @MethodSource("provideModeAlgorithmLengthParams") 404 | public void testInputStreamReadAfterClose(Mode mode, Algorithm algo, int bufferSize) 405 | throws IOException { 406 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 407 | QatDecompressorInputStream decompressedStream = 408 | new QatDecompressorInputStream( 409 | inputStream, bufferSize, new QatZipper.Builder().setAlgorithm(algo).setMode(mode)); 410 | decompressedStream.close(); 411 | try { 412 | int r = decompressedStream.read(); 413 | fail("Failed to catch IOException. Returned was " + r); 414 | } catch (IOException ioe) { 415 | assertTrue(true); 416 | } 417 | } 418 | 419 | @ParameterizedTest 420 | @MethodSource("provideModeAlgorithmLengthParams") 421 | public void testInputStreamAvailableAfterClose(Mode mode, Algorithm algo, int bufferSize) 422 | throws IOException { 423 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 424 | QatDecompressorInputStream decompressedStream = 425 | new QatDecompressorInputStream( 426 | inputStream, bufferSize, new QatZipper.Builder().setAlgorithm(algo).setMode(mode)); 427 | decompressedStream.close(); 428 | try { 429 | decompressedStream.available(); 430 | fail("Failed to catch IOException!"); 431 | } catch (IOException ioe) { 432 | assertTrue(true); 433 | } 434 | } 435 | 436 | @ParameterizedTest 437 | @MethodSource("provideModeAlgorithmLengthParams") 438 | public void testInputStreamDoubleClose(Mode mode, Algorithm algo, int bufferSize) 439 | throws IOException { 440 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 441 | QatDecompressorInputStream decompressedStream = 442 | new QatDecompressorInputStream( 443 | inputStream, bufferSize, new QatZipper.Builder().setAlgorithm(algo).setMode(mode)); 444 | decompressedStream.close(); 445 | decompressedStream.close(); 446 | assertTrue(true); 447 | } 448 | 449 | @ParameterizedTest 450 | @MethodSource("provideModeAlgorithmSkipLengthParams") 451 | public void testInputStreamSkip(Mode mode, Algorithm algo, int bytesToSkip) throws IOException { 452 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 453 | try (QatCompressorOutputStream compressedStream = 454 | new QatCompressorOutputStream( 455 | outputStream, 16 * 1024, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 456 | compressedStream.write(src); 457 | compressedStream.write(new byte[bytesToSkip]); 458 | compressedStream.write(src); 459 | } 460 | byte[] srcBytes = outputStream.toByteArray(); 461 | 462 | ByteArrayInputStream inputStream = new ByteArrayInputStream(srcBytes); 463 | byte[] result1 = new byte[src.length]; 464 | byte[] result2 = new byte[src.length]; 465 | try (QatDecompressorInputStream decompressedStream = 466 | new QatDecompressorInputStream( 467 | inputStream, 16 * 1024, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 468 | int read = decompressedStream.read(result1); 469 | assertTrue(result1.length == read); 470 | long skipped = decompressedStream.skip(bytesToSkip); 471 | assertTrue(skipped == bytesToSkip); 472 | read = decompressedStream.read(result2); 473 | assertTrue(result2.length == read); 474 | } 475 | assertTrue(Arrays.equals(src, result1)); 476 | assertTrue(Arrays.equals(src, result2)); 477 | } 478 | 479 | @ParameterizedTest 480 | @MethodSource("provideModeAlgorithmParams") 481 | public void testInputStreamSkipNegative(Mode mode, Algorithm algo) throws IOException { 482 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 483 | byte[] result = new byte[src.length]; 484 | try (QatDecompressorInputStream decompressedStream = 485 | new QatDecompressorInputStream( 486 | inputStream, 16 * 1024, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 487 | int skipped = (int) decompressedStream.skip(-5); 488 | assertEquals(0, skipped); 489 | int read = decompressedStream.read(result); 490 | assertEquals(result.length, read); 491 | assertEquals(-1, decompressedStream.read()); 492 | assertEquals(0, decompressedStream.available()); 493 | } 494 | assertTrue(Arrays.equals(src, result)); 495 | } 496 | 497 | @ParameterizedTest 498 | @MethodSource("provideModeAlgorithmParams") 499 | public void testInputStreamReadBadOffset(Mode mode, Algorithm algo) throws IOException { 500 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 501 | byte[] result = new byte[src.length]; 502 | try (QatDecompressorInputStream decompressedStream = 503 | new QatDecompressorInputStream( 504 | inputStream, 16 * 1024, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 505 | try { 506 | decompressedStream.read(result, -33, 100); 507 | fail("Failed to catch IndexOutOfBoundsException"); 508 | } catch (IndexOutOfBoundsException oob) { 509 | assertTrue(true); 510 | } 511 | } 512 | } 513 | 514 | @ParameterizedTest 515 | @MethodSource("provideModeAlgorithmParams") 516 | public void testInputStreamReadNullArray(Mode mode, Algorithm algo) throws IOException { 517 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 518 | byte[] result = new byte[src.length]; 519 | try (QatDecompressorInputStream decompressedStream = 520 | new QatDecompressorInputStream( 521 | inputStream, 16 * 1024, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 522 | try { 523 | decompressedStream.read(null, result.length - 1, -100); 524 | fail("Failed to catch NullPointerException"); 525 | } catch (NullPointerException npe) { 526 | assertTrue(true); 527 | } 528 | } 529 | } 530 | 531 | @ParameterizedTest 532 | @MethodSource("provideModeAlgorithmParams") 533 | public void testInputStreamReadBadLength(Mode mode, Algorithm algo) throws IOException { 534 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 535 | byte[] result = new byte[src.length]; 536 | try (QatDecompressorInputStream decompressedStream = 537 | new QatDecompressorInputStream( 538 | inputStream, 16 * 1024, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 539 | try { 540 | decompressedStream.read(result, result.length - 1, -100); 541 | fail("Failed to catch IndexOutOfBoundsException"); 542 | } catch (IndexOutOfBoundsException oob) { 543 | assertTrue(true); 544 | } 545 | } 546 | } 547 | 548 | @ParameterizedTest 549 | @MethodSource("provideModeAlgorithmParams") 550 | public void testInputStreamReadBadOffsetAndLength(Mode mode, Algorithm algo) throws IOException { 551 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 552 | byte[] result = new byte[src.length]; 553 | try (QatDecompressorInputStream decompressedStream = 554 | new QatDecompressorInputStream( 555 | inputStream, 16 * 1024, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 556 | try { 557 | int r = decompressedStream.read(result, result.length - 1, 100); 558 | fail("Failed to catch IndexOutOfBoundsException. Returned " + r); 559 | } catch (IndexOutOfBoundsException oob) { 560 | assertTrue(true); 561 | } 562 | } 563 | } 564 | 565 | @ParameterizedTest 566 | @MethodSource("provideModeAlgorithmParams") 567 | public void testInputStreamRead3EOF(Mode mode, Algorithm algo) throws IOException { 568 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 569 | byte[] result = new byte[src.length]; 570 | byte[] result2 = new byte[src.length]; 571 | try (QatDecompressorInputStream decompressedStream = 572 | new QatDecompressorInputStream( 573 | inputStream, 16 * 1024, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 574 | int read = decompressedStream.read(result); 575 | assertEquals(result.length, read); 576 | read = decompressedStream.read(result2); 577 | assertEquals(-1, read); 578 | assertEquals(0, decompressedStream.available()); 579 | } 580 | assertTrue(Arrays.equals(src, result)); 581 | } 582 | 583 | @ParameterizedTest 584 | @MethodSource("provideModeAlgorithmParams") 585 | public void testInputStreamReadEOF(Mode mode, Algorithm algo) throws IOException { 586 | ByteArrayInputStream inputStream = new ByteArrayInputStream(selectInputBytes(algo)); 587 | byte[] result = new byte[src.length]; 588 | try (QatDecompressorInputStream decompressedStream = 589 | new QatDecompressorInputStream( 590 | inputStream, 16 * 1024, new QatZipper.Builder().setAlgorithm(algo).setMode(mode))) { 591 | int read = decompressedStream.read(result); 592 | assertEquals(result.length, read); 593 | read = decompressedStream.read(); 594 | assertEquals(-1, read); 595 | assertEquals(0, decompressedStream.available()); 596 | } 597 | assertTrue(Arrays.equals(src, result)); 598 | } 599 | } 600 | -------------------------------------------------------------------------------- /src/test/java/com/intel/qat/QatTestSuite.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | 7 | package com.intel.qat; 8 | 9 | import org.junit.platform.suite.api.SelectPackages; 10 | import org.junit.platform.suite.api.Suite; 11 | 12 | @Suite 13 | @SelectPackages("com.intel.qat") 14 | public class QatTestSuite { 15 | private static String flag = System.getProperty("hardware.available"); 16 | public static final boolean FORCE_HARDWARE = (flag != null && flag.equals("true")); 17 | } 18 | -------------------------------------------------------------------------------- /src/test/java/com/intel/qat/fuzzing/FuzzerTest.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Copyright (C) 2023 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: BSD 5 | ******************************************************************************/ 6 | package com.intel.qat.fuzzing; 7 | 8 | import com.code_intelligence.jazzer.api.FuzzedDataProvider; 9 | import com.intel.qat.QatCompressorOutputStream; 10 | import com.intel.qat.QatDecompressorInputStream; 11 | import com.intel.qat.QatZipper; 12 | import java.io.ByteArrayInputStream; 13 | import java.io.ByteArrayOutputStream; 14 | import java.io.IOException; 15 | import java.nio.ByteBuffer; 16 | import java.util.Arrays; 17 | import java.util.Random; 18 | 19 | public class FuzzerTest { 20 | private static final Random RANDOM = new Random(); 21 | 22 | public static void fuzzerTestOneInput(FuzzedDataProvider data) throws IOException { 23 | try { 24 | if (data.remainingBytes() == 0) return; 25 | 26 | // Save remaining bytes as a source data 27 | byte[] src = data.consumeRemainingAsBytes(); 28 | 29 | testByteArray(src); 30 | testByteArrayWithParams(src); 31 | testByteBuffer(src); 32 | testDirectByteBuffer(src); 33 | testMixedTypesOne(src); 34 | testMixedTypesTwo(src); 35 | testMixedTypesThree(src); 36 | testWithCompressionLengthAndRetry(src); 37 | testByteArrayLZ4(src); 38 | testByteArrayWithParamsLZ4(src); 39 | testByteBufferLZ4(src); 40 | testDirectByteBufferLZ4(src); 41 | testDecompressionSrcBBDstDBB(src); 42 | testDecompressionSrcBBDstDBBLZ4(src); 43 | testDecompressionSrcDBBDstBB(src); 44 | testDecompressionSrcDBBDstBBLZ4(src); 45 | testDecompressionSrcBBRODstBB(src); 46 | testDecompressionSrcBBRODstBBLZ4(src); 47 | testMixedTypesOneLZ4(src); 48 | testMixedTypesTwoLZ4(src); 49 | testMixedTypesThreeLZ4(src); 50 | testWithCompressionLengthAndRetryLZ4(src); 51 | testQatStreamDeflate(src); 52 | testQatStreamLZ4(src); 53 | testQatStreamDeflate2(src); 54 | testQatStreamLZ42(src); 55 | } catch (RuntimeException e) { 56 | throw e; 57 | } 58 | } 59 | 60 | static void testByteArray(byte[] src) { 61 | QatZipper qzip = new QatZipper.Builder().build(); 62 | byte[] dst = new byte[qzip.maxCompressedLength(src.length)]; 63 | byte[] dec = new byte[src.length]; 64 | 65 | int compressedSize = qzip.compress(src, 0, src.length, dst, 0, dst.length); 66 | qzip.decompress(dst, 0, compressedSize, dec, 0, dec.length); 67 | qzip.end(); 68 | 69 | assert Arrays.equals(src, dec) : "The source and decompressed arrays do not match."; 70 | } 71 | 72 | static void testByteArrayLZ4(byte[] src) { 73 | QatZipper qzip = new QatZipper.Builder().setAlgorithm(QatZipper.Algorithm.LZ4).build(); 74 | byte[] dst = new byte[qzip.maxCompressedLength(src.length)]; 75 | byte[] dec = new byte[src.length]; 76 | 77 | int compressedSize = qzip.compress(src, 0, src.length, dst, 0, dst.length); 78 | qzip.decompress(dst, 0, compressedSize, dec, 0, dec.length); 79 | qzip.end(); 80 | 81 | assert Arrays.equals(src, dec) : "The source and decompressed arrays do not match."; 82 | } 83 | 84 | static void testByteArrayWithParams(byte[] src) { 85 | QatZipper qzip = new QatZipper.Builder().build(); 86 | 87 | int srcOffset = RANDOM.nextInt(src.length); 88 | 89 | byte[] dst = new byte[qzip.maxCompressedLength(src.length)]; 90 | byte[] dec = new byte[src.length]; 91 | 92 | int compressedSize = qzip.compress(src, srcOffset, src.length - srcOffset, dst, 0, dst.length); 93 | int decompressedSize = qzip.decompress(dst, 0, compressedSize, dec, 0, dec.length); 94 | 95 | qzip.end(); 96 | 97 | assert Arrays.equals( 98 | Arrays.copyOfRange(src, srcOffset, src.length), 99 | Arrays.copyOfRange(dec, 0, decompressedSize)) 100 | : "The source and decompressed arrays do not match."; 101 | } 102 | 103 | static void testByteArrayWithParamsLZ4(byte[] src) { 104 | QatZipper qzip = new QatZipper.Builder().setAlgorithm(QatZipper.Algorithm.LZ4).build(); 105 | 106 | int srcOffset = RANDOM.nextInt(src.length); 107 | 108 | byte[] dst = new byte[qzip.maxCompressedLength(src.length)]; 109 | byte[] dec = new byte[src.length]; 110 | 111 | int compressedSize = qzip.compress(src, srcOffset, src.length - srcOffset, dst, 0, dst.length); 112 | int decompressedSize = qzip.decompress(dst, 0, compressedSize, dec, 0, dec.length); 113 | 114 | qzip.end(); 115 | 116 | assert Arrays.equals( 117 | Arrays.copyOfRange(src, srcOffset, src.length), 118 | Arrays.copyOfRange(dec, 0, decompressedSize)) 119 | : "The source and decompressed arrays do not match."; 120 | } 121 | 122 | static void testByteBuffer(byte[] src) { 123 | ByteBuffer srcBuf = ByteBuffer.allocate(src.length); 124 | srcBuf.put(src, 0, src.length); 125 | srcBuf.flip(); 126 | 127 | QatZipper qzip = new QatZipper.Builder().build(); 128 | int compressedSize = qzip.maxCompressedLength(src.length); 129 | 130 | assert compressedSize > 0; 131 | 132 | ByteBuffer comBuf = ByteBuffer.allocate(compressedSize); 133 | qzip.compress(srcBuf, comBuf); 134 | comBuf.flip(); 135 | 136 | ByteBuffer decBuf = ByteBuffer.allocate(src.length); 137 | qzip.decompress(comBuf, decBuf); 138 | 139 | qzip.end(); 140 | 141 | assert srcBuf.compareTo(decBuf) == 0 : "The source and decompressed buffers do not match."; 142 | } 143 | 144 | static void testByteBufferLZ4(byte[] src) { 145 | ByteBuffer srcBuf = ByteBuffer.allocate(src.length); 146 | srcBuf.put(src, 0, src.length); 147 | srcBuf.flip(); 148 | 149 | QatZipper qzip = new QatZipper.Builder().setAlgorithm(QatZipper.Algorithm.LZ4).build(); 150 | int compressedSize = qzip.maxCompressedLength(src.length); 151 | 152 | assert compressedSize > 0; 153 | 154 | ByteBuffer comBuf = ByteBuffer.allocate(compressedSize); 155 | qzip.compress(srcBuf, comBuf); 156 | comBuf.flip(); 157 | 158 | ByteBuffer decBuf = ByteBuffer.allocate(src.length); 159 | qzip.decompress(comBuf, decBuf); 160 | 161 | qzip.end(); 162 | 163 | assert srcBuf.compareTo(decBuf) == 0 : "The source and decompressed buffers do not match."; 164 | } 165 | 166 | static void testDirectByteBuffer(byte[] src) { 167 | ByteBuffer srcBuf = ByteBuffer.allocateDirect(src.length); 168 | srcBuf.put(src, 0, src.length); 169 | srcBuf.flip(); 170 | 171 | QatZipper qzip = new QatZipper.Builder().build(); 172 | int compressedSize = qzip.maxCompressedLength(src.length); 173 | 174 | assert compressedSize > 0; 175 | 176 | ByteBuffer comBuf = ByteBuffer.allocateDirect(compressedSize); 177 | qzip.compress(srcBuf, comBuf); 178 | comBuf.flip(); 179 | 180 | ByteBuffer decBuf = ByteBuffer.allocateDirect(src.length); 181 | qzip.decompress(comBuf, decBuf); 182 | 183 | qzip.end(); 184 | 185 | assert srcBuf.compareTo(decBuf) == 0 : "The source and decompressed buffers do not match."; 186 | } 187 | 188 | static void testDecompressionSrcBBDstDBB(byte[] src) { 189 | ByteBuffer srcBuf = ByteBuffer.allocate(src.length); 190 | srcBuf.put(src, 0, src.length); 191 | srcBuf.flip(); 192 | 193 | QatZipper qzip = new QatZipper.Builder().build(); 194 | int compressedSize = qzip.maxCompressedLength(src.length); 195 | 196 | assert compressedSize > 0; 197 | 198 | ByteBuffer comBuf = ByteBuffer.allocate(compressedSize); 199 | qzip.compress(srcBuf, comBuf); 200 | comBuf.flip(); 201 | 202 | ByteBuffer decBuf = ByteBuffer.allocateDirect(src.length); 203 | qzip.decompress(comBuf, decBuf); 204 | 205 | qzip.end(); 206 | 207 | assert srcBuf.compareTo(decBuf) == 0 : "The source and decompressed buffers do not match."; 208 | } 209 | 210 | static void testDecompressionSrcBBDstDBBLZ4(byte[] src) { 211 | ByteBuffer srcBuf = ByteBuffer.allocate(src.length); 212 | srcBuf.put(src, 0, src.length); 213 | srcBuf.flip(); 214 | 215 | QatZipper qzip = new QatZipper.Builder().setAlgorithm(QatZipper.Algorithm.LZ4).build(); 216 | int compressedSize = qzip.maxCompressedLength(src.length); 217 | 218 | assert compressedSize > 0; 219 | 220 | ByteBuffer comBuf = ByteBuffer.allocate(compressedSize); 221 | qzip.compress(srcBuf, comBuf); 222 | comBuf.flip(); 223 | 224 | ByteBuffer decBuf = ByteBuffer.allocateDirect(src.length); 225 | qzip.decompress(comBuf, decBuf); 226 | 227 | qzip.end(); 228 | 229 | assert srcBuf.compareTo(decBuf) == 0 : "The source and decompressed buffers do not match."; 230 | } 231 | 232 | static void testDecompressionSrcDBBDstBB(byte[] src) { 233 | ByteBuffer srcBuf = ByteBuffer.allocate(src.length); 234 | srcBuf.put(src, 0, src.length); 235 | srcBuf.flip(); 236 | 237 | QatZipper qzip = new QatZipper.Builder().build(); 238 | int compressedSize = qzip.maxCompressedLength(src.length); 239 | 240 | assert compressedSize > 0; 241 | 242 | ByteBuffer comBuf = ByteBuffer.allocateDirect(compressedSize); 243 | qzip.compress(srcBuf, comBuf); 244 | comBuf.flip(); 245 | 246 | ByteBuffer decBuf = ByteBuffer.allocate(src.length); 247 | qzip.decompress(comBuf, decBuf); 248 | 249 | qzip.end(); 250 | 251 | assert srcBuf.compareTo(decBuf) == 0 : "The source and decompressed buffers do not match."; 252 | } 253 | 254 | static void testDecompressionSrcDBBDstBBLZ4(byte[] src) { 255 | ByteBuffer srcBuf = ByteBuffer.allocate(src.length); 256 | srcBuf.put(src, 0, src.length); 257 | srcBuf.flip(); 258 | 259 | QatZipper qzip = new QatZipper.Builder().build(); 260 | int compressedSize = qzip.maxCompressedLength(src.length); 261 | 262 | assert compressedSize > 0; 263 | 264 | ByteBuffer comBuf = ByteBuffer.allocateDirect(compressedSize); 265 | qzip.compress(srcBuf, comBuf); 266 | comBuf.flip(); 267 | 268 | ByteBuffer decBuf = ByteBuffer.allocate(src.length); 269 | qzip.decompress(comBuf, decBuf); 270 | 271 | qzip.end(); 272 | 273 | assert srcBuf.compareTo(decBuf) == 0 : "The source and decompressed buffers do not match."; 274 | } 275 | 276 | static void testDecompressionSrcBBRODstBB(byte[] src) { 277 | ByteBuffer srcBuf = ByteBuffer.allocate(src.length); 278 | srcBuf.put(src, 0, src.length); 279 | srcBuf.flip(); 280 | 281 | QatZipper qzip = new QatZipper.Builder().build(); 282 | int compressedSize = qzip.maxCompressedLength(src.length); 283 | 284 | assert compressedSize > 0; 285 | 286 | ByteBuffer comBuf = ByteBuffer.allocateDirect(compressedSize); 287 | qzip.compress(srcBuf, comBuf); 288 | comBuf.flip(); 289 | 290 | ByteBuffer decBuf = ByteBuffer.allocate(src.length); 291 | ByteBuffer comBufRO = comBuf.asReadOnlyBuffer(); 292 | qzip.decompress(comBufRO, decBuf); 293 | 294 | qzip.end(); 295 | 296 | assert srcBuf.compareTo(decBuf) == 0 : "The source and decompressed buffers do not match."; 297 | } 298 | 299 | static void testDecompressionSrcBBRODstBBLZ4(byte[] src) { 300 | ByteBuffer srcBuf = ByteBuffer.allocate(src.length); 301 | srcBuf.put(src, 0, src.length); 302 | srcBuf.flip(); 303 | 304 | QatZipper qzip = new QatZipper.Builder().build(); 305 | int compressedSize = qzip.maxCompressedLength(src.length); 306 | 307 | assert compressedSize > 0; 308 | 309 | ByteBuffer comBuf = ByteBuffer.allocateDirect(compressedSize); 310 | qzip.compress(srcBuf, comBuf); 311 | comBuf.flip(); 312 | 313 | ByteBuffer decBuf = ByteBuffer.allocate(src.length); 314 | ByteBuffer comBufRO = comBuf.asReadOnlyBuffer(); 315 | qzip.decompress(comBufRO, decBuf); 316 | 317 | qzip.end(); 318 | 319 | assert srcBuf.compareTo(decBuf) == 0 : "The source and decompressed buffers do not match."; 320 | } 321 | 322 | static void testDirectByteBufferLZ4(byte[] src) { 323 | ByteBuffer srcBuf = ByteBuffer.allocateDirect(src.length); 324 | srcBuf.put(src, 0, src.length); 325 | srcBuf.flip(); 326 | 327 | QatZipper qzip = new QatZipper.Builder().setAlgorithm(QatZipper.Algorithm.LZ4).build(); 328 | int compressedSize = qzip.maxCompressedLength(src.length); 329 | 330 | assert compressedSize > 0; 331 | 332 | ByteBuffer comBuf = ByteBuffer.allocateDirect(compressedSize); 333 | qzip.compress(srcBuf, comBuf); 334 | comBuf.flip(); 335 | 336 | ByteBuffer decBuf = ByteBuffer.allocateDirect(src.length); 337 | qzip.decompress(comBuf, decBuf); 338 | 339 | qzip.end(); 340 | 341 | assert srcBuf.compareTo(decBuf) == 0 : "The source and decompressed buffers do not match."; 342 | } 343 | 344 | static void testMixedTypesOne(byte[] src) { 345 | ByteBuffer srcBuf = ByteBuffer.allocate(src.length); 346 | srcBuf.put(src, 0, src.length); 347 | srcBuf.flip(); 348 | 349 | QatZipper qzip = new QatZipper.Builder().build(); 350 | int compressedSize = qzip.maxCompressedLength(src.length); 351 | 352 | assert compressedSize > 0; 353 | 354 | ByteBuffer comBuf = ByteBuffer.allocateDirect(compressedSize); 355 | qzip.compress(srcBuf, comBuf); 356 | comBuf.flip(); 357 | 358 | ByteBuffer decBuf = ByteBuffer.allocateDirect(src.length); 359 | qzip.decompress(comBuf, decBuf); 360 | 361 | qzip.end(); 362 | 363 | assert srcBuf.compareTo(decBuf) == 0 : "The source and decompressed buffers do not match."; 364 | } 365 | 366 | static void testMixedTypesOneLZ4(byte[] src) { 367 | ByteBuffer srcBuf = ByteBuffer.allocate(src.length); 368 | srcBuf.put(src, 0, src.length); 369 | srcBuf.flip(); 370 | 371 | QatZipper qzip = new QatZipper.Builder().setAlgorithm(QatZipper.Algorithm.LZ4).build(); 372 | int compressedSize = qzip.maxCompressedLength(src.length); 373 | 374 | assert compressedSize > 0; 375 | 376 | ByteBuffer comBuf = ByteBuffer.allocateDirect(compressedSize); 377 | qzip.compress(srcBuf, comBuf); 378 | comBuf.flip(); 379 | 380 | ByteBuffer decBuf = ByteBuffer.allocateDirect(src.length); 381 | qzip.decompress(comBuf, decBuf); 382 | 383 | qzip.end(); 384 | 385 | assert srcBuf.compareTo(decBuf) == 0 : "The source and decompressed buffers do not match."; 386 | } 387 | 388 | static void testMixedTypesTwo(byte[] src) { 389 | ByteBuffer srcBuf = ByteBuffer.allocateDirect(src.length); 390 | srcBuf.put(src, 0, src.length); 391 | srcBuf.flip(); 392 | 393 | QatZipper qzip = new QatZipper.Builder().build(); 394 | int compressedSize = qzip.maxCompressedLength(src.length); 395 | 396 | assert compressedSize > 0; 397 | 398 | ByteBuffer comBuf = ByteBuffer.allocate(compressedSize); 399 | qzip.compress(srcBuf, comBuf); 400 | comBuf.flip(); 401 | 402 | ByteBuffer decBuf = ByteBuffer.allocate(src.length); 403 | qzip.decompress(comBuf, decBuf); 404 | 405 | qzip.end(); 406 | 407 | assert srcBuf.compareTo(decBuf) == 0 : "The source and decompressed buffers do not match."; 408 | } 409 | 410 | static void testMixedTypesTwoLZ4(byte[] src) { 411 | ByteBuffer srcBuf = ByteBuffer.allocateDirect(src.length); 412 | srcBuf.put(src, 0, src.length); 413 | srcBuf.flip(); 414 | 415 | QatZipper qzip = new QatZipper.Builder().setAlgorithm(QatZipper.Algorithm.LZ4).build(); 416 | int compressedSize = qzip.maxCompressedLength(src.length); 417 | 418 | assert compressedSize > 0; 419 | 420 | ByteBuffer comBuf = ByteBuffer.allocate(compressedSize); 421 | qzip.compress(srcBuf, comBuf); 422 | comBuf.flip(); 423 | 424 | ByteBuffer decBuf = ByteBuffer.allocate(src.length); 425 | qzip.decompress(comBuf, decBuf); 426 | 427 | qzip.end(); 428 | 429 | assert srcBuf.compareTo(decBuf) == 0 : "The source and decompressed buffers do not match."; 430 | } 431 | 432 | static void testMixedTypesThree(byte[] src) { 433 | ByteBuffer srcBuf = ByteBuffer.allocate(src.length); 434 | srcBuf.put(src, 0, src.length); 435 | srcBuf.flip(); 436 | 437 | QatZipper qzip = new QatZipper.Builder().build(); 438 | int compressedSize = qzip.maxCompressedLength(src.length); 439 | 440 | assert compressedSize > 0; 441 | ByteBuffer readonlyBuf = srcBuf.asReadOnlyBuffer(); 442 | srcBuf.flip(); 443 | 444 | ByteBuffer comBuf = ByteBuffer.allocate(compressedSize); 445 | qzip.compress(readonlyBuf, comBuf); 446 | comBuf.flip(); 447 | 448 | ByteBuffer decBuf = ByteBuffer.allocate(src.length); 449 | qzip.decompress(comBuf, decBuf); 450 | 451 | qzip.end(); 452 | 453 | assert srcBuf.compareTo(decBuf) == 0 : "The source and decompressed buffers do not match."; 454 | } 455 | 456 | static void testMixedTypesThreeLZ4(byte[] src) { 457 | ByteBuffer srcBuf = ByteBuffer.allocate(src.length); 458 | srcBuf.put(src, 0, src.length); 459 | srcBuf.flip(); 460 | 461 | QatZipper qzip = new QatZipper.Builder().setAlgorithm(QatZipper.Algorithm.LZ4).build(); 462 | int compressedSize = qzip.maxCompressedLength(src.length); 463 | 464 | assert compressedSize > 0; 465 | ByteBuffer readonlyBuf = srcBuf.asReadOnlyBuffer(); 466 | srcBuf.flip(); 467 | 468 | ByteBuffer comBuf = ByteBuffer.allocate(compressedSize); 469 | qzip.compress(readonlyBuf, comBuf); 470 | comBuf.flip(); 471 | 472 | ByteBuffer decBuf = ByteBuffer.allocate(src.length); 473 | qzip.decompress(comBuf, decBuf); 474 | 475 | qzip.end(); 476 | 477 | assert srcBuf.compareTo(decBuf) == 0 : "The source and decompressed buffers do not match."; 478 | } 479 | 480 | static void testWithCompressionLengthAndRetry(byte[] src) { 481 | int comLevel = RANDOM.nextInt(9) + 1; 482 | int retryCount = RANDOM.nextInt(20); 483 | 484 | QatZipper qzip = new QatZipper.Builder().setLevel(comLevel).setRetryCount(retryCount).build(); 485 | 486 | byte[] dst = new byte[qzip.maxCompressedLength(src.length)]; 487 | byte[] dec = new byte[src.length]; 488 | 489 | int compressedSize = qzip.compress(src, 0, src.length, dst, 0, dst.length); 490 | qzip.decompress(dst, 0, compressedSize, dec, 0, dec.length); 491 | 492 | qzip.end(); 493 | 494 | assert Arrays.equals(src, dec) : "The source and decompressed arrays do not match."; 495 | } 496 | 497 | static void testWithCompressionLengthAndRetryLZ4(byte[] src) { 498 | int comLevel = RANDOM.nextInt(9) + 1; 499 | int retryCount = RANDOM.nextInt(20); 500 | 501 | QatZipper qzip = new QatZipper.Builder().setLevel(comLevel).setRetryCount(retryCount).build(); 502 | 503 | byte[] dst = new byte[qzip.maxCompressedLength(src.length)]; 504 | byte[] dec = new byte[src.length]; 505 | 506 | int compressedSize = qzip.compress(src, 0, src.length, dst, 0, dst.length); 507 | qzip.decompress(dst, 0, compressedSize, dec, 0, dec.length); 508 | 509 | qzip.end(); 510 | 511 | assert Arrays.equals(src, dec) : "The source and decompressed arrays do not match."; 512 | } 513 | 514 | static void testQatStreamDeflate(byte[] src) throws IOException { 515 | if (src.length < 32) return; 516 | int compressBufferSize = 1 + Math.abs(RANDOM.nextInt(1024 * 1024)); 517 | int decompressBufferSize = 1 + Math.abs(RANDOM.nextInt(1024 * 1024)); 518 | 519 | QatZipper.Algorithm algo = QatZipper.Algorithm.DEFLATE; 520 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 521 | try (QatCompressorOutputStream compressedStream = 522 | new QatCompressorOutputStream(outputStream, compressBufferSize, algo)) { 523 | compressedStream.write(src); 524 | } 525 | byte[] outputStreamBuf = outputStream.toByteArray(); 526 | byte[] buffer = new byte[1024]; 527 | ByteArrayOutputStream resultStream = new ByteArrayOutputStream(); 528 | ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStreamBuf); 529 | try (QatDecompressorInputStream decompressedStream = 530 | new QatDecompressorInputStream(inputStream, decompressBufferSize, algo)) { 531 | int bytesRead; 532 | while ((bytesRead = decompressedStream.read(buffer)) != -1) { 533 | resultStream.write(buffer, 0, bytesRead); 534 | } 535 | assert decompressedStream.available() == 0; 536 | } 537 | assert Arrays.equals(src, resultStream.toByteArray()) 538 | : "The source and decompressed arrays do not match. cb = " 539 | + compressBufferSize 540 | + " db = " 541 | + decompressBufferSize 542 | + " srcLen is " 543 | + src.length; 544 | } 545 | 546 | static void testQatStreamLZ4(byte[] src) throws IOException { 547 | if (src.length < 32) return; 548 | ByteBuffer.wrap(Arrays.copyOf(src, 8)); 549 | int compressBufferSize = 1 + Math.abs(RANDOM.nextInt(1024 * 1024)); 550 | int decompressBufferSize = 1 + Math.abs(RANDOM.nextInt(1024 * 1024)); 551 | 552 | QatZipper.Algorithm algo = QatZipper.Algorithm.LZ4; 553 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 554 | try (QatCompressorOutputStream compressedStream = 555 | new QatCompressorOutputStream(outputStream, compressBufferSize, algo)) { 556 | compressedStream.write(src); 557 | } 558 | byte[] outputStreamBuf = outputStream.toByteArray(); 559 | byte[] buffer = new byte[1024]; 560 | ByteArrayOutputStream resultStream = new ByteArrayOutputStream(); 561 | ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStreamBuf); 562 | try (QatDecompressorInputStream decompressedStream = 563 | new QatDecompressorInputStream(inputStream, decompressBufferSize, algo)) { 564 | int bytesRead; 565 | while ((bytesRead = decompressedStream.read(buffer)) != -1) { 566 | resultStream.write(buffer, 0, bytesRead); 567 | } 568 | assert decompressedStream.available() == 0; 569 | } 570 | assert Arrays.equals(src, resultStream.toByteArray()) 571 | : "The source and decompressed arrays do not match. cb = " 572 | + compressBufferSize 573 | + " db = " 574 | + decompressBufferSize 575 | + " srcLen is " 576 | + src.length; 577 | } 578 | 579 | static void testQatStreamDeflate2(byte[] src) throws IOException { 580 | if (src.length < 32) return; 581 | int compressBufferSize = 1 + Math.abs(RANDOM.nextInt(1024 * 1024)); 582 | int decompressBufferSize = 1 + Math.abs(RANDOM.nextInt(1024 * 1024)); 583 | 584 | QatZipper.Algorithm algo = QatZipper.Algorithm.DEFLATE; 585 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 586 | try (QatCompressorOutputStream compressedStream = 587 | new QatCompressorOutputStream(outputStream, compressBufferSize, algo)) { 588 | compressedStream.write(src); 589 | } 590 | byte[] outputStreamBuf = outputStream.toByteArray(); 591 | ByteArrayOutputStream resultStream = new ByteArrayOutputStream(); 592 | ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStreamBuf); 593 | try (QatDecompressorInputStream decompressedStream = 594 | new QatDecompressorInputStream(inputStream, decompressBufferSize, algo)) { 595 | int bytesRead; 596 | while ((bytesRead = decompressedStream.read()) != -1) { 597 | resultStream.write(bytesRead); 598 | } 599 | assert decompressedStream.available() == 0; 600 | } 601 | assert Arrays.equals(src, resultStream.toByteArray()) 602 | : "The source and decompressed arrays do not match. cb = " 603 | + compressBufferSize 604 | + " db = " 605 | + decompressBufferSize 606 | + " srcLen is " 607 | + src.length; 608 | } 609 | 610 | static void testQatStreamLZ42(byte[] src) throws IOException { 611 | if (src.length < 32) return; 612 | ByteBuffer.wrap(Arrays.copyOf(src, 8)); 613 | int compressBufferSize = 1 + Math.abs(RANDOM.nextInt(1024 * 1024)); 614 | int decompressBufferSize = 1 + Math.abs(RANDOM.nextInt(1024 * 1024)); 615 | 616 | QatZipper.Algorithm algo = QatZipper.Algorithm.LZ4; 617 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 618 | try (QatCompressorOutputStream compressedStream = 619 | new QatCompressorOutputStream(outputStream, compressBufferSize, algo)) { 620 | compressedStream.write(src); 621 | } 622 | byte[] outputStreamBuf = outputStream.toByteArray(); 623 | ByteArrayOutputStream resultStream = new ByteArrayOutputStream(); 624 | ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStreamBuf); 625 | try (QatDecompressorInputStream decompressedStream = 626 | new QatDecompressorInputStream(inputStream, decompressBufferSize, algo)) { 627 | int bytesRead; 628 | while ((bytesRead = decompressedStream.read()) != -1) { 629 | resultStream.write(bytesRead); 630 | } 631 | assert decompressedStream.available() == 0; 632 | } 633 | assert Arrays.equals(src, resultStream.toByteArray()) 634 | : "The source and decompressed arrays do not match. cb = " 635 | + compressBufferSize 636 | + " db = " 637 | + decompressBufferSize 638 | + " srcLen is " 639 | + src.length; 640 | } 641 | } 642 | --------------------------------------------------------------------------------