├── .gitattributes ├── .github └── workflows │ ├── gradle-wrapper-validation.yml │ └── nebula.yml ├── .gitignore ├── LICENSE ├── OSSMETADATA ├── README.md ├── build.gradle ├── ci-database-config └── mysql │ └── my.cnf ├── derand ├── build.gradle └── src │ ├── main │ └── java │ │ └── netflixoss │ │ └── derand │ │ ├── Derand.java │ │ └── StringToEncoderTranslator.java │ └── test │ └── java │ └── netflixoss │ └── derand │ └── DerandTest.java ├── derand_model.png ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat └── settings.gradle /.gitattributes: -------------------------------------------------------------------------------- 1 | build/ 2 | .gradle/ 3 | *.iml 4 | *.ipr 5 | *.iws 6 | .idea/ 7 | out/ 8 | secrets/signing-key 9 | -------------------------------------------------------------------------------- /.github/workflows/gradle-wrapper-validation.yml: -------------------------------------------------------------------------------- 1 | name: "Validate Gradle Wrapper" 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | validation: 6 | name: "Validation" 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | - uses: gradle/wrapper-validation-action@v1 11 | -------------------------------------------------------------------------------- /.github/workflows/nebula.yml: -------------------------------------------------------------------------------- 1 | name: Nebula Build 2 | on: 3 | push: 4 | branches: 5 | - '*' 6 | tags: 7 | - v*.*.* 8 | - v*.*.*-rc.* 9 | pull_request: 10 | 11 | jobs: 12 | validation: 13 | name: "Gradle Wrapper Validation" 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v2 17 | - uses: gradle/wrapper-validation-action@v1 18 | multijdk: 19 | if: | 20 | github.event_name == 'pull_request' || 21 | ((!startsWith(github.ref, 'refs/tags/v')) && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/main') 22 | needs: validation 23 | runs-on: ubuntu-latest 24 | strategy: 25 | matrix: 26 | # test against latest update of some major Java version(s), as well as specific LTS version(s) 27 | java: [ 8, 11 ] 28 | name: Gradle Build without Publish 29 | steps: 30 | - uses: actions/checkout@v1 31 | - name: Setup jdk 32 | uses: actions/setup-java@v1 33 | with: 34 | java-version: ${{ matrix.java }} 35 | - uses: actions/cache@v1 36 | id: gradle-cache 37 | with: 38 | path: ~/.gradle/caches 39 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }} 40 | restore-keys: | 41 | - ${{ runner.os }}-gradle- 42 | - uses: actions/cache@v1 43 | id: gradle-wrapper-cache 44 | with: 45 | path: ~/.gradle/wrapper 46 | key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }} 47 | restore-keys: | 48 | - ${{ runner.os }}-gradlewrapper- 49 | - name: docker test 50 | run: docker --help 51 | - name: Gradle build 52 | run: ./gradlew build 53 | publish: 54 | if: | 55 | github.event_name != 'pull_request' && 56 | (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main') 57 | needs: validation 58 | runs-on: ubuntu-latest 59 | name: Gradle Build and Publish 60 | steps: 61 | - uses: actions/checkout@v1 62 | - name: Setup jdk 8 63 | uses: actions/setup-java@v1 64 | with: 65 | java-version: 1.8 66 | - uses: actions/cache@v1 67 | id: gradle-cache 68 | with: 69 | path: ~/.gradle/caches 70 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }} 71 | restore-keys: | 72 | - ${{ runner.os }}-gradle- 73 | - uses: actions/cache@v1 74 | id: gradle-wrapper-cache 75 | with: 76 | path: ~/.gradle/wrapper 77 | key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }} 78 | restore-keys: | 79 | - ${{ runner.os }}-gradlewrapper- 80 | - name: Publish snapshot 81 | if: (!startsWith(github.ref, 'refs/tags/v')) 82 | run: ./gradlew build snapshot 83 | env: 84 | NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} 85 | NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} 86 | NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} 87 | NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} 88 | - name: Publish candidate 89 | if: | 90 | startsWith(github.ref, 'refs/tags/v') && 91 | contains(github.ref, '-rc.') 92 | run: ./gradlew -Prelease.useLastTag=true candidate 93 | env: 94 | NETFLIX_OSS_SONATYPE_USERNAME: ${{ secrets.ORG_SONATYPE_USERNAME }} 95 | NETFLIX_OSS_SONATYPE_PASSWORD: ${{ secrets.ORG_SONATYPE_PASSWORD }} 96 | NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} 97 | NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} 98 | NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} 99 | NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} 100 | - name: Publish release 101 | if: | 102 | startsWith(github.ref, 'refs/tags/v') && 103 | (!contains(github.ref, '-rc.')) 104 | run: ./gradlew -Prelease.useLastTag=true final 105 | env: 106 | NETFLIX_OSS_SONATYPE_USERNAME: ${{ secrets.ORG_SONATYPE_USERNAME }} 107 | NETFLIX_OSS_SONATYPE_PASSWORD: ${{ secrets.ORG_SONATYPE_PASSWORD }} 108 | NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} 109 | NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} 110 | NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} 111 | NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} 112 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | .gradle/ 3 | *.iml 4 | *.ipr 5 | *.iws 6 | .idea/ 7 | out/ 8 | secrets/signing-key 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | https://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | Copyright 2013-2018 Docker, Inc. 179 | 180 | Licensed under the Apache License, Version 2.0 (the "License"); 181 | you may not use this file except in compliance with the License. 182 | You may obtain a copy of the License at 183 | 184 | https://www.apache.org/licenses/LICENSE-2.0 185 | 186 | Unless required by applicable law or agreed to in writing, software 187 | distributed under the License is distributed on an "AS IS" BASIS, 188 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 189 | See the License for the specific language governing permissions and 190 | limitations under the License. -------------------------------------------------------------------------------- /OSSMETADATA: -------------------------------------------------------------------------------- 1 | osslifecycle=active -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Derand 2 | 3 | ## Text tokenization and cleanup from words that look random 4 | 5 | ```java 6 | Derand.tokenize("hello 3y29842ysjhfs world"); 7 | // "hello world" 8 | 9 | Derand.clean("hello 3y29842ysjhfs world") 10 | // "hello world 11 | ``` 12 | 13 | 14 | ### Maven 15 | 16 | ```shell 17 | 18 | com.netflix.netflixoss.derand 19 | derand 20 | 0.0.2 21 | 22 | ``` 23 | 24 | ### Gradle 25 | ```shell 26 | implementation 'com.netflix.netflixoss.derand:derand:0.0.2' 27 | ``` 28 | 29 | ### How it works? 30 | 31 | Derand uses pre-trained character level Convolutional Neural Network that operates on character embeddings. 32 | Derand neural net has slightly over 1000 params. 33 | Derand performs very well on CPUs and with default params achieves sub-millisecond inference on strings. 34 | 35 | The model of the network is in lightly optimized ONNX format that you can further optimize for your specific CPU. 36 | 37 | ### Does it scale? 38 | This package is being used in the logs ingestion pipeline at edge and scales to process gigabytes of data a second with a similar performance of regex. 39 | 40 | ### Is it thread-safe? 41 | Derand uses DJL and its predictors to load ONNX model and run inference. 42 | Each predictor is initialized in a ThreadLocal container. 43 | 44 | 45 | ### What are dependencies? 46 | This package depends on [Deep Java Library](https://djl.ai/), 47 | DJL's PyTorch and ONNX engines wrappers. 48 | 49 | ### How the model looks? 50 | ![Model](./derand_model.png) 51 | 52 | 53 | ### What is the input to the model? 54 | Input to the model is an array of character ids based on the character position in a pre-defined list of available chars. 55 | 56 | 57 | ### What is output of the model? 58 | Model outputs softmax probability of a word being random or non-random. 59 | 60 | ### How is model trained and what is its performance? 61 | Model is trained on about a million of examples in total, with %20 of the data in test set. 62 | On the test data model achieves 63 | ```shell 64 | acc: 0.9951 65 | precision: 0.9951 66 | recall: 0.9951 67 | ``` 68 | 69 | #### Performance optimizations 70 | Follow DJL inference optimization guide for PyTorch and ONNX. 71 | 72 | Usually, having ENV variable 73 | ```shell 74 | export OMP_NUM_THREADS=1 75 | ``` 76 | + JVM params 77 | ```shell 78 | -Dai.djl.pytorch.num_interop_threads=1 79 | -Dai.djl.pytorch.num_threads=1 80 | 81 | ``` 82 | achieves the highest throughput in a multi-threaded inference environment, such as 83 | * Scala / Akka parallel streams 84 | * Java parallel streams 85 | * Java Executors 86 | 87 | 88 | ### Examples of processing 89 | As Derand uses neural network and softmax probabilities to classify words, it is doing best effort tokenization. Below are some examples: 90 | 91 | 92 | The raw log examples are taken from Hadoop logs hosted on [Loghub](https://github.com/logpai/loghub) 93 | 94 | --- 95 | > Shilin He, Jieming Zhu, Pinjia He, Michael R. Lyu. [Loghub: A Large Collection of System Log Datasets towards Automated Log Analytics.](https://arxiv.org/abs/2008.06448) Arxiv, 2020. 96 | --- 97 | 98 | The Loghub Hadoop dataset was used just to provide the below examples: 99 | 100 | ```shell 101 | ----------- 102 | Before: org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Created MRAppMaster for application appattempt_1445144423722_0020_000001 103 | After: org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Created MRAppMaster for application 104 | ----------- 105 | 106 | ----------- 107 | Before: org.apache.hadoop.yarn.event.AsyncDispatcher: Registering class org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventType for class org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler 108 | After: org.apache.hadoop.yarn.event.AsyncDispatcher: Registering class org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventType for class org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler 109 | ----------- 110 | 111 | ----------- 112 | Before: org.apache.hadoop.http.HttpServer2: Jetty bound to port 62267 113 | After: Jetty bound to port 62267 # org.apache.hadoop.http.HttpServer2 considered to be random looking 114 | ----------- 115 | 116 | ----------- 117 | Before: Started HttpServer2$SelectChannelConnectorWithSafeStartup@0.0.0.0:62267 118 | After: Started 119 | ----------- 120 | 121 | ----------- 122 | Before: [IPC Server handler 13 on 62270] org.apache.hadoop.mapred.TaskAttemptListenerImpl: JVM with ID : jvm_1445144423722_0020_m_000002 asked for a task 123 | After: [IPC Server handler 13 on 62270] org.apache.hadoop.mapred.TaskAttemptListenerImpl: JVM with ID : asked for a task 124 | ----------- 125 | ``` 126 | 127 | Sometimes Derand tokenizes slightly non-random words, examples: 128 | 129 | ```shell 130 | ----------- 131 | Before: [RMCommunicator Allocator] org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator: Recalculating schedule, headroom= 132 | After: [RMCommunicator Allocator] org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator: Recalculating schedule, headroom= # Word "vCores:-27>" was considered random 133 | ----------- 134 | 135 | ----------- 136 | [RMCommunicator Allocator] org.apache.hadoop.ipc.Client: Retrying connect to server: msra-sa-41:8030. Already tried 0 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) 137 | [RMCommunicator Allocator] org.apache.hadoop.ipc.Client: Retrying connect to server: msra-sa-41:8030. Already tried 0 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=10, MILLISECONDS) # "sleepTime=1000" tokenized as 138 | ----------- 139 | ``` 140 | 141 | 142 | 143 | ### Slack community 144 | Working on open-source AI tools in observability space? Join our slack community 145 | [Open Source Observability Intelligence](https://join.slack.com/t/opensourceobs-fp54349/shared_invite/zt-mwnaslja-0mxk3dyyqB~WUKZ3ive7Dg) 146 | 147 | ### License 148 | This project is licensed under the [Apache-2.0 License](./LICENSE). -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | buildscript { 2 | repositories { 3 | maven { 4 | url = 'https://dl.bintray.com/nebula/gradle-plugins' 5 | } 6 | maven { 7 | url "https://plugins.gradle.org/m2/" 8 | } 9 | } 10 | dependencies { 11 | classpath "com.netflix.nebula:gradle-netflixoss-project-plugin:9.3.0" 12 | } 13 | } 14 | 15 | allprojects { 16 | group = 'com.netflix.netflixoss.derand' 17 | apply plugin: 'nebula.netflixoss' 18 | } 19 | 20 | subprojects { 21 | apply plugin: 'java' 22 | 23 | repositories { 24 | mavenCentral() 25 | } 26 | 27 | 28 | dependencies { 29 | constraints { 30 | // Define dependency versions as constraints 31 | implementation 'org.apache.commons:commons-text:1.9' 32 | } 33 | 34 | // Use JUnit Jupiter API for testing. 35 | testImplementation 'org.junit.jupiter:junit-jupiter-api:5.6.2' 36 | 37 | // Use JUnit Jupiter Engine for testing. 38 | testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' 39 | } 40 | 41 | tasks.named('test') { 42 | // Use junit platform for unit tests. 43 | useJUnitPlatform() 44 | } 45 | 46 | } 47 | 48 | gradleEnterprise { 49 | buildScan { 50 | publishAlways() 51 | termsOfServiceUrl = "https://gradle.com/terms-of-service" 52 | termsOfServiceAgree = "yes" 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /ci-database-config/mysql/my.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | innodb_file_per_table=ON 3 | innodb_large_prefix=ON 4 | innodb_file_format=barracuda 5 | -------------------------------------------------------------------------------- /derand/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'application' 2 | 3 | dependencies { 4 | 5 | implementation 'ai.djl.pytorch:pytorch-engine:0.10.0' 6 | implementation 'ai.djl.pytorch:pytorch-native-auto:1.7.1' 7 | 8 | implementation ('ai.djl.onnxruntime:onnxruntime-engine:0.10.0'){ 9 | exclude group: "com.microsoft.onnxruntime", module: "onnxruntime" 10 | } 11 | implementation "com.microsoft.onnxruntime:onnxruntime:1.7.0" 12 | 13 | 14 | } 15 | 16 | application { 17 | // Define the main class for the application. 18 | mainClass = 'netflixoss.derand.Derand' 19 | } 20 | -------------------------------------------------------------------------------- /derand/src/main/java/netflixoss/derand/Derand.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Copyright 2018 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package netflixoss.derand; 19 | 20 | import ai.djl.MalformedModelException; 21 | import ai.djl.inference.Predictor; 22 | import ai.djl.repository.zoo.Criteria; 23 | import ai.djl.repository.zoo.ModelNotFoundException; 24 | import ai.djl.repository.zoo.ModelZoo; 25 | import ai.djl.repository.zoo.ZooModel; 26 | import ai.djl.translate.TranslateException; 27 | 28 | import java.io.IOException; 29 | import java.util.*; 30 | 31 | public class Derand { 32 | 33 | private static final String MODEL_URL = "https://randomly-public-us-east-1.s3.amazonaws.com/derand.onnx.zip"; 34 | private static final String MODEL_NAME = "derand.onnx"; 35 | private static final String RND = ""; 36 | 37 | 38 | private static final ThreadLocal> predictorHolder = new ThreadLocal<>(); 39 | 40 | synchronized static Optional> init() throws IOException, ModelNotFoundException, MalformedModelException { 41 | if(model == null || !model.isPresent()){ 42 | return Optional.ofNullable(getModel()); 43 | } else { 44 | return model; 45 | } 46 | } 47 | 48 | private static ZooModel getModel() throws MalformedModelException, ModelNotFoundException, IOException { 49 | Criteria criteria = Criteria.builder() 50 | .setTypes(String.class, Boolean.class) 51 | .optTranslator(new StringToEncoderTranslator()) 52 | .optModelUrls(MODEL_URL) 53 | .optModelName(MODEL_NAME) 54 | .optEngine("OnnxRuntime") 55 | .build(); 56 | 57 | return ModelZoo.loadModel(criteria); 58 | } 59 | 60 | // TODO: add retry 61 | private static Optional> model; 62 | 63 | static { 64 | try { 65 | model = init(); 66 | } catch (IOException | ModelNotFoundException | MalformedModelException e) { 67 | model = Optional.empty(); 68 | } 69 | } 70 | 71 | 72 | private static boolean[] predictRandomnessPerWord(String[] words) throws TranslateException, IOException { 73 | 74 | if(!model.isPresent()){ 75 | throw new IOException("model can not be null"); 76 | } 77 | 78 | Predictor predictor = predictorHolder.get(); 79 | 80 | if(predictor == null){ 81 | predictor = model.get().newPredictor(); 82 | predictorHolder.set(predictor); 83 | } 84 | 85 | boolean[] result = new boolean[words.length]; 86 | 87 | int idx = 0; 88 | 89 | for (String word: words){ 90 | boolean isRandom = predictor.predict(word); 91 | result[idx] = isRandom; 92 | idx++; 93 | } 94 | 95 | return result; 96 | } 97 | 98 | private static String[] tokenizeWords(String[] words) throws TranslateException, IOException { 99 | 100 | String[] result = new String[words.length]; 101 | boolean[] randomMask = predictRandomnessPerWord(words); 102 | int idx = 0; 103 | 104 | for(boolean isRandom: randomMask){ 105 | if(isRandom){ 106 | result[idx] = RND; 107 | } else{ 108 | result[idx] = words[idx]; 109 | } 110 | idx++; 111 | } 112 | 113 | return result; 114 | } 115 | 116 | public static String tokenize(String text) throws TranslateException, IOException { 117 | 118 | if (isEmpty(text)) return ""; 119 | 120 | StringJoiner joiner = new StringJoiner(" "); 121 | 122 | for(String tokenizedWord :tokenizeWords(text.split(" "))){ 123 | joiner.add(tokenizedWord); 124 | } 125 | 126 | return joiner.toString(); 127 | } 128 | 129 | public static String clean(String text) throws TranslateException, IOException { 130 | 131 | if (isEmpty(text)) return ""; 132 | 133 | StringJoiner joiner = new StringJoiner(" "); 134 | 135 | for(String tokenizedWord :tokenizeWords(text.split(" "))){ 136 | if(!tokenizedWord.equals(RND)){ 137 | joiner.add(tokenizedWord); 138 | } 139 | } 140 | 141 | return joiner.toString(); 142 | 143 | } 144 | 145 | public static boolean[] classify(String text) throws TranslateException, IOException { 146 | /** 147 | * return boolean array that represents each word in a string, split by whitespace, as either random (true) or non-random (false) element 148 | */ 149 | 150 | if (isEmpty(text)) return new boolean[]{}; 151 | 152 | return predictRandomnessPerWord(text.split(" ")); 153 | } 154 | 155 | private static boolean isEmpty(String text) { 156 | return text == null || text.isEmpty() || text.trim().isEmpty(); 157 | } 158 | 159 | } 160 | -------------------------------------------------------------------------------- /derand/src/main/java/netflixoss/derand/StringToEncoderTranslator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Copyright 2018 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | 19 | package netflixoss.derand; 20 | 21 | import ai.djl.ndarray.NDArray; 22 | import ai.djl.ndarray.NDList; 23 | import ai.djl.ndarray.NDManager; 24 | import ai.djl.translate.Batchifier; 25 | import ai.djl.translate.Translator; 26 | import ai.djl.translate.TranslatorContext; 27 | 28 | import java.util.Arrays; 29 | import java.util.List; 30 | 31 | 32 | public final class StringToEncoderTranslator implements Translator { 33 | 34 | StringToEncoderTranslator() {} 35 | 36 | private static final List modelChars = Arrays.asList('!', '"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',', '-', '.', 37 | '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', 38 | '=', '>', '?', '@', '_', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 39 | 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 40 | 'x', 'y', 'z'); 41 | 42 | int maxLen = 16; 43 | 44 | @Override 45 | public NDList processInput(TranslatorContext ctx, String input) { 46 | 47 | /** 48 | * This method allows to convert word "hello" to a {@modelChars.lenght()} dimensional array with values representing character index 49 | * in the modelChars array and if the word is less than 50 characters, it will be left padded with 0f. 50 | * 51 | * For example, if {@modelChars.lenght()} == 50, "hello" will be converted to 52 | * [ 53 | * 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54 | * 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55 | * 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 37, 44, 56 | * 44, 47 57 | * ] 58 | * 59 | * which is a numerical input that neural network understand. 60 | */ 61 | NDManager manager = ctx.getNDManager(); 62 | 63 | if (input == null){ 64 | return new NDList(manager.create(new float[maxLen]).expandDims(0)); 65 | } 66 | 67 | int charArraySize = input.length(); 68 | 69 | // special case 70 | if (charArraySize == 0 ){ 71 | // return 0 filled array 72 | return new NDList(manager.create(new float[maxLen]).expandDims(0)); 73 | } 74 | 75 | float[] finalResult = new float[maxLen]; 76 | 77 | if(charArraySize > maxLen){ 78 | /* 79 | Removing extra characters from the left 80 | */ 81 | int startIdx = charArraySize - maxLen; 82 | int currentIdx = 0; 83 | int beginIndex = 0; 84 | for (Character charVal: input.toCharArray()){ 85 | if(currentIdx < startIdx){ 86 | currentIdx++; 87 | continue; 88 | } 89 | int charIdx = modelChars.indexOf(charVal); 90 | if(charIdx != -1){ 91 | finalResult[beginIndex] = (float) charIdx; 92 | } else { 93 | finalResult[beginIndex] = 0f; 94 | } 95 | beginIndex++; 96 | currentIdx++; 97 | } 98 | } else if (charArraySize < maxLen) { // pad with 0s on the left 99 | int startIdx = maxLen - charArraySize; 100 | int currentIdx = 0; 101 | for (Character charVal: input.toCharArray()){ 102 | if(currentIdx < startIdx){ 103 | finalResult[currentIdx] = 0f; 104 | } else { 105 | int charIdx = modelChars.indexOf(charVal); 106 | if(charIdx != -1){ 107 | finalResult[currentIdx] = (float) charIdx; 108 | } else { 109 | finalResult[currentIdx] = 0f; 110 | } 111 | } 112 | currentIdx++; 113 | } 114 | 115 | } else { // same size of maxLen and input 116 | int currentIdx = 0; 117 | for (Character charVal: input.toCharArray()){ 118 | int charIdx = modelChars.indexOf(charVal); 119 | if(charIdx != -1){ 120 | finalResult[currentIdx] = (float) charIdx; 121 | } else { 122 | finalResult[currentIdx] = 0f; 123 | } 124 | currentIdx++; 125 | } 126 | } 127 | 128 | return new NDList(manager.create(finalResult).expandDims(0)); 129 | } 130 | 131 | 132 | @Override 133 | public Boolean processOutput(TranslatorContext ctx, NDList list) { 134 | NDArray ndArray = list.singletonOrThrow(); 135 | NDArray ndArray1 = ndArray.get(0).argMax(); 136 | return ndArray1.toUint8Array()[0] == 1; 137 | } 138 | 139 | @Override 140 | public Batchifier getBatchifier() { 141 | return null; 142 | } 143 | } -------------------------------------------------------------------------------- /derand/src/test/java/netflixoss/derand/DerandTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Copyright 2018 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package netflixoss.derand; 19 | 20 | import ai.djl.translate.TranslateException; 21 | import org.junit.jupiter.api.Test; 22 | 23 | import java.io.IOException; 24 | 25 | import static org.junit.jupiter.api.Assertions.assertArrayEquals; 26 | import static org.junit.jupiter.api.Assertions.assertEquals; 27 | 28 | public class DerandTest { 29 | @Test public void testTokenize() throws TranslateException, IOException { 30 | assertEquals("hello world", Derand.tokenize("hello 3y29842ysjhfs world")); 31 | assertEquals("", Derand.tokenize("")); 32 | assertEquals("", Derand.tokenize(null)); 33 | assertEquals("", Derand.tokenize("3y29842ysjhfs")); 34 | } 35 | 36 | 37 | @Test public void testClean() throws TranslateException, IOException { 38 | assertEquals("hello world", Derand.clean("hello 3y29842ysjhfs world")); 39 | assertEquals("", Derand.clean("")); 40 | assertEquals("", Derand.clean(null)); 41 | assertEquals("", Derand.clean("3y29842ysjhfs")); 42 | assertEquals("hello hello", Derand.clean("y837sc42zsd hello sdyd8f7h34 hello 3y29842ysjhfs")); 43 | } 44 | 45 | 46 | @Test public void testClassify() throws TranslateException, IOException { 47 | assertArrayEquals(new boolean[]{false, true, false}, Derand.classify("hello 3y29842ysjhfs world")); 48 | assertArrayEquals(new boolean[]{}, Derand.classify("")); 49 | assertArrayEquals(new boolean[]{}, Derand.classify(null)); 50 | assertArrayEquals(new boolean[]{true}, Derand.classify("3y29842ysjhfs")); 51 | assertArrayEquals(new boolean[]{true, false, true, false, true}, Derand.classify("y837sc42zsd hello sdyd8f7h34 hello 3y29842ysjhfs")); 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /derand_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/derand/c27aba560e1676206abd11680f63d4a0890105b7/derand_model.png -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | netflixossPublishCandidatesToMavenCentral=false 2 | 3 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/derand/c27aba560e1676206abd11680f63d4a0890105b7/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # 4 | # Copyright 2015 the original author or authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | ## 21 | ## Gradle start up script for UN*X 22 | ## 23 | ############################################################################## 24 | 25 | # Attempt to set APP_HOME 26 | # Resolve links: $0 may be a link 27 | PRG="$0" 28 | # Need this for relative symlinks. 29 | while [ -h "$PRG" ] ; do 30 | ls=`ls -ld "$PRG"` 31 | link=`expr "$ls" : '.*-> \(.*\)$'` 32 | if expr "$link" : '/.*' > /dev/null; then 33 | PRG="$link" 34 | else 35 | PRG=`dirname "$PRG"`"/$link" 36 | fi 37 | done 38 | SAVED="`pwd`" 39 | cd "`dirname \"$PRG\"`/" >/dev/null 40 | APP_HOME="`pwd -P`" 41 | cd "$SAVED" >/dev/null 42 | 43 | APP_NAME="Gradle" 44 | APP_BASE_NAME=`basename "$0"` 45 | 46 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 47 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 48 | 49 | # Use the maximum available, or set MAX_FD != -1 to use that value. 50 | MAX_FD="maximum" 51 | 52 | warn () { 53 | echo "$*" 54 | } 55 | 56 | die () { 57 | echo 58 | echo "$*" 59 | echo 60 | exit 1 61 | } 62 | 63 | # OS specific support (must be 'true' or 'false'). 64 | cygwin=false 65 | msys=false 66 | darwin=false 67 | nonstop=false 68 | case "`uname`" in 69 | CYGWIN* ) 70 | cygwin=true 71 | ;; 72 | Darwin* ) 73 | darwin=true 74 | ;; 75 | MINGW* ) 76 | msys=true 77 | ;; 78 | NONSTOP* ) 79 | nonstop=true 80 | ;; 81 | esac 82 | 83 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 84 | 85 | 86 | # Determine the Java command to use to start the JVM. 87 | if [ -n "$JAVA_HOME" ] ; then 88 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 89 | # IBM's JDK on AIX uses strange locations for the executables 90 | JAVACMD="$JAVA_HOME/jre/sh/java" 91 | else 92 | JAVACMD="$JAVA_HOME/bin/java" 93 | fi 94 | if [ ! -x "$JAVACMD" ] ; then 95 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 96 | 97 | Please set the JAVA_HOME variable in your environment to match the 98 | location of your Java installation." 99 | fi 100 | else 101 | JAVACMD="java" 102 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 103 | 104 | Please set the JAVA_HOME variable in your environment to match the 105 | location of your Java installation." 106 | fi 107 | 108 | # Increase the maximum file descriptors if we can. 109 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 110 | MAX_FD_LIMIT=`ulimit -H -n` 111 | if [ $? -eq 0 ] ; then 112 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 113 | MAX_FD="$MAX_FD_LIMIT" 114 | fi 115 | ulimit -n $MAX_FD 116 | if [ $? -ne 0 ] ; then 117 | warn "Could not set maximum file descriptor limit: $MAX_FD" 118 | fi 119 | else 120 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 121 | fi 122 | fi 123 | 124 | # For Darwin, add options to specify how the application appears in the dock 125 | if $darwin; then 126 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 127 | fi 128 | 129 | # For Cygwin or MSYS, switch paths to Windows format before running java 130 | if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then 131 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 132 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 133 | 134 | JAVACMD=`cygpath --unix "$JAVACMD"` 135 | 136 | # We build the pattern for arguments to be converted via cygpath 137 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 138 | SEP="" 139 | for dir in $ROOTDIRSRAW ; do 140 | ROOTDIRS="$ROOTDIRS$SEP$dir" 141 | SEP="|" 142 | done 143 | OURCYGPATTERN="(^($ROOTDIRS))" 144 | # Add a user-defined pattern to the cygpath arguments 145 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 146 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 147 | fi 148 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 149 | i=0 150 | for arg in "$@" ; do 151 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 152 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 153 | 154 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 155 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 156 | else 157 | eval `echo args$i`="\"$arg\"" 158 | fi 159 | i=`expr $i + 1` 160 | done 161 | case $i in 162 | 0) set -- ;; 163 | 1) set -- "$args0" ;; 164 | 2) set -- "$args0" "$args1" ;; 165 | 3) set -- "$args0" "$args1" "$args2" ;; 166 | 4) set -- "$args0" "$args1" "$args2" "$args3" ;; 167 | 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 168 | 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 169 | 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 170 | 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 171 | 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 172 | esac 173 | fi 174 | 175 | # Escape application args 176 | save () { 177 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 178 | echo " " 179 | } 180 | APP_ARGS=`save "$@"` 181 | 182 | # Collect all arguments for the java command, following the shell quoting and substitution rules 183 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 184 | 185 | exec "$JAVACMD" "$@" 186 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | * 4 | * The settings file is used to specify which projects to include in your build. 5 | * 6 | * Detailed information about configuring a multi-project build in Gradle can be found 7 | * in the user manual at https://docs.gradle.org/6.8.1/userguide/multi_project_builds.html 8 | */ 9 | plugins { 10 | id "com.gradle.enterprise" version "3.5.2" 11 | } 12 | 13 | rootProject.name = 'netflixoss-derand' 14 | include('derand') 15 | 16 | --------------------------------------------------------------------------------