├── .github ├── dependabot.yml └── workflows │ └── build.yml ├── .gitignore ├── .scalafmt.conf ├── LICENSE ├── README.md ├── build.sbt ├── catalog-info.yaml ├── modules ├── cli │ └── src │ │ ├── graal │ │ ├── jni-config.json │ │ ├── predefined-classes-config.json │ │ ├── proxy-config.json │ │ ├── reflect-config.json │ │ ├── resource-config.json │ │ └── serialization-config.json │ │ └── main │ │ └── scala │ │ └── tfr │ │ ├── Cli.scala │ │ └── instances │ │ ├── OutputInstances.scala │ │ └── output │ │ └── package.scala └── core │ └── src │ ├── main │ ├── protobuf │ │ ├── google │ │ │ └── protobuf │ │ │ │ └── wrappers.proto │ │ ├── tensorflow │ │ │ └── core │ │ │ │ ├── example │ │ │ │ ├── example.proto │ │ │ │ └── feature.proto │ │ │ │ ├── framework │ │ │ │ ├── resource_handle.proto │ │ │ │ ├── tensor.proto │ │ │ │ ├── tensor_shape.proto │ │ │ │ └── types.proto │ │ │ │ └── protobuf │ │ │ │ └── named_tensor.proto │ │ └── tensorflow_serving │ │ │ ├── apis │ │ │ ├── classification.proto │ │ │ ├── inference.proto │ │ │ ├── input.proto │ │ │ ├── logging.proto │ │ │ ├── model.proto │ │ │ ├── predict.proto │ │ │ ├── prediction_log.proto │ │ │ ├── regression.proto │ │ │ └── session_service.proto │ │ │ └── config │ │ │ ├── log_collector_config.proto │ │ │ └── logging_config.proto │ └── scala │ │ └── tfr │ │ ├── Parsable.scala │ │ ├── Resources.scala │ │ ├── TFRecord.scala │ │ └── instances │ │ ├── ExampleInstances.scala │ │ ├── PredictionInstances.scala │ │ ├── ProtobufInstances.scala │ │ ├── example │ │ └── package.scala │ │ ├── prediction │ │ └── package.scala │ │ └── protobuf │ │ └── package.scala │ └── test │ ├── resources │ ├── part-00000-of-00004.tfrecords │ └── prediction_log.tfrecords │ └── scala │ └── tfr │ └── TfrSuite.scala ├── project ├── Dependencies.scala ├── build.properties └── plugins.sbt └── shell.nix /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: github-actions 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "04:00" 8 | open-pull-requests-limit: 10 9 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | checks: 6 | name: Checks 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v4 10 | - uses: graalvm/setup-graalvm@v1 11 | with: 12 | distribution: "graalvm-community" 13 | version: "latest" 14 | java-version: "21" 15 | cache: "sbt" 16 | - run: sbt scalafmtCheckAll scalafmtSbtCheck 17 | build: 18 | name: Build 19 | strategy: 20 | matrix: 21 | os: ["ubuntu-latest", "macos-latest", "macos-latest-xlarge"] 22 | fail-fast: true 23 | runs-on: ${{ matrix.os }} 24 | steps: 25 | - uses: actions/checkout@v4 26 | - uses: graalvm/setup-graalvm@v1 27 | with: 28 | distribution: "graalvm-community" 29 | version: "latest" 30 | java-version: "21" 31 | cache: "sbt" 32 | - name: Install Sbt (New MacOs ARM64 Runner doesn't have it) 33 | if: ${{ matrix.os == 'macos-latest-xlarge' }} 34 | run: curl -fL https://github.com/VirtusLab/coursier-m1/releases/latest/download/cs-aarch64-apple-darwin.gz | gzip -d > cs && chmod +x cs && (xattr -d com.apple.quarantine cs || true) && ./cs setup -y && echo "~/Library/Application Support/Coursier/bin" >> $GITHUB_PATH 35 | - run: sbt test graalvm-native-image:packageBin 36 | - uses: actions/upload-artifact@v4 37 | with: 38 | name: tfr-${{ runner.os }}-${{ runner.arch }} 39 | path: modules/cli/target/graalvm-native-image/tfr 40 | retention-days: 1 41 | release: 42 | name: Release 43 | runs-on: ubuntu-latest 44 | needs: [build, checks] 45 | if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') 46 | steps: 47 | - id: get_version 48 | run: | 49 | echo ${GITHUB_REF/refs\/tags\//} 50 | echo ::set-output name=version::${GITHUB_REF/refs\/tags\//} 51 | - uses: actions/download-artifact@v4 52 | - run: zip -j tfr-Linux-X64-${{ steps.get_version.outputs.version }}.zip tfr-Linux-X64/tfr 53 | - run: zip -j tfr-macOS-X64-${{ steps.get_version.outputs.version }}.zip tfr-macOS-X64/tfr 54 | - run: zip -j tfr-macOS-ARM64-${{ steps.get_version.outputs.version }}.zip tfr-macOS-ARM64/tfr 55 | - id: upload-release-assets 56 | uses: softprops/action-gh-release@v2 57 | with: 58 | files: | 59 | tfr-Linux-X64-${{ steps.get_version.outputs.version }}.zip 60 | tfr-macOS-X64-${{ steps.get_version.outputs.version }}.zip 61 | tfr-macOS-ARM64-${{ steps.get_version.outputs.version }}.zip 62 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | .DS_Store 3 | -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | version = "3.8.1" 2 | runner.dialect = scala3 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tfreader [![build](https://github.com/spotify/tfreader/actions/workflows/build.yml/badge.svg)](https://github.com/spotify/tfreader/actions/workflows/build.yml) 2 | 3 | Simple native CLI tool to read `TensorFlow` `TFRecords`. 4 | 5 | ## Install 6 | 7 | ### MacOs 8 | 9 | ```bash 10 | brew tap spotify/public 11 | brew install tfreader 12 | ``` 13 | 14 | ### Linux 15 | 16 | Right now we only have binaries available under [releases](https://github.com/spotify/tfreader/releases) 17 | 18 | ## Usage 19 | 20 | ```bash 21 | Usage: tfr [options] 22 | TensorFlow TFRecord reader CLI tool 23 | Options: 24 | 25 | -c, --check-crc32 Enable checks CRC32 on each record 26 | -f, --flat Output examples as flat JSON objects 27 | -n, --number Number of records to output 28 | -r, --record Record type to be read { example | prediction_log } 29 | -h, --help Show help message 30 | 31 | trailing arguments: 32 | files (not required) files? | STDIN 33 | ``` 34 | 35 | ## Examples 36 | 37 | #### Google Cloud Storage 38 | 39 | ```bash 40 | tfr -n 1 gs:////part-00000-of-00004.tfrecords | jq . 41 | ``` 42 | 43 | #### Local Filesystem 44 | 45 | ```bash 46 | tfr -n 1 core/src/test/resources/part-00000-of-00004.tfrecords | jq . 47 | ``` 48 | 49 | #### `stdin` 50 | 51 | ```bash 52 | cat core/src/test/resources/part-00000-of-00004.tfrecords | tfr -n 1 | jq . 53 | ``` 54 | 55 | ### Output 56 | 57 | #### Flat 58 | 59 | ```json 60 | { 61 | "tips": [0], 62 | "trip_seconds": [60], 63 | "payment_type": ["Cash"], 64 | "trip_miles": [0], 65 | "dropoff_longitude": [-87.63785], 66 | "dropoff_latitude": [41.893215], 67 | "pickup_longitude": [-87.63187], 68 | "pickup_latitude": [41.89204], 69 | "trip_start_timestamp": [1402934400], 70 | "trip_start_day": [2], 71 | "trip_start_hour": [16], 72 | "trip_start_month": [6], 73 | "fare": [3.25], 74 | "dropoff_census_tract": ["17031081800"], 75 | "dropoff_community_area": ["8"], 76 | "pickup_community_area": ["8"], 77 | "trip_id": ["8106c1f6-e6f3-426f-9aaf-b4e9703b4f10"] 78 | } 79 | ``` 80 | 81 | #### Default 82 | 83 | ```json 84 | { 85 | "features": { 86 | "feature": { 87 | "tips": { 88 | "floatList": { 89 | "value": [0] 90 | } 91 | }, 92 | "trip_seconds": { 93 | "int64List": { 94 | "value": [60] 95 | } 96 | }, 97 | "payment_type": { 98 | "bytesList": { 99 | "value": ["Cash"] 100 | } 101 | }, 102 | "trip_miles": { 103 | "floatList": { 104 | "value": [0] 105 | } 106 | }, 107 | "dropoff_longitude": { 108 | "floatList": { 109 | "value": [-87.63785] 110 | } 111 | }, 112 | "dropoff_latitude": { 113 | "floatList": { 114 | "value": [41.893215] 115 | } 116 | }, 117 | "pickup_longitude": { 118 | "floatList": { 119 | "value": [-87.63187] 120 | } 121 | }, 122 | "pickup_latitude": { 123 | "floatList": { 124 | "value": [41.89204] 125 | } 126 | }, 127 | "trip_start_timestamp": { 128 | "int64List": { 129 | "value": [1402934400] 130 | } 131 | }, 132 | "trip_start_day": { 133 | "int64List": { 134 | "value": [2] 135 | } 136 | }, 137 | "trip_start_hour": { 138 | "int64List": { 139 | "value": [16] 140 | } 141 | }, 142 | "trip_start_month": { 143 | "int64List": { 144 | "value": [6] 145 | } 146 | }, 147 | "fare": { 148 | "floatList": { 149 | "value": [3.25] 150 | } 151 | }, 152 | "dropoff_census_tract": { 153 | "bytesList": { 154 | "value": ["17031081800"] 155 | } 156 | }, 157 | "dropoff_community_area": { 158 | "bytesList": { 159 | "value": ["8"] 160 | } 161 | }, 162 | "pickup_community_area": { 163 | "bytesList": { 164 | "value": ["8"] 165 | } 166 | }, 167 | "trip_id": { 168 | "bytesList": { 169 | "value": ["8106c1f6-e6f3-426f-9aaf-b4e9703b4f10"] 170 | } 171 | } 172 | } 173 | } 174 | } 175 | ``` 176 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Spotify AB. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an 12 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | * KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations 15 | * under the License. 16 | */ 17 | import Dependencies._ 18 | 19 | ThisBuild / scalaVersion := "3.4.2" 20 | ThisBuild / scalacOptions ++= Seq( 21 | "-language:implicitConversions", 22 | "-Ykind-projector" 23 | ) 24 | ThisBuild / organization := "com.spotify" 25 | ThisBuild / organizationName := "spotify" 26 | ThisBuild / licenses := Seq( 27 | "APL2" -> url("http://www.apache.org/licenses/LICENSE-2.0.txt") 28 | ) 29 | ThisBuild / homepage := Some(url("https://github.com/spotify/tfreader")) 30 | ThisBuild / scmInfo := Some( 31 | ScmInfo( 32 | url("https://github.com/spotify/tfreader"), 33 | "scm:git@github.com:spotify/tfr.git" 34 | ) 35 | ) 36 | ThisBuild / developers := List( 37 | Developer( 38 | id = "regadas", 39 | name = "Filipe Regadas", 40 | email = "filiperegadas@gmail.com", 41 | url = url("https://github.com/regadas") 42 | ) 43 | ) 44 | 45 | lazy val noPublishSettings = Def.settings( 46 | publish / skip := true, 47 | Compile / packageDoc / publishArtifact := false 48 | ) 49 | 50 | lazy val tfr = project 51 | .in(file(".")) 52 | .settings(noPublishSettings) 53 | .aggregate(core, cli) 54 | 55 | lazy val core = project 56 | .in(file("modules/core")) 57 | .settings(noPublishSettings) 58 | .settings( 59 | name := "tfr-core", 60 | compileOrder := CompileOrder.JavaThenScala, 61 | libraryDependencies ++= Seq( 62 | gcs, 63 | guava, 64 | munit % Test, 65 | catsCore, 66 | fs2Io, 67 | circeCore, 68 | circeParser 69 | ), 70 | Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat, 71 | testFrameworks += new TestFramework("munit.Framework"), 72 | ProtobufConfig / version := protobufVersion 73 | ) 74 | .enablePlugins(ProtobufPlugin) 75 | 76 | lazy val cli = project 77 | .in(file("modules/cli")) 78 | .settings(noPublishSettings) 79 | .settings( 80 | name := "tfr-cli", 81 | buildInfoKeys := Seq[BuildInfoKey](version), 82 | buildInfoPackage := "tfr", 83 | libraryDependencies ++= Seq( 84 | catsCore, 85 | fs2Io, 86 | scallop 87 | ), 88 | GraalVMNativeImage / name := "tfr", 89 | graalVMNativeImageOptions ++= Seq( 90 | "-H:+UnlockExperimentalVMOptions", 91 | "-H:+StrictImageHeap", 92 | "-H:-CheckToolchain", 93 | "-H:+ReportExceptionStackTraces", 94 | "-H:EnableURLProtocols=http,https", 95 | "-H:ReflectionConfigurationFiles=" + baseDirectory.value / "src" / "graal" / "reflect-config.json", 96 | "--no-fallback", 97 | "-march=native" 98 | ), 99 | assembly / assemblyMergeStrategy := { 100 | case PathList("module-info.class") => MergeStrategy.rename 101 | case s if s.endsWith(".class") => MergeStrategy.last 102 | case x => 103 | val oldStrategy = (assembly / assemblyMergeStrategy).value 104 | oldStrategy(x) 105 | } 106 | ) 107 | .dependsOn(core) 108 | .enablePlugins(GraalVMNativeImagePlugin) 109 | .enablePlugins(BuildInfoPlugin) 110 | -------------------------------------------------------------------------------- /catalog-info.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: backstage.io/v1alpha1 2 | kind: Resource 3 | metadata: 4 | name: tfreader 5 | spec: 6 | type: resource 7 | owner: flatmap 8 | -------------------------------------------------------------------------------- /modules/cli/src/graal/jni-config.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name":"[Lcom.sun.management.internal.DiagnosticCommandArgumentInfo;" 4 | }, 5 | { 6 | "name":"[Lcom.sun.management.internal.DiagnosticCommandInfo;" 7 | }, 8 | { 9 | "name":"com.sun.management.internal.DiagnosticCommandArgumentInfo", 10 | "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","boolean","boolean","int"] }] 11 | }, 12 | { 13 | "name":"com.sun.management.internal.DiagnosticCommandInfo", 14 | "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","java.util.List"] }] 15 | }, 16 | { 17 | "name":"java.lang.Boolean", 18 | "methods":[{"name":"getBoolean","parameterTypes":["java.lang.String"] }] 19 | }, 20 | { 21 | "name":"java.lang.ClassLoader", 22 | "methods":[{"name":"getPlatformClassLoader","parameterTypes":[] }, {"name":"loadClass","parameterTypes":["java.lang.String"] }] 23 | }, 24 | { 25 | "name":"java.lang.String", 26 | "methods":[{"name":"lastIndexOf","parameterTypes":["int"] }, {"name":"substring","parameterTypes":["int"] }] 27 | }, 28 | { 29 | "name":"java.lang.System", 30 | "methods":[{"name":"getProperty","parameterTypes":["java.lang.String"] }, {"name":"setProperty","parameterTypes":["java.lang.String","java.lang.String"] }] 31 | }, 32 | { 33 | "name":"java.util.Arrays", 34 | "methods":[{"name":"asList","parameterTypes":["java.lang.Object[]"] }] 35 | }, 36 | { 37 | "name":"jdk.internal.loader.ClassLoaders$PlatformClassLoader" 38 | }, 39 | { 40 | "name":"org.graalvm.jniutils.JNIExceptionWrapperEntryPoints", 41 | "methods":[{"name":"getClassName","parameterTypes":["java.lang.Class"] }] 42 | }, 43 | { 44 | "name":"sun.management.VMManagementImpl", 45 | "fields":[{"name":"compTimeMonitoringSupport"}, {"name":"currentThreadCpuTimeSupport"}, {"name":"objectMonitorUsageSupport"}, {"name":"otherThreadCpuTimeSupport"}, {"name":"remoteDiagnosticCommandsSupport"}, {"name":"synchronizerUsageSupport"}, {"name":"threadAllocatedMemorySupport"}, {"name":"threadContentionMonitoringSupport"}] 46 | }, 47 | { 48 | "name":"tfr.Cli", 49 | "methods":[{"name":"main","parameterTypes":["java.lang.String[]"] }] 50 | } 51 | ] 52 | -------------------------------------------------------------------------------- /modules/cli/src/graal/predefined-classes-config.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "type":"agent-extracted", 4 | "classes":[ 5 | ] 6 | } 7 | ] 8 | 9 | -------------------------------------------------------------------------------- /modules/cli/src/graal/proxy-config.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "interfaces":["sun.misc.SignalHandler"] 4 | } 5 | ] 6 | -------------------------------------------------------------------------------- /modules/cli/src/graal/reflect-config.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "java.util.concurrent.ConcurrentSkipListMap$Values", 4 | "allDeclaredConstructors": true, 5 | "allPublicConstructors": true, 6 | "allDeclaredMethods": true, 7 | "allPublicMethods": true 8 | }, 9 | { 10 | "name":"[B" 11 | }, 12 | { 13 | "name":"[C" 14 | }, 15 | { 16 | "name":"[D" 17 | }, 18 | { 19 | "name":"[F" 20 | }, 21 | { 22 | "name":"[I" 23 | }, 24 | { 25 | "name":"[J" 26 | }, 27 | { 28 | "name":"[Ljava.lang.StackTraceElement;" 29 | }, 30 | { 31 | "name":"[Ljava.lang.String;" 32 | }, 33 | { 34 | "name":"[Ljava.lang.reflect.Method;" 35 | }, 36 | { 37 | "name":"[Ljavax.management.openmbean.CompositeData;" 38 | }, 39 | { 40 | "name":"[Lscala.Tuple2;" 41 | }, 42 | { 43 | "name":"[Lsun.security.pkcs.SignerInfo;" 44 | }, 45 | { 46 | "name":"[S" 47 | }, 48 | { 49 | "name":"[Z" 50 | }, 51 | { 52 | "name":"apple.security.AppleProvider", 53 | "methods":[{"name":"","parameterTypes":[] }] 54 | }, 55 | { 56 | "name":"cats.Later", 57 | "fields":[{"name":"0bitmap$1"}, {"name":"value$lzy1"}] 58 | }, 59 | { 60 | "name":"cats.effect.metrics.CpuStarvation", 61 | "queryAllPublicConstructors":true 62 | }, 63 | { 64 | "name":"cats.effect.metrics.CpuStarvationMBean", 65 | "queryAllPublicMethods":true 66 | }, 67 | { 68 | "name":"cats.effect.std.Console$", 69 | "fields":[{"name":"0bitmap$1"}] 70 | }, 71 | { 72 | "name":"cats.effect.std.Semaphore$impl", 73 | "fields":[{"name":"0bitmap$1"}] 74 | }, 75 | { 76 | "name":"cats.effect.unsafe.Head", 77 | "fields":[{"name":"head"}] 78 | }, 79 | { 80 | "name":"cats.effect.unsafe.IORuntimeCompanionPlatform", 81 | "fields":[{"name":"0bitmap$1"}] 82 | }, 83 | { 84 | "name":"cats.effect.unsafe.Tail", 85 | "fields":[{"name":"tailPublisher"}] 86 | }, 87 | { 88 | "name":"cats.effect.unsafe.TimerSkipListNodeBase", 89 | "fields":[{"name":"callback"}] 90 | }, 91 | { 92 | "name":"cats.effect.unsafe.WorkStealingThreadPool", 93 | "fields":[{"name":"0bitmap$1"}] 94 | }, 95 | { 96 | "name":"cats.effect.unsafe.metrics.ComputePoolSampler", 97 | "queryAllPublicConstructors":true 98 | }, 99 | { 100 | "name":"cats.effect.unsafe.metrics.ComputePoolSamplerMBean", 101 | "queryAllPublicMethods":true 102 | }, 103 | { 104 | "name":"cats.effect.unsafe.metrics.LiveFiberSnapshotTrigger", 105 | "queryAllPublicConstructors":true 106 | }, 107 | { 108 | "name":"cats.effect.unsafe.metrics.LiveFiberSnapshotTriggerMBean", 109 | "queryAllPublicMethods":true 110 | }, 111 | { 112 | "name":"cats.effect.unsafe.metrics.LocalQueueSampler", 113 | "queryAllPublicConstructors":true 114 | }, 115 | { 116 | "name":"cats.effect.unsafe.metrics.LocalQueueSamplerMBean", 117 | "queryAllPublicMethods":true 118 | }, 119 | { 120 | "name":"com.google.api.client.googleapis.services.AbstractGoogleClientRequest", 121 | "allDeclaredFields":true 122 | }, 123 | { 124 | "name":"com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest", 125 | "allDeclaredFields":true 126 | }, 127 | { 128 | "name":"com.google.api.client.http.GenericUrl", 129 | "allDeclaredFields":true 130 | }, 131 | { 132 | "name":"com.google.api.client.http.HttpHeaders", 133 | "allDeclaredFields":true, 134 | "queryAllDeclaredMethods":true, 135 | "methods":[{"name":"setAccept","parameterTypes":["java.lang.String"] }, {"name":"setAcceptEncoding","parameterTypes":["java.lang.String"] }, {"name":"setAge","parameterTypes":["java.lang.Long"] }, {"name":"setAuthenticate","parameterTypes":["java.lang.String"] }, {"name":"setAuthorization","parameterTypes":["java.lang.String"] }, {"name":"setAuthorization","parameterTypes":["java.util.List"] }, {"name":"setCacheControl","parameterTypes":["java.lang.String"] }, {"name":"setContentEncoding","parameterTypes":["java.lang.String"] }, {"name":"setContentLength","parameterTypes":["java.lang.Long"] }, {"name":"setContentMD5","parameterTypes":["java.lang.String"] }, {"name":"setContentRange","parameterTypes":["java.lang.String"] }, {"name":"setContentType","parameterTypes":["java.lang.String"] }, {"name":"setCookie","parameterTypes":["java.lang.String"] }, {"name":"setDate","parameterTypes":["java.lang.String"] }, {"name":"setETag","parameterTypes":["java.lang.String"] }, {"name":"setExpires","parameterTypes":["java.lang.String"] }, {"name":"setIfMatch","parameterTypes":["java.lang.String"] }, {"name":"setIfModifiedSince","parameterTypes":["java.lang.String"] }, {"name":"setIfNoneMatch","parameterTypes":["java.lang.String"] }, {"name":"setIfRange","parameterTypes":["java.lang.String"] }, {"name":"setIfUnmodifiedSince","parameterTypes":["java.lang.String"] }, {"name":"setLastModified","parameterTypes":["java.lang.String"] }, {"name":"setLocation","parameterTypes":["java.lang.String"] }, {"name":"setMimeVersion","parameterTypes":["java.lang.String"] }, {"name":"setRange","parameterTypes":["java.lang.String"] }, {"name":"setRetryAfter","parameterTypes":["java.lang.String"] }, {"name":"setUserAgent","parameterTypes":["java.lang.String"] }] 136 | }, 137 | { 138 | "name":"com.google.api.client.json.GenericJson", 139 | "allDeclaredFields":true, 140 | "methods":[{"name":"","parameterTypes":[] }] 141 | }, 142 | { 143 | "name":"com.google.api.client.util.GenericData", 144 | "allDeclaredFields":true, 145 | "methods":[{"name":"","parameterTypes":[] }] 146 | }, 147 | { 148 | "name":"com.google.api.services.storage.Storage$Objects$Get", 149 | "allDeclaredFields":true, 150 | "queryAllDeclaredMethods":true 151 | }, 152 | { 153 | "name":"com.google.api.services.storage.StorageRequest", 154 | "allDeclaredFields":true, 155 | "queryAllDeclaredMethods":true, 156 | "methods":[{"name":"setAlt","parameterTypes":["java.lang.String"] }] 157 | }, 158 | { 159 | "name":"com.google.api.services.storage.model.StorageObject", 160 | "allDeclaredFields":true, 161 | "queryAllDeclaredMethods":true 162 | }, 163 | { 164 | "name":"com.google.common.util.concurrent.AbstractFuture", 165 | "fields":[{"name":"listeners"}, {"name":"value"}, {"name":"waiters"}] 166 | }, 167 | { 168 | "name":"com.google.common.util.concurrent.AbstractFuture$Waiter", 169 | "fields":[{"name":"next"}, {"name":"thread"}] 170 | }, 171 | { 172 | "name":"com.google.protobuf.DescriptorProtos$FeatureSet", 173 | "methods":[{"name":"getEnumType","parameterTypes":[] }, {"name":"getFieldPresence","parameterTypes":[] }, {"name":"getJsonFormat","parameterTypes":[] }, {"name":"getMessageEncoding","parameterTypes":[] }, {"name":"getRepeatedFieldEncoding","parameterTypes":[] }, {"name":"getUtf8Validation","parameterTypes":[] }, {"name":"hasEnumType","parameterTypes":[] }, {"name":"hasFieldPresence","parameterTypes":[] }, {"name":"hasJsonFormat","parameterTypes":[] }, {"name":"hasMessageEncoding","parameterTypes":[] }, {"name":"hasRepeatedFieldEncoding","parameterTypes":[] }, {"name":"hasUtf8Validation","parameterTypes":[] }] 174 | }, 175 | { 176 | "name":"com.google.protobuf.DescriptorProtos$FeatureSet$Builder", 177 | "methods":[{"name":"clearEnumType","parameterTypes":[] }, {"name":"clearFieldPresence","parameterTypes":[] }, {"name":"clearJsonFormat","parameterTypes":[] }, {"name":"clearMessageEncoding","parameterTypes":[] }, {"name":"clearRepeatedFieldEncoding","parameterTypes":[] }, {"name":"clearUtf8Validation","parameterTypes":[] }, {"name":"getEnumType","parameterTypes":[] }, {"name":"getFieldPresence","parameterTypes":[] }, {"name":"getJsonFormat","parameterTypes":[] }, {"name":"getMessageEncoding","parameterTypes":[] }, {"name":"getRepeatedFieldEncoding","parameterTypes":[] }, {"name":"getUtf8Validation","parameterTypes":[] }, {"name":"hasEnumType","parameterTypes":[] }, {"name":"hasFieldPresence","parameterTypes":[] }, {"name":"hasJsonFormat","parameterTypes":[] }, {"name":"hasMessageEncoding","parameterTypes":[] }, {"name":"hasRepeatedFieldEncoding","parameterTypes":[] }, {"name":"hasUtf8Validation","parameterTypes":[] }, {"name":"setEnumType","parameterTypes":["com.google.protobuf.DescriptorProtos$FeatureSet$EnumType"] }, {"name":"setFieldPresence","parameterTypes":["com.google.protobuf.DescriptorProtos$FeatureSet$FieldPresence"] }, {"name":"setJsonFormat","parameterTypes":["com.google.protobuf.DescriptorProtos$FeatureSet$JsonFormat"] }, {"name":"setMessageEncoding","parameterTypes":["com.google.protobuf.DescriptorProtos$FeatureSet$MessageEncoding"] }, {"name":"setRepeatedFieldEncoding","parameterTypes":["com.google.protobuf.DescriptorProtos$FeatureSet$RepeatedFieldEncoding"] }, {"name":"setUtf8Validation","parameterTypes":["com.google.protobuf.DescriptorProtos$FeatureSet$Utf8Validation"] }] 178 | }, 179 | { 180 | "name":"com.google.protobuf.DescriptorProtos$FeatureSet$EnumType", 181 | "methods":[{"name":"getValueDescriptor","parameterTypes":[] }, {"name":"valueOf","parameterTypes":["com.google.protobuf.Descriptors$EnumValueDescriptor"] }] 182 | }, 183 | { 184 | "name":"com.google.protobuf.DescriptorProtos$FeatureSet$FieldPresence", 185 | "methods":[{"name":"getValueDescriptor","parameterTypes":[] }, {"name":"valueOf","parameterTypes":["com.google.protobuf.Descriptors$EnumValueDescriptor"] }] 186 | }, 187 | { 188 | "name":"com.google.protobuf.DescriptorProtos$FeatureSet$JsonFormat", 189 | "methods":[{"name":"getValueDescriptor","parameterTypes":[] }, {"name":"valueOf","parameterTypes":["com.google.protobuf.Descriptors$EnumValueDescriptor"] }] 190 | }, 191 | { 192 | "name":"com.google.protobuf.DescriptorProtos$FeatureSet$MessageEncoding", 193 | "methods":[{"name":"getValueDescriptor","parameterTypes":[] }, {"name":"valueOf","parameterTypes":["com.google.protobuf.Descriptors$EnumValueDescriptor"] }] 194 | }, 195 | { 196 | "name":"com.google.protobuf.DescriptorProtos$FeatureSet$RepeatedFieldEncoding", 197 | "methods":[{"name":"getValueDescriptor","parameterTypes":[] }, {"name":"valueOf","parameterTypes":["com.google.protobuf.Descriptors$EnumValueDescriptor"] }] 198 | }, 199 | { 200 | "name":"com.google.protobuf.DescriptorProtos$FeatureSet$Utf8Validation", 201 | "methods":[{"name":"getValueDescriptor","parameterTypes":[] }, {"name":"valueOf","parameterTypes":["com.google.protobuf.Descriptors$EnumValueDescriptor"] }] 202 | }, 203 | { 204 | "name":"com.google.protobuf.ExtensionRegistry", 205 | "methods":[{"name":"getEmptyRegistry","parameterTypes":[] }] 206 | }, 207 | { 208 | "name":"com.google.protobuf.Int64Value", 209 | "methods":[{"name":"getValue","parameterTypes":[] }, {"name":"newBuilder","parameterTypes":[] }] 210 | }, 211 | { 212 | "name":"com.google.protobuf.Int64Value$Builder", 213 | "methods":[{"name":"clearValue","parameterTypes":[] }, {"name":"getValue","parameterTypes":[] }, {"name":"setValue","parameterTypes":["long"] }] 214 | }, 215 | { 216 | "name":"com.sun.crypto.provider.AESCipher$General", 217 | "methods":[{"name":"","parameterTypes":[] }] 218 | }, 219 | { 220 | "name":"com.sun.crypto.provider.ARCFOURCipher", 221 | "methods":[{"name":"","parameterTypes":[] }] 222 | }, 223 | { 224 | "name":"com.sun.crypto.provider.ChaCha20Cipher$ChaCha20Poly1305", 225 | "methods":[{"name":"","parameterTypes":[] }] 226 | }, 227 | { 228 | "name":"com.sun.crypto.provider.DESCipher", 229 | "methods":[{"name":"","parameterTypes":[] }] 230 | }, 231 | { 232 | "name":"com.sun.crypto.provider.DESedeCipher", 233 | "methods":[{"name":"","parameterTypes":[] }] 234 | }, 235 | { 236 | "name":"com.sun.crypto.provider.DHParameters", 237 | "methods":[{"name":"","parameterTypes":[] }] 238 | }, 239 | { 240 | "name":"com.sun.crypto.provider.GaloisCounterMode$AESGCM", 241 | "methods":[{"name":"","parameterTypes":[] }] 242 | }, 243 | { 244 | "name":"com.sun.crypto.provider.HmacCore$HmacSHA384", 245 | "methods":[{"name":"","parameterTypes":[] }] 246 | }, 247 | { 248 | "name":"com.sun.crypto.provider.TlsMasterSecretGenerator", 249 | "methods":[{"name":"","parameterTypes":[] }] 250 | }, 251 | { 252 | "name":"com.sun.management.GarbageCollectorMXBean", 253 | "queryAllPublicMethods":true 254 | }, 255 | { 256 | "name":"com.sun.management.GcInfo", 257 | "queryAllPublicMethods":true 258 | }, 259 | { 260 | "name":"com.sun.management.HotSpotDiagnosticMXBean", 261 | "queryAllPublicMethods":true 262 | }, 263 | { 264 | "name":"com.sun.management.ThreadMXBean", 265 | "queryAllPublicMethods":true 266 | }, 267 | { 268 | "name":"com.sun.management.UnixOperatingSystemMXBean", 269 | "queryAllPublicMethods":true 270 | }, 271 | { 272 | "name":"com.sun.management.VMOption", 273 | "queryAllPublicMethods":true 274 | }, 275 | { 276 | "name":"com.sun.management.internal.GarbageCollectorExtImpl", 277 | "queryAllPublicConstructors":true 278 | }, 279 | { 280 | "name":"com.sun.management.internal.HotSpotDiagnostic", 281 | "queryAllPublicConstructors":true 282 | }, 283 | { 284 | "name":"com.sun.management.internal.HotSpotThreadImpl", 285 | "queryAllPublicConstructors":true 286 | }, 287 | { 288 | "name":"com.sun.management.internal.OperatingSystemImpl", 289 | "queryAllPublicConstructors":true 290 | }, 291 | { 292 | "name":"fs2.Chunk$", 293 | "fields":[{"name":"0bitmap$2"}, {"name":"IArraySlice$lzy1"}] 294 | }, 295 | { 296 | "name":"fs2.internal.ScopedResource$$anon$1", 297 | "fields":[{"name":"0bitmap$1"}, {"name":"TheLease$lzy1"}] 298 | }, 299 | { 300 | "name":"io.circe.Encoder$", 301 | "fields":[{"name":"0bitmap$1"}] 302 | }, 303 | { 304 | "name":"io.circe.JsonDecimal", 305 | "fields":[{"name":"0bitmap$1"}] 306 | }, 307 | { 308 | "name":"io.grpc.override.ContextStorageOverride" 309 | }, 310 | { 311 | "name":"io.opencensus.impl.trace.TraceComponentImpl" 312 | }, 313 | { 314 | "name":"io.opencensus.impllite.trace.TraceComponentImplLite" 315 | }, 316 | { 317 | "name":"io.opentelemetry.opencensusshim.OpenTelemetryContextManager" 318 | }, 319 | { 320 | "name":"io.opentelemetry.opencensusshim.OpenTelemetryTraceComponentImpl" 321 | }, 322 | { 323 | "name":"java.lang.Boolean", 324 | "fields":[{"name":"TYPE"}] 325 | }, 326 | { 327 | "name":"java.lang.Byte", 328 | "fields":[{"name":"TYPE"}] 329 | }, 330 | { 331 | "name":"java.lang.Character", 332 | "fields":[{"name":"TYPE"}] 333 | }, 334 | { 335 | "name":"java.lang.ClassValue" 336 | }, 337 | { 338 | "name":"java.lang.Deprecated", 339 | "queryAllPublicMethods":true 340 | }, 341 | { 342 | "name":"java.lang.Double", 343 | "fields":[{"name":"TYPE"}] 344 | }, 345 | { 346 | "name":"java.lang.Float", 347 | "fields":[{"name":"TYPE"}] 348 | }, 349 | { 350 | "name":"java.lang.Integer", 351 | "fields":[{"name":"TYPE"}] 352 | }, 353 | { 354 | "name":"java.lang.Long", 355 | "fields":[{"name":"TYPE"}] 356 | }, 357 | { 358 | "name":"java.lang.Object", 359 | "allDeclaredFields":true 360 | }, 361 | { 362 | "name":"java.lang.Short", 363 | "fields":[{"name":"TYPE"}] 364 | }, 365 | { 366 | "name":"java.lang.StackTraceElement", 367 | "queryAllPublicMethods":true 368 | }, 369 | { 370 | "name":"java.lang.String", 371 | "fields":[{"name":"TYPE"}] 372 | }, 373 | { 374 | "name":"java.lang.Thread", 375 | "fields":[{"name":"threadLocalRandomProbe"}] 376 | }, 377 | { 378 | "name":"java.lang.Throwable", 379 | "methods":[{"name":"addSuppressed","parameterTypes":["java.lang.Throwable"] }] 380 | }, 381 | { 382 | "name":"java.lang.Void", 383 | "fields":[{"name":"TYPE"}] 384 | }, 385 | { 386 | "name":"java.lang.invoke.VarHandle", 387 | "methods":[{"name":"releaseFence","parameterTypes":[] }] 388 | }, 389 | { 390 | "name":"java.lang.management.BufferPoolMXBean", 391 | "queryAllPublicMethods":true 392 | }, 393 | { 394 | "name":"java.lang.management.ClassLoadingMXBean", 395 | "queryAllPublicMethods":true 396 | }, 397 | { 398 | "name":"java.lang.management.CompilationMXBean", 399 | "queryAllPublicMethods":true 400 | }, 401 | { 402 | "name":"java.lang.management.LockInfo", 403 | "queryAllPublicMethods":true 404 | }, 405 | { 406 | "name":"java.lang.management.ManagementPermission", 407 | "methods":[{"name":"","parameterTypes":["java.lang.String"] }] 408 | }, 409 | { 410 | "name":"java.lang.management.MemoryMXBean", 411 | "queryAllPublicMethods":true 412 | }, 413 | { 414 | "name":"java.lang.management.MemoryManagerMXBean", 415 | "queryAllPublicMethods":true 416 | }, 417 | { 418 | "name":"java.lang.management.MemoryPoolMXBean", 419 | "queryAllPublicMethods":true 420 | }, 421 | { 422 | "name":"java.lang.management.MemoryUsage", 423 | "queryAllPublicMethods":true 424 | }, 425 | { 426 | "name":"java.lang.management.MonitorInfo", 427 | "queryAllPublicMethods":true 428 | }, 429 | { 430 | "name":"java.lang.management.PlatformLoggingMXBean", 431 | "queryAllPublicMethods":true, 432 | "methods":[{"name":"getLoggerLevel","parameterTypes":["java.lang.String"] }, {"name":"getLoggerNames","parameterTypes":[] }, {"name":"getParentLoggerName","parameterTypes":["java.lang.String"] }, {"name":"setLoggerLevel","parameterTypes":["java.lang.String","java.lang.String"] }] 433 | }, 434 | { 435 | "name":"java.lang.management.RuntimeMXBean", 436 | "queryAllPublicMethods":true 437 | }, 438 | { 439 | "name":"java.lang.management.ThreadInfo", 440 | "queryAllPublicMethods":true 441 | }, 442 | { 443 | "name":"java.math.BigDecimal" 444 | }, 445 | { 446 | "name":"java.math.BigInteger" 447 | }, 448 | { 449 | "name":"java.nio.Buffer", 450 | "fields":[{"name":"address"}] 451 | }, 452 | { 453 | "name":"java.security.AlgorithmParametersSpi" 454 | }, 455 | { 456 | "name":"java.security.KeyStoreSpi" 457 | }, 458 | { 459 | "name":"java.security.SecureRandomParameters" 460 | }, 461 | { 462 | "name":"java.security.interfaces.ECPrivateKey" 463 | }, 464 | { 465 | "name":"java.security.interfaces.ECPublicKey" 466 | }, 467 | { 468 | "name":"java.security.interfaces.RSAPrivateKey" 469 | }, 470 | { 471 | "name":"java.security.interfaces.RSAPublicKey" 472 | }, 473 | { 474 | "name":"java.sql.Date" 475 | }, 476 | { 477 | "name":"java.util.AbstractMap", 478 | "allDeclaredFields":true 479 | }, 480 | { 481 | "name":"java.util.Date" 482 | }, 483 | { 484 | "name":"java.util.PropertyPermission", 485 | "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String"] }] 486 | }, 487 | { 488 | "name":"java.util.concurrent.ConcurrentSkipListMap$Values", 489 | "allDeclaredMethods":true, 490 | "allPublicMethods":true, 491 | "allDeclaredConstructors":true, 492 | "allPublicConstructors":true 493 | }, 494 | { 495 | "name":"java.util.concurrent.atomic.AtomicBoolean", 496 | "fields":[{"name":"value"}] 497 | }, 498 | { 499 | "name":"java.util.concurrent.atomic.AtomicReference", 500 | "fields":[{"name":"value"}] 501 | }, 502 | { 503 | "name":"java.util.concurrent.atomic.Striped64", 504 | "fields":[{"name":"base"}, {"name":"cellsBusy"}] 505 | }, 506 | { 507 | "name":"java.util.concurrent.atomic.Striped64$Cell", 508 | "fields":[{"name":"value"}] 509 | }, 510 | { 511 | "name":"java.util.logging.LogManager", 512 | "methods":[{"name":"getLoggingMXBean","parameterTypes":[] }] 513 | }, 514 | { 515 | "name":"java.util.logging.LoggingMXBean", 516 | "queryAllPublicMethods":true 517 | }, 518 | { 519 | "name":"java.util.zip.CRC32C", 520 | "methods":[{"name":"","parameterTypes":[] }] 521 | }, 522 | { 523 | "name":"javax.management.MBeanOperationInfo", 524 | "queryAllPublicMethods":true, 525 | "methods":[{"name":"getSignature","parameterTypes":[] }] 526 | }, 527 | { 528 | "name":"javax.management.MBeanServerBuilder", 529 | "methods":[{"name":"","parameterTypes":[] }] 530 | }, 531 | { 532 | "name":"javax.management.ObjectName" 533 | }, 534 | { 535 | "name":"javax.management.StandardEmitterMBean", 536 | "methods":[{"name":"cacheMBeanInfo","parameterTypes":["javax.management.MBeanInfo"] }, {"name":"getCachedMBeanInfo","parameterTypes":[] }, {"name":"getMBeanInfo","parameterTypes":[] }] 537 | }, 538 | { 539 | "name":"javax.management.openmbean.CompositeData" 540 | }, 541 | { 542 | "name":"javax.management.openmbean.OpenMBeanOperationInfoSupport" 543 | }, 544 | { 545 | "name":"javax.management.openmbean.TabularData" 546 | }, 547 | { 548 | "name":"javax.security.auth.x500.X500Principal", 549 | "fields":[{"name":"thisX500Name"}], 550 | "methods":[{"name":"","parameterTypes":["sun.security.x509.X500Name"] }] 551 | }, 552 | { 553 | "name":"jdk.management.jfr.ConfigurationInfo", 554 | "queryAllPublicMethods":true 555 | }, 556 | { 557 | "name":"jdk.management.jfr.EventTypeInfo", 558 | "queryAllPublicMethods":true 559 | }, 560 | { 561 | "name":"jdk.management.jfr.FlightRecorderMXBean", 562 | "queryAllPublicMethods":true 563 | }, 564 | { 565 | "name":"jdk.management.jfr.FlightRecorderMXBeanImpl", 566 | "queryAllPublicConstructors":true, 567 | "methods":[{"name":"cacheMBeanInfo","parameterTypes":["javax.management.MBeanInfo"] }, {"name":"getCachedMBeanInfo","parameterTypes":[] }, {"name":"getMBeanInfo","parameterTypes":[] }, {"name":"getNotificationInfo","parameterTypes":[] }] 568 | }, 569 | { 570 | "name":"jdk.management.jfr.RecordingInfo", 571 | "queryAllPublicMethods":true 572 | }, 573 | { 574 | "name":"jdk.management.jfr.SettingDescriptorInfo", 575 | "queryAllPublicMethods":true 576 | }, 577 | { 578 | "name":"libcore.io.Memory" 579 | }, 580 | { 581 | "name":"org.robolectric.Robolectric" 582 | }, 583 | { 584 | "name":"org.rogach.scallop.Scallop", 585 | "fields":[{"name":"0bitmap$1"}] 586 | }, 587 | { 588 | "name":"org.rogach.scallop.ScallopConf", 589 | "queryAllPublicMethods":true 590 | }, 591 | { 592 | "name":"org.rogach.scallop.ScallopConfBase$$anon$1", 593 | "fields":[{"name":"0bitmap$1"}] 594 | }, 595 | { 596 | "name":"org.rogach.scallop.ScallopConfBase$$anon$4", 597 | "fields":[{"name":"0bitmap$3"}] 598 | }, 599 | { 600 | "name":"org.rogach.scallop.ScallopOption", 601 | "fields":[{"name":"0bitmap$1"}] 602 | }, 603 | { 604 | "name":"org.rogach.scallop.ScallopOption$$anon$1", 605 | "fields":[{"name":"0bitmap$2"}] 606 | }, 607 | { 608 | "name":"org.tensorflow.framework.DataType", 609 | "methods":[{"name":"getValueDescriptor","parameterTypes":[] }, {"name":"valueOf","parameterTypes":["com.google.protobuf.Descriptors$EnumValueDescriptor"] }] 610 | }, 611 | { 612 | "name":"org.tensorflow.framework.ResourceHandleProto", 613 | "methods":[{"name":"newBuilder","parameterTypes":[] }] 614 | }, 615 | { 616 | "name":"org.tensorflow.framework.TensorProto", 617 | "methods":[{"name":"getBoolVal","parameterTypes":["int"] }, {"name":"getBoolValCount","parameterTypes":[] }, {"name":"getBoolValList","parameterTypes":[] }, {"name":"getDcomplexVal","parameterTypes":["int"] }, {"name":"getDcomplexValCount","parameterTypes":[] }, {"name":"getDcomplexValList","parameterTypes":[] }, {"name":"getDoubleVal","parameterTypes":["int"] }, {"name":"getDoubleValCount","parameterTypes":[] }, {"name":"getDoubleValList","parameterTypes":[] }, {"name":"getDtype","parameterTypes":[] }, {"name":"getDtypeValue","parameterTypes":[] }, {"name":"getFloatVal","parameterTypes":["int"] }, {"name":"getFloatValCount","parameterTypes":[] }, {"name":"getFloatValList","parameterTypes":[] }, {"name":"getHalfVal","parameterTypes":["int"] }, {"name":"getHalfValCount","parameterTypes":[] }, {"name":"getHalfValList","parameterTypes":[] }, {"name":"getInt64Val","parameterTypes":["int"] }, {"name":"getInt64ValCount","parameterTypes":[] }, {"name":"getInt64ValList","parameterTypes":[] }, {"name":"getIntVal","parameterTypes":["int"] }, {"name":"getIntValCount","parameterTypes":[] }, {"name":"getIntValList","parameterTypes":[] }, {"name":"getResourceHandleVal","parameterTypes":["int"] }, {"name":"getResourceHandleValCount","parameterTypes":[] }, {"name":"getResourceHandleValList","parameterTypes":[] }, {"name":"getScomplexVal","parameterTypes":["int"] }, {"name":"getScomplexValCount","parameterTypes":[] }, {"name":"getScomplexValList","parameterTypes":[] }, {"name":"getStringVal","parameterTypes":["int"] }, {"name":"getStringValCount","parameterTypes":[] }, {"name":"getStringValList","parameterTypes":[] }, {"name":"getTensorContent","parameterTypes":[] }, {"name":"getTensorShape","parameterTypes":[] }, {"name":"getUint32Val","parameterTypes":["int"] }, {"name":"getUint32ValCount","parameterTypes":[] }, {"name":"getUint32ValList","parameterTypes":[] }, {"name":"getUint64Val","parameterTypes":["int"] }, {"name":"getUint64ValCount","parameterTypes":[] }, {"name":"getUint64ValList","parameterTypes":[] }, {"name":"getVariantVal","parameterTypes":["int"] }, {"name":"getVariantValCount","parameterTypes":[] }, {"name":"getVariantValList","parameterTypes":[] }, {"name":"getVersionNumber","parameterTypes":[] }, {"name":"hasTensorShape","parameterTypes":[] }] 618 | }, 619 | { 620 | "name":"org.tensorflow.framework.TensorProto$Builder", 621 | "methods":[{"name":"addBoolVal","parameterTypes":["boolean"] }, {"name":"addDcomplexVal","parameterTypes":["double"] }, {"name":"addDoubleVal","parameterTypes":["double"] }, {"name":"addFloatVal","parameterTypes":["float"] }, {"name":"addHalfVal","parameterTypes":["int"] }, {"name":"addInt64Val","parameterTypes":["long"] }, {"name":"addIntVal","parameterTypes":["int"] }, {"name":"addResourceHandleVal","parameterTypes":["org.tensorflow.framework.ResourceHandleProto"] }, {"name":"addScomplexVal","parameterTypes":["float"] }, {"name":"addStringVal","parameterTypes":["com.google.protobuf.ByteString"] }, {"name":"addUint32Val","parameterTypes":["int"] }, {"name":"addUint64Val","parameterTypes":["long"] }, {"name":"addVariantVal","parameterTypes":["org.tensorflow.framework.VariantTensorDataProto"] }, {"name":"clearBoolVal","parameterTypes":[] }, {"name":"clearDcomplexVal","parameterTypes":[] }, {"name":"clearDoubleVal","parameterTypes":[] }, {"name":"clearDtype","parameterTypes":[] }, {"name":"clearFloatVal","parameterTypes":[] }, {"name":"clearHalfVal","parameterTypes":[] }, {"name":"clearInt64Val","parameterTypes":[] }, {"name":"clearIntVal","parameterTypes":[] }, {"name":"clearResourceHandleVal","parameterTypes":[] }, {"name":"clearScomplexVal","parameterTypes":[] }, {"name":"clearStringVal","parameterTypes":[] }, {"name":"clearTensorContent","parameterTypes":[] }, {"name":"clearTensorShape","parameterTypes":[] }, {"name":"clearUint32Val","parameterTypes":[] }, {"name":"clearUint64Val","parameterTypes":[] }, {"name":"clearVariantVal","parameterTypes":[] }, {"name":"clearVersionNumber","parameterTypes":[] }, {"name":"getBoolVal","parameterTypes":["int"] }, {"name":"getBoolValCount","parameterTypes":[] }, {"name":"getBoolValList","parameterTypes":[] }, {"name":"getDcomplexVal","parameterTypes":["int"] }, {"name":"getDcomplexValCount","parameterTypes":[] }, {"name":"getDcomplexValList","parameterTypes":[] }, {"name":"getDoubleVal","parameterTypes":["int"] }, {"name":"getDoubleValCount","parameterTypes":[] }, {"name":"getDoubleValList","parameterTypes":[] }, {"name":"getDtype","parameterTypes":[] }, {"name":"getDtypeValue","parameterTypes":[] }, {"name":"getFloatVal","parameterTypes":["int"] }, {"name":"getFloatValCount","parameterTypes":[] }, {"name":"getFloatValList","parameterTypes":[] }, {"name":"getHalfVal","parameterTypes":["int"] }, {"name":"getHalfValCount","parameterTypes":[] }, {"name":"getHalfValList","parameterTypes":[] }, {"name":"getInt64Val","parameterTypes":["int"] }, {"name":"getInt64ValCount","parameterTypes":[] }, {"name":"getInt64ValList","parameterTypes":[] }, {"name":"getIntVal","parameterTypes":["int"] }, {"name":"getIntValCount","parameterTypes":[] }, {"name":"getIntValList","parameterTypes":[] }, {"name":"getResourceHandleVal","parameterTypes":["int"] }, {"name":"getResourceHandleValBuilder","parameterTypes":["int"] }, {"name":"getResourceHandleValCount","parameterTypes":[] }, {"name":"getResourceHandleValList","parameterTypes":[] }, {"name":"getScomplexVal","parameterTypes":["int"] }, {"name":"getScomplexValCount","parameterTypes":[] }, {"name":"getScomplexValList","parameterTypes":[] }, {"name":"getStringVal","parameterTypes":["int"] }, {"name":"getStringValCount","parameterTypes":[] }, {"name":"getStringValList","parameterTypes":[] }, {"name":"getTensorContent","parameterTypes":[] }, {"name":"getTensorShape","parameterTypes":[] }, {"name":"getTensorShapeBuilder","parameterTypes":[] }, {"name":"getUint32Val","parameterTypes":["int"] }, {"name":"getUint32ValCount","parameterTypes":[] }, {"name":"getUint32ValList","parameterTypes":[] }, {"name":"getUint64Val","parameterTypes":["int"] }, {"name":"getUint64ValCount","parameterTypes":[] }, {"name":"getUint64ValList","parameterTypes":[] }, {"name":"getVariantVal","parameterTypes":["int"] }, {"name":"getVariantValBuilder","parameterTypes":["int"] }, {"name":"getVariantValCount","parameterTypes":[] }, {"name":"getVariantValList","parameterTypes":[] }, {"name":"getVersionNumber","parameterTypes":[] }, {"name":"hasTensorShape","parameterTypes":[] }, {"name":"setBoolVal","parameterTypes":["int","boolean"] }, {"name":"setDcomplexVal","parameterTypes":["int","double"] }, {"name":"setDoubleVal","parameterTypes":["int","double"] }, {"name":"setDtype","parameterTypes":["org.tensorflow.framework.DataType"] }, {"name":"setDtypeValue","parameterTypes":["int"] }, {"name":"setFloatVal","parameterTypes":["int","float"] }, {"name":"setHalfVal","parameterTypes":["int","int"] }, {"name":"setInt64Val","parameterTypes":["int","long"] }, {"name":"setIntVal","parameterTypes":["int","int"] }, {"name":"setResourceHandleVal","parameterTypes":["int","org.tensorflow.framework.ResourceHandleProto"] }, {"name":"setScomplexVal","parameterTypes":["int","float"] }, {"name":"setStringVal","parameterTypes":["int","com.google.protobuf.ByteString"] }, {"name":"setTensorContent","parameterTypes":["com.google.protobuf.ByteString"] }, {"name":"setTensorShape","parameterTypes":["org.tensorflow.framework.TensorShapeProto"] }, {"name":"setUint32Val","parameterTypes":["int","int"] }, {"name":"setUint64Val","parameterTypes":["int","long"] }, {"name":"setVariantVal","parameterTypes":["int","org.tensorflow.framework.VariantTensorDataProto"] }, {"name":"setVersionNumber","parameterTypes":["int"] }] 622 | }, 623 | { 624 | "name":"org.tensorflow.framework.TensorShapeProto", 625 | "methods":[{"name":"getDim","parameterTypes":["int"] }, {"name":"getDimCount","parameterTypes":[] }, {"name":"getDimList","parameterTypes":[] }, {"name":"getUnknownRank","parameterTypes":[] }, {"name":"newBuilder","parameterTypes":[] }] 626 | }, 627 | { 628 | "name":"org.tensorflow.framework.TensorShapeProto$Builder", 629 | "methods":[{"name":"addDim","parameterTypes":["org.tensorflow.framework.TensorShapeProto$Dim"] }, {"name":"clearDim","parameterTypes":[] }, {"name":"clearUnknownRank","parameterTypes":[] }, {"name":"getDim","parameterTypes":["int"] }, {"name":"getDimBuilder","parameterTypes":["int"] }, {"name":"getDimCount","parameterTypes":[] }, {"name":"getDimList","parameterTypes":[] }, {"name":"getUnknownRank","parameterTypes":[] }, {"name":"setDim","parameterTypes":["int","org.tensorflow.framework.TensorShapeProto$Dim"] }, {"name":"setUnknownRank","parameterTypes":["boolean"] }] 630 | }, 631 | { 632 | "name":"org.tensorflow.framework.TensorShapeProto$Dim", 633 | "methods":[{"name":"getName","parameterTypes":[] }, {"name":"getNameBytes","parameterTypes":[] }, {"name":"getSize","parameterTypes":[] }, {"name":"newBuilder","parameterTypes":[] }] 634 | }, 635 | { 636 | "name":"org.tensorflow.framework.TensorShapeProto$Dim$Builder", 637 | "methods":[{"name":"clearName","parameterTypes":[] }, {"name":"clearSize","parameterTypes":[] }, {"name":"getName","parameterTypes":[] }, {"name":"getSize","parameterTypes":[] }, {"name":"setName","parameterTypes":["java.lang.String"] }, {"name":"setNameBytes","parameterTypes":["com.google.protobuf.ByteString"] }, {"name":"setSize","parameterTypes":["long"] }], 638 | "queriedMethods":[{"name":"getNameBytes","parameterTypes":[] }] 639 | }, 640 | { 641 | "name":"org.tensorflow.framework.VariantTensorDataProto", 642 | "methods":[{"name":"newBuilder","parameterTypes":[] }] 643 | }, 644 | { 645 | "name":"sun.management.ClassLoadingImpl", 646 | "queryAllPublicConstructors":true 647 | }, 648 | { 649 | "name":"sun.management.CompilationImpl", 650 | "queryAllPublicConstructors":true 651 | }, 652 | { 653 | "name":"sun.management.ManagementFactoryHelper$1", 654 | "queryAllPublicConstructors":true 655 | }, 656 | { 657 | "name":"sun.management.ManagementFactoryHelper$PlatformLoggingImpl", 658 | "queryAllPublicConstructors":true 659 | }, 660 | { 661 | "name":"sun.management.MemoryImpl", 662 | "queryAllPublicConstructors":true 663 | }, 664 | { 665 | "name":"sun.management.MemoryManagerImpl", 666 | "queryAllPublicConstructors":true 667 | }, 668 | { 669 | "name":"sun.management.MemoryPoolImpl", 670 | "queryAllPublicConstructors":true 671 | }, 672 | { 673 | "name":"sun.management.RuntimeImpl", 674 | "queryAllPublicConstructors":true 675 | }, 676 | { 677 | "name":"sun.misc.Signal", 678 | "methods":[{"name":"","parameterTypes":["java.lang.String"] }, {"name":"handle","parameterTypes":["sun.misc.Signal","sun.misc.SignalHandler"] }] 679 | }, 680 | { 681 | "name":"sun.misc.SignalHandler" 682 | }, 683 | { 684 | "name":"sun.misc.Unsafe", 685 | "allDeclaredFields":true, 686 | "methods":[{"name":"arrayBaseOffset","parameterTypes":["java.lang.Class"] }, {"name":"arrayIndexScale","parameterTypes":["java.lang.Class"] }, {"name":"copyMemory","parameterTypes":["long","long","long"] }, {"name":"copyMemory","parameterTypes":["java.lang.Object","long","java.lang.Object","long","long"] }, {"name":"getBoolean","parameterTypes":["java.lang.Object","long"] }, {"name":"getByte","parameterTypes":["long"] }, {"name":"getByte","parameterTypes":["java.lang.Object","long"] }, {"name":"getDouble","parameterTypes":["java.lang.Object","long"] }, {"name":"getFloat","parameterTypes":["java.lang.Object","long"] }, {"name":"getInt","parameterTypes":["long"] }, {"name":"getInt","parameterTypes":["java.lang.Object","long"] }, {"name":"getLong","parameterTypes":["long"] }, {"name":"getLong","parameterTypes":["java.lang.Object","long"] }, {"name":"getObject","parameterTypes":["java.lang.Object","long"] }, {"name":"objectFieldOffset","parameterTypes":["java.lang.reflect.Field"] }, {"name":"putBoolean","parameterTypes":["java.lang.Object","long","boolean"] }, {"name":"putByte","parameterTypes":["long","byte"] }, {"name":"putByte","parameterTypes":["java.lang.Object","long","byte"] }, {"name":"putDouble","parameterTypes":["java.lang.Object","long","double"] }, {"name":"putFloat","parameterTypes":["java.lang.Object","long","float"] }, {"name":"putInt","parameterTypes":["long","int"] }, {"name":"putInt","parameterTypes":["java.lang.Object","long","int"] }, {"name":"putLong","parameterTypes":["long","long"] }, {"name":"putLong","parameterTypes":["java.lang.Object","long","long"] }, {"name":"putObject","parameterTypes":["java.lang.Object","long","java.lang.Object"] }] 687 | }, 688 | { 689 | "name":"sun.security.pkcs12.PKCS12KeyStore", 690 | "methods":[{"name":"","parameterTypes":[] }] 691 | }, 692 | { 693 | "name":"sun.security.pkcs12.PKCS12KeyStore$DualFormatPKCS12", 694 | "methods":[{"name":"","parameterTypes":[] }] 695 | }, 696 | { 697 | "name":"sun.security.provider.DSA$SHA224withDSA", 698 | "methods":[{"name":"","parameterTypes":[] }] 699 | }, 700 | { 701 | "name":"sun.security.provider.DSA$SHA256withDSA", 702 | "methods":[{"name":"","parameterTypes":[] }] 703 | }, 704 | { 705 | "name":"sun.security.provider.JavaKeyStore$DualFormatJKS", 706 | "methods":[{"name":"","parameterTypes":[] }] 707 | }, 708 | { 709 | "name":"sun.security.provider.JavaKeyStore$JKS", 710 | "methods":[{"name":"","parameterTypes":[] }] 711 | }, 712 | { 713 | "name":"sun.security.provider.NativePRNG", 714 | "methods":[{"name":"","parameterTypes":[] }, {"name":"","parameterTypes":["java.security.SecureRandomParameters"] }] 715 | }, 716 | { 717 | "name":"sun.security.provider.SHA", 718 | "methods":[{"name":"","parameterTypes":[] }] 719 | }, 720 | { 721 | "name":"sun.security.provider.SHA2$SHA224", 722 | "methods":[{"name":"","parameterTypes":[] }] 723 | }, 724 | { 725 | "name":"sun.security.provider.SHA2$SHA256", 726 | "methods":[{"name":"","parameterTypes":[] }] 727 | }, 728 | { 729 | "name":"sun.security.provider.SHA5$SHA384", 730 | "methods":[{"name":"","parameterTypes":[] }] 731 | }, 732 | { 733 | "name":"sun.security.provider.SHA5$SHA512", 734 | "methods":[{"name":"","parameterTypes":[] }] 735 | }, 736 | { 737 | "name":"sun.security.provider.X509Factory", 738 | "methods":[{"name":"","parameterTypes":[] }] 739 | }, 740 | { 741 | "name":"sun.security.provider.certpath.PKIXCertPathValidator", 742 | "methods":[{"name":"","parameterTypes":[] }] 743 | }, 744 | { 745 | "name":"sun.security.rsa.PSSParameters", 746 | "methods":[{"name":"","parameterTypes":[] }] 747 | }, 748 | { 749 | "name":"sun.security.rsa.RSAKeyFactory$Legacy", 750 | "methods":[{"name":"","parameterTypes":[] }] 751 | }, 752 | { 753 | "name":"sun.security.rsa.RSAPSSSignature", 754 | "methods":[{"name":"","parameterTypes":[] }] 755 | }, 756 | { 757 | "name":"sun.security.rsa.RSASignature$SHA224withRSA", 758 | "methods":[{"name":"","parameterTypes":[] }] 759 | }, 760 | { 761 | "name":"sun.security.rsa.RSASignature$SHA256withRSA", 762 | "methods":[{"name":"","parameterTypes":[] }] 763 | }, 764 | { 765 | "name":"sun.security.ssl.KeyManagerFactoryImpl$SunX509", 766 | "methods":[{"name":"","parameterTypes":[] }] 767 | }, 768 | { 769 | "name":"sun.security.ssl.SSLContextImpl$DefaultSSLContext", 770 | "methods":[{"name":"","parameterTypes":[] }] 771 | }, 772 | { 773 | "name":"sun.security.ssl.TrustManagerFactoryImpl$PKIXFactory", 774 | "methods":[{"name":"","parameterTypes":[] }] 775 | }, 776 | { 777 | "name":"sun.security.util.ObjectIdentifier" 778 | }, 779 | { 780 | "name":"sun.security.x509.AuthorityInfoAccessExtension", 781 | "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] 782 | }, 783 | { 784 | "name":"sun.security.x509.AuthorityKeyIdentifierExtension", 785 | "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] 786 | }, 787 | { 788 | "name":"sun.security.x509.BasicConstraintsExtension", 789 | "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] 790 | }, 791 | { 792 | "name":"sun.security.x509.CRLDistributionPointsExtension", 793 | "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] 794 | }, 795 | { 796 | "name":"sun.security.x509.CertificateExtensions" 797 | }, 798 | { 799 | "name":"sun.security.x509.CertificatePoliciesExtension", 800 | "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] 801 | }, 802 | { 803 | "name":"sun.security.x509.ExtendedKeyUsageExtension", 804 | "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] 805 | }, 806 | { 807 | "name":"sun.security.x509.IssuerAlternativeNameExtension", 808 | "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] 809 | }, 810 | { 811 | "name":"sun.security.x509.KeyUsageExtension", 812 | "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] 813 | }, 814 | { 815 | "name":"sun.security.x509.NetscapeCertTypeExtension", 816 | "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] 817 | }, 818 | { 819 | "name":"sun.security.x509.PrivateKeyUsageExtension", 820 | "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] 821 | }, 822 | { 823 | "name":"sun.security.x509.SubjectAlternativeNameExtension", 824 | "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] 825 | }, 826 | { 827 | "name":"sun.security.x509.SubjectKeyIdentifierExtension", 828 | "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] 829 | }, 830 | { 831 | "name":"tensorflow.serving.Logging$LogMetadata", 832 | "methods":[{"name":"getModelSpec","parameterTypes":[] }, {"name":"getSamplingConfig","parameterTypes":[] }, {"name":"getSavedModelTags","parameterTypes":["int"] }, {"name":"getSavedModelTagsCount","parameterTypes":[] }, {"name":"getSavedModelTagsList","parameterTypes":[] }, {"name":"hasModelSpec","parameterTypes":[] }, {"name":"hasSamplingConfig","parameterTypes":[] }, {"name":"newBuilder","parameterTypes":[] }] 833 | }, 834 | { 835 | "name":"tensorflow.serving.Logging$LogMetadata$Builder", 836 | "methods":[{"name":"addSavedModelTags","parameterTypes":["java.lang.String"] }, {"name":"clearModelSpec","parameterTypes":[] }, {"name":"clearSamplingConfig","parameterTypes":[] }, {"name":"clearSavedModelTags","parameterTypes":[] }, {"name":"getModelSpec","parameterTypes":[] }, {"name":"getModelSpecBuilder","parameterTypes":[] }, {"name":"getSamplingConfig","parameterTypes":[] }, {"name":"getSamplingConfigBuilder","parameterTypes":[] }, {"name":"getSavedModelTags","parameterTypes":["int"] }, {"name":"getSavedModelTagsCount","parameterTypes":[] }, {"name":"getSavedModelTagsList","parameterTypes":[] }, {"name":"hasModelSpec","parameterTypes":[] }, {"name":"hasSamplingConfig","parameterTypes":[] }, {"name":"setModelSpec","parameterTypes":["tensorflow.serving.Model$ModelSpec"] }, {"name":"setSamplingConfig","parameterTypes":["tensorflow.serving.LoggingConfigOuterClass$SamplingConfig"] }, {"name":"setSavedModelTags","parameterTypes":["int","java.lang.String"] }] 837 | }, 838 | { 839 | "name":"tensorflow.serving.LoggingConfigOuterClass$SamplingConfig", 840 | "methods":[{"name":"newBuilder","parameterTypes":[] }] 841 | }, 842 | { 843 | "name":"tensorflow.serving.Model$ModelSpec", 844 | "methods":[{"name":"getName","parameterTypes":[] }, {"name":"getNameBytes","parameterTypes":[] }, {"name":"getSignatureName","parameterTypes":[] }, {"name":"getSignatureNameBytes","parameterTypes":[] }, {"name":"getVersion","parameterTypes":[] }, {"name":"getVersionChoiceCase","parameterTypes":[] }, {"name":"getVersionLabel","parameterTypes":[] }, {"name":"getVersionLabelBytes","parameterTypes":[] }, {"name":"hasVersion","parameterTypes":[] }, {"name":"hasVersionLabel","parameterTypes":[] }, {"name":"newBuilder","parameterTypes":[] }] 845 | }, 846 | { 847 | "name":"tensorflow.serving.Model$ModelSpec$Builder", 848 | "methods":[{"name":"clearName","parameterTypes":[] }, {"name":"clearSignatureName","parameterTypes":[] }, {"name":"clearVersion","parameterTypes":[] }, {"name":"clearVersionChoice","parameterTypes":[] }, {"name":"clearVersionLabel","parameterTypes":[] }, {"name":"getName","parameterTypes":[] }, {"name":"getSignatureName","parameterTypes":[] }, {"name":"getVersion","parameterTypes":[] }, {"name":"getVersionBuilder","parameterTypes":[] }, {"name":"getVersionChoiceCase","parameterTypes":[] }, {"name":"getVersionLabel","parameterTypes":[] }, {"name":"hasVersion","parameterTypes":[] }, {"name":"hasVersionLabel","parameterTypes":[] }, {"name":"setName","parameterTypes":["java.lang.String"] }, {"name":"setNameBytes","parameterTypes":["com.google.protobuf.ByteString"] }, {"name":"setSignatureName","parameterTypes":["java.lang.String"] }, {"name":"setSignatureNameBytes","parameterTypes":["com.google.protobuf.ByteString"] }, {"name":"setVersion","parameterTypes":["com.google.protobuf.Int64Value"] }, {"name":"setVersionLabel","parameterTypes":["java.lang.String"] }, {"name":"setVersionLabelBytes","parameterTypes":["com.google.protobuf.ByteString"] }], 849 | "queriedMethods":[{"name":"getNameBytes","parameterTypes":[] }, {"name":"getSignatureNameBytes","parameterTypes":[] }, {"name":"getVersionLabelBytes","parameterTypes":[] }] 850 | }, 851 | { 852 | "name":"tensorflow.serving.Predict$PredictRequest", 853 | "methods":[{"name":"getDefaultInstance","parameterTypes":[] }, {"name":"getModelSpec","parameterTypes":[] }, {"name":"getOutputFilter","parameterTypes":["int"] }, {"name":"getOutputFilterCount","parameterTypes":[] }, {"name":"getOutputFilterList","parameterTypes":[] }, {"name":"hasModelSpec","parameterTypes":[] }, {"name":"newBuilder","parameterTypes":[] }] 854 | }, 855 | { 856 | "name":"tensorflow.serving.Predict$PredictRequest$Builder", 857 | "methods":[{"name":"addOutputFilter","parameterTypes":["java.lang.String"] }, {"name":"clearModelSpec","parameterTypes":[] }, {"name":"clearOutputFilter","parameterTypes":[] }, {"name":"getModelSpec","parameterTypes":[] }, {"name":"getModelSpecBuilder","parameterTypes":[] }, {"name":"getOutputFilter","parameterTypes":["int"] }, {"name":"getOutputFilterCount","parameterTypes":[] }, {"name":"getOutputFilterList","parameterTypes":[] }, {"name":"hasModelSpec","parameterTypes":[] }, {"name":"setModelSpec","parameterTypes":["tensorflow.serving.Model$ModelSpec"] }, {"name":"setOutputFilter","parameterTypes":["int","java.lang.String"] }] 858 | }, 859 | { 860 | "name":"tensorflow.serving.Predict$PredictResponse", 861 | "methods":[{"name":"getDefaultInstance","parameterTypes":[] }, {"name":"getModelSpec","parameterTypes":[] }, {"name":"hasModelSpec","parameterTypes":[] }, {"name":"newBuilder","parameterTypes":[] }] 862 | }, 863 | { 864 | "name":"tensorflow.serving.Predict$PredictResponse$Builder", 865 | "methods":[{"name":"clearModelSpec","parameterTypes":[] }, {"name":"getModelSpec","parameterTypes":[] }, {"name":"getModelSpecBuilder","parameterTypes":[] }, {"name":"hasModelSpec","parameterTypes":[] }, {"name":"setModelSpec","parameterTypes":["tensorflow.serving.Model$ModelSpec"] }] 866 | }, 867 | { 868 | "name":"tensorflow.serving.PredictionLogOuterClass$ClassifyLog", 869 | "methods":[{"name":"newBuilder","parameterTypes":[] }] 870 | }, 871 | { 872 | "name":"tensorflow.serving.PredictionLogOuterClass$MultiInferenceLog", 873 | "methods":[{"name":"newBuilder","parameterTypes":[] }] 874 | }, 875 | { 876 | "name":"tensorflow.serving.PredictionLogOuterClass$PredictLog", 877 | "methods":[{"name":"getRequest","parameterTypes":[] }, {"name":"getResponse","parameterTypes":[] }, {"name":"hasRequest","parameterTypes":[] }, {"name":"hasResponse","parameterTypes":[] }, {"name":"newBuilder","parameterTypes":[] }] 878 | }, 879 | { 880 | "name":"tensorflow.serving.PredictionLogOuterClass$PredictLog$Builder", 881 | "methods":[{"name":"clearRequest","parameterTypes":[] }, {"name":"clearResponse","parameterTypes":[] }, {"name":"getRequest","parameterTypes":[] }, {"name":"getRequestBuilder","parameterTypes":[] }, {"name":"getResponse","parameterTypes":[] }, {"name":"getResponseBuilder","parameterTypes":[] }, {"name":"hasRequest","parameterTypes":[] }, {"name":"hasResponse","parameterTypes":[] }, {"name":"setRequest","parameterTypes":["tensorflow.serving.Predict$PredictRequest"] }, {"name":"setResponse","parameterTypes":["tensorflow.serving.Predict$PredictResponse"] }] 882 | }, 883 | { 884 | "name":"tensorflow.serving.PredictionLogOuterClass$PredictionLog", 885 | "methods":[{"name":"getClassifyLog","parameterTypes":[] }, {"name":"getLogMetadata","parameterTypes":[] }, {"name":"getLogTypeCase","parameterTypes":[] }, {"name":"getMultiInferenceLog","parameterTypes":[] }, {"name":"getPredictLog","parameterTypes":[] }, {"name":"getRegressLog","parameterTypes":[] }, {"name":"hasClassifyLog","parameterTypes":[] }, {"name":"hasLogMetadata","parameterTypes":[] }, {"name":"hasMultiInferenceLog","parameterTypes":[] }, {"name":"hasPredictLog","parameterTypes":[] }, {"name":"hasRegressLog","parameterTypes":[] }] 886 | }, 887 | { 888 | "name":"tensorflow.serving.PredictionLogOuterClass$PredictionLog$Builder", 889 | "methods":[{"name":"clearClassifyLog","parameterTypes":[] }, {"name":"clearLogMetadata","parameterTypes":[] }, {"name":"clearLogType","parameterTypes":[] }, {"name":"clearMultiInferenceLog","parameterTypes":[] }, {"name":"clearPredictLog","parameterTypes":[] }, {"name":"clearRegressLog","parameterTypes":[] }, {"name":"getClassifyLog","parameterTypes":[] }, {"name":"getClassifyLogBuilder","parameterTypes":[] }, {"name":"getLogMetadata","parameterTypes":[] }, {"name":"getLogMetadataBuilder","parameterTypes":[] }, {"name":"getLogTypeCase","parameterTypes":[] }, {"name":"getMultiInferenceLog","parameterTypes":[] }, {"name":"getMultiInferenceLogBuilder","parameterTypes":[] }, {"name":"getPredictLog","parameterTypes":[] }, {"name":"getPredictLogBuilder","parameterTypes":[] }, {"name":"getRegressLog","parameterTypes":[] }, {"name":"getRegressLogBuilder","parameterTypes":[] }, {"name":"hasClassifyLog","parameterTypes":[] }, {"name":"hasLogMetadata","parameterTypes":[] }, {"name":"hasMultiInferenceLog","parameterTypes":[] }, {"name":"hasPredictLog","parameterTypes":[] }, {"name":"hasRegressLog","parameterTypes":[] }, {"name":"setClassifyLog","parameterTypes":["tensorflow.serving.PredictionLogOuterClass$ClassifyLog"] }, {"name":"setLogMetadata","parameterTypes":["tensorflow.serving.Logging$LogMetadata"] }, {"name":"setMultiInferenceLog","parameterTypes":["tensorflow.serving.PredictionLogOuterClass$MultiInferenceLog"] }, {"name":"setPredictLog","parameterTypes":["tensorflow.serving.PredictionLogOuterClass$PredictLog"] }, {"name":"setRegressLog","parameterTypes":["tensorflow.serving.PredictionLogOuterClass$RegressLog"] }] 890 | }, 891 | { 892 | "name":"tensorflow.serving.PredictionLogOuterClass$RegressLog", 893 | "methods":[{"name":"newBuilder","parameterTypes":[] }] 894 | }, 895 | { 896 | "name":"tfr.Cli$", 897 | "fields":[{"name":"cats$effect$IOApp$$queue$lzy1"}] 898 | }, 899 | { 900 | "name":"tfr.Cli$Options", 901 | "queryAllPublicMethods":true, 902 | "methods":[{"name":"checkCrc32","parameterTypes":[] }, {"name":"files","parameterTypes":[] }, {"name":"flat","parameterTypes":[] }, {"name":"number","parameterTypes":[] }, {"name":"record","parameterTypes":[] }] 903 | }, 904 | { 905 | "name":"tfr.Cli$Options$", 906 | "fields":[{"name":"0bitmap$1"}, {"name":"recordValueConverter$lzy1"}] 907 | }, 908 | { 909 | "name":"tfr.instances.ExampleEncoderInstances$flat$", 910 | "fields":[{"name":"bytesListEncoder$lzy1"}, {"name":"exampleEncoder$lzy1"}, {"name":"featureEncoder$lzy1"}, {"name":"featuresEncoder$lzy1"}, {"name":"floatListEncoder$lzy1"}, {"name":"int64ListEncoder$lzy1"}] 911 | }, 912 | { 913 | "name":"tfr.instances.example.package$", 914 | "fields":[{"name":"0bitmap$1"}, {"name":"byteStringEncoder$lzy1"}, {"name":"bytesListEncoder$lzy1"}, {"name":"exampleEncoder$lzy1"}, {"name":"featureEncoder$lzy1"}, {"name":"featuresEncoder$lzy1"}, {"name":"flat$lzy1"}, {"name":"floatListEncoder$lzy1"}, {"name":"int64ListEncoder$lzy1"}] 915 | }, 916 | { 917 | "name":"tfr.instances.output.package$", 918 | "fields":[{"name":"0bitmap$1"}, {"name":"errorShow$lzy1"}] 919 | }, 920 | { 921 | "name":"tfr.instances.prediction.package$", 922 | "fields":[{"name":"0bitmap$1"}, {"name":"byteStringEncoder$lzy1"}, {"name":"predictionLogEncoder$lzy1"}] 923 | } 924 | ] 925 | -------------------------------------------------------------------------------- /modules/cli/src/graal/resource-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "resources":{ 3 | "includes":[{ 4 | "pattern":"\\QMETA-INF/maven/com.google.cloud/google-cloud-storage/pom.properties\\E" 5 | }, { 6 | "pattern":"\\QMETA-INF/services/com.google.auth.http.HttpTransportFactory\\E" 7 | }, { 8 | "pattern":"\\QMETA-INF/services/com.google.cloud.storage.StorageFactory\\E" 9 | }, { 10 | "pattern":"\\QMETA-INF/services/com.google.cloud.storage.spi.StorageRpcFactory\\E" 11 | }, { 12 | "pattern":"\\QMETA-INF/services/java.lang.System$LoggerFinder\\E" 13 | }, { 14 | "pattern":"\\QMETA-INF/services/java.net.spi.InetAddressResolverProvider\\E" 15 | }, { 16 | "pattern":"\\QMETA-INF/services/java.net.spi.URLStreamHandlerProvider\\E" 17 | }, { 18 | "pattern":"\\QMETA-INF/services/java.time.zone.ZoneRulesProvider\\E" 19 | }, { 20 | "pattern":"\\Qcom/google/api/client/googleapis/google-api-client.properties\\E" 21 | }, { 22 | "pattern":"\\Qcom/google/api/client/http/google-http-client.properties\\E" 23 | }, { 24 | "pattern":"\\Qcom/google/cloud/storage/META-INF/maven/com.google.cloud/google-cloud-storage/pom.properties\\E" 25 | }, { 26 | "pattern":"\\Qdependencies.properties\\E" 27 | }, { 28 | "pattern":"java.base:\\Qjdk/internal/icu/impl/data/icudt72b/nfc.nrm\\E" 29 | }, { 30 | "pattern":"java.base:\\Qjdk/internal/icu/impl/data/icudt72b/nfkc.nrm\\E" 31 | }, { 32 | "pattern":"java.base:\\Qjdk/internal/icu/impl/data/icudt72b/uprops.icu\\E" 33 | }, { 34 | "pattern":"java.base:\\Qsun/net/idn/uidna.spp\\E" 35 | }, { 36 | "pattern":"java.base:\\Qsun/net/www/content-types.properties\\E" 37 | }, { 38 | "pattern":"jdk.jfr:\\Qjdk/jfr/internal/query/view.ini\\E" 39 | }]}, 40 | "bundles":[] 41 | } 42 | -------------------------------------------------------------------------------- /modules/cli/src/graal/serialization-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "types":[ 3 | ], 4 | "lambdaCapturingTypes":[ 5 | ], 6 | "proxies":[ 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /modules/cli/src/main/scala/tfr/Cli.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Spotify AB. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an 12 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | * KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations 15 | * under the License. 16 | */ 17 | package tfr 18 | 19 | import java.io.InputStream 20 | 21 | import org.rogach.scallop._ 22 | import cats.Show 23 | import io.circe.Encoder 24 | import cats.effect.{IO, Resource, IOApp, ExitCode} 25 | import fs2._ 26 | import org.tensorflow.example.Example 27 | import tensorflow.serving.PredictionLogOuterClass.PredictionLog 28 | import tfr.instances.example.{given, _} 29 | import tfr.instances.prediction.{given, _} 30 | import tfr.instances.output.{given, _} 31 | import scala.collection.immutable.ArraySeq 32 | 33 | object Cli extends IOApp: 34 | 35 | object Options: 36 | enum RecordType(value: String): 37 | case Example extends RecordType("example") 38 | case PredictionLog extends RecordType("prediction_log") 39 | 40 | given recordValueConverter: ValueConverter[RecordType] = 41 | singleArgConverter[RecordType] { s => 42 | RecordType.valueOf(s.split("_").fold("")(_ + _.capitalize)) 43 | } 44 | 45 | final class Options(arguments: Seq[String]) extends ScallopConf(arguments): 46 | import Options.{given, _} 47 | printedName = "tfr" 48 | version(BuildInfo.version) 49 | banner("""Usage: tfr [options] 50 | |TensorFlow TFRecord reader CLI tool 51 | |Options: 52 | |""".stripMargin) 53 | 54 | val record: ScallopOption[RecordType] = 55 | opt[RecordType]( 56 | default = Some(RecordType.Example), 57 | descr = "Record type to be read { example | prediction_log }" 58 | ) 59 | val checkCrc32 = opt[Boolean]( 60 | default = Some(false), 61 | descr = "Enable checks CRC32 on each record" 62 | ) 63 | val number = opt[Int](descr = "Number of records to output") 64 | val flat = opt[Boolean]( 65 | default = Some(false), 66 | descr = "Output examples as flat JSON objects" 67 | ) 68 | val files = trailArg[List[String]]( 69 | required = false, 70 | descr = "files? | STDIN", 71 | default = Some(List.empty) 72 | ) 73 | verify() 74 | 75 | override def run(args: List[String]): IO[ExitCode] = 76 | val options = Options(args) 77 | val resources = options.files() match 78 | case Nil => Resources.stdin[IO] :: Nil 79 | case l => l.iterator.map(Resources.file[IO]).toList 80 | 81 | options.record() match 82 | case Options.RecordType.Example => 83 | given exampleEncoder: Encoder[Example] = 84 | if options.flat() then flat.exampleEncoder 85 | else tfr.instances.example.exampleEncoder 86 | 87 | run[Example](options, resources) 88 | case Options.RecordType.PredictionLog => 89 | given predictionLogEncoder: Encoder[PredictionLog] = 90 | tfr.instances.prediction.predictionLogEncoder 91 | 92 | run[PredictionLog](options, resources) 93 | 94 | def run[T: Parsable: Show]( 95 | options: Options, 96 | resources: List[Resource[IO, InputStream]] 97 | ): IO[ExitCode] = 98 | val reader = TFRecord.resourceReader[IO, T]( 99 | TFRecord.typedReader[T, IO](options.checkCrc32()) 100 | ) 101 | val records = 102 | Stream(resources.map(reader.run): _*).parJoin(resources.length) 103 | 104 | options.number 105 | .map(records.take(_)) 106 | .getOrElse(records) 107 | .printlns 108 | .compile 109 | .drain 110 | .as(ExitCode.Success) 111 | -------------------------------------------------------------------------------- /modules/cli/src/main/scala/tfr/instances/OutputInstances.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Spotify AB. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an 12 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | * KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations 15 | * under the License. 16 | */ 17 | package tfr.instances 18 | 19 | import cats.Show 20 | import tfr.TFRecord.Error 21 | 22 | trait OutputInstances: 23 | given eitherReadErrorShow[T](using 24 | es: Show[T], 25 | ts: Show[Error] 26 | ): Show[Either[Error, T]] with 27 | override def show(t: Either[Error, T]): String = 28 | t.fold(ts.show, es.show) 29 | 30 | given errorShow: Show[Error] with 31 | override def show(t: Error): String = 32 | t match 33 | case Error.EmptyHeader => "empty header" 34 | case Error.InvalidCrc32 => "invalid crc32" 35 | case Error.ReadError => "unexpected read error" 36 | -------------------------------------------------------------------------------- /modules/cli/src/main/scala/tfr/instances/output/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Spotify AB. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an 12 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | * KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations 15 | * under the License. 16 | */ 17 | package tfr.instances 18 | 19 | package object output extends OutputInstances 20 | -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/google/protobuf/wrappers.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers - Google's data interchange format 2 | // Copyright 2008 Google Inc. All rights reserved. 3 | // https://developers.google.com/protocol-buffers/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions are 7 | // met: 8 | // 9 | // * Redistributions of source code must retain the above copyright 10 | // notice, this list of conditions and the following disclaimer. 11 | // * Redistributions in binary form must reproduce the above 12 | // copyright notice, this list of conditions and the following disclaimer 13 | // in the documentation and/or other materials provided with the 14 | // distribution. 15 | // * Neither the name of Google Inc. nor the names of its 16 | // contributors may be used to endorse or promote products derived from 17 | // this software without specific prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | // Wrappers for primitive (non-message) types. These types are useful 32 | // for embedding primitives in the `google.protobuf.Any` type and for places 33 | // where we need to distinguish between the absence of a primitive 34 | // typed field and its default value. 35 | // 36 | // These wrappers have no meaningful use within repeated fields as they lack 37 | // the ability to detect presence on individual elements. 38 | // These wrappers have no meaningful use within a map or a oneof since 39 | // individual entries of a map or fields of a oneof can already detect presence. 40 | 41 | syntax = "proto3"; 42 | 43 | package google.protobuf; 44 | 45 | option csharp_namespace = "Google.Protobuf.WellKnownTypes"; 46 | option cc_enable_arenas = true; 47 | option go_package = "google.golang.org/protobuf/types/known/wrapperspb"; 48 | option java_package = "com.google.protobuf"; 49 | option java_outer_classname = "WrappersProto"; 50 | option java_multiple_files = true; 51 | option objc_class_prefix = "GPB"; 52 | 53 | // Wrapper message for `double`. 54 | // 55 | // The JSON representation for `DoubleValue` is JSON number. 56 | message DoubleValue { 57 | // The double value. 58 | double value = 1; 59 | } 60 | 61 | // Wrapper message for `float`. 62 | // 63 | // The JSON representation for `FloatValue` is JSON number. 64 | message FloatValue { 65 | // The float value. 66 | float value = 1; 67 | } 68 | 69 | // Wrapper message for `int64`. 70 | // 71 | // The JSON representation for `Int64Value` is JSON string. 72 | message Int64Value { 73 | // The int64 value. 74 | int64 value = 1; 75 | } 76 | 77 | // Wrapper message for `uint64`. 78 | // 79 | // The JSON representation for `UInt64Value` is JSON string. 80 | message UInt64Value { 81 | // The uint64 value. 82 | uint64 value = 1; 83 | } 84 | 85 | // Wrapper message for `int32`. 86 | // 87 | // The JSON representation for `Int32Value` is JSON number. 88 | message Int32Value { 89 | // The int32 value. 90 | int32 value = 1; 91 | } 92 | 93 | // Wrapper message for `uint32`. 94 | // 95 | // The JSON representation for `UInt32Value` is JSON number. 96 | message UInt32Value { 97 | // The uint32 value. 98 | uint32 value = 1; 99 | } 100 | 101 | // Wrapper message for `bool`. 102 | // 103 | // The JSON representation for `BoolValue` is JSON `true` and `false`. 104 | message BoolValue { 105 | // The bool value. 106 | bool value = 1; 107 | } 108 | 109 | // Wrapper message for `string`. 110 | // 111 | // The JSON representation for `StringValue` is JSON string. 112 | message StringValue { 113 | // The string value. 114 | string value = 1; 115 | } 116 | 117 | // Wrapper message for `bytes`. 118 | // 119 | // The JSON representation for `BytesValue` is JSON string. 120 | message BytesValue { 121 | // The bytes value. 122 | bytes value = 1; 123 | } 124 | -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/tensorflow/core/example/example.proto: -------------------------------------------------------------------------------- 1 | // Protocol messages for describing input data Examples for machine learning 2 | // model training or inference. 3 | syntax = "proto3"; 4 | 5 | package tensorflow; 6 | 7 | import "tensorflow/core/example/feature.proto"; 8 | 9 | option cc_enable_arenas = true; 10 | option java_outer_classname = "ExampleProtos"; 11 | option java_multiple_files = true; 12 | option java_package = "org.tensorflow.example"; 13 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/example/example_protos_go_proto"; 14 | 15 | // An Example is a mostly-normalized data format for storing data for 16 | // training and inference. It contains a key-value store (features); where 17 | // each key (string) maps to a Feature message (which is oneof packed BytesList, 18 | // FloatList, or Int64List). This flexible and compact format allows the 19 | // storage of large amounts of typed data, but requires that the data shape 20 | // and use be determined by the configuration files and parsers that are used to 21 | // read and write this format. That is, the Example is mostly *not* a 22 | // self-describing format. In TensorFlow, Examples are read in row-major 23 | // format, so any configuration that describes data with rank-2 or above 24 | // should keep this in mind. For example, to store an M x N matrix of Bytes, 25 | // the BytesList must contain M*N bytes, with M rows of N contiguous values 26 | // each. That is, the BytesList value must store the matrix as: 27 | // .... row 0 .... .... row 1 .... // ........... // ... row M-1 .... 28 | // 29 | // An Example for a movie recommendation application: 30 | // features { 31 | // feature { 32 | // key: "age" 33 | // value { float_list { 34 | // value: 29.0 35 | // }} 36 | // } 37 | // feature { 38 | // key: "movie" 39 | // value { bytes_list { 40 | // value: "The Shawshank Redemption" 41 | // value: "Fight Club" 42 | // }} 43 | // } 44 | // feature { 45 | // key: "movie_ratings" 46 | // value { float_list { 47 | // value: 9.0 48 | // value: 9.7 49 | // }} 50 | // } 51 | // feature { 52 | // key: "suggestion" 53 | // value { bytes_list { 54 | // value: "Inception" 55 | // }} 56 | // } 57 | // # Note that this feature exists to be used as a label in training. 58 | // # E.g., if training a logistic regression model to predict purchase 59 | // # probability in our learning tool we would set the label feature to 60 | // # "suggestion_purchased". 61 | // feature { 62 | // key: "suggestion_purchased" 63 | // value { float_list { 64 | // value: 1.0 65 | // }} 66 | // } 67 | // # Similar to "suggestion_purchased" above this feature exists to be used 68 | // # as a label in training. 69 | // # E.g., if training a linear regression model to predict purchase 70 | // # price in our learning tool we would set the label feature to 71 | // # "purchase_price". 72 | // feature { 73 | // key: "purchase_price" 74 | // value { float_list { 75 | // value: 9.99 76 | // }} 77 | // } 78 | // } 79 | // 80 | // A conformant Example data set obeys the following conventions: 81 | // - If a Feature K exists in one example with data type T, it must be of 82 | // type T in all other examples when present. It may be omitted. 83 | // - The number of instances of Feature K list data may vary across examples, 84 | // depending on the requirements of the model. 85 | // - If a Feature K doesn't exist in an example, a K-specific default will be 86 | // used, if configured. 87 | // - If a Feature K exists in an example but contains no items, the intent 88 | // is considered to be an empty tensor and no default will be used. 89 | 90 | message Example { 91 | Features features = 1; 92 | } 93 | 94 | // A SequenceExample is an Example representing one or more sequences, and 95 | // some context. The context contains features which apply to the entire 96 | // example. The feature_lists contain a key, value map where each key is 97 | // associated with a repeated set of Features (a FeatureList). 98 | // A FeatureList thus represents the values of a feature identified by its key 99 | // over time / frames. 100 | // 101 | // Below is a SequenceExample for a movie recommendation application recording a 102 | // sequence of ratings by a user. The time-independent features ("locale", 103 | // "age", "favorites") describing the user are part of the context. The sequence 104 | // of movies the user rated are part of the feature_lists. For each movie in the 105 | // sequence we have information on its name and actors and the user's rating. 106 | // This information is recorded in three separate feature_list(s). 107 | // In the example below there are only two movies. All three feature_list(s), 108 | // namely "movie_ratings", "movie_names", and "actors" have a feature value for 109 | // both movies. Note, that "actors" is itself a bytes_list with multiple 110 | // strings per movie. 111 | // 112 | // context: { 113 | // feature: { 114 | // key : "locale" 115 | // value: { 116 | // bytes_list: { 117 | // value: [ "pt_BR" ] 118 | // } 119 | // } 120 | // } 121 | // feature: { 122 | // key : "age" 123 | // value: { 124 | // float_list: { 125 | // value: [ 19.0 ] 126 | // } 127 | // } 128 | // } 129 | // feature: { 130 | // key : "favorites" 131 | // value: { 132 | // bytes_list: { 133 | // value: [ "Majesty Rose", "Savannah Outen", "One Direction" ] 134 | // } 135 | // } 136 | // } 137 | // } 138 | // feature_lists: { 139 | // feature_list: { 140 | // key : "movie_ratings" 141 | // value: { 142 | // feature: { 143 | // float_list: { 144 | // value: [ 4.5 ] 145 | // } 146 | // } 147 | // feature: { 148 | // float_list: { 149 | // value: [ 5.0 ] 150 | // } 151 | // } 152 | // } 153 | // } 154 | // feature_list: { 155 | // key : "movie_names" 156 | // value: { 157 | // feature: { 158 | // bytes_list: { 159 | // value: [ "The Shawshank Redemption" ] 160 | // } 161 | // } 162 | // feature: { 163 | // bytes_list: { 164 | // value: [ "Fight Club" ] 165 | // } 166 | // } 167 | // } 168 | // } 169 | // feature_list: { 170 | // key : "actors" 171 | // value: { 172 | // feature: { 173 | // bytes_list: { 174 | // value: [ "Tim Robbins", "Morgan Freeman" ] 175 | // } 176 | // } 177 | // feature: { 178 | // bytes_list: { 179 | // value: [ "Brad Pitt", "Edward Norton", "Helena Bonham Carter" ] 180 | // } 181 | // } 182 | // } 183 | // } 184 | // } 185 | // 186 | // A conformant SequenceExample data set obeys the following conventions: 187 | // 188 | // Context: 189 | // - All conformant context features K must obey the same conventions as 190 | // a conformant Example's features (see above). 191 | // Feature lists: 192 | // - A FeatureList L may be missing in an example; it is up to the 193 | // parser configuration to determine if this is allowed or considered 194 | // an empty list (zero length). 195 | // - If a FeatureList L exists, it may be empty (zero length). 196 | // - If a FeatureList L is non-empty, all features within the FeatureList 197 | // must have the same data type T. Even across SequenceExamples, the type T 198 | // of the FeatureList identified by the same key must be the same. An entry 199 | // without any values may serve as an empty feature. 200 | // - If a FeatureList L is non-empty, it is up to the parser configuration 201 | // to determine if all features within the FeatureList must 202 | // have the same size. The same holds for this FeatureList across multiple 203 | // examples. 204 | // - For sequence modeling, e.g.: 205 | // http://colah.github.io/posts/2015-08-Understanding-LSTMs/ 206 | // https://github.com/tensorflow/nmt 207 | // the feature lists represent a sequence of frames. 208 | // In this scenario, all FeatureLists in a SequenceExample have the same 209 | // number of Feature messages, so that the ith element in each FeatureList 210 | // is part of the ith frame (or time step). 211 | // Examples of conformant and non-conformant examples' FeatureLists: 212 | // 213 | // Conformant FeatureLists: 214 | // feature_lists: { feature_list: { 215 | // key: "movie_ratings" 216 | // value: { feature: { float_list: { value: [ 4.5 ] } } 217 | // feature: { float_list: { value: [ 5.0 ] } } } 218 | // } } 219 | // 220 | // Non-conformant FeatureLists (mismatched types): 221 | // feature_lists: { feature_list: { 222 | // key: "movie_ratings" 223 | // value: { feature: { float_list: { value: [ 4.5 ] } } 224 | // feature: { int64_list: { value: [ 5 ] } } } 225 | // } } 226 | // 227 | // Conditionally conformant FeatureLists, the parser configuration determines 228 | // if the feature sizes must match: 229 | // feature_lists: { feature_list: { 230 | // key: "movie_ratings" 231 | // value: { feature: { float_list: { value: [ 4.5 ] } } 232 | // feature: { float_list: { value: [ 5.0, 6.0 ] } } } 233 | // } } 234 | // 235 | // Conformant pair of SequenceExample 236 | // feature_lists: { feature_list: { 237 | // key: "movie_ratings" 238 | // value: { feature: { float_list: { value: [ 4.5 ] } } 239 | // feature: { float_list: { value: [ 5.0 ] } } } 240 | // } } 241 | // and: 242 | // feature_lists: { feature_list: { 243 | // key: "movie_ratings" 244 | // value: { feature: { float_list: { value: [ 4.5 ] } } 245 | // feature: { float_list: { value: [ 5.0 ] } } 246 | // feature: { float_list: { value: [ 2.0 ] } } } 247 | // } } 248 | // 249 | // Conformant pair of SequenceExample 250 | // feature_lists: { feature_list: { 251 | // key: "movie_ratings" 252 | // value: { feature: { float_list: { value: [ 4.5 ] } } 253 | // feature: { float_list: { value: [ 5.0 ] } } } 254 | // } } 255 | // and: 256 | // feature_lists: { feature_list: { 257 | // key: "movie_ratings" 258 | // value: { } 259 | // } } 260 | // 261 | // Conditionally conformant pair of SequenceExample, the parser configuration 262 | // determines if the second feature_lists is consistent (zero-length) or 263 | // invalid (missing "movie_ratings"): 264 | // feature_lists: { feature_list: { 265 | // key: "movie_ratings" 266 | // value: { feature: { float_list: { value: [ 4.5 ] } } 267 | // feature: { float_list: { value: [ 5.0 ] } } } 268 | // } } 269 | // and: 270 | // feature_lists: { } 271 | // 272 | // Non-conformant pair of SequenceExample (mismatched types) 273 | // feature_lists: { feature_list: { 274 | // key: "movie_ratings" 275 | // value: { feature: { float_list: { value: [ 4.5 ] } } 276 | // feature: { float_list: { value: [ 5.0 ] } } } 277 | // } } 278 | // and: 279 | // feature_lists: { feature_list: { 280 | // key: "movie_ratings" 281 | // value: { feature: { int64_list: { value: [ 4 ] } } 282 | // feature: { int64_list: { value: [ 5 ] } } 283 | // feature: { int64_list: { value: [ 2 ] } } } 284 | // } } 285 | // 286 | // Conditionally conformant pair of SequenceExample; the parser configuration 287 | // determines if the feature sizes must match: 288 | // feature_lists: { feature_list: { 289 | // key: "movie_ratings" 290 | // value: { feature: { float_list: { value: [ 4.5 ] } } 291 | // feature: { float_list: { value: [ 5.0 ] } } } 292 | // } } 293 | // and: 294 | // feature_lists: { feature_list: { 295 | // key: "movie_ratings" 296 | // value: { feature: { float_list: { value: [ 4.0 ] } } 297 | // feature: { float_list: { value: [ 5.0, 3.0 ] } } 298 | // } } 299 | 300 | message SequenceExample { 301 | Features context = 1; 302 | FeatureLists feature_lists = 2; 303 | } 304 | -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/tensorflow/core/example/feature.proto: -------------------------------------------------------------------------------- 1 | // Protocol messages for describing features for machine learning model 2 | // training or inference. 3 | // 4 | // There are three base Feature types: 5 | // - bytes 6 | // - float 7 | // - int64 8 | // 9 | // A Feature contains Lists which may hold zero or more values. These 10 | // lists are the base values BytesList, FloatList, Int64List. 11 | // 12 | // Features are organized into categories by name. The Features message 13 | // contains the mapping from name to Feature. 14 | // 15 | // Example Features for a movie recommendation application: 16 | // feature { 17 | // key: "age" 18 | // value { float_list { 19 | // value: 29.0 20 | // }} 21 | // } 22 | // feature { 23 | // key: "movie" 24 | // value { bytes_list { 25 | // value: "The Shawshank Redemption" 26 | // value: "Fight Club" 27 | // }} 28 | // } 29 | // feature { 30 | // key: "movie_ratings" 31 | // value { float_list { 32 | // value: 9.0 33 | // value: 9.7 34 | // }} 35 | // } 36 | // feature { 37 | // key: "suggestion" 38 | // value { bytes_list { 39 | // value: "Inception" 40 | // }} 41 | // } 42 | // feature { 43 | // key: "suggestion_purchased" 44 | // value { int64_list { 45 | // value: 1 46 | // }} 47 | // } 48 | // feature { 49 | // key: "purchase_price" 50 | // value { float_list { 51 | // value: 9.99 52 | // }} 53 | // } 54 | // 55 | 56 | syntax = "proto3"; 57 | 58 | package tensorflow; 59 | 60 | option cc_enable_arenas = true; 61 | option java_outer_classname = "FeatureProtos"; 62 | option java_multiple_files = true; 63 | option java_package = "org.tensorflow.example"; 64 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/example/example_protos_go_proto"; 65 | 66 | // Containers to hold repeated fundamental values. 67 | message BytesList { 68 | repeated bytes value = 1; 69 | } 70 | message FloatList { 71 | repeated float value = 1 [packed = true]; 72 | } 73 | message Int64List { 74 | repeated int64 value = 1 [packed = true]; 75 | } 76 | 77 | // Containers for non-sequential data. 78 | message Feature { 79 | // Each feature can be exactly one kind. 80 | oneof kind { 81 | BytesList bytes_list = 1; 82 | FloatList float_list = 2; 83 | Int64List int64_list = 3; 84 | } 85 | } 86 | 87 | message Features { 88 | // Map from feature name to feature. 89 | map feature = 1; 90 | } 91 | 92 | // Containers for sequential data. 93 | // 94 | // A FeatureList contains lists of Features. These may hold zero or more 95 | // Feature values. 96 | // 97 | // FeatureLists are organized into categories by name. The FeatureLists message 98 | // contains the mapping from name to FeatureList. 99 | // 100 | message FeatureList { 101 | repeated Feature feature = 1; 102 | } 103 | 104 | message FeatureLists { 105 | // Map from feature name to feature list. 106 | map feature_list = 1; 107 | } 108 | -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/tensorflow/core/framework/resource_handle.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | import "tensorflow/core/framework/tensor_shape.proto"; 6 | import "tensorflow/core/framework/types.proto"; 7 | 8 | option cc_enable_arenas = true; 9 | option java_outer_classname = "ResourceHandle"; 10 | option java_multiple_files = true; 11 | option java_package = "org.tensorflow.framework"; 12 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/resource_handle_go_proto"; 13 | 14 | // Protocol buffer representing a handle to a tensorflow resource. Handles are 15 | // not valid across executions, but can be serialized back and forth from within 16 | // a single run. 17 | message ResourceHandleProto { 18 | // Unique name for the device containing the resource. 19 | string device = 1; 20 | 21 | // Container in which this resource is placed. 22 | string container = 2; 23 | 24 | // Unique name of this resource. 25 | string name = 3; 26 | 27 | // Hash code for the type of the resource. Is only valid in the same device 28 | // and in the same execution. 29 | uint64 hash_code = 4; 30 | 31 | // For debug-only, the name of the type pointed to by this handle, if 32 | // available. 33 | string maybe_type_name = 5; 34 | 35 | // Protocol buffer representing a pair of (data type, tensor shape). 36 | message DtypeAndShape { 37 | DataType dtype = 1; 38 | TensorShapeProto shape = 2; 39 | } 40 | 41 | // Data types and shapes for the underlying resource. 42 | repeated DtypeAndShape dtypes_and_shapes = 6; 43 | 44 | // A set of devices containing the resource. If empty, the resource only 45 | // exists on `device`. 46 | repeated string allowed_devices = 7; 47 | } 48 | -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/tensorflow/core/framework/tensor.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | import "tensorflow/core/framework/resource_handle.proto"; 6 | import "tensorflow/core/framework/tensor_shape.proto"; 7 | import "tensorflow/core/framework/types.proto"; 8 | 9 | option cc_enable_arenas = true; 10 | option java_outer_classname = "TensorProtos"; 11 | option java_multiple_files = true; 12 | option java_package = "org.tensorflow.framework"; 13 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/tensor_go_proto"; 14 | 15 | // Protocol buffer representing a tensor. 16 | message TensorProto { 17 | DataType dtype = 1; 18 | 19 | // Shape of the tensor. TODO(touts): sort out the 0-rank issues. 20 | TensorShapeProto tensor_shape = 2; 21 | 22 | // Only one of the representations below is set, one of "tensor_contents" and 23 | // the "xxx_val" attributes. We are not using oneof because as oneofs cannot 24 | // contain repeated fields it would require another extra set of messages. 25 | 26 | // Version number. 27 | // 28 | // In version 0, if the "repeated xxx" representations contain only one 29 | // element, that element is repeated to fill the shape. This makes it easy 30 | // to represent a constant Tensor with a single value. 31 | int32 version_number = 3; 32 | 33 | // Serialized raw tensor content from either Tensor::AsProtoTensorContent or 34 | // memcpy in tensorflow::grpc::EncodeTensorToByteBuffer. This representation 35 | // can be used for all tensor types. The purpose of this representation is to 36 | // reduce serialization overhead during RPC call by avoiding serialization of 37 | // many repeated small items. 38 | bytes tensor_content = 4; 39 | 40 | // Type specific representations that make it easy to create tensor protos in 41 | // all languages. Only the representation corresponding to "dtype" can 42 | // be set. The values hold the flattened representation of the tensor in 43 | // row major order. 44 | 45 | // DT_HALF, DT_BFLOAT16. Note that since protobuf has no int16 type, we'll 46 | // have some pointless zero padding for each value here. 47 | repeated int32 half_val = 13 [packed = true]; 48 | 49 | // DT_FLOAT. 50 | repeated float float_val = 5 [packed = true]; 51 | 52 | // DT_DOUBLE. 53 | repeated double double_val = 6 [packed = true]; 54 | 55 | // DT_INT32, DT_INT16, DT_INT8, DT_UINT8. 56 | repeated int32 int_val = 7 [packed = true]; 57 | 58 | // DT_STRING 59 | repeated bytes string_val = 8; 60 | 61 | // DT_COMPLEX64. scomplex_val(2*i) and scomplex_val(2*i+1) are real 62 | // and imaginary parts of i-th single precision complex. 63 | repeated float scomplex_val = 9 [packed = true]; 64 | 65 | // DT_INT64 66 | repeated int64 int64_val = 10 [packed = true]; 67 | 68 | // DT_BOOL 69 | repeated bool bool_val = 11 [packed = true]; 70 | 71 | // DT_COMPLEX128. dcomplex_val(2*i) and dcomplex_val(2*i+1) are real 72 | // and imaginary parts of i-th double precision complex. 73 | repeated double dcomplex_val = 12 [packed = true]; 74 | 75 | // DT_RESOURCE 76 | repeated ResourceHandleProto resource_handle_val = 14; 77 | 78 | // DT_VARIANT 79 | repeated VariantTensorDataProto variant_val = 15; 80 | 81 | // DT_UINT32 82 | repeated uint32 uint32_val = 16 [packed = true]; 83 | 84 | // DT_UINT64 85 | repeated uint64 uint64_val = 17 [packed = true]; 86 | } 87 | 88 | // Protocol buffer representing the serialization format of DT_VARIANT tensors. 89 | message VariantTensorDataProto { 90 | // Name of the type of objects being serialized. 91 | string type_name = 1; 92 | // Portions of the object that are not Tensors. 93 | bytes metadata = 2; 94 | // Tensors contained within objects being serialized. 95 | repeated TensorProto tensors = 3; 96 | } 97 | -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/tensorflow/core/framework/tensor_shape.proto: -------------------------------------------------------------------------------- 1 | // Protocol buffer representing the shape of tensors. 2 | 3 | syntax = "proto3"; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "TensorShapeProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/tensor_shape_go_proto"; 9 | 10 | package tensorflow; 11 | 12 | // Dimensions of a tensor. 13 | message TensorShapeProto { 14 | // One dimension of the tensor. 15 | message Dim { 16 | // Size of the tensor in that dimension. 17 | // This value must be >= -1, but values of -1 are reserved for "unknown" 18 | // shapes (values of -1 mean "unknown" dimension). Certain wrappers 19 | // that work with TensorShapeProto may fail at runtime when deserializing 20 | // a TensorShapeProto containing a dim value of -1. 21 | int64 size = 1; 22 | 23 | // Optional name of the tensor dimension. 24 | string name = 2; 25 | }; 26 | 27 | // Dimensions of the tensor, such as {"input", 30}, {"output", 40} 28 | // for a 30 x 40 2D tensor. If an entry has size -1, this 29 | // corresponds to a dimension of unknown size. The names are 30 | // optional. 31 | // 32 | // The order of entries in "dim" matters: It indicates the layout of the 33 | // values in the tensor in-memory representation. 34 | // 35 | // The first entry in "dim" is the outermost dimension used to layout the 36 | // values, the last entry is the innermost dimension. This matches the 37 | // in-memory layout of RowMajor Eigen tensors. 38 | // 39 | // If "dim.size()" > 0, "unknown_rank" must be false. 40 | repeated Dim dim = 2; 41 | 42 | // If true, the number of dimensions in the shape is unknown. 43 | // 44 | // If true, "dim.size()" must be 0. 45 | bool unknown_rank = 3; 46 | }; 47 | -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/tensorflow/core/framework/types.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "TypesProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/types_go_proto"; 9 | 10 | // (== suppress_warning documentation-presence ==) 11 | // LINT.IfChange 12 | enum DataType { 13 | // Not a legal value for DataType. Used to indicate a DataType field 14 | // has not been set. 15 | DT_INVALID = 0; 16 | 17 | // Data types that all computation devices are expected to be 18 | // capable to support. 19 | DT_FLOAT = 1; 20 | DT_DOUBLE = 2; 21 | DT_INT32 = 3; 22 | DT_UINT8 = 4; 23 | DT_INT16 = 5; 24 | DT_INT8 = 6; 25 | DT_STRING = 7; 26 | DT_COMPLEX64 = 8; // Single-precision complex 27 | DT_INT64 = 9; 28 | DT_BOOL = 10; 29 | DT_QINT8 = 11; // Quantized int8 30 | DT_QUINT8 = 12; // Quantized uint8 31 | DT_QINT32 = 13; // Quantized int32 32 | DT_BFLOAT16 = 14; // Float32 truncated to 16 bits. Only for cast ops. 33 | DT_QINT16 = 15; // Quantized int16 34 | DT_QUINT16 = 16; // Quantized uint16 35 | DT_UINT16 = 17; 36 | DT_COMPLEX128 = 18; // Double-precision complex 37 | DT_HALF = 19; 38 | DT_RESOURCE = 20; 39 | DT_VARIANT = 21; // Arbitrary C++ data types 40 | DT_UINT32 = 22; 41 | DT_UINT64 = 23; 42 | 43 | // Do not use! These are only for parameters. Every enum above 44 | // should have a corresponding value below (verified by types_test). 45 | DT_FLOAT_REF = 101; 46 | DT_DOUBLE_REF = 102; 47 | DT_INT32_REF = 103; 48 | DT_UINT8_REF = 104; 49 | DT_INT16_REF = 105; 50 | DT_INT8_REF = 106; 51 | DT_STRING_REF = 107; 52 | DT_COMPLEX64_REF = 108; 53 | DT_INT64_REF = 109; 54 | DT_BOOL_REF = 110; 55 | DT_QINT8_REF = 111; 56 | DT_QUINT8_REF = 112; 57 | DT_QINT32_REF = 113; 58 | DT_BFLOAT16_REF = 114; 59 | DT_QINT16_REF = 115; 60 | DT_QUINT16_REF = 116; 61 | DT_UINT16_REF = 117; 62 | DT_COMPLEX128_REF = 118; 63 | DT_HALF_REF = 119; 64 | DT_RESOURCE_REF = 120; 65 | DT_VARIANT_REF = 121; 66 | DT_UINT32_REF = 122; 67 | DT_UINT64_REF = 123; 68 | } 69 | // LINT.ThenChange( 70 | // https://www.tensorflow.org/code/tensorflow/c/tf_datatype.h, 71 | // https://www.tensorflow.org/code/tensorflow/go/tensor.go, 72 | // https://www.tensorflow.org/code/tensorflow/core/framework/tensor.cc, 73 | // https://www.tensorflow.org/code/tensorflow/core/framework/types.h, 74 | // https://www.tensorflow.org/code/tensorflow/core/framework/types.cc, 75 | // https://www.tensorflow.org/code/tensorflow/python/framework/dtypes.py, 76 | // https://www.tensorflow.org/code/tensorflow/python/framework/function.py) 77 | -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/tensorflow/core/protobuf/named_tensor.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | import "tensorflow/core/framework/tensor.proto"; 6 | 7 | option cc_enable_arenas = true; 8 | option java_outer_classname = "NamedTensorProtos"; 9 | option java_multiple_files = true; 10 | option java_package = "org.tensorflow.framework"; 11 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; 12 | 13 | // A pair of tensor name and tensor values. 14 | message NamedTensorProto { 15 | // Name of the tensor. 16 | string name = 1; 17 | 18 | // The client can populate a TensorProto using a tensorflow::Tensor`, or 19 | // directly using the protobuf field accessors. 20 | // 21 | // The client specifies whether the returned tensor values should be 22 | // filled tensor fields (float_val, int_val, etc.) or encoded in a 23 | // compact form in tensor.tensor_content. 24 | TensorProto tensor = 2; 25 | } -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/tensorflow_serving/apis/classification.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | 5 | import "tensorflow_serving/apis/input.proto"; 6 | import "tensorflow_serving/apis/model.proto"; 7 | 8 | option cc_enable_arenas = true; 9 | 10 | // A single class. 11 | message Class { 12 | // Label or name of the class. 13 | string label = 1; 14 | // Score for this class (e.g., the probability the item belongs to this 15 | // class). As per the proto3 default-value semantics, if the score is missing, 16 | // it should be treated as 0. 17 | float score = 2; 18 | } 19 | 20 | // List of classes for a single item (tensorflow.Example). 21 | message Classifications { 22 | repeated Class classes = 1; 23 | } 24 | 25 | // Contains one result per input example, in the same order as the input in 26 | // ClassificationRequest. 27 | message ClassificationResult { 28 | repeated Classifications classifications = 1; 29 | } 30 | 31 | // RPC Interfaces 32 | 33 | message ClassificationRequest { 34 | // Model Specification. If version is not specified, will use the latest 35 | // (numerical) version. 36 | ModelSpec model_spec = 1; 37 | 38 | // Input data. 39 | tensorflow.serving.Input input = 2; 40 | } 41 | 42 | message ClassificationResponse { 43 | // Effective Model Specification used for classification. 44 | ModelSpec model_spec = 2; 45 | 46 | // Result of the classification. 47 | ClassificationResult result = 1; 48 | } -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/tensorflow_serving/apis/inference.proto: -------------------------------------------------------------------------------- 1 | // This file contains messages for various machine learning inferences 2 | // such as regression and classification. 3 | // 4 | // In many applications more than one type of inference is desired for a single 5 | // input. For example, given meteorologic data an application may want to 6 | // perform a classification to determine if we should expect rain, snow or sun 7 | // and also perform a regression to predict the temperature. 8 | // Sharing the single input data between two inference tasks can be accomplished 9 | // using MultiInferenceRequest and MultiInferenceResponse. 10 | 11 | syntax = "proto3"; 12 | 13 | option cc_enable_arenas = true; 14 | 15 | import "tensorflow_serving/apis/classification.proto"; 16 | import "tensorflow_serving/apis/input.proto"; 17 | import "tensorflow_serving/apis/model.proto"; 18 | import "tensorflow_serving/apis/regression.proto"; 19 | 20 | package tensorflow.serving; 21 | 22 | // Inference request such as classification, regression, etc... 23 | message InferenceTask { 24 | // Model Specification. If version is not specified, will use the latest 25 | // (numerical) version. 26 | // All ModelSpecs in a MultiInferenceRequest must access the same model name. 27 | ModelSpec model_spec = 1; 28 | 29 | // Signature's method_name. Should be one of the method names defined in 30 | // third_party/tensorflow/python/saved_model/signature_constants.py. 31 | // e.g. "tensorflow/serving/classify". 32 | string method_name = 2; 33 | } 34 | 35 | // Inference result, matches the type of request or is an error. 36 | message InferenceResult { 37 | ModelSpec model_spec = 1; 38 | 39 | oneof result { 40 | ClassificationResult classification_result = 2; 41 | RegressionResult regression_result = 3; 42 | } 43 | } 44 | 45 | // Inference request containing one or more requests. 46 | message MultiInferenceRequest { 47 | // Inference tasks. 48 | repeated InferenceTask tasks = 1; 49 | 50 | // Input data. 51 | Input input = 2; 52 | } 53 | 54 | // Inference request containing one or more responses. 55 | message MultiInferenceResponse { 56 | // List of results; one for each InferenceTask in the request, returned in the 57 | // same order as the request. 58 | repeated InferenceResult results = 1; 59 | } -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/tensorflow_serving/apis/input.proto: -------------------------------------------------------------------------------- 1 | // Input used in serving APIs. Based on the tensorflow.Example family of 2 | // feature representations. 3 | 4 | syntax = "proto3"; 5 | 6 | option cc_enable_arenas = true; 7 | 8 | import "tensorflow/core/example/example.proto"; 9 | 10 | package tensorflow.serving; 11 | 12 | // Specifies one or more fully independent input Examples. 13 | // See examples at: 14 | // https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/example.proto 15 | message ExampleList { 16 | repeated tensorflow.Example examples = 1; 17 | } 18 | 19 | // Specifies one or more independent input Examples, with a common context 20 | // Example. 21 | // 22 | // The common use case for context is to cleanly and optimally specify some 23 | // features that are common across multiple examples. 24 | // 25 | // See example below with a search query as the context and multiple restaurants 26 | // to perform some inference on. 27 | // 28 | // context: { 29 | // features: { 30 | // feature: { 31 | // key : "query" 32 | // value: { 33 | // bytes_list: { 34 | // value: [ "pizza" ] 35 | // } 36 | // } 37 | // } 38 | // } 39 | // } 40 | // examples: { 41 | // features: { 42 | // feature: { 43 | // key : "cuisine" 44 | // value: { 45 | // bytes_list: { 46 | // value: [ "Pizzeria" ] 47 | // } 48 | // } 49 | // } 50 | // } 51 | // } 52 | // examples: { 53 | // features: { 54 | // feature: { 55 | // key : "cuisine" 56 | // value: { 57 | // bytes_list: { 58 | // value: [ "Taqueria" ] 59 | // } 60 | // } 61 | // } 62 | // } 63 | // } 64 | // 65 | // Implementations of ExampleListWithContext merge the context Example into each 66 | // of the Examples. Note that feature keys must not be duplicated between the 67 | // Examples and context Example, or the behavior is undefined. 68 | // 69 | // See also: 70 | // tensorflow/core/example/example.proto 71 | // https://developers.google.com/protocol-buffers/docs/proto3#maps 72 | message ExampleListWithContext { 73 | repeated tensorflow.Example examples = 1; 74 | tensorflow.Example context = 2; 75 | } 76 | 77 | message Input { 78 | oneof kind { 79 | ExampleList example_list = 1 [lazy = true]; 80 | ExampleListWithContext example_list_with_context = 2 [lazy = true]; 81 | } 82 | } -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/tensorflow_serving/apis/logging.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | 5 | import "tensorflow_serving/apis/model.proto"; 6 | import "tensorflow_serving/config/logging_config.proto"; 7 | 8 | option cc_enable_arenas = true; 9 | 10 | // Metadata logged along with the request logs. 11 | message LogMetadata { 12 | ModelSpec model_spec = 1; 13 | SamplingConfig sampling_config = 2; 14 | // List of tags used to load the relevant MetaGraphDef from SavedModel. 15 | repeated string saved_model_tags = 3; 16 | // TODO(b/33279154): Add more metadata as mentioned in the bug. 17 | } 18 | -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/tensorflow_serving/apis/model.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | 5 | import "google/protobuf/wrappers.proto"; 6 | 7 | option cc_enable_arenas = true; 8 | 9 | // Metadata for an inference request such as the model name and version. 10 | message ModelSpec { 11 | // Required servable name. 12 | string name = 1; 13 | 14 | // Optional choice of which version of the model to use. 15 | // 16 | // Expected to be left unset in the common case. Should be specified when 17 | // there is a strong version consistency requirement (e.g. when the model 18 | // signature changes across versions and requests need to be 19 | // version-specific). 20 | // 21 | // When left unspecified, the system will serve the best available version. 22 | // This is typically the latest version, though during version transitions, 23 | // notably when serving on a fleet of instances, may be either the previous or 24 | // new version. 25 | oneof version_choice { 26 | // Use this specific version number. 27 | google.protobuf.Int64Value version = 2; 28 | 29 | // Use the version associated with the given label. 30 | string version_label = 4; 31 | } 32 | 33 | // A named signature to evaluate. If unspecified, the default signature will 34 | // be used. 35 | string signature_name = 3; 36 | } -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/tensorflow_serving/apis/predict.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | 5 | import "tensorflow/core/framework/tensor.proto"; 6 | import "tensorflow_serving/apis/model.proto"; 7 | 8 | option cc_enable_arenas = true; 9 | 10 | // PredictRequest specifies which TensorFlow model to run, as well as 11 | // how inputs are mapped to tensors and how outputs are filtered before 12 | // returning to user. 13 | message PredictRequest { 14 | // Model Specification. If version is not specified, will use the latest 15 | // (numerical) version. 16 | ModelSpec model_spec = 1; 17 | 18 | // Input tensors. 19 | // Names of input tensor are alias names. The mapping from aliases to real 20 | // input tensor names is stored in the SavedModel export as a prediction 21 | // SignatureDef under the 'inputs' field. 22 | map inputs = 2; 23 | 24 | // Output filter. 25 | // Names specified are alias names. The mapping from aliases to real output 26 | // tensor names is stored in the SavedModel export as a prediction 27 | // SignatureDef under the 'outputs' field. 28 | // Only tensors specified here will be run/fetched and returned, with the 29 | // exception that when none is specified, all tensors specified in the 30 | // named signature will be run/fetched and returned. 31 | repeated string output_filter = 3; 32 | 33 | // Reserved field 4. 34 | reserved 4; 35 | 36 | // Options for streaming requests to control how multiple requests/responses 37 | // are handled within a single stream. 38 | PredictStreamedOptions predict_streamed_options = 5; 39 | 40 | // Client identifier to group requests belonging to a specific entity. 41 | // Example entities can be product ids, service names, user ids etc. 42 | // Servers can use this to optimize placement, caching and colocation. 43 | // TODO(b/329897437): Migrate to client_id in RequestOptions. 44 | optional bytes client_id = 6; 45 | 46 | // Options for PredictRequest. 47 | message RequestOptions { 48 | // Client identifier to group requests belonging to a specific entity. 49 | // Example entities can be product ids, service names, user ids etc. 50 | // Servers can use this to optimize placement, caching and colocation. 51 | optional bytes client_id = 1; 52 | 53 | // Deterministic mode for the request. When specified, model servers will 54 | // reduce numeric instability based on different mode selections. 55 | enum DeterministicMode { 56 | DETERMINISTIC_MODE_UNSPECIFIED = 0; 57 | 58 | // Only supported in disaggregated serving. When set, the request will be 59 | // pinned to a fixed decoder slot index that's deterministic across 60 | // processes. 61 | FIXED_DECODER_SLOT = 1; 62 | } 63 | 64 | optional DeterministicMode deterministic_mode = 2; 65 | } 66 | 67 | optional RequestOptions request_options = 7; 68 | } 69 | 70 | // Options only used for streaming requests that control how inputs/ouputs are 71 | // handled in the stream. 72 | message PredictStreamedOptions { 73 | // Request state used to handle splitting of requests. NONE is the 74 | // default when the stream request is not split. 75 | // 76 | // SPLIT is used when multiple streamed requests are used to generate a 77 | // logical request. END_SPLIT should be called for the last split of the 78 | // logical request. NONE can not be interspersed with SPLIT before END_SPLIT 79 | // is called. If another request is sent on the same 80 | // stream after END_SPLIT, it can be any of the RequestState since a new 81 | // logical request has started. If END_SPLIT is called on its own the 82 | // behavior is the same as NONE. 83 | // 84 | // Some examples with a mix of request states and the logical request. 85 | // 86 | // Example 1 : 87 | // SPLIT 88 | // SPLIT 89 | // END_SPLIT 90 | // 91 | // Will be treated as a single logical request. 92 | // 93 | // Example 2: 94 | // NONE 95 | // END_SPLIT 96 | // NONE 97 | // 98 | // Will be treated as three logical requests (1. NONE 2. END_SPLIT, 3. NONE) 99 | // 100 | // Example 3: 101 | // SPLIT 102 | // SPLIT 103 | // 104 | // Invalid because END_SPLIT is never call. 105 | // 106 | // Example 4: 107 | // SPLIT 108 | // NONE 109 | // SPLIT 110 | // END_SPLIT 111 | // 112 | // Invalid because is interspersed with SPLIT. 113 | // 114 | // Example 5: 115 | // SPLIT 116 | // END_SPLIT 117 | // SPLIT 118 | // SPLIT 119 | // END_SPLIT 120 | // 121 | // Will be treated as two logical requests (1. SPLIT, END_SPLIT 2. SPLIT, 122 | // SPLIT, END_SPLIT) 123 | 124 | enum RequestState { 125 | NONE = 0; 126 | SPLIT = 1; 127 | END_SPLIT = 2; 128 | } 129 | 130 | // Request state used to handle segmentation of requests. 131 | RequestState request_state = 1; 132 | 133 | // Input tensors split dimensions. 134 | // Defines the dimension used to split input tensors specified 135 | // in PredictRequest.inputs. The dimension will be used 136 | // for concatenation of multiple SPLIT requests. 137 | // 138 | // For input tensor in PredictRequest.inputs that are not contained in this 139 | // map, the tensors from the first SPLIT request will be used. 140 | // 141 | // For example, with an original input tensor of [[1, 2, 3, 4], [5, 6, 7, 8]]. 142 | // 143 | // For a split dimension of 0 and two requests (SPLIT and END_SPLIT), the 144 | // input tensors for request 1 should be [1, 2, 3, 4] and request 2 should be 145 | // be [5, 6, 7, 8]. 146 | // 147 | // For a split dimension of 1 and two requests (SPLIT and END_SPLIT), the 148 | // input tensors for request 1 should be [[1, 2], [5, 6]] and request 2 should 149 | // be [[3, 4], [7, 8]]. 150 | map split_dimensions = 2; 151 | } 152 | 153 | // Response for PredictRequest on successful run. 154 | message PredictResponse { 155 | // Effective Model Specification used to process PredictRequest. 156 | ModelSpec model_spec = 2; 157 | 158 | // Output tensors. 159 | map outputs = 1; 160 | } -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/tensorflow_serving/apis/prediction_log.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | 5 | import "tensorflow_serving/apis/classification.proto"; 6 | import "tensorflow_serving/apis/inference.proto"; 7 | import "tensorflow_serving/apis/logging.proto"; 8 | import "tensorflow_serving/apis/predict.proto"; 9 | import "tensorflow_serving/apis/regression.proto"; 10 | import "tensorflow_serving/apis/session_service.proto"; 11 | 12 | option cc_enable_arenas = true; 13 | 14 | message ClassifyLog { 15 | ClassificationRequest request = 1; 16 | ClassificationResponse response = 2; 17 | } 18 | 19 | message RegressLog { 20 | RegressionRequest request = 1; 21 | RegressionResponse response = 2; 22 | } 23 | 24 | message PredictLog { 25 | PredictRequest request = 1; 26 | PredictResponse response = 2; 27 | } 28 | 29 | message PredictStreamedLog { 30 | repeated PredictRequest request = 1; 31 | repeated PredictResponse response = 2; 32 | } 33 | 34 | message MultiInferenceLog { 35 | MultiInferenceRequest request = 1; 36 | MultiInferenceResponse response = 2; 37 | } 38 | 39 | message SessionRunLog { 40 | SessionRunRequest request = 1; 41 | SessionRunResponse response = 2; 42 | } 43 | 44 | // Logged model inference request. 45 | message PredictionLog { 46 | LogMetadata log_metadata = 1; 47 | oneof log_type { 48 | ClassifyLog classify_log = 2; 49 | RegressLog regress_log = 3; 50 | PredictLog predict_log = 6; 51 | PredictStreamedLog predict_streamed_log = 7; 52 | MultiInferenceLog multi_inference_log = 4; 53 | SessionRunLog session_run_log = 5; 54 | } 55 | } -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/tensorflow_serving/apis/regression.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | 5 | import "tensorflow_serving/apis/input.proto"; 6 | import "tensorflow_serving/apis/model.proto"; 7 | 8 | option cc_enable_arenas = true; 9 | 10 | // Regression result for a single item (tensorflow.Example). 11 | message Regression { 12 | float value = 1; 13 | } 14 | 15 | // Contains one result per input example, in the same order as the input in 16 | // RegressionRequest. 17 | message RegressionResult { 18 | repeated Regression regressions = 1; 19 | } 20 | 21 | // RPC interfaces. 22 | 23 | message RegressionRequest { 24 | // Model Specification. If version is not specified, will use the latest 25 | // (numerical) version. 26 | ModelSpec model_spec = 1; 27 | 28 | // Input data. 29 | tensorflow.serving.Input input = 2; 30 | } 31 | 32 | message RegressionResponse { 33 | // Effective Model Specification used for regression. 34 | ModelSpec model_spec = 2; 35 | 36 | RegressionResult result = 1; 37 | } -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/tensorflow_serving/apis/session_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | 5 | import "tensorflow/core/protobuf/named_tensor.proto"; 6 | import "tensorflow_serving/apis/model.proto"; 7 | 8 | option cc_enable_arenas = true; 9 | 10 | message SessionRunRequest { 11 | // Model Specification. If version is not specified, will use the latest 12 | // (numerical) version. 13 | ModelSpec model_spec = 1; 14 | 15 | // Tensors to be fed in the step. Each feed is a named tensor. 16 | repeated NamedTensorProto feed = 2; 17 | 18 | // Fetches. A list of tensor names. The caller expects a tensor to 19 | // be returned for each fetch[i] (see RunResponse.tensor). The 20 | // order of specified fetches does not change the execution order. 21 | repeated string fetch = 3; 22 | 23 | // Target Nodes. A list of node names. The named nodes will be run 24 | // to but their outputs will not be fetched. 25 | repeated string target = 4; 26 | 27 | // If true, treat names in feed/fetch/target as alias names than actual tensor 28 | // names (that appear in the TF graph). Alias names are resolved to actual 29 | // names using `SignatureDef` in SavedModel associated with the model. 30 | bool tensor_name_is_alias = 6; 31 | } 32 | 33 | message SessionRunResponse { 34 | // Effective Model Specification used for session run. 35 | ModelSpec model_spec = 3; 36 | 37 | // NOTE: The order of the returned tensors may or may not match 38 | // the fetch order specified in RunRequest. 39 | repeated NamedTensorProto tensor = 1; 40 | } -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/tensorflow_serving/config/log_collector_config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | option cc_enable_arenas = true; 5 | 6 | message LogCollectorConfig { 7 | // Identifies the type of the LogCollector we will use to collect these logs. 8 | string type = 1; 9 | 10 | // The prefix to use for the filenames of the logs. 11 | string filename_prefix = 2; 12 | } -------------------------------------------------------------------------------- /modules/core/src/main/protobuf/tensorflow_serving/config/logging_config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | 5 | import "tensorflow_serving/config/log_collector_config.proto"; 6 | 7 | option cc_enable_arenas = true; 8 | 9 | message SamplingConfig { 10 | // Requests will be logged uniformly at random with this probability. 11 | // Valid range: [0, 1.0]. 12 | double sampling_rate = 1; 13 | 14 | // Attributes of requests that can be optionally sampled. 15 | // Note: Enabling more attributes will increase logging storage requirements. 16 | enum Attributes { 17 | ATTR_DEFAULT = 0x0; 18 | ATTR_REQUEST_ORIGIN = 0x1; 19 | ATTR_REQUEST_CRITICALITY = 0x2; 20 | } 21 | // Bitwise OR of above attributes 22 | int32 attributes = 2; 23 | } 24 | 25 | // Configuration for logging query/responses. 26 | message LoggingConfig { 27 | LogCollectorConfig log_collector_config = 1; 28 | SamplingConfig sampling_config = 2; 29 | } -------------------------------------------------------------------------------- /modules/core/src/main/scala/tfr/Parsable.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Spotify AB. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an 12 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | * KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations 15 | * under the License. 16 | */ 17 | package tfr 18 | 19 | import cats.data.Kleisli 20 | import cats.effect.Sync 21 | import org.tensorflow.example.Example 22 | import tensorflow.serving.PredictionLogOuterClass.PredictionLog 23 | 24 | trait Parsable[T]: 25 | extension [F[_]: Sync](x: Array[Byte]) def parse: F[T] 26 | 27 | def parser[F[_]: Sync]: Kleisli[F, Array[Byte], T] 28 | 29 | object Parsable: 30 | given tfExampleParsable: Parsable[Example] with 31 | extension [F[_]](x: Array[Byte]) 32 | def parse(using sync: Sync[F]): F[Example] = 33 | sync.delay(Example.parseFrom(x)) 34 | 35 | override def parser[F[_]](using 36 | sync: Sync[F] 37 | ): Kleisli[F, Array[Byte], Example] = 38 | Kleisli(_.parse) 39 | 40 | given tfPredictionLogParsable: Parsable[PredictionLog] with 41 | extension [F[_]](x: Array[Byte]) 42 | def parse(using sync: Sync[F]): F[PredictionLog] = 43 | sync.delay(PredictionLog.parseFrom(x)) 44 | 45 | override def parser[F[_]](using 46 | sync: Sync[F] 47 | ): Kleisli[F, Array[Byte], PredictionLog] = 48 | Kleisli(_.parse) 49 | -------------------------------------------------------------------------------- /modules/core/src/main/scala/tfr/Resources.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Spotify AB. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an 12 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | * KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations 15 | * under the License. 16 | */ 17 | package tfr 18 | 19 | import java.io.InputStream 20 | import java.net.URI 21 | import java.nio.channels.Channels 22 | import java.nio.file.{Files, Paths} 23 | 24 | import cats.effect.{Resource, Sync} 25 | import com.google.cloud.storage.StorageOptions 26 | import java.util.zip.GZIPInputStream 27 | 28 | object Resources: 29 | 30 | final def stdin[F[_]](using sync: Sync[F]): Resource[F, InputStream] = 31 | Resource.fromAutoCloseable(sync.delay(System.in)) 32 | 33 | final def file[F[_]]( 34 | path: String 35 | )(using sync: Sync[F]): Resource[F, InputStream] = 36 | Resource 37 | .fromAutoCloseable(sync.delay { 38 | URI.create(path) match { 39 | case gcsUri if gcsUri.getScheme == "gs" => 40 | val service = StorageOptions.getDefaultInstance.getService 41 | val blobPath = gcsUri.getPath.tail match { 42 | case s if s.endsWith("/") => s.init 43 | case s => s 44 | } 45 | val readChannel = service.reader(gcsUri.getHost, blobPath) 46 | Channels.newInputStream(readChannel) 47 | case uri => 48 | Files.newInputStream(Paths.get(uri.getPath)) 49 | } 50 | }) 51 | .map { input => 52 | path match { 53 | case _ if path.endsWith(".gz") => new GZIPInputStream(input) 54 | case _ => input 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /modules/core/src/main/scala/tfr/TFRecord.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Spotify AB. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an 12 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | * KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations 15 | * under the License. 16 | */ 17 | package tfr 18 | 19 | import java.io.InputStream 20 | import java.nio.{ByteBuffer, ByteOrder} 21 | 22 | import scala.util.Try 23 | 24 | import cats.data.Kleisli 25 | import cats.effect.Sync 26 | import com.google.common.hash.Hashing 27 | import fs2.{Pull, Stream} 28 | import cats.effect.Resource 29 | import scala.annotation.tailrec 30 | 31 | object TFRecord: 32 | enum Error: 33 | case EmptyHeader, InvalidCrc32, ReadError 34 | 35 | private[this] val HeaderLength: Int = 36 | (java.lang.Long.SIZE + java.lang.Integer.SIZE) / java.lang.Byte.SIZE 37 | private[this] val FooterLength: Int = 38 | java.lang.Integer.SIZE / java.lang.Byte.SIZE 39 | private[this] val Crc32c = Hashing.crc32c() 40 | 41 | private def mask(crc: Int): Int = ((crc >>> 15) | (crc << 17)) + 0xa282ead8 42 | private def hashLong(x: Long): Int = mask(Crc32c.hashLong(x).asInt()) 43 | private def hashBytes(x: Array[Byte]): Int = mask(Crc32c.hashBytes(x).asInt()) 44 | 45 | private def reader_( 46 | checkCrc32: Boolean 47 | ): InputStream => Either[Error, Array[Byte]] = 48 | input => 49 | read(input, HeaderLength) 50 | .filterOrElse(_.nonEmpty, Error.EmptyHeader) 51 | .map { headerBytes => 52 | val buffer = 53 | ByteBuffer.wrap(headerBytes).order(ByteOrder.LITTLE_ENDIAN) 54 | (buffer.getLong, buffer.getInt) 55 | } 56 | .filterOrElse( 57 | header => !checkCrc32 || hashLong(header._1) == header._2, 58 | Error.InvalidCrc32 59 | ) 60 | .flatMap(header => read(input, header._1.toInt)) 61 | .filterOrElse( 62 | data => { 63 | read(input, FooterLength) 64 | .forall { footer => 65 | !checkCrc32 || hashBytes(data) == ByteBuffer 66 | .wrap(footer) 67 | .order(ByteOrder.LITTLE_ENDIAN) 68 | .getInt 69 | } 70 | }, 71 | Error.InvalidCrc32 72 | ) 73 | 74 | def reader[F[_]]( 75 | checkCrc32: Boolean 76 | )(using 77 | sync: Sync[F] 78 | ): Kleisli[F, InputStream, Either[Error, Array[Byte]]] = 79 | Kleisli(input => sync.delay(reader_(checkCrc32).apply(input))) 80 | 81 | def typedReader[T, F[_]]( 82 | checkCrc32: Boolean 83 | )(using 84 | parsable: Parsable[T], 85 | sync: Sync[F] 86 | ): Kleisli[F, InputStream, Either[Error, T]] = 87 | TFRecord.reader[F](checkCrc32).andThen { 88 | case Left(value) => 89 | sync.delay(Left(value): Either[Error, T]) 90 | case Right(value) => 91 | parsable.parser 92 | .andThen(ex => sync.delay(Right(ex): Either[Error, T])) 93 | .run(value) 94 | } 95 | 96 | def streamReader[F[_], A]( 97 | reader: Kleisli[F, InputStream, Either[Error, A]] 98 | ): Kleisli[Stream[F, *], InputStream, Either[Error, A]] = 99 | Kleisli { input => 100 | Stream 101 | .repeatEval(reader.apply(input)) 102 | .repeatPull(_.uncons1.flatMap { 103 | case None => Pull.pure(None) 104 | case Some((Left(Error.EmptyHeader), _)) => Pull.pure(None) 105 | case Some((elem, stream)) => Pull.output1(elem).as(Some(stream)) 106 | }) 107 | } 108 | 109 | def resourceReader[F[_]: Sync, A]( 110 | reader: Kleisli[F, InputStream, Either[Error, A]] 111 | ): Kleisli[Stream[F, *], Resource[F, InputStream], Either[Error, A]] = 112 | Kleisli { resource => 113 | Stream.resource(resource).flatMap(streamReader(reader).run) 114 | } 115 | 116 | private def read( 117 | input: InputStream, 118 | length: Int 119 | ): Either[Error, Array[Byte]] = { 120 | @tailrec 121 | def read(off: Int, data: Array[Byte]): Array[Byte] = { 122 | val r = input.read(data, off, data.length - off) 123 | if (r > 0) then read(off + r, data) 124 | else if off == 0 then Array.emptyByteArray 125 | else data 126 | } 127 | 128 | Try(read(0, Array.ofDim[Byte](length))).toEither.left.map(_ => 129 | Error.ReadError 130 | ) 131 | } 132 | -------------------------------------------------------------------------------- /modules/core/src/main/scala/tfr/instances/ExampleInstances.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Spotify AB. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an 12 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | * KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations 15 | * under the License. 16 | */ 17 | package tfr.instances 18 | 19 | import cats.Show 20 | import io.circe.{Printer => CircePrinter, Encoder, Json} 21 | import java.lang.{Float => JFloat, Long => JLong} 22 | 23 | import com.google.protobuf.ByteString 24 | import org.tensorflow.example.{ 25 | BytesList, 26 | Example, 27 | Feature, 28 | Features, 29 | FloatList, 30 | Int64List 31 | } 32 | 33 | import scala.collection.mutable 34 | import scala.jdk.CollectionConverters._ 35 | 36 | trait ExampleInstances extends ExampleShowInstances with ExampleEncoderInstances 37 | 38 | trait ExampleShowInstances: 39 | given showExample(using encoder: Encoder[Example]): Show[Example] with 40 | private[this] val Printer = 41 | CircePrinter.noSpaces.copy(dropNullValues = true) 42 | 43 | override def show(t: Example): String = 44 | Printer.print(Encoder[Example].apply(t)) 45 | 46 | trait ExampleEncoderInstances extends ProtobufEncoderInstances: 47 | 48 | given bytesListEncoder: Encoder[BytesList] with 49 | override def apply(a: BytesList): Json = 50 | Json.obj( 51 | "value" -> Encoder 52 | .encodeIterable[ByteString, Iterable] 53 | .contramap[BytesList](_.getValueList.asScala) 54 | .apply(a) 55 | ) 56 | 57 | given floatListEncoder: Encoder[FloatList] with 58 | override def apply(a: FloatList): Json = 59 | Json.obj( 60 | "value" -> Encoder 61 | .encodeIterable[JFloat, Iterable] 62 | .contramap[FloatList](_.getValueList.asScala) 63 | .apply(a) 64 | ) 65 | 66 | given int64ListEncoder: Encoder[Int64List] with 67 | override def apply(a: Int64List): Json = 68 | Json.obj( 69 | "value" -> Encoder 70 | .encodeIterable[JLong, Iterable] 71 | .contramap[Int64List](_.getValueList.asScala) 72 | .apply(a) 73 | ) 74 | 75 | given featureEncoder: Encoder[Feature] with 76 | final def apply(f: Feature): Json = 77 | f match 78 | case _ if f.hasBytesList => 79 | Json.obj("bytesList" -> Encoder[BytesList].apply(f.getBytesList)) 80 | case _ if f.hasFloatList => 81 | Json.obj("floatList" -> Encoder[FloatList].apply(f.getFloatList)) 82 | case _ if f.hasInt64List => 83 | Json.obj("int64List" -> Encoder[Int64List].apply(f.getInt64List)) 84 | case _ => Json.Null 85 | 86 | given featuresEncoder: Encoder[Features] with 87 | override def apply(a: Features): Json = 88 | Json.obj( 89 | "feature" -> Encoder 90 | .encodeMapLike[String, Feature, mutable.Map] 91 | .contramap[Features](_.getFeatureMap.asScala) 92 | .apply(a) 93 | ) 94 | 95 | given exampleEncoder: Encoder[Example] with 96 | override def apply(a: Example): Json = 97 | Json.obj( 98 | "features" -> Encoder[Features] 99 | .contramap[Example](_.getFeatures) 100 | .apply(a) 101 | ) 102 | 103 | object flat: 104 | given featureEncoder: Encoder[Feature] with 105 | final def apply(f: Feature): Json = 106 | f match 107 | case _ if f.hasBytesList => Encoder[BytesList].apply(f.getBytesList) 108 | case _ if f.hasFloatList => Encoder[FloatList].apply(f.getFloatList) 109 | case _ if f.hasInt64List => Encoder[Int64List].apply(f.getInt64List) 110 | case _ => Json.Null 111 | 112 | given featuresEncoder: Encoder[Features] = Encoder 113 | .encodeMapLike[String, Feature, mutable.Map] 114 | .contramap(_.getFeatureMap.asScala) 115 | 116 | given exampleEncoder: Encoder[Example] = 117 | Encoder[Features].contramap(_.getFeatures) 118 | 119 | given bytesListEncoder: Encoder[BytesList] = Encoder 120 | .encodeIterable[ByteString, Iterable] 121 | .contramap(_.getValueList.asScala) 122 | 123 | given floatListEncoder: Encoder[FloatList] = 124 | Encoder.encodeIterable[JFloat, Iterable].contramap(_.getValueList.asScala) 125 | 126 | given int64ListEncoder: Encoder[Int64List] = 127 | Encoder.encodeIterable[JLong, Iterable].contramap(_.getValueList.asScala) 128 | -------------------------------------------------------------------------------- /modules/core/src/main/scala/tfr/instances/PredictionInstances.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Spotify AB. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an 12 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | * KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations 15 | * under the License. 16 | */ 17 | package tfr.instances 18 | 19 | import cats.Show 20 | import com.google.protobuf.util.JsonFormat 21 | import io.circe.{Encoder, Printer => CircePrinter} 22 | import tensorflow.serving.PredictionLogOuterClass.PredictionLog 23 | 24 | trait PredictionInstances 25 | extends PredictionLogShowInstances 26 | with PredictionLogEncoderInstances 27 | 28 | trait PredictionLogShowInstances: 29 | private[this] val Printer = 30 | CircePrinter.noSpaces.copy(dropNullValues = true) 31 | 32 | given showPredictionLog(using 33 | encoder: Encoder[PredictionLog] 34 | ): Show[PredictionLog] with 35 | override def show(t: PredictionLog): String = 36 | Printer.print(Encoder[PredictionLog].apply(t)) 37 | 38 | trait PredictionLogEncoderInstances extends ProtobufEncoderInstances: 39 | import io.circe._ 40 | import io.circe.parser._ 41 | 42 | private[this] val ProtoPrinter = 43 | JsonFormat.printer().omittingInsignificantWhitespace() 44 | 45 | given predictionLogEncoder: Encoder[PredictionLog] with 46 | override def apply(a: PredictionLog): Json = 47 | parse(ProtoPrinter.print(a)).getOrElse(Json.fromString("")) 48 | -------------------------------------------------------------------------------- /modules/core/src/main/scala/tfr/instances/ProtobufInstances.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Spotify AB. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an 12 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | * KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations 15 | * under the License. 16 | */ 17 | package tfr.instances 18 | 19 | import java.util.Base64 20 | 21 | import cats.Show 22 | import com.google.protobuf.{ByteString, Message} 23 | import com.google.protobuf.util.JsonFormat 24 | import io.circe.Encoder 25 | 26 | trait ProtobufInstances 27 | extends ProtobufShowInstances 28 | with ProtobufEncoderInstances 29 | 30 | trait ProtobufShowInstances: 31 | given showProtobuf[A <: Message]: Show[A] with 32 | private[this] val Printer = 33 | JsonFormat.printer().omittingInsignificantWhitespace() 34 | 35 | override def show(t: A): String = Printer.print(t) 36 | 37 | trait ProtobufEncoderInstances: 38 | given byteStringEncoder: Encoder[ByteString] = 39 | Encoder.encodeString.contramap { s => 40 | if s.isValidUtf8 then s.toStringUtf8 41 | else Base64.getEncoder.encodeToString(s.toByteArray) 42 | } 43 | -------------------------------------------------------------------------------- /modules/core/src/main/scala/tfr/instances/example/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Spotify AB. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an 12 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | * KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations 15 | * under the License. 16 | */ 17 | package tfr.instances 18 | 19 | package object example extends ExampleInstances 20 | -------------------------------------------------------------------------------- /modules/core/src/main/scala/tfr/instances/prediction/package.scala: -------------------------------------------------------------------------------- 1 | package tfr.instances 2 | 3 | package object prediction extends PredictionInstances 4 | -------------------------------------------------------------------------------- /modules/core/src/main/scala/tfr/instances/protobuf/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Spotify AB. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an 12 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | * KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations 15 | * under the License. 16 | */ 17 | package tfr.instances 18 | 19 | package object protobuf extends ProtobufInstances 20 | -------------------------------------------------------------------------------- /modules/core/src/test/resources/part-00000-of-00004.tfrecords: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spotify/tfreader/cdbff9a8a0a7184252a0c45a366db307f3cd8b98/modules/core/src/test/resources/part-00000-of-00004.tfrecords -------------------------------------------------------------------------------- /modules/core/src/test/resources/prediction_log.tfrecords: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spotify/tfreader/cdbff9a8a0a7184252a0c45a366db307f3cd8b98/modules/core/src/test/resources/prediction_log.tfrecords -------------------------------------------------------------------------------- /modules/core/src/test/scala/tfr/TfrSuite.scala: -------------------------------------------------------------------------------- 1 | package tfr 2 | 3 | import java.io.{File, FileInputStream} 4 | 5 | import cats.effect.IO 6 | import org.tensorflow.example.Example 7 | import tensorflow.serving.PredictionLogOuterClass.PredictionLog 8 | import cats.effect.unsafe.implicits.global 9 | 10 | class TfrSuite extends munit.FunSuite { 11 | 12 | private[this] val ExampleResourceFile = new File( 13 | getClass.getResource("/part-00000-of-00004.tfrecords").toURI 14 | ) 15 | 16 | private[this] val PredictionLogResourceFile = new File( 17 | getClass.getResource("/prediction_log.tfrecords").toURI 18 | ) 19 | 20 | test("read as example") { 21 | val example = TFRecord 22 | .typedReader[Example, IO](false) 23 | .apply(new FileInputStream(ExampleResourceFile)) 24 | .unsafeRunSync() 25 | 26 | assert(example.isRight) 27 | } 28 | 29 | test("read as example with crc32 check") { 30 | val example = TFRecord 31 | .typedReader[Example, IO](true) 32 | .apply(new FileInputStream(ExampleResourceFile)) 33 | .unsafeRunSync() 34 | 35 | assert(example.isRight) 36 | } 37 | 38 | test("read all as example") { 39 | val examples = TFRecord 40 | .streamReader(TFRecord.typedReader[Example, IO](false)) 41 | .apply(new FileInputStream(ExampleResourceFile)) 42 | .compile 43 | .toList 44 | .unsafeRunSync() 45 | .filter(_.isRight) 46 | 47 | assertEquals(examples.length, 3750) 48 | } 49 | 50 | test("read as prediction log") { 51 | val predictionLog = TFRecord 52 | .typedReader[PredictionLog, IO](false) 53 | .apply(new FileInputStream(PredictionLogResourceFile)) 54 | .unsafeRunSync() 55 | 56 | assert(predictionLog.isRight) 57 | } 58 | 59 | test("read as prediction log") { 60 | val predictionLog = TFRecord 61 | .typedReader[PredictionLog, IO](true) 62 | .apply(new FileInputStream(PredictionLogResourceFile)) 63 | .unsafeRunSync() 64 | 65 | assert(predictionLog.isRight) 66 | } 67 | 68 | test("read all as example") { 69 | val predictionLog = TFRecord 70 | .streamReader(TFRecord.typedReader[PredictionLog, IO](false)) 71 | .apply(new FileInputStream(PredictionLogResourceFile)) 72 | .compile 73 | .toList 74 | .unsafeRunSync() 75 | .filter(_.isRight) 76 | 77 | assertEquals(predictionLog.length, 10) 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /project/Dependencies.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | 3 | object Dependencies { 4 | lazy val protobufVersion = "4.27.1" 5 | lazy val protobuf = 6 | "com.google.protobuf" % "protobuf-java-util" % protobufVersion 7 | lazy val guava = "com.google.guava" % "guava" % "33.2.1-jre" 8 | lazy val catsCore = "org.typelevel" %% "cats-core" % "2.12.0" 9 | lazy val fs2Io = "co.fs2" %% "fs2-io" % "3.10.2" 10 | lazy val scallop = "org.rogach" %% "scallop" % "5.1.0" 11 | lazy val gcs = "com.google.cloud" % "google-cloud-storage" % "2.40.0" 12 | lazy val circeCore = "io.circe" %% "circe-core" % "0.14.7" 13 | lazy val munit = "org.scalameta" %% "munit" % "1.0.0" 14 | lazy val circeParser = "io.circe" %% "circe-parser" % "0.14.7" 15 | } 16 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.10.0 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2") 2 | addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.10.0") 3 | addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.2.1") 4 | addSbtPlugin("com.github.sbt" % "sbt-dynver" % "5.0.1") 5 | addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.10.0") 6 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.2.0") 7 | addSbtPlugin("com.github.sbt" % "sbt-protobuf" % "0.8.0") 8 | addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.12.0") 9 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import { } }: 2 | 3 | pkgs.mkShell { 4 | buildInputs = [ 5 | (pkgs.sbt.override { jre = pkgs.graalvm-ce; }) 6 | pkgs.graalvm-ce 7 | ]; 8 | } --------------------------------------------------------------------------------