├── .bazelrc ├── .bazelversion ├── BUILD ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── RELEASE.md ├── WORKSPACE ├── build_common.sh ├── build_pip_package.sh ├── configure.sh ├── docker-compose.yml ├── examples └── prensor_playground.ipynb ├── g3doc └── api_docs │ └── python │ ├── _toc.yaml │ ├── expression_impl.md │ ├── expression_impl │ ├── _api_cache.json │ ├── _toc.yaml │ ├── all_symbols.md │ ├── apply_schema.md │ ├── apply_schema │ │ └── apply_schema.md │ ├── broadcast.md │ ├── broadcast │ │ ├── broadcast.md │ │ └── broadcast_anonymous.md │ ├── depth_limit.md │ ├── depth_limit │ │ └── limit_depth.md │ ├── filter_expression.md │ ├── filter_expression │ │ ├── filter_by_child.md │ │ └── filter_by_sibling.md │ ├── index.md │ ├── index │ │ ├── get_index_from_end.md │ │ └── get_positional_index.md │ ├── map_prensor.md │ ├── map_prensor │ │ ├── map_ragged_tensor.md │ │ └── map_sparse_tensor.md │ ├── map_prensor_to_prensor.md │ ├── map_prensor_to_prensor │ │ ├── Schema.md │ │ ├── create_schema.md │ │ └── map_prensor_to_prensor.md │ ├── map_values.md │ ├── map_values │ │ ├── map_many_values.md │ │ ├── map_values.md │ │ └── map_values_anonymous.md │ ├── parquet.md │ ├── parquet │ │ ├── ParquetDataset.md │ │ ├── calculate_parquet_values.md │ │ └── create_expression_from_parquet_file.md │ ├── placeholder.md │ ├── placeholder │ │ ├── create_expression_from_schema.md │ │ └── get_placeholder_paths_from_graph.md │ ├── project.md │ ├── project │ │ └── project.md │ ├── promote.md │ ├── promote │ │ ├── PromoteChildExpression.md │ │ ├── PromoteExpression.md │ │ ├── promote.md │ │ └── promote_anonymous.md │ ├── promote_and_broadcast.md │ ├── promote_and_broadcast │ │ ├── promote_and_broadcast.md │ │ └── promote_and_broadcast_anonymous.md │ ├── proto.md │ ├── proto │ │ ├── DescriptorPool.md │ │ ├── FileDescriptorSet.md │ │ ├── ProtoExpression.md │ │ ├── TransformFn.md │ │ ├── create_expression_from_file_descriptor_set.md │ │ ├── create_expression_from_proto.md │ │ ├── create_transformed_field.md │ │ └── is_proto_expression.md │ ├── reroot.md │ ├── reroot │ │ ├── create_proto_index_field.md │ │ └── reroot.md │ ├── size.md │ ├── size │ │ ├── SizeExpression.md │ │ ├── has.md │ │ ├── size.md │ │ └── size_anonymous.md │ ├── slice_expression.md │ └── slice_expression │ │ ├── IndexValue.md │ │ └── slice_expression.md │ ├── s2t.md │ └── s2t │ ├── ChildNodeTensor.md │ ├── Expression.md │ ├── LeafNodeTensor.md │ ├── NodeTensor.md │ ├── Path.md │ ├── Prensor.md │ ├── RootNodeTensor.md │ ├── Step.md │ ├── _api_cache.json │ ├── _toc.yaml │ ├── all_symbols.md │ ├── calculate_prensors.md │ ├── calculate_prensors_with_graph.md │ ├── calculate_prensors_with_source_paths.md │ ├── create_expression_from_file_descriptor_set.md │ ├── create_expression_from_prensor.md │ ├── create_expression_from_proto.md │ ├── create_path.md │ ├── create_prensor_from_descendant_nodes.md │ ├── create_prensor_from_root_and_children.md │ ├── get_default_options.md │ ├── get_options_with_minimal_checks.md │ ├── get_ragged_tensor.md │ ├── get_ragged_tensors.md │ ├── get_sparse_tensor.md │ └── get_sparse_tensors.md ├── setup.py ├── struct2tensor ├── BUILD ├── __init__.py ├── benchmarks │ ├── BUILD │ ├── benchmark.proto │ ├── g3doc │ │ ├── README.md │ │ └── images │ │ │ ├── broadcast_depth_2.png │ │ │ ├── broadcast_depth_5.png │ │ │ ├── project_deep_1.png │ │ │ ├── project_deep_5.png │ │ │ ├── project_flat_1.png │ │ │ ├── project_flat_5.png │ │ │ ├── promote_depth_1.png │ │ │ ├── promote_depth_4.png │ │ │ ├── reroot_depth_1.png │ │ │ └── reroot_depth_4.png │ ├── ops_benchmark.py │ ├── serialization_benchmark.cc │ ├── struct2tensor_benchmark.py │ ├── struct2tensor_benchmark_util.py │ └── testdata │ │ ├── BUILD │ │ ├── deep_all_types_4096.tfrecord.gz │ │ ├── deep_all_types_4096_positive.tfrecord.gz │ │ ├── flat_100_int_features_4096.tfrecord.gz │ │ ├── flat_all_types_100_int_values_4096.tfrecord.gz │ │ ├── flat_all_types_4096.tfrecord.gz │ │ ├── flat_all_types_4096_positive.tfrecord.gz │ │ ├── tf_example_100_int_features_4096.tfrecord.gz │ │ ├── tf_example_1_int_feature_100_values_4096.tfrecord.gz │ │ └── tf_example_all_types_4096.tfrecord.gz ├── calculate.py ├── calculate_options.py ├── calculate_test.py ├── calculate_with_source_paths.py ├── calculate_with_source_paths_test.py ├── create_expression.py ├── create_expression_test.py ├── expression.py ├── expression_add.py ├── expression_add_test.py ├── expression_impl │ ├── __init__.py │ ├── apply_schema.py │ ├── apply_schema_test.py │ ├── broadcast.py │ ├── broadcast_test.py │ ├── depth_limit.py │ ├── depth_limit_test.py │ ├── filter_expression.py │ ├── filter_expression_test.py │ ├── index.py │ ├── index_test.py │ ├── map_prensor.py │ ├── map_prensor_test.py │ ├── map_prensor_to_prensor.py │ ├── map_prensor_to_prensor_test.py │ ├── map_values.py │ ├── map_values_test.py │ ├── parquet.py │ ├── parquet_test.py │ ├── parse_message_level_ex.py │ ├── parse_message_level_ex_test.py │ ├── placeholder.py │ ├── placeholder_test.py │ ├── project.py │ ├── project_test.py │ ├── promote.py │ ├── promote_and_broadcast.py │ ├── promote_and_broadcast_test.py │ ├── promote_test.py │ ├── proto.py │ ├── proto_test.py │ ├── proto_test_util.py │ ├── raw_parquet_dataset_test.py │ ├── reroot.py │ ├── reroot_test.py │ ├── size.py │ ├── size_test.py │ ├── slice_expression.py │ └── slice_expression_test.py ├── expression_test.py ├── kernels │ ├── BUILD │ ├── decode_proto_map_op.cc │ ├── decode_proto_sparse_op.cc │ ├── equi_join_any_indices_op.cc │ ├── equi_join_indices_op.cc │ ├── parquet │ │ ├── BUILD │ │ ├── parent_indices_builder.cc │ │ ├── parent_indices_builder.h │ │ ├── parquet_dataset_kernel.cc │ │ ├── parquet_reader.cc │ │ ├── parquet_reader.h │ │ ├── parquet_reader_util.cc │ │ └── parquet_reader_util.h │ ├── run_length_before_op.cc │ ├── streaming_proto_reader.cc │ ├── streaming_proto_reader.h │ └── vector_to_tensor.h ├── ops │ ├── BUILD │ ├── __init__.py │ ├── decode_proto_map_op.cc │ ├── decode_proto_sparse_op.cc │ ├── equi_join_any_indices_op.cc │ ├── equi_join_indices_op.cc │ ├── file_descriptor_set.py │ ├── file_descriptor_set_test.py │ ├── gen_decode_proto_map_op.py │ ├── gen_decode_proto_sparse.py │ ├── gen_equi_join_any_indices.py │ ├── gen_equi_join_indices.py │ ├── gen_parquet_dataset.py │ ├── gen_run_length_before.py │ ├── op_kernel_registration_test.cc │ ├── parquet_dataset_op.cc │ ├── run_length_before_op.cc │ ├── struct2tensor_ops.py │ └── struct2tensor_ops_test.py ├── path.py ├── path_test.py ├── prensor.py ├── prensor_test.py ├── prensor_to_structured_tensor.py ├── prensor_to_structured_tensor_test.py ├── prensor_util.py ├── prensor_value.py ├── prensor_value_test.py ├── proto │ ├── BUILD │ ├── __init__.py │ └── query_metadata.proto ├── struct2tensor.bzl ├── struct2tensor_module_test.py ├── structured_tensor_to_prensor.py ├── structured_tensor_to_prensor_test.py ├── test │ ├── BUILD │ ├── __init__.py │ ├── dependent_test.proto │ ├── expression_test_util.py │ ├── prensor_test_util.py │ ├── test.proto │ ├── test_any.proto │ ├── test_extension.proto │ ├── test_map.proto │ └── test_proto3.proto ├── testdata │ └── parquet_testdata │ │ ├── BUILD │ │ ├── all_data_types.parquet │ │ ├── dremel_example.parquet │ │ └── dremel_example_two_row_groups.parquet ├── tools │ ├── BUILD │ ├── build_docs.py │ ├── docker_build │ │ ├── Dockerfile.manylinux2014 │ │ └── build_manylinux.sh │ └── tf_serving_docker │ │ └── Dockerfile ├── version.py └── workspace.bzl ├── tf ├── BUILD ├── BUILD.tpl └── tf_configure.bzl └── third_party ├── BUILD ├── arrow.BUILD ├── local_python.BUILD.tpl ├── python_configure.bzl └── thrift.BUILD /.bazelrc: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # This file is based on tensorflow's (v2.2.0) .bazelrc found here: 16 | # https://github.com/tensorflow/tensorflow/blob/v2.2.0/.bazelrc 17 | # 18 | # We need to provide this file because we need to be as close as possible to 19 | # Tensorflow's building environment. 20 | 21 | # Sets the default Apple platform to macOS. 22 | build --apple_platform_type=macos 23 | 24 | # Enable using platform specific build settings 25 | build --enable_platform_specific_config 26 | 27 | # Enable using 'cc_shared_library' rule. 28 | build --experimental_cc_shared_library 29 | 30 | # Flag to enable remote config 31 | common --experimental_repo_remote_exec 32 | 33 | # Suppress C++ compiler warnings, otherwise build logs become 10s of MBs. 34 | build:linux --copt=-w 35 | build:macos --copt=-w 36 | 37 | # By default, build TF in C++ 17 mode. 38 | build:linux --cxxopt=-std=c++17 39 | build:linux --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=1 40 | build:linux --host_cxxopt=-std=c++17 41 | build:macos --cxxopt=-std=c++17 42 | build:macos --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=1 43 | build:macos --host_cxxopt=-std=c++17 44 | 45 | 46 | # Suppress all warning messages. 47 | build:short_logs --output_filter=DONT_MATCH_ANYTHING 48 | 49 | build:macos --macos_minimum_os=10.12 50 | -------------------------------------------------------------------------------- /.bazelversion: -------------------------------------------------------------------------------- 1 | 6.5.0 -------------------------------------------------------------------------------- /BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # CPU kernels for struct2tensors. 16 | 17 | licenses(["notice"]) # Apache 2.0 18 | 19 | package(default_visibility = ["//visibility:public"]) 20 | 21 | sh_binary( 22 | name = "build_pip_package", 23 | testonly = True, # Some files are testonly 24 | srcs = ["build_pip_package.sh"], 25 | data = [ 26 | "//struct2tensor/ops:gen_parquet_dataset_py", 27 | "//struct2tensor/ops:struct2tensor_ops", 28 | "//struct2tensor/proto:query_metadata_py_pb2", 29 | "//struct2tensor/test:dependent_test_py_pb2", 30 | "//struct2tensor/test:test_any_py_pb2", 31 | "//struct2tensor/test:test_extension_py_pb2", 32 | "//struct2tensor/test:test_map_py_pb2", 33 | "//struct2tensor/test:test_proto3_py_pb2", 34 | "//struct2tensor/test:test_py_pb2", 35 | ], 36 | ) 37 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Struct2Tensor 2 | 3 | Contributions to Struct2Tensor are welcome. One can create a pull request. 4 | 5 | ## Contributor License Agreement 6 | 7 | Contributions to this project must be accompanied by a Contributor License 8 | Agreement. You (or your employer) retain the copyright to your contribution, 9 | this simply gives us permission to use and redistribute your contributions as 10 | part of the project. Head over to to see 11 | your current agreements on file or to sign a new one. 12 | 13 | You generally only need to submit a CLA once, so if you've already submitted one 14 | (even if it was for a different project), you probably don't need to do it 15 | again. 16 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include struct2tensor/ *.so 2 | recursive-include struct2tensor/ *.py 3 | -------------------------------------------------------------------------------- /build_pip_package.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | # 17 | # Convenience binary to build struct2tensor from source. 18 | # 19 | # Usage: build_pip_package.sh [--python_bin_path PYTHON_BIN_PATH] 20 | 21 | if [[ -z "$1" ]]; then 22 | PYTHON_BIN_PATH=python 23 | else 24 | if [[ "$1" == --python_bin_path ]]; then 25 | shift 26 | PYTHON_BIN_PATH=$1 27 | else 28 | printf "Unrecognized argument $1" 29 | exit 1 30 | fi 31 | fi 32 | 33 | set -u -x 34 | 35 | function main() { 36 | # We will build the wheel in a temp directory. 37 | TMPDIR=$(mktemp -d) 38 | 39 | # If run by "bazel run", $(pwd) is the .runfiles dir that contains all the 40 | # data dependencies. 41 | RUNFILES_DIR=$(pwd) 42 | 43 | cp ${BUILD_WORKSPACE_DIRECTORY}/setup.py "${TMPDIR}" 44 | cp ${BUILD_WORKSPACE_DIRECTORY}/MANIFEST.in "${TMPDIR}" 45 | cp ${BUILD_WORKSPACE_DIRECTORY}/LICENSE "${TMPDIR}" 46 | cp ${BUILD_WORKSPACE_DIRECTORY}/README.md "${TMPDIR}" 47 | 48 | rsync -avm -L --exclude='*_test.py' ${BUILD_WORKSPACE_DIRECTORY}/struct2tensor "${TMPDIR}" 49 | 50 | cp -f ${RUNFILES_DIR}/struct2tensor/proto/*_pb2.py "${TMPDIR}"/struct2tensor/proto/ 51 | cp -f ${RUNFILES_DIR}/struct2tensor/test/*_pb2.py "${TMPDIR}"/struct2tensor/test/ 52 | cp -f ${RUNFILES_DIR}/struct2tensor/ops/*.so "${TMPDIR}"/struct2tensor/ops/ 53 | 54 | chmod +w "${TMPDIR}"/struct2tensor/ops/* 55 | 56 | 57 | cd ${TMPDIR} 58 | ${PYTHON_BIN_PATH} setup.py bdist_wheel 59 | 60 | mkdir -p ${BUILD_WORKSPACE_DIRECTORY}/dist/ 61 | cp dist/*.whl ${BUILD_WORKSPACE_DIRECTORY}/dist/ 62 | rm -rf ${TMPDIR} 63 | } 64 | 65 | main "$@" 66 | -------------------------------------------------------------------------------- /configure.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | # Later changes made by Martin Zinkevich. 17 | # 18 | # This script prepares the bazel workspace for build, by configuring the 19 | # .bazelrc file. 20 | # 21 | # Usage: configure.sh [--python_bin_path PYTHON_BIN_PATH] 22 | 23 | function write_to_bazelrc() { 24 | echo "$1" >> .bazelrc 25 | } 26 | 27 | function write_action_env_to_bazelrc() { 28 | write_to_bazelrc "build --action_env $1=\"$2\"" 29 | } 30 | 31 | function has_tensorflow() { 32 | ${PYTHON_BIN_PATH} -c "import tensorflow" > /dev/null 33 | } 34 | 35 | function ensure_tensorflow() { 36 | if has_tensorflow; then 37 | echo "Using installed tf..." 38 | else 39 | echo "Building struct2tensor requires tensorflow. Please install tensorflow." 40 | exit 1 41 | fi 42 | } 43 | 44 | if [[ -z "$1" ]]; then 45 | PYTHON_BIN_PATH=python 46 | else 47 | if [[ "$1" == --python_bin_path ]]; then 48 | shift 49 | PYTHON_BIN_PATH=$1 50 | else 51 | printf "Unrecognized argument $1" 52 | exit 1 53 | fi 54 | fi 55 | 56 | ensure_tensorflow 57 | TF_CFLAGS=( $(${PYTHON_BIN_PATH} -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))') ) 58 | TF_LFLAGS="$(${PYTHON_BIN_PATH} -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))')" 59 | 60 | write_action_env_to_bazelrc "TF_HEADER_DIR" ${TF_CFLAGS:2} 61 | SHARED_LIBRARY_DIR=${TF_LFLAGS:2} 62 | SHARED_LIBRARY_NAME=$(echo $TF_LFLAGS | rev | cut -d":" -f1 | rev) 63 | if ! [[ $TF_LFLAGS =~ .*:.* ]]; then 64 | if [[ "$(uname)" == "Darwin" ]]; then 65 | SHARED_LIBRARY_NAME="libtensorflow_framework.2.dylib" 66 | else 67 | SHARED_LIBRARY_NAME="libtensorflow_framework.so" 68 | fi 69 | fi 70 | write_action_env_to_bazelrc "TF_SHARED_LIBRARY_DIR" ${SHARED_LIBRARY_DIR} 71 | write_action_env_to_bazelrc "TF_SHARED_LIBRARY_NAME" ${SHARED_LIBRARY_NAME} 72 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | version: '3.1' 16 | 17 | # Extensions are not support until 3.4 thus the repeated boilerplate below. 18 | 19 | # We mount the struct2tensor project root at /struct2tensor (which is the WORKDIR of the image) 20 | # in the container. 21 | # 22 | # We need to set PYTHON_VERSION in `build` command which is used to pick the 23 | # base TF SIG Build image. 24 | # The TF_VERSION environment variables is expected when run the pipeline 25 | # Valid TF_VERSION are NIGHTLY_TF|NIGHTLY_TF_2|RELEASED_TF|PRERELEASED_TF|RELEASED_TF_2|PRERELEASED_TF_2 26 | # Sample usage: 27 | # docker-compose build --build-arg PYTHON_VERSION=3.7 manylinux2014 28 | # docker-compose run -e TF_VERSION=NIGHTLY_TF manylinux2014 29 | 30 | services: 31 | manylinux2014: 32 | image: struct2tensor-build:manylinux2014 33 | build: 34 | context: . 35 | dockerfile: struct2tensor/tools/docker_build/Dockerfile.manylinux2014 36 | volumes: 37 | - .:/struct2tensor:delegated 38 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/_toc.yaml: -------------------------------------------------------------------------------- 1 | toc: 2 | - include: /api_docs/python/s2t/_toc.yaml 3 | - break: true 4 | - include: /api_docs/python/expression_impl/_toc.yaml 5 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl.md: -------------------------------------------------------------------------------- 1 | description: Import all modules in expression_impl. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # Module: expression_impl 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Import all modules in expression_impl. 24 | 25 | 26 | The modules in this file should be accessed like the following: 27 | 28 | ``` 29 | import struct2tensor as s2t 30 | from struct2tensor import expression_impl 31 | 32 | s2t.expression_impl.apply_schema 33 | ``` 34 | 35 | ## Modules 36 | 37 | [`apply_schema`](./expression_impl/apply_schema.md) module: Apply a schema to an expression. 38 | 39 | [`broadcast`](./expression_impl/broadcast.md) module: Methods for broadcasting a path in a tree. 40 | 41 | [`depth_limit`](./expression_impl/depth_limit.md) module: Caps the depth of an expression. 42 | 43 | [`filter_expression`](./expression_impl/filter_expression.md) module: Create a new expression that is a filtered version of an original one. 44 | 45 | [`index`](./expression_impl/index.md) module: get_positional_index and get_index_from_end methods. 46 | 47 | [`map_prensor`](./expression_impl/map_prensor.md) module: Arbitrary operations from sparse and ragged tensors to a leaf field. 48 | 49 | [`map_prensor_to_prensor`](./expression_impl/map_prensor_to_prensor.md) module: Arbitrary operations from prensors to prensors in an expression. 50 | 51 | [`map_values`](./expression_impl/map_values.md) module: Maps the values of various leaves of the same child to a single result. 52 | 53 | [`parquet`](./expression_impl/parquet.md) module: Apache Parquet Dataset. 54 | 55 | [`placeholder`](./expression_impl/placeholder.md) module: Placeholder expression. 56 | 57 | [`project`](./expression_impl/project.md) module: project selects a subtree of an expression. 58 | 59 | [`promote`](./expression_impl/promote.md) module: Promote an expression to be a child of its grandparent. 60 | 61 | [`promote_and_broadcast`](./expression_impl/promote_and_broadcast.md) module: promote_and_broadcast a set of nodes. 62 | 63 | [`proto`](./expression_impl/proto.md) module: Expressions to parse a proto. 64 | 65 | [`reroot`](./expression_impl/reroot.md) module: Reroot to a subtree, maintaining an input proto index. 66 | 67 | [`size`](./expression_impl/size.md) module: Functions for creating new size or has expression. 68 | 69 | [`slice_expression`](./expression_impl/slice_expression.md) module: Implementation of slice. 70 | 71 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/apply_schema.md: -------------------------------------------------------------------------------- 1 | description: Apply a schema to an expression. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # Module: expression_impl.apply_schema 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Apply a schema to an expression. 24 | 25 | 26 | A tensorflow metadata schema ( 27 | detailed information about the data: specifically, it presents domain 28 | information (e.g., not just integers, but integers between 0 and 10), and more 29 | detailed structural information (e.g., this field occurs in at least 70% of its 30 | parents, and when it occurs, it shows up 5 to 7 times). 31 | 32 | Applying a schema attaches a tensorflow metadata schema to an expression: 33 | namely, it aligns the features in the schema with the expression's children by 34 | name (possibly recursively). 35 | 36 | After applying a schema to an expression, one can use promote, broadcast, et 37 | cetera, and the schema for new expressions will be inferred. If you write a 38 | custom expression, you can write code that determines the schema information of 39 | the result. 40 | 41 | To get the schema back, call get_schema(). 42 | 43 | This does not filter out fields not in the schema. 44 | 45 | 46 | my_expr = ... 47 | my_schema = ...schema here... 48 | my_new_schema = my_expr.apply_schema(my_schema).get_schema() 49 | my_new_schema has semantically identical information on the fields as my_schema. 50 | 51 | 52 | 1. Get the (non-deprecated) paths from a schema. 53 | 2. Check if any paths in the schema are not in the expression. 54 | 3. Check if any paths in the expression are not in the schema. 55 | 4. Project the expression to paths in the schema. 56 | 57 | ## Functions 58 | 59 | [`apply_schema(...)`](../expression_impl/apply_schema/apply_schema.md) 60 | 61 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/apply_schema/apply_schema.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 |
5 | 6 | # expression_impl.apply_schema.apply_schema 7 | 8 | 9 | 10 | 11 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 
24 | expression_impl.apply_schema.apply_schema(
25 |     expr: expression.Expression,
26 |     schema: schema_pb2.Schema
27 | ) -> expression.Expression
28 |
29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/broadcast.md: -------------------------------------------------------------------------------- 1 | description: Methods for broadcasting a path in a tree. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # Module: expression_impl.broadcast 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Methods for broadcasting a path in a tree. 24 | 25 | 26 | This provides methods for broadcasting a field anonymously (that is used in 27 | promote_and_broadcast), or with an explicitly given name. 28 | 29 | Suppose you have an expr representing: 30 | 31 | ``` 32 | + 33 | | 34 | +-session* (stars indicate repeated) 35 | | 36 | +-event* 37 | | 38 | +-val*-int64 39 | 40 | session: { 41 | event: {} 42 | event: {} 43 | val: 10 44 | val: 11 45 | } 46 | session: { 47 | event: {} 48 | event: {} 49 | val: 20 50 | } 51 | ``` 52 | 53 | #### Then: 54 | 55 | 56 | 57 | ``` 58 | broadcast.broadcast(expr, path.Path(["session","val"]), "event", "nv") 59 | ``` 60 | 61 | becomes: 62 | 63 | ``` 64 | + 65 | | 66 | +---session* (stars indicate repeated) 67 | | 68 | +-event* 69 | | | 70 | | +---nv*-int64 71 | | 72 | +-val*-int64 73 | 74 | session: { 75 | event: { 76 | nv: 10 77 | nv:11 78 | } 79 | event: { 80 | nv: 10 81 | nv:11 82 | } 83 | val: 10 84 | val: 11 85 | } 86 | session: { 87 | event: {nv: 20} 88 | event: {nv: 20} 89 | val: 20 90 | } 91 | ``` 92 | 93 | ## Functions 94 | 95 | [`broadcast(...)`](../expression_impl/broadcast/broadcast.md) 96 | 97 | [`broadcast_anonymous(...)`](../expression_impl/broadcast/broadcast_anonymous.md) 98 | 99 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/broadcast/broadcast.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 |
5 | 6 | # expression_impl.broadcast.broadcast 7 | 8 | 9 | 10 | 11 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 
24 | expression_impl.broadcast.broadcast(
25 |     root: expression.Expression,
26 |     origin: path.Path,
27 |     sibling_name: path.Step,
28 |     new_field_name: path.Step
29 | ) -> expression.Expression
30 |
31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/broadcast/broadcast_anonymous.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 |
5 | 6 | # expression_impl.broadcast.broadcast_anonymous 7 | 8 | 9 | 10 | 11 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 
24 | expression_impl.broadcast.broadcast_anonymous(
25 |     root: expression.Expression,
26 |     origin: path.Path,
27 |     sibling: path.Step
28 | ) -> Tuple[expression.Expression, path.Path]
29 |
30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/depth_limit.md: -------------------------------------------------------------------------------- 1 | description: Caps the depth of an expression. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # Module: expression_impl.depth_limit 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Caps the depth of an expression. 24 | 25 | 26 | Suppose you have an expression expr modeled as: 27 | 28 | ``` 29 | * 30 | \ 31 | A 32 | / \ 33 | D B 34 | \ 35 | C 36 | ``` 37 | 38 | if expr_2 = depth_limit.limit_depth(expr, 2) 39 | You get: 40 | 41 | ``` 42 | * 43 | \ 44 | A 45 | / \ 46 | D B 47 | ``` 48 | 49 | ## Functions 50 | 51 | [`limit_depth(...)`](../expression_impl/depth_limit/limit_depth.md): Limit the depth to nodes k steps from expr. 52 | 53 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/depth_limit/limit_depth.md: -------------------------------------------------------------------------------- 1 | description: Limit the depth to nodes k steps from expr. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.depth_limit.limit_depth 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Limit the depth to nodes k steps from expr. 24 | 25 | 
26 | expression_impl.depth_limit.limit_depth(
27 |     expr: expression.Expression,
28 |     depth_limit: int
29 | ) -> expression.Expression
30 |
31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/filter_expression/filter_by_child.md: -------------------------------------------------------------------------------- 1 | description: Filter an expression by an optional boolean child field. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.filter_expression.filter_by_child 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Filter an expression by an optional boolean child field. 24 | 25 | 
26 | expression_impl.filter_expression.filter_by_child(
27 |     expr: expression.Expression,
28 |     p: path.Path,
29 |     child_field_name: path.Step,
30 |     new_field_name: path.Step
31 | ) -> expression.Expression
32 |
33 | 34 | 35 | 36 | 37 | 38 | If the child field is present and True, then keep that parent. 39 | Otherwise, drop the parent. 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 50 | 53 | 54 | 57 | 60 | 61 | 64 | 67 | 68 | 71 | 74 | 75 |

Args

48 | `expr` 49 | 51 | the original expression 52 |
55 | `p` 56 | 58 | the path to filter. 59 |
62 | `child_field_name` 63 | 65 | the boolean child field to use to filter. 66 |
69 | `new_field_name` 70 | 72 | the new, filtered version of path. 73 |
76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 87 | 88 | 89 |

Returns

85 | The new root expression. 86 |
90 | 91 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/filter_expression/filter_by_sibling.md: -------------------------------------------------------------------------------- 1 | description: Filter an expression by its sibling. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.filter_expression.filter_by_sibling 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Filter an expression by its sibling. 24 | 25 | 
26 | expression_impl.filter_expression.filter_by_sibling(
27 |     expr: expression.Expression,
28 |     p: path.Path,
29 |     sibling_field_name: path.Step,
30 |     new_field_name: path.Step
31 | ) -> expression.Expression
32 |
33 | 34 | 35 | 36 | 37 | 38 | 39 | This is similar to boolean_mask. The shape of the path being filtered and 40 | the sibling must be identical (e.g., each parent object must have an 41 | equal number of source and sibling children). 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 52 | 55 | 56 | 59 | 62 | 63 | 66 | 69 | 70 | 73 | 76 | 77 |

Args

50 | `expr` 51 | 53 | the root expression. 54 |
57 | `p` 58 | 60 | a path to the source to be filtered. 61 |
64 | `sibling_field_name` 65 | 67 | the sibling to use as a mask. 68 |
71 | `new_field_name` 72 | 74 | a new sibling to create. 75 |
78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 89 | 90 | 91 |

Returns

87 | a new root. 88 |
92 | 93 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/index/get_index_from_end.md: -------------------------------------------------------------------------------- 1 | description: Gets the number of steps from the end of the array. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.index.get_index_from_end 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Gets the number of steps from the end of the array. 24 | 25 | 
26 | expression_impl.index.get_index_from_end(
27 |     t: expression.Expression,
28 |     source_path: path.Path,
29 |     new_field_name: path.Step
30 | ) -> Tuple[expression.Expression, path.Path]
31 |
32 | 33 | 34 | 35 | 36 | 37 | Given an array ["a", "b", "c"], with indices [0, 1, 2], the result of this 38 | is [-3,-2,-1]. 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 49 | 52 | 53 | 56 | 59 | 60 | 63 | 66 | 67 |

Args

47 | `t` 48 | 50 | original expression 51 |
54 | `source_path` 55 | 57 | path in expression to get index of. 58 |
61 | `new_field_name` 62 | 64 | the name of the new field. 65 |
68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 79 | 80 | 81 |

Returns

77 | The new expression and the new path as a pair. 78 |
82 | 83 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/index/get_positional_index.md: -------------------------------------------------------------------------------- 1 | description: Gets the positional index. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.index.get_positional_index 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Gets the positional index. 24 | 25 | 
26 | expression_impl.index.get_positional_index(
27 |     expr: expression.Expression,
28 |     source_path: path.Path,
29 |     new_field_name: path.Step
30 | ) -> Tuple[expression.Expression, path.Path]
31 |
32 | 33 | 34 | 35 | 36 | 37 | Given a field with parent_index [0,1,1,2,3,4,4], this returns: 38 | parent_index [0,1,1,2,3,4,4] and value [0,0,1,0,0,0,1] 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 49 | 52 | 53 | 56 | 59 | 60 | 63 | 66 | 67 |

Args

47 | `expr` 48 | 50 | original expression 51 |
54 | `source_path` 55 | 57 | path in expression to get index of. 58 |
61 | `new_field_name` 62 | 64 | the name of the new field. 65 |
68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 79 | 80 | 81 |

Returns

77 | The new expression and the new path as a pair. 78 |
82 | 83 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/map_prensor.md: -------------------------------------------------------------------------------- 1 | description: Arbitrary operations from sparse and ragged tensors to a leaf field. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # Module: expression_impl.map_prensor 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Arbitrary operations from sparse and ragged tensors to a leaf field. 24 | 25 | 26 | There are two public methods of note right now: map_sparse_tensor 27 | and map_ragged_tensor. 28 | 29 | #### Assume expr is: 30 | 31 | 32 | 33 | ``` 34 | session: { 35 | event: { 36 | val_a: 10 37 | val_b: 1 38 | } 39 | event: { 40 | val_a: 20 41 | val_b: 2 42 | } 43 | event: { 44 | } 45 | event: { 46 | val_a: 40 47 | } 48 | event: { 49 | val_b: 5 50 | } 51 | } 52 | ``` 53 | 54 | Either of the following alternatives will add val_a and val_b 55 | to create val_sum. 56 | 57 | map_sparse_tensor converts val_a and val_b to sparse tensors, 58 | and then add them to produce val_sum. 59 | 60 | ``` 61 | new_root = map_prensor.map_sparse_tensor( 62 | expr, 63 | path.Path(["event"]), 64 | [path.Path(["val_a"]), path.Path(["val_b"])], 65 | lambda x,y: x + y, 66 | False, 67 | tf.int32, 68 | "val_sum") 69 | ``` 70 | 71 | map_ragged_tensor converts val_a and val_b to ragged tensors, 72 | and then add them to produce val_sum. 73 | 74 | ``` 75 | new_root = map_prensor.map_ragged_tensor( 76 | expr, 77 | path.Path(["event"]), 78 | [path.Path(["val_a"]), path.Path(["val_b"])], 79 | lambda x,y: x + y, 80 | False, 81 | tf.int32, 82 | "val_sum") 83 | ``` 84 | 85 | The result of either is: 86 | 87 | ``` 88 | session: { 89 | event: { 90 | val_a: 10 91 | val_b: 1 92 | val_sum: 11 93 | } 94 | event: { 95 | val_a: 20 96 | val_b: 2 97 | val_sum: 22 98 | } 99 | event: { 100 | } 101 | event: { 102 | val_a: 40 103 | val_sum: 40 104 | } 105 | event: { 106 | val_b: 5 107 | val_sum: 5 108 | } 109 | } 110 | ``` 111 | 112 | ## Functions 113 | 114 | [`map_ragged_tensor(...)`](../expression_impl/map_prensor/map_ragged_tensor.md): Map a ragged tensor. 115 | 116 | [`map_sparse_tensor(...)`](../expression_impl/map_prensor/map_sparse_tensor.md): Maps a sparse tensor. 117 | 118 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/map_prensor/map_ragged_tensor.md: -------------------------------------------------------------------------------- 1 | description: Map a ragged tensor. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.map_prensor.map_ragged_tensor 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Map a ragged tensor. 24 | 25 | 
 26 | expression_impl.map_prensor.map_ragged_tensor(
 27 |     root: expression.Expression,
 28 |     root_path: path.Path,
 29 |     paths: Sequence[path.Path],
 30 |     operation: Callable[..., tf.RaggedTensor],
 31 |     is_repeated: bool,
 32 |     dtype: tf.DType,
 33 |     new_field_name: path.Step
 34 | ) -> expression.Expression
 35 |
36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 51 | 54 | 55 | 58 | 61 | 62 | 65 | 68 | 69 | 72 | 76 | 77 | 80 | 83 | 84 | 87 | 90 | 91 | 94 | 98 | 99 |

Args

49 | `root` 50 | 52 | the root of the expression. 53 |
56 | `root_path` 57 | 59 | the path relative to which the ragged tensors are calculated. 60 |
63 | `paths` 64 | 66 | the input paths relative to the root_path 67 |
70 | `operation` 71 | 73 | a method that takes the list of ragged tensors as input and 74 | returns a ragged tensor. 75 |
78 | `is_repeated` 79 | 81 | true if the result of operation is repeated. 82 |
85 | `dtype` 86 | 88 | dtype of the result of the operation. 89 |
92 | `new_field_name` 93 | 95 | root_path.get_child(new_field_name) is the path of the 96 | result. 97 |
100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 112 | 113 | 114 |

Returns

109 | A new root expression containing the old root expression plus the new path, 110 | root_path.get_child(new_field_name), with the result of the operation. 111 |
115 | 116 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/map_prensor/map_sparse_tensor.md: -------------------------------------------------------------------------------- 1 | description: Maps a sparse tensor. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.map_prensor.map_sparse_tensor 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Maps a sparse tensor. 24 | 25 | 
 26 | expression_impl.map_prensor.map_sparse_tensor(
 27 |     root: expression.Expression,
 28 |     root_path: path.Path,
 29 |     paths: Sequence[path.Path],
 30 |     operation: Callable[..., tf.SparseTensor],
 31 |     is_repeated: bool,
 32 |     dtype: tf.DType,
 33 |     new_field_name: path.Step
 34 | ) -> expression.Expression
 35 |
36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 51 | 54 | 55 | 58 | 61 | 62 | 65 | 68 | 69 | 72 | 76 | 77 | 80 | 83 | 84 | 87 | 90 | 91 | 94 | 98 | 99 |

Args

49 | `root` 50 | 52 | the root of the expression. 53 |
56 | `root_path` 57 | 59 | the path relative to which the sparse tensors are calculated. 60 |
63 | `paths` 64 | 66 | the input paths relative to the root_path 67 |
70 | `operation` 71 | 73 | a method that takes the list of sparse tensors as input and 74 | returns a sparse tensor. 75 |
78 | `is_repeated` 79 | 81 | true if the result of operation is repeated. 82 |
85 | `dtype` 86 | 88 | dtype of the result of the operation. 89 |
92 | `new_field_name` 93 | 95 | root_path.get_child(new_field_name) is the path of the 96 | result. 97 |
100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 112 | 113 | 114 |

Returns

109 | A new root expression containing the old root expression plus the new path, 110 | root_path.get_child(new_field_name), with the result of the operation. 111 |
115 | 116 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/map_prensor_to_prensor.md: -------------------------------------------------------------------------------- 1 | description: Arbitrary operations from prensors to prensors in an expression. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # Module: expression_impl.map_prensor_to_prensor 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Arbitrary operations from prensors to prensors in an expression. 24 | 25 | 26 | This is useful if a single op generates an entire structure. In general, it is 27 | better to use the existing expressions framework or design a custom expression 28 | than use this op. So long as any of the output is required, all of the input 29 | is required. 30 | 31 | For example, suppose you have an op my_op, that takes a prensor of the form: 32 | 33 | ``` 34 | event 35 | / \ 36 | foo bar 37 | ``` 38 | 39 | and produces a prensor of the form my_result_schema: 40 | 41 | ``` 42 | event 43 | / \ 44 | foo2 bar2 45 | ``` 46 | 47 | ``` 48 | my_result_schema = create_schema( 49 | is_repeated=True, 50 | children={"foo2":{is_repeated:True, dtype:tf.int64}, 51 | "bar2":{is_repeated:False, dtype:tf.int64}}) 52 | ``` 53 | 54 | If you give it an expression original with the schema: 55 | 56 | ``` 57 | session 58 | | 59 | event 60 | / \ 61 | foo bar 62 | 63 | result = map_prensor_to_prensor( 64 | original, 65 | path.Path(["session","event"]), 66 | my_op, 67 | my_result_schema) 68 | ``` 69 | 70 | Result will have the schema: 71 | 72 | ``` 73 | session 74 | | 75 | event-------- 76 | / \ \ \ 77 | foo bar foo2 bar2 78 | ``` 79 | 80 | ## Classes 81 | 82 | [`class Schema`](../expression_impl/map_prensor_to_prensor/Schema.md): A finite schema for a prensor. 83 | 84 | ## Functions 85 | 86 | [`create_schema(...)`](../expression_impl/map_prensor_to_prensor/create_schema.md): Create a schema recursively. 87 | 88 | [`map_prensor_to_prensor(...)`](../expression_impl/map_prensor_to_prensor/map_prensor_to_prensor.md): Maps an expression to a prensor, and merges that prensor. 89 | 90 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/map_prensor_to_prensor/create_schema.md: -------------------------------------------------------------------------------- 1 | description: Create a schema recursively. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.map_prensor_to_prensor.create_schema 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Create a schema recursively. 24 | 25 | 
26 | expression_impl.map_prensor_to_prensor.create_schema(
27 |     is_repeated: bool = True,
28 |     dtype: Optional[tf.DType] = None,
29 |     schema_feature: Optional[schema_pb2.Feature] = None,
30 |     children: Optional[Dict[path.Step, Any]] = None
31 | ) -> expression_impl.map_prensor_to_prensor.Schema
32 |
33 | 34 | 35 | 36 | 37 | 38 | 39 | #### Example: 40 | 41 | 42 | my_result_schema = create_schema( 43 | is_repeated=True, 44 | children={"foo2":{is_repeated=True, dtype=tf.int64}, 45 | "bar2":{is_repeated=False, dtype=tf.int64}}) 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 56 | 59 | 60 | 63 | 66 | 67 | 70 | 74 | 75 | 78 | 82 | 83 |

Args

54 | `is_repeated` 55 | 57 | whether the root is repeated. 58 |
61 | `dtype` 62 | 64 | the dtype of a leaf (None for non-leaves). 65 |
68 | `schema_feature` 69 | 71 | the schema_pb2.Feature describing this expression. name and 72 | struct_domain need not be specified. 73 |
76 | `children` 77 | 79 | the child schemas. Note that the value type of children is either 80 | a Schema or a dictionary of arguments to create_schema. 81 |
84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 95 | 96 | 97 |

Returns

93 | a new Schema represented by the inputs. 94 |
98 | 99 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/map_values.md: -------------------------------------------------------------------------------- 1 | description: Maps the values of various leaves of the same child to a single result. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # Module: expression_impl.map_values 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Maps the values of various leaves of the same child to a single result. 24 | 25 | 26 | All inputs must have the same shape (parent_index must be equal). 27 | 28 | The output is given the same shape (output of function must be of equal length). 29 | 30 | Note that the operations are on 1-D tensors (as opposed to scalars). 31 | 32 | ## Functions 33 | 34 | [`map_many_values(...)`](../expression_impl/map_values/map_many_values.md): Map multiple sibling fields into a new sibling. 35 | 36 | [`map_values(...)`](../expression_impl/map_values/map_values.md): Map field into a new sibling. 37 | 38 | [`map_values_anonymous(...)`](../expression_impl/map_values/map_values_anonymous.md): Map field into a new sibling. 39 | 40 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/map_values/map_many_values.md: -------------------------------------------------------------------------------- 1 | description: Map multiple sibling fields into a new sibling. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.map_values.map_many_values 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Map multiple sibling fields into a new sibling. 24 | 25 | 
 26 | expression_impl.map_values.map_many_values(
 27 |     root: expression.Expression,
 28 |     parent_path: path.Path,
 29 |     source_fields: Sequence[path.Step],
 30 |     operation: Callable[..., tf.Tensor],
 31 |     dtype: tf.DType,
 32 |     new_field_name: path.Step
 33 | ) -> Tuple[expression.Expression, path.Path]
 34 |
35 | 36 | 37 | 38 | 39 | 40 | All source fields must have the same shape, and the shape of the output 41 | must be the same as well. 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 52 | 55 | 56 | 59 | 62 | 63 | 66 | 69 | 70 | 73 | 76 | 77 | 80 | 83 | 84 | 87 | 90 | 91 |

Args

50 | `root` 51 | 53 | original root. 54 |
57 | `parent_path` 58 | 60 | parent path of all sources and the new field. 61 |
64 | `source_fields` 65 | 67 | source fields of the operation. Must have the same shape. 68 |
71 | `operation` 72 | 74 | operation from source_fields to new field. 75 |
78 | `dtype` 79 | 81 | type of new field. 82 |
85 | `new_field_name` 86 | 88 | name of the new field. 89 |
92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 103 | 104 | 105 |

Returns

101 | The new expression and the new path as a pair. 102 |
106 | 107 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/map_values/map_values.md: -------------------------------------------------------------------------------- 1 | description: Map field into a new sibling. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.map_values.map_values 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Map field into a new sibling. 24 | 25 | 
26 | expression_impl.map_values.map_values(
27 |     root: expression.Expression,
28 |     source_path: path.Path,
29 |     operation: Callable[[tf.Tensor], tf.Tensor],
30 |     dtype: tf.DType,
31 |     new_field_name: path.Step
32 | ) -> expression.Expression
33 |
34 | 35 | 36 | 37 | 38 | 39 | The shape of the output must be the same as the input. 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 50 | 53 | 54 | 57 | 60 | 61 | 64 | 67 | 68 | 71 | 74 | 75 | 78 | 81 | 82 |

Args

48 | `root` 49 | 51 | original root. 52 |
55 | `source_path` 56 | 58 | source of the operation. 59 |
62 | `operation` 63 | 65 | operation from source_fields to new field. 66 |
69 | `dtype` 70 | 72 | type of new field. 73 |
76 | `new_field_name` 77 | 79 | name of the new field. 80 |
83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 94 | 95 | 96 |

Returns

92 | The new expression. 93 |
97 | 98 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/map_values/map_values_anonymous.md: -------------------------------------------------------------------------------- 1 | description: Map field into a new sibling. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.map_values.map_values_anonymous 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Map field into a new sibling. 24 | 25 | 
26 | expression_impl.map_values.map_values_anonymous(
27 |     root: expression.Expression,
28 |     source_path: path.Path,
29 |     operation: Callable[[tf.Tensor], tf.Tensor],
30 |     dtype: tf.DType
31 | ) -> Tuple[expression.Expression, path.Path]
32 |
33 | 34 | 35 | 36 | 37 | 38 | The shape of the output must be the same as the input. 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 49 | 52 | 53 | 56 | 59 | 60 | 63 | 66 | 67 | 70 | 73 | 74 |

Args

47 | `root` 48 | 50 | original root. 51 |
54 | `source_path` 55 | 57 | source of the operation. 58 |
61 | `operation` 62 | 64 | operation from source_fields to new field. 65 |
68 | `dtype` 69 | 71 | type of new field. 72 |
75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 86 | 87 | 88 |

Returns

84 | The new expression and the new path as a pair. 85 |
89 | 90 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/parquet.md: -------------------------------------------------------------------------------- 1 | description: Apache Parquet Dataset. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # Module: expression_impl.parquet 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Apache Parquet Dataset. 24 | 25 | 26 | 27 | #### Example usage: 28 | 29 | 30 | 31 | ``` 32 | exp = create_expression_from_parquet_file(filenames) 33 | docid_project_exp = project.project(exp, [path.Path(["DocId"])]) 34 | pqds = parquet_dataset.calculate_parquet_values([docid_project_exp], exp, 35 | filenames, batch_size) 36 | 37 | for prensors in pqds: 38 | doc_id_prensor = prensors[0] 39 | ``` 40 | 41 | ## Classes 42 | 43 | [`class ParquetDataset`](../expression_impl/parquet/ParquetDataset.md): A dataset which reads columns from a parquet file and returns a prensor. 44 | 45 | ## Functions 46 | 47 | [`calculate_parquet_values(...)`](../expression_impl/parquet/calculate_parquet_values.md): Calculates expressions and returns a parquet dataset. 48 | 49 | [`create_expression_from_parquet_file(...)`](../expression_impl/parquet/create_expression_from_parquet_file.md): Creates a placeholder expression from a parquet file. 50 | 51 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/parquet/calculate_parquet_values.md: -------------------------------------------------------------------------------- 1 | description: Calculates expressions and returns a parquet dataset. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.parquet.calculate_parquet_values 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Calculates expressions and returns a parquet dataset. 24 | 25 | 
26 | expression_impl.parquet.calculate_parquet_values(
27 |     expressions: List[expression.Expression],
28 |     root_exp: placeholder._PlaceholderRootExpression,
29 |     filenames: List[str],
30 |     batch_size: int,
31 |     options: Optional[calculate_options.Options] = None
32 | )
33 |
34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 49 | 52 | 53 | 56 | 59 | 60 | 63 | 66 | 67 | 70 | 73 | 74 | 77 | 80 | 81 |

Args

47 | `expressions` 48 | 50 | A list of expressions to calculate. 51 |
54 | `root_exp` 55 | 57 | The root placeholder expression to use as the feed dict. 58 |
61 | `filenames` 62 | 64 | A list of parquet files. 65 |
68 | `batch_size` 69 | 71 | The number of messages to batch. 72 |
75 | `options` 76 | 78 | calculate options. 79 |
82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 93 | 94 | 95 |

Returns

91 | A parquet dataset. 92 |
96 | 97 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/parquet/create_expression_from_parquet_file.md: -------------------------------------------------------------------------------- 1 | description: Creates a placeholder expression from a parquet file. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.parquet.create_expression_from_parquet_file 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Creates a placeholder expression from a parquet file. 24 | 25 | 
26 | expression_impl.parquet.create_expression_from_parquet_file(
27 |     filenames: List[str]
28 | ) -> placeholder._PlaceholderRootExpression
29 |
30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 45 | 48 | 49 |

Args

43 | `filenames` 44 | 46 | A list of parquet files. 47 |
50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 62 | 63 | 64 |

Returns

59 | A PlaceholderRootExpression that should be used as the root of an expression 60 | graph. 61 |
65 | 66 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/placeholder.md: -------------------------------------------------------------------------------- 1 | description: Placeholder expression. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # Module: expression_impl.placeholder 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Placeholder expression. 24 | 25 | 26 | A placeholder expression represents prensor nodes, however a prensor is not 27 | needed until calculate is called. This allows the user to apply expression 28 | queries to a placeholder expression before having an actual prensor object. 29 | When calculate is called on a placeholder expression (or a descendant of a 30 | placeholder expression), the feed_dict will need to be passed in. Then calculate 31 | will bind the prensor with the appropriate placeholder expression. 32 | 33 | #### Sample usage: 34 | 35 | 36 | 37 | ``` 38 | placeholder_exp = placeholder.create_expression_from_schema(schema) 39 | new_exp = expression_queries(placeholder_exp, ..) 40 | result = calculate.calculate_values([new_exp], 41 | feed_dict={placeholder_exp: pren}) 42 | # placeholder_exp requires a feed_dict to be passed in when calculating 43 | ``` 44 | 45 | ## Functions 46 | 47 | [`create_expression_from_schema(...)`](../expression_impl/placeholder/create_expression_from_schema.md): Creates a placeholder expression from a parquet schema. 48 | 49 | [`get_placeholder_paths_from_graph(...)`](../expression_impl/placeholder/get_placeholder_paths_from_graph.md): Gets all placeholder paths from an expression graph. 50 | 51 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/placeholder/create_expression_from_schema.md: -------------------------------------------------------------------------------- 1 | description: Creates a placeholder expression from a parquet schema. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.placeholder.create_expression_from_schema 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Creates a placeholder expression from a parquet schema. 24 | 25 | 
26 | expression_impl.placeholder.create_expression_from_schema(
27 |     schema: expression_impl.map_prensor_to_prensor.Schema
28 | ) -> "_PlaceholderRootExpression"
29 |
30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 45 | 49 | 50 |

Args

43 | `schema` 44 | 46 | The schema that describes the prensor tree that this placeholder 47 | represents. 48 |
51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 63 | 64 | 65 |

Returns

60 | A PlaceholderRootExpression that should be used as the root of an expression 61 | graph. 62 |
66 | 67 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/placeholder/get_placeholder_paths_from_graph.md: -------------------------------------------------------------------------------- 1 | description: Gets all placeholder paths from an expression graph. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.placeholder.get_placeholder_paths_from_graph 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Gets all placeholder paths from an expression graph. 24 | 25 | 
26 | expression_impl.placeholder.get_placeholder_paths_from_graph(
27 |     graph: calculate.ExpressionGraph
28 | ) -> List[path.Path]
29 |
30 | 31 | 32 | 33 | 34 | 35 | This finds all leaf placeholder expressions in an expression graph, and gets 36 | the path of these expressions. 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 47 | 50 | 51 |

Args

45 | `graph` 46 | 48 | expression graph 49 |
52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 63 | 64 | 65 |

Returns

61 | a list of paths of placeholder expressions 62 |
66 | 67 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/project.md: -------------------------------------------------------------------------------- 1 | description: project selects a subtree of an expression. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # Module: expression_impl.project 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | project selects a subtree of an expression. 24 | 25 | 26 | project is often used right before calculating the value. 27 | 28 | #### Example: 29 | 30 | 31 | 32 | ``` 33 | expr = ... 34 | new_expr = project.project(expr, [path.Path(["foo","bar"]), 35 | path.Path(["x", "y"])]) 36 | [prensor_result] = calculate.calculate_prensors([new_expr]) 37 | ``` 38 | 39 | prensor_result now has two paths, "foo.bar" and "x.y". 40 | 41 | ## Functions 42 | 43 | [`project(...)`](../expression_impl/project/project.md): select a subtree. 44 | 45 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/project/project.md: -------------------------------------------------------------------------------- 1 | description: select a subtree. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.project.project 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | select a subtree. 24 | 25 | 
26 | expression_impl.project.project(
27 |     expr: expression.Expression,
28 |     paths: Sequence[path.Path]
29 | ) -> expression.Expression
30 |
31 | 32 | 33 | 34 | 35 | 36 | Paths not selected are removed. 37 | Paths that are selected are "known", such that if calculate_prensors is 38 | called, they will be in the result. 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 49 | 52 | 53 | 56 | 59 | 60 |

Args

47 | `expr` 48 | 50 | the original expression. 51 |
54 | `paths` 55 | 57 | the paths to include. 58 |
61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 72 | 73 | 74 |

Returns

70 | A projected expression. 71 |
75 | 76 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/promote.md: -------------------------------------------------------------------------------- 1 | description: Promote an expression to be a child of its grandparent. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # Module: expression_impl.promote 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Promote an expression to be a child of its grandparent. 24 | 25 | 26 | Promote is part of the standard flattening of data, promote_and_broadcast, 27 | which takes structured data and flattens it. By directly accessing promote, 28 | one can perform simpler operations. 29 | 30 | For example, suppose an expr represents: 31 | 32 | ``` 33 | + 34 | | 35 | +-session* (stars indicate repeated) 36 | | 37 | +-event* 38 | | 39 | +-val*-int64 40 | 41 | session: { 42 | event: { 43 | val: 111 44 | } 45 | event: { 46 | val: 121 47 | val: 122 48 | } 49 | } 50 | 51 | session: { 52 | event: { 53 | val: 10 54 | val: 7 55 | } 56 | event: { 57 | val: 1 58 | } 59 | } 60 | 61 | ``` 62 | 63 | ``` 64 | promote.promote(expr, path.Path(["session", "event", "val"]), nval) 65 | ``` 66 | 67 | produces: 68 | 69 | ``` 70 | + 71 | | 72 | +-session* (stars indicate repeated) 73 | | 74 | +-event* 75 | | | 76 | | +-val*-int64 77 | | 78 | +-nval*-int64 79 | 80 | session: { 81 | event: { 82 | val: 111 83 | } 84 | event: { 85 | val: 121 86 | val: 122 87 | } 88 | nval: 111 89 | nval: 121 90 | nval: 122 91 | } 92 | 93 | session: { 94 | event: { 95 | val: 10 96 | val: 7 97 | } 98 | event: { 99 | val: 1 100 | } 101 | nval: 10 102 | nval: 7 103 | nval: 1 104 | } 105 | ``` 106 | 107 | ## Classes 108 | 109 | [`class PromoteChildExpression`](../expression_impl/promote/PromoteChildExpression.md): The root of the promoted sub tree. 110 | 111 | [`class PromoteExpression`](../expression_impl/promote/PromoteExpression.md): A promoted leaf. 112 | 113 | ## Functions 114 | 115 | [`promote(...)`](../expression_impl/promote/promote.md): Promote a path to be a child of its grandparent, and give it a name. 116 | 117 | [`promote_anonymous(...)`](../expression_impl/promote/promote_anonymous.md): Promote a path to be a new anonymous child of its grandparent. 118 | 119 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/promote/promote.md: -------------------------------------------------------------------------------- 1 | description: Promote a path to be a child of its grandparent, and give it a name. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.promote.promote 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Promote a path to be a child of its grandparent, and give it a name. 24 | 25 | 
26 | expression_impl.promote.promote(
27 |     root: expression.Expression,
28 |     p: path.Path,
29 |     new_field_name: path.Step
30 | ) -> expression.Expression
31 |
32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/promote/promote_anonymous.md: -------------------------------------------------------------------------------- 1 | description: Promote a path to be a new anonymous child of its grandparent. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.promote.promote_anonymous 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Promote a path to be a new anonymous child of its grandparent. 24 | 25 | 
26 | expression_impl.promote.promote_anonymous(
27 |     root: expression.Expression,
28 |     p: path.Path
29 | ) -> Tuple[expression.Expression, path.Path]
30 |
31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/promote_and_broadcast.md: -------------------------------------------------------------------------------- 1 | description: promote_and_broadcast a set of nodes. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # Module: expression_impl.promote_and_broadcast 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | promote_and_broadcast a set of nodes. 24 | 25 | 26 | For example, suppose an expr represents: 27 | 28 | ``` 29 | + 30 | | 31 | +-session* (stars indicate repeated) 32 | | 33 | +-event* 34 | | | 35 | | +-val*-int64 36 | | 37 | +-user_info? (question mark indicates optional) 38 | | 39 | +-age? int64 40 | 41 | session: { 42 | event: { 43 | val: 1 44 | } 45 | event: { 46 | val: 4 47 | val: 5 48 | } 49 | user_info: { 50 | age: 25 51 | } 52 | } 53 | 54 | session: { 55 | event: { 56 | val: 7 57 | } 58 | event: { 59 | val: 8 60 | val: 9 61 | } 62 | user_info: { 63 | age: 20 64 | } 65 | } 66 | ``` 67 | 68 | ``` 69 | promote_and_broadcast.promote_and_broadcast( 70 | path.Path(["event"]),{"nage":path.Path(["user_info","age"])}) 71 | ``` 72 | 73 | creates: 74 | 75 | ``` 76 | + 77 | | 78 | +-session* (stars indicate repeated) 79 | | 80 | +-event* 81 | | | 82 | | +-val*-int64 83 | | | 84 | | +-nage*-int64 85 | | 86 | +-user_info? (question mark indicates optional) 87 | | 88 | +-age? int64 89 | 90 | session: { 91 | event: { 92 | nage: 25 93 | val: 1 94 | } 95 | event: { 96 | nage: 25 97 | val: 4 98 | val: 5 99 | } 100 | user_info: { 101 | age: 25 102 | } 103 | } 104 | 105 | session: { 106 | event: { 107 | nage: 20 108 | val: 7 109 | } 110 | event: { 111 | nage: 20 112 | val: 8 113 | val: 9 114 | } 115 | user_info: { 116 | age: 20 117 | } 118 | } 119 | ``` 120 | 121 | ## Functions 122 | 123 | [`promote_and_broadcast(...)`](../expression_impl/promote_and_broadcast/promote_and_broadcast.md): Promote and broadcast a set of paths to a particular location. 124 | 125 | [`promote_and_broadcast_anonymous(...)`](../expression_impl/promote_and_broadcast/promote_and_broadcast_anonymous.md): Promotes then broadcasts the origin until its parent is new_parent. 126 | 127 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/promote_and_broadcast/promote_and_broadcast.md: -------------------------------------------------------------------------------- 1 | description: Promote and broadcast a set of paths to a particular location. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.promote_and_broadcast.promote_and_broadcast 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Promote and broadcast a set of paths to a particular location. 24 | 25 | 
26 | expression_impl.promote_and_broadcast.promote_and_broadcast(
27 |     root: expression.Expression,
28 |     path_dictionary: Mapping[path.Step, path.Path],
29 |     dest_path_parent: path.Path
30 | ) -> expression.Expression
31 |
32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 47 | 50 | 51 | 54 | 57 | 58 | 61 | 64 | 65 |

Args

45 | `root` 46 | 48 | the original expression. 49 |
52 | `path_dictionary` 53 | 55 | a map from destination fields to origin paths. 56 |
59 | `dest_path_parent` 60 | 62 | a map from destination strings. 63 |
66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 78 | 79 | 80 |

Returns

75 | A new expression, where all the origin paths are promoted and broadcast 76 | until they are children of dest_path_parent. 77 |
81 | 82 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/promote_and_broadcast/promote_and_broadcast_anonymous.md: -------------------------------------------------------------------------------- 1 | description: Promotes then broadcasts the origin until its parent is new_parent. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.promote_and_broadcast.promote_and_broadcast_anonymous 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Promotes then broadcasts the origin until its parent is new_parent. 24 | 25 | 
26 | expression_impl.promote_and_broadcast.promote_and_broadcast_anonymous(
27 |     root: expression.Expression,
28 |     origin: path.Path,
29 |     new_parent: path.Path
30 | ) -> Tuple[expression.Expression, path.Path]
31 |
32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/proto.md: -------------------------------------------------------------------------------- 1 | description: Expressions to parse a proto. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # Module: expression_impl.proto 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Expressions to parse a proto. 24 | 25 | 26 | These expressions return values with more information than standard node values. 27 | Specifically, each node calculates additional tensors that are used as inputs 28 | for its children. 29 | 30 | ## Classes 31 | 32 | [`class DescriptorPool`](../expression_impl/proto/DescriptorPool.md): A collection of protobufs dynamically constructed by descriptor protos. 33 | 34 | [`class FileDescriptorSet`](../expression_impl/proto/FileDescriptorSet.md): A ProtocolMessage 35 | 36 | ## Functions 37 | 38 | [`create_expression_from_file_descriptor_set(...)`](../expression_impl/proto/create_expression_from_file_descriptor_set.md): Create an expression from a 1D tensor of serialized protos. 39 | 40 | [`create_expression_from_proto(...)`](../expression_impl/proto/create_expression_from_proto.md): Create an expression from a 1D tensor of serialized protos. 41 | 42 | [`create_transformed_field(...)`](../expression_impl/proto/create_transformed_field.md): Create an expression that transforms serialized proto tensors. 43 | 44 | [`is_proto_expression(...)`](../expression_impl/proto/is_proto_expression.md): Returns true if an expression is a ProtoExpression. 45 | 46 | ## Type Aliases 47 | 48 | [`ProtoExpression`](../expression_impl/proto/ProtoExpression.md) 49 | 50 | [`TransformFn`](../expression_impl/proto/TransformFn.md) 51 | 52 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/proto/FileDescriptorSet.md: -------------------------------------------------------------------------------- 1 | description: A ProtocolMessage 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.proto.FileDescriptorSet 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | A ProtocolMessage 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 34 | 37 | 38 |

Attributes

32 | `file` 33 | 35 | `repeated FileDescriptorProto file` 36 |
39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/proto/ProtoExpression.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 |
5 | 6 | # expression_impl.proto.ProtoExpression 7 | 8 | 9 | This symbol is a **type alias**. 10 | 11 | 12 | 13 | #### Source: 14 | 15 | 
16 | ProtoExpression = Union[
17 |     struct2tensor.expression_impl.proto._ProtoRootExpression,
18 |     struct2tensor.expression_impl.proto._ProtoChildExpression,
19 |     struct2tensor.expression_impl.proto._ProtoLeafExpression
20 | ]
21 |
22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/proto/TransformFn.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 |
5 | 6 | # expression_impl.proto.TransformFn 7 | 8 | 9 | This symbol is a **type alias**. 10 | 11 | 12 | 13 | #### Source: 14 | 15 | 
16 | TransformFn = Callable[
17 |     tensorflow.python.framework.ops.Tensor,
18 |     tensorflow.python.framework.ops.Tensor,
19 |     Tuple[tensorflow.python.framework.ops.Tensor, tensorflow.python.framework.ops.Tensor]
20 | ]
21 |
22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/proto/create_expression_from_file_descriptor_set.md: -------------------------------------------------------------------------------- 1 | description: Create an expression from a 1D tensor of serialized protos. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.proto.create_expression_from_file_descriptor_set 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Create an expression from a 1D tensor of serialized protos. 24 | 25 | 
26 | expression_impl.proto.create_expression_from_file_descriptor_set(
27 |     tensor_of_protos: tf.Tensor,
28 |     proto_name: ProtoFullName,
29 |     file_descriptor_set: expression_impl.proto.FileDescriptorSet,
30 |     message_format: str = 'binary'
31 | ) -> expression.Expression
32 |
33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 48 | 51 | 52 | 55 | 59 | 60 | 63 | 69 | 70 | 73 | 77 | 78 |

Args

46 | `tensor_of_protos` 47 | 49 | 1D tensor of serialized protos. 50 |
53 | `proto_name` 54 | 56 | fully qualified name (e.g. "some.package.SomeProto") of the 57 | proto in `tensor_of_protos`. 58 |
61 | `file_descriptor_set` 62 | 64 | The FileDescriptorSet proto containing `proto_name`'s 65 | and all its dependencies' FileDescriptorProto. Note that if file1 imports 66 | file2, then file2's FileDescriptorProto must precede file1's in 67 | file_descriptor_set.file. 68 |
71 | `message_format` 72 | 74 | Indicates the format of the protocol buffer: is one of 75 | 'text' or 'binary'. 76 |
79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 90 | 91 | 92 |

Returns

88 | An expression. 89 |
93 | 94 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/proto/create_expression_from_proto.md: -------------------------------------------------------------------------------- 1 | description: Create an expression from a 1D tensor of serialized protos. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.proto.create_expression_from_proto 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Create an expression from a 1D tensor of serialized protos. 24 | 25 | 
26 | expression_impl.proto.create_expression_from_proto(
27 |     tensor_of_protos: tf.Tensor,
28 |     desc: descriptor.Descriptor,
29 |     message_format: str = 'binary'
30 | ) -> expression.Expression
31 |
32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 47 | 50 | 51 | 54 | 57 | 58 | 61 | 65 | 66 |

Args

45 | `tensor_of_protos` 46 | 48 | 1D tensor of serialized protos. 49 |
52 | `desc` 53 | 55 | a descriptor of protos in tensor of protos. 56 |
59 | `message_format` 60 | 62 | Indicates the format of the protocol buffer: is one of 63 | 'text' or 'binary'. 64 |
67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 78 | 79 | 80 |

Returns

76 | An expression. 77 |
81 | 82 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/proto/is_proto_expression.md: -------------------------------------------------------------------------------- 1 | description: Returns true if an expression is a ProtoExpression. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.proto.is_proto_expression 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Returns true if an expression is a ProtoExpression. 24 | 25 | 
26 | expression_impl.proto.is_proto_expression(
27 |     expr: expression.Expression
28 | ) -> bool
29 |
30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/reroot.md: -------------------------------------------------------------------------------- 1 | description: Reroot to a subtree, maintaining an input proto index. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # Module: expression_impl.reroot 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Reroot to a subtree, maintaining an input proto index. 24 | 25 | 26 | reroot is similar to get_descendant_or_error. However, this method allows 27 | you to call create_proto_index(...) later on, that gives you a reference to the 28 | original proto. 29 | 30 | ## Functions 31 | 32 | [`create_proto_index_field(...)`](../expression_impl/reroot/create_proto_index_field.md) 33 | 34 | [`reroot(...)`](../expression_impl/reroot/reroot.md): Reroot to a new path, maintaining a input proto index. 35 | 36 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/reroot/create_proto_index_field.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 |
5 | 6 | # expression_impl.reroot.create_proto_index_field 7 | 8 | 9 | 10 | 11 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 
24 | expression_impl.reroot.create_proto_index_field(
25 |     root: expression.Expression,
26 |     new_field_name: path.Step
27 | ) -> expression.Expression
28 |
29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/reroot/reroot.md: -------------------------------------------------------------------------------- 1 | description: Reroot to a new path, maintaining a input proto index. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.reroot.reroot 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Reroot to a new path, maintaining a input proto index. 24 | 25 | 
26 | expression_impl.reroot.reroot(
27 |     root: expression.Expression,
28 |     source_path: path.Path
29 | ) -> expression.Expression
30 |
31 | 32 | 33 | 34 | 35 | 36 | Similar to root.get_descendant_or_error(source_path): however, this 37 | method retains the ability to get a map to the original index. 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 48 | 51 | 52 | 55 | 58 | 59 |

Args

46 | `root` 47 | 49 | the original root. 50 |
53 | `source_path` 54 | 56 | the path to the new root. 57 |
60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 71 | 72 | 73 |

Returns

69 | the new root. 70 |
74 | 75 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/size.md: -------------------------------------------------------------------------------- 1 | description: Functions for creating new size or has expression. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # Module: expression_impl.size 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Functions for creating new size or has expression. 24 | 25 | 26 | Given a field "foo.bar", 27 | 28 | ``` 29 | root = size(expr, path.Path(["foo","bar"]), "bar_size") 30 | ``` 31 | 32 | creates a new expression root that has an optional field "foo.bar_size", which 33 | is always present, and contains the number of bar in a particular foo. 34 | 35 | ``` 36 | root_2 = has(expr, path.Path(["foo","bar"]), "bar_has") 37 | ``` 38 | 39 | creates a new expression root that has an optional field "foo.bar_has", which 40 | is always present, and is true if there are one or more bar in foo. 41 | 42 | ## Classes 43 | 44 | [`class SizeExpression`](../expression_impl/size/SizeExpression.md): Size of the given expression. 45 | 46 | ## Functions 47 | 48 | [`has(...)`](../expression_impl/size/has.md): Get the has of a field as a new sibling field. 49 | 50 | [`size(...)`](../expression_impl/size/size.md): Get the size of a field as a new sibling field. 51 | 52 | [`size_anonymous(...)`](../expression_impl/size/size_anonymous.md): Calculate the size of a field, and store it as an anonymous sibling. 53 | 54 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/size/has.md: -------------------------------------------------------------------------------- 1 | description: Get the has of a field as a new sibling field. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.size.has 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Get the has of a field as a new sibling field. 24 | 25 | 
26 | expression_impl.size.has(
27 |     root: expression.Expression,
28 |     source_path: path.Path,
29 |     new_field_name: path.Step
30 | ) -> expression.Expression
31 |
32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 47 | 50 | 51 | 54 | 57 | 58 | 61 | 64 | 65 |

Args

45 | `root` 46 | 48 | the original expression. 49 |
52 | `source_path` 53 | 55 | the source path to measure. Cannot be root. 56 |
59 | `new_field_name` 60 | 62 | the name of the sibling field. 63 |
66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 77 | 78 | 79 |

Returns

75 | The new expression. 76 |
80 | 81 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/size/size.md: -------------------------------------------------------------------------------- 1 | description: Get the size of a field as a new sibling field. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.size.size 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Get the size of a field as a new sibling field. 24 | 25 | 
26 | expression_impl.size.size(
27 |     root: expression.Expression,
28 |     source_path: path.Path,
29 |     new_field_name: path.Step
30 | ) -> expression.Expression
31 |
32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 47 | 50 | 51 | 54 | 57 | 58 | 61 | 64 | 65 |

Args

45 | `root` 46 | 48 | the original expression. 49 |
52 | `source_path` 53 | 55 | the source path to measure. Cannot be root. 56 |
59 | `new_field_name` 60 | 62 | the name of the sibling field. 63 |
66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 77 | 78 | 79 |

Returns

75 | The new expression. 76 |
80 | 81 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/size/size_anonymous.md: -------------------------------------------------------------------------------- 1 | description: Calculate the size of a field, and store it as an anonymous sibling. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.size.size_anonymous 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Calculate the size of a field, and store it as an anonymous sibling. 24 | 25 | 
26 | expression_impl.size.size_anonymous(
27 |     root: expression.Expression,
28 |     source_path: path.Path
29 | ) -> Tuple[expression.Expression, path.Path]
30 |
31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 46 | 49 | 50 | 53 | 56 | 57 |

Args

44 | `root` 45 | 47 | the original expression. 48 |
51 | `source_path` 52 | 54 | the source path to measure. Cannot be root. 55 |
58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 69 | 70 | 71 |

Returns

67 | The new expression and the new field as a pair. 68 |
72 | 73 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/slice_expression/IndexValue.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 |
5 | 6 | # expression_impl.slice_expression.IndexValue 7 | 8 | 9 | This symbol is a **type alias**. 10 | 11 | 12 | 13 | #### Source: 14 | 15 | 
16 | IndexValue = Union[
17 |     int,
18 |     tensorflow.python.framework.ops.Tensor,
19 |     tensorflow.python.ops.variables.Variable
20 | ]
21 |
22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/expression_impl/slice_expression/slice_expression.md: -------------------------------------------------------------------------------- 1 | description: Creates a new subtree with a sliced expression. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # expression_impl.slice_expression.slice_expression 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Creates a new subtree with a sliced expression. 24 | 25 | 
26 | expression_impl.slice_expression.slice_expression(
27 |     expr: expression.Expression,
28 |     p: path.Path,
29 |     new_field_name: path.Step,
30 |     begin: Optional[IndexValue],
31 |     end: Optional[IndexValue]
32 | ) -> expression.Expression
33 |
34 | 35 | 36 | 37 | 38 | 39 | This follows the pattern of python slice() method. 40 | See module-level comments for examples. 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 51 | 54 | 55 | 58 | 61 | 62 | 65 | 68 | 69 | 72 | 75 | 76 | 79 | 82 | 83 |

Args

49 | `expr` 50 | 52 | the original root expression 53 |
56 | `p` 57 | 59 | the path to the source to be sliced. 60 |
63 | `new_field_name` 64 | 66 | the name of the new subtree. 67 |
70 | `begin` 71 | 73 | beginning index 74 |
77 | `end` 78 | 80 | end index. 81 |
84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 95 | 96 | 97 |

Returns

93 | A new root expression. 94 |
98 | 99 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/NodeTensor.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 |
5 | 6 | # s2t.NodeTensor 7 | 8 | 9 | This symbol is a **type alias**. 10 | 11 | 12 | 13 | #### Source: 14 | 15 | 
16 | NodeTensor = Union[
17 |     s2t.LeafNodeTensor,
18 |     s2t.ChildNodeTensor,
19 |     s2t.RootNodeTensor
20 | ]
21 |
22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/RootNodeTensor.md: -------------------------------------------------------------------------------- 1 | description: The value of the root. 2 | 3 |
4 | 5 | 6 | 7 | 8 |
9 | 10 | # s2t.RootNodeTensor 11 | 12 | 13 | 14 | 15 | 21 | 22 | 23 | 24 | 25 | The value of the root. 26 | 27 | 
 28 | s2t.RootNodeTensor(
 29 |     size: tf.Tensor
 30 | )
 31 |
32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 47 | 50 | 51 |

Args

45 | `size` 46 | 48 | A scalar int64 tensor saying how many root objects there are. 49 |
52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 66 | 69 | 70 | 73 | 76 | 77 |

Attributes

64 | `is_repeated` 65 | 67 | 68 |
71 | `size` 72 | 74 | 75 |
78 | 79 | 80 | 81 | ## Methods 82 | 83 |

get_positional_index

84 | 85 | View source 86 | 87 | 
 88 | get_positional_index() -> tf.Tensor
 89 |
90 | 91 | Gets the positional index for this RootNodeTensor. 92 | 93 | The positional index relative to the node's parent, and thus is always 94 | monotonically increasing at step size 1 for a RootNodeTensor. 95 | 96 | 97 | 98 | 99 | 100 | 101 | 104 | 105 | 106 |
Returns
102 | A tensor of positional indices. 103 |
107 | 108 | 109 | 110 | 111 | 112 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/Step.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 |
5 | 6 | # s2t.Step 7 | 8 | 9 | This symbol is a **type alias**. 10 | 11 | 12 | 13 | #### Source: 14 | 15 | 
16 | Step = Union[
17 |     int,
18 |     str
19 | ]
20 |
21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/_toc.yaml: -------------------------------------------------------------------------------- 1 | toc: 2 | - title: s2t 3 | section: 4 | - title: Overview 5 | path: /api_docs/python/s2t 6 | - title: ChildNodeTensor 7 | path: /api_docs/python/s2t/ChildNodeTensor 8 | - title: Expression 9 | path: /api_docs/python/s2t/Expression 10 | - title: LeafNodeTensor 11 | path: /api_docs/python/s2t/LeafNodeTensor 12 | - title: NodeTensor 13 | path: /api_docs/python/s2t/NodeTensor 14 | - title: Path 15 | path: /api_docs/python/s2t/Path 16 | - title: Prensor 17 | path: /api_docs/python/s2t/Prensor 18 | - title: RootNodeTensor 19 | path: /api_docs/python/s2t/RootNodeTensor 20 | - title: Step 21 | path: /api_docs/python/s2t/Step 22 | - title: calculate_prensors 23 | path: /api_docs/python/s2t/calculate_prensors 24 | - title: calculate_prensors_with_graph 25 | path: /api_docs/python/s2t/calculate_prensors_with_graph 26 | - title: calculate_prensors_with_source_paths 27 | path: /api_docs/python/s2t/calculate_prensors_with_source_paths 28 | - title: create_expression_from_file_descriptor_set 29 | path: /api_docs/python/s2t/create_expression_from_file_descriptor_set 30 | - title: create_expression_from_prensor 31 | path: /api_docs/python/s2t/create_expression_from_prensor 32 | - title: create_expression_from_proto 33 | path: /api_docs/python/s2t/create_expression_from_proto 34 | - title: create_path 35 | path: /api_docs/python/s2t/create_path 36 | - title: create_prensor_from_descendant_nodes 37 | path: /api_docs/python/s2t/create_prensor_from_descendant_nodes 38 | - title: create_prensor_from_root_and_children 39 | path: /api_docs/python/s2t/create_prensor_from_root_and_children 40 | - title: get_default_options 41 | path: /api_docs/python/s2t/get_default_options 42 | - title: get_options_with_minimal_checks 43 | path: /api_docs/python/s2t/get_options_with_minimal_checks 44 | - title: get_ragged_tensor 45 | status: deprecated 46 | path: /api_docs/python/s2t/get_ragged_tensor 47 | - title: get_ragged_tensors 48 | status: deprecated 49 | path: /api_docs/python/s2t/get_ragged_tensors 50 | - title: get_sparse_tensor 51 | status: deprecated 52 | path: /api_docs/python/s2t/get_sparse_tensor 53 | - title: get_sparse_tensors 54 | status: deprecated 55 | path: /api_docs/python/s2t/get_sparse_tensors 56 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/all_symbols.md: -------------------------------------------------------------------------------- 1 | # All symbols in Struct2Tensor 2 | 3 | 4 | 5 | ## Primary symbols 6 | * s2t 7 | * s2t.ChildNodeTensor 8 | * s2t.Expression 9 | * s2t.LeafNodeTensor 10 | * s2t.NodeTensor 11 | * s2t.Path 12 | * s2t.Prensor 13 | * s2t.RootNodeTensor 14 | * s2t.Step 15 | * s2t.calculate_prensors 16 | * s2t.calculate_prensors_with_graph 17 | * s2t.calculate_prensors_with_source_paths 18 | * s2t.create_expression_from_file_descriptor_set 19 | * s2t.create_expression_from_prensor 20 | * s2t.create_expression_from_proto 21 | * s2t.create_path 22 | * s2t.create_prensor_from_descendant_nodes 23 | * s2t.create_prensor_from_root_and_children 24 | * s2t.get_default_options 25 | * s2t.get_options_with_minimal_checks 26 | * s2t.get_ragged_tensor 27 | * s2t.get_ragged_tensors 28 | * s2t.get_sparse_tensor 29 | * s2t.get_sparse_tensors -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/calculate_prensors.md: -------------------------------------------------------------------------------- 1 | description: Gets the prensor value of the expressions. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # s2t.calculate_prensors 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Gets the prensor value of the expressions. 24 | 25 | 
26 | s2t.calculate_prensors(
27 |     expressions: Sequence[s2t.Expression],
28 |     options: Optional[calculate_options.Options] = None,
29 |     feed_dict: Optional[Dict[expression.Expression, prensor.Prensor]] = None
30 | ) -> Sequence[s2t.Prensor]
31 |
32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 47 | 50 | 51 | 54 | 57 | 58 | 61 | 65 | 66 |

Args

45 | `expressions` 46 | 48 | expressions to calculate prensors for. 49 |
52 | `options` 53 | 55 | options for calculate(...). 56 |
59 | `feed_dict` 60 | 62 | a dictionary, mapping expression to prensor that will be used 63 | as the initial expression in the expression graph. 64 |
67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 78 | 79 | 80 |

Returns

76 | a list of prensors. 77 |
81 | 82 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/calculate_prensors_with_graph.md: -------------------------------------------------------------------------------- 1 | description: Gets the prensor value of the expressions and the graph used. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # s2t.calculate_prensors_with_graph 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Gets the prensor value of the expressions and the graph used. 24 | 25 | 
26 | s2t.calculate_prensors_with_graph(
27 |     expressions: Sequence[s2t.Expression],
28 |     options: Optional[calculate_options.Options] = None,
29 |     feed_dict: Optional[Dict[expression.Expression, prensor.Prensor]] = None
30 | ) -> Tuple[Sequence[prensor.Prensor], 'ExpressionGraph']
31 |
32 | 33 | 34 | 35 | 36 | 37 | This method is useful for getting information like the protobuf fields parsed 38 | to create an expression. 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 49 | 52 | 53 | 56 | 59 | 60 | 63 | 67 | 68 |

Args

47 | `expressions` 48 | 50 | expressions to calculate prensors for. 51 |
54 | `options` 55 | 57 | options for calculate(...) methods. 58 |
61 | `feed_dict` 62 | 64 | a dictionary, mapping expression to prensor that will be used 65 | as the initial expression in the expression graph. 66 |
69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 80 | 81 | 82 |

Returns

78 | a list of prensors, and the graph used to calculate them. 79 |
83 | 84 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/calculate_prensors_with_source_paths.md: -------------------------------------------------------------------------------- 1 | description: Returns a list of prensor trees, and proto summaries. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # s2t.calculate_prensors_with_source_paths 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Returns a list of prensor trees, and proto summaries. 24 | 25 | 
26 | s2t.calculate_prensors_with_source_paths(
27 |     trees: Sequence[s2t.Expression],
28 |     options: Optional[calculate_options.Options] = None
29 | ) -> Tuple[Sequence[prensor.Prensor], Sequence[ProtoRequirements]]
30 |
31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/create_expression_from_file_descriptor_set.md: -------------------------------------------------------------------------------- 1 | description: Create an expression from a 1D tensor of serialized protos. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # s2t.create_expression_from_file_descriptor_set 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Create an expression from a 1D tensor of serialized protos. 24 | 25 | 
26 | s2t.create_expression_from_file_descriptor_set(
27 |     tensor_of_protos: tf.Tensor,
28 |     proto_name: ProtoFullName,
29 |     file_descriptor_set: FileDescriptorSet,
30 |     message_format: str = 'binary'
31 | ) -> s2t.Expression
32 |
33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 48 | 51 | 52 | 55 | 59 | 60 | 63 | 69 | 70 | 73 | 77 | 78 |

Args

46 | `tensor_of_protos` 47 | 49 | 1D tensor of serialized protos. 50 |
53 | `proto_name` 54 | 56 | fully qualified name (e.g. "some.package.SomeProto") of the 57 | proto in `tensor_of_protos`. 58 |
61 | `file_descriptor_set` 62 | 64 | The FileDescriptorSet proto containing `proto_name`'s 65 | and all its dependencies' FileDescriptorProto. Note that if file1 imports 66 | file2, then file2's FileDescriptorProto must precede file1's in 67 | file_descriptor_set.file. 68 |
71 | `message_format` 72 | 74 | Indicates the format of the protocol buffer: is one of 75 | 'text' or 'binary'. 76 |
79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 90 | 91 | 92 |

Returns

88 | An expression. 89 |
93 | 94 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/create_expression_from_prensor.md: -------------------------------------------------------------------------------- 1 | description: Gets an expression representing the prensor. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # s2t.create_expression_from_prensor 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Gets an expression representing the prensor. 24 | 25 | 
26 | s2t.create_expression_from_prensor(
27 |     t: s2t.Prensor
28 | ) -> s2t.Expression
29 |
30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 45 | 48 | 49 |

Args

43 | `t` 44 | 46 | The prensor to represent. 47 |
50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 61 | 62 | 63 |

Returns

59 | An expression representing the prensor. 60 |
64 | 65 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/create_expression_from_proto.md: -------------------------------------------------------------------------------- 1 | description: Create an expression from a 1D tensor of serialized protos. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # s2t.create_expression_from_proto 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Create an expression from a 1D tensor of serialized protos. 24 | 25 | 
26 | s2t.create_expression_from_proto(
27 |     tensor_of_protos: tf.Tensor,
28 |     desc: descriptor.Descriptor,
29 |     message_format: str = 'binary'
30 | ) -> s2t.Expression
31 |
32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 47 | 50 | 51 | 54 | 57 | 58 | 61 | 65 | 66 |

Args

45 | `tensor_of_protos` 46 | 48 | 1D tensor of serialized protos. 49 |
52 | `desc` 53 | 55 | a descriptor of protos in tensor of protos. 56 |
59 | `message_format` 60 | 62 | Indicates the format of the protocol buffer: is one of 63 | 'text' or 'binary'. 64 |
67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 78 | 79 | 80 |

Returns

76 | An expression. 77 |
81 | 82 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/create_path.md: -------------------------------------------------------------------------------- 1 | description: Create a path from an object. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # s2t.create_path 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Create a path from an object. 24 | 25 | 
26 | s2t.create_path(
27 |     path_source: s2t.Path
28 | ) -> s2t.Path
29 |
30 | 31 | 32 | 33 | 34 | 35 | 36 | #### The BNF for a path is: 37 | 38 | 39 | letter := [A-Za-z] 40 | digit := [0-9] 41 | := "_"|"-"| | letter | digit 42 | := + 43 | := "(" ( ".")* ")" 44 | := | 45 | := (( ".") * )? 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 58 | 61 | 62 |

Args

56 | `path_source` 57 | 59 | a string or a Path object. 60 |
63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 74 | 75 | 76 |

Returns

72 | A Path. 73 |
77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 89 | 92 | 93 |

Raises

87 | `ValueError` 88 | 90 | if this is not a valid path. 91 |
94 | 95 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/create_prensor_from_descendant_nodes.md: -------------------------------------------------------------------------------- 1 | description: Create a prensor from a map of paths to NodeTensor. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # s2t.create_prensor_from_descendant_nodes 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Create a prensor from a map of paths to NodeTensor. 24 | 25 | 
26 | s2t.create_prensor_from_descendant_nodes(
27 |     nodes: Mapping[s2t.Path, s2t.NodeTensor]
28 | ) -> "Prensor"
29 |
30 | 31 | 32 | 33 | 34 | 35 | If a path is a key in the map, all prefixes of that path must be present. 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 46 | 49 | 50 |

Args

44 | `nodes` 45 | 47 | A map from paths to NodeTensors. 48 |
51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 62 | 63 | 64 |

Returns

60 | A Prensor. 61 |
65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 77 | 80 | 81 |

Raises

75 | `ValueError` 76 | 78 | if there is a prefix of a path missing. 79 |
82 | 83 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/create_prensor_from_root_and_children.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 |
5 | 6 | # s2t.create_prensor_from_root_and_children 7 | 8 | 9 | 10 | 11 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 
24 | s2t.create_prensor_from_root_and_children(
25 |     root: s2t.NodeTensor,
26 |     children: Mapping[s2t.Step, s2t.Prensor]
27 | ) -> s2t.Prensor
28 |
29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/get_default_options.md: -------------------------------------------------------------------------------- 1 | description: Get the default options. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # s2t.get_default_options 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Get the default options. 24 | 25 | 
26 | s2t.get_default_options() -> Options
27 |
28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/get_options_with_minimal_checks.md: -------------------------------------------------------------------------------- 1 | description: Options for calculation with minimal runtime checks. 2 | 3 |
4 | 5 | 6 |
7 | 8 | # s2t.get_options_with_minimal_checks 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Options for calculation with minimal runtime checks. 24 | 25 | 
26 | s2t.get_options_with_minimal_checks() -> Options
27 |
28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/get_ragged_tensor.md: -------------------------------------------------------------------------------- 1 | description: Get a ragged tensor for a path. (deprecated) 2 | 3 |
4 | 5 | 6 |
7 | 8 | # s2t.get_ragged_tensor 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Get a ragged tensor for a path. (deprecated) 24 | 25 | 
26 | s2t.get_ragged_tensor(
27 |     t: s2t.Prensor,
28 |     p: s2t.Path,
29 |     options: calculate_options.Options = calculate_options.get_default_options()
30 | ) -> tf.RaggedTensor
31 |
32 | 33 | 34 | 35 | 36 | 37 | Warning: THIS FUNCTION IS DEPRECATED. It will be removed in a future version. 38 | Instructions for updating: 39 | Use the Prensor class method instead. 40 | 41 | All steps are represented in the ragged tensor. 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 52 | 55 | 56 | 59 | 62 | 63 | 66 | 69 | 70 |

Args

50 | `t` 51 | 53 | The Prensor to extract tensors from. 54 |
57 | `p` 58 | 60 | the path to a leaf node in `t`. 61 |
64 | `options` 65 | 67 | used to pass options for calculating ragged tensors. 68 |
71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 83 | 84 | 85 |

Returns

80 | A ragged tensor containing values of the leaf node, preserving the 81 | structure along the path. Raises an error if the path is not found. 82 |
86 | 87 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/get_ragged_tensors.md: -------------------------------------------------------------------------------- 1 | description: Gets ragged tensors for all the leaves of the prensor expression. (deprecated) 2 | 3 |
4 | 5 | 6 |
7 | 8 | # s2t.get_ragged_tensors 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Gets ragged tensors for all the leaves of the prensor expression. (deprecated) 24 | 25 | 
26 | s2t.get_ragged_tensors(
27 |     t: s2t.Prensor,
28 |     options: calculate_options.Options = calculate_options.get_default_options()
29 | ) -> Mapping[s2t.Path, tf.RaggedTensor]
30 |
31 | 32 | 33 | 34 | 35 | 36 | Warning: THIS FUNCTION IS DEPRECATED. It will be removed in a future version. 37 | Instructions for updating: 38 | Use the Prensor class method instead. 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 49 | 52 | 53 | 56 | 59 | 60 |

Args

47 | `t` 48 | 50 | The Prensor to extract tensors from. 51 |
54 | `options` 55 | 57 | used to pass options for calculating ragged tensors. 58 |
61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 72 | 73 | 74 |

Returns

70 | A map from paths to ragged tensors. 71 |
75 | 76 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/get_sparse_tensor.md: -------------------------------------------------------------------------------- 1 | description: Gets a sparse tensor for path p. (deprecated) 2 | 3 |
4 | 5 | 6 |
7 | 8 | # s2t.get_sparse_tensor 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Gets a sparse tensor for path p. (deprecated) 24 | 25 | 
26 | s2t.get_sparse_tensor(
27 |     t: s2t.Prensor,
28 |     p: s2t.Path,
29 |     options: calculate_options.Options = calculate_options.get_default_options()
30 | ) -> tf.SparseTensor
31 |
32 | 33 | 34 | 35 | 36 | 37 | Warning: THIS FUNCTION IS DEPRECATED. It will be removed in a future version. 38 | Instructions for updating: 39 | Use the Prensor class method instead. 40 | 41 | Note that any optional fields are not registered as dimensions, as they can't 42 | be represented in a sparse tensor. 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 53 | 56 | 57 | 60 | 63 | 64 | 67 | 70 | 71 |

Args

51 | `t` 52 | 54 | The Prensor to extract tensors from. 55 |
58 | `p` 59 | 61 | The path to a leaf node in `t`. 62 |
65 | `options` 66 | 68 | Currently unused. 69 |
72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 84 | 85 | 86 |

Returns

81 | A sparse tensor containing values of the leaf node, preserving the 82 | structure along the path. Raises an error if the path is not found. 83 |
87 | 88 | -------------------------------------------------------------------------------- /g3doc/api_docs/python/s2t/get_sparse_tensors.md: -------------------------------------------------------------------------------- 1 | description: Gets sparse tensors for all the leaves of the prensor expression. (deprecated) 2 | 3 |
4 | 5 | 6 |
7 | 8 | # s2t.get_sparse_tensors 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | Gets sparse tensors for all the leaves of the prensor expression. (deprecated) 24 | 25 | 
26 | s2t.get_sparse_tensors(
27 |     t: s2t.Prensor,
28 |     options: calculate_options.Options = calculate_options.get_default_options()
29 | ) -> Mapping[s2t.Path, tf.SparseTensor]
30 |
31 | 32 | 33 | 34 | 35 | 36 | Warning: THIS FUNCTION IS DEPRECATED. It will be removed in a future version. 37 | Instructions for updating: 38 | Use the Prensor class method instead. 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 49 | 52 | 53 | 56 | 59 | 60 |

Args

47 | `t` 48 | 50 | The Prensor to extract tensors from. 51 |
54 | `options` 55 | 57 | Currently unused. 58 |
61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 72 | 73 | 74 |

Returns

70 | A map from paths to sparse tensors. 71 |
75 | 76 | -------------------------------------------------------------------------------- /struct2tensor/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Import core names for struct2tensor.""" 15 | 16 | # Import calculate API. 17 | from struct2tensor.calculate import calculate_prensors 18 | from struct2tensor.calculate import calculate_prensors_with_graph 19 | from struct2tensor.calculate_options import get_default_options 20 | from struct2tensor.calculate_options import get_options_with_minimal_checks 21 | from struct2tensor.calculate_with_source_paths import calculate_prensors_with_source_paths 22 | 23 | # Import expressions API. 24 | from struct2tensor.create_expression import create_expression_from_prensor 25 | from struct2tensor.expression import Expression 26 | 27 | # Import expression queries API 28 | from struct2tensor.expression_impl.proto import create_expression_from_file_descriptor_set 29 | from struct2tensor.expression_impl.proto import create_expression_from_proto 30 | 31 | # Import path API 32 | from struct2tensor.path import create_path 33 | from struct2tensor.path import Path 34 | from struct2tensor.path import Step 35 | 36 | # Import prensor API 37 | from struct2tensor.prensor import ChildNodeTensor 38 | from struct2tensor.prensor import create_prensor_from_descendant_nodes 39 | from struct2tensor.prensor import create_prensor_from_root_and_children 40 | from struct2tensor.prensor import LeafNodeTensor 41 | from struct2tensor.prensor import NodeTensor 42 | from struct2tensor.prensor import Prensor 43 | from struct2tensor.prensor import RootNodeTensor 44 | 45 | # TODO(b/163167832): Remove these after 0.32.0 is released. 46 | from struct2tensor.prensor_util import get_ragged_tensor 47 | from struct2tensor.prensor_util import get_ragged_tensors 48 | from struct2tensor.prensor_util import get_sparse_tensor 49 | from struct2tensor.prensor_util import get_sparse_tensors 50 | 51 | # Importing this will register the session handler for PrensorValue, and 52 | # tf.compat.v1.Session.run() will be able to take a Prensor and return a 53 | # PrensorValue. 54 | import struct2tensor.prensor_value 55 | -------------------------------------------------------------------------------- /struct2tensor/benchmarks/g3doc/images/broadcast_depth_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/g3doc/images/broadcast_depth_2.png -------------------------------------------------------------------------------- /struct2tensor/benchmarks/g3doc/images/broadcast_depth_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/g3doc/images/broadcast_depth_5.png -------------------------------------------------------------------------------- /struct2tensor/benchmarks/g3doc/images/project_deep_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/g3doc/images/project_deep_1.png -------------------------------------------------------------------------------- /struct2tensor/benchmarks/g3doc/images/project_deep_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/g3doc/images/project_deep_5.png -------------------------------------------------------------------------------- /struct2tensor/benchmarks/g3doc/images/project_flat_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/g3doc/images/project_flat_1.png -------------------------------------------------------------------------------- /struct2tensor/benchmarks/g3doc/images/project_flat_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/g3doc/images/project_flat_5.png -------------------------------------------------------------------------------- /struct2tensor/benchmarks/g3doc/images/promote_depth_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/g3doc/images/promote_depth_1.png -------------------------------------------------------------------------------- /struct2tensor/benchmarks/g3doc/images/promote_depth_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/g3doc/images/promote_depth_4.png -------------------------------------------------------------------------------- /struct2tensor/benchmarks/g3doc/images/reroot_depth_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/g3doc/images/reroot_depth_1.png -------------------------------------------------------------------------------- /struct2tensor/benchmarks/g3doc/images/reroot_depth_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/g3doc/images/reroot_depth_4.png -------------------------------------------------------------------------------- /struct2tensor/benchmarks/testdata/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | licenses(["notice"]) 16 | 17 | package( 18 | default_visibility = ["//struct2tensor/benchmarks:__subpackages__"], 19 | ) 20 | 21 | filegroup( 22 | name = "data_files", 23 | srcs = glob(["*.tfrecord.gz"]), 24 | ) 25 | -------------------------------------------------------------------------------- /struct2tensor/benchmarks/testdata/deep_all_types_4096.tfrecord.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/testdata/deep_all_types_4096.tfrecord.gz -------------------------------------------------------------------------------- /struct2tensor/benchmarks/testdata/deep_all_types_4096_positive.tfrecord.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/testdata/deep_all_types_4096_positive.tfrecord.gz -------------------------------------------------------------------------------- /struct2tensor/benchmarks/testdata/flat_100_int_features_4096.tfrecord.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/testdata/flat_100_int_features_4096.tfrecord.gz -------------------------------------------------------------------------------- /struct2tensor/benchmarks/testdata/flat_all_types_100_int_values_4096.tfrecord.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/testdata/flat_all_types_100_int_values_4096.tfrecord.gz -------------------------------------------------------------------------------- /struct2tensor/benchmarks/testdata/flat_all_types_4096.tfrecord.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/testdata/flat_all_types_4096.tfrecord.gz -------------------------------------------------------------------------------- /struct2tensor/benchmarks/testdata/flat_all_types_4096_positive.tfrecord.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/testdata/flat_all_types_4096_positive.tfrecord.gz -------------------------------------------------------------------------------- /struct2tensor/benchmarks/testdata/tf_example_100_int_features_4096.tfrecord.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/testdata/tf_example_100_int_features_4096.tfrecord.gz -------------------------------------------------------------------------------- /struct2tensor/benchmarks/testdata/tf_example_1_int_feature_100_values_4096.tfrecord.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/testdata/tf_example_1_int_feature_100_values_4096.tfrecord.gz -------------------------------------------------------------------------------- /struct2tensor/benchmarks/testdata/tf_example_all_types_4096.tfrecord.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/benchmarks/testdata/tf_example_all_types_4096.tfrecord.gz -------------------------------------------------------------------------------- /struct2tensor/calculate_options.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Set options for struct2tensor. 15 | 16 | This object can be passed to several methods. It is passed 17 | as an argument to calculate, get_sparse_tensor, and get_ragged_tensor. 18 | 19 | """ 20 | 21 | 22 | class Options(object): 23 | """Options for calculate functions. 24 | 25 | Do not construct Options directly. The preferred method of creating an object 26 | is calling get_default_options() or get_options_with_minimal_checks() below. 27 | Any fine-tuning can be done by modifying the properties of the Options object 28 | after creation. 29 | 30 | When a method takes an optional Options object but none is provided, it will 31 | replace it with get_default_options() . 32 | 33 | Available options: 34 | ragged_checks: if True, add assertion ops when converting a Prensor object 35 | to RaggedTensors. 36 | sparse_checks: if True, add assertion ops when converting a Prensor object 37 | to SparseTensors. 38 | use_string_view: if True, decode sub-messages into string views to avoid 39 | copying. 40 | experimental_honor_proto3_optional_semantics: if True, if a proto3 primitive 41 | optional field without the presence semantic (i.e. the field is without 42 | the "optional" or "repeated" label) is requested to be parsed, it will 43 | always have a value for each input parent message. If a value is not 44 | present on wire, the default value (0 or "") will be used. 45 | """ 46 | 47 | def __init__(self, ragged_checks: bool, sparse_checks: bool): 48 | """Create options.""" 49 | self.ragged_checks = ragged_checks 50 | self.sparse_checks = sparse_checks 51 | self.use_string_view = False 52 | self.experimental_honor_proto3_optional_semantics = False 53 | 54 | def __str__(self): 55 | return ("{ragged_checks:" + str(self.ragged_checks) + ", sparse_checks: " + 56 | str(self.sparse_checks) + "}") 57 | 58 | 59 | def get_default_options() -> Options: 60 | """Get the default options.""" 61 | return Options(ragged_checks=True, sparse_checks=True) 62 | 63 | 64 | def get_options_with_minimal_checks() -> Options: 65 | """Options for calculation with minimal runtime checks.""" 66 | return Options(ragged_checks=False, sparse_checks=False) 67 | -------------------------------------------------------------------------------- /struct2tensor/expression_impl/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Import all modules in expression_impl. 16 | 17 | The modules in this file should be accessed like the following: 18 | 19 | ``` 20 | import struct2tensor as s2t 21 | from struct2tensor import expression_impl 22 | 23 | s2t.expression_impl.apply_schema 24 | ``` 25 | """ 26 | 27 | from struct2tensor.expression_impl import apply_schema 28 | from struct2tensor.expression_impl import broadcast 29 | from struct2tensor.expression_impl import depth_limit 30 | from struct2tensor.expression_impl import filter_expression 31 | from struct2tensor.expression_impl import index 32 | from struct2tensor.expression_impl import map_prensor 33 | from struct2tensor.expression_impl import map_prensor_to_prensor 34 | from struct2tensor.expression_impl import map_values 35 | from struct2tensor.expression_impl import parquet 36 | from struct2tensor.expression_impl import placeholder 37 | from struct2tensor.expression_impl import project 38 | from struct2tensor.expression_impl import promote 39 | from struct2tensor.expression_impl import promote_and_broadcast 40 | from struct2tensor.expression_impl import proto 41 | from struct2tensor.expression_impl import reroot 42 | from struct2tensor.expression_impl import size 43 | from struct2tensor.expression_impl import slice_expression 44 | -------------------------------------------------------------------------------- /struct2tensor/expression_impl/project_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Tests for struct2tensor.project.""" 15 | 16 | from absl.testing import absltest 17 | from struct2tensor import create_expression 18 | from struct2tensor import path 19 | from struct2tensor.expression_impl import project 20 | from struct2tensor.test import prensor_test_util 21 | 22 | 23 | class ProjectTest(absltest.TestCase): 24 | 25 | def test_project(self): 26 | expr = create_expression.create_expression_from_prensor( 27 | prensor_test_util.create_nested_prensor()) 28 | projected = project.project( 29 | expr, [path.Path(["user", "friends"]), 30 | path.Path(["doc", "keep_me"])]) 31 | self.assertIsNotNone( 32 | projected.get_descendant(path.Path(["user", "friends"]))) 33 | self.assertIsNotNone( 34 | projected.get_descendant(path.Path(["doc", "keep_me"]))) 35 | self.assertIsNone(projected.get_descendant(path.Path(["doc", "bar"]))) 36 | 37 | def test_project_lenient_format(self): 38 | expr = create_expression.create_expression_from_prensor( 39 | prensor_test_util.create_nested_prensor_with_lenient_field_names(), 40 | validate_step_format=False, 41 | ) 42 | projected = project.project( 43 | expr, [path.Path(["doc", "keep_me/x"], validate_step_format=False)] 44 | ) 45 | self.assertLen(projected.get_known_descendants(), 3) 46 | 47 | 48 | if __name__ == "__main__": 49 | absltest.main() 50 | -------------------------------------------------------------------------------- /struct2tensor/expression_impl/promote_and_broadcast_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Tests for struct2tensor.broadcast.""" 15 | 16 | from absl.testing import absltest 17 | from struct2tensor import create_expression 18 | from struct2tensor import path 19 | from struct2tensor.expression_impl import promote_and_broadcast 20 | from struct2tensor.test import expression_test_util 21 | from struct2tensor.test import prensor_test_util 22 | import tensorflow as tf 23 | 24 | 25 | class PromoteAndBroadcastTest(absltest.TestCase): 26 | 27 | def test_promote_and_broadcast_anonymous(self): 28 | """A basic promote and broadcast.""" 29 | expr = create_expression.create_expression_from_prensor( 30 | prensor_test_util.create_big_prensor()) 31 | new_root, p = promote_and_broadcast.promote_and_broadcast_anonymous( 32 | expr, path.Path(["user", "friends"]), path.Path(["doc"])) 33 | new_field = new_root.get_descendant_or_error(p) 34 | self.assertTrue(new_field.is_repeated) 35 | self.assertEqual(new_field.type, tf.string) 36 | self.assertTrue(new_field.is_leaf) 37 | self.assertTrue(new_field.calculation_equal(new_field)) 38 | self.assertFalse(new_field.calculation_equal(expr)) 39 | leaf_node = expression_test_util.calculate_value_slowly(new_field) 40 | self.assertEqual(leaf_node.values.dtype, tf.string) 41 | self.assertEqual(new_field.known_field_names(), frozenset()) 42 | 43 | 44 | if __name__ == "__main__": 45 | absltest.main() 46 | -------------------------------------------------------------------------------- /struct2tensor/kernels/parquet/parquet_reader_util.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Google LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | https://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | #include "parquet/api/reader.h" 16 | #include "tensorflow/core/lib/core/errors.h" 17 | #include "tensorflow/core/lib/core/status.h" 18 | 19 | namespace struct2tensor { 20 | namespace parquet_dataset { 21 | 22 | tensorflow::Status OpenFileWithStatus( 23 | const std::string& filename, 24 | std::unique_ptr* file_reader) { 25 | try { 26 | *file_reader = parquet::ParquetFileReader::OpenFile(filename, false); 27 | return absl::OkStatus(); 28 | } catch (const parquet::ParquetException& e) { 29 | return tensorflow::errors::Internal( 30 | absl::StrCat("Invalid File: ", filename)); 31 | } 32 | } 33 | } // namespace parquet_dataset 34 | } // namespace struct2tensor 35 | -------------------------------------------------------------------------------- /struct2tensor/kernels/parquet/parquet_reader_util.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Google LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | https://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | #ifndef THIRD_PARTY_PY_STRUCT2TENSOR_GOOGLE_PARQUET_KERNEL_PARQUET_READER_UTIL_H_ 16 | #define THIRD_PARTY_PY_STRUCT2TENSOR_GOOGLE_PARQUET_KERNEL_PARQUET_READER_UTIL_H_ 17 | 18 | #include "parquet/api/reader.h" 19 | #include "tensorflow/core/lib/core/errors.h" 20 | #include "tensorflow/core/lib/core/status.h" 21 | 22 | namespace struct2tensor { 23 | namespace parquet_dataset { 24 | 25 | // This wraps parquet's open to handle exceptions. It's a standalone library 26 | // to be compiled with exception enabled so that the rest of the kernel can be 27 | // exception free. 28 | // Sets file_reader as the file handle and returns status::ok() if successful. 29 | // Returns an internal error if the file is unable to be opened. 30 | tensorflow::Status OpenFileWithStatus( 31 | const std::string& filename, 32 | std::unique_ptr* file_reader); 33 | } // namespace parquet_dataset 34 | } // namespace struct2tensor 35 | 36 | #endif // THIRD_PARTY_PY_STRUCT2TENSOR_GOOGLE_PARQUET_KERNEL_PARQUET_READER_UTIL_H_ 37 | -------------------------------------------------------------------------------- /struct2tensor/kernels/run_length_before_op.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2019 Google LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | https://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | // An op on a 1-D tensor that, given [a_0,...,a_n], returns [b_0,...,b_n] where: 16 | // b_n := \sum_{i=0}^{n-1} I(a_i=a_n) 17 | // This assumes that for all a_i, a_j, if i <= j, then a_i <= a_j. 18 | 19 | #include "tensorflow/core/framework/op.h" 20 | #include "tensorflow/core/framework/op_kernel.h" 21 | #include "tensorflow/core/framework/shape_inference.h" 22 | #include "tensorflow/core/framework/tensor_types.h" 23 | #include "tensorflow/core/platform/types.h" 24 | 25 | namespace struct2tensor { 26 | 27 | namespace { 28 | 29 | using ::tensorflow::DEVICE_CPU; 30 | using ::tensorflow::OpKernel; 31 | using ::tensorflow::OpKernelConstruction; 32 | using ::tensorflow::OpKernelContext; 33 | using ::tensorflow::Status; 34 | using ::tensorflow::Tensor; 35 | 36 | class RunLengthBeforeOp : public OpKernel { 37 | public: 38 | explicit RunLengthBeforeOp(OpKernelConstruction* context) 39 | : OpKernel(context) {} 40 | 41 | void Compute(OpKernelContext* context) override { 42 | // Grab the input tensor 43 | const Tensor& input_tensor = context->input(0); 44 | auto input = input_tensor.flat(); 45 | 46 | // Create an output tensor 47 | Tensor* output_tensor = nullptr; 48 | OP_REQUIRES_OK(context, context->allocate_output(0, input_tensor.shape(), 49 | &output_tensor)); 50 | const int64_t input_length = input.size(); 51 | if (input_length > 0) { 52 | typename tensorflow::TTypes::Flat output_flat = 53 | output_tensor->flat(); 54 | 55 | int64_t repeats_so_far = 0; 56 | output_flat(0) = 0; 57 | int64_t last_value = input(0); 58 | for (int64_t i = 1; i < input_length; i++) { 59 | const int64_t current_value = input(i); 60 | if (current_value == last_value) { 61 | ++repeats_so_far; 62 | } else { 63 | repeats_so_far = 0; 64 | } 65 | output_flat(i) = repeats_so_far; 66 | last_value = current_value; 67 | } 68 | } 69 | } 70 | }; 71 | 72 | REGISTER_KERNEL_BUILDER(Name("RunLengthBefore").Device(DEVICE_CPU), 73 | RunLengthBeforeOp); 74 | 75 | } // namespace 76 | } // namespace struct2tensor 77 | -------------------------------------------------------------------------------- /struct2tensor/kernels/vector_to_tensor.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2019 Google LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | https://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | #ifndef THIRD_PARTY_PY_STRUCT2TENSOR_KERNELS_VECTOR_TO_TENSOR_H_ 16 | #define THIRD_PARTY_PY_STRUCT2TENSOR_KERNELS_VECTOR_TO_TENSOR_H_ 17 | 18 | #include "absl/strings/string_view.h" 19 | #include "tensorflow/core/framework/tensor.h" 20 | #include "tensorflow/core/framework/types.h" 21 | #include "tensorflow/core/lib/core/errors.h" 22 | #include "tensorflow/core/platform/tstring.h" 23 | 24 | namespace struct2tensor { 25 | 26 | // Populate `tensor` from a vector of `T`. This assumes `tensor`'s type is also 27 | // `T`, with the exception of int64 types. 28 | template 29 | inline void VectorToTensor(const std::vector& v, tensorflow::Tensor* tensor, 30 | bool produce_string_view) { 31 | std::copy_n(v.begin(), v.size(), tensor->flat().data()); 32 | } 33 | 34 | // Specialization for vector - copies the strings into a string 35 | // tensor. 36 | template <> 37 | inline void VectorToTensor( 38 | const std::vector& v, tensorflow::Tensor* tensor, 39 | bool produce_string_view) { 40 | tensorflow::tstring* output = tensor->flat().data(); 41 | for (auto sv : v) { 42 | if (produce_string_view) { 43 | (output++)->assign_as_view(sv); 44 | } else { 45 | (output++)->assign(sv.data(), sv.size()); 46 | } 47 | } 48 | } 49 | } // namespace struct2tensor 50 | 51 | #endif // THIRD_PARTY_PY_STRUCT2TENSOR_KERNELS_VECTOR_TO_TENSOR_H_ 52 | -------------------------------------------------------------------------------- /struct2tensor/ops/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | -------------------------------------------------------------------------------- /struct2tensor/ops/equi_join_any_indices_op.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2021 Google LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | https://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/op.h" 17 | #include "tensorflow/core/framework/op_kernel.h" 18 | #include "tensorflow/core/framework/shape_inference.h" 19 | 20 | using tensorflow::shape_inference::InferenceContext; 21 | 22 | REGISTER_OP("EquiJoinAnyIndices") 23 | .Input("a: int64") 24 | .Input("b: int64") 25 | .Output("index_a: int64") 26 | .Output("index_b: int64") 27 | .SetShapeFn([](InferenceContext* c) { 28 | c->set_output(0, c->Vector(InferenceContext::kUnknownDim)); 29 | c->set_output(1, c->Vector(InferenceContext::kUnknownDim)); 30 | return absl::OkStatus(); 31 | }) 32 | .Doc(R"doc( 33 | This op is similiar to EquiJoinIndices. However this op does not assume that 34 | `a` and `b` are monotonically increasing. Prefer to use EquiJoinIndices if 35 | possible. 36 | 37 | )doc"); 38 | -------------------------------------------------------------------------------- /struct2tensor/ops/equi_join_indices_op.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2019 Google LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | https://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/op.h" 17 | #include "tensorflow/core/framework/op_kernel.h" 18 | #include "tensorflow/core/framework/shape_inference.h" 19 | 20 | using tensorflow::shape_inference::InferenceContext; 21 | 22 | REGISTER_OP("EquiJoinIndices") 23 | .Input("a: int64") 24 | .Input("b: int64") 25 | .Output("index_a: int64") 26 | .Output("index_b: int64") 27 | .SetShapeFn([](InferenceContext* c) { 28 | c->set_output(0, c->Vector(InferenceContext::kUnknownDim)); 29 | c->set_output(1, c->Vector(InferenceContext::kUnknownDim)); 30 | return absl::OkStatus(); 31 | }) 32 | .Doc(R"doc( 33 | An op on two 1-D int64 tensors a,b that 34 | returns two 1-D int64 tensors [index_a, index_b] where: 35 | 1. For every k, a[index_a[k]] = b[index_b[k]] 36 | 2. For every i,j, iff a[i]==b[j], then there exists a k where 37 | index_a[k]=i and index_b[k]=j. 38 | 3. For any k, k' where k < k', 39 | index_a[k] <= index_a[k'], and if index_a[k] == index_a[k'], then 40 | index_b[k] <= index_b[k']. 41 | 42 | Imagine if you had two tables, A with fields "a_key" and "a_value", and 43 | B with fields "b_key" and "b_value", where a_key is monotonically increasing 44 | int64, and b_key is monotonically increasing int64. 45 | 46 | C = SELECT * FROM A, B WHERE A.a_key = B.b_key; 47 | 48 | Imagine that A.a_key, B.b_key, A.a_value, and B.b_value are all 1-D tensors. 49 | 50 | Then we can create the result C: 51 | a_index, b_index = equi_join_indices(A.a_key,B.b_key) 52 | C.a_key = tf.gather(A.a_key, a_index) 53 | C.a_value = tf.gather(A.a_value, a_index) 54 | C.b_key = tf.gather(B.b_key, b_index) 55 | C.b_value = tf.gather(B.b_value, b_index) 56 | 57 | 58 | a: a 1-D tensor where for all i, a[i] <= a[i+1] 59 | b: a 1-D tensor where for all i, b[i] <= b[i+1] 60 | index_a: a 1-D tensor of indices of a 61 | index_b: a 1-D tensor of indices of b 62 | 63 | )doc"); 64 | -------------------------------------------------------------------------------- /struct2tensor/ops/file_descriptor_set_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Tests for struct2tensor.ops.file_descriptor_set.""" 16 | 17 | from absl.testing import absltest 18 | from struct2tensor.ops import file_descriptor_set 19 | from struct2tensor.test import dependent_test_pb2 20 | 21 | # Notice that while text_extension_pb2 is not used directly, 22 | # it must be linked in and imported so that the extension can be found in the 23 | # pool. 24 | from struct2tensor.test import test_extension_pb2 # pylint: disable=unused-import 25 | from struct2tensor.test import test_map_pb2 26 | from struct2tensor.test import test_pb2 27 | 28 | 29 | def _get_base_directory(): 30 | return "" # pylint: disable=unreachable 31 | 32 | 33 | class FileDescriptorSetTest(absltest.TestCase): 34 | 35 | def test_get_file_descriptor_set_proto_simple_test_map(self): 36 | file_set_proto = file_descriptor_set.get_file_descriptor_set_proto( 37 | test_map_pb2.SubMessage.DESCRIPTOR, []) 38 | self.assertLen(file_set_proto.file, 1) 39 | self.assertEqual( 40 | file_set_proto.file[0].name, 41 | _get_base_directory() + "struct2tensor/test/test_map.proto") 42 | 43 | 44 | if __name__ == "__main__": 45 | absltest.main() 46 | -------------------------------------------------------------------------------- /struct2tensor/ops/gen_decode_proto_map_op.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Wrapper for _decode_proto_map_op.so.""" 15 | 16 | from tensorflow.python.framework import load_library 17 | from tensorflow.python.platform import resource_loader 18 | 19 | decode_proto_map_module = load_library.load_op_library( 20 | resource_loader.get_path_to_datafile('_decode_proto_map_op.so')) 21 | 22 | decode_proto_map = decode_proto_map_module.decode_proto_map 23 | decode_proto_map_v2 = decode_proto_map_module.decode_proto_map_v2 24 | -------------------------------------------------------------------------------- /struct2tensor/ops/gen_decode_proto_sparse.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Wrapper for _decode_proto_sparse_op.so.""" 15 | 16 | from tensorflow.python.framework import load_library 17 | from tensorflow.python.platform import resource_loader 18 | 19 | decode_proto_sparse_module = load_library.load_op_library( 20 | resource_loader.get_path_to_datafile('_decode_proto_sparse_op.so')) 21 | 22 | decode_proto_sparse_v2 = decode_proto_sparse_module.decode_proto_sparse_v2 23 | decode_proto_sparse_v3 = decode_proto_sparse_module.decode_proto_sparse_v3 24 | decode_proto_sparse_v4 = decode_proto_sparse_module.decode_proto_sparse_v4 25 | -------------------------------------------------------------------------------- /struct2tensor/ops/gen_equi_join_any_indices.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Wrapper for _equi_join_any_indices_op.so.""" 15 | 16 | from tensorflow.python.framework import load_library 17 | from tensorflow.python.platform import resource_loader 18 | 19 | equi_join_any_indices_module = load_library.load_op_library( 20 | resource_loader.get_path_to_datafile('_equi_join_any_indices_op.so')) 21 | equi_join_any_indices = equi_join_any_indices_module.equi_join_any_indices 22 | -------------------------------------------------------------------------------- /struct2tensor/ops/gen_equi_join_indices.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Wrapper for _equi_join_indices_op.so.""" 15 | 16 | from tensorflow.python.framework import load_library 17 | from tensorflow.python.platform import resource_loader 18 | 19 | equi_join_indices_module = load_library.load_op_library( 20 | resource_loader.get_path_to_datafile('_equi_join_indices_op.so')) 21 | equi_join_indices = equi_join_indices_module.equi_join_indices 22 | -------------------------------------------------------------------------------- /struct2tensor/ops/gen_parquet_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Wrapper for _parquet_dataset_op.so.""" 15 | 16 | from tensorflow.python.framework import load_library 17 | from tensorflow.python.platform import resource_loader 18 | 19 | parquet_dataset_module = load_library.load_op_library( 20 | resource_loader.get_path_to_datafile('_parquet_dataset_op.so')) 21 | 22 | parquet_dataset = parquet_dataset_module.parquet_dataset 23 | -------------------------------------------------------------------------------- /struct2tensor/ops/gen_run_length_before.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Wrapper for _run_length_before_op.so.""" 15 | 16 | from tensorflow.python.framework import load_library 17 | from tensorflow.python.platform import resource_loader 18 | 19 | run_length_before_module = load_library.load_op_library( 20 | resource_loader.get_path_to_datafile('_run_length_before_op.so')) 21 | run_length_before = run_length_before_module.run_length_before 22 | -------------------------------------------------------------------------------- /struct2tensor/ops/op_kernel_registration_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Google LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | https://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | #include 16 | 17 | #include 18 | #include "tensorflow/core/framework/op.h" 19 | #include "tensorflow/core/framework/op_kernel.h" 20 | 21 | namespace { 22 | 23 | TEST(OpAndKernelRegistrationTest, Struct2TensorOpsAndKernelsAreRegistered) { 24 | static constexpr char const* kStruct2TensorOps[] = { 25 | "DecodeProtoMap", 26 | "EquiJoinIndices", 27 | "EquiJoinAnyIndices", 28 | "DecodeProtoSparseV3", 29 | "RunLengthBefore", 30 | "ParquetDataset", 31 | }; 32 | 33 | const auto* global_op_registry = tensorflow::OpRegistry::Global(); 34 | for (const char* op_name : kStruct2TensorOps) { 35 | const tensorflow::OpRegistrationData* ord; 36 | EXPECT_TRUE(global_op_registry->LookUp(op_name, &ord).ok()) << op_name; 37 | EXPECT_EQ(1, tensorflow::GetRegisteredKernelsForOp(op_name).kernel_size()) 38 | << op_name; 39 | } 40 | } 41 | 42 | } // namespace 43 | -------------------------------------------------------------------------------- /struct2tensor/ops/run_length_before_op.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2019 Google LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | https://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/op.h" 17 | #include "tensorflow/core/framework/shape_inference.h" 18 | 19 | REGISTER_OP("RunLengthBefore") 20 | .Input("ordered_indices: int64") 21 | .Output("run_length_before: int64") 22 | .SetShapeFn([](::tensorflow::shape_inference::InferenceContext* context) { 23 | context->set_output(0, context->input(0)); 24 | return absl::OkStatus(); 25 | }) 26 | .Doc(R"doc( 27 | The `run_length_before` op, given [a_0,...,a_n], returns [b_0,...,b_n] where: 28 | b_n := \sum_{i=0}^{n-1} I(a_i=a_n) 29 | This assumes that for all a_i, a_j, if i <= j, then a_i <= a_j. 30 | 31 | This is useful for creating the last index column of a ragged array, or from 32 | converting from global orderings to local orderings or dewey orderings. 33 | 34 | For example: 35 | input: [0, 0, 7, 7, 8, 9, 9] 36 | output: [0, 1, 0, 1, 0, 0, 1] 37 | 38 | ordered_indices: a int64 vector where for all i, a[i] <= a[i+1] 39 | run_length_before: for all n: 40 | run_length_before[n] := \sum_{i=0}^{n-1} I(a_i=a_n) 41 | 42 | )doc"); 43 | -------------------------------------------------------------------------------- /struct2tensor/proto/BUILD: -------------------------------------------------------------------------------- 1 | load("//struct2tensor:struct2tensor.bzl", "s2t_proto_library", "s2t_proto_library_cc", "s2t_proto_library_py") 2 | 3 | package( 4 | default_visibility = ["@//:__subpackages__"], 5 | ) 6 | 7 | licenses(["notice"]) 8 | 9 | s2t_proto_library( 10 | name = "query_metadata_proto", 11 | srcs = ["query_metadata.proto"], 12 | deps = ["@com_github_tensorflow_metadata//tensorflow_metadata/proto/v0:cc_metadata_v0_proto_cc"], 13 | ) 14 | 15 | s2t_proto_library_cc( 16 | name = "query_metadata_cc_proto", 17 | deps = [":query_metadata_proto"], 18 | ) 19 | 20 | s2t_proto_library_py( 21 | name = "query_metadata_py_pb2", 22 | srcs = ["query_metadata.proto"], 23 | api_version = 2, 24 | oss_deps = [ 25 | "@com_github_tensorflow_metadata//tensorflow_metadata/proto/v0:py_metadata_v0_proto_py", 26 | ], 27 | proto_library = "query_metadata_proto", 28 | ) 29 | -------------------------------------------------------------------------------- /struct2tensor/proto/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | -------------------------------------------------------------------------------- /struct2tensor/proto/query_metadata.proto: -------------------------------------------------------------------------------- 1 | /* Copyright 2019 Google LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | https://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | // Contains messages related to Prensor query metadata. 16 | syntax = "proto3"; 17 | 18 | package struct2tensor; 19 | 20 | import "tensorflow_metadata/proto/v0/path.proto"; 21 | 22 | // Information about a proto parsed by a Prensor query. 23 | // Next TAG to use: 3 24 | message ParsedProtoInfo { 25 | // The fully qualified name of the proto message parsed by the query. 26 | string message_name = 1; 27 | // All the proto field paths that will be parsed by the query. 28 | repeated tensorflow.metadata.v0.Path field_paths = 2; 29 | } 30 | 31 | // Metadata of a PrensorQuery. 32 | // Next TAG to use: 4 33 | message QueryMetadata { 34 | string message_name = 1 [deprecated = true]; 35 | repeated string field_paths_to_parse = 2 [deprecated = true]; 36 | // Contains information about the protos being parsed by the query. There 37 | // could be multiple types of protos parsed by a single query (i.e. a 38 | // compilation of projections.) 39 | repeated ParsedProtoInfo parsed_proto_info = 3; 40 | } 41 | -------------------------------------------------------------------------------- /struct2tensor/struct2tensor_module_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Tests for struct2tensor.__init__.py.""" 15 | 16 | from absl.testing import absltest 17 | import struct2tensor as s2t 18 | 19 | 20 | class Struct2tensorModuleTest(absltest.TestCase): 21 | 22 | def test_importing_struct2tensor_modules(self): 23 | """This tests that the exposed packages in root __init__.py are found.""" 24 | # pylint: disable=pointless-statement 25 | 26 | # calculate APIs 27 | s2t.calculate_prensors 28 | s2t.calculate_prensors_with_graph 29 | s2t.get_default_options 30 | s2t.get_options_with_minimal_checks 31 | s2t.calculate_prensors_with_source_paths 32 | 33 | # expression APIs 34 | s2t.create_expression_from_prensor 35 | s2t.create_expression_from_file_descriptor_set 36 | s2t.create_expression_from_proto 37 | s2t.Expression 38 | 39 | # path API 40 | s2t.create_path 41 | s2t.Path 42 | s2t.Step 43 | 44 | # prensor APIs 45 | s2t.ChildNodeTensor 46 | s2t.LeafNodeTensor 47 | s2t.NodeTensor 48 | s2t.Prensor 49 | s2t.RootNodeTensor 50 | s2t.create_prensor_from_descendant_nodes 51 | s2t.create_prensor_from_root_and_children 52 | s2t.prensor_value 53 | # pylint: enable=pointless-statement 54 | 55 | def test_importing_expression_impl_modules(self): 56 | """This tests that the expression_impl/__init__.py imports are found.""" 57 | 58 | from struct2tensor import expression_impl # pylint: disable=g-import-not-at-top 59 | 60 | modules = [ 61 | 'apply_schema', 'broadcast', 'depth_limit', 'filter_expression', 62 | 'index', 'map_prensor', 'map_prensor_to_prensor', 'map_values', 63 | 'parquet', 'placeholder', 'project', 'promote', 'promote_and_broadcast', 64 | 'proto', 'reroot', 'size', 'slice_expression' 65 | ] 66 | 67 | for module in modules: 68 | getattr(expression_impl, module) 69 | 70 | 71 | if __name__ == '__main__': 72 | absltest.main() 73 | -------------------------------------------------------------------------------- /struct2tensor/test/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | -------------------------------------------------------------------------------- /struct2tensor/test/dependent_test.proto: -------------------------------------------------------------------------------- 1 | /* Copyright 2019 Google LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | https://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | syntax = "proto2"; 16 | 17 | package struct2tensor.test; 18 | 19 | import "struct2tensor/test/test.proto"; 20 | 21 | message NotSoSimple { 22 | optional AllSimple simple = 1; 23 | } 24 | -------------------------------------------------------------------------------- /struct2tensor/test/test_any.proto: -------------------------------------------------------------------------------- 1 | /* Copyright 2019 Google LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | https://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | syntax = "proto3"; 16 | 17 | package struct2tensor.test; 18 | 19 | import "google/protobuf/any.proto"; 20 | 21 | message MessageWithAny { 22 | google.protobuf.Any my_any = 1; 23 | } 24 | -------------------------------------------------------------------------------- /struct2tensor/test/test_extension.proto: -------------------------------------------------------------------------------- 1 | /* Copyright 2019 Google LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | https://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | syntax = "proto2"; 16 | 17 | package struct2tensor.test; 18 | 19 | 20 | import "struct2tensor/test/test.proto"; 21 | 22 | // Testing placing an extension in an external proto. 23 | message MyExternalExtension { 24 | extend UserInfo { 25 | optional MyExternalExtension ext = 188564378; 26 | } 27 | optional string special = 1; 28 | } 29 | 30 | message MyMessageSet { 31 | 32 | optional string special_message = 1; 33 | } 34 | -------------------------------------------------------------------------------- /struct2tensor/test/test_map.proto: -------------------------------------------------------------------------------- 1 | /* Copyright 2019 Google LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | https://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | syntax = "proto3"; 16 | 17 | package struct2tensor.test; 18 | 19 | message SubMessage { 20 | repeated float repeated_float = 1; 21 | repeated int64 repeated_int64 = 2; 22 | } 23 | 24 | enum TestEnum { 25 | DEFAULT = 0; 26 | BAR = 1; 27 | BAZ = 2; 28 | } 29 | 30 | message MessageWithMap { 31 | map string_message_map = 1; 32 | map int64_message_map = 2; 33 | 34 | map int32_string_map = 3; 35 | map int64_string_map = 4; 36 | map uint64_string_map = 6; 37 | map uint32_string_map = 7; 38 | map fixed64_string_map = 8; 39 | map fixed32_string_map = 9; 40 | map sfixed64_string_map = 10; 41 | map sfixed32_string_map = 11; 42 | map sint64_string_map = 12; 43 | map sint32_string_map = 13; 44 | map string_string_map = 14; 45 | map bool_string_map = 15; 46 | 47 | map string_double_map = 16; 48 | map string_float_map = 17; 49 | map string_int64_map = 18; 50 | map string_int32_map = 19; 51 | map string_uint64_map = 20; 52 | map string_uint32_map = 21; 53 | map string_fixed64_map = 22; 54 | map string_fixed32_map = 23; 55 | map string_sfixed64_map = 24; 56 | map string_sfixed32_map = 25; 57 | map string_sint64_map = 26; 58 | map string_sint32_map = 27; 59 | map string_bool_map = 28; 60 | map string_enum_map = 29; 61 | } 62 | -------------------------------------------------------------------------------- /struct2tensor/test/test_proto3.proto: -------------------------------------------------------------------------------- 1 | /* Copyright 2021 Google LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | https://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | syntax = "proto3"; 16 | 17 | package struct2tensor.test; 18 | 19 | message Proto3Message { 20 | string optional_string = 1; 21 | int32 optional_int32 = 2; 22 | uint32 optional_uint32 = 3; 23 | int64 optional_int64 = 4; 24 | uint64 optional_uint64 = 5; 25 | float optional_float = 6; 26 | double optional_double = 7; 27 | bytes optional_bytes = 8; 28 | repeated string repeated_string = 101; 29 | 30 | Proto3Message nested = 1001; 31 | repeated Proto3Message repeated_nested = 1002; 32 | } 33 | -------------------------------------------------------------------------------- /struct2tensor/testdata/parquet_testdata/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # Sample parquet files for testing. The parquet file schemas are described in 16 | # raw_parquet_dataset_test.py. 17 | 18 | licenses(["notice"]) 19 | 20 | filegroup( 21 | name = "parquet_testdata", 22 | srcs = glob(["**"]), 23 | visibility = ["//visibility:public"], 24 | ) 25 | -------------------------------------------------------------------------------- /struct2tensor/testdata/parquet_testdata/all_data_types.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/testdata/parquet_testdata/all_data_types.parquet -------------------------------------------------------------------------------- /struct2tensor/testdata/parquet_testdata/dremel_example.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/testdata/parquet_testdata/dremel_example.parquet -------------------------------------------------------------------------------- /struct2tensor/testdata/parquet_testdata/dremel_example_two_row_groups.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/struct2tensor/testdata/parquet_testdata/dremel_example_two_row_groups.parquet -------------------------------------------------------------------------------- /struct2tensor/tools/BUILD: -------------------------------------------------------------------------------- 1 | # Placeholder: load py_binary 2 | # Placeholder: load py_library 3 | # Placeholder: load py_test 4 | 5 | # Copyright 2020 Google LLC 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | licenses(["notice"]) 19 | 20 | package( 21 | default_visibility = ["//struct2tensor:__subpackages__"], 22 | ) 23 | 24 | py_library( 25 | name = "build_docs_lib", 26 | srcs = ["build_docs.py"], 27 | deps = [ 28 | "//struct2tensor", 29 | "//struct2tensor:struct2tensor_expression_impl", 30 | "//third_party/py/absl:app", 31 | "//third_party/py/tensorflow_docs/api_generator:generate_lib", 32 | "//third_party/py/tensorflow_docs/api_generator:public_api", 33 | "//third_party/py/yaml", 34 | "@absl_py//absl/flags", 35 | ], 36 | ) 37 | 38 | py_binary( 39 | name = "build_docs", 40 | srcs = ["build_docs.py"], 41 | deps = [ 42 | ":build_docs_lib", 43 | ], 44 | ) 45 | -------------------------------------------------------------------------------- /struct2tensor/tools/docker_build/Dockerfile.manylinux2014: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Dockerfile for building a manylinux2014 struct2tensor wheel. 16 | ARG PYTHON_VERSION 17 | FROM tensorflow/build:latest-python${PYTHON_VERSION} 18 | WORKDIR /struct2tensor 19 | 20 | # Uses manylinux2014 compatible devtoolset. See below for the detail: 21 | # https://github.com/tensorflow/build/blob/master/tf_sig_build_dockerfiles/builder.devtoolset/build_devtoolset.sh 22 | ENV CC=/dt9/usr/bin/gcc 23 | 24 | # tensorflow/build images already contains nightly packages. Clean them up. 25 | RUN pip uninstall -y tf-nightly tb-nightly tf-estimator-nightly keras-nightly 26 | 27 | RUN pip install auditwheel 28 | 29 | # Update binutils to avoid linker(gold) issue. See b/227299577#comment9 30 | RUN wget http://old-releases.ubuntu.com/ubuntu/pool/main/b/binutils/binutils_2.35.1-1ubuntu1_amd64.deb \ 31 | && wget http://old-releases.ubuntu.com/ubuntu/pool/main/b/binutils/binutils-x86-64-linux-gnu_2.35.1-1ubuntu1_amd64.deb \ 32 | && wget http://old-releases.ubuntu.com/ubuntu/pool/main/b/binutils/binutils-common_2.35.1-1ubuntu1_amd64.deb \ 33 | && wget http://old-releases.ubuntu.com/ubuntu/pool/main/b/binutils/libbinutils_2.35.1-1ubuntu1_amd64.deb 34 | RUN dpkg -i binutils_2.35.1-1ubuntu1_amd64.deb \ 35 | binutils-x86-64-linux-gnu_2.35.1-1ubuntu1_amd64.deb \ 36 | binutils-common_2.35.1-1ubuntu1_amd64.deb \ 37 | libbinutils_2.35.1-1ubuntu1_amd64.deb 38 | 39 | CMD ["struct2tensor/tools/docker_build/build_manylinux.sh"] 40 | -------------------------------------------------------------------------------- /struct2tensor/version.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Contains the version string of struct2tensor.""" 16 | 17 | # Note that setup.py uses this version. 18 | __version__ = '0.49.0.dev' 19 | -------------------------------------------------------------------------------- /tf/BUILD: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/struct2tensor/dd7a2521c45793127407e9c32973c11da1da7e16/tf/BUILD -------------------------------------------------------------------------------- /tf/BUILD.tpl: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Template for a BUILD file that defines TF library tagets. 15 | Dynamic libraries and the PIP package in Struct2Tensor depend upon Tensorflow 16 | through these targets. The template is populated by tf_configure.bzl by 17 | running ../configure.sh. 18 | 19 | Static libraries for building a tensorflow serving binary do not depend upon 20 | this. 21 | """ 22 | 23 | 24 | package(default_visibility = ["//visibility:public"]) 25 | 26 | cc_library( 27 | name = "tf_header_lib", 28 | hdrs = [":tf_header_include"], 29 | includes = ["include"], 30 | visibility = ["//visibility:public"], 31 | ) 32 | 33 | cc_library( 34 | name = "libtensorflow_framework", 35 | srcs = [":libtensorflow_framework.so"], 36 | visibility = ["//visibility:public"], 37 | ) 38 | 39 | %{TF_HEADER_GENRULE} 40 | %{TF_SHARED_LIBRARY_GENRULE} 41 | -------------------------------------------------------------------------------- /third_party/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | licenses(["notice"]) 17 | 18 | exports_files([ 19 | "python_configure.bzl", 20 | ]) 21 | -------------------------------------------------------------------------------- /third_party/local_python.BUILD.tpl: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | licenses(["restricted"]) 15 | 16 | package(default_visibility = ["//visibility:public"]) 17 | 18 | # Point both runtimes to the same python binary to ensure we always 19 | # use the python binary specified by ./configure.py script. 20 | load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair") 21 | 22 | py_runtime( 23 | name = "py2_runtime", 24 | interpreter_path = "%{PYTHON_BIN_PATH}", 25 | python_version = "PY2", 26 | ) 27 | 28 | py_runtime( 29 | name = "py3_runtime", 30 | interpreter_path = "%{PYTHON_BIN_PATH}", 31 | python_version = "PY3", 32 | ) 33 | 34 | py_runtime_pair( 35 | name = "py_runtime_pair", 36 | py2_runtime = ":py2_runtime", 37 | py3_runtime = ":py3_runtime", 38 | ) 39 | 40 | toolchain( 41 | name = "py_toolchain", 42 | toolchain = ":py_runtime_pair", 43 | toolchain_type = "@bazel_tools//tools/python:toolchain_type", 44 | target_compatible_with = [%{PLATFORM_CONSTRAINT}], 45 | exec_compatible_with = [%{PLATFORM_CONSTRAINT}], 46 | ) 47 | 48 | # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib 49 | # See https://docs.python.org/3/extending/windows.html 50 | cc_import( 51 | name = "python_lib", 52 | interface_library = select({ 53 | ":windows": ":python_import_lib", 54 | # A placeholder for Unix platforms which makes --no_build happy. 55 | "//conditions:default": "not-existing.lib", 56 | }), 57 | system_provided = 1, 58 | ) 59 | 60 | cc_library( 61 | name = "python_headers", 62 | hdrs = [":python_include"], 63 | deps = select({ 64 | ":windows": [":python_lib"], 65 | "//conditions:default": [], 66 | }), 67 | includes = ["python_include"], 68 | ) 69 | 70 | cc_library( 71 | name = "numpy_headers", 72 | hdrs = [":numpy_include"], 73 | includes = ["numpy_include"], 74 | ) 75 | 76 | config_setting( 77 | name = "windows", 78 | values = {"cpu": "x64_windows"}, 79 | visibility = ["//visibility:public"], 80 | ) 81 | 82 | %{PYTHON_INCLUDE_GENRULE} 83 | %{NUMPY_INCLUDE_GENRULE} 84 | %{PYTHON_IMPORT_LIB_GENRULE} 85 | -------------------------------------------------------------------------------- /third_party/thrift.BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Description: 16 | # Apache Thrift library 17 | load("@org_tensorflow//tensorflow:tensorflow.bzl", "clean_dep") 18 | 19 | package(default_visibility = ["//visibility:public"]) 20 | 21 | licenses(["notice"]) # Apache 2.0 22 | 23 | exports_files(["LICENSE"]) 24 | 25 | cc_library( 26 | name = "thrift", 27 | srcs = glob([ 28 | "lib/cpp/src/thrift/**/*.h", 29 | ]) + [ 30 | "lib/cpp/src/thrift/protocol/TProtocol.cpp", 31 | "lib/cpp/src/thrift/transport/TBufferTransports.cpp", 32 | "lib/cpp/src/thrift/transport/TTransportException.cpp", 33 | ], 34 | hdrs = [ 35 | "compiler/cpp/src/thrift/version.h", 36 | "lib/cpp/src/thrift/config.h", 37 | ], 38 | includes = [ 39 | "lib/cpp/src", 40 | ], 41 | # boost needs functions in librt.so if built with glibc<2.17 (which is the 42 | # case in our manylinux docker image). 43 | linkopts = select({ 44 | clean_dep("@org_tensorflow//tensorflow:macos"): None, 45 | "//conditions:default": ["-lrt"], 46 | }), 47 | textual_hdrs = [ 48 | "lib/cpp/src/thrift/protocol/TBinaryProtocol.tcc", 49 | "lib/cpp/src/thrift/protocol/TCompactProtocol.tcc", 50 | ], 51 | deps = [ 52 | "@boost//:geometry", 53 | ], 54 | ) 55 | 56 | # LINT.IfChange(thrift_gen_version) 57 | genrule( 58 | name = "version_h", 59 | srcs = [ 60 | "compiler/cpp/src/thrift/version.h.in", 61 | ], 62 | outs = [ 63 | "compiler/cpp/src/thrift/version.h", 64 | ], 65 | cmd = "sed 's/@PACKAGE_VERSION@/0.12.0/g' $< > $@", 66 | ) 67 | 68 | genrule( 69 | name = "config_h", 70 | srcs = ["build/cmake/config.h.in"], 71 | outs = ["lib/cpp/src/thrift/config.h"], 72 | cmd = ("sed " + 73 | "-e 's/cmakedefine/define/g' " + 74 | "-e 's/$${PACKAGE}/thrift/g' " + 75 | "-e 's/$${PACKAGE_BUGREPORT}//g' " + 76 | "-e 's/$${PACKAGE_NAME}/thrift/g' " + 77 | "-e 's/$${PACKAGE_TARNAME}/thrift/g' " + 78 | "-e 's/$${PACKAGE_URL}//g' " + 79 | "-e 's/$${PACKAGE_VERSION}/0.12.0/g' " + 80 | "-e 's/$${PACKAGE_STRING}/thrift 0.12.0/g' " + 81 | "$< >$@"), 82 | ) 83 | # LINT.ThenChange(../workspace.bzl:thrift_archive_version) 84 | --------------------------------------------------------------------------------