├── tests ├── __init__.py ├── test_lmdb │ └── data.mdb ├── test_image │ ├── cradle.gif │ ├── sample.png │ ├── small.tiff │ ├── sample.webp │ ├── small-00.png │ ├── small-01.png │ ├── small-02.png │ ├── small-03.png │ └── small-04.png ├── test_libsvm │ └── sample ├── test_mnist │ ├── mnist.npz │ ├── t10k-images-idx3-ubyte │ ├── t10k-images-idx3-ubyte.gz │ └── t10k-labels-idx1-ubyte.gz ├── test_video │ └── small.mp4 ├── test_hadoop │ └── string.seq ├── test_text │ └── lorem.txt.gz ├── test_parquet │ └── parquet_cpp_example.parquet ├── test_ignite │ ├── stop_ignite.sh │ ├── bin │ │ ├── start-igfs.sh │ │ └── start-plain.sh │ ├── sql │ │ └── init.sql │ ├── config │ │ ├── ignite-config-plain.xml │ │ └── ignite-config-igfs.xml │ └── start_ignite.sh ├── test_kinesis │ └── kinesis_test.sh ├── test_pubsub │ └── pubsub_test.sh ├── test_text.py ├── test_lmdb.py └── test_hadoop.py ├── third_party ├── BUILD ├── tf │ ├── BUILD │ └── BUILD.tpl ├── giflib.BUILD ├── lmdb.BUILD ├── zlib.BUILD ├── libwebp.BUILD ├── kafka.patch ├── parquet.header ├── ffmpeg_3_4.BUILD ├── libav_9_20.BUILD ├── ffmpeg_2_8.BUILD └── aws.BUILD ├── tensorflow_io ├── arrow │ ├── python │ │ ├── __init__.py │ │ └── ops │ │ │ └── __init__.py │ ├── BUILD │ ├── __init__.py │ ├── kernels │ │ └── arrow_stream_client.h │ └── ops │ │ └── dataset_ops.cc ├── cifar │ ├── python │ │ ├── __init__.py │ │ └── ops │ │ │ └── __init__.py │ ├── BUILD │ ├── __init__.py │ └── ops │ │ └── dataset_ops.cc ├── image │ ├── python │ │ ├── __init__.py │ │ └── ops │ │ │ └── __init__.py │ ├── BUILD │ ├── __init__.py │ └── ops │ │ └── dataset_ops.cc ├── kafka │ ├── python │ │ ├── __init__.py │ │ └── ops │ │ │ ├── __init__.py │ │ │ └── kafka_ops.py │ ├── BUILD │ ├── __init__.py │ └── ops │ │ ├── kafka_ops.cc │ │ └── dataset_ops.cc ├── lmdb │ ├── python │ │ ├── __init__.py │ │ └── ops │ │ │ ├── __init__.py │ │ │ └── lmdb_dataset_ops.py │ ├── BUILD │ ├── __init__.py │ └── ops │ │ └── dataset_ops.cc ├── mnist │ ├── python │ │ ├── __init__.py │ │ └── ops │ │ │ └── __init__.py │ ├── BUILD │ ├── __init__.py │ └── ops │ │ └── dataset_ops.cc ├── text │ ├── python │ │ ├── __init__.py │ │ └── ops │ │ │ ├── __init__.py │ │ │ └── text_ops.py │ ├── BUILD │ ├── __init__.py │ └── ops │ │ └── text_ops.cc ├── video │ ├── python │ │ ├── __init__.py │ │ └── ops │ │ │ └── __init__.py │ ├── __init__.py │ ├── ops │ │ └── dataset_ops.cc │ ├── BUILD │ └── kernels │ │ └── video_reader.h ├── hadoop │ ├── python │ │ ├── __init__.py │ │ └── ops │ │ │ └── __init__.py │ ├── BUILD │ ├── ops │ │ └── dataset_ops.cc │ └── __init__.py ├── ignite │ ├── python │ │ ├── __init__.py │ │ └── ops │ │ │ ├── __init__.py │ │ │ └── igfs_ops.py │ ├── ops │ │ ├── igfs_ops.cc │ │ └── dataset_ops.cc │ ├── kernels │ │ ├── igfs │ │ │ ├── igfs_client.cc │ │ │ ├── igfs_writable_file.h │ │ │ ├── igfs_random_access_file.h │ │ │ ├── igfs_extended_tcp_client.h │ │ │ ├── igfs_random_access_file.cc │ │ │ ├── igfs_writable_file.cc │ │ │ └── igfs.h │ │ ├── client │ │ │ ├── ignite_plain_client.h │ │ │ └── ignite_ssl_wrapper.h │ │ └── dataset │ │ │ └── ignite_dataset.h │ ├── BUILD │ └── __init__.py ├── kinesis │ ├── python │ │ ├── __init__.py │ │ └── ops │ │ │ └── __init__.py │ ├── BUILD │ ├── __init__.py │ └── ops │ │ └── dataset_ops.cc ├── libsvm │ ├── python │ │ ├── __init__.py │ │ └── ops │ │ │ └── __init__.py │ ├── BUILD │ ├── __init__.py │ └── ops │ │ └── libsvm_ops.cc ├── parquet │ ├── python │ │ ├── __init__.py │ │ └── ops │ │ │ └── __init__.py │ ├── BUILD │ ├── __init__.py │ └── ops │ │ └── dataset_ops.cc ├── pubsub │ ├── python │ │ ├── __init__.py │ │ └── ops │ │ │ ├── __init__.py │ │ │ └── pubsub_dataset_ops.py │ ├── BUILD │ ├── __init__.py │ └── ops │ │ └── dataset_ops.cc ├── core │ └── BUILD ├── bigtable │ ├── python │ │ └── ops │ │ │ └── __init__.py │ ├── ops │ │ └── bigtable_test_ops.cc │ ├── __init__.py │ └── kernels │ │ └── bigtable_range_helpers.cc └── __init__.py ├── R-package ├── .gitignore ├── .Rbuildignore ├── tests │ ├── testthat.R │ └── testthat │ │ ├── testdata │ │ ├── data.mdb │ │ ├── sample.webp │ │ ├── small.mp4 │ │ ├── small.tiff │ │ ├── string.seq │ │ └── parquet_cpp_example.parquet │ │ ├── utils.R │ │ └── test-datasets-ops.R ├── README.md ├── man │ ├── pipe.Rd │ ├── tfio.Rd │ ├── from_schema.Rd │ ├── mnist_image_dataset.Rd │ ├── mnist_label_dataset.Rd │ ├── pubsub_dataset.Rd │ ├── tiff_dataset.Rd │ ├── webp_dataset.Rd │ ├── lmdb_dataset.Rd │ ├── video_dataset.Rd │ ├── sequence_file_dataset.Rd │ ├── kinesis_dataset.Rd │ ├── from_schema.arrow_stream_dataset.Rd │ ├── from_schema.arrow_feather_dataset.Rd │ ├── kafka_dataset.Rd │ ├── parquet_dataset.Rd │ ├── arrow_stream_dataset.Rd │ ├── arrow_feather_dataset.Rd │ ├── make_libsvm_dataset.Rd │ └── ignite_dataset.Rd ├── tfio.Rproj ├── cran-comments.md ├── scripts │ ├── Dockerfile │ └── gen_wrappers.R ├── R │ ├── dataset_utils.R │ ├── pubsub_dataset.R │ ├── lmdb_dataset.R │ ├── video_dataset.R │ ├── hadoop_dataset.R │ ├── kinesis_dataset.R │ ├── mnist_dataset.R │ ├── kafka_dataset.R │ ├── parquet_dataset.R │ ├── image_dataset.R │ ├── package.R │ └── libsvm_dataset.R ├── DESCRIPTION └── NEWS.md ├── .travis ├── bazel.build.sh ├── lint.bazel.sh ├── lint.python.sh ├── wheel.build.sh ├── build.test.sh ├── bazel.configure.sh ├── wheel.test.sh ├── wheel.configure.sh ├── python.release.sh ├── wheel.r.test.sh └── python3.7+.release.sh ├── .gitignore ├── configure.sh └── dev └── Dockerfile /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /third_party/BUILD: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /third_party/tf/BUILD: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/arrow/python/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/cifar/python/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/image/python/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/kafka/python/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/lmdb/python/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/mnist/python/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/text/python/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/video/python/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/arrow/python/ops/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/cifar/python/ops/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/hadoop/python/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/hadoop/python/ops/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/ignite/python/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/ignite/python/ops/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/image/python/ops/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/kafka/python/ops/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/kinesis/python/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/libsvm/python/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/libsvm/python/ops/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/lmdb/python/ops/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/mnist/python/ops/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/parquet/python/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/pubsub/python/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/pubsub/python/ops/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/text/python/ops/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/video/python/ops/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/kinesis/python/ops/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow_io/parquet/python/ops/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /R-package/.gitignore: -------------------------------------------------------------------------------- 1 | .Rproj.user 2 | .Rhistory 3 | .RData 4 | .Ruserdata 5 | .DS_Store 6 | -------------------------------------------------------------------------------- /tests/test_lmdb/data.mdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/tests/test_lmdb/data.mdb -------------------------------------------------------------------------------- /tests/test_image/cradle.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/tests/test_image/cradle.gif -------------------------------------------------------------------------------- /tests/test_image/sample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/tests/test_image/sample.png -------------------------------------------------------------------------------- /tests/test_image/small.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/tests/test_image/small.tiff -------------------------------------------------------------------------------- /tests/test_libsvm/sample: -------------------------------------------------------------------------------- 1 | 1 1:3.4 2:0.5 4:0.231 2 | 1 2:2.5 3:inf 5:0.503 3 | 2 3:2.5 2:nan 1:0.105 4 | -------------------------------------------------------------------------------- /tests/test_mnist/mnist.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/tests/test_mnist/mnist.npz -------------------------------------------------------------------------------- /tests/test_video/small.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/tests/test_video/small.mp4 -------------------------------------------------------------------------------- /tests/test_hadoop/string.seq: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/tests/test_hadoop/string.seq -------------------------------------------------------------------------------- /tests/test_image/sample.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/tests/test_image/sample.webp -------------------------------------------------------------------------------- /tests/test_image/small-00.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/tests/test_image/small-00.png -------------------------------------------------------------------------------- /tests/test_image/small-01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/tests/test_image/small-01.png -------------------------------------------------------------------------------- /tests/test_image/small-02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/tests/test_image/small-02.png -------------------------------------------------------------------------------- /tests/test_image/small-03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/tests/test_image/small-03.png -------------------------------------------------------------------------------- /tests/test_image/small-04.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/tests/test_image/small-04.png -------------------------------------------------------------------------------- /tests/test_text/lorem.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/tests/test_text/lorem.txt.gz -------------------------------------------------------------------------------- /R-package/.Rbuildignore: -------------------------------------------------------------------------------- 1 | ^.*\.Rproj$ 2 | ^\.Rproj\.user$ 3 | ^man-roxygen/ 4 | scripts 5 | ^cran-comments\.md$ 6 | -------------------------------------------------------------------------------- /R-package/tests/testthat.R: -------------------------------------------------------------------------------- 1 | library(testthat) 2 | library(tensorflow) 3 | library(tfio) 4 | 5 | test_check("tfio") 6 | -------------------------------------------------------------------------------- /tests/test_mnist/t10k-images-idx3-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/tests/test_mnist/t10k-images-idx3-ubyte -------------------------------------------------------------------------------- /R-package/tests/testthat/testdata/data.mdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/R-package/tests/testthat/testdata/data.mdb -------------------------------------------------------------------------------- /tests/test_mnist/t10k-images-idx3-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/tests/test_mnist/t10k-images-idx3-ubyte.gz -------------------------------------------------------------------------------- /tests/test_mnist/t10k-labels-idx1-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/tests/test_mnist/t10k-labels-idx1-ubyte.gz -------------------------------------------------------------------------------- /R-package/tests/testthat/testdata/sample.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/R-package/tests/testthat/testdata/sample.webp -------------------------------------------------------------------------------- /R-package/tests/testthat/testdata/small.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/R-package/tests/testthat/testdata/small.mp4 -------------------------------------------------------------------------------- /R-package/tests/testthat/testdata/small.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/R-package/tests/testthat/testdata/small.tiff -------------------------------------------------------------------------------- /R-package/tests/testthat/testdata/string.seq: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/R-package/tests/testthat/testdata/string.seq -------------------------------------------------------------------------------- /tests/test_parquet/parquet_cpp_example.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/tests/test_parquet/parquet_cpp_example.parquet -------------------------------------------------------------------------------- /R-package/README.md: -------------------------------------------------------------------------------- 1 | ## R interface to TensorFlow IO 2 | 3 | This is the R interface to datasets and filesystem extensions maintained by SIG-IO. 4 | -------------------------------------------------------------------------------- /R-package/tests/testthat/testdata/parquet_cpp_example.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0101011/io/master/R-package/tests/testthat/testdata/parquet_cpp_example.parquet -------------------------------------------------------------------------------- /.travis/bazel.build.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | bazel build \ 4 | --noshow_progress \ 5 | --noshow_loading_progress \ 6 | --verbose_failures \ 7 | --test_output=errors \ 8 | -- //tensorflow_io/... 9 | -------------------------------------------------------------------------------- /.travis/lint.bazel.sh: -------------------------------------------------------------------------------- 1 | set -x -e 2 | 3 | docker run -i -t --rm -v $PWD:/v -w /v --net=host golang:1.12 bash -x -e -c 'go get github.com/bazelbuild/buildtools/buildifier && buildifier --mode=diff $(find . -type f \( -iname WORKSPACE -or -iname BUILD -or -iname *.BUILD \))' 4 | -------------------------------------------------------------------------------- /.travis/lint.python.sh: -------------------------------------------------------------------------------- 1 | curl https://bootstrap.pypa.io/get-pip.py | python 2 | python -m pip install pylint 3 | 4 | curl -o .pylint -sSL https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/tools/ci_build/pylintrc 5 | 6 | find . -name \*.py | xargs pylint --rcfile=.pylint 7 | -------------------------------------------------------------------------------- /R-package/man/pipe.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/reexports.R 3 | \name{\%>\%} 4 | \alias{\%>\%} 5 | \title{Pipe operator} 6 | \usage{ 7 | lhs \%>\% rhs 8 | } 9 | \description{ 10 | See \code{\link[magrittr]{\%>\%}} for more details. 11 | } 12 | \keyword{internal} 13 | -------------------------------------------------------------------------------- /R-package/man/tfio.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/package.R 3 | \docType{package} 4 | \name{tfio} 5 | \alias{tfio} 6 | \alias{tfio-package} 7 | \title{TensorFlow IO API for R} 8 | \description{ 9 | This library provides an R interface to the 10 | \href{https://github.com/tensorflow/io}{TensorFlow IO} API 11 | that provides datasets and filesystem extensions maintained by SIG-IO. 12 | } 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bazelrc 2 | /bazel-* 3 | /artifacts 4 | /get-pip.py 5 | .DS_Store 6 | *.pyc 7 | __pycache__ 8 | *.swp 9 | *.whl 10 | 11 | # IDE specific 12 | .vscode/ 13 | .idea/ 14 | *.pbxproj 15 | *.xcworkspacedata 16 | .ipynb_checkpoints 17 | 18 | # Auto-generated files by `R CMD check` 19 | tfio.Rcheck/ 20 | tfio_*.tar.gz 21 | .Rproj.user 22 | *.Rcheck 23 | .cache 24 | 25 | # Setuptools 26 | *.egg-info 27 | build 28 | dist 29 | wheelhouse 30 | 31 | # Lint 32 | .pylint 33 | -------------------------------------------------------------------------------- /R-package/tfio.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | 3 | RestoreWorkspace: Default 4 | SaveWorkspace: Default 5 | AlwaysSaveHistory: Default 6 | 7 | EnableCodeIndexing: Yes 8 | UseSpacesForTab: Yes 9 | NumSpacesForTab: 2 10 | Encoding: UTF-8 11 | 12 | RnwWeave: Sweave 13 | LaTeX: pdfLaTeX 14 | 15 | AutoAppendNewline: Yes 16 | StripTrailingWhitespace: Yes 17 | 18 | BuildType: Package 19 | PackageUseDevtools: Yes 20 | PackageInstallArgs: --no-multiarch --with-keep.source 21 | PackageRoxygenize: rd,collate,namespace 22 | -------------------------------------------------------------------------------- /third_party/tf/BUILD.tpl: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | cc_library( 4 | name = "tf_header_lib", 5 | hdrs = [":tf_header_include"], 6 | includes = ["include"], 7 | visibility = ["//visibility:public"], 8 | ) 9 | 10 | cc_library( 11 | name = "libtensorflow_framework", 12 | srcs = [":libtensorflow_framework.so"], 13 | #data = ["lib/libtensorflow_framework.so"], 14 | visibility = ["//visibility:public"], 15 | ) 16 | 17 | %{TF_HEADER_GENRULE} 18 | %{TF_SHARED_LIBRARY_GENRULE} -------------------------------------------------------------------------------- /.travis/wheel.build.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | if [[ "$1" == "--"* ]]; then 4 | VERSION_CHOICE=$1 5 | VERSION_NUMBER=$2 6 | shift 7 | shift 8 | fi 9 | 10 | for entry in "$@" ; do 11 | $entry --version 12 | $entry -m pip --version 13 | $entry setup.py --data bazel-bin -q bdist_wheel $VERSION_CHOICE $VERSION_NUMBER 14 | done 15 | ls dist/* 16 | for f in dist/*.whl; do 17 | if [[ $(uname) == "Darwin" ]]; then 18 | delocate-wheel -w wheelhouse $f 19 | else 20 | auditwheel repair $f 21 | fi 22 | done 23 | ls wheelhouse/* 24 | -------------------------------------------------------------------------------- /R-package/man/from_schema.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/arrow_dataset.R 3 | \name{from_schema} 4 | \alias{from_schema} 5 | \title{Create an Arrow Dataset from the given Arrow schema.} 6 | \usage{ 7 | from_schema(object, ...) 8 | } 9 | \arguments{ 10 | \item{object}{An \R object.} 11 | 12 | \item{...}{Optional arguments passed on to implementing methods.} 13 | } 14 | \description{ 15 | Infer output types and shapes from the given Arrow schema and create an Arrow 16 | Dataset. 17 | } 18 | -------------------------------------------------------------------------------- /R-package/cran-comments.md: -------------------------------------------------------------------------------- 1 | ## Test environments 2 | 3 | * local OS X install, R 3.5 4 | * ubuntu 14.04 (on travis-ci), R 3.5 5 | * ubuntu 18.04 (on travis-ci), R 3.5 6 | * win-builder (devel) 7 | 8 | ## R CMD check results 9 | 10 | ``` 11 | 0 errors | 0 warnings | 0 note 12 | ``` 13 | 14 | ## Comments 15 | 16 | The examples are wrapped in `\dontrun{}` block and most of the tests are skipped via `skip_on_cran()` since they can only be run when both Python and TensorFlow are installed but this is currently not viable on CRAN test machines. 17 | -------------------------------------------------------------------------------- /R-package/scripts/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM r-base 2 | COPY . . 3 | 4 | RUN apt-get update && \ 5 | apt-get install -y --no-install-recommends \ 6 | build-essential \ 7 | python-dev \ 8 | python-setuptools \ 9 | python-pip && \ 10 | rm -rf /var/lib/apt/lists/* 11 | 12 | # Dependencies 13 | RUN pip install tensorflow-io 14 | RUN Rscript -e 'install.packages(c("Rcpp", "reticulate", "knitr", "tensorflow", "tfdatasets", "forge", "tidyselect"))' 15 | 16 | # tfio package installation 17 | RUN R CMD build R-package/ 18 | RUN R CMD INSTALL tfio_*.gz 19 | -------------------------------------------------------------------------------- /tensorflow_io/hadoop/BUILD: -------------------------------------------------------------------------------- 1 | licenses(["notice"]) # Apache 2.0 2 | 3 | package(default_visibility = ["//visibility:public"]) 4 | 5 | cc_binary( 6 | name = "python/ops/_hadoop_ops.so", 7 | srcs = [ 8 | "kernels/hadoop_dataset_ops.cc", 9 | "ops/dataset_ops.cc", 10 | ], 11 | copts = [ 12 | "-pthread", 13 | "-std=c++11", 14 | "-DNDEBUG", 15 | ], 16 | linkshared = 1, 17 | deps = [ 18 | "@local_config_tf//:libtensorflow_framework", 19 | "@local_config_tf//:tf_header_lib", 20 | ], 21 | ) 22 | -------------------------------------------------------------------------------- /tensorflow_io/libsvm/BUILD: -------------------------------------------------------------------------------- 1 | licenses(["notice"]) # Apache 2.0 2 | 3 | package(default_visibility = ["//visibility:public"]) 4 | 5 | cc_binary( 6 | name = "python/ops/_libsvm_ops.so", 7 | srcs = [ 8 | "kernels/decode_libsvm_op.cc", 9 | "ops/libsvm_ops.cc", 10 | ], 11 | copts = [ 12 | "-pthread", 13 | "-std=c++11", 14 | "-DNDEBUG", 15 | ], 16 | linkshared = 1, 17 | deps = [ 18 | "@local_config_tf//:libtensorflow_framework", 19 | "@local_config_tf//:tf_header_lib", 20 | ], 21 | ) 22 | -------------------------------------------------------------------------------- /tensorflow_io/lmdb/BUILD: -------------------------------------------------------------------------------- 1 | licenses(["notice"]) # Apache 2.0 2 | 3 | package(default_visibility = ["//visibility:public"]) 4 | 5 | cc_binary( 6 | name = "python/ops/_lmdb_ops.so", 7 | srcs = [ 8 | "kernels/lmdb_dataset_ops.cc", 9 | "ops/dataset_ops.cc", 10 | ], 11 | copts = [ 12 | "-pthread", 13 | "-std=c++11", 14 | "-DNDEBUG", 15 | ], 16 | linkshared = 1, 17 | deps = [ 18 | "@lmdb", 19 | "@local_config_tf//:libtensorflow_framework", 20 | "@local_config_tf//:tf_header_lib", 21 | ], 22 | ) 23 | -------------------------------------------------------------------------------- /tensorflow_io/parquet/BUILD: -------------------------------------------------------------------------------- 1 | licenses(["notice"]) # Apache 2.0 2 | 3 | package(default_visibility = ["//visibility:public"]) 4 | 5 | cc_binary( 6 | name = "python/ops/_parquet_ops.so", 7 | srcs = [ 8 | "kernels/parquet_dataset_ops.cc", 9 | "ops/dataset_ops.cc", 10 | ], 11 | copts = [ 12 | "-pthread", 13 | "-std=c++11", 14 | "-DNDEBUG", 15 | ], 16 | linkshared = 1, 17 | deps = [ 18 | "@arrow", 19 | "@local_config_tf//:libtensorflow_framework", 20 | "@local_config_tf//:tf_header_lib", 21 | ], 22 | ) 23 | -------------------------------------------------------------------------------- /R-package/R/dataset_utils.R: -------------------------------------------------------------------------------- 1 | as_tf_dataset <- function (dataset, tags = NULL) { 2 | if (!is_dataset(dataset)) 3 | stop("Provided dataset is not a TensorFlow Dataset") 4 | if (!inherits(dataset, "tf_dataset")) 5 | class(dataset) <- c("tf_dataset", class(dataset), tags) 6 | dataset 7 | } 8 | 9 | is_dataset <- function (x) { 10 | inherits(x, "tensorflow.python.data.ops.dataset_ops.Dataset") || 11 | inherits(x, "tensorflow.python.data.ops.dataset_ops.DatasetV2") || 12 | is_tfio_dataset(x) 13 | } 14 | 15 | is_tfio_dataset <- function(x) { 16 | grepl("tensorflow_io", class(x)) 17 | } 18 | -------------------------------------------------------------------------------- /tensorflow_io/kinesis/BUILD: -------------------------------------------------------------------------------- 1 | licenses(["notice"]) # Apache 2.0 2 | 3 | package(default_visibility = ["//visibility:public"]) 4 | 5 | cc_binary( 6 | name = "python/ops/_kinesis_ops.so", 7 | srcs = [ 8 | "kernels/kinesis_dataset_ops.cc", 9 | "ops/dataset_ops.cc", 10 | ], 11 | copts = [ 12 | "-pthread", 13 | "-std=c++11", 14 | "-DNDEBUG", 15 | ], 16 | linkshared = 1, 17 | deps = [ 18 | "@aws", 19 | "@curl", 20 | "@local_config_tf//:libtensorflow_framework", 21 | "@local_config_tf//:tf_header_lib", 22 | ], 23 | ) 24 | -------------------------------------------------------------------------------- /tensorflow_io/cifar/BUILD: -------------------------------------------------------------------------------- 1 | licenses(["notice"]) # Apache 2.0 2 | 3 | package(default_visibility = ["//visibility:public"]) 4 | 5 | cc_binary( 6 | name = "python/ops/_cifar_ops.so", 7 | srcs = [ 8 | "kernels/cifar_dataset_ops.cc", 9 | "ops/dataset_ops.cc", 10 | ], 11 | copts = [ 12 | "-pthread", 13 | "-std=c++11", 14 | "-DNDEBUG", 15 | ], 16 | linkshared = 1, 17 | deps = [ 18 | "//tensorflow_io/core:dataset_ops", 19 | "@libarchive", 20 | "@local_config_tf//:libtensorflow_framework", 21 | "@local_config_tf//:tf_header_lib", 22 | ], 23 | ) 24 | -------------------------------------------------------------------------------- /tensorflow_io/mnist/BUILD: -------------------------------------------------------------------------------- 1 | licenses(["notice"]) # Apache 2.0 2 | 3 | package(default_visibility = ["//visibility:public"]) 4 | 5 | cc_binary( 6 | name = "python/ops/_mnist_ops.so", 7 | srcs = [ 8 | "kernels/mnist_dataset_ops.cc", 9 | "ops/dataset_ops.cc", 10 | ], 11 | copts = [ 12 | "-pthread", 13 | "-std=c++11", 14 | "-DNDEBUG", 15 | ], 16 | linkshared = 1, 17 | deps = [ 18 | "//tensorflow_io/core:dataset_ops", 19 | "@libarchive", 20 | "@local_config_tf//:libtensorflow_framework", 21 | "@local_config_tf//:tf_header_lib", 22 | ], 23 | ) 24 | -------------------------------------------------------------------------------- /R-package/man/mnist_image_dataset.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/mnist_dataset.R 3 | \name{mnist_image_dataset} 4 | \alias{mnist_image_dataset} 5 | \title{Creates a \code{MNISTImageDataset}.} 6 | \usage{ 7 | mnist_image_dataset(filenames, compression_type = NULL) 8 | } 9 | \arguments{ 10 | \item{filenames}{A \code{tf.string} tensor containing one or more filenames.} 11 | 12 | \item{compression_type}{A \code{tf.string} scalar evaluating to one 13 | of \code{""} (no compression), \code{"ZLIB"}, or \code{"GZIP"}.} 14 | } 15 | \description{ 16 | This creates a dataset for MNIST images. 17 | } 18 | -------------------------------------------------------------------------------- /R-package/man/mnist_label_dataset.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/mnist_dataset.R 3 | \name{mnist_label_dataset} 4 | \alias{mnist_label_dataset} 5 | \title{Creates a \code{MNISTLabelDataset}.} 6 | \usage{ 7 | mnist_label_dataset(filenames, compression_type = NULL) 8 | } 9 | \arguments{ 10 | \item{filenames}{A \code{tf.string} tensor containing one or more filenames.} 11 | 12 | \item{compression_type}{A \code{tf.string} scalar evaluating to one 13 | of \code{""} (no compression), \code{"ZLIB"}, or \code{"GZIP"}.} 14 | } 15 | \description{ 16 | This creates a dataset for MNIST labels. 17 | } 18 | -------------------------------------------------------------------------------- /.travis/build.test.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | export TENSORFLOW_INSTALL="${1}" 4 | 5 | # Path to shared libraries for running pytest 6 | export TFIO_DATAPATH="bazel-bin" 7 | 8 | if [[ $(uname) == "Linux" ]]; then 9 | apt-get -y -qq update 10 | apt-get -y -qq install python python3 ffmpeg 11 | curl -sSOL https://bootstrap.pypa.io/get-pip.py 12 | for entry in python3 python ; do 13 | $entry get-pip.py -q 14 | $entry -m pip --version 15 | $entry -m pip install -q "${TENSORFLOW_INSTALL}" 16 | $entry -m pip install -q pytest boto3 google-cloud-pubsub==0.39.1 pyarrow==0.11.1 pandas==0.19.2 17 | $entry -m pytest tests 18 | done 19 | fi 20 | -------------------------------------------------------------------------------- /R-package/tests/testthat/utils.R: -------------------------------------------------------------------------------- 1 | library(tensorflow) 2 | 3 | skip_if_no_tensorflow_io <- function() { 4 | if (!reticulate::py_module_available("tensorflow_io")) 5 | skip("tensorflow_io Python module is not available for testing") 6 | } 7 | 8 | test_succeeds <- function(desc, expr) { 9 | test_that(desc, { 10 | skip_if_no_tensorflow_io() 11 | expect_error(force(expr), NA) 12 | }) 13 | } 14 | 15 | iterate_all_batches <- function(dataset) { 16 | sess <- tf$Session() 17 | iterator <- make_iterator_one_shot(dataset) 18 | next_batch <- iterator_get_next(iterator) 19 | 20 | until_out_of_range({ 21 | sess$run(next_batch) 22 | }) 23 | } 24 | -------------------------------------------------------------------------------- /tensorflow_io/arrow/BUILD: -------------------------------------------------------------------------------- 1 | licenses(["notice"]) # Apache 2.0 2 | 3 | package(default_visibility = ["//visibility:public"]) 4 | 5 | cc_binary( 6 | name = "python/ops/_arrow_ops.so", 7 | srcs = [ 8 | "kernels/arrow_dataset_ops.cc", 9 | "kernels/arrow_stream_client.h", 10 | "kernels/arrow_stream_client_unix.cc", 11 | "ops/dataset_ops.cc", 12 | ], 13 | copts = [ 14 | "-pthread", 15 | "-std=c++11", 16 | "-DNDEBUG", 17 | ], 18 | linkshared = 1, 19 | deps = [ 20 | "@arrow", 21 | "@local_config_tf//:libtensorflow_framework", 22 | "@local_config_tf//:tf_header_lib", 23 | ], 24 | ) 25 | -------------------------------------------------------------------------------- /third_party/giflib.BUILD: -------------------------------------------------------------------------------- 1 | # Description: 2 | # A library for decoding and encoding GIF images 3 | 4 | licenses(["notice"]) # MIT 5 | 6 | exports_files(["COPYING"]) 7 | 8 | cc_library( 9 | name = "giflib", 10 | srcs = [ 11 | "lib/dgif_lib.c", 12 | "lib/egif_lib.c", 13 | "lib/gif_err.c", 14 | "lib/gif_font.c", 15 | "lib/gif_hash.c", 16 | "lib/gif_hash.h", 17 | "lib/gif_lib_private.h", 18 | "lib/gifalloc.c", 19 | "lib/openbsd-reallocarray.c", 20 | "lib/quantize.c", 21 | ], 22 | hdrs = [ 23 | "lib/gif_lib.h", 24 | ], 25 | includes = ["lib"], 26 | visibility = ["//visibility:public"], 27 | ) 28 | -------------------------------------------------------------------------------- /tensorflow_io/kafka/BUILD: -------------------------------------------------------------------------------- 1 | licenses(["notice"]) # Apache 2.0 2 | 3 | package(default_visibility = ["//visibility:public"]) 4 | 5 | cc_binary( 6 | name = "python/ops/_kafka_ops.so", 7 | srcs = [ 8 | "kernels/kafka_dataset_ops.cc", 9 | "kernels/kafka_sequence.cc", 10 | "ops/dataset_ops.cc", 11 | "ops/kafka_ops.cc", 12 | ], 13 | copts = [ 14 | "-pthread", 15 | "-std=c++11", 16 | "-DNDEBUG", 17 | ], 18 | linkshared = 1, 19 | deps = [ 20 | "//tensorflow_io/core:sequence_ops", 21 | "@kafka", 22 | "@local_config_tf//:libtensorflow_framework", 23 | "@local_config_tf//:tf_header_lib", 24 | ], 25 | ) 26 | -------------------------------------------------------------------------------- /R-package/man/pubsub_dataset.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/pubsub_dataset.R 3 | \name{pubsub_dataset} 4 | \alias{pubsub_dataset} 5 | \title{Creates a \code{PubSubDataset}.} 6 | \usage{ 7 | pubsub_dataset(subscriptions, server = NULL, eof = FALSE, 8 | timeout = 1000) 9 | } 10 | \arguments{ 11 | \item{subscriptions}{A \code{tf.string} tensor containing one or more 12 | subscriptions.} 13 | 14 | \item{server}{The pubsub server.} 15 | 16 | \item{eof}{If True, the pubsub reader will stop on EOF.} 17 | 18 | \item{timeout}{The timeout value for the PubSub to wait (in millisecond).} 19 | } 20 | \description{ 21 | This creates a dataset for consuming PubSub messages. 22 | } 23 | -------------------------------------------------------------------------------- /tensorflow_io/image/BUILD: -------------------------------------------------------------------------------- 1 | licenses(["notice"]) # Apache 2.0 2 | 3 | package(default_visibility = ["//visibility:public"]) 4 | 5 | cc_binary( 6 | name = "python/ops/_image_ops.so", 7 | srcs = [ 8 | "kernels/gif_dataset_ops.cc", 9 | "kernels/tiff_dataset_ops.cc", 10 | "kernels/webp_dataset_ops.cc", 11 | "ops/dataset_ops.cc", 12 | ], 13 | copts = [ 14 | "-pthread", 15 | "-std=c++11", 16 | "-DNDEBUG", 17 | ], 18 | linkshared = 1, 19 | deps = [ 20 | "@giflib", 21 | "@libtiff", 22 | "@libwebp", 23 | "@local_config_tf//:libtensorflow_framework", 24 | "@local_config_tf//:tf_header_lib", 25 | ], 26 | ) 27 | -------------------------------------------------------------------------------- /tensorflow_io/pubsub/BUILD: -------------------------------------------------------------------------------- 1 | licenses(["notice"]) # Apache 2.0 2 | 3 | package(default_visibility = ["//visibility:public"]) 4 | 5 | load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library") 6 | 7 | cc_binary( 8 | name = "python/ops/_pubsub_ops.so", 9 | srcs = [ 10 | "kernels/pubsub_dataset_ops.cc", 11 | "ops/dataset_ops.cc", 12 | ], 13 | copts = [ 14 | "-pthread", 15 | "-std=c++11", 16 | "-DNDEBUG", 17 | ], 18 | linkshared = 1, 19 | deps = [ 20 | "@com_github_grpc_grpc//:grpc++", 21 | "@com_google_googleapis//google/pubsub/v1:pubsub_proto", 22 | "@local_config_tf//:libtensorflow_framework", 23 | "@local_config_tf//:tf_header_lib", 24 | ], 25 | ) 26 | -------------------------------------------------------------------------------- /tensorflow_io/text/BUILD: -------------------------------------------------------------------------------- 1 | licenses(["notice"]) # Apache 2.0 2 | 3 | package(default_visibility = ["//visibility:public"]) 4 | 5 | cc_binary( 6 | name = "python/ops/_text_ops.so", 7 | srcs = [ 8 | "kernels/text_input.cc", 9 | "kernels/text_sequence.cc", 10 | "ops/text_ops.cc", 11 | ], 12 | copts = [ 13 | "-pthread", 14 | "-std=c++11", 15 | "-DNDEBUG", 16 | ], 17 | includes = [ 18 | ".", 19 | ], 20 | linkshared = 1, 21 | deps = [ 22 | "//tensorflow_io/core:dataset_ops", 23 | "//tensorflow_io/core:sequence_ops", 24 | "@libarchive", 25 | "@local_config_tf//:libtensorflow_framework", 26 | "@local_config_tf//:tf_header_lib", 27 | ], 28 | ) 29 | -------------------------------------------------------------------------------- /third_party/lmdb.BUILD: -------------------------------------------------------------------------------- 1 | # Description: 2 | # LMDB is the Lightning Memory-mapped Database. 3 | 4 | licenses(["notice"]) # OpenLDAP Public License 5 | 6 | exports_files(["LICENSE"]) 7 | 8 | cc_library( 9 | name = "lmdb", 10 | srcs = [ 11 | "mdb.c", 12 | "midl.c", 13 | ], 14 | hdrs = [ 15 | "lmdb.h", 16 | "midl.h", 17 | ], 18 | copts = [ 19 | "-w", 20 | ], 21 | linkopts = select({ 22 | ":windows": ["-DEFAULTLIB:advapi32.lib"], # InitializeSecurityDescriptor, SetSecurityDescriptorDacl 23 | "//conditions:default": ["-lpthread"], 24 | }), 25 | visibility = ["//visibility:public"], 26 | ) 27 | 28 | config_setting( 29 | name = "windows", 30 | values = {"cpu": "x64_windows"}, 31 | ) 32 | -------------------------------------------------------------------------------- /R-package/R/pubsub_dataset.R: -------------------------------------------------------------------------------- 1 | #' Creates a `PubSubDataset`. 2 | #' 3 | #' This creates a dataset for consuming PubSub messages. 4 | #' 5 | #' @param subscriptions A `tf.string` tensor containing one or more 6 | #' subscriptions. 7 | #' @param server The pubsub server. 8 | #' @param eof If True, the pubsub reader will stop on EOF. 9 | #' @param timeout The timeout value for the PubSub to wait (in millisecond). 10 | #' 11 | #' @export 12 | pubsub_dataset <- function( 13 | subscriptions, 14 | server = NULL, 15 | eof = FALSE, 16 | timeout = 1000) { 17 | dataset <- tfio_lib$pubsub$PubSubDataset( 18 | subscriptions = subscriptions, 19 | server = server, 20 | eof = cast_logical(eof), 21 | timeout = cast_scalar_integer(timeout) 22 | ) 23 | as_tf_dataset(dataset) 24 | } 25 | -------------------------------------------------------------------------------- /R-package/R/lmdb_dataset.R: -------------------------------------------------------------------------------- 1 | #' Create a `LMDBDataset`. 2 | #' 3 | #' This function allows a user to read data from a LMDB 4 | #' file. A lmdb file consists of (key value) pairs sequentially. 5 | #' 6 | #' @param filenames A `tf.string` tensor containing one or more filenames. 7 | #' 8 | #' @examples \dontrun{ 9 | #' dataset <- sequence_file_dataset("testdata/data.mdb") %>% 10 | #' dataset_repeat(1) 11 | #' 12 | #' sess <- tf$Session() 13 | #' iterator <- make_iterator_one_shot(dataset) 14 | #' next_batch <- iterator_get_next(iterator) 15 | #' 16 | #' until_out_of_range({ 17 | #' batch <- sess$run(next_batch) 18 | #' print(batch) 19 | #' }) 20 | #' } 21 | #' 22 | #' @export 23 | lmdb_dataset <- function(filenames) { 24 | dataset <- tfio_lib$lmdb$LMDBDataset(filenames = filenames) 25 | as_tf_dataset(dataset) 26 | } 27 | -------------------------------------------------------------------------------- /R-package/man/tiff_dataset.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/image_dataset.R 3 | \name{tiff_dataset} 4 | \alias{tiff_dataset} 5 | \title{Create a \code{TIFFDataset}.} 6 | \usage{ 7 | tiff_dataset(filenames) 8 | } 9 | \arguments{ 10 | \item{filenames}{A \code{tf.string} tensor containing one or more filenames.} 11 | } 12 | \description{ 13 | A TIFF Image File Dataset that reads the TIFF file. 14 | } 15 | \examples{ 16 | \dontrun{ 17 | dataset <- tiff_dataset( 18 | filenames = list("testdata/small.tiff")) \%>\% 19 | dataset_repeat(1) 20 | 21 | sess <- tf$Session() 22 | iterator <- make_iterator_one_shot(dataset) 23 | next_batch <- iterator_get_next(iterator) 24 | 25 | until_out_of_range({ 26 | batch <- sess$run(next_batch) 27 | print(batch) 28 | }) 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /R-package/man/webp_dataset.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/image_dataset.R 3 | \name{webp_dataset} 4 | \alias{webp_dataset} 5 | \title{Create a \code{WebPDataset}.} 6 | \usage{ 7 | webp_dataset(filenames) 8 | } 9 | \arguments{ 10 | \item{filenames}{A \code{tf.string} tensor containing one or more filenames.} 11 | } 12 | \description{ 13 | A WebP Image File Dataset that reads the WebP file. 14 | } 15 | \examples{ 16 | \dontrun{ 17 | dataset <- webp_dataset( 18 | filenames = list("testdata/sample.webp")) \%>\% 19 | dataset_repeat(1) 20 | 21 | sess <- tf$Session() 22 | iterator <- make_iterator_one_shot(dataset) 23 | next_batch <- iterator_get_next(iterator) 24 | 25 | until_out_of_range({ 26 | batch <- sess$run(next_batch) 27 | print(batch) 28 | }) 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /tests/test_ignite/stop_ignite.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | docker rm -f ignite-plain 18 | docker rm -f ignite-igfs -------------------------------------------------------------------------------- /.travis/bazel.configure.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | export TENSORFLOW_INSTALL="${1}" 4 | 5 | gcc -v 6 | python --version 7 | export BAZEL_VERSION=0.20.0 BAZEL_OS=$(uname | tr '[:upper:]' '[:lower:]') 8 | curl -sSOL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh 9 | bash -e bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh 2>&1 > bazel-install.log || (cat bazel-install.log && false) 10 | bazel version 11 | curl -sSOL https://bootstrap.pypa.io/get-pip.py 12 | python get-pip.py -q 13 | python -m pip --version 14 | if [[ $(uname) == "Darwin" ]]; then 15 | python -m pip install -q -U matplotlib numpy --ignore-installed six 16 | fi 17 | python -m pip install -q --ignore-installed six "${TENSORFLOW_INSTALL}" 18 | python -c 'import tensorflow as tf; print(tf.version.VERSION)' 19 | bash -e -x configure.sh 20 | -------------------------------------------------------------------------------- /R-package/man/lmdb_dataset.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/lmdb_dataset.R 3 | \name{lmdb_dataset} 4 | \alias{lmdb_dataset} 5 | \title{Create a \code{LMDBDataset}.} 6 | \usage{ 7 | lmdb_dataset(filenames) 8 | } 9 | \arguments{ 10 | \item{filenames}{A \code{tf.string} tensor containing one or more filenames.} 11 | } 12 | \description{ 13 | This function allows a user to read data from a LMDB 14 | file. A lmdb file consists of (key value) pairs sequentially. 15 | } 16 | \examples{ 17 | \dontrun{ 18 | dataset <- sequence_file_dataset("testdata/data.mdb") \%>\% 19 | dataset_repeat(1) 20 | 21 | sess <- tf$Session() 22 | iterator <- make_iterator_one_shot(dataset) 23 | next_batch <- iterator_get_next(iterator) 24 | 25 | until_out_of_range({ 26 | batch <- sess$run(next_batch) 27 | print(batch) 28 | }) 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /R-package/R/video_dataset.R: -------------------------------------------------------------------------------- 1 | #' Create a `VideoDataset` that reads the video file. 2 | #' 3 | #' This allows a user to read data from a video file with ffmpeg. The output of 4 | #' VideoDataset is a sequence of (height, weight, 3) tensor in rgb24 format. 5 | #' 6 | #' @param filenames A `tf.string` tensor containing one or more filenames. 7 | #' 8 | #' @examples \dontrun{ 9 | #' dataset <- video_dataset( 10 | #' filenames = list("testdata/small.mp4")) %>% 11 | #' dataset_repeat(2) 12 | #' 13 | #' sess <- tf$Session() 14 | #' iterator <- make_iterator_one_shot(dataset) 15 | #' next_batch <- iterator_get_next(iterator) 16 | #' 17 | #' until_out_of_range({ 18 | #' batch <- sess$run(next_batch) 19 | #' print(batch) 20 | #' }) 21 | #' } 22 | #' 23 | #' @export 24 | video_dataset <- function(filenames) { 25 | dataset <- tfio_lib$video$VideoDataset(filenames = filenames) 26 | as_tf_dataset(dataset) 27 | } 28 | -------------------------------------------------------------------------------- /.travis/wheel.test.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | run_test() { 4 | entry=$1 5 | CPYTHON_VERSION=$($entry -c 'import sys; print(str(sys.version_info[0])+str(sys.version_info[1]))') 6 | (cd wheelhouse && $entry -m pip install *-cp${CPYTHON_VERSION}-*.whl) 7 | $entry -m pip install -q pytest boto3 google-cloud-pubsub==0.39.1 pyarrow==0.11.1 pandas==0.19.2 8 | (cd tests && $entry -m pytest -v --import-mode=append $(find . -type f \( -iname "test_*.py" ! -iname "test_*_eager.py" \))) 9 | (cd tests && $entry -m pytest -v --import-mode=append $(find . -type f \( -iname "test_*_eager.py" \))) 10 | } 11 | 12 | PYTHON_VERSION=python 13 | if [[ "$#" -gt 0 ]]; then 14 | PYTHON_VERSION="${1}" 15 | shift 16 | fi 17 | if [[ $(uname) == "Linux" ]]; then 18 | apt-get -y -qq update 19 | apt-get -y -qq install $PYTHON_VERSION ffmpeg 20 | curl -sSOL https://bootstrap.pypa.io/get-pip.py 21 | $PYTHON_VERSION get-pip.py -q 22 | fi 23 | run_test $PYTHON_VERSION 24 | -------------------------------------------------------------------------------- /R-package/man/video_dataset.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/video_dataset.R 3 | \name{video_dataset} 4 | \alias{video_dataset} 5 | \title{Create a \code{VideoDataset} that reads the video file.} 6 | \usage{ 7 | video_dataset(filenames) 8 | } 9 | \arguments{ 10 | \item{filenames}{A \code{tf.string} tensor containing one or more filenames.} 11 | } 12 | \description{ 13 | This allows a user to read data from a video file with ffmpeg. The output of 14 | VideoDataset is a sequence of (height, weight, 3) tensor in rgb24 format. 15 | } 16 | \examples{ 17 | \dontrun{ 18 | dataset <- video_dataset( 19 | filenames = list("testdata/small.mp4")) \%>\% 20 | dataset_repeat(2) 21 | 22 | sess <- tf$Session() 23 | iterator <- make_iterator_one_shot(dataset) 24 | next_batch <- iterator_get_next(iterator) 25 | 26 | until_out_of_range({ 27 | batch <- sess$run(next_batch) 28 | print(batch) 29 | }) 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /tensorflow_io/core/BUILD: -------------------------------------------------------------------------------- 1 | licenses(["notice"]) # Apache 2.0 2 | 3 | package(default_visibility = ["//visibility:public"]) 4 | 5 | cc_library( 6 | name = "sequence_ops", 7 | srcs = [ 8 | "kernels/sequence_ops.h", 9 | ], 10 | copts = [ 11 | "-pthread", 12 | "-std=c++11", 13 | "-DNDEBUG", 14 | ], 15 | includes = [ 16 | ".", 17 | ], 18 | deps = [ 19 | "@local_config_tf//:libtensorflow_framework", 20 | "@local_config_tf//:tf_header_lib", 21 | ], 22 | ) 23 | 24 | cc_library( 25 | name = "dataset_ops", 26 | srcs = [ 27 | "kernels/dataset_ops.h", 28 | ], 29 | copts = [ 30 | "-pthread", 31 | "-std=c++11", 32 | "-DNDEBUG", 33 | ], 34 | includes = [ 35 | ".", 36 | ], 37 | deps = [ 38 | "@local_config_tf//:libtensorflow_framework", 39 | "@local_config_tf//:tf_header_lib", 40 | ], 41 | ) 42 | -------------------------------------------------------------------------------- /tests/test_ignite/bin/start-igfs.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | nohup apache-ignite-fabric/bin/ignite.sh /data/config/ignite-config-igfs.xml & 18 | sleep 10 # Wait Apache Ignite to be started 19 | 20 | tail -f nohup.out 21 | -------------------------------------------------------------------------------- /configure.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | rm -f .bazelrc 18 | if python -c "import tensorflow" &> /dev/null; then 19 | echo 'using installed tensorflow' 20 | else 21 | pip install tensorflow 22 | fi 23 | 24 | python config_helper.py 25 | -------------------------------------------------------------------------------- /tensorflow_io/ignite/ops/igfs_ops.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/platform/env.h" 17 | 18 | #include "tensorflow_io/ignite/kernels/igfs/igfs.h" 19 | 20 | namespace tensorflow { 21 | 22 | REGISTER_FILE_SYSTEM("igfs", IGFS); 23 | 24 | } // namespace tensorflow 25 | -------------------------------------------------------------------------------- /tensorflow_io/bigtable/python/ops/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """This module contains the Python API for the Cloud Bigtable integration.""" 17 | 18 | from __future__ import absolute_import 19 | from __future__ import division 20 | from __future__ import print_function 21 | -------------------------------------------------------------------------------- /third_party/zlib.BUILD: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | licenses(["notice"]) # BSD/MIT-like license (for zlib) 4 | 5 | cc_library( 6 | name = "zlib", 7 | srcs = [ 8 | "adler32.c", 9 | "compress.c", 10 | "crc32.c", 11 | "crc32.h", 12 | "deflate.c", 13 | "deflate.h", 14 | "gzclose.c", 15 | "gzguts.h", 16 | "gzlib.c", 17 | "gzread.c", 18 | "gzwrite.c", 19 | "infback.c", 20 | "inffast.c", 21 | "inffast.h", 22 | "inffixed.h", 23 | "inflate.c", 24 | "inflate.h", 25 | "inftrees.c", 26 | "inftrees.h", 27 | "trees.c", 28 | "trees.h", 29 | "uncompr.c", 30 | "zconf.h", 31 | "zutil.c", 32 | "zutil.h", 33 | ], 34 | hdrs = ["zlib.h"], 35 | copts = [ 36 | "-Wno-shift-negative-value", 37 | "-DZ_HAVE_UNISTD_H", 38 | ], 39 | includes = ["."], 40 | ) 41 | -------------------------------------------------------------------------------- /R-package/R/hadoop_dataset.R: -------------------------------------------------------------------------------- 1 | #' Create a `SequenceFileDataset`. 2 | #' 3 | #' This function allows a user to read data from a hadoop sequence 4 | #' file. A sequence file consists of (key value) pairs sequentially. At 5 | #' the moment, `org.apache.hadoop.io.Text` is the only serialization type 6 | #' being supported, and there is no compression support. 7 | #' 8 | #' @param filenames A `tf.string` tensor containing one or more filenames. 9 | #' 10 | #' @examples \dontrun{ 11 | #' dataset <- sequence_file_dataset("testdata/string.seq") %>% 12 | #' dataset_repeat(1) 13 | #' 14 | #' sess <- tf$Session() 15 | #' iterator <- make_iterator_one_shot(dataset) 16 | #' next_batch <- iterator_get_next(iterator) 17 | #' 18 | #' until_out_of_range({ 19 | #' batch <- sess$run(next_batch) 20 | #' print(batch) 21 | #' }) 22 | #' } 23 | #' 24 | #' @export 25 | sequence_file_dataset <- function(filenames) { 26 | dataset <- tfio_lib$hadoop$SequenceFileDataset(filenames = filenames) 27 | as_tf_dataset(dataset) 28 | } 29 | -------------------------------------------------------------------------------- /third_party/libwebp.BUILD: -------------------------------------------------------------------------------- 1 | # Description: 2 | # WebP codec library 3 | 4 | licenses(["notice"]) # WebM license 5 | 6 | exports_files(["COPYING"]) 7 | 8 | cc_library( 9 | name = "libwebp", 10 | srcs = glob([ 11 | "src/dsp/*.c", 12 | "src/dsp/*.h", 13 | "src/utils/*.c", 14 | "src/utils/*.h", 15 | "src/dec/*.c", 16 | "src/dec/*.h", 17 | "src/demux/*.c", 18 | "src/demux/*.h", 19 | "src/enc/*.c", 20 | "src/enc/*.h", 21 | ]) + [ 22 | "imageio/imageio_util.c", 23 | "imageio/webpdec.c", 24 | "imageio/metadata.c", 25 | ], 26 | hdrs = glob([ 27 | "src/webp/*.h", 28 | ]) + [ 29 | "imageio/webpdec.h", 30 | "imageio/metadata.h", 31 | "imageio/imageio_util.h", 32 | ], 33 | copts = [], 34 | defines = [], 35 | includes = [ 36 | "src", 37 | ], 38 | linkopts = [], 39 | visibility = ["//visibility:public"], 40 | deps = [], 41 | ) 42 | -------------------------------------------------------------------------------- /tests/test_ignite/sql/init.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | -- ============================================================================== 15 | 16 | CREATE TABLE TEST_CACHE (ID LONG PRIMARY KEY, NAME VARCHAR, VAL LONG); 17 | 18 | INSERT INTO TEST_CACHE VALUES (1, 'TEST1', 42); 19 | INSERT INTO TEST_CACHE VALUES (2, 'TEST2', 43); 20 | INSERT INTO TEST_CACHE VALUES (3, 'TEST3', 44); 21 | -------------------------------------------------------------------------------- /.travis/wheel.configure.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | python --version 4 | python -m pip --version 5 | if [[ $(uname) == "Darwin" ]]; then 6 | python -m pip install -q delocate 7 | delocate-wheel --version 8 | else 9 | apt-get -y -qq update 10 | apt-get -y -qq install software-properties-common apt-transport-https 11 | add-apt-repository -y ppa:deadsnakes/ppa 12 | apt-get -y -qq update 13 | apt-get -y -qq install python3 python3.5 python3.6 14 | 15 | # Install patchelf 16 | curl -sSOL https://nixos.org/releases/patchelf/patchelf-0.9/patchelf-0.9.tar.bz2 17 | tar xfa patchelf-0.9.tar.bz2 18 | bash -c -e '(cd patchelf-0.9 && ./configure -q --prefix=/usr && make -s && make -s install)' 19 | rm -rf patchelf-0.9* 20 | 21 | # Install the latest version of pip 22 | curl -sSOL https://bootstrap.pypa.io/get-pip.py 23 | python3.6 get-pip.py -q 24 | python3.5 get-pip.py -q 25 | python3 get-pip.py -q 26 | 27 | # Install auditwheel 28 | python3 -m pip install -q auditwheel==1.5.0 29 | python3 -m pip install -q wheel==0.31.1 30 | auditwheel --version 31 | fi 32 | -------------------------------------------------------------------------------- /R-package/man/sequence_file_dataset.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/hadoop_dataset.R 3 | \name{sequence_file_dataset} 4 | \alias{sequence_file_dataset} 5 | \title{Create a \code{SequenceFileDataset}.} 6 | \usage{ 7 | sequence_file_dataset(filenames) 8 | } 9 | \arguments{ 10 | \item{filenames}{A \code{tf.string} tensor containing one or more filenames.} 11 | } 12 | \description{ 13 | This function allows a user to read data from a hadoop sequence 14 | file. A sequence file consists of (key value) pairs sequentially. At 15 | the moment, \code{org.apache.hadoop.io.Text} is the only serialization type 16 | being supported, and there is no compression support. 17 | } 18 | \examples{ 19 | \dontrun{ 20 | dataset <- sequence_file_dataset("testdata/string.seq") \%>\% 21 | dataset_repeat(1) 22 | 23 | sess <- tf$Session() 24 | iterator <- make_iterator_one_shot(dataset) 25 | next_batch <- iterator_get_next(iterator) 26 | 27 | until_out_of_range({ 28 | batch <- sess$run(next_batch) 29 | print(batch) 30 | }) 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /R-package/man/kinesis_dataset.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/kinesis_dataset.R 3 | \name{kinesis_dataset} 4 | \alias{kinesis_dataset} 5 | \title{Creates a \code{KinesisDataset}.} 6 | \usage{ 7 | kinesis_dataset(stream, shard = "", read_indefinitely = TRUE, 8 | interval = 1e+05) 9 | } 10 | \arguments{ 11 | \item{stream}{A \code{tf.string} tensor containing the name of the stream.} 12 | 13 | \item{shard}{A \code{tf.string} tensor containing the id of the shard.} 14 | 15 | \item{read_indefinitely}{If \code{True}, the Kinesis dataset will keep retry again 16 | on \code{EOF} after the \code{interval} period. If \code{False}, then the dataset will 17 | stop on \code{EOF}. The default value is \code{True}.} 18 | 19 | \item{interval}{The interval for the Kinesis Client to wait before it tries 20 | to get records again (in millisecond).} 21 | } 22 | \description{ 23 | Kinesis is a managed service provided by AWS for data streaming. 24 | This dataset reads messages from Kinesis with each message presented 25 | as a \code{tf.string}. 26 | } 27 | -------------------------------------------------------------------------------- /tests/test_ignite/bin/start-plain.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | nohup apache-ignite-fabric/bin/ignite.sh /data/config/ignite-config-plain.xml & 18 | sleep 10 # Wait Apache Ignite to be started 19 | 20 | ./apache-ignite-fabric/bin/sqlline.sh \ 21 | -u "jdbc:ignite:thin://127.0.0.1/" \ 22 | --run=/data/sql/init.sql 23 | 24 | tail -f nohup.out 25 | -------------------------------------------------------------------------------- /R-package/man/from_schema.arrow_stream_dataset.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/arrow_dataset.R 3 | \name{from_schema.arrow_stream_dataset} 4 | \alias{from_schema.arrow_stream_dataset} 5 | \title{Create an Arrow Dataset from an input stream, inferring output types and 6 | shapes from the given Arrow schema.} 7 | \usage{ 8 | \method{from_schema}{arrow_stream_dataset}(object, schema, 9 | columns = NULL, host = NULL, filenames = NULL, ...) 10 | } 11 | \arguments{ 12 | \item{object}{An \R object.} 13 | 14 | \item{schema}{Arrow schema defining the record batch data in the stream.} 15 | 16 | \item{columns}{A list of column indices to be used in the Dataset.} 17 | 18 | \item{host}{A \code{tf.string} tensor or string defining the input stream. 19 | For a socket client, use ":", for stdin use "STDIN".} 20 | 21 | \item{filenames}{Not used.} 22 | 23 | \item{...}{Optional arguments passed on to implementing methods.} 24 | } 25 | \description{ 26 | Create an Arrow Dataset from an input stream, inferring output types and 27 | shapes from the given Arrow schema. 28 | } 29 | -------------------------------------------------------------------------------- /R-package/R/kinesis_dataset.R: -------------------------------------------------------------------------------- 1 | #' Creates a `KinesisDataset`. 2 | #' 3 | #' Kinesis is a managed service provided by AWS for data streaming. 4 | #' This dataset reads messages from Kinesis with each message presented 5 | #' as a `tf.string`. 6 | #' 7 | #' @param stream A `tf.string` tensor containing the name of the stream. 8 | #' @param shard A `tf.string` tensor containing the id of the shard. 9 | #' @param read_indefinitely If `True`, the Kinesis dataset will keep retry again 10 | #' on `EOF` after the `interval` period. If `False`, then the dataset will 11 | #' stop on `EOF`. The default value is `True`. 12 | #' @param interval The interval for the Kinesis Client to wait before it tries 13 | #' to get records again (in millisecond). 14 | #' 15 | #' @export 16 | kinesis_dataset <- function( 17 | stream, 18 | shard = "", 19 | read_indefinitely = TRUE, 20 | interval = 100000) { 21 | dataset <- tfio_lib$kinesis$KinesisDataset( 22 | stream = stream, 23 | shard = shard, 24 | read_indefinitely = cast_logical(read_indefinitely), 25 | interval = cast_scalar_integer(interval) 26 | ) 27 | as_tf_dataset(dataset) 28 | } 29 | -------------------------------------------------------------------------------- /R-package/man/from_schema.arrow_feather_dataset.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/arrow_dataset.R 3 | \name{from_schema.arrow_feather_dataset} 4 | \alias{from_schema.arrow_feather_dataset} 5 | \title{Create an Arrow Dataset for reading record batches from Arrow feather files, 6 | inferring output types and shapes from the given Arrow schema.} 7 | \usage{ 8 | \method{from_schema}{arrow_feather_dataset}(object, schema, 9 | columns = NULL, host = NULL, filenames = NULL, ...) 10 | } 11 | \arguments{ 12 | \item{object}{An \R object.} 13 | 14 | \item{schema}{Arrow schema defining the record batch data in the stream.} 15 | 16 | \item{columns}{A list of column indices to be used in the Dataset.} 17 | 18 | \item{host}{Not used.} 19 | 20 | \item{filenames}{A \code{tf.string} tensor, list or scalar containing files in 21 | Arrow Feather format.} 22 | 23 | \item{...}{Optional arguments passed on to implementing methods.} 24 | } 25 | \description{ 26 | Create an Arrow Dataset for reading record batches from Arrow feather files, 27 | inferring output types and shapes from the given Arrow schema. 28 | } 29 | -------------------------------------------------------------------------------- /tensorflow_io/bigtable/ops/bigtable_test_ops.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/common_shape_fns.h" 17 | #include "tensorflow/core/framework/op.h" 18 | 19 | namespace tensorflow { 20 | 21 | REGISTER_OP("BigtableTestClient") 22 | .Attr("container: string = ''") 23 | .Attr("shared_name: string = ''") 24 | .Output("client: resource") 25 | .SetShapeFn(shape_inference::ScalarShape); 26 | 27 | } // namespace tensorflow 28 | -------------------------------------------------------------------------------- /tensorflow_io/lmdb/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """LMDB Dataset. 16 | 17 | @@LMDBDataset 18 | """ 19 | 20 | from __future__ import absolute_import 21 | from __future__ import division 22 | from __future__ import print_function 23 | 24 | from tensorflow_io.lmdb.python.ops.lmdb_dataset_ops import LMDBDataset 25 | 26 | from tensorflow.python.util.all_util import remove_undocumented 27 | 28 | _allowed_symbols = [ 29 | "LMDBDataset", 30 | ] 31 | 32 | remove_undocumented(__name__) 33 | -------------------------------------------------------------------------------- /R-package/R/mnist_dataset.R: -------------------------------------------------------------------------------- 1 | #' Creates a `MNISTImageDataset`. 2 | #' 3 | #' This creates a dataset for MNIST images. 4 | #' 5 | #' @param filenames A `tf.string` tensor containing one or more filenames. 6 | #' @param compression_type A `tf.string` scalar evaluating to one 7 | #' of `""` (no compression), `"ZLIB"`, or `"GZIP"`. 8 | #' 9 | #' @export 10 | mnist_image_dataset <- function( 11 | filenames, 12 | compression_type = NULL) { 13 | dataset <- tfio_lib$mnist$MNISTImageDataset( 14 | filenames = filenames, 15 | compression_type = compression_type 16 | ) 17 | as_tf_dataset(dataset) 18 | } 19 | 20 | #' Creates a `MNISTLabelDataset`. 21 | #' 22 | #' This creates a dataset for MNIST labels. 23 | #' 24 | #' @param filenames A `tf.string` tensor containing one or more filenames. 25 | #' @param compression_type A `tf.string` scalar evaluating to one 26 | #' of `""` (no compression), `"ZLIB"`, or `"GZIP"`. 27 | #' 28 | #' @export 29 | mnist_label_dataset <- function( 30 | filenames, 31 | compression_type = NULL) { 32 | dataset <- tfio_lib$mnist$MNISTLabelDataset( 33 | filenames = filenames, 34 | compression_type = compression_type 35 | ) 36 | as_tf_dataset(dataset) 37 | } 38 | -------------------------------------------------------------------------------- /tensorflow_io/video/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Video Dataset. 16 | 17 | @@VideoDataset 18 | """ 19 | 20 | from __future__ import absolute_import 21 | from __future__ import division 22 | from __future__ import print_function 23 | 24 | from tensorflow_io.video.python.ops.video_dataset_ops import VideoDataset 25 | 26 | from tensorflow.python.util.all_util import remove_undocumented 27 | 28 | _allowed_symbols = [ 29 | "VideoDataset", 30 | ] 31 | 32 | remove_undocumented(__name__) 33 | -------------------------------------------------------------------------------- /tensorflow_io/pubsub/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """PubSub Dataset. 16 | 17 | @@PubSubDataset 18 | """ 19 | 20 | from __future__ import absolute_import 21 | from __future__ import division 22 | from __future__ import print_function 23 | 24 | from tensorflow_io.pubsub.python.ops.pubsub_dataset_ops import PubSubDataset 25 | 26 | from tensorflow.python.util.all_util import remove_undocumented 27 | 28 | _allowed_symbols = [ 29 | "PubSubDataset", 30 | ] 31 | 32 | remove_undocumented(__name__) 33 | -------------------------------------------------------------------------------- /tensorflow_io/kinesis/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Kinesis Dataset. 16 | 17 | @@KinesisDataset 18 | """ 19 | 20 | from __future__ import absolute_import 21 | from __future__ import division 22 | from __future__ import print_function 23 | 24 | from tensorflow_io.kinesis.python.ops.kinesis_dataset_ops import KinesisDataset 25 | 26 | from tensorflow.python.util.all_util import remove_undocumented 27 | 28 | _allowed_symbols = [ 29 | "KinesisDataset", 30 | ] 31 | 32 | remove_undocumented(__name__) 33 | -------------------------------------------------------------------------------- /tensorflow_io/parquet/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Parquet Dataset. 16 | 17 | @@ParquetDataset 18 | """ 19 | 20 | from __future__ import absolute_import 21 | from __future__ import division 22 | from __future__ import print_function 23 | 24 | from tensorflow_io.parquet.python.ops.parquet_dataset_ops import ParquetDataset 25 | 26 | from tensorflow.python.util.all_util import remove_undocumented 27 | 28 | _allowed_symbols = [ 29 | "ParquetDataset", 30 | ] 31 | 32 | remove_undocumented(__name__) 33 | -------------------------------------------------------------------------------- /tensorflow_io/hadoop/ops/dataset_ops.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/common_shape_fns.h" 17 | #include "tensorflow/core/framework/op.h" 18 | #include "tensorflow/core/framework/shape_inference.h" 19 | 20 | namespace tensorflow { 21 | 22 | REGISTER_OP("SequenceFileDataset") 23 | .Input("filenames: string") 24 | .Output("handle: variant") 25 | .Attr("output_types: list(type) >= 1") 26 | .SetIsStateful() 27 | .SetShapeFn(shape_inference::ScalarShape); 28 | 29 | } // namespace tensorflow 30 | -------------------------------------------------------------------------------- /tensorflow_io/hadoop/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Sequence File Dataset. 16 | 17 | @@SequenceFileDataset 18 | """ 19 | 20 | from __future__ import absolute_import 21 | from __future__ import division 22 | from __future__ import print_function 23 | 24 | from tensorflow_io.hadoop.python.ops.hadoop_dataset_ops import SequenceFileDataset 25 | 26 | from tensorflow.python.util.all_util import remove_undocumented 27 | 28 | _allowed_symbols = [ 29 | "SequenceFileDataset", 30 | ] 31 | 32 | remove_undocumented(__name__) 33 | -------------------------------------------------------------------------------- /R-package/man/kafka_dataset.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/kafka_dataset.R 3 | \name{kafka_dataset} 4 | \alias{kafka_dataset} 5 | \title{Creates a \code{KafkaDataset}.} 6 | \usage{ 7 | kafka_dataset(topics, servers = "localhost", group = "", eof = FALSE, 8 | timeout = 1000) 9 | } 10 | \arguments{ 11 | \item{topics}{A \code{tf.string} tensor containing one or more subscriptions, in 12 | the format of \code{[topic:partition:offset:length]}, by default length is -1 13 | for unlimited.} 14 | 15 | \item{servers}{A list of bootstrap servers.} 16 | 17 | \item{group}{The consumer group id.} 18 | 19 | \item{eof}{If True, the kafka reader will stop on EOF.} 20 | 21 | \item{timeout}{The timeout value for the Kafka Consumer to wait (in 22 | millisecond).} 23 | } 24 | \description{ 25 | Creates a \code{KafkaDataset}. 26 | } 27 | \examples{ 28 | \dontrun{ 29 | dataset <- kafka_dataset( 30 | topics = list("test:0:0:4"), group = "test", eof = TRUE) \%>\% 31 | dataset_repeat(1) 32 | 33 | sess <- tf$Session() 34 | iterator <- make_iterator_one_shot(dataset) 35 | next_batch <- iterator_get_next(iterator) 36 | 37 | until_out_of_range({ 38 | batch <- sess$run(next_batch) 39 | print(batch) 40 | }) 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /tensorflow_io/lmdb/ops/dataset_ops.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/common_shape_fns.h" 17 | #include "tensorflow/core/framework/op.h" 18 | #include "tensorflow/core/framework/shape_inference.h" 19 | 20 | namespace tensorflow { 21 | 22 | REGISTER_OP("LMDBDataset") 23 | .Input("filenames: string") 24 | .Output("handle: variant") 25 | .Attr("output_types: list(type) >= 1") 26 | .Attr("output_shapes: list(shape) >= 1") 27 | .SetIsStateful() 28 | .SetShapeFn(shape_inference::ScalarShape); 29 | 30 | } // namespace tensorflow 31 | -------------------------------------------------------------------------------- /tensorflow_io/pubsub/ops/dataset_ops.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/common_shape_fns.h" 17 | #include "tensorflow/core/framework/op.h" 18 | #include "tensorflow/core/framework/shape_inference.h" 19 | 20 | namespace tensorflow { 21 | 22 | REGISTER_OP("PubSubDataset") 23 | .Input("subscriptions: string") 24 | .Input("server: string") 25 | .Input("eof: bool") 26 | .Input("timeout: int64") 27 | .Output("handle: variant") 28 | .SetIsStateful() 29 | .SetShapeFn(shape_inference::ScalarShape); 30 | 31 | } // namespace tensorflow 32 | -------------------------------------------------------------------------------- /tensorflow_io/video/ops/dataset_ops.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/common_shape_fns.h" 17 | #include "tensorflow/core/framework/op.h" 18 | #include "tensorflow/core/framework/shape_inference.h" 19 | 20 | namespace tensorflow { 21 | 22 | REGISTER_OP("VideoDataset") 23 | .Input("filenames: string") 24 | .Output("handle: variant") 25 | .SetIsStateful() 26 | .SetShapeFn([](shape_inference::InferenceContext* c) { 27 | c->set_output(0, c->MakeShape({c->UnknownDim(), c->UnknownDim(), 3})); 28 | return Status::OK(); 29 | }); 30 | 31 | } // namespace tensorflow 32 | -------------------------------------------------------------------------------- /tensorflow_io/text/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """TextInput/TextOutput 16 | 17 | @@TextOutputSequence 18 | @@TextDataset 19 | """ 20 | 21 | from __future__ import absolute_import 22 | from __future__ import division 23 | from __future__ import print_function 24 | 25 | from tensorflow_io.text.python.ops.text_ops import TextOutputSequence 26 | from tensorflow_io.text.python.ops.text_ops import TextDataset 27 | 28 | from tensorflow.python.util.all_util import remove_undocumented 29 | 30 | _allowed_symbols = [ 31 | "TextOutputSequence", 32 | "TextDataset", 33 | ] 34 | 35 | remove_undocumented(__name__) 36 | -------------------------------------------------------------------------------- /R-package/R/kafka_dataset.R: -------------------------------------------------------------------------------- 1 | #' Creates a `KafkaDataset`. 2 | #' 3 | #' @param topics A `tf.string` tensor containing one or more subscriptions, in 4 | #' the format of `[topic:partition:offset:length]`, by default length is -1 5 | #' for unlimited. 6 | #' @param servers A list of bootstrap servers. 7 | #' @param group The consumer group id. 8 | #' @param eof If True, the kafka reader will stop on EOF. 9 | #' @param timeout The timeout value for the Kafka Consumer to wait (in 10 | #' millisecond). 11 | #' 12 | #' @examples \dontrun{ 13 | #' dataset <- kafka_dataset( 14 | #' topics = list("test:0:0:4"), group = "test", eof = TRUE) %>% 15 | #' dataset_repeat(1) 16 | #' 17 | #' sess <- tf$Session() 18 | #' iterator <- make_iterator_one_shot(dataset) 19 | #' next_batch <- iterator_get_next(iterator) 20 | #' 21 | #' until_out_of_range({ 22 | #' batch <- sess$run(next_batch) 23 | #' print(batch) 24 | #' }) 25 | #' } 26 | #' 27 | #' @export 28 | kafka_dataset <- function( 29 | topics, 30 | servers = "localhost", 31 | group = "", 32 | eof = FALSE, 33 | timeout = 1000) { 34 | dataset <- tfio_lib$kafka$KafkaDataset( 35 | topics = topics, 36 | servers = servers, 37 | group = group, 38 | eof = cast_logical(eof), 39 | timeout = cast_scalar_integer(timeout) 40 | ) 41 | as_tf_dataset(dataset) 42 | } 43 | -------------------------------------------------------------------------------- /tensorflow_io/cifar/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """CIFAR File Dataset. 16 | 17 | @@CIFAR10Dataset 18 | @@CIFAR100Dataset 19 | """ 20 | 21 | from __future__ import absolute_import 22 | from __future__ import division 23 | from __future__ import print_function 24 | 25 | from tensorflow_io.cifar.python.ops.cifar_dataset_ops import CIFAR10Dataset 26 | from tensorflow_io.cifar.python.ops.cifar_dataset_ops import CIFAR100Dataset 27 | 28 | from tensorflow.python.util.all_util import remove_undocumented 29 | 30 | _allowed_symbols = [ 31 | "CIFAR10Dataset", 32 | "CIFAR100Dataset", 33 | ] 34 | 35 | remove_undocumented(__name__) 36 | -------------------------------------------------------------------------------- /R-package/man/parquet_dataset.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/parquet_dataset.R 3 | \name{parquet_dataset} 4 | \alias{parquet_dataset} 5 | \title{Create a \code{ParquetDataset}.} 6 | \usage{ 7 | parquet_dataset(filenames, columns, output_types) 8 | } 9 | \arguments{ 10 | \item{filenames}{A 0-D or 1-D \code{tf.string} tensor containing one or more 11 | filenames.} 12 | 13 | \item{columns}{A 0-D or 1-D \code{tf.int32} tensor containing the columns to 14 | extract.} 15 | 16 | \item{output_types}{A tuple of \code{tf.DType} objects representing the types of 17 | the columns returned.} 18 | } 19 | \description{ 20 | This allows a user to read data from a parquet file. 21 | } 22 | \examples{ 23 | \dontrun{ 24 | dtypes <- tf$python$framework$dtypes 25 | output_types <- reticulate::tuple( 26 | dtypes$bool, dtypes$int32, dtypes$int64, dtypes$float32, dtypes$float64) 27 | dataset <- parquet_dataset( 28 | filenames = list("testdata/parquet_cpp_example.parquet"), 29 | columns = list(0, 1, 2, 4, 5), 30 | output_types = output_types) \%>\% 31 | dataset_repeat(2) 32 | 33 | sess <- tf$Session() 34 | iterator <- make_iterator_one_shot(dataset) 35 | next_batch <- iterator_get_next(iterator) 36 | 37 | until_out_of_range({ 38 | batch <- sess$run(next_batch) 39 | print(batch) 40 | }) 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /tensorflow_io/libsvm/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """LibSVM Dataset. 16 | 17 | @@make_libsvm_dataset 18 | @@decode_libsvm 19 | """ 20 | 21 | from __future__ import absolute_import 22 | from __future__ import division 23 | from __future__ import print_function 24 | 25 | from tensorflow_io.libsvm.python.ops.libsvm_dataset_ops import make_libsvm_dataset 26 | from tensorflow_io.libsvm.python.ops.libsvm_dataset_ops import decode_libsvm 27 | 28 | from tensorflow.python.util.all_util import remove_undocumented 29 | 30 | _allowed_symbols = [ 31 | "make_libsvm_dataset", 32 | "decode_libsvm", 33 | ] 34 | 35 | remove_undocumented(__name__) 36 | -------------------------------------------------------------------------------- /third_party/kafka.patch: -------------------------------------------------------------------------------- 1 | diff -Naur config.h config.h 2 | --- config.h 1970-01-01 00:00:00.000000000 +0000 3 | +++ config.h 2017-10-28 00:57:03.316957390 +0000 4 | @@ -0,0 +1,40 @@ 5 | +#pragma once 6 | +#define WITHOUT_OPTIMIZATION 0 7 | +#define ENABLE_DEVEL 0 8 | +#define ENABLE_REFCNT_DEBUG 0 9 | +#define ENABLE_SHAREDPTR_DEBUG 0 10 | + 11 | +#define HAVE_ATOMICS_32 1 12 | +#define HAVE_ATOMICS_32_SYNC 1 13 | + 14 | +#if (HAVE_ATOMICS_32) 15 | +# if (HAVE_ATOMICS_32_SYNC) 16 | +# define ATOMIC_OP32(OP1,OP2,PTR,VAL) __sync_ ## OP1 ## _and_ ## OP2(PTR, VAL) 17 | +# else 18 | +# define ATOMIC_OP32(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST) 19 | +# endif 20 | +#endif 21 | + 22 | +#define HAVE_ATOMICS_64 1 23 | +#define HAVE_ATOMICS_64_SYNC 1 24 | + 25 | +#if (HAVE_ATOMICS_64) 26 | +# if (HAVE_ATOMICS_64_SYNC) 27 | +# define ATOMIC_OP64(OP1,OP2,PTR,VAL) __sync_ ## OP1 ## _and_ ## OP2(PTR, VAL) 28 | +# else 29 | +# define ATOMIC_OP64(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST) 30 | +# endif 31 | +#endif 32 | + 33 | + 34 | +#define WITH_ZLIB 1 35 | +#define WITH_LIBDL 1 36 | +#define WITH_PLUGINS 0 37 | +#define WITH_SNAPPY 1 38 | +#define WITH_SOCKEM 1 39 | +#define WITH_SSL 1 40 | +#define WITH_SASL 0 41 | +#define WITH_SASL_SCRAM 0 42 | +#define WITH_SASL_CYRUS 0 43 | +#define HAVE_REGEX 1 44 | +#define HAVE_STRNDUP 1 45 | -------------------------------------------------------------------------------- /.travis/python.release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | set -e -x 17 | 18 | # Release: 19 | # docker run -i -t --rm -v $PWD:/v -w /v --net=host buildpack-deps:14.04 /v/.travis/python.release.sh 20 | 21 | export TENSORFLOW_INSTALL="$(python setup.py --package-version)" 22 | export PYTHON_VERSION="python2.7 python3.4 python3.5 python3.6" 23 | if [[ "$#" -gt 0 ]]; then 24 | export TENSORFLOW_INSTALL="${1}" 25 | shift 26 | PYTHON_VERSION="$@" 27 | fi 28 | 29 | bash -x -e .travis/bazel.configure.sh "${TENSORFLOW_INSTALL}" 30 | bash -x -e .travis/bazel.build.sh 31 | bash -x -e .travis/wheel.configure.sh 32 | bash -x -e .travis/wheel.build.sh ${PYTHON_VERSION} 33 | -------------------------------------------------------------------------------- /dev/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM tensorflow/tensorflow:custom-op 2 | 3 | RUN apt-get update && \ 4 | apt-get install -y \ 5 | curl \ 6 | nano \ 7 | unzip \ 8 | ffmpeg 9 | 10 | ARG BAZEL_VERSION=0.24.1 11 | ARG BAZEL_OS=linux 12 | 13 | RUN curl -sL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh -o bazel-install.sh && \ 14 | bash -x bazel-install.sh && \ 15 | rm bazel-install.sh 16 | 17 | ARG CONDA_OS=Linux 18 | 19 | # Miniconda - Python 3.6, 64-bit, x86, latest 20 | RUN curl -sL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o mconda-install.sh && \ 21 | bash -x mconda-install.sh -b -p miniconda && \ 22 | rm mconda-install.sh 23 | 24 | ENV PATH="/miniconda/bin:$PATH" 25 | 26 | ARG CONDA_ADD_PACKAGES="" 27 | 28 | RUN conda create -y -q -n tfio-dev python=3.6 ${CONDA_ADD_PACKAGES} 29 | 30 | ARG ARROW_VERSION=0.11.1 31 | 32 | RUN echo ". /miniconda/etc/profile.d/conda.sh" >> ~/.bashrc && \ 33 | echo "source activate tfio-dev" >> ~/.bashrc 34 | 35 | ARG PIP_ADD_PACKAGES="" 36 | 37 | RUN /bin/bash -c "source activate tfio-dev && python -m pip install \ 38 | pytest \ 39 | pylint \ 40 | boto3 \ 41 | google-cloud-pubsub==0.39.1 \ 42 | pyarrow==${ARROW_VERSION} \ 43 | pandas \ 44 | ${PIP_ADD_PACKAGES} \ 45 | " 46 | 47 | ENV TFIO_DATAPATH=bazel-bin 48 | -------------------------------------------------------------------------------- /R-package/R/parquet_dataset.R: -------------------------------------------------------------------------------- 1 | #' Create a `ParquetDataset`. 2 | #' 3 | #' This allows a user to read data from a parquet file. 4 | #' 5 | #' @param filenames A 0-D or 1-D `tf.string` tensor containing one or more 6 | #' filenames. 7 | #' @param columns A 0-D or 1-D `tf.int32` tensor containing the columns to 8 | #' extract. 9 | #' @param output_types A tuple of `tf.DType` objects representing the types of 10 | #' the columns returned. 11 | #' 12 | #' @examples \dontrun{ 13 | #' dtypes <- tf$python$framework$dtypes 14 | #' output_types <- reticulate::tuple( 15 | #' dtypes$bool, dtypes$int32, dtypes$int64, dtypes$float32, dtypes$float64) 16 | #' dataset <- parquet_dataset( 17 | #' filenames = list("testdata/parquet_cpp_example.parquet"), 18 | #' columns = list(0, 1, 2, 4, 5), 19 | #' output_types = output_types) %>% 20 | #' dataset_repeat(2) 21 | #' 22 | #' sess <- tf$Session() 23 | #' iterator <- make_iterator_one_shot(dataset) 24 | #' next_batch <- iterator_get_next(iterator) 25 | #' 26 | #' until_out_of_range({ 27 | #' batch <- sess$run(next_batch) 28 | #' print(batch) 29 | #' }) 30 | #' } 31 | #' 32 | #' @export 33 | parquet_dataset <- function(filenames, columns, output_types) { 34 | dataset <- tfio_lib$parquet$ParquetDataset( 35 | filenames = filenames, 36 | columns = cast_integer_list(columns), 37 | output_types = output_types 38 | ) 39 | as_tf_dataset(dataset) 40 | } 41 | -------------------------------------------------------------------------------- /R-package/man/arrow_stream_dataset.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/arrow_dataset.R 3 | \name{arrow_stream_dataset} 4 | \alias{arrow_stream_dataset} 5 | \title{Creates a \code{ArrowStreamDataset}.} 6 | \usage{ 7 | arrow_stream_dataset(host, columns, output_types, output_shapes = NULL) 8 | } 9 | \arguments{ 10 | \item{host}{A \code{tf.string} tensor or string defining the input stream. 11 | For a socket client, use ":", for stdin use "STDIN".} 12 | 13 | \item{columns}{A list of column indices to be used in the Dataset.} 14 | 15 | \item{output_types}{Tensor dtypes of the output tensors.} 16 | 17 | \item{output_shapes}{TensorShapes of the output tensors or \code{NULL} to infer 18 | partial.} 19 | } 20 | \description{ 21 | An Arrow Dataset for reading record batches from an input stream. Currently 22 | supported input streams are a socket client or stdin. 23 | } 24 | \examples{ 25 | \dontrun{ 26 | dataset <- arrow_stream_dataset( 27 | host, 28 | columns = reticulate::tuple(0L, 1L), 29 | output_types = reticulate::tuple(tf$int32, tf$float32), 30 | output_shapes = reticulate::tuple(list(), list())) \%>\% 31 | dataset_repeat(1) 32 | 33 | sess <- tf$Session() 34 | iterator <- make_iterator_one_shot(dataset) 35 | next_batch <- iterator_get_next(iterator) 36 | 37 | until_out_of_range({ 38 | batch <- sess$run(next_batch) 39 | print(batch) 40 | }) 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /tensorflow_io/kafka/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Kafka Dataset. 16 | 17 | @@KafkaOutputSequence 18 | @@KafkaDataset 19 | @@write_kafka 20 | """ 21 | 22 | from __future__ import absolute_import 23 | from __future__ import division 24 | from __future__ import print_function 25 | 26 | from tensorflow_io.kafka.python.ops.kafka_ops import KafkaOutputSequence 27 | from tensorflow_io.kafka.python.ops.kafka_dataset_ops import KafkaDataset 28 | from tensorflow_io.kafka.python.ops.kafka_dataset_ops import write_kafka 29 | 30 | from tensorflow.python.util.all_util import remove_undocumented 31 | 32 | _allowed_symbols = [ 33 | "KafkaOutputSequence", 34 | "KafkaDataset", 35 | "write_kafka", 36 | ] 37 | 38 | remove_undocumented(__name__) 39 | -------------------------------------------------------------------------------- /R-package/DESCRIPTION: -------------------------------------------------------------------------------- 1 | Package: tfio 2 | Type: Package 3 | Title: Interface to 'TensorFlow IO' 4 | Version: 0.4.0 5 | Authors@R: c( 6 | person("TensorFlow IO Contributors", role = c("aut", "cph"), 7 | email = "io@tensorflow.org", 8 | comment = "Full list of contributors can be found at "), 9 | person("Yuan", "Tang", role = c("aut", "cre"), 10 | email = "terrytangyuan@gmail.com", 11 | comment = c(ORCID = "0000-0001-5243-233X")), 12 | person(family = "TensorFlow Authors", role = c("cph")), 13 | person("Ant Financial", role = c("cph")), 14 | person("RStudio", role = c("cph")) 15 | ) 16 | Description: Interface to 'TensorFlow IO', Datasets and filesystem extensions maintained by `TensorFlow SIG-IO` . 17 | License: Apache License 2.0 18 | URL: https://github.com/tensorflow/io 19 | BugReports: https://github.com/tensorflow/io/issues 20 | SystemRequirements: TensorFlow >= 1.13.0 (https://www.tensorflow.org/) and TensorFlow IO >= 0.4.0 (https://github.com/tensorflow/io) 21 | Encoding: UTF-8 22 | LazyData: true 23 | Depends: 24 | R (>= 3.1) 25 | Imports: 26 | reticulate (>= 1.10), 27 | tensorflow (>= 1.9), 28 | tfdatasets (>= 1.9), 29 | forge, 30 | magrittr 31 | Roxygen: list(markdown = TRUE) 32 | RoxygenNote: 6.1.0 33 | Suggests: 34 | testthat, 35 | knitr 36 | VignetteBuilder: knitr 37 | -------------------------------------------------------------------------------- /tensorflow_io/mnist/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """MNIST Dataset. 16 | 17 | @@MNISTDataset 18 | @@MNISTImageDataset 19 | @@MNISTLabelDataset 20 | """ 21 | 22 | from __future__ import absolute_import 23 | from __future__ import division 24 | from __future__ import print_function 25 | 26 | from tensorflow_io.mnist.python.ops.mnist_dataset_ops import MNISTDataset 27 | from tensorflow_io.mnist.python.ops.mnist_dataset_ops import MNISTImageDataset 28 | from tensorflow_io.mnist.python.ops.mnist_dataset_ops import MNISTLabelDataset 29 | 30 | from tensorflow.python.util.all_util import remove_undocumented 31 | 32 | _allowed_symbols = [ 33 | "MNISTDataset", 34 | "MNISTImageDataset", 35 | "MNISTLabelDataset", 36 | ] 37 | 38 | remove_undocumented(__name__) 39 | -------------------------------------------------------------------------------- /tensorflow_io/arrow/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Arrow Dataset. 16 | 17 | @@ArrowDataset 18 | @@ArrowFeatherDataset 19 | @@ArrowStreamDataset 20 | """ 21 | 22 | from __future__ import absolute_import 23 | from __future__ import division 24 | from __future__ import print_function 25 | 26 | from tensorflow_io.arrow.python.ops.arrow_dataset_ops import ArrowDataset 27 | from tensorflow_io.arrow.python.ops.arrow_dataset_ops import ArrowFeatherDataset 28 | from tensorflow_io.arrow.python.ops.arrow_dataset_ops import ArrowStreamDataset 29 | 30 | from tensorflow.python.util.all_util import remove_undocumented 31 | 32 | _allowed_symbols = [ 33 | "ArrowDataset", 34 | "ArrowFeatherDataset", 35 | "ArrowStreamDataset", 36 | ] 37 | 38 | remove_undocumented(__name__) 39 | -------------------------------------------------------------------------------- /R-package/man/arrow_feather_dataset.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/arrow_dataset.R 3 | \name{arrow_feather_dataset} 4 | \alias{arrow_feather_dataset} 5 | \title{Creates a \code{ArrowFeatherDataset}.} 6 | \usage{ 7 | arrow_feather_dataset(filenames, columns, output_types, 8 | output_shapes = NULL) 9 | } 10 | \arguments{ 11 | \item{filenames}{A \code{tf.string} tensor, list or scalar containing files in 12 | Arrow Feather format.} 13 | 14 | \item{columns}{A list of column indices to be used in the Dataset.} 15 | 16 | \item{output_types}{Tensor dtypes of the output tensors.} 17 | 18 | \item{output_shapes}{TensorShapes of the output tensors or \code{NULL} to infer 19 | partial.} 20 | } 21 | \description{ 22 | An Arrow Dataset for reading record batches from Arrow feather files. Feather 23 | is a light-weight columnar format ideal for simple writing of Pandas 24 | DataFrames. 25 | } 26 | \examples{ 27 | \dontrun{ 28 | dataset <- arrow_feather_dataset( 29 | list('/path/to/a.feather', '/path/to/b.feather'), 30 | columns = reticulate::tuple(0L, 1L), 31 | output_types = reticulate::tuple(tf$int32, tf$float32), 32 | output_shapes = reticulate::tuple(list(), list())) \%>\% 33 | dataset_repeat(1) 34 | 35 | sess <- tf$Session() 36 | iterator <- make_iterator_one_shot(dataset) 37 | next_batch <- iterator_get_next(iterator) 38 | 39 | until_out_of_range({ 40 | batch <- sess$run(next_batch) 41 | print(batch) 42 | }) 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /tensorflow_io/bigtable/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Cloud Bigtable Client for TensorFlow. 16 | 17 | This contrib package allows TensorFlow to interface directly with Cloud Bigtable 18 | for high-speed data loading. 19 | 20 | @@BigtableClient 21 | @@BigtableTable 22 | 23 | """ 24 | 25 | from __future__ import absolute_import 26 | from __future__ import division 27 | from __future__ import print_function 28 | 29 | from tensorflow.python.util.all_util import remove_undocumented 30 | from tensorflow_io.bigtable.python.ops.bigtable_api import BigtableClient 31 | from tensorflow_io.bigtable.python.ops.bigtable_api import BigtableTable 32 | 33 | _allowed_symbols = [ 34 | 'BigtableClient', 35 | 'BigtableTable', 36 | ] 37 | 38 | remove_undocumented(__name__, _allowed_symbols) 39 | -------------------------------------------------------------------------------- /R-package/R/image_dataset.R: -------------------------------------------------------------------------------- 1 | #' Create a `WebPDataset`. 2 | #' 3 | #' A WebP Image File Dataset that reads the WebP file. 4 | #' 5 | #' @param filenames A `tf.string` tensor containing one or more filenames. 6 | #' 7 | #' @examples \dontrun{ 8 | #' dataset <- webp_dataset( 9 | #' filenames = list("testdata/sample.webp")) %>% 10 | #' dataset_repeat(1) 11 | #' 12 | #' sess <- tf$Session() 13 | #' iterator <- make_iterator_one_shot(dataset) 14 | #' next_batch <- iterator_get_next(iterator) 15 | #' 16 | #' until_out_of_range({ 17 | #' batch <- sess$run(next_batch) 18 | #' print(batch) 19 | #' }) 20 | #' } 21 | #' 22 | #' @export 23 | webp_dataset <- function(filenames) { 24 | dataset <- tfio_lib$image$WebPDataset(filenames = filenames) 25 | as_tf_dataset(dataset) 26 | } 27 | #' Create a `TIFFDataset`. 28 | #' 29 | #' A TIFF Image File Dataset that reads the TIFF file. 30 | #' 31 | #' @param filenames A `tf.string` tensor containing one or more filenames. 32 | #' 33 | #' @examples \dontrun{ 34 | #' dataset <- tiff_dataset( 35 | #' filenames = list("testdata/small.tiff")) %>% 36 | #' dataset_repeat(1) 37 | #' 38 | #' sess <- tf$Session() 39 | #' iterator <- make_iterator_one_shot(dataset) 40 | #' next_batch <- iterator_get_next(iterator) 41 | #' 42 | #' until_out_of_range({ 43 | #' batch <- sess$run(next_batch) 44 | #' print(batch) 45 | #' }) 46 | #' } 47 | #' 48 | #' @export 49 | tiff_dataset <- function(filenames) { 50 | dataset <- tfio_lib$image$TIFFDataset(filenames = filenames) 51 | as_tf_dataset(dataset) 52 | } 53 | -------------------------------------------------------------------------------- /tensorflow_io/image/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Image Dataset. 16 | 17 | @@WebPDataset 18 | @@TIFFDataset 19 | @@GIFDataset 20 | @@decode_webp 21 | """ 22 | 23 | from __future__ import absolute_import 24 | from __future__ import division 25 | from __future__ import print_function 26 | 27 | from tensorflow_io.image.python.ops.image_dataset_ops import WebPDataset 28 | from tensorflow_io.image.python.ops.image_dataset_ops import TIFFDataset 29 | from tensorflow_io.image.python.ops.image_dataset_ops import GIFDataset 30 | from tensorflow_io.image.python.ops.image_dataset_ops import decode_webp 31 | 32 | from tensorflow.python.util.all_util import remove_undocumented 33 | 34 | _allowed_symbols = [ 35 | "WebPDataset", 36 | "TIFFDataset", 37 | "GIFDataset", 38 | "decode_webp", 39 | ] 40 | 41 | remove_undocumented(__name__) 42 | -------------------------------------------------------------------------------- /tensorflow_io/kafka/python/ops/kafka_ops.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """KafkaOutputSequence.""" 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | from tensorflow_io import _load_library 21 | kafka_ops = _load_library('_kafka_ops.so') 22 | 23 | 24 | class KafkaOutputSequence(object): 25 | """KafkaOutputSequence""" 26 | 27 | def __init__(self, topic, servers="localhost"): 28 | """Create a `KafkaOutputSequence`. 29 | """ 30 | self._topic = topic 31 | self._resource = kafka_ops.kafka_output_sequence( 32 | topic=topic, servers=servers) 33 | 34 | def setitem(self, index, item): 35 | kafka_ops.kafka_output_sequence_set_item(self._resource, index, item) 36 | 37 | def flush(self): 38 | kafka_ops.kafka_output_sequence_flush(self._resource) 39 | -------------------------------------------------------------------------------- /tensorflow_io/parquet/ops/dataset_ops.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/common_shape_fns.h" 17 | #include "tensorflow/core/framework/op.h" 18 | #include "tensorflow/core/framework/shape_inference.h" 19 | 20 | namespace tensorflow { 21 | 22 | REGISTER_OP("ParquetDataset") 23 | .Input("filenames: string") 24 | .Input("columns: int64") 25 | .Output("handle: variant") 26 | .Attr("output_types: list(type) >= 1") 27 | .Attr("output_shapes: list(shape) >= 1") 28 | .SetIsStateful() 29 | .SetShapeFn(shape_inference::ScalarShape); // TODO (yongtang): check that 30 | // filenames and columns are 31 | // scalars, and check 32 | // output_types and shapes. 33 | 34 | } // namespace tensorflow 35 | -------------------------------------------------------------------------------- /.travis/wheel.r.test.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | apt-get -y -qq update 4 | apt-get -y -qq install python ffmpeg 5 | curl -sSOL https://bootstrap.pypa.io/get-pip.py 6 | python get-pip.py -q 7 | 8 | CPYTHON_VERSION=$(python -c 'import sys; print(str(sys.version_info[0])+str(sys.version_info[1]))') 9 | (cd wheelhouse && python -m pip install *-cp${CPYTHON_VERSION}-*.whl) 10 | python -m pip install -q pytest boto3 google-cloud-pubsub==0.39.1 pyarrow==0.11.1 pandas==0.19.2 11 | 12 | DEBIAN_FRONTEND=noninteractive apt-get -y -qq install software-properties-common apt-transport-https 13 | DEBIAN_FRONTEND=noninteractive apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 14 | DEBIAN_FRONTEND=noninteractive add-apt-repository -y "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran35/" 15 | DEBIAN_FRONTEND=noninteractive apt-get -y -qq update 16 | # Note: libpython-dev libpython3-dev are needed to for reticulate to see python binding (/usr/lib/libpython2.7.so) 17 | DEBIAN_FRONTEND=noninteractive apt-get -y -qq install libpython-dev libpython3-dev 18 | DEBIAN_FRONTEND=noninteractive apt-get -y -qq install r-base 19 | echo "options(repos = c(CRAN='http://cran.rstudio.com'))" >> ~/.Rprofile 20 | R -e 'install.packages(c("tensorflow"), quiet = TRUE)' 21 | R -e 'install.packages(c("testthat", "devtools"), quiet = TRUE)' 22 | R -e 'install.packages(c("forge"), quiet = TRUE)' 23 | R -e 'library("devtools"); install_github("rstudio/tfdatasets", ref="c6fc59b", quiet = TRUE)' 24 | 25 | (cd R-package && R -e 'v <- data.frame(devtools::test()); stopifnot(all(!v$failed && !v$skipped && !v$error))') 26 | -------------------------------------------------------------------------------- /tensorflow_io/ignite/kernels/igfs/igfs_client.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_io/ignite/kernels/igfs/igfs_client.h" 17 | 18 | namespace tensorflow { 19 | 20 | IGFSClient::IGFSClient(const string &host, int port, const string &fs_name, 21 | const string &user_name) 22 | : fs_name_(fs_name), 23 | user_name_(user_name), 24 | client_(ExtendedTCPClient(host, port, true)) { 25 | client_.Connect(); 26 | } 27 | 28 | IGFSClient::~IGFSClient() { client_.Disconnect(); } 29 | 30 | Status IGFSClient::SendRequestGetResponse(const Request &request, 31 | Response *response) { 32 | TF_RETURN_IF_ERROR(request.Write(&client_)); 33 | client_.reset(); 34 | 35 | if (response != nullptr) { 36 | TF_RETURN_IF_ERROR(response->Read(&client_)); 37 | client_.reset(); 38 | } 39 | 40 | return Status::OK(); 41 | } 42 | 43 | } // namespace tensorflow 44 | -------------------------------------------------------------------------------- /third_party/parquet.header: -------------------------------------------------------------------------------- 1 | # Parquet needs generated parquet_types.h and parquet_types.cpp which are generated 2 | # from thrift proto. 3 | # 4 | # Generating parquet_types.h and parquet_types.cpp, however, needs both bison and flex 5 | # installed, which is really an unnecessary step. 6 | # 7 | # We use the following step to generate the parquet_types.h and parquet_types.cpp files: 8 | # - In third_party directory, run `docker run -i -t --rm -v $PWD:/v -w /v ubuntu:16.04 bash -x /v/parquet.header` 9 | # - Once complete, a parquet.patch file will be generated which could be used as a patch in bazel 10 | # 11 | # docker run -i -t --rm -v $PWD:/v -w /v ubuntu:16.04 bash -x /v/parquet.header 12 | apt-get -y -qq update && apt-get -y -qq install libtool automake curl g++ pkg-config make bison flex libboost-dev libboost-filesystem-dev libboost-program-options-dev libboost-regex-dev libboost-system-dev libboost-test-dev libssl-dev libtool bison flex pkg-config cmake 13 | curl -OL https://github.com/apache/thrift/archive/0.11.0.tar.gz 14 | tar -xzf 0.11.0.tar.gz 15 | (cd thrift-0.11.0 && ./bootstrap.sh && ./configure --prefix=/usr && make install) 16 | curl -OL https://github.com/apache/arrow/archive/apache-arrow-0.11.1.tar.gz 17 | tar -xzf apache-arrow-0.11.1.tar.gz 18 | cp -r arrow-apache-arrow-0.11.1 a 19 | cp -r arrow-apache-arrow-0.11.1 b 20 | mkdir arrow-apache-arrow-0.11.1/cpp/release 21 | (cd arrow-apache-arrow-0.11.1/cpp/release && cmake .. -DARROW_PARQUET=ON && make parquet) 22 | cp arrow-apache-arrow-0.11.1/cpp/release/src/parquet/*.h b/cpp/src/parquet 23 | cp arrow-apache-arrow-0.11.1/cpp/release/src/parquet/*.cpp b/cpp/src/parquet 24 | diff -Naur a b > parquet.patch 25 | -------------------------------------------------------------------------------- /tensorflow_io/ignite/kernels/client/ignite_plain_client.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_CONTRIB_IGNITE_KERNELS_CLIENT_IGNITE_PLAIN_CLIENT_H_ 17 | #define TENSORFLOW_CONTRIB_IGNITE_KERNELS_CLIENT_IGNITE_PLAIN_CLIENT_H_ 18 | 19 | #include "tensorflow_io/ignite/kernels/client/ignite_client.h" 20 | 21 | namespace tensorflow { 22 | 23 | class PlainClient : public Client { 24 | public: 25 | PlainClient(string host, int port, bool big_endian); 26 | ~PlainClient(); 27 | 28 | Status Connect() override; 29 | Status Disconnect() override; 30 | bool IsConnected() override; 31 | int GetSocketDescriptor() override; 32 | Status ReadData(uint8_t* buf, const int32_t length) override; 33 | Status WriteData(const uint8_t* buf, const int32_t length) override; 34 | 35 | private: 36 | const string host_; 37 | const int port_; 38 | int sock_; 39 | }; 40 | 41 | } // namespace tensorflow 42 | 43 | #endif // TENSORFLOW_CONTRIB_IGNITE_KERNELS_CLIENT_IGNITE_PLAIN_CLIENT_H_ 44 | -------------------------------------------------------------------------------- /third_party/ffmpeg_3_4.BUILD: -------------------------------------------------------------------------------- 1 | # Description: 2 | # FFmpeg 3 | 4 | licenses(["notice"]) # LGPL v2.1+ license 5 | 6 | exports_files(["LICENSE.md"]) 7 | 8 | cc_import( 9 | name = "ffmpeg", 10 | hdrs = [ 11 | "libavcodec/avcodec.h", 12 | "libavcodec/version.h", 13 | "libavformat/avformat.h", 14 | "libavformat/avio.h", 15 | "libavformat/version.h", 16 | "libavutil/attributes.h", 17 | "libavutil/avconfig.h", 18 | "libavutil/avutil.h", 19 | "libavutil/buffer.h", 20 | "libavutil/channel_layout.h", 21 | "libavutil/common.h", 22 | "libavutil/cpu.h", 23 | "libavutil/dict.h", 24 | "libavutil/error.h", 25 | "libavutil/frame.h", 26 | "libavutil/imgutils.h", 27 | "libavutil/intfloat.h", 28 | "libavutil/log.h", 29 | "libavutil/macros.h", 30 | "libavutil/mathematics.h", 31 | "libavutil/mem.h", 32 | "libavutil/pixdesc.h", 33 | "libavutil/pixfmt.h", 34 | "libavutil/rational.h", 35 | "libavutil/samplefmt.h", 36 | "libavutil/version.h", 37 | "libswscale/swscale.h", 38 | "libswscale/version.h", 39 | ], 40 | visibility = ["//visibility:public"], 41 | ) 42 | 43 | genrule( 44 | name = "libavutil_avconfig_h", 45 | outs = ["libavutil/avconfig.h"], 46 | cmd = "\n".join([ 47 | "cat <<'EOF' >$@", 48 | "#ifndef AVUTIL_AVCONFIG_H", 49 | "#define AVUTIL_AVCONFIG_H", 50 | "#define AV_HAVE_BIGENDIAN 0", 51 | "#define AV_HAVE_FAST_UNALIGNED 1", 52 | "#endif /* AVUTIL_AVCONFIG_H */", 53 | "EOF", 54 | ]), 55 | ) 56 | -------------------------------------------------------------------------------- /tensorflow_io/arrow/kernels/arrow_stream_client.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_IO_ARROW_STREAM_CLIENT_H_ 17 | #define TENSORFLOW_IO_ARROW_STREAM_CLIENT_H_ 18 | 19 | #include "arrow/io/api.h" 20 | 21 | namespace tensorflow { 22 | 23 | // Class to wrap a socket as a readable Arrow InputStream 24 | class ArrowStreamClient : public arrow::io::InputStream { 25 | public: 26 | ArrowStreamClient(const std::string& host); 27 | ~ArrowStreamClient() override; 28 | 29 | arrow::Status Connect(); 30 | arrow::Status Close() override; 31 | arrow::Status Tell(int64_t* position) const override; 32 | arrow::Status Read(int64_t nbytes, int64_t* bytes_read, void* out) override; 33 | arrow::Status Read(int64_t nbytes, 34 | std::shared_ptr* out) override; 35 | 36 | private: 37 | const std::string host_; 38 | int sock_; 39 | int64_t pos_; 40 | }; 41 | 42 | } // namespace tensorflow 43 | 44 | #endif // TENSORFLOW_IO_ARROW_STREAM_CLIENT_H_ 45 | -------------------------------------------------------------------------------- /third_party/libav_9_20.BUILD: -------------------------------------------------------------------------------- 1 | # Description: 2 | # FFmpeg 3 | 4 | licenses(["notice"]) # LGPL v2.1+ license 5 | 6 | exports_files(["LICENSE.md"]) 7 | 8 | cc_import( 9 | name = "libav", 10 | hdrs = [ 11 | "libavcodec/avcodec.h", 12 | "libavcodec/old_codec_ids.h", 13 | "libavcodec/version.h", 14 | "libavformat/avformat.h", 15 | "libavformat/avio.h", 16 | "libavformat/version.h", 17 | "libavutil/attributes.h", 18 | "libavutil/avconfig.h", 19 | "libavutil/avutil.h", 20 | "libavutil/channel_layout.h", 21 | "libavutil/common.h", 22 | "libavutil/cpu.h", 23 | "libavutil/dict.h", 24 | "libavutil/error.h", 25 | "libavutil/imgutils.h", 26 | "libavutil/intfloat.h", 27 | "libavutil/log.h", 28 | "libavutil/mathematics.h", 29 | "libavutil/mem.h", 30 | "libavutil/old_pix_fmts.h", 31 | "libavutil/pixdesc.h", 32 | "libavutil/pixfmt.h", 33 | "libavutil/rational.h", 34 | "libavutil/samplefmt.h", 35 | "libavutil/time.h", 36 | "libavutil/version.h", 37 | "libswscale/swscale.h", 38 | "libswscale/version.h", 39 | ], 40 | visibility = ["//visibility:public"], 41 | ) 42 | 43 | genrule( 44 | name = "libavutil_avconfig_h", 45 | outs = ["libavutil/avconfig.h"], 46 | cmd = "\n".join([ 47 | "cat <<'EOF' >$@", 48 | "#ifndef AVUTIL_AVCONFIG_H", 49 | "#define AVUTIL_AVCONFIG_H", 50 | "#define AV_HAVE_BIGENDIAN 0", 51 | "#define AV_HAVE_FAST_UNALIGNED 1", 52 | "#endif /* AVUTIL_AVCONFIG_H */", 53 | "EOF", 54 | ]), 55 | ) 56 | -------------------------------------------------------------------------------- /tensorflow_io/ignite/kernels/igfs/igfs_writable_file.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGFS_IGFS_WRITABLE_FILE_H_ 17 | #define TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGFS_IGFS_WRITABLE_FILE_H_ 18 | 19 | #include "tensorflow_io/ignite/kernels/igfs/igfs_client.h" 20 | #include "tensorflow/core/platform/file_system.h" 21 | 22 | namespace tensorflow { 23 | 24 | class IGFSWritableFile : public WritableFile { 25 | public: 26 | IGFSWritableFile(const string &file_name, int64_t resource_id, 27 | std::unique_ptr &&client); 28 | ~IGFSWritableFile() override; 29 | Status Append(StringPiece data) override; 30 | Status Close() override; 31 | Status Flush() override; 32 | Status Sync() override; 33 | 34 | private: 35 | const string file_name_; 36 | int64_t resource_id_; 37 | std::unique_ptr client_; 38 | }; 39 | 40 | } // namespace tensorflow 41 | 42 | #endif // TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGFS_IGFS_WRITABLE_FILE_H_ 43 | -------------------------------------------------------------------------------- /tensorflow_io/ignite/kernels/igfs/igfs_random_access_file.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGFS_IGFS_RANDOM_ACCESS_FILE_H_ 17 | #define TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGFS_IGFS_RANDOM_ACCESS_FILE_H_ 18 | 19 | #include "tensorflow_io/ignite/kernels/igfs/igfs_client.h" 20 | #include "tensorflow/core/platform/file_system.h" 21 | 22 | namespace tensorflow { 23 | 24 | class IGFSRandomAccessFile : public RandomAccessFile { 25 | public: 26 | IGFSRandomAccessFile(const string &file_name, int64_t resource_id, 27 | std::unique_ptr &&client); 28 | ~IGFSRandomAccessFile() override; 29 | Status Read(uint64 offset, size_t n, StringPiece *result, 30 | char *scratch) const override; 31 | 32 | private: 33 | const string file_name_; 34 | const int64_t resource_id_; 35 | std::unique_ptr client_; 36 | }; 37 | 38 | } // namespace tensorflow 39 | 40 | #endif // TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGFS_IGFS_RANDOM_ACCESS_FILE_H_ 41 | -------------------------------------------------------------------------------- /tests/test_kinesis/kinesis_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | set -e 18 | set -o pipefail 19 | 20 | if [ "$#" -ne 2 ]; then 21 | echo "Usage: $0 start|stop " >&2 22 | exit 1 23 | fi 24 | 25 | if [[ $(uname) == "Darwin" ]]; then 26 | pip install -q --user localstack 27 | $HOME/Library/Python/2.7/bin/localstack start & 28 | exit 0 29 | fi 30 | 31 | action=$1 32 | container=$2 33 | if [ "$action" == "start" ]; then 34 | echo pull localstack/localstack:0.8.10 35 | docker pull localstack/localstack:0.8.10 36 | echo pull localstack/localstack:0.8.10 successfully 37 | docker run -d --rm -p 4568:4568 --name=$container localstack/localstack:0.8.10 38 | echo Container $container started successfully 39 | elif [ "$action" == "stop" ]; then 40 | docker rm -f $container 41 | echo Container $container removed successfully 42 | else 43 | echo "Usage: $0 start|stop " >&2 44 | exit 1 45 | fi 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /tensorflow_io/ignite/python/ops/igfs_ops.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Ignite File System for checkpointing and communication with TensorBoard. 16 | 17 | Apache Ignite is a memory-centric distributed database, caching, and 18 | processing platform for transactional, analytical, and streaming workloads, 19 | delivering in-memory speeds at petabyte scale. In addition to database 20 | functionality Apache Ignite provides a distributed file system called 21 | IGFS (https://ignite.apache.org/features/igfs.html). IGFS delivers a similar 22 | functionality to Hadoop HDFS, but only in-memory. In fact, in addition to 23 | its own APIs, IGFS implements Hadoop FileSystem API and can be transparently 24 | plugged into Hadoop or Spark deployments. This contrib package contains an 25 | integration between IGFS and TensorFlow. 26 | """ 27 | 28 | from __future__ import absolute_import 29 | from __future__ import division 30 | from __future__ import print_function 31 | 32 | from tensorflow_io import _load_library 33 | _load_library("_ignite_ops.so", "file_system") 34 | -------------------------------------------------------------------------------- /third_party/ffmpeg_2_8.BUILD: -------------------------------------------------------------------------------- 1 | # Description: 2 | # FFmpeg 3 | 4 | licenses(["notice"]) # LGPL v2.1+ license 5 | 6 | exports_files(["LICENSE.md"]) 7 | 8 | cc_import( 9 | name = "ffmpeg", 10 | hdrs = [ 11 | "libavcodec/avcodec.h", 12 | "libavcodec/old_codec_ids.h", 13 | "libavcodec/version.h", 14 | "libavformat/avformat.h", 15 | "libavformat/avio.h", 16 | "libavformat/version.h", 17 | "libavutil/attributes.h", 18 | "libavutil/avconfig.h", 19 | "libavutil/avutil.h", 20 | "libavutil/buffer.h", 21 | "libavutil/channel_layout.h", 22 | "libavutil/common.h", 23 | "libavutil/cpu.h", 24 | "libavutil/dict.h", 25 | "libavutil/error.h", 26 | "libavutil/frame.h", 27 | "libavutil/imgutils.h", 28 | "libavutil/intfloat.h", 29 | "libavutil/log.h", 30 | "libavutil/macros.h", 31 | "libavutil/mathematics.h", 32 | "libavutil/mem.h", 33 | "libavutil/old_pix_fmts.h", 34 | "libavutil/pixdesc.h", 35 | "libavutil/pixfmt.h", 36 | "libavutil/rational.h", 37 | "libavutil/samplefmt.h", 38 | "libavutil/version.h", 39 | "libswscale/swscale.h", 40 | "libswscale/version.h", 41 | ], 42 | visibility = ["//visibility:public"], 43 | ) 44 | 45 | genrule( 46 | name = "libavutil_avconfig_h", 47 | outs = ["libavutil/avconfig.h"], 48 | cmd = "\n".join([ 49 | "cat <<'EOF' >$@", 50 | "#ifndef AVUTIL_AVCONFIG_H", 51 | "#define AVUTIL_AVCONFIG_H", 52 | "#define AV_HAVE_BIGENDIAN 0", 53 | "#define AV_HAVE_FAST_UNALIGNED 1", 54 | "#endif /* AVUTIL_AVCONFIG_H */", 55 | "EOF", 56 | ]), 57 | ) 58 | -------------------------------------------------------------------------------- /tests/test_ignite/config/ignite-config-plain.xml: -------------------------------------------------------------------------------- 1 | 2 | 14 | 15 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 127.0.0.1 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /tensorflow_io/ignite/BUILD: -------------------------------------------------------------------------------- 1 | licenses(["notice"]) # Apache 2.0 2 | 3 | package(default_visibility = ["//visibility:public"]) 4 | 5 | cc_binary( 6 | name = "python/ops/_ignite_ops.so", 7 | srcs = [ 8 | "kernels/client/ignite_byte_swapper.h", 9 | "kernels/client/ignite_client.h", 10 | "kernels/client/ignite_plain_client.h", 11 | "kernels/client/ignite_plain_client_unix.cc", 12 | "kernels/client/ignite_ssl_wrapper.cc", 13 | "kernels/client/ignite_ssl_wrapper.h", 14 | "kernels/dataset/ignite_binary_object_parser.cc", 15 | "kernels/dataset/ignite_binary_object_parser.h", 16 | "kernels/dataset/ignite_dataset.cc", 17 | "kernels/dataset/ignite_dataset.h", 18 | "kernels/dataset/ignite_dataset_iterator.cc", 19 | "kernels/dataset/ignite_dataset_iterator.h", 20 | "kernels/dataset/ignite_dataset_ops.cc", 21 | "kernels/igfs/igfs.cc", 22 | "kernels/igfs/igfs.h", 23 | "kernels/igfs/igfs_client.cc", 24 | "kernels/igfs/igfs_client.h", 25 | "kernels/igfs/igfs_extended_tcp_client.cc", 26 | "kernels/igfs/igfs_extended_tcp_client.h", 27 | "kernels/igfs/igfs_messages.cc", 28 | "kernels/igfs/igfs_messages.h", 29 | "kernels/igfs/igfs_random_access_file.cc", 30 | "kernels/igfs/igfs_random_access_file.h", 31 | "kernels/igfs/igfs_writable_file.cc", 32 | "kernels/igfs/igfs_writable_file.h", 33 | "ops/dataset_ops.cc", 34 | "ops/igfs_ops.cc", 35 | ], 36 | copts = [ 37 | "-pthread", 38 | "-std=c++11", 39 | "-DNDEBUG", 40 | ], 41 | linkshared = 1, 42 | deps = [ 43 | "@boringssl//:ssl", 44 | "@local_config_tf//:libtensorflow_framework", 45 | "@local_config_tf//:tf_header_lib", 46 | ], 47 | ) 48 | -------------------------------------------------------------------------------- /tensorflow_io/kinesis/ops/dataset_ops.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/common_shape_fns.h" 17 | #include "tensorflow/core/framework/op.h" 18 | #include "tensorflow/core/framework/shape_inference.h" 19 | 20 | namespace tensorflow { 21 | 22 | REGISTER_OP("KinesisDataset") 23 | .Input("stream: string") 24 | .Input("shard: string") 25 | .Input("read_indefinitely: bool") 26 | .Input("interval: int64") 27 | .Output("handle: variant") 28 | .SetIsStateful() 29 | .SetShapeFn(shape_inference::ScalarShape) 30 | .Doc(R"doc( 31 | Creates a dataset that emits the messages of one or more Kinesis topics. 32 | 33 | stream: A `tf.string` tensor containing the name of the stream. 34 | shard: A `tf.string` tensor containing the id of the shard. 35 | read_indefinitely: If `True`, the Kinesis dataset will keep retry 36 | again on `EOF` after the `interval` period. If `False`, then 37 | the dataset will stop on `EOF`. The default value is `True`. 38 | interval: The interval for the Kinesis Client to wait before 39 | it tries to get records again (in millisecond). 40 | )doc"); 41 | 42 | } // namespace tensorflow 43 | -------------------------------------------------------------------------------- /R-package/man/make_libsvm_dataset.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/libsvm_dataset.R 3 | \name{make_libsvm_dataset} 4 | \alias{make_libsvm_dataset} 5 | \title{Create a Dataset from LibSVM files.} 6 | \usage{ 7 | make_libsvm_dataset(file_names, num_features, dtype = NULL, 8 | label_dtype = NULL, batch_size = 1, compression_type = "", 9 | buffer_size = NULL, num_parallel_parser_calls = NULL, 10 | drop_final_batch = FALSE, prefetch_buffer_size = 0) 11 | } 12 | \arguments{ 13 | \item{file_names}{A \code{tf.string} tensor containing one or more filenames.} 14 | 15 | \item{num_features}{The number of features.} 16 | 17 | \item{dtype}{The type of the output feature tensor. Default to \code{tf.float32}.} 18 | 19 | \item{label_dtype}{The type of the output label tensor. Default to 20 | \code{tf.int64}.} 21 | 22 | \item{batch_size}{An integer representing the number of records to combine in 23 | a single batch, default 1.} 24 | 25 | \item{compression_type}{A \code{tf.string} scalar evaluating to one of \code{""} (no 26 | compression), \code{"ZLIB"}, or \code{"GZIP"}.} 27 | 28 | \item{buffer_size}{A \code{tf.int64} scalar denoting the number of bytes to 29 | buffer. A value of 0 results in the default buffering values chosen based 30 | on the compression type.} 31 | 32 | \item{num_parallel_parser_calls}{Number of parallel records to parse in 33 | parallel. Defaults to an automatic selection.} 34 | 35 | \item{drop_final_batch}{Whether the last batch should be dropped in case its 36 | size is smaller than \code{batch_size}; the default behavior is not to drop the 37 | smaller batch.} 38 | 39 | \item{prefetch_buffer_size}{An integer specifying the number of feature 40 | batches to prefetch for performance improvement. Defaults to auto-tune. Set 41 | to 0 to disable prefetching.} 42 | } 43 | \description{ 44 | Create a Dataset from LibSVM files. 45 | } 46 | -------------------------------------------------------------------------------- /tensorflow_io/ignite/kernels/client/ignite_ssl_wrapper.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_CONTRIB_IGNITE_KERNELS_CLIENT_IGNITE_SSL_WRAPPER_H_ 17 | #define TENSORFLOW_CONTRIB_IGNITE_KERNELS_CLIENT_IGNITE_SSL_WRAPPER_H_ 18 | 19 | #include "tensorflow_io/ignite/kernels/client/ignite_client.h" 20 | 21 | #include 22 | 23 | namespace tensorflow { 24 | 25 | class SslWrapper : public Client { 26 | public: 27 | SslWrapper(std::shared_ptr client, string certfile, string keyfile, 28 | string cert_password, bool big_endian); 29 | ~SslWrapper(); 30 | 31 | Status Connect() override; 32 | Status Disconnect() override; 33 | bool IsConnected() override; 34 | int GetSocketDescriptor() override; 35 | Status ReadData(uint8_t* buf, const int32_t length) override; 36 | Status WriteData(const uint8_t* buf, const int32_t length) override; 37 | 38 | private: 39 | Status InitSslContext(); 40 | 41 | std::shared_ptr client_; 42 | string certfile_; 43 | string keyfile_; 44 | string cert_password_; 45 | SSL_CTX* ctx_; 46 | SSL* ssl_; 47 | }; 48 | 49 | } // namespace tensorflow 50 | 51 | #endif // TENSORFLOW_CONTRIB_IGNITE_KERNELS_CLIENT_IGNITE_SSL_WRAPPER_H_ 52 | -------------------------------------------------------------------------------- /tensorflow_io/ignite/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """IgniteDataset that allows to get data from Apache Ignite. 16 | 17 | Apache Ignite is a memory-centric distributed database, caching, and 18 | processing platform for transactional, analytical, and streaming workloads, 19 | delivering in-memory speeds at petabyte scale. This contrib package 20 | contains an integration between Apache Ignite and TensorFlow. The 21 | integration is based on tf.data from TensorFlow side and Binary Client 22 | Protocol from Apache Ignite side. It allows to use Apache Ignite as a 23 | datasource for neural network training, inference and all other 24 | computations supported by TensorFlow. Ignite Dataset is based on Apache 25 | Ignite Binary Client Protocol: 26 | https://apacheignite.readme.io/v2.6/docs/binary-client-protocol. 27 | 28 | @@IgniteDataset 29 | """ 30 | 31 | from __future__ import absolute_import 32 | from __future__ import division 33 | from __future__ import print_function 34 | 35 | from tensorflow_io.ignite.python.ops.ignite_dataset_ops import IgniteDataset 36 | from tensorflow.python.util.all_util import remove_undocumented 37 | 38 | _allowed_symbols = [ 39 | "IgniteDataset", 40 | ] 41 | 42 | remove_undocumented(__name__) 43 | -------------------------------------------------------------------------------- /tensorflow_io/kafka/ops/kafka_ops.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/common_shape_fns.h" 17 | #include "tensorflow/core/framework/op.h" 18 | #include "tensorflow/core/framework/shape_inference.h" 19 | 20 | namespace tensorflow { 21 | 22 | REGISTER_OP("KafkaOutputSequence") 23 | .Input("topic: string") 24 | .Input("servers: string") 25 | .Output("sequence: resource") 26 | .Attr("container: string = ''") 27 | .Attr("shared_name: string = ''") 28 | .SetIsStateful() 29 | .SetShapeFn([](shape_inference::InferenceContext* c) { 30 | shape_inference::ShapeHandle unused; 31 | TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused)); 32 | TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); 33 | c->set_output(0, c->Scalar()); 34 | return Status::OK(); 35 | }); 36 | REGISTER_OP("KafkaOutputSequenceSetItem") 37 | .Input("sequence: resource") 38 | .Input("index: int64") 39 | .Input("item: string") 40 | .SetIsStateful() 41 | .SetShapeFn(shape_inference::ScalarShape); 42 | REGISTER_OP("KafkaOutputSequenceFlush") 43 | .Input("sequence: resource") 44 | .SetIsStateful() 45 | .SetShapeFn(shape_inference::ScalarShape); 46 | 47 | } // namespace tensorflow 48 | -------------------------------------------------------------------------------- /R-package/R/package.R: -------------------------------------------------------------------------------- 1 | #' TensorFlow IO API for R 2 | #' 3 | #' This library provides an R interface to the 4 | #' \href{https://github.com/tensorflow/io}{TensorFlow IO} API 5 | #' that provides datasets and filesystem extensions maintained by SIG-IO. 6 | #' 7 | #' @docType package 8 | #' @name tfio 9 | NULL 10 | 11 | #' @importFrom reticulate py_last_error tuple py_str py_has_attr import 12 | #' @import tfdatasets 13 | #' @import forge 14 | NULL 15 | 16 | tfio_lib <- NULL 17 | 18 | .onLoad <- function(libname, pkgname) { 19 | 20 | # Delay load handler 21 | displayed_warning <- FALSE 22 | delay_load <- list( 23 | 24 | priority = 5, 25 | 26 | environment = "r-tensorflow-io", 27 | 28 | on_load = function() { 29 | check_tensorflow_version(displayed_warning) 30 | }, 31 | 32 | on_error = function(e) { 33 | stop(tf_config()$error_message, call. = FALSE) 34 | } 35 | ) 36 | 37 | if (!reticulate::py_module_available("tensorflow_io")) { 38 | tfio_module_not_available_message() 39 | } else { 40 | tfio_lib <<- import("tensorflow_io", delay_load = delay_load) 41 | } 42 | } 43 | 44 | tfio_module_not_available_message <- function() { 45 | packageStartupMessage( 46 | paste0("tensorflow_io Python module is not available. ", 47 | "Please install it and try load it via library(tfio) again.")) 48 | } 49 | 50 | check_tensorflow_version <- function(displayed_warning) { 51 | current_tf_ver <- tf_version() 52 | min_ver <- "1.13.0" 53 | if (current_tf_ver < min_ver) { 54 | if (!displayed_warning) { 55 | packageStartupMessage( 56 | "tfio requires TensorFlow version >= ", min_ver, " ", 57 | "(you are currently running version ", current_tf_ver, ").\n") 58 | displayed_warning <<- TRUE 59 | } 60 | } 61 | } 62 | 63 | .onUnload <- function(libpath) { 64 | 65 | } 66 | 67 | .onAttach <- function(libname, pkgname) { 68 | 69 | } 70 | 71 | .onDetach <- function(libpath) { 72 | 73 | } 74 | -------------------------------------------------------------------------------- /tensorflow_io/ignite/kernels/igfs/igfs_extended_tcp_client.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGFS_IGFS_EXTENDED_TCP_CLIENT_H_ 17 | #define TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGFS_IGFS_EXTENDED_TCP_CLIENT_H_ 18 | 19 | #include "tensorflow_io/ignite/kernels/client/ignite_plain_client.h" 20 | 21 | namespace tensorflow { 22 | 23 | class ExtendedTCPClient : public PlainClient { 24 | public: 25 | ExtendedTCPClient(const string &host, int port, bool big_endian); 26 | Status ReadData(uint8_t *buf, const int32_t length) override; 27 | Status WriteData(const uint8_t *buf, const int32_t length) override; 28 | Status Ignore(int n); 29 | Status SkipToPos(int target_pos); 30 | Status ReadBool(bool *res); 31 | Status ReadNullableString(string *res); 32 | Status ReadString(string *res); 33 | Status ReadStringMap(std::map *res); 34 | Status WriteSize(std::map::size_type s); 35 | Status FillWithZerosUntil(int n); 36 | Status WriteBool(bool val); 37 | Status WriteString(string str); 38 | Status WriteStringMap(std::map map); 39 | void reset(); 40 | 41 | private: 42 | int pos_; 43 | }; 44 | 45 | } // namespace tensorflow 46 | 47 | #endif // TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGFS_IGFS_EXTENDED_TCP_CLIENT_H_ 48 | -------------------------------------------------------------------------------- /tensorflow_io/video/BUILD: -------------------------------------------------------------------------------- 1 | licenses(["notice"]) # Apache 2.0 2 | 3 | package(default_visibility = ["//visibility:public"]) 4 | 5 | cc_binary( 6 | name = "python/ops/_video_ops_ffmpeg_3.4.so", 7 | srcs = [ 8 | "kernels/ffmpeg_3_4.cc", 9 | "kernels/video_dataset_ops.cc", 10 | "kernels/video_reader.h", 11 | "ops/dataset_ops.cc", 12 | ], 13 | copts = [ 14 | "-pthread", 15 | "-std=c++11", 16 | "-DNDEBUG", 17 | ], 18 | data = [], 19 | includes = ["."], 20 | linkopts = [], 21 | linkshared = 1, 22 | deps = [ 23 | "@ffmpeg_3_4//:ffmpeg", 24 | "@local_config_tf//:libtensorflow_framework", 25 | "@local_config_tf//:tf_header_lib", 26 | ], 27 | ) 28 | 29 | cc_binary( 30 | name = "python/ops/_video_ops_ffmpeg_2.8.so", 31 | srcs = [ 32 | "kernels/ffmpeg_2_8.cc", 33 | "kernels/video_dataset_ops.cc", 34 | "kernels/video_reader.h", 35 | "ops/dataset_ops.cc", 36 | ], 37 | copts = [ 38 | "-pthread", 39 | "-std=c++11", 40 | "-DNDEBUG", 41 | ], 42 | data = [], 43 | includes = ["."], 44 | linkopts = [], 45 | linkshared = 1, 46 | deps = [ 47 | "@ffmpeg_2_8//:ffmpeg", 48 | "@local_config_tf//:libtensorflow_framework", 49 | "@local_config_tf//:tf_header_lib", 50 | ], 51 | ) 52 | 53 | cc_binary( 54 | name = "python/ops/_video_ops_libav_9.20.so", 55 | srcs = [ 56 | "kernels/libav_9_20.cc", 57 | "kernels/video_dataset_ops.cc", 58 | "kernels/video_reader.h", 59 | "ops/dataset_ops.cc", 60 | ], 61 | copts = [ 62 | "-pthread", 63 | "-std=c++11", 64 | "-DNDEBUG", 65 | ], 66 | data = [], 67 | includes = ["."], 68 | linkopts = [], 69 | linkshared = 1, 70 | deps = [ 71 | "@libav_9_20//:libav", 72 | "@local_config_tf//:libtensorflow_framework", 73 | "@local_config_tf//:tf_header_lib", 74 | ], 75 | ) 76 | -------------------------------------------------------------------------------- /tests/test_ignite/start_ignite.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | IGNITE_VERSION=2.6.0 18 | SCRIPT_PATH="$( cd "$(dirname "$0")" ; pwd -P )" 19 | 20 | if [[ "$(uname)" == "Darwin" ]]; then 21 | set -e 22 | curl -sSOL https://archive.apache.org/dist/ignite/${IGNITE_VERSION}/apache-ignite-fabric-${IGNITE_VERSION}-bin.zip 23 | unzip -qq apache-ignite-fabric-${IGNITE_VERSION}-bin.zip 24 | apache-ignite-fabric-${IGNITE_VERSION}-bin/bin/ignite.sh tests/test_ignite/config/ignite-config-plain.xml & 25 | sleep 10 # Wait Apache Ignite to be started 26 | apache-ignite-fabric-${IGNITE_VERSION}-bin/bin/sqlline.sh -u "jdbc:ignite:thin://127.0.0.1/" --run=tests/test_ignite/sql/init.sql 27 | apache-ignite-fabric-${IGNITE_VERSION}-bin/bin/ignite.sh tests/test_ignite/config/ignite-config-igfs.xml & 28 | sleep 10 # Wait Apache Ignite to be started 29 | exit 0 30 | fi 31 | 32 | # Start Apache Ignite with plain client listener. 33 | docker run -itd --name ignite-plain -p 10800:10800 \ 34 | -v ${SCRIPT_PATH}:/data apacheignite/ignite:${IGNITE_VERSION} /data/bin/start-plain.sh 35 | 36 | # Start Apache Ignite with IGFS. 37 | docker run -itd --name ignite-igfs -p 10500:10500 \ 38 | -v ${SCRIPT_PATH}:/data apacheignite/ignite:${IGNITE_VERSION} /data/bin/start-igfs.sh 39 | 40 | # Wait Apache Ignite to be started 41 | #sleep 10 42 | -------------------------------------------------------------------------------- /tensorflow_io/ignite/kernels/igfs/igfs_random_access_file.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_io/ignite/kernels/igfs/igfs_random_access_file.h" 17 | #include "tensorflow_io/ignite/kernels/igfs/igfs_messages.h" 18 | 19 | namespace tensorflow { 20 | 21 | IGFSRandomAccessFile::IGFSRandomAccessFile(const string &file_name, 22 | int64_t resource_id, 23 | std::unique_ptr &&client) 24 | : file_name_(file_name), 25 | resource_id_(resource_id), 26 | client_(std::move(client)) {} 27 | 28 | IGFSRandomAccessFile::~IGFSRandomAccessFile() { 29 | CtrlResponse close_response = {false}; 30 | Status status = client_->Close(&close_response, resource_id_); 31 | 32 | if (!status.ok()) LOG(ERROR) << status.ToString(); 33 | } 34 | 35 | Status IGFSRandomAccessFile::Read(uint64 offset, size_t n, StringPiece *result, 36 | char *scratch) const { 37 | ReadBlockCtrlResponse response = ReadBlockCtrlResponse((uint8_t *)scratch); 38 | TF_RETURN_IF_ERROR(client_->ReadBlock(&response, resource_id_, offset, n)); 39 | 40 | std::streamsize sz = response.res.GetSuccessfullyRead(); 41 | if (sz == 0) return errors::OutOfRange("End of file"); 42 | 43 | *result = StringPiece(scratch, sz); 44 | 45 | return Status::OK(); 46 | } 47 | 48 | } // namespace tensorflow 49 | -------------------------------------------------------------------------------- /tensorflow_io/video/kernels/video_reader.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/dataset.h" 17 | #include "tensorflow/core/lib/io/buffered_inputstream.h" 18 | #include "tensorflow/core/platform/file_system.h" 19 | 20 | extern "C" { 21 | 22 | #include "libavcodec/avcodec.h" 23 | #include "libavformat/avformat.h" 24 | #include "libavutil/imgutils.h" 25 | #include "libswscale/swscale.h" 26 | #include 27 | 28 | } 29 | 30 | namespace tensorflow { 31 | namespace data { 32 | namespace video { 33 | 34 | class VideoReader { 35 | public: 36 | explicit VideoReader(const string &filename) : filename_(filename) {} 37 | 38 | Status ReadHeader(); 39 | 40 | bool ReadAhead(bool first); 41 | 42 | Status ReadFrame(int *num_bytes, uint8_t**value, int *height, int *width); 43 | 44 | virtual ~VideoReader(); 45 | 46 | private: 47 | std::string ahead_; 48 | std::string filename_; 49 | bool frame_more_ = false; 50 | bool packet_more_ = false; 51 | bool buffer_more_ = false; 52 | int stream_index_ = -1; 53 | size_t num_bytes_ = 0; 54 | uint8_t *buffer_rgb_ = 0; 55 | AVFrame *frame_rgb_ = 0; 56 | struct SwsContext *sws_context_ = 0; 57 | AVFormatContext *format_context_ = 0; 58 | AVCodecContext *codec_context_ = 0; 59 | AVFrame *frame_ = 0; 60 | AVPacket packet_; 61 | TF_DISALLOW_COPY_AND_ASSIGN(VideoReader); 62 | }; 63 | 64 | } // namespace 65 | } // namespace data 66 | } // namespace tensorflow 67 | -------------------------------------------------------------------------------- /R-package/scripts/gen_wrappers.R: -------------------------------------------------------------------------------- 1 | # Instructions: 2 | # * Ensure you have the following R libraries installed via the following: 3 | # install.packages(c("reticulate", "stringr", "roxygen2))` 4 | # * Change variable `pkg_location` below to point to your R package directory 5 | 6 | library(reticulate) 7 | library(stringr) 8 | library(roxygen2) 9 | pkg_location <- "~/repos/io/R-package" 10 | 11 | setwd(pkg_location) 12 | tfio_lib <- import("tensorflow_io") 13 | 14 | source(paste0(pkg_location, "scripts/gen_wrapper_utils.R")) 15 | 16 | generated_wrappers_file_name <- "R/auto_generated_wrappers.R" 17 | write(paste0( 18 | "# ------------------------------------------------------------------------------- 19 | # *** WARNING: DO NOT MODIFY THIS FILE *** 20 | # 21 | # Instead, modify scripts/gen_wrappers.R which automatically generates this file.\n#\n", 22 | "# Generated by reticulate v", packageVersion("reticulate"), 23 | "\n# ------------------------------------------------------------------------------- \n" 24 | ), file = generated_wrappers_file_name) 25 | 26 | py_function_custom_scaffold( 27 | "tfio_lib$kafka$KafkaDataset", 28 | r_function = "kafka_dataset", 29 | process_param_fn = process_int_param_fn, 30 | postprocess_fn = tf_dataset_postprocess_fn, 31 | file_name = generated_wrappers_file_name) 32 | 33 | py_function_custom_scaffold( 34 | "tfio_lib$ignite$IgniteDataset", 35 | r_function = "ignite_dataset", 36 | process_param_fn = process_int_param_fn, 37 | postprocess_fn = tf_dataset_postprocess_fn, 38 | file_name = generated_wrappers_file_name) 39 | 40 | py_function_custom_scaffold( 41 | "tfio_lib$kinesis$KinesisDataset", 42 | r_function = "kinesis_dataset", 43 | process_param_fn = process_int_param_fn, 44 | postprocess_fn = tf_dataset_postprocess_fn, 45 | file_name = generated_wrappers_file_name) 46 | 47 | py_function_custom_scaffold( 48 | "tfio_lib$hadoop$SequenceFileDataset", 49 | r_function = "sequence_file_dataset", 50 | process_param_fn = process_int_param_fn, 51 | postprocess_fn = tf_dataset_postprocess_fn, 52 | file_name = generated_wrappers_file_name) 53 | 54 | # Regenerate NAMESPACE and .Rd files 55 | roxygen2::roxygenise(pkg_location) 56 | -------------------------------------------------------------------------------- /tests/test_pubsub/pubsub_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | set -x 18 | set -e 19 | set -o pipefail 20 | 21 | if [ "$#" -ne 2 ]; then 22 | echo "Usage: $0 start|stop " >&2 23 | exit 1 24 | fi 25 | 26 | if [[ $(uname) == "Darwin" ]]; then 27 | curl -sSOL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-236.0.0-darwin-x86_64.tar.gz 28 | tar -xzf google-cloud-sdk-236.0.0-darwin-x86_64.tar.gz 29 | google-cloud-sdk/install.sh -q 30 | google-cloud-sdk/bin/gcloud -q components install beta 31 | google-cloud-sdk/bin/gcloud -q components install pubsub-emulator 32 | google-cloud-sdk/bin/gcloud -q beta emulators pubsub start & 33 | exit 0 34 | fi 35 | 36 | script=$(readlink -f "$0") 37 | base=$(dirname "$script") 38 | echo running from "$base" 39 | 40 | action=$1 41 | container=$2 42 | if [ "$action" == "start" ]; then 43 | echo pull google/cloud-sdk 44 | docker pull google/cloud-sdk:236.0.0 45 | echo pull google/cloud-sdk successfully 46 | docker run -d --rm --net=host --name=$container -v $base:/v -w /v google/cloud-sdk:236.0.0 bash -x -c 'gcloud beta emulators pubsub start' 47 | #echo wait 10 secs until pubsub is up and running 48 | #sleep 10 49 | elif [ "$action" == "stop" ]; then 50 | docker rm -f $container 51 | echo container $container removed successfully 52 | else 53 | echo "usage: $0 start|stop " >&2 54 | exit 1 55 | fi 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /R-package/tests/testthat/test-datasets-ops.R: -------------------------------------------------------------------------------- 1 | context("TensorFlow IO dataset ops") 2 | 3 | source("utils.R") 4 | 5 | test_succeeds("sequence_file_dataset() works successfully", { 6 | dataset <- sequence_file_dataset("testdata/string.seq") %>% 7 | dataset_repeat(2) 8 | iterate_all_batches(dataset) 9 | }) 10 | 11 | test_succeeds("kafka_dataset() works successfully", { 12 | dataset <- kafka_dataset( 13 | topics = list("test:0:0:4"), group = "test", eof = TRUE) %>% 14 | dataset_repeat(1) 15 | iterate_all_batches(dataset) 16 | }) 17 | 18 | test_succeeds("ignite_dataset() works successfully", { 19 | skip_on_travis() 20 | dataset <- ignite_dataset( 21 | cache_name = "SQL_PUBLIC_TEST_CACHE", port = 10800) 22 | iterate_all_batches(dataset) 23 | }) 24 | 25 | test_succeeds("parquet_dataset() works successfully", { 26 | skip_on_travis() 27 | dtypes <- tf$python$framework$dtypes 28 | output_types <- reticulate::tuple( 29 | dtypes$bool, dtypes$int32, dtypes$int64, dtypes$float32, dtypes$float64) 30 | dataset <- parquet_dataset( 31 | filenames = list("testdata/parquet_cpp_example.parquet"), 32 | columns = list(0, 1, 2, 4, 5), 33 | output_types = output_types) %>% 34 | dataset_repeat(2) 35 | iterate_all_batches(dataset) 36 | }) 37 | 38 | test_succeeds("webp_dataset() works successfully", { 39 | skip_on_travis() 40 | dataset <- webp_dataset( 41 | filenames = list("testdata/sample.webp")) %>% 42 | dataset_repeat(2) 43 | iterate_all_batches(dataset) 44 | }) 45 | 46 | test_succeeds("video_dataset() works successfully", { 47 | skip_on_travis() 48 | dataset <- video_dataset( 49 | filenames = list("testdata/small.mp4")) %>% 50 | dataset_repeat(2) 51 | iterate_all_batches(dataset) 52 | }) 53 | 54 | test_succeeds("lmdb_dataset() works successfully", { 55 | skip_on_travis() 56 | dataset <- lmdb_dataset( 57 | filenames = list("testdata/data.mdb")) %>% 58 | dataset_repeat(2) 59 | iterate_all_batches(dataset) 60 | }) 61 | 62 | test_succeeds("tiff_dataset() works successfully", { 63 | skip_on_travis() 64 | dataset <- tiff_dataset( 65 | filenames = list("testdata/small.tiff")) %>% 66 | dataset_repeat(2) 67 | iterate_all_batches(dataset) 68 | }) 69 | -------------------------------------------------------------------------------- /tensorflow_io/text/ops/text_ops.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/common_shape_fns.h" 17 | #include "tensorflow/core/framework/op.h" 18 | #include "tensorflow/core/framework/shape_inference.h" 19 | 20 | namespace tensorflow { 21 | 22 | REGISTER_OP("TextInput") 23 | .Input("source: string") 24 | .Output("handle: variant") 25 | .Attr("filters: list(string) = []") 26 | .SetShapeFn([](shape_inference::InferenceContext* c) { 27 | c->set_output(0, c->MakeShape({c->UnknownDim()})); 28 | return Status::OK(); 29 | }); 30 | 31 | REGISTER_OP("TextDataset") 32 | .Input("input: T") 33 | .Input("batch: int64") 34 | .Output("handle: variant") 35 | .Attr("output_types: list(type) >= 1") 36 | .Attr("output_shapes: list(shape) >= 1") 37 | .Attr("T: {string, variant} = DT_VARIANT") 38 | .SetIsStateful() 39 | .SetShapeFn([](shape_inference::InferenceContext* c) { 40 | c->set_output(0, c->MakeShape({})); 41 | return Status::OK(); 42 | }); 43 | 44 | REGISTER_OP("TextOutputSequence") 45 | .Input("destination: string") 46 | .Output("sequence: resource") 47 | .Attr("container: string = ''") 48 | .Attr("shared_name: string = ''") 49 | .SetIsStateful() 50 | .SetShapeFn(shape_inference::ScalarShape); 51 | REGISTER_OP("TextOutputSequenceSetItem") 52 | .Input("sequence: resource") 53 | .Input("index: int64") 54 | .Input("item: string") 55 | .SetIsStateful() 56 | .SetShapeFn(shape_inference::ScalarShape); 57 | 58 | } // namespace tensorflow 59 | -------------------------------------------------------------------------------- /R-package/R/libsvm_dataset.R: -------------------------------------------------------------------------------- 1 | #' Create a Dataset from LibSVM files. 2 | #' 3 | #' @param file_names A `tf.string` tensor containing one or more filenames. 4 | #' @param num_features The number of features. 5 | #' @param dtype The type of the output feature tensor. Default to `tf.float32`. 6 | #' @param label_dtype The type of the output label tensor. Default to 7 | #' `tf.int64`. 8 | #' @param batch_size An integer representing the number of records to combine in 9 | #' a single batch, default 1. 10 | #' @param compression_type A `tf.string` scalar evaluating to one of `""` (no 11 | #' compression), `"ZLIB"`, or `"GZIP"`. 12 | #' @param buffer_size A `tf.int64` scalar denoting the number of bytes to 13 | #' buffer. A value of 0 results in the default buffering values chosen based 14 | #' on the compression type. 15 | #' @param num_parallel_parser_calls Number of parallel records to parse in 16 | #' parallel. Defaults to an automatic selection. 17 | #' @param drop_final_batch Whether the last batch should be dropped in case its 18 | #' size is smaller than `batch_size`; the default behavior is not to drop the 19 | #' smaller batch. 20 | #' @param prefetch_buffer_size An integer specifying the number of feature 21 | #' batches to prefetch for performance improvement. Defaults to auto-tune. Set 22 | #' to 0 to disable prefetching. 23 | #' 24 | #' @export 25 | make_libsvm_dataset <- function( 26 | file_names, 27 | num_features, 28 | dtype = NULL, 29 | label_dtype = NULL, 30 | batch_size = 1, 31 | compression_type = '', 32 | buffer_size = NULL, 33 | num_parallel_parser_calls = NULL, 34 | drop_final_batch = FALSE, 35 | prefetch_buffer_size = 0) { 36 | dataset <- tfio_lib$libsvm$make_libsvm_dataset( 37 | file_names = file_names, 38 | num_features = num_features, 39 | dtype = dtype, 40 | label_dtype = label_dtype, 41 | batch_size = cast_scalar_integer(batch_size), 42 | compression_type = compression_type, 43 | buffer_size = cast_nullable_scalar_integer(buffer_size), 44 | num_parallel_parser_calls = cast_nullable_scalar_integer(num_parallel_parser_calls), 45 | drop_final_batch = cast_logical(drop_final_batch), 46 | prefetch_buffer_size = cast_scalar_integer(prefetch_buffer_size) 47 | ) 48 | as_tf_dataset(dataset) 49 | } 50 | -------------------------------------------------------------------------------- /tensorflow_io/cifar/ops/dataset_ops.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/common_shape_fns.h" 17 | #include "tensorflow/core/framework/op.h" 18 | #include "tensorflow/core/framework/shape_inference.h" 19 | 20 | namespace tensorflow { 21 | 22 | REGISTER_OP("CIFAR10Dataset") 23 | .Input("input: T") 24 | .Input("batch: int64") 25 | .Output("handle: variant") 26 | .Attr("output_types: list(type) >= 1") 27 | .Attr("output_shapes: list(shape) >= 1") 28 | .Attr("T: {string, variant} = DT_VARIANT") 29 | .SetIsStateful() 30 | .SetShapeFn(shape_inference::ScalarShape); 31 | 32 | REGISTER_OP("CIFAR100Dataset") 33 | .Input("input: T") 34 | .Input("batch: int64") 35 | .Output("handle: variant") 36 | .Attr("output_types: list(type) >= 1") 37 | .Attr("output_shapes: list(shape) >= 1") 38 | .Attr("T: {string, variant} = DT_VARIANT") 39 | .SetIsStateful() 40 | .SetShapeFn(shape_inference::ScalarShape); 41 | 42 | REGISTER_OP("CIFAR10Input") 43 | .Input("source: string") 44 | .Output("handle: variant") 45 | .Attr("filters: list(string) = []") 46 | .SetShapeFn([](shape_inference::InferenceContext* c) { 47 | c->set_output(0, c->MakeShape({c->UnknownDim()})); 48 | return Status::OK(); 49 | }); 50 | 51 | REGISTER_OP("CIFAR100Input") 52 | .Input("source: string") 53 | .Output("handle: variant") 54 | .Attr("filters: list(string) = []") 55 | .SetShapeFn([](shape_inference::InferenceContext* c) { 56 | c->set_output(0, c->MakeShape({c->UnknownDim()})); 57 | return Status::OK(); 58 | }); 59 | 60 | 61 | } // namespace tensorflow 62 | -------------------------------------------------------------------------------- /tensorflow_io/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """tensorflow-io""" 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import os 21 | import sys 22 | import inspect 23 | 24 | import tensorflow 25 | from tensorflow import errors 26 | 27 | def _load_library(filename, lib="op"): 28 | """_load_library""" 29 | f = inspect.getfile(sys._getframe(1)) # pylint: disable=protected-access 30 | 31 | # Construct filename 32 | f = os.path.join(os.path.dirname(f), filename) 33 | filenames = [f] 34 | 35 | # Add datapath to load if en var is set, used for running tests where shared 36 | # libraries are built in a different path 37 | datapath = os.environ.get('TFIO_DATAPATH') 38 | if datapath is not None: 39 | # Build filename from `datapath` + `package_name` + `relpath_to_library` 40 | f = os.path.join( 41 | datapath, __name__, os.path.relpath(f, os.path.dirname(__file__))) 42 | filenames.append(f) 43 | 44 | # Function to load the library, return True if file system library is loaded 45 | load_fn = tensorflow.load_op_library if lib == "op" \ 46 | else lambda f: tensorflow.compat.v1.load_file_system_library(f) is None 47 | 48 | # Try to load all paths for file, fail if none succeed 49 | errs = [] 50 | for f in filenames: 51 | try: 52 | l = load_fn(f) 53 | if l is not None: 54 | return l 55 | except errors.NotFoundError as e: 56 | errs.append(str(e)) 57 | raise NotImplementedError( 58 | "unable to open file: " + 59 | "{}, from paths: {}\ncaused by: {}".format(filename, filenames, errs)) 60 | -------------------------------------------------------------------------------- /tensorflow_io/bigtable/kernels/bigtable_range_helpers.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_io/bigtable/kernels/bigtable_range_helpers.h" 17 | 18 | #include "tensorflow/core/platform/logging.h" 19 | 20 | namespace tensorflow { 21 | 22 | namespace { 23 | 24 | string MakePrefixEndKey(const string& prefix) { 25 | string end = prefix; 26 | while (true) { 27 | if (end.empty()) { 28 | return end; 29 | } 30 | ++end[end.size() - 1]; 31 | if (end[end.size() - 1] == 0) { 32 | // Handle wraparound case. 33 | end = end.substr(0, end.size() - 1); 34 | } else { 35 | return end; 36 | } 37 | } 38 | } 39 | 40 | } // namespace 41 | 42 | /* static */ MultiModeKeyRange MultiModeKeyRange::FromPrefix(string prefix) { 43 | string end = MakePrefixEndKey(prefix); 44 | VLOG(1) << "Creating MultiModeKeyRange from Prefix: " << prefix 45 | << ", with end key: " << end; 46 | return MultiModeKeyRange(std::move(prefix), std::move(end)); 47 | } 48 | 49 | /* static */ MultiModeKeyRange MultiModeKeyRange::FromRange(string begin, 50 | string end) { 51 | return MultiModeKeyRange(std::move(begin), std::move(end)); 52 | } 53 | 54 | const string& MultiModeKeyRange::begin_key() const { return begin_; } 55 | 56 | const string& MultiModeKeyRange::end_key() const { return end_; } 57 | 58 | bool MultiModeKeyRange::contains_key(StringPiece key) const { 59 | if (StringPiece(begin_) > key) { 60 | return false; 61 | } 62 | if (StringPiece(end_) <= key && !end_.empty()) { 63 | return false; 64 | } 65 | return true; 66 | } 67 | 68 | } // namespace tensorflow 69 | -------------------------------------------------------------------------------- /tensorflow_io/kafka/ops/dataset_ops.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/common_shape_fns.h" 17 | #include "tensorflow/core/framework/op.h" 18 | #include "tensorflow/core/framework/shape_inference.h" 19 | 20 | namespace tensorflow { 21 | 22 | REGISTER_OP("KafkaDataset") 23 | .Input("topics: string") 24 | .Input("servers: string") 25 | .Input("group: string") 26 | .Input("eof: bool") 27 | .Input("timeout: int64") 28 | .Output("handle: variant") 29 | .SetIsStateful() 30 | .SetShapeFn(shape_inference::ScalarShape) 31 | .Doc(R"doc( 32 | Creates a dataset that emits the messages of one or more Kafka topics. 33 | 34 | topics: A `tf.string` tensor containing one or more subscriptions, 35 | in the format of [topic:partition:offset:length], 36 | by default length is -1 for unlimited. 37 | servers: A list of bootstrap servers. 38 | group: The consumer group id. 39 | eof: If True, the kafka reader will stop on EOF. 40 | timeout: The timeout value for the Kafka Consumer to wait 41 | (in millisecond). 42 | )doc"); 43 | 44 | REGISTER_OP("WriteKafka") 45 | .Input("message: string") 46 | .Input("topic: string") 47 | .Input("servers: string") 48 | .Output("content: string") 49 | .SetShapeFn([](shape_inference::InferenceContext* c) { 50 | shape_inference::ShapeHandle unused; 51 | TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused)); 52 | TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); 53 | TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused)); 54 | c->set_output(0, c->Scalar()); 55 | return Status::OK(); 56 | }); 57 | 58 | } // namespace tensorflow 59 | -------------------------------------------------------------------------------- /third_party/aws.BUILD: -------------------------------------------------------------------------------- 1 | # Description: 2 | # AWS C++ SDK 3 | 4 | package(default_visibility = ["//visibility:public"]) 5 | 6 | licenses(["notice"]) # Apache 2.0 7 | 8 | exports_files(["LICENSE"]) 9 | 10 | cc_library( 11 | name = "aws", 12 | srcs = glob([ 13 | "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp", 14 | "aws-cpp-sdk-core/include/**/*.h", 15 | "aws-cpp-sdk-core/source/*.cpp", 16 | "aws-cpp-sdk-core/source/auth/**/*.cpp", 17 | "aws-cpp-sdk-core/source/config/**/*.cpp", 18 | "aws-cpp-sdk-core/source/client/**/*.cpp", 19 | "aws-cpp-sdk-core/source/external/**/*.cpp", 20 | "aws-cpp-sdk-core/source/internal/**/*.cpp", 21 | "aws-cpp-sdk-core/source/http/*.cpp", 22 | "aws-cpp-sdk-core/source/http/curl/**/*.cpp", 23 | "aws-cpp-sdk-core/source/http/standard/**/*.cpp", 24 | "aws-cpp-sdk-core/source/utils/*.cpp", 25 | "aws-cpp-sdk-core/source/utils/base64/**/*.cpp", 26 | "aws-cpp-sdk-core/source/utils/json/**/*.cpp", 27 | "aws-cpp-sdk-core/source/utils/logging/**/*.cpp", 28 | "aws-cpp-sdk-core/source/utils/memory/**/*.cpp", 29 | "aws-cpp-sdk-core/source/utils/stream/**/*.cpp", 30 | "aws-cpp-sdk-core/source/utils/threading/**/*.cpp", 31 | "aws-cpp-sdk-core/source/utils/xml/**/*.cpp", 32 | "aws-cpp-sdk-core/source/utils/crypto/*.cpp", 33 | "aws-cpp-sdk-core/source/utils/crypto/factory/**/*.cpp", 34 | "aws-cpp-sdk-kinesis/include/**/*.h", 35 | "aws-cpp-sdk-kinesis/source/**/*.cpp", 36 | "aws-cpp-sdk-s3/include/**/*.h", 37 | "aws-cpp-sdk-s3/source/**/*.cpp", 38 | ]), 39 | hdrs = [ 40 | "aws-cpp-sdk-core/include/aws/core/SDKConfig.h", 41 | ], 42 | defines = [ 43 | "PLATFORM_LINUX", 44 | "ENABLE_CURL_CLIENT", 45 | "ENABLE_NO_ENCRYPTION", 46 | ], 47 | includes = [ 48 | "aws-cpp-sdk-core/include/", 49 | "aws-cpp-sdk-kinesis/include/", 50 | "aws-cpp-sdk-s3/include/", 51 | ], 52 | deps = [ 53 | "@curl", 54 | ], 55 | ) 56 | 57 | genrule( 58 | name = "SDKConfig_h", 59 | srcs = [ 60 | "aws-cpp-sdk-core/include/aws/core/SDKConfig.h.in", 61 | ], 62 | outs = [ 63 | "aws-cpp-sdk-core/include/aws/core/SDKConfig.h", 64 | ], 65 | cmd = "sed 's/cmakedefine/define/g' $< > $@", 66 | ) 67 | -------------------------------------------------------------------------------- /tests/test_text.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | # ============================================================================== 15 | """Tests for Text Input.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import os 22 | import pytest 23 | import tensorflow 24 | tensorflow.compat.v1.disable_eager_execution() 25 | 26 | from tensorflow import errors # pylint: disable=wrong-import-position 27 | import tensorflow_io.text as text_io # pylint: disable=wrong-import-position 28 | 29 | def test_text_input(): 30 | """test_text_input 31 | """ 32 | text_filename = os.path.join( 33 | os.path.dirname(os.path.abspath(__file__)), "test_text", "lorem.txt") 34 | with open(text_filename, 'rb') as f: 35 | lines = [line.strip() for line in f] 36 | text_filename = "file://" + text_filename 37 | 38 | gzip_text_filename = os.path.join( 39 | os.path.dirname(os.path.abspath(__file__)), "test_text", "lorem.txt.gz") 40 | gzip_text_filename = "file://" + gzip_text_filename 41 | 42 | lines = lines * 3 43 | filenames = [text_filename, gzip_text_filename, text_filename] 44 | dataset = text_io.TextDataset(filenames, batch=2) 45 | iterator = dataset.make_initializable_iterator() 46 | init_op = iterator.initializer 47 | get_next = iterator.get_next() 48 | with tensorflow.compat.v1.Session() as sess: 49 | sess.run(init_op) 50 | for i in range(0, len(lines) - 2, 2): 51 | v = sess.run(get_next) 52 | assert lines[i] == v[0] 53 | assert lines[i + 1] == v[1] 54 | v = sess.run(get_next) 55 | assert lines[len(lines) - 1] == v[0] 56 | with pytest.raises(errors.OutOfRangeError): 57 | sess.run(get_next) 58 | 59 | if __name__ == "__main__": 60 | test.main() 61 | -------------------------------------------------------------------------------- /tensorflow_io/ignite/kernels/igfs/igfs_writable_file.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_io/ignite/kernels/igfs/igfs_writable_file.h" 17 | #include "tensorflow_io/ignite/kernels/igfs/igfs_messages.h" 18 | 19 | namespace tensorflow { 20 | 21 | IGFSWritableFile::IGFSWritableFile(const string &file_name, int64_t resource_id, 22 | std::unique_ptr &&client) 23 | : file_name_(file_name), 24 | resource_id_(resource_id), 25 | client_(std::move(client)) {} 26 | 27 | IGFSWritableFile::~IGFSWritableFile() { 28 | if (resource_id_ >= 0) { 29 | CtrlResponse close_response = {false}; 30 | 31 | Status status = client_->Close(&close_response, resource_id_); 32 | if (!status.ok()) LOG(ERROR) << status.ToString(); 33 | } 34 | } 35 | 36 | Status IGFSWritableFile::Append(StringPiece data) { 37 | return client_->WriteBlock(resource_id_, (uint8_t *)data.data(), data.size()); 38 | } 39 | 40 | Status IGFSWritableFile::Close() { 41 | int64_t resource_to_be_closed = resource_id_; 42 | resource_id_ = -1; 43 | 44 | CtrlResponse close_response = {false}; 45 | return client_->Close(&close_response, resource_to_be_closed); 46 | } 47 | 48 | Status IGFSWritableFile::Flush() { return Sync(); } 49 | 50 | Status IGFSWritableFile::Sync() { 51 | CtrlResponse close_response = {false}; 52 | TF_RETURN_IF_ERROR(client_->Close(&close_response, resource_id_)); 53 | 54 | CtrlResponse open_append_resp(false); 55 | TF_RETURN_IF_ERROR(client_->OpenAppend(&open_append_resp, file_name_)); 56 | 57 | resource_id_ = open_append_resp.res.stream_id; 58 | 59 | return Status::OK(); 60 | } 61 | 62 | } // namespace tensorflow 63 | -------------------------------------------------------------------------------- /tensorflow_io/arrow/ops/dataset_ops.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/common_shape_fns.h" 17 | #include "tensorflow/core/framework/op.h" 18 | #include "tensorflow/core/framework/shape_inference.h" 19 | 20 | namespace tensorflow { 21 | 22 | REGISTER_OP("ArrowDataset") 23 | .Input("serialized_batches: string") 24 | .Input("columns: int32") 25 | .Output("handle: variant") 26 | .Attr("output_types: list(type) >= 1") 27 | .Attr("output_shapes: list(shape) >= 1") 28 | .SetIsStateful() 29 | .SetShapeFn(shape_inference::ScalarShape) 30 | .Doc(R"doc( 31 | Creates a dataset that reads serialized Arrow RecordBatches in file format. 32 | 33 | serialized_batches: Serialized Arrow RecordBatches. 34 | )doc"); 35 | 36 | REGISTER_OP("ArrowFeatherDataset") 37 | .Input("filenames: string") 38 | .Input("columns: int32") 39 | .Output("handle: variant") 40 | .Attr("output_types: list(type) >= 1") 41 | .Attr("output_shapes: list(shape) >= 1") 42 | .SetIsStateful() 43 | .SetShapeFn(shape_inference::ScalarShape) 44 | .Doc(R"doc( 45 | Creates a dataset that reads files in Arrow Feather format. 46 | 47 | filenames: One or more file paths. 48 | )doc"); 49 | 50 | REGISTER_OP("ArrowStreamDataset") 51 | .Input("host: string") 52 | .Input("columns: int32") 53 | .Output("handle: variant") 54 | .Attr("output_types: list(type) >= 1") 55 | .Attr("output_shapes: list(shape) >= 1") 56 | .SetIsStateful() 57 | .SetShapeFn(shape_inference::ScalarShape) 58 | .Doc(R"doc( 59 | Creates a dataset that connects to a host serving Arrow RecordBatches in stream format. 60 | 61 | host: A host address that is serving an Arrow stream. 62 | )doc"); 63 | 64 | } // namespace tensorflow 65 | -------------------------------------------------------------------------------- /tensorflow_io/libsvm/ops/libsvm_ops.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/common_shape_fns.h" 17 | #include "tensorflow/core/framework/op.h" 18 | #include "tensorflow/core/framework/shape_inference.h" 19 | 20 | namespace tensorflow { 21 | 22 | using shape_inference::InferenceContext; 23 | 24 | REGISTER_OP("DecodeLibsvm") 25 | .Input("input: string") 26 | .Output("label: label_dtype") 27 | .Output("feature_indices: int64") 28 | .Output("feature_values: dtype") 29 | .Output("feature_shape: int64") 30 | .Attr("dtype: {float, double, int32, int64} = DT_FLOAT") 31 | .Attr("label_dtype: {float, double, int32, int64} = DT_INT64") 32 | .Attr("num_features: int >= 1") 33 | .SetShapeFn([](InferenceContext* c) { 34 | c->set_output(0, c->input(0)); 35 | 36 | c->set_output(1, c->Matrix(InferenceContext::kUnknownDim, 37 | InferenceContext::kUnknownDim)); 38 | c->set_output(2, c->Vector(InferenceContext::kUnknownDim)); 39 | c->set_output(3, c->Vector(InferenceContext::kUnknownDim)); 40 | 41 | return Status::OK(); 42 | }) 43 | 44 | .Doc(R"doc( 45 | Convert LibSVM input to tensors. The output consists of 46 | a label and a feature tensor. The shape of the label tensor 47 | is the same as input and the shape of the feature tensor is 48 | `[input_shape, num_features]`. 49 | 50 | input: Each string is a record in the LibSVM. 51 | label: A tensor of the same shape as input. 52 | feature_indices: A 2-D int64 tensor of dense_shape [N, ndims]. 53 | feature_values: A 1-D tensor of any type and dense_shape [N]. 54 | feature_shape: A 1-D int64 tensor of dense_shape [ndims]. 55 | num_features: The number of features. 56 | )doc"); 57 | 58 | } // namespace tensorflow 59 | -------------------------------------------------------------------------------- /R-package/NEWS.md: -------------------------------------------------------------------------------- 1 | # tfio 0.4.0 2 | 3 | ## Major Features 4 | * Support Google Cloud Pub/Sub Dataset. 5 | * Support MNIST file format. 6 | * Support `decode_webp` in image. 7 | * Support `write_kafka` in kafka. 8 | 9 | ## Thanks to our Contributors 10 | 11 | This release contains contributions from many people: 12 | 13 | Bryan Cutler, Jongwook Choi, Sergii Khomenko, Stephan Uphoff, 14 | Yong Tang, Yuan (Terry) Tang 15 | 16 | We are also grateful to all who filed issues or helped resolve them, asked and 17 | answered questions, and were part of inspiring discussions. 18 | 19 | # tfio 0.3.0 20 | 21 | ## Major Features 22 | * Support TIFF image Dataset. 23 | * Support LMDB Dataset. 24 | 25 | ## Thanks to our Contributors 26 | 27 | This release contains contributions from many people: 28 | 29 | Bryan Cutler, Yong Tang, Yuan (Terry) Tang 30 | 31 | We are also grateful to all who filed issues or helped resolve them, asked and 32 | answered questions, and were part of inspiring discussions. 33 | 34 | # tfio 0.2.0 35 | 36 | ## Major Features 37 | * Support Apache Arrow Datasets. 38 | * Support WebP image Dataset. 39 | * Support LIBSVM Dataset. 40 | * Support Apache Parquet Dataset. 41 | * Support Video Dataset (from FFmpeg). 42 | 43 | ## Thanks to our Contributors 44 | 45 | This release contains contributions from many people: 46 | 47 | Anton Dmitriev, Bryan Cutler, Peng Yu, Yong Tang, Yuan (Terry) Tang 48 | 49 | We are also grateful to all who filed issues or helped resolve them, asked and 50 | answered questions, and were part of inspiring discussions. 51 | 52 | # tfio 0.1.0 53 | 54 | Initial release of TensorFlow I/O. 55 | 56 | ## Major Features 57 | * `tensorflow_io.ignite`: Data source for Apache Ignite and File System (IGFS). 58 | * `tensorflow_io.kafka`: Apache Kafka stream-processing support. 59 | * `tensorflow_io.kinesis`: Amazon Kinesis data streams support. 60 | * `tensorflow_io.hadoop`: Hadoop SequenceFile format support. 61 | 62 | ## Thanks to our Contributors 63 | 64 | This release contains contributions from many people: 65 | 66 | Anjali Sridhar, Anton Dmitriev, Artem Malykh, Brennan Saeta, Derek Murray, 67 | Gunhan Gulsoy, Jacques Pienaar, Jianwei Xie, Jiri Simsa, knight, Loo Rong Jie, 68 | Martin Wicke, Michael Case, Sergei Lebedev, Sourabh Bajaj, Yifei Feng, 69 | Yong Tang, Yuan (Terry) Tang, Yun Peng 70 | 71 | We are also grateful to all who filed issues or helped resolve them, asked and 72 | answered questions, and were part of inspiring discussions. 73 | -------------------------------------------------------------------------------- /R-package/man/ignite_dataset.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/ignite_dataset.R 3 | \name{ignite_dataset} 4 | \alias{ignite_dataset} 5 | \title{Create a \code{IgniteDataset}.} 6 | \usage{ 7 | ignite_dataset(cache_name, host = "localhost", port = 10800, 8 | local = FALSE, part = -1, page_size = 100, username = NULL, 9 | password = NULL, certfile = NULL, keyfile = NULL, 10 | cert_password = NULL) 11 | } 12 | \arguments{ 13 | \item{cache_name}{Cache name to be used as datasource.} 14 | 15 | \item{host}{Apache Ignite Thin Client host to be connected.} 16 | 17 | \item{port}{Apache Ignite Thin Client port to be connected.} 18 | 19 | \item{local}{Local flag that defines to query only local data.} 20 | 21 | \item{part}{Number of partitions to be queried.} 22 | 23 | \item{page_size}{Apache Ignite Thin Client page size.} 24 | 25 | \item{username}{Apache Ignite Thin Client authentication username.} 26 | 27 | \item{password}{Apache Ignite Thin Client authentication password.} 28 | 29 | \item{certfile}{File in PEM format containing the certificate as well as any 30 | number of CA certificates needed to establish the certificate's 31 | authenticity.} 32 | 33 | \item{keyfile}{File containing the private key (otherwise the private key 34 | will be taken from certfile as well).} 35 | 36 | \item{cert_password}{Password to be used if the private key is encrypted and 37 | a password is necessary.} 38 | } 39 | \description{ 40 | Apache Ignite is a memory-centric distributed database, caching, and 41 | processing platform for transactional, analytical, and streaming workloads, 42 | delivering in-memory speeds at petabyte scale. This contrib package 43 | contains an integration between Apache Ignite and TensorFlow. The 44 | integration is based on tf.data from TensorFlow side and Binary Client 45 | Protocol from Apache Ignite side. It allows to use Apache Ignite as a 46 | datasource for neural network training, inference and all other 47 | computations supported by TensorFlow. Ignite Dataset is based on Apache 48 | Ignite Binary Client Protocol. 49 | } 50 | \examples{ 51 | \dontrun{ 52 | dataset <- ignite_dataset( 53 | cache_name = "SQL_PUBLIC_TEST_CACHE", port = 10800) \%>\% 54 | dataset_repeat(1) 55 | 56 | sess <- tf$Session() 57 | iterator <- make_iterator_one_shot(dataset) 58 | next_batch <- iterator_get_next(iterator) 59 | 60 | until_out_of_range({ 61 | batch <- sess$run(next_batch) 62 | print(batch) 63 | }) 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /tests/test_lmdb.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Tests for LMDBDatasetOp.""" 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import os 21 | import shutil 22 | 23 | import tensorflow 24 | tensorflow.compat.v1.disable_eager_execution() 25 | from tensorflow import errors # pylint: disable=wrong-import-position 26 | from tensorflow import test # pylint: disable=wrong-import-position 27 | 28 | import tensorflow_io.lmdb as lmdb_io # pylint: disable=wrong-import-position 29 | 30 | 31 | class LMDBDatasetTest(test.TestCase): 32 | """LMDBDatasetTest""" 33 | 34 | def test_read_from_file(self): 35 | """test_read_from_file""" 36 | super(LMDBDatasetTest, self).setUp() 37 | # Copy database out because we need the path to be writable to use locks. 38 | path = os.path.join( 39 | os.path.dirname(os.path.abspath(__file__)), "test_lmdb", "data.mdb") 40 | self.db_path = os.path.join(self.get_temp_dir(), "data.mdb") 41 | shutil.copy(path, self.db_path) 42 | 43 | filename = self.db_path 44 | 45 | num_repeats = 2 46 | 47 | dataset = lmdb_io.LMDBDataset([filename]).repeat(num_repeats) 48 | iterator = dataset.make_initializable_iterator() 49 | init_op = iterator.initializer 50 | get_next = iterator.get_next() 51 | 52 | with self.cached_session() as sess: 53 | sess.run(init_op) 54 | for _ in range(num_repeats): 55 | for i in range(10): 56 | k = str(i).encode() 57 | v = str(chr(ord("a") + i)).encode() 58 | self.assertEqual((k, v), sess.run(get_next)) 59 | with self.assertRaises(errors.OutOfRangeError): 60 | sess.run(get_next) 61 | 62 | 63 | if __name__ == "__main__": 64 | test.main() 65 | -------------------------------------------------------------------------------- /tensorflow_io/image/ops/dataset_ops.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/common_shape_fns.h" 17 | #include "tensorflow/core/framework/op.h" 18 | #include "tensorflow/core/framework/shape_inference.h" 19 | 20 | namespace tensorflow { 21 | 22 | REGISTER_OP("WebPDataset") 23 | .Input("filenames: string") 24 | .Output("handle: variant") 25 | .SetIsStateful() 26 | .SetShapeFn([](shape_inference::InferenceContext* c) { 27 | c->set_output(0, c->MakeShape({c->UnknownDim(), c->UnknownDim(), c->UnknownDim()})); 28 | return Status::OK(); 29 | }); 30 | 31 | REGISTER_OP("TIFFDataset") 32 | .Input("filenames: string") 33 | .Output("handle: variant") 34 | .SetIsStateful() 35 | .SetShapeFn([](shape_inference::InferenceContext* c) { 36 | c->set_output(0, c->MakeShape({c->UnknownDim(), c->UnknownDim(), c->UnknownDim()})); 37 | return Status::OK(); 38 | }); 39 | 40 | REGISTER_OP("GIFDataset") 41 | .Input("filenames: string") 42 | .Output("handle: variant") 43 | .SetIsStateful() 44 | .SetShapeFn([](shape_inference::InferenceContext* c) { 45 | c->set_output(0, c->MakeShape({c->UnknownDim(), c->UnknownDim(), c->UnknownDim()})); 46 | return Status::OK(); 47 | }); 48 | 49 | REGISTER_OP("DecodeWebP") 50 | .Input("contents: string") 51 | .Output("image: uint8") 52 | .SetShapeFn([](shape_inference::InferenceContext* c) { 53 | shape_inference::ShapeHandle unused; 54 | TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused)); 55 | c->set_output(0, c->MakeShape({ 56 | shape_inference::InferenceContext::kUnknownDim, 57 | shape_inference::InferenceContext::kUnknownDim, 4})); 58 | return Status::OK(); 59 | }); 60 | 61 | } // namespace tensorflow 62 | -------------------------------------------------------------------------------- /tensorflow_io/lmdb/python/ops/lmdb_dataset_ops.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """LMDBDataset.""" 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow 21 | from tensorflow import dtypes 22 | from tensorflow.compat.v1 import data 23 | from tensorflow_io import _load_library 24 | lmdb_ops = _load_library('_lmdb_ops.so') 25 | 26 | class LMDBDataset(data.Dataset): 27 | """A LMDB Dataset that reads the lmdb file.""" 28 | 29 | def __init__(self, filenames): 30 | """Create a `LMDBDataset`. 31 | 32 | `LMDBDataset` allows a user to read data from a mdb file as 33 | (key value) pairs sequentially. 34 | 35 | For example: 36 | ```python 37 | tf.enable_eager_execution() 38 | dataset = LMDBDataset("/foo/bar.mdb") 39 | # Prints the (key, value) pairs inside a lmdb file. 40 | for key, value in dataset: 41 | print(key, value) 42 | ``` 43 | Args: 44 | filenames: A `tf.string` tensor containing one or more filenames. 45 | """ 46 | self._filenames = tensorflow.convert_to_tensor( 47 | filenames, dtype=dtypes.string, name="filenames") 48 | super(LMDBDataset, self).__init__() 49 | 50 | def _inputs(self): 51 | return [] 52 | 53 | def _as_variant_tensor(self): 54 | return lmdb_ops.lmdb_dataset( 55 | self._filenames, 56 | (dtypes.string, dtypes.string), 57 | (tensorflow.TensorShape([]), tensorflow.TensorShape([]))) 58 | 59 | @property 60 | def output_classes(self): 61 | return tensorflow.Tensor, tensorflow.Tensor 62 | 63 | @property 64 | def output_shapes(self): 65 | return (tensorflow.TensorShape([]), tensorflow.TensorShape([])) 66 | 67 | @property 68 | def output_types(self): 69 | return dtypes.string, dtypes.string 70 | -------------------------------------------------------------------------------- /tensorflow_io/ignite/ops/dataset_ops.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/common_shape_fns.h" 17 | #include "tensorflow/core/framework/op.h" 18 | #include "tensorflow/core/framework/shape_inference.h" 19 | 20 | namespace tensorflow { 21 | 22 | REGISTER_OP("IgniteDataset") 23 | .Input("cache_name: string") 24 | .Input("host: string") 25 | .Input("port: int32") 26 | .Input("local: bool") 27 | .Input("part: int32") 28 | .Input("page_size: int32") 29 | .Input("schema: int32") 30 | .Input("permutation: int32") 31 | .Output("handle: variant") 32 | .SetIsStateful() 33 | .SetShapeFn(shape_inference::ScalarShape) 34 | .Doc(R"doc( 35 | IgniteDataset that allows to get data from Apache Ignite. 36 | 37 | Apache Ignite is a memory-centric distributed database, caching, and processing 38 | platform for transactional, analytical, and streaming workloads, delivering 39 | in-memory speeds at petabyte scale. This contrib package contains an 40 | integration between Apache Ignite and TensorFlow. The integration is based on 41 | tf.data from TensorFlow side and Binary Client Protocol from Apache Ignite side. 42 | It allows to use Apache Ignite as a datasource for neural network training, 43 | inference and all other computations supported by TensorFlow. Ignite Dataset 44 | is based on Apache Ignite Binary Client Protocol. 45 | 46 | cache_name: Ignite Cache Name. 47 | host: Ignite Thin Client Host. 48 | port: Ignite Thin Client Port. 49 | local: Local flag that defines that data should be fetched from local host only. 50 | part: Partition data should be fetched from. 51 | page_size: Page size for Ignite Thin Client. 52 | schema: Internal structure that defines schema of cache objects. 53 | permutation: Internal structure that defines permutation of cache objects. 54 | )doc"); 55 | 56 | } // namespace tensorflow 57 | -------------------------------------------------------------------------------- /tests/test_ignite/config/ignite-config-igfs.xml: -------------------------------------------------------------------------------- 1 | 2 | 14 | 15 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 127.0.0.1 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /tensorflow_io/ignite/kernels/igfs/igfs.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGFS_IGFS_H_ 17 | #define TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGFS_IGFS_H_ 18 | 19 | #include "tensorflow_io/ignite/kernels/igfs/igfs_client.h" 20 | #include "tensorflow/core/platform/file_system.h" 21 | 22 | namespace tensorflow { 23 | 24 | class IGFS : public FileSystem { 25 | public: 26 | IGFS(); 27 | ~IGFS(); 28 | Status NewRandomAccessFile( 29 | const string& file_name, 30 | std::unique_ptr* result) override; 31 | Status NewWritableFile(const string& fname, 32 | std::unique_ptr* result) override; 33 | Status NewAppendableFile(const string& fname, 34 | std::unique_ptr* result) override; 35 | Status NewReadOnlyMemoryRegionFromFile( 36 | const string& fname, 37 | std::unique_ptr* result) override; 38 | Status FileExists(const string& fname) override; 39 | Status GetChildren(const string& dir, std::vector* result) override; 40 | Status GetMatchingPaths(const string& pattern, 41 | std::vector* results) override; 42 | Status DeleteFile(const string& fname) override; 43 | Status CreateDir(const string& name) override; 44 | Status DeleteDir(const string& name) override; 45 | Status GetFileSize(const string& fname, uint64* size) override; 46 | Status RenameFile(const string& src, const string& target) override; 47 | Status Stat(const string& fname, FileStatistics* stat) override; 48 | string TranslateName(const string& name) const override; 49 | 50 | private: 51 | std::unique_ptr CreateClient() const; 52 | 53 | const string host_; 54 | const int port_; 55 | const string fs_name_; 56 | }; 57 | 58 | } // namespace tensorflow 59 | 60 | #endif // TENSORFLOW_CONTRIB_IGNITE_KERNELS_IGFS_IGFS_H_ 61 | -------------------------------------------------------------------------------- /tensorflow_io/text/python/ops/text_ops.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """TextInput/TextOutput.""" 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow 21 | from tensorflow import dtypes 22 | from tensorflow.compat.v1 import data 23 | from tensorflow_io import _load_library 24 | text_ops = _load_library('_text_ops.so') 25 | 26 | class TextDataset(data.Dataset): 27 | """A Text Dataset 28 | """ 29 | 30 | def __init__(self, filename, batch=None): 31 | """Create a Text Reader. 32 | 33 | Args: 34 | filename: A `tf.string` tensor containing one or more filenames. 35 | """ 36 | self._data_input = text_ops.text_input(filename, ["none", "gz"]) 37 | self._batch = 0 if batch is None else batch 38 | super(TextDataset, self).__init__() 39 | 40 | def _inputs(self): 41 | return [] 42 | 43 | def _as_variant_tensor(self): 44 | return text_ops.text_dataset( 45 | self._data_input, 46 | self._batch, 47 | output_types=self.output_types, 48 | output_shapes=self.output_shapes) 49 | 50 | @property 51 | def output_shapes(self): 52 | return tuple([ 53 | tensorflow.TensorShape([])]) if self._batch == 0 else tuple([ 54 | tensorflow.TensorShape([None])]) 55 | 56 | @property 57 | def output_classes(self): 58 | return tensorflow.Tensor 59 | 60 | @property 61 | def output_types(self): 62 | return tuple([dtypes.string]) 63 | 64 | class TextOutputSequence(object): 65 | """TextOutputSequence""" 66 | 67 | def __init__(self, filenames): 68 | """Create a `TextOutputSequence`. 69 | """ 70 | self._filenames = filenames 71 | self._resource = text_ops.text_output_sequence(destination=filenames) 72 | 73 | def setitem(self, index, item): 74 | text_ops.text_output_sequence_set_item(self._resource, index, item) 75 | -------------------------------------------------------------------------------- /tensorflow_io/mnist/ops/dataset_ops.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/common_shape_fns.h" 17 | #include "tensorflow/core/framework/op.h" 18 | #include "tensorflow/core/framework/shape_inference.h" 19 | 20 | namespace tensorflow { 21 | 22 | REGISTER_OP("MNISTImageDataset") 23 | .Input("input: T") 24 | .Input("batch: int64") 25 | .Output("handle: variant") 26 | .Attr("output_types: list(type) >= 1") 27 | .Attr("output_shapes: list(shape) >= 1") 28 | .Attr("T: {string, variant} = DT_VARIANT") 29 | .SetIsStateful() 30 | .SetShapeFn([](shape_inference::InferenceContext* c) { 31 | c->set_output(0, c->MakeShape({c->UnknownDim(), c->UnknownDim(), c->UnknownDim()})); 32 | return Status::OK(); 33 | }); 34 | REGISTER_OP("MNISTLabelDataset") 35 | .Input("input: T") 36 | .Input("batch: int64") 37 | .Output("handle: variant") 38 | .Attr("output_types: list(type) >= 1") 39 | .Attr("output_shapes: list(shape) >= 1") 40 | .Attr("T: {string, variant} = DT_VARIANT") 41 | .SetIsStateful() 42 | .SetShapeFn([](shape_inference::InferenceContext* c) { 43 | c->set_output(0, c->MakeShape({c->UnknownDim()})); 44 | return Status::OK(); 45 | }); 46 | 47 | 48 | REGISTER_OP("MNISTLabelInput") 49 | .Input("source: string") 50 | .Output("handle: variant") 51 | .Attr("filters: list(string) = []") 52 | .SetShapeFn([](shape_inference::InferenceContext* c) { 53 | c->set_output(0, c->MakeShape({c->UnknownDim()})); 54 | return Status::OK(); 55 | }); 56 | 57 | REGISTER_OP("MNISTImageInput") 58 | .Input("source: string") 59 | .Output("handle: variant") 60 | .Attr("filters: list(string) = []") 61 | .SetShapeFn([](shape_inference::InferenceContext* c) { 62 | c->set_output(0, c->MakeShape({c->UnknownDim()})); 63 | return Status::OK(); 64 | }); 65 | 66 | } // namespace tensorflow 67 | -------------------------------------------------------------------------------- /tensorflow_io/ignite/kernels/dataset/ignite_dataset.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_CONTRIB_IGNITE_KERNELS_DATASET_IGNITE_DATASET_H_ 17 | #define TENSORFLOW_CONTRIB_IGNITE_KERNELS_DATASET_IGNITE_DATASET_H_ 18 | 19 | #include "tensorflow/core/framework/dataset.h" 20 | 21 | namespace tensorflow { 22 | 23 | class IgniteDataset : public DatasetBase { 24 | public: 25 | IgniteDataset(OpKernelContext* ctx, string cache_name, string host, 26 | int32 port, bool local, int32 part, int32 page_size, 27 | string username, string password, string certfile, 28 | string keyfile, string cert_password, std::vector schema, 29 | std::vector permutation, DataTypeVector dtypes, 30 | std::vector shapes); 31 | ~IgniteDataset(); 32 | std::unique_ptr MakeIteratorInternal( 33 | const string& prefix) const override; 34 | const DataTypeVector& output_dtypes() const override; 35 | const std::vector& output_shapes() const override; 36 | string DebugString() const override; 37 | 38 | protected: 39 | Status AsGraphDefInternal(SerializationContext* ctx, 40 | DatasetGraphDefBuilder* b, 41 | Node** output) const override; 42 | 43 | private: 44 | const string cache_name_; 45 | const string host_; 46 | const int32 port_; 47 | const bool local_; 48 | const int32 part_; 49 | const int32 page_size_; 50 | const string username_; 51 | const string password_; 52 | const string certfile_; 53 | const string keyfile_; 54 | const string cert_password_; 55 | const std::vector schema_; 56 | const std::vector permutation_; 57 | const DataTypeVector dtypes_; 58 | const std::vector shapes_; 59 | }; 60 | 61 | } // namespace tensorflow 62 | 63 | #endif // TENSORFLOW_CONTRIB_IGNITE_KERNELS_DATASET_IGNITE_DATASET_H_ 64 | -------------------------------------------------------------------------------- /tensorflow_io/pubsub/python/ops/pubsub_dataset_ops.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """PubSub Dataset.""" 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow 21 | from tensorflow import dtypes 22 | from tensorflow.compat.v1 import data 23 | from tensorflow_io import _load_library 24 | pubsub_ops = _load_library('_pubsub_ops.so') 25 | 26 | class PubSubDataset(data.Dataset): 27 | """A PubSub Dataset that consumes the message. 28 | """ 29 | 30 | def __init__(self, 31 | subscriptions, 32 | server=None, 33 | eof=False, 34 | timeout=1000): 35 | """Create a PubSubDataset. 36 | 37 | Args: 38 | subscriptions: A `tf.string` tensor containing one or more subscriptions. 39 | server: The pubsub server. 40 | eof: If True, the pubsub reader will stop on EOF. 41 | timeout: The timeout value for the PubSub to wait 42 | (in millisecond). 43 | """ 44 | self._subscriptions = tensorflow.convert_to_tensor( 45 | subscriptions, dtype=dtypes.string, name="subscriptions") 46 | self._server = tensorflow.convert_to_tensor( 47 | server, dtype=dtypes.string, name="server") 48 | self._eof = tensorflow.convert_to_tensor(eof, dtype=dtypes.bool, name="eof") 49 | self._timeout = tensorflow.convert_to_tensor( 50 | timeout, dtype=dtypes.int64, name="timeout") 51 | super(PubSubDataset, self).__init__() 52 | 53 | def _inputs(self): 54 | return [] 55 | 56 | def _as_variant_tensor(self): 57 | return pubsub_ops.pub_sub_dataset(self._subscriptions, self._server, 58 | self._eof, self._timeout) 59 | 60 | @property 61 | def output_classes(self): 62 | return tensorflow.Tensor 63 | 64 | @property 65 | def output_shapes(self): 66 | return tensorflow.TensorShape([]) 67 | 68 | @property 69 | def output_types(self): 70 | return dtypes.string 71 | -------------------------------------------------------------------------------- /tests/test_hadoop.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | # ============================================================================== 15 | """Tests for SequenceFileDataset.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import os 22 | 23 | import tensorflow 24 | tensorflow.compat.v1.disable_eager_execution() 25 | 26 | from tensorflow import errors # pylint: disable=wrong-import-position 27 | from tensorflow import test # pylint: disable=wrong-import-position 28 | from tensorflow.compat.v1 import data # pylint: disable=wrong-import-position 29 | 30 | import tensorflow_io.hadoop as hadoop_io # pylint: disable=wrong-import-position 31 | 32 | 33 | class SequenceFileDatasetTest(test.TestCase): 34 | """SequenceFileDatasetTest""" 35 | 36 | def test_sequence_file_dataset(self): 37 | """Test case for SequenceFileDataset. 38 | 39 | The file is generated with `org.apache.hadoop.io.Text` for key/value. 40 | There are 25 records in the file with the format of: 41 | key = XXX 42 | value = VALUEXXX 43 | where XXX is replaced as the line number (starts with 001). 44 | """ 45 | filename = os.path.join( 46 | os.path.dirname(os.path.abspath(__file__)), "test_hadoop", "string.seq") 47 | 48 | num_repeats = 2 49 | 50 | dataset = hadoop_io.SequenceFileDataset([filename]).repeat( 51 | num_repeats) 52 | iterator = data.make_initializable_iterator(dataset) 53 | init_op = iterator.initializer 54 | get_next = iterator.get_next() 55 | 56 | with self.cached_session() as sess: 57 | sess.run(init_op) 58 | for _ in range(num_repeats): # Dataset is repeated. 59 | for i in range(25): # 25 records. 60 | v0 = ("%03d" % (i + 1)).encode() 61 | v1 = ("VALUE%03d" % (i + 1)).encode() 62 | self.assertEqual((v0, v1), sess.run(get_next)) 63 | with self.assertRaises(errors.OutOfRangeError): 64 | sess.run(get_next) 65 | 66 | 67 | if __name__ == "__main__": 68 | test.main() 69 | -------------------------------------------------------------------------------- /.travis/python3.7+.release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | set -e -x 17 | 18 | # Release: 19 | # docker run -i -t --rm -v $PWD:/v -w /v --net=host ubuntu:16.04 /v/.travis/python3.7+.release.sh 20 | 21 | 22 | apt-get -y -qq update && apt-get -y -qq install \ 23 | software-properties-common \ 24 | gcc g++ make patch \ 25 | unzip curl patchelf 26 | 27 | add-apt-repository -y ppa:deadsnakes/ppa 28 | 29 | apt-get -y -qq update 30 | 31 | curl -sSOL https://bootstrap.pypa.io/get-pip.py 32 | 33 | export PYTHON_VERSION="python3.7" 34 | if [[ "$#" -gt 0 ]]; then 35 | export PYTHON_VERSION="${1}" 36 | shift 37 | fi 38 | 39 | apt-get -y -qq update && apt-get -y -qq install $PYTHON_VERSION 40 | $PYTHON_VERSION get-pip.py -q 41 | $PYTHON_VERSION -m pip --version 42 | 43 | export TENSORFLOW_INSTALL="$(${PYTHON_VERSION} setup.py --package-version)" 44 | if [[ "$#" -gt 0 ]]; then 45 | export TENSORFLOW_INSTALL="${1}" 46 | shift 47 | fi 48 | export BAZEL_VERSION=0.24.1 BAZEL_OS=$(uname | tr '[:upper:]' '[:lower:]') 49 | 50 | 51 | 52 | curl -sSOL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh 53 | bash -e bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh 2>&1 > bazel-install.log || (cat bazel-install.log && false) 54 | bazel version 55 | 56 | $PYTHON_VERSION -m pip install -q "${TENSORFLOW_INSTALL}" 57 | $PYTHON_VERSION -c 'import tensorflow as tf; print(tf.version.VERSION)' 58 | $PYTHON_VERSION config_helper.py 59 | 60 | bash -x -e .travis/bazel.build.sh 61 | 62 | 63 | if [[ "$1" == "--"* ]]; then 64 | VERSION_CHOICE=$1 65 | VERSION_NUMBER=$2 66 | shift 67 | shift 68 | fi 69 | 70 | $PYTHON_VERSION setup.py --data bazel-bin -q bdist_wheel $VERSION_CHOICE $VERSION_NUMBER 71 | ls dist/* 72 | 73 | $PYTHON_VERSION -m pip install -q wheel==0.31.1 74 | $PYTHON_VERSION -m pip install -q auditwheel==1.5.0 75 | auditwheel --version 76 | for f in dist/*.whl; do 77 | auditwheel repair $f 78 | done 79 | ls wheelhouse/* 80 | 81 | --------------------------------------------------------------------------------