├── .Rbuildignore ├── .covrignore ├── .github ├── .gitignore ├── CODE_OF_CONDUCT.md └── workflows │ ├── R-CMD-check.yaml │ ├── debug.yaml │ ├── pkgdown.yaml │ ├── pr-commands.yaml │ ├── rhub.yaml │ └── test-coverage.yaml ├── .gitignore ├── DESCRIPTION ├── LICENSE ├── LICENSE.md ├── MAINTENANCE.md ├── Makefile ├── NAMESPACE ├── NEWS.md ├── R ├── altrep.R ├── col_types.R ├── collector.R ├── cpp11.R ├── date.R ├── example.R ├── generator.R ├── locale.R ├── path.R ├── problems.R ├── sysdata.rda ├── tidyselect.R ├── utils.R ├── vroom-package.R ├── vroom.R ├── vroom_fwf.R ├── vroom_lines.R ├── vroom_write.R └── zzz.R ├── README.Rmd ├── README.md ├── _pkgdown.yml ├── codecov.yml ├── cran-comments.md ├── data-raw ├── date_symbols.R └── mtcars-multi-zip.R ├── debug └── debug.R ├── img └── taylor.gif ├── inst ├── COPYRIGHTS ├── WORDLIST ├── bench │ ├── .gitignore │ ├── GNUmakefile │ ├── README.md │ ├── all_character-long.tsv │ ├── all_character-long │ │ ├── data.table-data.table.R │ │ ├── data.table-data.table.tsv │ │ ├── input.R │ │ ├── read.delim-base.R │ │ ├── read.delim-base.tsv │ │ ├── readr-dplyr.R │ │ ├── readr-dplyr.tsv │ │ ├── vroom-base.R │ │ ├── vroom-base.tsv │ │ ├── vroom-dplyr.R │ │ ├── vroom-dplyr.tsv │ │ ├── vroom_no_altrep-dplyr.R │ │ └── vroom_no_altrep-dplyr.tsv │ ├── all_character-wide.tsv │ ├── all_character-wide │ │ ├── data.table-data.table.R │ │ ├── data.table-data.table.tsv │ │ ├── input.R │ │ ├── read.delim-base.R │ │ ├── read.delim-base.tsv │ │ ├── readr-dplyr.R │ │ ├── readr-dplyr.tsv │ │ ├── vroom-base.R │ │ ├── vroom-base.tsv │ │ ├── vroom-dplyr.R │ │ ├── vroom-dplyr.tsv │ │ ├── vroom_no_altrep-dplyr.R │ │ └── vroom_no_altrep-dplyr.tsv │ ├── all_numeric-long.tsv │ ├── all_numeric-long │ │ ├── data.table-data.table.R │ │ ├── data.table-data.table.tsv │ │ ├── input.R │ │ ├── read.delim-base.R │ │ ├── read.delim-base.tsv │ │ ├── readr-dplyr.R │ │ ├── readr-dplyr.tsv │ │ ├── vroom-base.R │ │ ├── vroom-base.tsv │ │ ├── vroom-dplyr.R │ │ ├── vroom-dplyr.tsv │ │ ├── vroom_no_altrep-base.R │ │ ├── vroom_no_altrep-base.tsv │ │ ├── vroom_no_altrep-dplyr.R │ │ └── vroom_no_altrep-dplyr.tsv │ ├── all_numeric-wide.tsv │ ├── all_numeric-wide │ │ ├── data.table-data.table.R │ │ ├── data.table-data.table.tsv │ │ ├── input.R │ │ ├── read.delim-base.R │ │ ├── read.delim-base.tsv │ │ ├── readr-dplyr.R │ │ ├── readr-dplyr.tsv │ │ ├── vroom-base.R │ │ ├── vroom-base.tsv │ │ ├── vroom-dplyr.R │ │ ├── vroom-dplyr.tsv │ │ ├── vroom_no_altrep-base.R │ │ ├── vroom_no_altrep-base.tsv │ │ ├── vroom_no_altrep-dplyr.R │ │ └── vroom_no_altrep-dplyr.tsv │ ├── download-data.sh │ ├── fwf.tsv │ ├── fwf │ │ ├── read.delim-base.R │ │ ├── read.delim-base.tsv │ │ ├── readr-dplyr.R │ │ ├── readr-dplyr.tsv │ │ ├── vroom-base.R │ │ ├── vroom-base.tsv │ │ ├── vroom-dplyr.R │ │ ├── vroom-dplyr.tsv │ │ ├── vroom_no_altrep-dplyr.R │ │ └── vroom_no_altrep-dplyr.tsv │ ├── run-bench-fwf.R │ ├── run-bench.R │ ├── script.sh │ ├── session_info.R │ ├── session_info.tsv │ ├── summarise-benchmarks.R │ ├── taxi.tsv │ ├── taxi │ │ ├── data.table-data.table.R │ │ ├── data.table-data.table.tsv │ │ ├── read.delim-base.R │ │ ├── read.delim-base.tsv │ │ ├── readr-dplyr.R │ │ ├── readr-dplyr.tsv │ │ ├── vroom-base.R │ │ ├── vroom-base.tsv │ │ ├── vroom-dplyr.R │ │ ├── vroom-dplyr.tsv │ │ ├── vroom_no_altrep-dplyr.R │ │ └── vroom_no_altrep-dplyr.tsv │ ├── taxi_multiple.tsv │ ├── taxi_multiple │ │ ├── data.table-data.table.R │ │ ├── data.table-data.table.tsv │ │ ├── readr-dplyr.R │ │ ├── readr-dplyr.tsv │ │ ├── vroom-base.R │ │ ├── vroom-base.tsv │ │ ├── vroom-dplyr.R │ │ ├── vroom-dplyr.tsv │ │ ├── vroom_no_altrep-dplyr.R │ │ └── vroom_no_altrep-dplyr.tsv │ ├── taxi_writing.tsv │ └── taxi_writing │ │ ├── base-gzip.R │ │ ├── base-gzip.tsv │ │ ├── base-multithreaded_gzip.R │ │ ├── base-multithreaded_gzip.tsv │ │ ├── base-uncompressed.R │ │ ├── base-uncompressed.tsv │ │ ├── base-zstandard.R │ │ ├── base-zstandard.tsv │ │ ├── data.table-gzip.R │ │ ├── data.table-gzip.tsv │ │ ├── data.table-multithreaded_gzip.R │ │ ├── data.table-multithreaded_gzip.tsv │ │ ├── data.table-uncompressed.R │ │ ├── data.table-uncompressed.tsv │ │ ├── readr-gzip.R │ │ ├── readr-gzip.tsv │ │ ├── readr-multithreaded_gzip.R │ │ ├── readr-multithreaded_gzip.tsv │ │ ├── readr-uncompressed.R │ │ ├── readr-uncompressed.tsv │ │ ├── readr-zstandard.R │ │ ├── readr-zstandard.tsv │ │ ├── vroom-gzip.R │ │ ├── vroom-gzip.tsv │ │ ├── vroom-multithreaded_gzip.R │ │ ├── vroom-multithreaded_gzip.tsv │ │ ├── vroom-uncompressed.R │ │ ├── vroom-uncompressed.tsv │ │ ├── vroom-zstandard.R │ │ └── vroom-zstandard.tsv ├── extdata │ ├── fwf-sample.txt │ ├── mtcars-4.csv │ ├── mtcars-6.csv │ ├── mtcars-8.csv │ ├── mtcars-multi-cyl.zip │ ├── mtcars.csv │ ├── mtcars.csv.bz2 │ ├── mtcars.csv.gz │ ├── mtcars.csv.xz │ └── mtcars.csv.zip └── words │ ├── adjectives.txt │ └── animals.txt ├── man ├── as.col_spec.Rd ├── cols.Rd ├── date_names.Rd ├── figures │ ├── lifecycle-archived.svg │ ├── lifecycle-defunct.svg │ ├── lifecycle-deprecated.svg │ ├── lifecycle-experimental.svg │ ├── lifecycle-maturing.svg │ ├── lifecycle-questioning.svg │ ├── lifecycle-retired.svg │ ├── lifecycle-soft-deprecated.svg │ ├── lifecycle-stable.svg │ ├── lifecycle-superseded.svg │ └── logo.png ├── gen_tbl.Rd ├── generators.Rd ├── guess_type.Rd ├── locale.Rd ├── output_column.Rd ├── problems.Rd ├── reexports.Rd ├── spec.Rd ├── vroom-package.Rd ├── vroom.Rd ├── vroom_altrep.Rd ├── vroom_altrep_opts.Rd ├── vroom_example.Rd ├── vroom_format.Rd ├── vroom_fwf.Rd ├── vroom_lines.Rd ├── vroom_progress.Rd ├── vroom_str.Rd ├── vroom_write.Rd └── vroom_write_lines.Rd ├── revdep ├── .gitignore ├── README.md ├── cran.md ├── email.yml ├── failures.md └── problems.md ├── src ├── .clang-format ├── .gitignore ├── DateTime.h ├── DateTimeParser.h ├── Iconv.cpp ├── Iconv.h ├── LocaleInfo.cpp ├── LocaleInfo.h ├── Makevars ├── altrep.cc ├── altrep.h ├── collectors.h ├── columns.h ├── connection.h ├── cpp11.cpp ├── delimited_index.cc ├── delimited_index.h ├── delimited_index_connection.cc ├── delimited_index_connection.h ├── fixed_width_index.h ├── fixed_width_index_connection.cc ├── fixed_width_index_connection.h ├── gen.cc ├── grisu3.c ├── grisu3.h ├── guess_type.cc ├── iconv_file.cc ├── index.h ├── index_collection.cc ├── index_collection.h ├── iterator.h ├── mio │ ├── .gitignore │ ├── CMakeLists.txt │ ├── LICENSE │ ├── README.md │ ├── cmake │ │ └── WinApiLevels.cmake │ ├── include │ │ └── mio │ │ │ ├── CMakeLists.txt │ │ │ ├── detail │ │ │ ├── CMakeLists.txt │ │ │ ├── mmap.ipp │ │ │ └── string_util.hpp │ │ │ ├── mmap.hpp │ │ │ ├── page.hpp │ │ │ └── shared_mmap.hpp │ ├── single_include │ │ └── mio │ │ │ └── mio.hpp │ ├── test │ │ ├── CMakeLists.txt │ │ ├── example.cpp │ │ └── test.cpp │ └── third_party │ │ ├── LICENSE.md │ │ ├── amalgamate.py │ │ └── config.json ├── multi_progress.h ├── parallel.h ├── r_utils.h ├── spdlog │ ├── .clang-format │ ├── .clang-tidy │ ├── .gitattributes │ ├── .gitignore │ ├── .travis.yml │ ├── CMakeLists.txt │ ├── INSTALL │ ├── LICENSE │ ├── README.md │ ├── appveyor.yml │ ├── bench │ │ ├── CMakeLists.txt │ │ ├── async_bench.cpp │ │ ├── bench.cpp │ │ ├── formatter-bench.cpp │ │ ├── latency.cpp │ │ └── utils.h │ ├── cmake │ │ ├── ide.cmake │ │ ├── pch.h.in │ │ ├── spdlog.pc.in │ │ ├── spdlogCPack.cmake │ │ ├── spdlogConfig.cmake.in │ │ ├── utils.cmake │ │ └── version.rc.in │ ├── example │ │ ├── CMakeLists.txt │ │ └── example.cpp │ ├── include │ │ └── spdlog │ │ │ ├── async.h │ │ │ ├── async_logger-inl.h │ │ │ ├── async_logger.h │ │ │ ├── cfg │ │ │ ├── argv.h │ │ │ ├── env.h │ │ │ ├── helpers-inl.h │ │ │ └── helpers.h │ │ │ ├── common-inl.h │ │ │ ├── common.h │ │ │ ├── details │ │ │ ├── backtracer-inl.h │ │ │ ├── backtracer.h │ │ │ ├── circular_q.h │ │ │ ├── console_globals.h │ │ │ ├── file_helper-inl.h │ │ │ ├── file_helper.h │ │ │ ├── fmt_helper.h │ │ │ ├── log_msg-inl.h │ │ │ ├── log_msg.h │ │ │ ├── log_msg_buffer-inl.h │ │ │ ├── log_msg_buffer.h │ │ │ ├── mpmc_blocking_q.h │ │ │ ├── null_mutex.h │ │ │ ├── os-inl.h │ │ │ ├── os.h │ │ │ ├── periodic_worker-inl.h │ │ │ ├── periodic_worker.h │ │ │ ├── registry-inl.h │ │ │ ├── registry.h │ │ │ ├── synchronous_factory.h │ │ │ ├── tcp_client-windows.h │ │ │ ├── tcp_client.h │ │ │ ├── thread_pool-inl.h │ │ │ ├── thread_pool.h │ │ │ └── windows_include.h │ │ │ ├── fmt │ │ │ ├── bin_to_hex.h │ │ │ ├── bundled │ │ │ │ ├── LICENSE.rst │ │ │ │ ├── chrono.h │ │ │ │ ├── color.h │ │ │ │ ├── compile.h │ │ │ │ ├── core.h │ │ │ │ ├── format-inl.h │ │ │ │ ├── format.h │ │ │ │ ├── locale.h │ │ │ │ ├── os.h │ │ │ │ ├── ostream.h │ │ │ │ ├── posix.h │ │ │ │ ├── printf.h │ │ │ │ └── ranges.h │ │ │ ├── chrono.h │ │ │ ├── fmt.h │ │ │ └── ostr.h │ │ │ ├── formatter.h │ │ │ ├── fwd.h │ │ │ ├── logger-inl.h │ │ │ ├── logger.h │ │ │ ├── pattern_formatter-inl.h │ │ │ ├── pattern_formatter.h │ │ │ ├── sinks │ │ │ ├── android_sink.h │ │ │ ├── ansicolor_sink-inl.h │ │ │ ├── ansicolor_sink.h │ │ │ ├── base_sink-inl.h │ │ │ ├── base_sink.h │ │ │ ├── basic_file_sink-inl.h │ │ │ ├── basic_file_sink.h │ │ │ ├── daily_file_sink.h │ │ │ ├── dist_sink.h │ │ │ ├── dup_filter_sink.h │ │ │ ├── msvc_sink.h │ │ │ ├── null_sink.h │ │ │ ├── ostream_sink.h │ │ │ ├── ringbuffer_sink.h │ │ │ ├── rotating_file_sink-inl.h │ │ │ ├── rotating_file_sink.h │ │ │ ├── sink-inl.h │ │ │ ├── sink.h │ │ │ ├── stdout_color_sinks-inl.h │ │ │ ├── stdout_color_sinks.h │ │ │ ├── stdout_sinks-inl.h │ │ │ ├── stdout_sinks.h │ │ │ ├── syslog_sink.h │ │ │ ├── systemd_sink.h │ │ │ ├── tcp_sink.h │ │ │ ├── win_eventlog_sink.h │ │ │ ├── wincolor_sink-inl.h │ │ │ └── wincolor_sink.h │ │ │ ├── spdlog-inl.h │ │ │ ├── spdlog.h │ │ │ ├── stopwatch.h │ │ │ ├── tweakme.h │ │ │ └── version.h │ ├── logos │ │ └── jetbrains-variant-4.svg │ ├── scripts │ │ ├── extract_version.py │ │ └── format.sh │ ├── src │ │ ├── async.cpp │ │ ├── cfg.cpp │ │ ├── color_sinks.cpp │ │ ├── file_sinks.cpp │ │ ├── fmt.cpp │ │ ├── spdlog.cpp │ │ └── stdout_sinks.cpp │ └── tests │ │ ├── CMakeLists.txt │ │ ├── catch.hpp │ │ ├── catch.license │ │ ├── includes.h │ │ ├── main.cpp │ │ ├── test_async.cpp │ │ ├── test_backtrace.cpp │ │ ├── test_cfg.cpp │ │ ├── test_create_dir.cpp │ │ ├── test_daily_logger.cpp │ │ ├── test_dup_filter.cpp │ │ ├── test_errors.cpp │ │ ├── test_eventlog.cpp │ │ ├── test_file_helper.cpp │ │ ├── test_file_logging.cpp │ │ ├── test_fmt_helper.cpp │ │ ├── test_macros.cpp │ │ ├── test_misc.cpp │ │ ├── test_mpmc_q.cpp │ │ ├── test_pattern_formatter.cpp │ │ ├── test_registry.cpp │ │ ├── test_sink.h │ │ ├── test_stdout_api.cpp │ │ ├── test_stopwatch.cpp │ │ ├── test_systemd.cpp │ │ ├── test_time_point.cpp │ │ ├── utils.cpp │ │ └── utils.h ├── tzfile.h ├── unicode_fopen.h ├── utils.h ├── vroom.cc ├── vroom.h ├── vroom_big_int.cc ├── vroom_big_int.h ├── vroom_chr.cc ├── vroom_chr.h ├── vroom_date.cc ├── vroom_date.h ├── vroom_dbl.cc ├── vroom_dbl.h ├── vroom_dttm.cc ├── vroom_dttm.h ├── vroom_errors.cpp ├── vroom_errors.h ├── vroom_fct.cc ├── vroom_fct.h ├── vroom_fwf.cc ├── vroom_int.cc ├── vroom_int.h ├── vroom_lgl.h ├── vroom_num.cc ├── vroom_num.h ├── vroom_rle.cc ├── vroom_rle.h ├── vroom_time.cc ├── vroom_time.h ├── vroom_types.h ├── vroom_vec.h └── vroom_write.cc ├── standalone ├── .gitignore ├── CLI11.hpp ├── Makefile ├── README.md ├── generate_tests.R └── vroom.cc ├── tests ├── spelling.R ├── testthat.R └── testthat │ ├── _snaps │ ├── col_types.md │ ├── path.md │ └── problems.md │ ├── empty-file │ ├── enc-iso-8859-1.txt │ ├── fwf-trailing-crlf.txt │ ├── fwf-trailing.txt │ ├── helper.R │ ├── multi-byte-ascii.txt │ ├── multi-byte-unicode.txt │ ├── multi-file │ ├── bar │ ├── baz │ ├── foo │ └── qux │ ├── raw.csv │ ├── test-big-int.R │ ├── test-chr.R │ ├── test-col_types.R │ ├── test-connection.R │ ├── test-datetime.R │ ├── test-dbl.R │ ├── test-factor.R │ ├── test-int.R │ ├── test-logical.R │ ├── test-multi-byte.R │ ├── test-multi-file.R │ ├── test-num.R │ ├── test-path.R │ ├── test-problems.R │ ├── test-select.R │ ├── test-vroom.R │ ├── test-vroom_fwf.R │ ├── test-vroom_lines.R │ └── test-vroom_write.R ├── vignettes ├── .gitignore ├── benchmarks.Rmd └── vroom.Rmd └── vroom.Rproj /.Rbuildignore: -------------------------------------------------------------------------------- 1 | ^Meta$ 2 | ^doc$ 3 | ^codecov\.yml$ 4 | ^appveyor\.yml$ 5 | ^pkgdown$ 6 | ^_pkgdown\.yml$ 7 | ^docs$ 8 | ^\.travis\.yml$ 9 | ^README\.Rmd$ 10 | ^LICENSE\.md$ 11 | ^src/.clang-format$ 12 | ^script.R$ 13 | ^.*\.Rproj$ 14 | ^\.Rproj\.user$ 15 | ^.covrignore$ 16 | ^vroom\.Rcheck$ 17 | ^vroom.*\.tar\.gz$ 18 | ^vroom.*\.tgz$ 19 | ^src/spdlog$ 20 | ^Makefile$ 21 | ^data-raw$ 22 | ^logs$ 23 | ^standalone$ 24 | ^src/mio/third_party$ 25 | ^cran-comments\.md$ 26 | ^CRAN-RELEASE$ 27 | ^presentation$ 28 | ^tools$ 29 | ^revdep$ 30 | ^\.github$ 31 | ^src/\.ycm_extra_conf.py$ 32 | ^\.clangd$ 33 | ^compile_commands.json$ 34 | ^inst/bench/.+/.*\.tsv$ 35 | ^src/mio/cmake/ 36 | ^src/mio/single_include/ 37 | ^src/mio/test/ 38 | ^src/mio/.*CMakeLists\.txt$ 39 | ^\.cache$ 40 | ^MAINTENANCE\.md$ 41 | ^img$ 42 | ^investigations$ 43 | ^debug$ 44 | ^\.vscode$ 45 | ^rchk.*$ 46 | ^CRAN-SUBMISSION$ 47 | -------------------------------------------------------------------------------- /.covrignore: -------------------------------------------------------------------------------- 1 | src/mio 2 | src/spdlog 3 | -------------------------------------------------------------------------------- /.github/.gitignore: -------------------------------------------------------------------------------- 1 | *.html 2 | -------------------------------------------------------------------------------- /.github/workflows/pkgdown.yaml: -------------------------------------------------------------------------------- 1 | # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples 2 | # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help 3 | on: 4 | push: 5 | branches: [main, master] 6 | pull_request: 7 | release: 8 | types: [published] 9 | workflow_dispatch: 10 | 11 | name: pkgdown.yaml 12 | 13 | permissions: read-all 14 | 15 | jobs: 16 | pkgdown: 17 | runs-on: ubuntu-latest 18 | # Only restrict concurrency for non-PR jobs 19 | concurrency: 20 | group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }} 21 | env: 22 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 23 | permissions: 24 | contents: write 25 | steps: 26 | - uses: actions/checkout@v4 27 | 28 | - uses: r-lib/actions/setup-pandoc@v2 29 | 30 | - uses: r-lib/actions/setup-r@v2 31 | with: 32 | use-public-rspm: true 33 | 34 | - uses: r-lib/actions/setup-r-dependencies@v2 35 | with: 36 | extra-packages: any::pkgdown, local::. 37 | needs: website 38 | 39 | - name: Build site 40 | run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE) 41 | shell: Rscript {0} 42 | 43 | - name: Deploy to GitHub pages 🚀 44 | if: github.event_name != 'pull_request' 45 | uses: JamesIves/github-pages-deploy-action@v4.5.0 46 | with: 47 | clean: false 48 | branch: gh-pages 49 | folder: docs 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | Meta 2 | doc 3 | inst/doc 4 | docs 5 | .Rproj.user 6 | vroom.Rcheck/ 7 | vroom*.tar.gz 8 | vroom*.tgz 9 | logs/ 10 | script.R 11 | presentation/ 12 | compile_commands.json 13 | .clangd/ 14 | .cache/ 15 | investigations 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | YEAR: 2023 2 | COPYRIGHT HOLDER: vroom authors 3 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # MIT License 2 | 3 | Copyright (c) 2023 vroom authors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | @echo "make: Entering directory '/Users/jhester/p/vroom/src'" 3 | @Rscript -e 'pkgload::load_all(quiet = FALSE)' 4 | @echo "make: Leaving directory '/Users/jhester/p/vroom/src'" 5 | 6 | test: 7 | @echo "make: Entering directory '/Users/jhester/p/vroom/tests/testthat'" 8 | @Rscript -e 'devtools::test()' 9 | @echo "make: Leaving directory '/Users/jhester/p/vroom/tests/testthat'" 10 | 11 | clean: 12 | @Rscript -e 'pkgbuild::clean_dll()' 13 | 14 | BENCH_TAXI := $(wildcard ~/data/taxi_trip_fare*csv) 15 | BENCH_FWF := ~/data/PUMS5_06.TXT 16 | BENCH_ROWS := 1000000 17 | BENCH_COLS := 25 18 | BENCH_SRC := $(wildcard inst/bench/*-benchmark.R) 19 | BENCH_OUT := $(BENCH_SRC:-benchmark.R=-times.tsv) 20 | 21 | .NOTPARALLEL: bench 22 | 23 | bench: $(BENCH_OUT) 24 | 25 | %-times.tsv : %-benchmark.R 26 | R -q --vanilla -f $< --args $(BENCH_INPUTS) 27 | 28 | inst/bench/all_%-times.tsv: inst/bench/all_%-benchmark.R 29 | R -q --vanilla -f $< --args $(BENCH_ROWS) $(BENCH_COLS) 30 | 31 | inst/bench/fwf-times.tsv: inst/bench/fwf-benchmark.R 32 | R -q --vanilla -f $< --args $(BENCH_FWF) 33 | 34 | bench-clean: 35 | rm -f $(BENCH_OUT) 36 | 37 | install: 38 | R CMD INSTALL . 39 | -------------------------------------------------------------------------------- /R/altrep.R: -------------------------------------------------------------------------------- 1 | #' Structure of objects 2 | #' 3 | #' Similar to `str()` but with more information for Altrep objects. 4 | #' 5 | #' @param x a vector 6 | #' @examples 7 | #' # when used on non-altrep objects altrep will always be false 8 | #' vroom_str(mtcars) 9 | #' 10 | #' mt <- vroom(vroom_example("mtcars.csv"), ",", altrep = c("chr", "dbl")) 11 | #' vroom_str(mt) 12 | #' @export 13 | vroom_str <- function(x) { 14 | UseMethod("vroom_str") 15 | } 16 | 17 | #' @export 18 | vroom_str.data.frame <- function(x) { 19 | classes <- glue::glue_collapse(glue::single_quote(class(x)), ", ", last = ", and ") 20 | rows <- nrow(x) 21 | cols <- ncol(x) 22 | 23 | cat(glue::glue("{classes}: {rows} obs., {cols} vars.:\n\n"), sep = "") 24 | nms <- names(x) 25 | for (i in seq_along(x)) { 26 | cat("$", nms[[i]], ":\t", sep = "") 27 | vroom_str(x[[i]]) 28 | } 29 | } 30 | 31 | #' @export 32 | vroom_str.default <- function(x) { 33 | cat(vroom_str_(x)) 34 | } 35 | -------------------------------------------------------------------------------- /R/collector.R: -------------------------------------------------------------------------------- 1 | collector <- function(type, ...) { 2 | structure(list(...), class = c(paste0("collector_", type), "collector")) 3 | } 4 | 5 | is.collector <- function(x) inherits(x, "collector") 6 | 7 | # Conditionally exported in zzz.R 8 | # @export 9 | print.collector <- function(x, ...) { 10 | cat("<", class(x)[1], ">\n", sep = "") 11 | } 12 | -------------------------------------------------------------------------------- /R/example.R: -------------------------------------------------------------------------------- 1 | #' Get path to vroom examples 2 | #' 3 | #' vroom comes bundled with a number of sample files in 4 | #' its 'inst/extdata' directory. Use `vroom_examples()` to list all the 5 | #' available examples and `vroom_example()` to retrieve the path to one 6 | #' example. 7 | #' @param path Name of file. 8 | #' @param pattern A regular expression of filenames to match. If `NULL`, all 9 | #' available files are returned. 10 | #' @export 11 | #' @examples 12 | #' # List all available examples 13 | #' vroom_examples() 14 | #' 15 | #' # Get path to one example 16 | #' vroom_example("mtcars.csv") 17 | vroom_example <- function (path) { 18 | system.file("extdata", path, package = "vroom", mustWork = TRUE) 19 | } 20 | 21 | #' @rdname vroom_example 22 | #' @export 23 | vroom_examples <- function (pattern = NULL) { 24 | list.files(system.file("extdata", package = "vroom"), pattern = pattern) 25 | } 26 | -------------------------------------------------------------------------------- /R/problems.R: -------------------------------------------------------------------------------- 1 | #' Retrieve parsing problems 2 | #' 3 | #' vroom will only fail to parse a file if the file is invalid in a way that is 4 | #' unrecoverable. However there are a number of non-fatal problems that you 5 | #' might want to know about. You can retrieve a data frame of these problems 6 | #' with this function. 7 | #' 8 | #' @param x A data frame from `vroom::vroom()`. 9 | #' @param lazy If `TRUE`, just the problems found so far are returned. If 10 | #' `FALSE` (the default) the lazy data is first read completely and all 11 | #' problems are returned. 12 | #' @return A data frame with one row for each problem and four columns: 13 | #' - row,col - Row and column number that caused the problem, referencing the 14 | #' original input 15 | #' - expected - What vroom expected to find 16 | #' - actual - What it actually found 17 | #' - file - The file with the problem 18 | #' @export 19 | problems <- function(x = .Last.value, lazy = FALSE) { 20 | if(!inherits(x, "tbl_df")) { 21 | cli::cli_abort(c( 22 | "The {.arg x} argument of {.fun vroom::problems} must be a data frame created by vroom:", 23 | x = "{.arg x} has class {.cls {class(x)}}" 24 | )) 25 | } 26 | 27 | if (!isTRUE(lazy)) { 28 | vroom_materialize(x, replace = FALSE) 29 | } 30 | 31 | probs <- attr(x, "problems") 32 | if (typeof(probs) != "externalptr") { 33 | cli::cli_abort(c( 34 | "The {.arg x} argument of {.fun vroom::problems} must be a data frame created by vroom:", 35 | x = "{.arg x} seems to have been created with something else, maybe readr?" 36 | )) 37 | } 38 | probs <- vroom_errors_(probs) 39 | probs <- probs[!duplicated(probs), ] 40 | probs <- probs[order(probs$file, probs$row, probs$col), ] 41 | 42 | tibble::as_tibble(probs) 43 | } 44 | -------------------------------------------------------------------------------- /R/sysdata.rda: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidyverse/vroom/968a3312d00f6f9de2a0c9cd3b1f2fa50a3e353e/R/sysdata.rda -------------------------------------------------------------------------------- /R/tidyselect.R: -------------------------------------------------------------------------------- 1 | #' @aliases select_helpers 2 | #' @importFrom tidyselect contains 3 | #' @export 4 | tidyselect::contains 5 | #' @importFrom tidyselect ends_with 6 | #' @export 7 | tidyselect::ends_with 8 | #' @importFrom tidyselect everything 9 | #' @export 10 | tidyselect::everything 11 | #' @importFrom tidyselect matches 12 | #' @export 13 | tidyselect::matches 14 | #' @importFrom tidyselect num_range 15 | #' @export 16 | tidyselect::num_range 17 | #' @importFrom tidyselect one_of 18 | #' @export 19 | tidyselect::one_of 20 | #' @importFrom tidyselect starts_with 21 | #' @export 22 | tidyselect::starts_with 23 | #' @importFrom tidyselect last_col 24 | #' @export 25 | tidyselect::last_col 26 | -------------------------------------------------------------------------------- /R/vroom-package.R: -------------------------------------------------------------------------------- 1 | #' @keywords internal 2 | #' @aliases vroom-package 3 | #' @useDynLib vroom, .registration = TRUE 4 | "_PACKAGE" 5 | 6 | ## usethis namespace: start 7 | #' @import rlang 8 | #' @importFrom bit64 integer64 9 | #' @importFrom crayon blue 10 | #' @importFrom crayon bold 11 | #' @importFrom crayon cyan 12 | #' @importFrom crayon green 13 | #' @importFrom crayon reset 14 | #' @importFrom crayon silver 15 | #' @importFrom glue glue 16 | #' @importFrom lifecycle deprecate_warn 17 | #' @importFrom lifecycle deprecated 18 | ## usethis namespace: end 19 | NULL 20 | -------------------------------------------------------------------------------- /R/zzz.R: -------------------------------------------------------------------------------- 1 | .onUnload <- function(libpath) { 2 | library.dynam.unload("vroom", libpath) 3 | } 4 | 5 | .onLoad <- function(...) { 6 | tzdb::tzdb_initialize() 7 | 8 | # only register conflicting S3 methods if readr is not already loaded. 9 | if (!"readr" %in% loadedNamespaces()) { 10 | s3_register("base::format", "col_spec") 11 | s3_register("base::print", "col_spec") 12 | s3_register("base::print", "collector") 13 | s3_register("base::print", "date_names") 14 | s3_register("base::print", "locale") 15 | s3_register("utils::str", "col_spec") 16 | s3_register("base::all.equal", "spec_tbl_df") 17 | s3_register("base::as.data.frame", "spec_tbl_df") 18 | s3_register("tibble::as_tibble", "spec_tbl_df") 19 | s3_register("testthat::compare", "spec_tbl_df") 20 | s3_register("waldo::compare_proxy", "spec_tbl_df") 21 | } 22 | } 23 | 24 | .conflicts.OK <- TRUE 25 | 26 | s3_register <- function(generic, class, method = NULL) { 27 | stopifnot(is.character(generic), length(generic) == 1) 28 | stopifnot(is.character(class), length(class) == 1) 29 | 30 | pieces <- strsplit(generic, "::")[[1]] 31 | stopifnot(length(pieces) == 2) 32 | package <- pieces[[1]] 33 | generic <- pieces[[2]] 34 | 35 | if (is.null(method)) { 36 | method <- get(paste0(generic, ".", class), envir = parent.frame()) 37 | } 38 | stopifnot(is.function(method)) 39 | 40 | if (package %in% loadedNamespaces()) { 41 | registerS3method(generic, class, method, envir = asNamespace(package)) 42 | } 43 | 44 | # Always register hook in case package is later unloaded & reloaded 45 | setHook( 46 | packageEvent(package, "onLoad"), 47 | function(...) { 48 | registerS3method(generic, class, method, envir = asNamespace(package)) 49 | } 50 | ) 51 | } 52 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | comment: false 2 | 3 | coverage: 4 | status: 5 | project: 6 | default: 7 | target: auto 8 | threshold: 1% 9 | informational: true 10 | patch: 11 | default: 12 | target: auto 13 | threshold: 1% 14 | informational: true 15 | -------------------------------------------------------------------------------- /cran-comments.md: -------------------------------------------------------------------------------- 1 | This small patch release is at the request of Kurt Hornik: 2 | 3 | > Specifically, plase see the URLs with 4 | 5 | > Message: Invalid URL: missing authority part 6 | 7 | > in the CRAN incoming feasibility check. 8 | 9 | vroom also has a NOTE about non-API calls to R which I know I need to address but I will do that in a near-term minor or major release, where I will also do full reverse dependency checks (involving vroom and readr). 10 | 11 | My near term goal is to just fix the bad URL in vroom's README. 12 | 13 | ## revdepcheck results 14 | 15 | THESE RESULTS PERTAIN TO vroom v1.6.5! 16 | 17 | We checked 40 reverse dependencies, comparing R CMD check results across CRAN and dev versions of this package. 18 | 19 | * We saw 0 new problems 20 | * We failed to check 0 packages 21 | -------------------------------------------------------------------------------- /data-raw/date_symbols.R: -------------------------------------------------------------------------------- 1 | library(stringi) 2 | 3 | locs <- stri_locale_list() 4 | base <- unique(stri_split_fixed(locs, "_", n = 2, simplify = TRUE)[, 1]) 5 | 6 | locale_info <- function(x) { 7 | 8 | full <- stri_datetime_symbols(x, context = "format", width = "wide") 9 | abbr <- stri_datetime_symbols(x, context = "format", width = "abbreviated") 10 | 11 | date_names( 12 | mon = full$Month, 13 | mon_ab = abbr$Month, 14 | day = full$Weekday, 15 | day_ab = abbr$Weekday, 16 | am_pm = full$AmPm 17 | ) 18 | } 19 | 20 | date_symbols <- lapply(base, locale_info) 21 | names(date_symbols) <- base 22 | 23 | usethis::use_data(date_symbols, internal = TRUE, overwrite = TRUE) 24 | -------------------------------------------------------------------------------- /data-raw/mtcars-multi-zip.R: -------------------------------------------------------------------------------- 1 | library(vroom) 2 | 3 | # this is productive in terms of how the files are named inside the zip archive 4 | # i.e. I prefer just a filename vs. a full path that refers to my machine 5 | withr::local_dir("inst/extdata/") 6 | 7 | files <- grep("mtcars-[468].csv", list.files(), value = TRUE) 8 | files 9 | 10 | target <- "mtcars-multi-cyl.zip" 11 | # if target exists, nuke it first 12 | if (file.exists(target)) unlink(target) 13 | 14 | zip(target, files) 15 | 16 | # check that the contents look as expected 17 | unzip(target, list = TRUE) 18 | 19 | withr::deferred_run() 20 | -------------------------------------------------------------------------------- /debug/debug.R: -------------------------------------------------------------------------------- 1 | devtools::clean_dll() 2 | devtools::load_all() 3 | 4 | vroom:::vroom_format( 5 | tibble::tibble(col1 = c("NATHAN", "NA", "PETER", NA)), 6 | delim = ",", num_threads = 1 7 | ) 8 | 9 | -------------------------------------------------------------------------------- /img/taylor.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidyverse/vroom/968a3312d00f6f9de2a0c9cd3b1f2fa50a3e353e/img/taylor.gif -------------------------------------------------------------------------------- /inst/WORDLIST: -------------------------------------------------------------------------------- 1 | ALTREP 2 | Altrep 3 | Altrepisode 4 | BCP 5 | BOM 6 | CMD 7 | CRAN's 8 | Codecov 9 | Datetime 10 | Dowle 11 | EBS 12 | François 13 | Gabe 14 | Grisu 15 | Kalibera 16 | KiB 17 | Lifecycle 18 | Microdata 19 | NUL 20 | ORCID 21 | PBC 22 | POSIXct 23 | Parsers 24 | Preprocess 25 | RStudio 26 | RStudio's 27 | RStudio’s 28 | Rcpp 29 | Rdatatable 30 | Romain 31 | TSV 32 | Tierney 33 | UBSAN 34 | UNK 35 | UseR 36 | Zstandard 37 | afl 38 | backslashed 39 | behaviour 40 | benchmarking 41 | bool 42 | bz 43 | bzip 44 | centric 45 | cpp 46 | csv 47 | datetime 48 | datetimes 49 | dbi 50 | de 51 | delim 52 | deterministically 53 | dplyr 54 | durations 55 | extdata 56 | fread 57 | funder 58 | fwf 59 | gcc 60 | github 61 | grisu 62 | gz 63 | gzip 64 | gzipped 65 | https 66 | lbzip 67 | libc 68 | libstdc 69 | macOS 70 | mandreyel 71 | mio 72 | mis 73 | natively 74 | nycflights 75 | parsers 76 | pigz 77 | pixz 78 | purrr 79 | rchk 80 | readr 81 | readr's 82 | recognised 83 | relicensing 84 | stringi 85 | testthat 86 | tibble 87 | tidyverse 88 | tokenizer 89 | tsv 90 | tz 91 | tzdb 92 | unicode 93 | unterminated 94 | vCPUs 95 | vroom's 96 | xlarge 97 | xz 98 | xzip 99 | -------------------------------------------------------------------------------- /inst/bench/.gitignore: -------------------------------------------------------------------------------- 1 | input.tsv 2 | -------------------------------------------------------------------------------- /inst/bench/GNUmakefile: -------------------------------------------------------------------------------- 1 | MAKEFLAGS += --no-builtin-rules 2 | 3 | BENCH_LONG_ROWS := 10000 4 | BENCH_LONG_COLS := 25 5 | BENCH_WIDE_ROWS := 1000 6 | BENCH_WIDE_COLS := 100 7 | 8 | BENCH_INPUTS := all_numeric-long/input.tsv all_numeric-wide/input.tsv all_character-long/input.tsv all_character-wide/input.tsv 9 | TAXI_INPUTS := $(wildcard ~/data/small_trip_fare_*.csv) 10 | FWF_INPUT := ~/data/small_PUMS5_06.TXT 11 | BENCH_SRC := $(wildcard */*.R) 12 | BENCH_MARKS := $(BENCH_SRC:.R=.tsv) 13 | BENCH_OUT := $(patsubst %/,%.tsv, $(wildcard */)) 14 | 15 | all: $(BENCH_OUT) session_info.tsv 16 | 17 | $(BENCH_OUT) : $(BENCH_MARKS) 18 | Rscript summarise-benchmarks.R 19 | 20 | session_info.tsv: session_info.R 21 | ./$< 22 | 23 | %-long/input.tsv: %-long/input.R 24 | Rscript $< $(BENCH_LONG_ROWS) $(BENCH_LONG_COLS) $@ 25 | 26 | %-wide/input.tsv: %-wide/input.R 27 | Rscript $< $(BENCH_WIDE_ROWS) $(BENCH_WIDE_COLS) $@ 28 | 29 | taxi/%.tsv : taxi/%.R $(TAXI_INPUTS) 30 | run-bench.R $< $@ $(word 1, $(TAXI_INPUTS)) 31 | 32 | taxi_multiple/%.tsv : taxi_multiple/%.R $(TAXI_INPUTS) 33 | run-bench.R $< $@ $(TAXI_INPUTS) 34 | 35 | taxi_writing/%.tsv : taxi_writing/%.R $(TAXI_INPUTS) 36 | run-bench.R $< $@ $(word 1, $(TAXI_INPUTS)) 37 | 38 | fwf/%.tsv : fwf/%.R $(FWF_INPUT) 39 | run-bench-fwf.R $< $@ $(FWF_INPUT) 40 | 41 | %.tsv : %.R $(BENCH_INPUTS) 42 | run-bench.R $< $@ $(@D)/input.tsv 43 | 44 | clean: 45 | rm -f $(BENCH_INPUTS) $(BENCH_MARKS) $(BENCH_OUT) session_info.tsv 46 | -------------------------------------------------------------------------------- /inst/bench/README.md: -------------------------------------------------------------------------------- 1 | # Benchmarks for vroom 2 | 3 | The benchmarks are run with a makefile, run `make` to run them. 4 | 5 | ## Personal notes on running benchmarks on AWS 6 | 7 | ### Starting up 8 | - Create new volume from previous vroom snapshot - `io1` volume type - 2500 IOPS 9 | - Attach volume to instance - `/dev/sda1` 10 | 11 | ### Attach to instance 12 | 13 | `ssh vroom-bench` 14 | 15 | When you first start there may be some unattended upgrades installing, use top 16 | to monitor and wait until they are done. 17 | 18 | ### running 19 | 20 | ``` 21 | make -j 1 \ 22 | TAXI_INPUTS='$(wildcard ~/data/trip_fare*csv)' \ 23 | FWF_INPUT=~/data/PUMS5_06.TXT \ 24 | BENCH_LONG_ROWS=1000000 \ 25 | BENCH_LOG_COLS=25 \ 26 | BENCH_WIDE_ROWS=100000 \ 27 | BENCH_WIDE_COLS=1000 28 | ``` 29 | 30 | ### Tearing down 31 | - Shut down instance 32 | - Detach volume 33 | - Create snapshot 34 | - Delete volume 35 | -------------------------------------------------------------------------------- /inst/bench/all_character-long/data.table-data.table.R: -------------------------------------------------------------------------------- 1 | library(data.table) 2 | x <- fread(file, sep = "\t", quote = "", strip.white = FALSE, na.strings = NULL) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- x[sample(NROW(x), 100), ] 7 | d <- x[X1 == "helpless_sheep", ] 8 | e <- x[ , .(mean(nchar(X2))), by = X1] 9 | -------------------------------------------------------------------------------- /inst/bench/all_character-long/data.table-data.table.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | library(data.table) 0.05641532699999985 0.05641937255859375 400248755 1000000 25 5080014848 3 | "x <- fread(file, sep = ""\t"", quote = """", strip.white = FALSE, na.strings = NULL)" 45.553360995 42.82739567756653 400248755 1000000 25 5080014848 4 | print(x) 0.01535388299999596 0.015356302261352539 400248755 1000000 25 5080014848 5 | a <- head(x) 2.4213100000025634e-4 2.4390220642089844e-4 400248755 1000000 25 5080014848 6 | b <- tail(x) 2.0244400000279938e-4 0.00020360946655273438 400248755 1000000 25 5080014848 7 | c <- x[sample(NROW(x), 100), ] 0.003910263000001635 0.003912210464477539 400248755 1000000 25 5080014848 8 | "d <- x[X1 == ""helpless_sheep"", ]" 0.06275645499999882 0.01575303077697754 400248755 1000000 25 5080014848 9 | e <- x[, .(mean(nchar(X2))), by = X1] 0.19549771300000174 0.14883995056152344 400248755 1000000 25 5080014848 10 | -------------------------------------------------------------------------------- /inst/bench/all_character-long/input.R: -------------------------------------------------------------------------------- 1 | args <- commandArgs(trailingOnly = TRUE) 2 | rows <- as.integer(args[[1]]) 3 | cols <- as.integer(args[[2]]) 4 | output <- args[[3]] 5 | 6 | set.seed(42) 7 | RNGversion("3.5.3") 8 | 9 | library(vroom) 10 | 11 | # We want ~ 1000 rows to filter 12 | num_levels <- 5 13 | levels <- c("helpless_sheep", gen_name(num_levels - 1)) 14 | 15 | filt_p <- 1000 / rows 16 | 17 | # The prob for the rest should just be evenly spaced 18 | rest_p <- rep((1 - filt_p) / (num_levels - 1), num_levels - 1) 19 | 20 | col_types <- stats::setNames( 21 | c(list( 22 | col_factor(levels = levels, prob = c(filt_p, rest_p))), 23 | rep(list(col_character()), cols - 1) 24 | ), make.names(seq_len(cols))) 25 | 26 | data <- gen_tbl(rows, cols, col_types = col_types) 27 | 28 | vroom_write(data, output, "\t") 29 | -------------------------------------------------------------------------------- /inst/bench/all_character-long/read.delim-base.R: -------------------------------------------------------------------------------- 1 | ({}) 2 | x <- read.delim(file, quote = "", na.strings = NULL, stringsAsFactors = FALSE) 3 | print(head(x, 10)) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- x[sample(NROW(x), 100), ] 7 | d <- x[x$X1 == "helpless_sheep", ] 8 | e <- tapply(x$X2, x$X1, function(x) mean(nchar(x))) 9 | -------------------------------------------------------------------------------- /inst/bench/all_character-long/read.delim-base.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ }) 4.0029999999191546e-6 6.198883056640625e-6 400248755 1000000 25 4863918080 3 | "x <- read.delim(file, quote = """", na.strings = NULL, stringsAsFactors = FALSE)" 103.065713264 103.06781911849976 400248755 1000000 25 4863918080 4 | print(head(x, 10)) 0.007192154999998479 0.007193565368652344 400248755 1000000 25 4863918080 5 | a <- head(x) 2.5602699999183187e-4 2.5725364685058594e-4 400248755 1000000 25 4863918080 6 | b <- tail(x) 4.7244900000009693e-4 4.737377166748047e-4 400248755 1000000 25 4863918080 7 | c <- x[sample(NROW(x), 100), ] 0.0015474770000025728 0.0015492439270019531 400248755 1000000 25 4863918080 8 | "d <- x[x$X1 == ""helpless_sheep"", ]" 0.027589560000009783 0.027591228485107422 400248755 1000000 25 4863918080 9 | e <- tapply(x$X2, x$X1, function(x) mean(nchar(x))) 0.29284924599998874 0.29285216331481934 400248755 1000000 25 4863918080 10 | -------------------------------------------------------------------------------- /inst/bench/all_character-long/readr-dplyr.R: -------------------------------------------------------------------------------- 1 | ({ library(readr); library(dplyr) }) 2 | x <- read_tsv(file, trim_ws = FALSE, quote = "", na = character()) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- sample_n(x, 100) 7 | d <- filter(x, X1 == "helpless_sheep") 8 | e <- group_by(x, X1) %>% summarise(avg_nchar = mean(nchar(X2))) 9 | -------------------------------------------------------------------------------- /inst/bench/all_character-long/readr-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(readr) library(dplyr) }) 0.27801725900000007 0.27802062034606934 400248755 1000000 25 4671979520 3 | "x <- read_tsv(file, trim_ws = FALSE, quote = """", na = character())" 62.577469189 62.581751346588135 400248755 1000000 25 4671979520 4 | print(x) 0.07374980299999834 0.10151100158691406 400248755 1000000 25 4671979520 5 | a <- head(x) 0.0011762729999986732 0.0011777877807617188 400248755 1000000 25 4671979520 6 | b <- tail(x) 5.460859999999457e-4 5.471706390380859e-4 400248755 1000000 25 4671979520 7 | c <- sample_n(x, 100) 0.016387897000001317 0.01638960838317871 400248755 1000000 25 4671979520 8 | "d <- filter(x, X1 == ""helpless_sheep"")" 0.019892996999999468 0.019894838333129883 400248755 1000000 25 4671979520 9 | e <- group_by(x, X1) %>% summarise(avg_nchar = mean(nchar(X2))) 0.21462222699999245 0.21462440490722656 400248755 1000000 25 4671979520 10 | -------------------------------------------------------------------------------- /inst/bench/all_character-long/vroom-base.R: -------------------------------------------------------------------------------- 1 | library(vroom) 2 | x <- vroom(file, trim_ws = FALSE, quote = "", escape_double = FALSE, na = character()) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- x[sample(NROW(x), 100), ] 7 | d <- x[x$X1 == "helpless_sheep", ] 8 | e <- tapply(x$X2, x$X1, function(x) mean(nchar(x))) 9 | -------------------------------------------------------------------------------- /inst/bench/all_character-long/vroom-base.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | library(vroom) 0.23405636800000007 0.23406624794006348 400248755 1000000 25 3461603328 3 | "x <- vroom(file, trim_ws = FALSE, quote = """", escape_double = FALSE, na = cha..." 3.9978824160000004 0.5949602127075195 400248755 1000000 25 3461603328 4 | print(x) 0.045972403000000384 0.0459742546081543 400248755 1000000 25 3461603328 5 | a <- head(x) 9.583050000001592e-4 9.601116180419922e-4 400248755 1000000 25 3461603328 6 | b <- tail(x) 5.036210000000096e-4 5.049705505371094e-4 400248755 1000000 25 3461603328 7 | c <- x[sample(NROW(x), 100), ] 0.002841930999999853 0.0028433799743652344 400248755 1000000 25 3461603328 8 | "d <- x[x$X1 == ""helpless_sheep"", ]" 0.16209093400000008 0.1620922088623047 400248755 1000000 25 3461603328 9 | e <- tapply(x$X2, x$X1, function(x) mean(nchar(x))) 2.1265156109999994 2.1265347003936768 400248755 1000000 25 3461603328 10 | -------------------------------------------------------------------------------- /inst/bench/all_character-long/vroom-dplyr.R: -------------------------------------------------------------------------------- 1 | ({ library(vroom); library(dplyr) }) 2 | x <- vroom(file, trim_ws = FALSE, quote = "", escape_double = FALSE, na = character()) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- sample_n(x, 100) 7 | d <- filter(x, X1 == "helpless_sheep") 8 | e <- group_by(x, X1) %>% summarise(avg_nchar = mean(nchar(X2))) 9 | -------------------------------------------------------------------------------- /inst/bench/all_character-long/vroom-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(vroom) library(dplyr) }) 0.3563563940000001 0.35636043548583984 400248755 1000000 25 3446657024 3 | "x <- vroom(file, trim_ws = FALSE, quote = """", escape_double = FALSE, na = cha..." 3.6933613389999995 0.6395699977874756 400248755 1000000 25 3446657024 4 | print(x) 0.057654061000000034 0.057656049728393555 400248755 1000000 25 3446657024 5 | a <- head(x) 0.0010614000000002122 0.0010628700256347656 400248755 1000000 25 3446657024 6 | b <- tail(x) 5.792140000000501e-4 5.805492401123047e-4 400248755 1000000 25 3446657024 7 | c <- sample_n(x, 100) 0.015052698999999947 0.015054941177368164 400248755 1000000 25 3446657024 8 | "d <- filter(x, X1 == ""helpless_sheep"")" 0.1843323209999994 0.18434882164001465 400248755 1000000 25 3446657024 9 | e <- group_by(x, X1) %>% summarise(avg_nchar = mean(nchar(X2))) 1.2399041610000001 1.2399187088012695 400248755 1000000 25 3446657024 10 | -------------------------------------------------------------------------------- /inst/bench/all_character-long/vroom_no_altrep-dplyr.R: -------------------------------------------------------------------------------- 1 | ({library(vroom); library(dplyr)}) 2 | x <- vroom(file, trim_ws = FALSE, quote = "", escape_double = FALSE, na = character(), altrep = FALSE) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- sample_n(x, 100) 7 | d <- filter(x, X1 == "helpless_sheep") 8 | e <- group_by(x, X1) %>% summarise(avg_nchar = mean(nchar(X2))) 9 | -------------------------------------------------------------------------------- /inst/bench/all_character-long/vroom_no_altrep-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(vroom) library(dplyr) }) 0.33683956000000004 0.33684349060058594 400248755 1000000 25 4617994240 3 | "x <- vroom(file, trim_ws = FALSE, quote = """", escape_double = FALSE, na = cha..." 53.144246626 50.454383850097656 400248755 1000000 25 4617994240 4 | print(x) 0.04912083899999686 0.04912281036376953 400248755 1000000 25 4617994240 5 | a <- head(x) 0.0010306440000036332 0.001032114028930664 400248755 1000000 25 4617994240 6 | b <- tail(x) 5.248319999964224e-4 5.261898040771484e-4 400248755 1000000 25 4617994240 7 | c <- sample_n(x, 100) 0.015900661999999954 0.01590275764465332 400248755 1000000 25 4617994240 8 | "d <- filter(x, X1 == ""helpless_sheep"")" 0.020508452000001398 0.02051067352294922 400248755 1000000 25 4617994240 9 | e <- group_by(x, X1) %>% summarise(avg_nchar = mean(nchar(X2))) 0.14906932099999892 0.14907217025756836 400248755 1000000 25 4617994240 10 | -------------------------------------------------------------------------------- /inst/bench/all_character-wide/data.table-data.table.R: -------------------------------------------------------------------------------- 1 | ../all_character-long/data.table-data.table.R -------------------------------------------------------------------------------- /inst/bench/all_character-wide/data.table-data.table.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | library(data.table) 0.05824787900000006 0.05825209617614746 1600080624 100000 1000 13596225536 3 | "x <- fread(file, sep = ""\t"", quote = """", strip.white = FALSE, na.strings = NULL)" 205.61311071900002 201.76576399803162 1600080624 100000 1000 13596225536 4 | print(x) 0.13403993000000014 0.13405299186706543 1600080624 100000 1000 13596225536 5 | a <- head(x) 0.001164771000020437 0.0011682510375976562 1600080624 100000 1000 13596225536 6 | b <- tail(x) 0.0012540889999854699 0.0012569427490234375 1600080624 100000 1000 13596225536 7 | c <- x[sample(NROW(x), 100), ] 0.03256328700001632 0.03256726264953613 1600080624 100000 1000 13596225536 8 | "d <- x[X1 == ""helpless_sheep"", ]" 0.20218088300001114 0.16787242889404297 1600080624 100000 1000 13596225536 9 | e <- x[, .(mean(nchar(X2))), by = X1] 0.048374496000008094 0.014370918273925781 1600080624 100000 1000 13596225536 10 | -------------------------------------------------------------------------------- /inst/bench/all_character-wide/input.R: -------------------------------------------------------------------------------- 1 | ../all_character-long/input.R -------------------------------------------------------------------------------- /inst/bench/all_character-wide/read.delim-base.R: -------------------------------------------------------------------------------- 1 | ../all_character-long/read.delim-base.R -------------------------------------------------------------------------------- /inst/bench/all_character-wide/read.delim-base.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ }) 3.6660000000132698e-6 5.7220458984375e-6 1600080624 100000 1000 14050525184 3 | "x <- read.delim(file, quote = """", na.strings = NULL, stringsAsFactors = FALSE)" 510.41117117600004 510.42269372940063 1600080624 100000 1000 14050525184 4 | print(head(x, 10)) 0.14892288600003667 0.1489250659942627 1600080624 100000 1000 14050525184 5 | a <- head(x) 0.006831051000062871 0.006834268569946289 1600080624 100000 1000 14050525184 6 | b <- tail(x) 0.007016168999939509 0.007021188735961914 1600080624 100000 1000 14050525184 7 | c <- x[sample(NROW(x), 100), ] 0.025838453000005757 0.025842905044555664 1600080624 100000 1000 14050525184 8 | "d <- x[x$X1 == ""helpless_sheep"", ]" 0.22350397200000316 0.2235558032989502 1600080624 100000 1000 14050525184 9 | e <- tapply(x$X2, x$X1, function(x) mean(nchar(x))) 0.05853724200005672 0.05854058265686035 1600080624 100000 1000 14050525184 10 | -------------------------------------------------------------------------------- /inst/bench/all_character-wide/readr-dplyr.R: -------------------------------------------------------------------------------- 1 | ../all_character-long/readr-dplyr.R -------------------------------------------------------------------------------- /inst/bench/all_character-wide/readr-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(readr) library(dplyr) }) 0.33448946199999985 0.33449411392211914 1600080624 100000 1000 13111070720 3 | "x <- read_tsv(file, trim_ws = FALSE, quote = """", na = character())" 459.43154994500003 459.44729495048523 1600080624 100000 1000 13111070720 4 | print(x) 0.13655689199998733 0.21697449684143066 1600080624 100000 1000 13111070720 5 | a <- head(x) 0.0032427959999949962 0.0032444000244140625 1600080624 100000 1000 13111070720 6 | b <- tail(x) 0.0025115230000096744 0.002513408660888672 1600080624 100000 1000 13111070720 7 | c <- sample_n(x, 100) 0.02890705900000512 0.02890944480895996 1600080624 100000 1000 13111070720 8 | "d <- filter(x, X1 == ""helpless_sheep"")" 0.037404203999983565 0.03740739822387695 1600080624 100000 1000 13111070720 9 | e <- group_by(x, X1) %>% summarise(avg_nchar = mean(nchar(X2))) 0.056806200999972134 0.056809425354003906 1600080624 100000 1000 13111070720 10 | -------------------------------------------------------------------------------- /inst/bench/all_character-wide/vroom-base.R: -------------------------------------------------------------------------------- 1 | ../all_character-long/vroom-base.R -------------------------------------------------------------------------------- /inst/bench/all_character-wide/vroom-base.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | library(vroom) 0.27099322299999984 0.27102088928222656 1600080624 100000 1000 7050588160 3 | "x <- vroom(file, trim_ws = FALSE, quote = """", escape_double = FALSE, na = cha..." 16.096324971999998 3.1437342166900635 1600080624 100000 1000 7050588160 4 | print(x) 0.06175371999999868 0.06175589561462402 1600080624 100000 1000 7050588160 5 | a <- head(x) 0.004202636999998788 0.004204511642456055 1600080624 100000 1000 7050588160 6 | b <- tail(x) 0.003743497999998624 0.003745555877685547 1600080624 100000 1000 7050588160 7 | c <- x[sample(NROW(x), 100), ] 0.004824033999998534 0.0048258304595947266 1600080624 100000 1000 7050588160 8 | "d <- x[x$X1 == ""helpless_sheep"", ]" 0.05477418799999967 0.054776668548583984 1600080624 100000 1000 7050588160 9 | e <- tapply(x$X2, x$X1, function(x) mean(nchar(x))) 0.2514594879999983 0.2514626979827881 1600080624 100000 1000 7050588160 10 | -------------------------------------------------------------------------------- /inst/bench/all_character-wide/vroom-dplyr.R: -------------------------------------------------------------------------------- 1 | ../all_character-long/vroom-dplyr.R -------------------------------------------------------------------------------- /inst/bench/all_character-wide/vroom-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(vroom) library(dplyr) }) 0.36314286799999995 0.3631572723388672 1600080624 100000 1000 7050731520 3 | "x <- vroom(file, trim_ws = FALSE, quote = """", escape_double = FALSE, na = cha..." 15.844784293 3.108229875564575 1600080624 100000 1000 7050731520 4 | print(x) 0.06392608800000232 0.06392836570739746 1600080624 100000 1000 7050731520 5 | a <- head(x) 0.004407775999997199 0.004410743713378906 1600080624 100000 1000 7050731520 6 | b <- tail(x) 0.0039727230000004 0.0039751529693603516 1600080624 100000 1000 7050731520 7 | c <- sample_n(x, 100) 0.026044930000001187 0.026047945022583008 1600080624 100000 1000 7050731520 8 | "d <- filter(x, X1 == ""helpless_sheep"")" 0.08114447399999847 0.0811469554901123 1600080624 100000 1000 7050731520 9 | e <- group_by(x, X1) %>% summarise(avg_nchar = mean(nchar(X2))) 0.1597604110000006 0.15976333618164062 1600080624 100000 1000 7050731520 10 | -------------------------------------------------------------------------------- /inst/bench/all_character-wide/vroom_no_altrep-dplyr.R: -------------------------------------------------------------------------------- 1 | ../all_character-long/vroom_no_altrep-dplyr.R -------------------------------------------------------------------------------- /inst/bench/all_character-wide/vroom_no_altrep-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(vroom) library(dplyr) }) 0.349206017 0.3492105007171631 1600080624 100000 1000 13033082880 3 | "x <- vroom(file, trim_ws = FALSE, quote = """", escape_double = FALSE, na = cha..." 258.09181479800003 247.31999611854553 1600080624 100000 1000 13033082880 4 | print(x) 0.06623288399998728 0.06623482704162598 1600080624 100000 1000 13033082880 5 | a <- head(x) 0.0027959219999615925 0.002797365188598633 1600080624 100000 1000 13033082880 6 | b <- tail(x) 0.0024252139999703104 0.0024268627166748047 1600080624 100000 1000 13033082880 7 | c <- sample_n(x, 100) 0.027183294999986174 0.02718496322631836 1600080624 100000 1000 13033082880 8 | "d <- filter(x, X1 == ""helpless_sheep"")" 0.034791021999978966 0.03479290008544922 1600080624 100000 1000 13033082880 9 | e <- group_by(x, X1) %>% summarise(avg_nchar = mean(nchar(X2))) 0.036543052000013176 0.0365450382232666 1600080624 100000 1000 13033082880 10 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-long/data.table-data.table.R: -------------------------------------------------------------------------------- 1 | library(data.table) 2 | x <- fread(file, sep = "\t", quote = "", strip.white = FALSE, na.strings = NULL) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- x[sample(NROW(x), 100), ] 7 | d <- x[X1 > 3, ] 8 | e <- x[ , .(mean(X1)), by = as.integer(X2)] 9 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-long/data.table-data.table.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | library(data.table) 0.05067038700000004 0.050673484802246094 490798139 1000000 25 2920148992 3 | "x <- fread(file, sep = ""\t"", quote = """", strip.white = FALSE, na.strings = NULL)" 1.339093691 0.25513172149658203 490798139 1000000 25 2920148992 4 | print(x) 0.012984569999999973 0.01298666000366211 490798139 1000000 25 2920148992 5 | a <- head(x) 2.257949999999731e-4 0.00022745132446289062 490798139 1000000 25 2920148992 6 | b <- tail(x) 1.8293999999974275e-4 1.842975616455078e-4 490798139 1000000 25 2920148992 7 | c <- x[sample(NROW(x), 100), ] 0.0032215970000000205 0.0032236576080322266 490798139 1000000 25 2920148992 8 | d <- x[X1 > 3, ] 0.00553502799999972 0.005329132080078125 490798139 1000000 25 2920148992 9 | e <- x[, .(mean(X1)), by = as.integer(X2)] 0.15706757400000004 0.024374723434448242 490798139 1000000 25 2920148992 10 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-long/input.R: -------------------------------------------------------------------------------- 1 | args <- commandArgs(trailingOnly = TRUE) 2 | rows <- as.integer(args[[1]]) 3 | cols <- as.integer(args[[2]]) 4 | output <- args[[3]] 5 | 6 | set.seed(42) 7 | RNGversion("3.5.3") 8 | 9 | data <- vroom::gen_tbl(rows, cols, col_types = strrep("d", cols)) 10 | 11 | vroom::vroom_write(data, output, "\t") 12 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-long/read.delim-base.R: -------------------------------------------------------------------------------- 1 | ({}) 2 | x <- read.delim(file, quote = "", na.strings = NULL, stringsAsFactors = FALSE) 3 | print(head(x, 10)) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- x[sample(NROW(x), 100), ] 7 | d <- x[x$X1 > 3, ] 8 | e <- tapply(x$X1, as.integer(x$X2), mean) 9 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-long/read.delim-base.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ }) 3.1689999999695573e-6 5.0067901611328125e-6 490798139 1000000 25 5142343680 3 | "x <- read.delim(file, quote = """", na.strings = NULL, stringsAsFactors = FALSE)" 111.410966158 111.41260313987732 490798139 1000000 25 5142343680 4 | print(head(x, 10)) 1.4099199399999947 1.4099228382110596 490798139 1000000 25 5142343680 5 | a <- head(x) 2.753560000030575e-4 2.765655517578125e-4 490798139 1000000 25 5142343680 6 | b <- tail(x) 4.943510000003926e-4 4.954338073730469e-4 490798139 1000000 25 5142343680 7 | c <- x[sample(NROW(x), 100), ] 0.0011230939999933298 0.0011248588562011719 490798139 1000000 25 5142343680 8 | d <- x[x$X1 > 3, ] 4.450478324000002 4.450565814971924 490798139 1000000 25 5142343680 9 | e <- tapply(x$X1, as.integer(x$X2), mean) 0.03682857900000158 0.03683161735534668 490798139 1000000 25 5142343680 10 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-long/readr-dplyr.R: -------------------------------------------------------------------------------- 1 | ({ library(readr); library(dplyr) }) 2 | x <- read_tsv(file, trim_ws = FALSE, quote = "", na = character()) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- sample_n(x, 100) 7 | d <- filter(x, X1 > 3) 8 | e <- group_by(x, as.integer(X2)) %>% summarise(avg_X1 = mean(X1)) 9 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-long/readr-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(readr) library(dplyr) }) 0.2670424519999999 0.267045259475708 490798139 1000000 25 3029172224 3 | "x <- read_tsv(file, trim_ws = FALSE, quote = """", na = character())" 13.121372993 13.124317646026611 490798139 1000000 25 3029172224 4 | print(x) 0.0572878620000008 0.06383562088012695 490798139 1000000 25 3029172224 5 | a <- head(x) 0.001062892000000204 0.001064300537109375 490798139 1000000 25 3029172224 6 | b <- tail(x) 4.806849999994256e-4 4.818439483642578e-4 490798139 1000000 25 3029172224 7 | c <- sample_n(x, 100) 0.015185605999999297 0.015187501907348633 490798139 1000000 25 3029172224 8 | d <- filter(x, X1 > 3) 0.017675308999999473 0.01767706871032715 490798139 1000000 25 3029172224 9 | e <- group_by(x, as.integer(X2)) %>% summarise(avg_X1 = mean(X1)) 0.05443157900000095 0.05443406105041504 490798139 1000000 25 3029172224 10 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-long/vroom-base.R: -------------------------------------------------------------------------------- 1 | library(vroom) 2 | x <- vroom(file, trim_ws = FALSE, quote = "", escape_double = FALSE, na = character()) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- x[sample(NROW(x), 100), ] 7 | d <- x[x$X1 > 3, ] 8 | e <- tapply(x$X1, as.integer(x$X2), mean) 9 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-long/vroom-base.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | library(vroom) 0.22593527800000013 0.2259387969970703 490798139 1000000 25 3523780608 3 | "x <- vroom(file, trim_ws = FALSE, quote = """", escape_double = FALSE, na = cha..." 3.8099602220000004 0.5808789730072021 490798139 1000000 25 3523780608 4 | print(x) 0.05762982400000016 0.05461859703063965 490798139 1000000 25 3523780608 5 | a <- head(x) 9.245710000005403e-4 0.00092601776123046875 490798139 1000000 25 3523780608 6 | b <- tail(x) 4.916189999999432e-4 4.928112030029297e-4 490798139 1000000 25 3523780608 7 | c <- x[sample(NROW(x), 100), ] 0.0027541729999995823 0.0027556419372558594 490798139 1000000 25 3523780608 8 | d <- x[x$X1 > 3, ] 0.2551561959999997 0.028618335723876953 490798139 1000000 25 3523780608 9 | e <- tapply(x$X1, as.integer(x$X2), mean) 0.2509484510000002 0.25095057487487793 490798139 1000000 25 3523780608 10 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-long/vroom-dplyr.R: -------------------------------------------------------------------------------- 1 | ({ library(vroom); library(dplyr) }) 2 | x <- vroom(file, trim_ws = FALSE, quote = "", escape_double = FALSE, na = character()) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- sample_n(x, 100) 7 | d <- filter(x, X1 > 3) 8 | e <- group_by(x, as.integer(X2)) %>% summarise(avg_X1 = mean(X1)) 9 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-long/vroom-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(vroom) library(dplyr) }) 0.29869805699999996 0.29870104789733887 490798139 1000000 25 3531214848 3 | "x <- vroom(file, trim_ws = FALSE, quote = """", escape_double = FALSE, na = cha..." 3.8170653839999997 0.603179931640625 490798139 1000000 25 3531214848 4 | print(x) 0.06604458400000013 0.06310796737670898 490798139 1000000 25 3531214848 5 | a <- head(x) 9.071970000000817e-4 0.00090885162353515625 490798139 1000000 25 3531214848 6 | b <- tail(x) 4.918530000006527e-4 0.00049304962158203125 490798139 1000000 25 3531214848 7 | c <- sample_n(x, 100) 0.013811199000000052 0.013812541961669922 490798139 1000000 25 3531214848 8 | d <- filter(x, X1 > 3) 0.2886524169999998 0.041882991790771484 490798139 1000000 25 3531214848 9 | e <- group_by(x, as.integer(X2)) %>% summarise(avg_X1 = mean(X1)) 0.23468923700000044 0.23469161987304688 490798139 1000000 25 3531214848 10 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-long/vroom_no_altrep-base.R: -------------------------------------------------------------------------------- 1 | library(vroom) 2 | x <- vroom(file, trim_ws = FALSE, quote = "", escape_double = FALSE, na = character(), altrep = FALSE) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- x[sample(NROW(x), 100), ] 7 | d <- x[x$X1 > 3, ] 8 | e <- tapply(x$X1, as.integer(x$X2), mean) 9 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-long/vroom_no_altrep-base.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | library(vroom) 0.22674703399999996 0.22675061225891113 490798139 1000000 25 2886311936 3 | "x <- vroom(file, trim_ws = FALSE, quote = """", escape_double = FALSE, na = cha..." 9.917269908 1.325624942779541 490798139 1000000 25 2886311936 4 | print(x) 0.04774370600000033 0.04775238037109375 490798139 1000000 25 2886311936 5 | a <- head(x) 8.502769999996218e-4 0.00085163116455078125 490798139 1000000 25 2886311936 6 | b <- tail(x) 4.878420000000716e-4 4.889965057373047e-4 490798139 1000000 25 2886311936 7 | c <- x[sample(NROW(x), 100), ] 0.0028887550000007423 0.0028901100158691406 490798139 1000000 25 2886311936 8 | d <- x[x$X1 > 3, ] 0.005902124000000342 0.005903720855712891 490798139 1000000 25 2886311936 9 | e <- tapply(x$X1, as.integer(x$X2), mean) 0.054749883000001276 0.05475163459777832 490798139 1000000 25 2886311936 10 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-long/vroom_no_altrep-dplyr.R: -------------------------------------------------------------------------------- 1 | ({ library(vroom); library(dplyr) }) 2 | x <- vroom(file, trim_ws = FALSE, quote = "", escape_double = FALSE, na = character(), altrep = FALSE) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- sample_n(x, 100) 7 | d <- filter(x, X1 > 3) 8 | e <- group_by(x, as.integer(X2)) %>% summarise(avg_X1 = mean(X1)) 9 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-long/vroom_no_altrep-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(vroom) library(dplyr) }) 0.2972948229999999 0.29729771614074707 490798139 1000000 25 2957496320 3 | "x <- vroom(file, trim_ws = FALSE, quote = """", escape_double = FALSE, na = cha..." 10.102860680000001 1.3375225067138672 490798139 1000000 25 2957496320 4 | print(x) 0.04706370399999926 0.04706525802612305 490798139 1000000 25 2957496320 5 | a <- head(x) 8.443339999999466e-4 8.456707000732422e-4 490798139 1000000 25 2957496320 6 | b <- tail(x) 4.372789999997906e-4 4.3845176696777344e-4 490798139 1000000 25 2957496320 7 | c <- sample_n(x, 100) 0.013971556000001328 0.013973236083984375 490798139 1000000 25 2957496320 8 | d <- filter(x, X1 > 3) 0.01792857100000056 0.017930269241333008 490798139 1000000 25 2957496320 9 | e <- group_by(x, as.integer(X2)) %>% summarise(avg_X1 = mean(X1)) 0.045251213000000234 0.04525351524353027 490798139 1000000 25 2957496320 10 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-wide/data.table-data.table.R: -------------------------------------------------------------------------------- 1 | ../all_numeric-long/data.table-data.table.R -------------------------------------------------------------------------------- /inst/bench/all_numeric-wide/data.table-data.table.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | library(data.table) 0.051875126999999965 0.051879167556762695 1963219523 100000 1000 5880053760 3 | "x <- fread(file, sep = ""\t"", quote = """", strip.white = FALSE, na.strings = NULL)" 5.577086213 1.2563517093658447 1963219523 100000 1000 5880053760 4 | print(x) 0.09918665400000037 0.09919476509094238 1963219523 100000 1000 5880053760 5 | a <- head(x) 4.1016100000046407e-4 4.124641418457031e-4 1963219523 100000 1000 5880053760 6 | b <- tail(x) 4.3974700000060096e-4 4.417896270751953e-4 1963219523 100000 1000 5880053760 7 | c <- x[sample(NROW(x), 100), ] 0.002832650000000214 0.002834796905517578 1963219523 100000 1000 5880053760 8 | d <- x[X1 > 3, ] 0.003388150000000145 0.003390073776245117 1963219523 100000 1000 5880053760 9 | e <- x[, .(mean(X1)), by = as.integer(X2)] 0.014421183999999698 0.003484010696411133 1963219523 100000 1000 5880053760 10 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-wide/input.R: -------------------------------------------------------------------------------- 1 | ../all_numeric-long/input.R -------------------------------------------------------------------------------- /inst/bench/all_numeric-wide/read.delim-base.R: -------------------------------------------------------------------------------- 1 | ../all_numeric-long/read.delim-base.R -------------------------------------------------------------------------------- /inst/bench/all_numeric-wide/read.delim-base.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ }) 3.0330000000411417e-6 4.76837158203125e-6 1963219523 100000 1000 15470862336 3 | "x <- read.delim(file, quote = """", na.strings = NULL, stringsAsFactors = FALSE)" 520.970175058 520.9826147556305 1963219523 100000 1000 15470862336 4 | print(head(x, 10)) 0.1300029069999482 0.13001203536987305 1963219523 100000 1000 15470862336 5 | a <- head(x) 0.006249463000017386 0.006251096725463867 1963219523 100000 1000 15470862336 6 | b <- tail(x) 0.006446566000022358 0.006448507308959961 1963219523 100000 1000 15470862336 7 | c <- x[sample(NROW(x), 100), ] 0.008708142000045882 0.008709907531738281 1963219523 100000 1000 15470862336 8 | d <- x[x$X1 > 3, ] 0.07440451399997983 0.07440614700317383 1963219523 100000 1000 15470862336 9 | e <- tapply(x$X1, as.integer(x$X2), mean) 0.00439676700000291 0.0043985843658447266 1963219523 100000 1000 15470862336 10 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-wide/readr-dplyr.R: -------------------------------------------------------------------------------- 1 | ../all_numeric-long/readr-dplyr.R -------------------------------------------------------------------------------- /inst/bench/all_numeric-wide/readr-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(readr) library(dplyr) }) 0.26801563699999975 0.2680184841156006 1963219523 100000 1000 5863723008 3 | "x <- read_tsv(file, trim_ws = FALSE, quote = """", na = character())" 56.104649359 56.11158347129822 1963219523 100000 1000 5863723008 4 | print(x) 0.0787199290000018 0.0954742431640625 1963219523 100000 1000 5863723008 5 | a <- head(x) 0.0027141720000045666 0.0027158260345458984 1963219523 100000 1000 5863723008 6 | b <- tail(x) 0.0022157200000023636 0.002217531204223633 1963219523 100000 1000 5863723008 7 | c <- sample_n(x, 100) 0.02533303300000256 0.025334835052490234 1963219523 100000 1000 5863723008 8 | d <- filter(x, X1 > 3) 0.017108968999998808 0.017110586166381836 1963219523 100000 1000 5863723008 9 | e <- group_by(x, as.integer(X2)) %>% summarise(avg_X1 = mean(X1)) 0.038782436000005305 0.03878426551818848 1963219523 100000 1000 5863723008 10 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-wide/vroom-base.R: -------------------------------------------------------------------------------- 1 | ../all_numeric-long/vroom-base.R -------------------------------------------------------------------------------- /inst/bench/all_numeric-wide/vroom-base.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | library(vroom) 0.22703830000000025 0.2270522117614746 1963219523 100000 1000 7791280128 3 | "x <- vroom(file, trim_ws = FALSE, quote = """", escape_double = FALSE, na = cha..." 16.174510832 2.998760223388672 1963219523 100000 1000 7791280128 4 | print(x) 0.07004265899999851 0.06718945503234863 1963219523 100000 1000 7791280128 5 | a <- head(x) 0.003972434999997887 0.003973960876464844 1963219523 100000 1000 7791280128 6 | b <- tail(x) 0.00348148499999823 0.0034830570220947266 1963219523 100000 1000 7791280128 7 | c <- x[sample(NROW(x), 100), ] 0.00464282200000099 0.0046443939208984375 1963219523 100000 1000 7791280128 8 | d <- x[x$X1 > 3, ] 0.049587988999999055 0.010494709014892578 1963219523 100000 1000 7791280128 9 | e <- tapply(x$X1, as.integer(x$X2), mean) 0.041909286999999296 0.04191112518310547 1963219523 100000 1000 7791280128 10 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-wide/vroom-dplyr.R: -------------------------------------------------------------------------------- 1 | ../all_numeric-long/vroom-dplyr.R -------------------------------------------------------------------------------- /inst/bench/all_numeric-wide/vroom-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(vroom) library(dplyr) }) 0.2996260059999998 0.29964709281921387 1963219523 100000 1000 7792242688 3 | "x <- vroom(file, trim_ws = FALSE, quote = """", escape_double = FALSE, na = cha..." 16.673107875 2.9666857719421387 1963219523 100000 1000 7792242688 4 | print(x) 0.07086207600000094 0.06784224510192871 1963219523 100000 1000 7792242688 5 | a <- head(x) 0.003975565000001069 0.0039768218994140625 1963219523 100000 1000 7792242688 6 | b <- tail(x) 0.013946671999999438 0.013948440551757812 1963219523 100000 1000 7792242688 7 | c <- sample_n(x, 100) 0.02246668400000118 0.022468090057373047 1963219523 100000 1000 7792242688 8 | d <- filter(x, X1 > 3) 0.05683080300000043 0.019407272338867188 1963219523 100000 1000 7792242688 9 | e <- group_by(x, as.integer(X2)) %>% summarise(avg_X1 = mean(X1)) 0.07652678499999865 0.07652854919433594 1963219523 100000 1000 7792242688 10 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-wide/vroom_no_altrep-base.R: -------------------------------------------------------------------------------- 1 | ../all_numeric-long/vroom_no_altrep-base.R -------------------------------------------------------------------------------- /inst/bench/all_numeric-wide/vroom_no_altrep-base.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | library(vroom) 0.22699760899999988 0.22700119018554688 1963219523 100000 1000 5739347968 3 | "x <- vroom(file, trim_ws = FALSE, quote = """", escape_double = FALSE, na = cha..." 40.352723725000004 6.934685707092285 1963219523 100000 1000 5739347968 4 | print(x) 0.060421132000001876 0.06042838096618652 1963219523 100000 1000 5739347968 5 | a <- head(x) 0.0024158189999994306 0.0024175643920898438 1963219523 100000 1000 5739347968 6 | b <- tail(x) 0.0020524330000029067 0.002053976058959961 1963219523 100000 1000 5739347968 7 | c <- x[sample(NROW(x), 100), ] 0.004886046000002864 0.00488734245300293 1963219523 100000 1000 5739347968 8 | d <- x[x$X1 > 3, ] 0.005605907000003185 0.00560760498046875 1963219523 100000 1000 5739347968 9 | e <- tapply(x$X1, as.integer(x$X2), mean) 0.0062097089999966215 0.006211280822753906 1963219523 100000 1000 5739347968 10 | -------------------------------------------------------------------------------- /inst/bench/all_numeric-wide/vroom_no_altrep-dplyr.R: -------------------------------------------------------------------------------- 1 | ../all_numeric-long/vroom_no_altrep-dplyr.R -------------------------------------------------------------------------------- /inst/bench/all_numeric-wide/vroom_no_altrep-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(vroom) library(dplyr) }) 0.2939437899999999 0.2939589023590088 1963219523 100000 1000 5740023808 3 | "x <- vroom(file, trim_ws = FALSE, quote = """", escape_double = FALSE, na = cha..." 41.10774825 6.923159122467041 1963219523 100000 1000 5740023808 4 | print(x) 0.06295269499999989 0.06295442581176758 1963219523 100000 1000 5740023808 5 | a <- head(x) 0.0024358220000024744 0.0024373531341552734 1963219523 100000 1000 5740023808 6 | b <- tail(x) 0.002060569999997597 0.002062082290649414 1963219523 100000 1000 5740023808 7 | c <- sample_n(x, 100) 0.09410682600000086 0.09410929679870605 1963219523 100000 1000 5740023808 8 | d <- filter(x, X1 > 3) 0.013741070000001798 0.013743400573730469 1963219523 100000 1000 5740023808 9 | e <- group_by(x, as.integer(X2)) %>% summarise(avg_X1 = mean(X1)) 0.030236498000000722 0.030238628387451172 1963219523 100000 1000 5740023808 10 | -------------------------------------------------------------------------------- /inst/bench/download-data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # download and extract taxi data 4 | mkdir ~/data/ 5 | cd ~/data 6 | 7 | wget -O ~/data/trip_fare.7z https://archive.org/download/nycTaxiTripData2013/trip_fare.7z && \ 8 | sudo apt install p7zip-full && \ 9 | 7z x trip_fare.7z &> data.out 10 | 11 | # fix trailing space in header for every file 12 | ls *trip_fare*.csv | xargs -P 16 sed -i '1 s/, /,/g' 13 | 14 | # download the US census data 15 | wget -O ~/data/all_California.zip https://www2.census.gov/census_2000/datasets/PUMS/FivePercent/California/all_California.zip && \ 16 | sudo apt install unzip && \ 17 | unzip all_California.zip 18 | -------------------------------------------------------------------------------- /inst/bench/fwf/read.delim-base.R: -------------------------------------------------------------------------------- 1 | ({}) 2 | x <- read.fwf(file, widths = fields$width, col.names = fields$col_names) 3 | print(head(x, 10)) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- x[sample(NROW(x), 100), ] 7 | d <- x[x$PERSONS== "06", ] 8 | e <- tapply(x$TOTPUMA5, x$STATE, mean) 9 | -------------------------------------------------------------------------------- /inst/bench/fwf/read.delim-base.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ }) 3.063999999941558e-6 5.245208740234375e-6 709936880 2342339 37 6621343744 3 | x <- read.fwf(file, widths = fields$width, col.names = fields$col_names) 1089.464910198 1089.5732316970825 709936880 2342339 37 6621343744 4 | print(head(x, 10)) 0.015051766999931715 0.015721797943115234 709936880 2342339 37 6621343744 5 | a <- head(x) 8.085290000963141e-4 8.099079132080078e-4 709936880 2342339 37 6621343744 6 | b <- tail(x) 0.0010067789999084198 0.0010082721710205078 709936880 2342339 37 6621343744 7 | c <- x[sample(NROW(x), 100), ] 0.0029781670000375016 0.0029802322387695312 709936880 2342339 37 6621343744 8 | "d <- x[x$PERSONS == ""06"", ]" 0.49151955999991515 0.49154090881347656 709936880 2342339 37 6621343744 9 | e <- tapply(x$TOTPUMA5, x$STATE, mean) 0.08967751000000135 0.08968019485473633 709936880 2342339 37 6621343744 10 | -------------------------------------------------------------------------------- /inst/bench/fwf/readr-dplyr.R: -------------------------------------------------------------------------------- 1 | ({ library(readr); library(dplyr) }) 2 | x <- read_fwf(file, fields, col_types = types) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- sample_n(x, 100) 7 | d <- filter(x, PERSONS == "06") 8 | e <- group_by(x, STATE) %>% summarise(avg_TOTPUM5 = mean(TOTPUMA5)) 9 | -------------------------------------------------------------------------------- /inst/bench/fwf/readr-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(readr) library(dplyr) }) 0.12630891799999988 0.1263124942779541 709936880 2342339 37 6644965376 3 | x <- read_fwf(file, fields, col_types = types) 32.593650083 32.59428787231445 709936880 2342339 37 6644965376 4 | print(x) 0.04750896499999868 0.04751086235046387 709936880 2342339 37 6644965376 5 | a <- head(x) 0.0011317969999993238 0.0011332035064697266 709936880 2342339 37 6644965376 6 | b <- tail(x) 6.720350000009034e-4 6.732940673828125e-4 709936880 2342339 37 6644965376 7 | c <- sample_n(x, 100) 0.016384803000001114 0.01638650894165039 709936880 2342339 37 6644965376 8 | "d <- filter(x, PERSONS == ""06"")" 0.09458435599999859 0.09458684921264648 709936880 2342339 37 6644965376 9 | e <- group_by(x, STATE) %>% summarise(avg_TOTPUM5 = mean(TOTPUMA5)) 0.09379347599999477 0.0937960147857666 709936880 2342339 37 6644965376 10 | -------------------------------------------------------------------------------- /inst/bench/fwf/vroom-base.R: -------------------------------------------------------------------------------- 1 | library(vroom) 2 | x <- vroom_fwf(file, fields, col_types = types) 3 | print(head(x, 10)) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- x[sample(NROW(x), 100), ] 7 | d <- x[x$PERSONS== "06", ] 8 | e <- tapply(x$TOTPUMA5, x$STATE, mean) 9 | -------------------------------------------------------------------------------- /inst/bench/fwf/vroom-base.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | library(vroom) 7.52309999998424e-5 7.796287536621094e-5 709936880 2342339 37 4989079552 3 | x <- vroom_fwf(file, fields, col_types = types) 0.16403843000000018 0.16362786293029785 709936880 2342339 37 4989079552 4 | print(head(x, 10)) 0.056157112999999814 0.05505537986755371 709936880 2342339 37 4989079552 5 | a <- head(x) 5.586259999996734e-4 5.600452423095703e-4 709936880 2342339 37 4989079552 6 | b <- tail(x) 8.067250000003412e-4 8.080005645751953e-4 709936880 2342339 37 4989079552 7 | c <- x[sample(NROW(x), 100), ] 0.00669999900000029 0.006701469421386719 709936880 2342339 37 4989079552 8 | "d <- x[x$PERSONS == ""06"", ]" 0.284221665 0.2842233180999756 709936880 2342339 37 4989079552 9 | e <- tapply(x$TOTPUMA5, x$STATE, mean) 2.1088666509999996 1.7768657207489014 709936880 2342339 37 4989079552 10 | -------------------------------------------------------------------------------- /inst/bench/fwf/vroom-dplyr.R: -------------------------------------------------------------------------------- 1 | ({ library(vroom); library(dplyr) }) 2 | x <- vroom_fwf(file, fields, col_types = types) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- sample_n(x, 100) 7 | d <- filter(x, PERSONS == "06") 8 | e <- group_by(x, STATE) %>% summarise(avg_TOTPUM5 = mean(TOTPUMA5)) 9 | -------------------------------------------------------------------------------- /inst/bench/fwf/vroom-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(vroom) library(dplyr) }) 0.08314487400000026 0.08314847946166992 709936880 2342339 37 4959260672 3 | x <- vroom_fwf(file, fields, col_types = types) 0.1629412410000004 0.16250014305114746 709936880 2342339 37 4959260672 4 | print(x) 0.04872845899999989 0.047605037689208984 709936880 2342339 37 4959260672 5 | a <- head(x) 0.001250199999999868 0.0012516975402832031 709936880 2342339 37 4959260672 6 | b <- tail(x) 7.490030000001369e-4 0.00075054168701171875 709936880 2342339 37 4959260672 7 | c <- sample_n(x, 100) 0.01527529800000016 0.01527714729309082 709936880 2342339 37 4959260672 8 | "d <- filter(x, PERSONS == ""06"")" 0.305116822 0.30511975288391113 709936880 2342339 37 4959260672 9 | e <- group_by(x, STATE) %>% summarise(avg_TOTPUM5 = mean(TOTPUMA5)) 1.2913172750000004 1.2913398742675781 709936880 2342339 37 4959260672 10 | -------------------------------------------------------------------------------- /inst/bench/fwf/vroom_no_altrep-dplyr.R: -------------------------------------------------------------------------------- 1 | ({library(vroom); library(dplyr)}) 2 | x <- vroom_fwf(file, fields, col_types = types, altrep_opts = FALSE) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- sample_n(x, 100) 7 | d <- filter(x, PERSONS == "06") 8 | e <- group_by(x, STATE) %>% summarise(avg_TOTPUM5 = mean(TOTPUMA5)) 9 | -------------------------------------------------------------------------------- /inst/bench/fwf/vroom_no_altrep-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(vroom) library(dplyr) }) 0.08015654499999991 0.08015990257263184 709936880 2342339 37 6397005824 3 | x <- vroom_fwf(file, fields, col_types = types, altrep_opts = FALSE) 34.306891702 14.682522296905518 709936880 2342339 37 6397005824 4 | print(x) 0.04313145500000104 0.04313325881958008 709936880 2342339 37 6397005824 5 | a <- head(x) 8.676749999949607e-4 8.692741394042969e-4 709936880 2342339 37 6397005824 6 | b <- tail(x) 5.269520000013017e-4 5.283355712890625e-4 709936880 2342339 37 6397005824 7 | c <- sample_n(x, 100) 0.01451422600000285 0.014516115188598633 709936880 2342339 37 6397005824 8 | "d <- filter(x, PERSONS == ""06"")" 0.4679828829999977 0.46799564361572266 709936880 2342339 37 6397005824 9 | e <- group_by(x, STATE) %>% summarise(avg_TOTPUM5 = mean(TOTPUMA5)) 0.09054751499999725 0.09055042266845703 709936880 2342339 37 6397005824 10 | -------------------------------------------------------------------------------- /inst/bench/run-bench.R: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env Rscript 2 | 3 | args <- commandArgs(trailingOnly = TRUE) 4 | source_file <- args[[1]] 5 | out_file <- args[[2]] 6 | 7 | file <- args[-c(1:2)] 8 | 9 | cat(source_file, "\n") 10 | out <- bench::workout_expressions(as.list(parse(source_file, keep.source = FALSE))) 11 | 12 | x <- vroom::vroom(file, col_types = list()) 13 | 14 | out$size <- sum(file.size(file)) 15 | out$rows <- nrow(x) 16 | out$cols <- ncol(x) 17 | out$process <- as.numeric(out$process) 18 | out$real <- as.numeric(out$real) 19 | out$max_memory <- as.numeric(bench::bench_process_memory()[["max"]]) 20 | 21 | vroom::vroom_write(out, out_file) 22 | -------------------------------------------------------------------------------- /inst/bench/session_info.R: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env Rscript 2 | 3 | vroom::vroom_write( 4 | sessioninfo::package_info(c("vroom", "readr", "dplyr", "data.table", "base"), dependencies = FALSE, include_base = TRUE), 5 | here::here("inst", "bench", "session_info.tsv"), 6 | delim = "\t" 7 | ) 8 | -------------------------------------------------------------------------------- /inst/bench/session_info.tsv: -------------------------------------------------------------------------------- 1 | package ondiskversion loadedversion path loadedpath attached is_base date source md5ok library 2 | base 4.1.0 NA /opt/R/4.1.0/lib/R/library/base NA TRUE TRUE 2021-05-18 local NA /opt/R/4.1.0/lib/R/library 3 | data.table 1.14.0 NA /home/ubuntu/R/x86_64-pc-linux-gnu-library/4.1/data.table NA FALSE FALSE 2021-02-21 RSPM (R 4.1.0) NA /home/ubuntu/R/x86_64-pc-linux-gnu-library/4.1 4 | dplyr 1.0.7 NA /home/ubuntu/R/x86_64-pc-linux-gnu-library/4.1/dplyr NA FALSE FALSE 2021-06-18 RSPM (R 4.1.0) NA /home/ubuntu/R/x86_64-pc-linux-gnu-library/4.1 5 | readr 1.4.0 NA /home/ubuntu/R/x86_64-pc-linux-gnu-library/4.1/readr NA FALSE FALSE 2020-10-05 RSPM (R 4.1.0) NA /home/ubuntu/R/x86_64-pc-linux-gnu-library/4.1 6 | vroom 1.5.1 1.5.1 /home/ubuntu/R/x86_64-pc-linux-gnu-library/4.1/vroom /home/ubuntu/R/x86_64-pc-linux-gnu-library/4.1/vroom FALSE FALSE 2021-06-22 local NA /home/ubuntu/R/x86_64-pc-linux-gnu-library/4.1 7 | -------------------------------------------------------------------------------- /inst/bench/taxi/data.table-data.table.R: -------------------------------------------------------------------------------- 1 | library(data.table) 2 | x <- fread(file, sep = ",", quote = "", strip.white = FALSE, na.strings = NULL) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- x[sample(NROW(x), 100), ] 7 | d <- x[payment_type == "UNK", ] 8 | e <- x[ , .(mean(tip_amount)), by = payment_type] 9 | -------------------------------------------------------------------------------- /inst/bench/taxi/data.table-data.table.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | library(data.table) 0.0535136759999999 0.05351758003234863 1641999923 14388451 11 6853369856 3 | "x <- fread(file, sep = "","", quote = """", strip.white = FALSE, na.strings = NULL)" 28.634132631 5.829838037490845 1641999923 14388451 11 6853369856 4 | print(x) 0.011972201999999044 0.011974573135375977 1641999923 14388451 11 6853369856 5 | a <- head(x) 2.2678200000214588e-4 2.281665802001953e-4 1641999923 14388451 11 6853369856 6 | b <- tail(x) 1.8586699999900702e-4 0.00018739700317382812 1641999923 14388451 11 6853369856 7 | c <- x[sample(NROW(x), 100), ] 6.299819999995293e-4 6.320476531982422e-4 1641999923 14388451 11 6853369856 8 | "d <- x[payment_type == ""UNK"", ]" 0.34992646499999935 0.10343503952026367 1641999923 14388451 11 6853369856 9 | e <- x[, .(mean(tip_amount)), by = payment_type] 1.6930106420000008 0.76350998878479 1641999923 14388451 11 6853369856 10 | -------------------------------------------------------------------------------- /inst/bench/taxi/read.delim-base.R: -------------------------------------------------------------------------------- 1 | ({}) 2 | x <- read.delim(file, sep = ",", quote = "", na.strings = NULL, stringsAsFactors = FALSE) 3 | print(head(x, 10)) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- x[sample(NROW(x), 100), ] 7 | d <- x[x$payment_type == "UNK", ] 8 | e <- tapply(x$tip_amount, x$payment_type, mean) 9 | -------------------------------------------------------------------------------- /inst/bench/taxi/read.delim-base.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ }) 2.617000000038061e-6 4.291534423828125e-6 1641999923 14388451 11 6637604864 3 | "x <- read.delim(file, sep = "","", quote = """", na.strings = NULL, stringsAsFact..." 72.30426375399999 72.30614757537842 1641999923 14388451 11 6637604864 4 | print(head(x, 10)) 0.0059510929999930795 0.0059528350830078125 1641999923 14388451 11 6637604864 5 | a <- head(x) 1.9695099999239574e-4 1.983642578125e-4 1641999923 14388451 11 6637604864 6 | b <- tail(x) 4.1288200000622055e-4 4.1413307189941406e-4 1641999923 14388451 11 6637604864 7 | c <- x[sample(NROW(x), 100), ] 3.71048000005203e-4 3.726482391357422e-4 1641999923 14388451 11 6637604864 8 | "d <- x[x$payment_type == ""UNK"", ]" 1.2673573149999982 1.2681455612182617 1641999923 14388451 11 6637604864 9 | e <- tapply(x$tip_amount, x$payment_type, mean) 0.8941533619999973 0.8941662311553955 1641999923 14388451 11 6637604864 10 | -------------------------------------------------------------------------------- /inst/bench/taxi/readr-dplyr.R: -------------------------------------------------------------------------------- 1 | ({ library(readr); library(dplyr) }) 2 | x <- read_csv(file, col_types = c(pickup_datetime = "c"), quote = "", trim_ws = FALSE, na = character()) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- sample_n(x, 100) 7 | d <- filter(x, payment_type == "UNK") 8 | e <- group_by(x, payment_type) %>% summarise(avg_tip = mean(tip_amount)) 9 | -------------------------------------------------------------------------------- /inst/bench/taxi/readr-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(readr) library(dplyr) }) 0.33623017600000016 0.3362598419189453 1641999923 14388451 11 7415922688 3 | "x <- read_csv(file, col_types = c(pickup_datetime = ""c""), quote = """", trim_ws..." 37.30245180599999 37.303383588790894 1641999923 14388451 11 7415922688 4 | print(x) 0.12003345499999796 0.14695501327514648 1641999923 14388451 11 7415922688 5 | a <- head(x) 0.0013562000000035823 0.0013580322265625 1641999923 14388451 11 7415922688 6 | b <- tail(x) 6.067529999995713e-4 6.082057952880859e-4 1641999923 14388451 11 7415922688 7 | c <- sample_n(x, 100) 0.01610033400000077 0.01610279083251953 1641999923 14388451 11 7415922688 8 | "d <- filter(x, payment_type == ""UNK"")" 0.24887825199999725 0.24891424179077148 1641999923 14388451 11 7415922688 9 | e <- group_by(x, payment_type) %>% summarise(avg_tip = mean(tip_amount)) 0.5376497970000003 0.5376534461975098 1641999923 14388451 11 7415922688 10 | -------------------------------------------------------------------------------- /inst/bench/taxi/vroom-base.R: -------------------------------------------------------------------------------- 1 | library(vroom) 2 | x <- vroom(file, col_types = c(pickup_datetime = "c"), trim_ws = FALSE, quote = "", escape_double = FALSE, na = character()) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- x[sample(NROW(x), 100), ] 7 | d <- x[x$payment_type == "UNK", ] 8 | e <- tapply(x$tip_amount, x$payment_type, mean) 9 | -------------------------------------------------------------------------------- /inst/bench/taxi/vroom-base.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | library(vroom) 0.233707 0.23376321099931374 1666833417 14776615 11 6815055872 3 | "x <- vroom(file, col_types = c(pickup_datetime = ""c""), trim_ws = FALSE, quote..." 17.640452000000003 1.3566655699978583 1666833417 14776615 11 6815055872 4 | print(x) 0.1533760000000015 0.15729006798937917 1666833417 14776615 11 6815055872 5 | a <- head(x) 0.0020479999999984955 0.0020507239969447255 1666833417 14776615 11 6815055872 6 | b <- tail(x) 4.980000000003315e-4 5.036990041844547e-4 1666833417 14776615 11 6815055872 7 | c <- x[sample(NROW(x), 100), ] 2.119999999976585e-4 2.1384801948443055e-4 1666833417 14776615 11 6815055872 8 | "d <- x[x$payment_type == ""UNK"", ]" 1.1206159999999983 1.1206832609605044 1666833417 14776615 11 6815055872 9 | e <- tapply(x$tip_amount, x$payment_type, mean) 8.975451 7.352519184001721 1666833417 14776615 11 6815055872 10 | -------------------------------------------------------------------------------- /inst/bench/taxi/vroom-dplyr.R: -------------------------------------------------------------------------------- 1 | ({ library(vroom); library(dplyr) }) 2 | x <- vroom(file, col_types = c(pickup_datetime = "c"), trim_ws = FALSE, quote = "", escape_double = FALSE, na = character()) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- sample_n(x, 100) 7 | d <- filter(x, payment_type == "UNK") 8 | e <- group_by(x, payment_type) %>% summarise(avg_tip = mean(tip_amount)) 9 | -------------------------------------------------------------------------------- /inst/bench/taxi/vroom-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(vroom) library(dplyr) }) 0.443401 0.4445208079996519 1666833417 14776615 11 6881202176 3 | "x <- vroom(file, col_types = c(pickup_datetime = ""c""), trim_ws = FALSE, quote..." 16.979603 1.2900599709828384 1666833417 14776615 11 6881202176 4 | print(x) 0.07167099999999849 0.07509800099069253 1666833417 14776615 11 6881202176 5 | a <- head(x) 0.0017300000000020077 0.00173125701257959 1666833417 14776615 11 6881202176 6 | b <- tail(x) 3.6899999999917554e-4 3.7027703365311027e-4 1666833417 14776615 11 6881202176 7 | c <- sample_n(x, 100) 0.010666000000000508 0.010672853037249297 1666833417 14776615 11 6881202176 8 | "d <- filter(x, payment_type == ""UNK"")" 1.3317319999999988 1.3320457949885167 1666833417 14776615 11 6881202176 9 | e <- group_by(x, payment_type) %>% summarise(avg_tip = mean(tip_amount)) 3.956316000000001 3.9592064779717475 1666833417 14776615 11 6881202176 10 | -------------------------------------------------------------------------------- /inst/bench/taxi/vroom_no_altrep-dplyr.R: -------------------------------------------------------------------------------- 1 | ({library(vroom); library(dplyr)}) 2 | x <- vroom(file, col_types = c(pickup_datetime = "c"), trim_ws = FALSE, quote = "", escape_double = FALSE, na = character(), altrep = FALSE) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- sample_n(x, 100) 7 | d <- filter(x, payment_type == "UNK") 8 | e <- group_by(x, payment_type) %>% summarise(avg_tip = mean(tip_amount)) 9 | -------------------------------------------------------------------------------- /inst/bench/taxi/vroom_no_altrep-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(vroom) library(dplyr) }) 0.301629645 0.3016328811645508 1641999923 14388451 11 7029014528 3 | "x <- vroom(file, col_types = c(pickup_datetime = ""c""), trim_ws = FALSE, quote..." 47.375392178000006 18.417577743530273 1641999923 14388451 11 7029014528 4 | print(x) 0.1009485880000014 0.11608505249023438 1641999923 14388451 11 7029014528 5 | a <- head(x) 0.001024977000000149 0.0010266304016113281 1641999923 14388451 11 7029014528 6 | b <- tail(x) 4.7993700000148465e-4 4.813671112060547e-4 1641999923 14388451 11 7029014528 7 | c <- sample_n(x, 100) 0.013319092999999782 0.0133209228515625 1641999923 14388451 11 7029014528 8 | "d <- filter(x, payment_type == ""UNK"")" 0.960716675999997 0.9607300758361816 1641999923 14388451 11 7029014528 9 | e <- group_by(x, payment_type) %>% summarise(avg_tip = mean(tip_amount)) 1.2211422310000017 1.2211759090423584 1641999923 14388451 11 7029014528 10 | -------------------------------------------------------------------------------- /inst/bench/taxi_multiple/data.table-data.table.R: -------------------------------------------------------------------------------- 1 | library(data.table) 2 | x <- rbindlist(idcol = "path", 3 | lapply(stats::setNames(file, file), fread, sep = ",", quote = "", strip.white = FALSE, na.strings = NULL) 4 | ) 5 | print(x) 6 | a <- head(x) 7 | b <- tail(x) 8 | c <- x[sample(NROW(x), 100), ] 9 | d <- x[payment_type == "UNK", ] 10 | e <- x[ , .(mean(tip_amount)), by = payment_type] 11 | -------------------------------------------------------------------------------- /inst/bench/taxi_multiple/data.table-data.table.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | library(data.table) 0.049356014000000004 0.05027127265930176 19747530143 173179759 11 63974539264 3 | "x <- rbindlist(idcol = ""path"", lapply(stats::setNames(file, file), fread, sep..." 365.63006013800003 95.29237675666809 19747530143 173179759 11 63974539264 4 | print(x) 0.006093298999985564 0.006696939468383789 19747530143 173179759 11 63974539264 5 | a <- head(x) 2.292589999797201e-4 2.3102760314941406e-4 19747530143 173179759 11 63974539264 6 | b <- tail(x) 1.8573600004856416e-4 1.8715858459472656e-4 19747530143 173179759 11 63974539264 7 | c <- x[sample(NROW(x), 100), ] 6.668740000463913e-4 6.690025329589844e-4 19747530143 173179759 11 63974539264 8 | "d <- x[payment_type == ""UNK"", ]" 3.5333796889999576 1.0772333145141602 19747530143 173179759 11 63974539264 9 | e <- x[, .(mean(tip_amount)), by = payment_type] 14.88072779800001 4.706362962722778 19747530143 173179759 11 63974539264 10 | -------------------------------------------------------------------------------- /inst/bench/taxi_multiple/readr-dplyr.R: -------------------------------------------------------------------------------- 1 | ({ library(readr); library(dplyr); library(purrr) }) 2 | x <- map_dfr(set_names(file), .id = "path", 3 | ~ read_csv(.x, col_types = c(pickup_datetime = "c"), quote = "", trim_ws = FALSE, na = character()) 4 | ) 5 | print(x) 6 | a <- head(x) 7 | b <- tail(x) 8 | c <- sample_n(x, 100) 9 | d <- filter(x, payment_type == "UNK") 10 | e <- group_by(x, payment_type) %>% summarise(avg_tip = mean(tip_amount)) 11 | -------------------------------------------------------------------------------- /inst/bench/taxi_multiple/readr-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(readr) library(dplyr) library(purrr) }) 0.2852784450000001 0.2862823009490967 19747530143 173179759 11 68142870528 3 | "x <- map_dfr(set_names(file), .id = ""path"", ~read_csv(.x, col_types = c(picku..." 475.032626874 475.0412588119507 19747530143 173179759 11 68142870528 4 | print(x) 0.4224021850000099 0.836052656173706 19747530143 173179759 11 68142870528 5 | a <- head(x) 5.290510000008908e-4 5.30242919921875e-4 19747530143 173179759 11 68142870528 6 | b <- tail(x) 5.061389999809762e-4 5.07354736328125e-4 19747530143 173179759 11 68142870528 7 | c <- sample_n(x, 100) 0.014291614999990543 0.014293432235717773 19747530143 173179759 11 68142870528 8 | "d <- filter(x, payment_type == ""UNK"")" 4.235514479999949 4.235594272613525 19747530143 173179759 11 68142870528 9 | e <- group_by(x, payment_type) %>% summarise(avg_tip = mean(tip_amount)) 13.503164607999963 13.503409147262573 19747530143 173179759 11 68142870528 10 | -------------------------------------------------------------------------------- /inst/bench/taxi_multiple/vroom-base.R: -------------------------------------------------------------------------------- 1 | library(vroom) 2 | x <- vroom(file, id = "path", col_types = c(pickup_datetime = "c"), trim_ws = FALSE, quote = "", escape_double = FALSE, na = character(), altrep = TRUE) 3 | print(head(x, 10)) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- x[sample(NROW(x), 100), ] 7 | d <- x[x$payment_type == "UNK", ] 8 | e <- tapply(x$tip_amount, x$payment_type, mean) 9 | -------------------------------------------------------------------------------- /inst/bench/taxi_multiple/vroom-base.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | library(vroom) 0.24334018300000015 0.24334383010864258 19747530143 173179759 11 94792810496 3 | "x <- vroom(file, id = ""path"", col_types = c(pickup_datetime = ""c""), trim_ws =..." 280.08802939099996 20.344595670700073 19747530143 173179759 11 94792810496 4 | print(head(x, 10)) 2.7922258899999974 3.047116756439209 19747530143 173179759 11 94792810496 5 | a <- head(x) 2.6394799999707175e-4 2.655982971191406e-4 19747530143 173179759 11 94792810496 6 | b <- tail(x) 4.8234499996624436e-4 4.837512969970703e-4 19747530143 173179759 11 94792810496 7 | c <- x[sample(NROW(x), 100), ] 2.3426100000278893e-4 2.353191375732422e-4 19747530143 173179759 11 94792810496 8 | "d <- x[x$payment_type == ""UNK"", ]" 21.519468974000006 21.5198495388031 19747530143 173179759 11 94792810496 9 | e <- tapply(x$tip_amount, x$payment_type, mean) 162.64416488300003 142.6223108768463 19747530143 173179759 11 94792810496 10 | -------------------------------------------------------------------------------- /inst/bench/taxi_multiple/vroom-dplyr.R: -------------------------------------------------------------------------------- 1 | ({library(vroom); library(dplyr)}) 2 | x <- vroom(file, id = "path", col_types = c(pickup_datetime = "c"), trim_ws = FALSE, quote = "", escape_double = FALSE, na = character(), altrep = TRUE) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- sample_n(x, 100) 7 | d <- filter(x, payment_type == "UNK") 8 | e <- group_by(x, payment_type) %>% summarise(avg_tip = mean(tip_amount)) 9 | -------------------------------------------------------------------------------- /inst/bench/taxi_multiple/vroom-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(vroom) library(dplyr) }) 0.3229789220000001 0.3229827880859375 19747530143 173179759 11 94496784384 3 | "x <- vroom(file, id = ""path"", col_types = c(pickup_datetime = ""c""), trim_ws =..." 281.055523347 20.365679502487183 19747530143 173179759 11 94496784384 4 | print(x) 2.542836164999983 2.785041570663452 19747530143 173179759 11 94496784384 5 | a <- head(x) 9.97338999980002e-4 9.989738464355469e-4 19747530143 173179759 11 94496784384 6 | b <- tail(x) 4.8055199999907927e-4 4.818439483642578e-4 19747530143 173179759 11 94496784384 7 | c <- sample_n(x, 100) 0.01243754600000102 0.012439489364624023 19747530143 173179759 11 94496784384 8 | "d <- filter(x, payment_type == ""UNK"")" 23.858187990000033 23.858795642852783 19747530143 173179759 11 94496784384 9 | e <- group_by(x, payment_type) %>% summarise(avg_tip = mean(tip_amount)) 65.63478174599999 65.63620686531067 19747530143 173179759 11 94496784384 10 | -------------------------------------------------------------------------------- /inst/bench/taxi_multiple/vroom_no_altrep-dplyr.R: -------------------------------------------------------------------------------- 1 | ({library(vroom); library(dplyr)}) 2 | x <- vroom(file, id = "path", col_types = c(pickup_datetime = "c"), trim_ws = FALSE, quote = "", escape_double = FALSE, na = character(), altrep = FALSE) 3 | print(x) 4 | a <- head(x) 5 | b <- tail(x) 6 | c <- sample_n(x, 100) 7 | d <- filter(x, payment_type == "UNK") 8 | e <- group_by(x, payment_type) %>% summarise(avg_tip = mean(tip_amount)) 9 | -------------------------------------------------------------------------------- /inst/bench/taxi_multiple/vroom_no_altrep-dplyr.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | ({ library(vroom) library(dplyr) }) 0.317607953 0.31838297843933105 19747530143 173179759 11 67751387136 3 | "x <- vroom(file, id = ""path"", col_types = c(pickup_datetime = ""c""), trim_ws =..." 607.194956447 232.3293113708496 19747530143 173179759 11 67751387136 4 | print(x) 2.005394704000082 2.162087917327881 19747530143 173179759 11 67751387136 5 | a <- head(x) 0.0010485940000535265 0.0010504722595214844 19747530143 173179759 11 67751387136 6 | b <- tail(x) 4.88735000089946e-4 4.901885986328125e-4 19747530143 173179759 11 67751387136 7 | c <- sample_n(x, 100) 0.0136381549999669 0.013639688491821289 19747530143 173179759 11 67751387136 8 | "d <- filter(x, payment_type == ""UNK"")" 10.491835902000048 10.491984844207764 19747530143 173179759 11 67751387136 9 | e <- group_by(x, payment_type) %>% summarise(avg_tip = mean(tip_amount)) 7.2028114820000155 7.202939510345459 19747530143 173179759 11 67751387136 10 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/base-gzip.R: -------------------------------------------------------------------------------- 1 | { 2 | library(vroom) 3 | data <- vroom(file, col_types = c(pickup_datetime = "c")) 4 | vroom:::vroom_materialize(data, replace = TRUE) 5 | } 6 | 7 | { 8 | con <- gzfile(tempfile(fileext = ".gz"), "wb") 9 | write.table(data, con, sep = "\t", quote = FALSE, row.names = FALSE) 10 | close(con) 11 | } 12 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/base-gzip.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | "{ library(vroom) data <- vroom(file, col_types = c(pickup_datetime = ""c"")) vr..." 56.307022157 16.904335260391235 1641999923 14388451 11 6515875840 3 | "{ con <- gzfile(tempfile(fileext = "".gz""), ""wb"") write.table(data, con, sep =..." 197.11310746799998 197.12195253372192 1641999923 14388451 11 6515875840 4 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/base-multithreaded_gzip.R: -------------------------------------------------------------------------------- 1 | { 2 | library(vroom) 3 | data <- vroom(file, col_types = c(pickup_datetime = "c")) 4 | vroom:::vroom_materialize(data, replace = TRUE) 5 | } 6 | 7 | { 8 | con <- pipe(sprintf("pigz > %s", tempfile(fileext = ".gz")), "wb") 9 | write.table(data, con, sep = "\t", quote = FALSE, row.names = FALSE) 10 | close(con) 11 | } 12 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/base-multithreaded_gzip.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | "{ library(vroom) data <- vroom(file, col_types = c(pickup_datetime = ""c"")) vr..." 49.800825008 15.864639043807983 1641999923 14388451 11 6515879936 3 | "{ con <- pipe(sprintf(""pigz > %s"", tempfile(fileext = "".gz"")), ""wb"") write.ta..." 97.815085082 97.83104109764099 1641999923 14388451 11 6515879936 4 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/base-uncompressed.R: -------------------------------------------------------------------------------- 1 | { 2 | library(vroom) 3 | data <- vroom(file, col_types = c(pickup_datetime = "c")) 4 | vroom:::vroom_materialize(data, replace = TRUE) 5 | } 6 | 7 | write.table(data, tempfile(fileext = ".tsv"), sep = "\t", quote = FALSE, row.names = FALSE) 8 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/base-uncompressed.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | "{ library(vroom) data <- vroom(file, col_types = c(pickup_datetime = ""c"")) vr..." 48.850021397 16.385770320892334 1641999923 14388451 11 6515879936 3 | "write.table(data, tempfile(fileext = "".tsv""), sep = ""\t"", quote = FALSE, row...." 97.42563251000001 97.43097257614136 1641999923 14388451 11 6515879936 4 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/base-zstandard.R: -------------------------------------------------------------------------------- 1 | { 2 | library(vroom) 3 | data <- vroom(file, col_types = c(pickup_datetime = "c")) 4 | vroom:::vroom_materialize(data, replace = TRUE) 5 | } 6 | 7 | { 8 | con <- pipe(sprintf("zstd > %s", tempfile(fileext = ".zst")), "wb") 9 | write.table(data, con, sep = "\t", quote = FALSE, row.names = FALSE) 10 | close(con) 11 | } 12 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/base-zstandard.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | "{ library(vroom) data <- vroom(file, col_types = c(pickup_datetime = ""c"")) vr..." 57.130642056999996 18.90769624710083 1641999923 14388451 11 6515888128 3 | "{ con <- pipe(sprintf(""zstd > %s"", tempfile(fileext = "".zst"")), ""wb"") write.t..." 99.85411830200002 99.8676381111145 1641999923 14388451 11 6515888128 4 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/data.table-gzip.R: -------------------------------------------------------------------------------- 1 | { 2 | library(vroom) 3 | data <- vroom(file, col_types = c(pickup_datetime = "c")) 4 | vroom:::vroom_materialize(data, replace = TRUE) 5 | } 6 | 7 | data.table::fwrite(data, tempfile(fileext = ".gz"), sep = "\t", nThread = 1) 8 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/data.table-gzip.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | "{ library(vroom) data <- vroom(file, col_types = c(pickup_datetime = ""c"")) vr..." 57.120600576 19.275273323059082 1641999923 14388451 11 6518173696 3 | "data.table::fwrite(data, tempfile(fileext = "".gz""), sep = ""\t"", nThread = 1)" 67.75096147299999 67.75405859947205 1641999923 14388451 11 6518173696 4 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/data.table-multithreaded_gzip.R: -------------------------------------------------------------------------------- 1 | { 2 | library(vroom) 3 | data <- vroom(file, col_types = c(pickup_datetime = "c")) 4 | vroom:::vroom_materialize(data, replace = TRUE) 5 | } 6 | 7 | data.table::fwrite(data, tempfile(fileext = ".gz"), sep = "\t") 8 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/data.table-multithreaded_gzip.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | "{ library(vroom) data <- vroom(file, col_types = c(pickup_datetime = ""c"")) vr..." 52.228046424 18.820579051971436 1641999923 14388451 11 7046705152 3 | "data.table::fwrite(data, tempfile(fileext = "".gz""), sep = ""\t"")" 67.654484228 8.864737272262573 1641999923 14388451 11 7046705152 4 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/data.table-uncompressed.R: -------------------------------------------------------------------------------- 1 | { 2 | library(vroom) 3 | data <- vroom(file, col_types = c(pickup_datetime = "c")) 4 | vroom:::vroom_materialize(data, replace = TRUE) 5 | } 6 | 7 | data.table::fwrite(data, tempfile(fileext = ".tsv"), sep = "\t") 8 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/data.table-uncompressed.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | "{ library(vroom) data <- vroom(file, col_types = c(pickup_datetime = ""c"")) vr..." 51.447035740000004 16.117270708084106 1641999923 14388451 11 6576988160 3 | "data.table::fwrite(data, tempfile(fileext = "".tsv""), sep = ""\t"")" 9.219789825999996 1.4708619117736816 1641999923 14388451 11 6576988160 4 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/readr-gzip.R: -------------------------------------------------------------------------------- 1 | { 2 | library(vroom) 3 | data <- vroom(file, col_types = c(pickup_datetime = "c")) 4 | vroom:::vroom_materialize(data, replace = TRUE) 5 | } 6 | 7 | readr::write_tsv(data, tempfile(fileext = ".gz")) 8 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/readr-gzip.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | "{ library(vroom) data <- vroom(file, col_types = c(pickup_datetime = ""c"")) vr..." 52.427885992 16.014821529388428 1641999923 14388451 11 6816428032 3 | "readr::write_tsv(data, tempfile(fileext = "".gz""))" 120.15885149100001 120.1623752117157 1641999923 14388451 11 6816428032 4 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/readr-multithreaded_gzip.R: -------------------------------------------------------------------------------- 1 | { 2 | library(vroom) 3 | data <- vroom(file, col_types = c(pickup_datetime = "c")) 4 | vroom:::vroom_materialize(data, replace = TRUE) 5 | } 6 | 7 | readr::write_tsv(data, pipe(sprintf("pigz > %s", tempfile(fileext = ".gz")))) 8 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/readr-multithreaded_gzip.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | "{ library(vroom) data <- vroom(file, col_types = c(pickup_datetime = ""c"")) vr..." 52.571174903999996 16.05108642578125 1641999923 14388451 11 6816079872 3 | "readr::write_tsv(data, pipe(sprintf(""pigz > %s"", tempfile(fileext = "".gz""))))" 53.408188828 53.427462577819824 1641999923 14388451 11 6816079872 4 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/readr-uncompressed.R: -------------------------------------------------------------------------------- 1 | { 2 | library(vroom) 3 | data <- vroom(file, col_types = c(pickup_datetime = "c")) 4 | vroom:::vroom_materialize(data, replace = TRUE) 5 | } 6 | 7 | readr::write_tsv(data, tempfile(fileext = ".tsv")) 8 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/readr-uncompressed.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | "{ library(vroom) data <- vroom(file, col_types = c(pickup_datetime = ""c"")) vr..." 50.408979955 16.516019582748413 1641999923 14388451 11 6815977472 3 | "readr::write_tsv(data, tempfile(fileext = "".tsv""))" 52.224211567000005 52.229180574417114 1641999923 14388451 11 6815977472 4 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/readr-zstandard.R: -------------------------------------------------------------------------------- 1 | { 2 | library(vroom) 3 | data <- vroom(file, col_types = c(pickup_datetime = "c")) 4 | vroom:::vroom_materialize(data, replace = TRUE) 5 | } 6 | 7 | readr::write_tsv(data, pipe(sprintf("zstd > %s", tempfile(fileext = ".zst")))) 8 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/readr-zstandard.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | "{ library(vroom) data <- vroom(file, col_types = c(pickup_datetime = ""c"")) vr..." 53.053004752 17.89621663093567 1641999923 14388451 11 6815940608 3 | "readr::write_tsv(data, pipe(sprintf(""zstd > %s"", tempfile(fileext = "".zst""))))" 54.141594968999996 54.15725231170654 1641999923 14388451 11 6815940608 4 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/vroom-gzip.R: -------------------------------------------------------------------------------- 1 | { 2 | library(vroom) 3 | data <- vroom(file, col_types = c(pickup_datetime = "c")) 4 | vroom:::vroom_materialize(data, replace = TRUE) 5 | } 6 | 7 | vroom_write(data, tempfile(fileext = ".gz"), delim = "\t") 8 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/vroom-gzip.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | "{ library(vroom) data <- vroom(file, col_types = c(pickup_datetime = ""c"")) vr..." 58.208655639999996 18.150486946105957 1641999923 14388451 11 6657384448 3 | "vroom_write(data, tempfile(fileext = "".gz""), delim = ""\t"")" 93.763101656999993 74.44156193733215 1641999923 14388451 11 6657384448 4 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/vroom-multithreaded_gzip.R: -------------------------------------------------------------------------------- 1 | { 2 | library(vroom) 3 | data <- vroom(file, col_types = c(pickup_datetime = "c")) 4 | vroom:::vroom_materialize(data, replace = TRUE) 5 | } 6 | 7 | vroom_write(data, pipe(sprintf("pigz > %s", tempfile(fileext = ".gz"))), delim = "\t") 8 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/vroom-multithreaded_gzip.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | "{ library(vroom) data <- vroom(file, col_types = c(pickup_datetime = ""c"")) vr..." 52.529673003999996 17.085912942886353 1641999923 14388451 11 6656368640 3 | "vroom_write(data, pipe(sprintf(""pigz > %s"", tempfile(fileext = "".gz""))), deli..." 23.073533718 8.076940298080444 1641999923 14388451 11 6656368640 4 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/vroom-uncompressed.R: -------------------------------------------------------------------------------- 1 | { 2 | library(vroom) 3 | data <- vroom(file, col_types = c(pickup_datetime = "c")) 4 | vroom:::vroom_materialize(data, replace = TRUE) 5 | } 6 | 7 | vroom_write(data, tempfile(fileext = ".tsv"), delim = "\t") 8 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/vroom-uncompressed.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | "{ library(vroom) data <- vroom(file, col_types = c(pickup_datetime = ""c"")) vr..." 52.838888093 18.450870513916016 1641999923 14388451 11 7582793728 3 | "vroom_write(data, tempfile(fileext = "".tsv""), delim = ""\t"")" 22.587353183000005 1.6948139667510986 1641999923 14388451 11 7582793728 4 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/vroom-zstandard.R: -------------------------------------------------------------------------------- 1 | { 2 | library(vroom) 3 | data <- vroom(file, col_types = c(pickup_datetime = "c")) 4 | vroom:::vroom_materialize(data, replace = TRUE) 5 | } 6 | 7 | vroom_write(data, pipe(sprintf("zstd > %s", tempfile(fileext = ".zst"))), delim = "\t") 8 | -------------------------------------------------------------------------------- /inst/bench/taxi_writing/vroom-zstandard.tsv: -------------------------------------------------------------------------------- 1 | exprs process real size rows cols max_memory 2 | "{ library(vroom) data <- vroom(file, col_types = c(pickup_datetime = ""c"")) vr..." 51.565757179 16.19530463218689 1641999923 14388451 11 6656360448 3 | "vroom_write(data, pipe(sprintf(""zstd > %s"", tempfile(fileext = "".zst""))), del..." 22.155763127999997 12.370139837265015 1641999923 14388451 11 6656360448 4 | -------------------------------------------------------------------------------- /inst/extdata/fwf-sample.txt: -------------------------------------------------------------------------------- 1 | John Smith WA 418-Y11-4111 2 | Mary Hartford CA 319-Z19-4341 3 | Evan Nolan IL 219-532-c301 4 | -------------------------------------------------------------------------------- /inst/extdata/mtcars-4.csv: -------------------------------------------------------------------------------- 1 | model,mpg,cyl,disp,hp,drat,wt,qsec,vs,am,gear,carb 2 | Datsun 710,22.8,4,108,93,3.85,2.32,18.61,1,1,4,1 3 | Merc 240D,24.4,4,146.7,62,3.69,3.19,20,1,0,4,2 4 | Merc 230,22.8,4,140.8,95,3.92,3.15,22.9,1,0,4,2 5 | Fiat 128,32.4,4,78.7,66,4.08,2.2,19.47,1,1,4,1 6 | Honda Civic,30.4,4,75.7,52,4.93,1.615,18.52,1,1,4,2 7 | Toyota Corolla,33.9,4,71.1,65,4.22,1.835,19.9,1,1,4,1 8 | Toyota Corona,21.5,4,120.1,97,3.7,2.465,20.01,1,0,3,1 9 | Fiat X1-9,27.3,4,79,66,4.08,1.935,18.9,1,1,4,1 10 | Porsche 914-2,26,4,120.3,91,4.43,2.14,16.7,0,1,5,2 11 | Lotus Europa,30.4,4,95.1,113,3.77,1.513,16.9,1,1,5,2 12 | Volvo 142E,21.4,4,121,109,4.11,2.78,18.6,1,1,4,2 13 | -------------------------------------------------------------------------------- /inst/extdata/mtcars-6.csv: -------------------------------------------------------------------------------- 1 | model,mpg,cyl,disp,hp,drat,wt,qsec,vs,am,gear,carb 2 | Mazda RX4,21,6,160,110,3.9,2.62,16.46,0,1,4,4 3 | Mazda RX4 Wag,21,6,160,110,3.9,2.875,17.02,0,1,4,4 4 | Hornet 4 Drive,21.4,6,258,110,3.08,3.215,19.44,1,0,3,1 5 | Valiant,18.1,6,225,105,2.76,3.46,20.22,1,0,3,1 6 | Merc 280,19.2,6,167.6,123,3.92,3.44,18.3,1,0,4,4 7 | Merc 280C,17.8,6,167.6,123,3.92,3.44,18.9,1,0,4,4 8 | Ferrari Dino,19.7,6,145,175,3.62,2.77,15.5,0,1,5,6 9 | -------------------------------------------------------------------------------- /inst/extdata/mtcars-8.csv: -------------------------------------------------------------------------------- 1 | model,mpg,cyl,disp,hp,drat,wt,qsec,vs,am,gear,carb 2 | Hornet Sportabout,18.7,8,360,175,3.15,3.44,17.02,0,0,3,2 3 | Duster 360,14.3,8,360,245,3.21,3.57,15.84,0,0,3,4 4 | Merc 450SE,16.4,8,275.8,180,3.07,4.07,17.4,0,0,3,3 5 | Merc 450SL,17.3,8,275.8,180,3.07,3.73,17.6,0,0,3,3 6 | Merc 450SLC,15.2,8,275.8,180,3.07,3.78,18,0,0,3,3 7 | Cadillac Fleetwood,10.4,8,472,205,2.93,5.25,17.98,0,0,3,4 8 | Lincoln Continental,10.4,8,460,215,3,5.424,17.82,0,0,3,4 9 | Chrysler Imperial,14.7,8,440,230,3.23,5.345,17.42,0,0,3,4 10 | Dodge Challenger,15.5,8,318,150,2.76,3.52,16.87,0,0,3,2 11 | AMC Javelin,15.2,8,304,150,3.15,3.435,17.3,0,0,3,2 12 | Camaro Z28,13.3,8,350,245,3.73,3.84,15.41,0,0,3,4 13 | Pontiac Firebird,19.2,8,400,175,3.08,3.845,17.05,0,0,3,2 14 | Ford Pantera L,15.8,8,351,264,4.22,3.17,14.5,0,1,5,4 15 | Maserati Bora,15,8,301,335,3.54,3.57,14.6,0,1,5,8 16 | -------------------------------------------------------------------------------- /inst/extdata/mtcars-multi-cyl.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidyverse/vroom/968a3312d00f6f9de2a0c9cd3b1f2fa50a3e353e/inst/extdata/mtcars-multi-cyl.zip -------------------------------------------------------------------------------- /inst/extdata/mtcars.csv.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidyverse/vroom/968a3312d00f6f9de2a0c9cd3b1f2fa50a3e353e/inst/extdata/mtcars.csv.bz2 -------------------------------------------------------------------------------- /inst/extdata/mtcars.csv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidyverse/vroom/968a3312d00f6f9de2a0c9cd3b1f2fa50a3e353e/inst/extdata/mtcars.csv.gz -------------------------------------------------------------------------------- /inst/extdata/mtcars.csv.xz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidyverse/vroom/968a3312d00f6f9de2a0c9cd3b1f2fa50a3e353e/inst/extdata/mtcars.csv.xz -------------------------------------------------------------------------------- /inst/extdata/mtcars.csv.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidyverse/vroom/968a3312d00f6f9de2a0c9cd3b1f2fa50a3e353e/inst/extdata/mtcars.csv.zip -------------------------------------------------------------------------------- /man/as.col_spec.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/col_types.R 3 | \name{as.col_spec} 4 | \alias{as.col_spec} 5 | \title{Coerce to a column specification} 6 | \usage{ 7 | as.col_spec(x) 8 | } 9 | \arguments{ 10 | \item{x}{Input object} 11 | } 12 | \description{ 13 | This is most useful for generating a specification using the short form or coercing from a list. 14 | } 15 | \examples{ 16 | as.col_spec("cccnnn") 17 | } 18 | \keyword{internal} 19 | -------------------------------------------------------------------------------- /man/date_names.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/date.R 3 | \name{date_names} 4 | \alias{date_names} 5 | \alias{date_names_lang} 6 | \alias{date_names_langs} 7 | \title{Create or retrieve date names} 8 | \usage{ 9 | date_names(mon, mon_ab = mon, day, day_ab = day, am_pm = c("AM", "PM")) 10 | 11 | date_names_lang(language) 12 | 13 | date_names_langs() 14 | } 15 | \arguments{ 16 | \item{mon, mon_ab}{Full and abbreviated month names.} 17 | 18 | \item{day, day_ab}{Full and abbreviated week day names. Starts with Sunday.} 19 | 20 | \item{am_pm}{Names used for AM and PM.} 21 | 22 | \item{language}{A BCP 47 locale, made up of a language and a region, 23 | e.g. \code{"en_US"} for American English. See \code{date_names_langs()} 24 | for a complete list of available locales.} 25 | } 26 | \description{ 27 | When parsing dates, you often need to know how weekdays of the week and 28 | months are represented as text. This pair of functions allows you to either 29 | create your own, or retrieve from a standard list. The standard list is 30 | derived from ICU (\verb{https://site.icu-project.org}) via the \emph{stringi} package. 31 | } 32 | \examples{ 33 | date_names_lang("en") 34 | date_names_lang("ko") 35 | date_names_lang("fr") 36 | } 37 | -------------------------------------------------------------------------------- /man/figures/lifecycle-archived.svg: -------------------------------------------------------------------------------- 1 | 2 | lifecycle: archived 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | lifecycle 18 | 19 | archived 20 | 21 | 22 | -------------------------------------------------------------------------------- /man/figures/lifecycle-defunct.svg: -------------------------------------------------------------------------------- 1 | 2 | lifecycle: defunct 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | lifecycle 18 | 19 | defunct 20 | 21 | 22 | -------------------------------------------------------------------------------- /man/figures/lifecycle-deprecated.svg: -------------------------------------------------------------------------------- 1 | 2 | lifecycle: deprecated 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | lifecycle 18 | 19 | deprecated 20 | 21 | 22 | -------------------------------------------------------------------------------- /man/figures/lifecycle-experimental.svg: -------------------------------------------------------------------------------- 1 | 2 | lifecycle: experimental 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | lifecycle 18 | 19 | experimental 20 | 21 | 22 | -------------------------------------------------------------------------------- /man/figures/lifecycle-maturing.svg: -------------------------------------------------------------------------------- 1 | 2 | lifecycle: maturing 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | lifecycle 18 | 19 | maturing 20 | 21 | 22 | -------------------------------------------------------------------------------- /man/figures/lifecycle-questioning.svg: -------------------------------------------------------------------------------- 1 | 2 | lifecycle: questioning 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | lifecycle 18 | 19 | questioning 20 | 21 | 22 | -------------------------------------------------------------------------------- /man/figures/lifecycle-retired.svg: -------------------------------------------------------------------------------- 1 | lifecyclelifecycleretiredretired -------------------------------------------------------------------------------- /man/figures/lifecycle-soft-deprecated.svg: -------------------------------------------------------------------------------- 1 | 2 | lifecycle: soft-deprecated 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | lifecycle 18 | 19 | soft-deprecated 20 | 21 | 22 | -------------------------------------------------------------------------------- /man/figures/lifecycle-stable.svg: -------------------------------------------------------------------------------- 1 | 2 | lifecycle: stable 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 19 | 20 | lifecycle 21 | 22 | 25 | 26 | stable 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /man/figures/lifecycle-superseded.svg: -------------------------------------------------------------------------------- 1 | 2 | lifecycle: superseded 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | lifecycle 18 | 19 | superseded 20 | 21 | 22 | -------------------------------------------------------------------------------- /man/figures/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidyverse/vroom/968a3312d00f6f9de2a0c9cd3b1f2fa50a3e353e/man/figures/logo.png -------------------------------------------------------------------------------- /man/guess_type.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/col_types.R 3 | \name{guess_type} 4 | \alias{guess_type} 5 | \title{Guess the type of a vector} 6 | \usage{ 7 | guess_type( 8 | x, 9 | na = c("", "NA"), 10 | locale = default_locale(), 11 | guess_integer = FALSE 12 | ) 13 | } 14 | \arguments{ 15 | \item{x}{Character vector of values to parse.} 16 | 17 | \item{na}{Character vector of strings to interpret as missing values. Set this 18 | option to \code{character()} to indicate no missing values.} 19 | 20 | \item{locale}{The locale controls defaults that vary from place to place. 21 | The default locale is US-centric (like R), but you can use 22 | \code{\link[readr:locale]{locale()}} to create your own locale that controls things like 23 | the default time zone, encoding, decimal mark, big mark, and day/month 24 | names.} 25 | 26 | \item{guess_integer}{If \code{TRUE}, guess integer types for whole numbers, if 27 | \code{FALSE} guess numeric type for all numbers.} 28 | } 29 | \description{ 30 | Guess the type of a vector 31 | } 32 | \examples{ 33 | # Logical vectors 34 | guess_type(c("FALSE", "TRUE", "F", "T")) 35 | # Integers and doubles 36 | guess_type(c("1","2","3")) 37 | guess_type(c("1.6","2.6","3.4")) 38 | # Numbers containing grouping mark 39 | guess_type("1,234,566") 40 | # ISO 8601 date times 41 | guess_type(c("2010-10-10")) 42 | guess_type(c("2010-10-10 01:02:03")) 43 | guess_type(c("01:02:03 AM")) 44 | } 45 | -------------------------------------------------------------------------------- /man/output_column.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/vroom_write.R 3 | \name{output_column} 4 | \alias{output_column} 5 | \title{Preprocess column for output} 6 | \usage{ 7 | output_column(x) 8 | } 9 | \arguments{ 10 | \item{x}{A vector} 11 | } 12 | \description{ 13 | This is a generic function that applied to each column before it is saved 14 | to disk. It provides a hook for S3 classes that need special handling. 15 | } 16 | \examples{ 17 | # Most types are returned unchanged 18 | output_column(1) 19 | output_column("x") 20 | 21 | # datetimes are formatted in ISO 8601 22 | output_column(Sys.Date()) 23 | output_column(Sys.time()) 24 | } 25 | \keyword{internal} 26 | -------------------------------------------------------------------------------- /man/problems.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/problems.R 3 | \name{problems} 4 | \alias{problems} 5 | \title{Retrieve parsing problems} 6 | \usage{ 7 | problems(x = .Last.value, lazy = FALSE) 8 | } 9 | \arguments{ 10 | \item{x}{A data frame from \code{vroom::vroom()}.} 11 | 12 | \item{lazy}{If \code{TRUE}, just the problems found so far are returned. If 13 | \code{FALSE} (the default) the lazy data is first read completely and all 14 | problems are returned.} 15 | } 16 | \value{ 17 | A data frame with one row for each problem and four columns: 18 | \itemize{ 19 | \item row,col - Row and column number that caused the problem, referencing the 20 | original input 21 | \item expected - What vroom expected to find 22 | \item actual - What it actually found 23 | \item file - The file with the problem 24 | } 25 | } 26 | \description{ 27 | vroom will only fail to parse a file if the file is invalid in a way that is 28 | unrecoverable. However there are a number of non-fatal problems that you 29 | might want to know about. You can retrieve a data frame of these problems 30 | with this function. 31 | } 32 | -------------------------------------------------------------------------------- /man/reexports.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/tidyselect.R 3 | \docType{import} 4 | \name{reexports} 5 | \alias{reexports} 6 | \alias{contains} 7 | \alias{select_helpers} 8 | \alias{ends_with} 9 | \alias{everything} 10 | \alias{matches} 11 | \alias{num_range} 12 | \alias{one_of} 13 | \alias{starts_with} 14 | \alias{last_col} 15 | \title{Objects exported from other packages} 16 | \keyword{internal} 17 | \description{ 18 | These objects are imported from other packages. Follow the links 19 | below to see their documentation. 20 | 21 | \describe{ 22 | \item{tidyselect}{\code{\link[tidyselect:starts_with]{contains}}, \code{\link[tidyselect:starts_with]{ends_with}}, \code{\link[tidyselect]{everything}}, \code{\link[tidyselect:everything]{last_col}}, \code{\link[tidyselect:starts_with]{matches}}, \code{\link[tidyselect:starts_with]{num_range}}, \code{\link[tidyselect]{one_of}}, \code{\link[tidyselect]{starts_with}}} 23 | }} 24 | 25 | -------------------------------------------------------------------------------- /man/spec.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/col_types.R 3 | \name{cols_condense} 4 | \alias{cols_condense} 5 | \alias{spec} 6 | \title{Examine the column specifications for a data frame} 7 | \usage{ 8 | cols_condense(x) 9 | 10 | spec(x) 11 | } 12 | \arguments{ 13 | \item{x}{The data frame object to extract from} 14 | } 15 | \value{ 16 | A col_spec object. 17 | } 18 | \description{ 19 | \code{cols_condense()} takes a spec object and condenses its definition by setting 20 | the default column type to the most frequent type and only listing columns 21 | with a different type. 22 | 23 | \code{spec()} extracts the full column specification from a tibble 24 | created by readr. 25 | } 26 | \examples{ 27 | df <- vroom(vroom_example("mtcars.csv")) 28 | s <- spec(df) 29 | s 30 | 31 | cols_condense(s) 32 | } 33 | \concept{parsers} 34 | -------------------------------------------------------------------------------- /man/vroom-package.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/vroom-package.R 3 | \docType{package} 4 | \name{vroom-package} 5 | \alias{vroom-package} 6 | \title{vroom: Read and Write Rectangular Text Data Quickly} 7 | \description{ 8 | \if{html}{\figure{logo.png}{options: style='float: right' alt='logo' width='120'}} 9 | 10 | The goal of 'vroom' is to read and write data (like 'csv', 'tsv' and 'fwf') quickly. When reading it uses a quick initial indexing step, then reads the values lazily , so only the data you actually use needs to be read. The writer formats the data in parallel and writes to disk asynchronously from formatting. 11 | } 12 | \seealso{ 13 | Useful links: 14 | \itemize{ 15 | \item \url{https://vroom.r-lib.org} 16 | \item \url{https://github.com/tidyverse/vroom} 17 | \item Report bugs at \url{https://github.com/tidyverse/vroom/issues} 18 | } 19 | 20 | } 21 | \author{ 22 | \strong{Maintainer}: Jennifer Bryan \email{jenny@posit.co} (\href{https://orcid.org/0000-0002-6983-2759}{ORCID}) 23 | 24 | Authors: 25 | \itemize{ 26 | \item Jim Hester (\href{https://orcid.org/0000-0002-2739-7082}{ORCID}) 27 | \item Hadley Wickham \email{hadley@posit.co} (\href{https://orcid.org/0000-0003-4757-117X}{ORCID}) 28 | } 29 | 30 | Other contributors: 31 | \itemize{ 32 | \item Shelby Bearrows [contributor] 33 | \item https://github.com/mandreyel/ (mio library) [copyright holder] 34 | \item Jukka Jylänki (grisu3 implementation) [copyright holder] 35 | \item Mikkel Jørgensen (grisu3 implementation) [copyright holder] 36 | \item Posit Software, PBC [copyright holder, funder] 37 | } 38 | 39 | } 40 | \keyword{internal} 41 | -------------------------------------------------------------------------------- /man/vroom_altrep_opts.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/vroom.R 3 | \name{vroom_altrep_opts} 4 | \alias{vroom_altrep_opts} 5 | \title{Show which column types are using Altrep} 6 | \usage{ 7 | vroom_altrep_opts(which = NULL) 8 | } 9 | \arguments{ 10 | \item{which}{A character vector of column types to use Altrep for. Can also 11 | take \code{TRUE} or \code{FALSE} to use Altrep for all possible or none of the 12 | types} 13 | } 14 | \description{ 15 | \Sexpr[results=rd, stage=render]{lifecycle::badge("deprecated")} 16 | This function is deprecated in favor of \code{vroom_altrep()}. 17 | } 18 | -------------------------------------------------------------------------------- /man/vroom_example.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/example.R 3 | \name{vroom_example} 4 | \alias{vroom_example} 5 | \alias{vroom_examples} 6 | \title{Get path to vroom examples} 7 | \usage{ 8 | vroom_example(path) 9 | 10 | vroom_examples(pattern = NULL) 11 | } 12 | \arguments{ 13 | \item{path}{Name of file.} 14 | 15 | \item{pattern}{A regular expression of filenames to match. If \code{NULL}, all 16 | available files are returned.} 17 | } 18 | \description{ 19 | vroom comes bundled with a number of sample files in 20 | its 'inst/extdata' directory. Use \code{vroom_examples()} to list all the 21 | available examples and \code{vroom_example()} to retrieve the path to one 22 | example. 23 | } 24 | \examples{ 25 | # List all available examples 26 | vroom_examples() 27 | 28 | # Get path to one example 29 | vroom_example("mtcars.csv") 30 | } 31 | -------------------------------------------------------------------------------- /man/vroom_progress.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/vroom.R 3 | \name{vroom_progress} 4 | \alias{vroom_progress} 5 | \title{Determine whether progress bars should be shown} 6 | \usage{ 7 | vroom_progress() 8 | } 9 | \description{ 10 | By default, vroom shows progress bars. However, progress reporting is 11 | suppressed if any of the following conditions hold: 12 | \itemize{ 13 | \item The bar is explicitly disabled by setting the environment variable 14 | \code{VROOM_SHOW_PROGRESS} to \code{"false"}. 15 | \item The code is run in a non-interactive session, as determined by 16 | \code{\link[rlang:is_interactive]{rlang::is_interactive()}}. 17 | \item The code is run in an RStudio notebook chunk, as determined by 18 | \code{getOption("rstudio.notebook.executing")}. 19 | } 20 | } 21 | \examples{ 22 | vroom_progress() 23 | } 24 | -------------------------------------------------------------------------------- /man/vroom_str.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/altrep.R 3 | \name{vroom_str} 4 | \alias{vroom_str} 5 | \title{Structure of objects} 6 | \usage{ 7 | vroom_str(x) 8 | } 9 | \arguments{ 10 | \item{x}{a vector} 11 | } 12 | \description{ 13 | Similar to \code{str()} but with more information for Altrep objects. 14 | } 15 | \examples{ 16 | # when used on non-altrep objects altrep will always be false 17 | vroom_str(mtcars) 18 | 19 | mt <- vroom(vroom_example("mtcars.csv"), ",", altrep = c("chr", "dbl")) 20 | vroom_str(mt) 21 | } 22 | -------------------------------------------------------------------------------- /man/vroom_write_lines.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/vroom_write.R 3 | \name{vroom_write_lines} 4 | \alias{vroom_write_lines} 5 | \title{Write lines to a file} 6 | \usage{ 7 | vroom_write_lines( 8 | x, 9 | file, 10 | eol = "\\n", 11 | na = "NA", 12 | append = FALSE, 13 | num_threads = vroom_threads() 14 | ) 15 | } 16 | \arguments{ 17 | \item{x}{A character vector.} 18 | 19 | \item{file}{File or connection to write to.} 20 | 21 | \item{eol}{The end of line character to use. Most commonly either \code{"\n"} for 22 | Unix style newlines, or \code{"\r\n"} for Windows style newlines.} 23 | 24 | \item{na}{String used for missing values. Defaults to 'NA'.} 25 | 26 | \item{append}{If \code{FALSE}, will overwrite existing file. If \code{TRUE}, 27 | will append to existing file. In both cases, if the file does not exist a new 28 | file is created.} 29 | 30 | \item{num_threads}{Number of threads to use when reading and materializing 31 | vectors. If your data contains newlines within fields the parser will 32 | automatically be forced to use a single thread only.} 33 | } 34 | \description{ 35 | Write lines to a file 36 | } 37 | -------------------------------------------------------------------------------- /revdep/.gitignore: -------------------------------------------------------------------------------- 1 | checks 2 | library 3 | checks.noindex 4 | library.noindex 5 | data.sqlite 6 | *.html 7 | cloud.noindex 8 | -------------------------------------------------------------------------------- /revdep/README.md: -------------------------------------------------------------------------------- 1 | # Revdeps 2 | 3 | -------------------------------------------------------------------------------- /revdep/cran.md: -------------------------------------------------------------------------------- 1 | ## revdepcheck results 2 | 3 | We checked 40 reverse dependencies, comparing R CMD check results across CRAN and dev versions of this package. 4 | 5 | * We saw 0 new problems 6 | * We failed to check 0 packages 7 | 8 | -------------------------------------------------------------------------------- /revdep/email.yml: -------------------------------------------------------------------------------- 1 | release_date: ??? 2 | rel_release_date: ??? 3 | my_news_url: ??? 4 | release_version: ??? 5 | release_details: ??? 6 | -------------------------------------------------------------------------------- /revdep/failures.md: -------------------------------------------------------------------------------- 1 | *Wow, no problems at all. :)* -------------------------------------------------------------------------------- /revdep/problems.md: -------------------------------------------------------------------------------- 1 | *Wow, no problems at all. :)* -------------------------------------------------------------------------------- /src/.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | BasedOnStyle: LLVM 4 | Standard: Cpp11 5 | AlignAfterOpenBracket: AlwaysBreak 6 | AllowShortBlocksOnASingleLine: false 7 | BinPackArguments: false 8 | BinPackParameters: false 9 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 10 | PointerAlignment: Left 11 | --- 12 | -------------------------------------------------------------------------------- /src/.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.so 3 | *.dll 4 | -------------------------------------------------------------------------------- /src/Iconv.h: -------------------------------------------------------------------------------- 1 | #ifndef READ_ICONV_H_ 2 | #define READ_ICONV_H_ 3 | 4 | #include 5 | 6 | #include "R_ext/Riconv.h" 7 | #include 8 | #include 9 | 10 | class Iconv { 11 | void* cd_; 12 | std::string buffer_; 13 | 14 | public: 15 | Iconv(const std::string& from, const std::string& to = "UTF-8"); 16 | virtual ~Iconv(); 17 | 18 | SEXP makeSEXP(const char* start, const char* end, bool hasNull = true); 19 | std::string makeString(const char* start, const char* end); 20 | 21 | private: 22 | // Returns number of characters in buffer 23 | size_t convert(const char* start, const char* end); 24 | }; 25 | 26 | #endif 27 | -------------------------------------------------------------------------------- /src/LocaleInfo.cpp: -------------------------------------------------------------------------------- 1 | #include "cpp11/as.hpp" 2 | #include "cpp11/list.hpp" 3 | #include "cpp11/strings.hpp" 4 | #include 5 | #include 6 | 7 | #include "LocaleInfo.h" 8 | 9 | LocaleInfo::LocaleInfo(const cpp11::list& x) 10 | : encoding_(cpp11::as_cpp(x["encoding"])), 11 | encoder_(Iconv(encoding_)) { 12 | std::string klass = cpp11::as_cpp(x.attr("class")); 13 | if (klass != "locale") 14 | cpp11::stop("Invalid input: must be of class locale"); 15 | 16 | cpp11::list date_names(x["date_names"]); 17 | mon_ = cpp11::as_cpp>(date_names["mon"]); 18 | monAb_ = cpp11::as_cpp>(date_names["mon_ab"]); 19 | day_ = cpp11::as_cpp>(date_names["day"]); 20 | dayAb_ = cpp11::as_cpp>(date_names["day_ab"]); 21 | amPm_ = cpp11::as_cpp>(date_names["am_pm"]); 22 | 23 | decimalMark_ = cpp11::as_cpp(x["decimal_mark"]); 24 | groupingMark_ = cpp11::as_cpp(x["grouping_mark"]); 25 | 26 | dateFormat_ = cpp11::as_cpp(x["date_format"]); 27 | timeFormat_ = cpp11::as_cpp(x["time_format"]); 28 | 29 | tz_ = cpp11::as_cpp(x["tz"]); 30 | } 31 | -------------------------------------------------------------------------------- /src/LocaleInfo.h: -------------------------------------------------------------------------------- 1 | #ifndef FASTREAD_LOCALINFO 2 | #define FASTREAD_LOCALINFO 3 | 4 | #include "Iconv.h" 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | class LocaleInfo { 11 | 12 | public: 13 | // LC_TIME 14 | std::vector mon_, monAb_, day_, dayAb_, amPm_; 15 | std::string dateFormat_, timeFormat_; 16 | 17 | // LC_NUMERIC 18 | std::string decimalMark_, groupingMark_; 19 | 20 | // LC_MISC 21 | std::string tz_; 22 | std::string encoding_; 23 | Iconv encoder_; 24 | 25 | LocaleInfo(const cpp11::list&); 26 | }; 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /src/Makevars: -------------------------------------------------------------------------------- 1 | PKG_CPPFLAGS=-Imio/include -DWIN32_LEAN_AND_MEAN -Ispdlog/include -DFMT_HEADER_ONLY 2 | -------------------------------------------------------------------------------- /src/altrep.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | #if R_VERSION >= R_Version(3, 5, 0) 6 | #define HAS_ALTREP 7 | #endif 8 | 9 | #ifdef HAS_ALTREP 10 | #if R_VERSION < R_Version(3, 6, 0) 11 | 12 | // workaround because R's not so conveniently uses `class` 13 | // as a variable name, and C++ is not happy about that 14 | // 15 | // SEXP R_new_altrep(R_altrep_class_t class, SEXP data1, SEXP data2); 16 | // 17 | 18 | // clang-format off 19 | #ifdef __clang__ 20 | # pragma clang diagnostic push 21 | # pragma clang diagnostic ignored "-Wkeyword-macro" 22 | #define class klass 23 | # pragma clang diagnostic pop 24 | #else 25 | #define class klass 26 | #endif 27 | // clang-format on 28 | 29 | // Because functions declared in have C linkage 30 | extern "C" { 31 | #include 32 | } 33 | 34 | // undo the workaround 35 | #undef class 36 | 37 | #else 38 | extern "C" { 39 | #include 40 | } 41 | #endif 42 | #endif 43 | -------------------------------------------------------------------------------- /src/delimited_index_connection.h: -------------------------------------------------------------------------------- 1 | #include "delimited_index.h" 2 | 3 | namespace vroom { 4 | 5 | class delimited_index_connection : public delimited_index { 6 | std::string filename_; 7 | 8 | public: 9 | delimited_index_connection( 10 | SEXP in, 11 | const char* delim, 12 | const char quote, 13 | const bool trim_ws, 14 | const bool escape_double, 15 | const bool escape_backslash, 16 | const bool has_header, 17 | const size_t skip, 18 | const size_t n_max, 19 | const char* comment, 20 | const bool skip_empty_rows, 21 | const std::shared_ptr errors, 22 | const size_t chunk_size, 23 | const bool progress); 24 | 25 | ~delimited_index_connection() { remove(filename_.c_str()); } 26 | }; 27 | 28 | } // namespace vroom 29 | -------------------------------------------------------------------------------- /src/fixed_width_index_connection.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "fixed_width_index.h" 4 | 5 | namespace vroom { 6 | 7 | class fixed_width_index_connection : public fixed_width_index { 8 | std::string filename_; 9 | 10 | public: 11 | fixed_width_index_connection( 12 | SEXP in, 13 | std::vector col_starts, 14 | std::vector col_ends, 15 | bool trim_ws, 16 | const size_t skip, 17 | const char* comment, 18 | const bool skip_empty_rows, 19 | const size_t n_max, 20 | const bool progress, 21 | const size_t chunk_size); 22 | 23 | ~fixed_width_index_connection() { remove(filename_.c_str()); } 24 | }; 25 | 26 | } // namespace vroom 27 | -------------------------------------------------------------------------------- /src/gen.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | 6 | [[cpp11::register]] cpp11::strings gen_character_( 7 | int n, 8 | int min, 9 | int max, 10 | std::string values, 11 | uint32_t seed, 12 | uint32_t seed2) { 13 | std::mt19937 gen1(seed); 14 | std::mt19937 gen2(seed2); 15 | 16 | cpp11::writable::strings out(n); 17 | 18 | std::uniform_int_distribution<> char_dis(0, values.length() - 1); 19 | 20 | std::uniform_int_distribution<> len_dis(min, max); 21 | 22 | for (int i = 0; i < n; ++i) { 23 | std::string str; 24 | auto str_len = len_dis(gen1); 25 | for (int j = 0; j < str_len; ++j) { 26 | auto c = char_dis(gen2); 27 | str.push_back(values[c]); 28 | } 29 | out[i] = str.c_str(); 30 | } 31 | 32 | return out; 33 | } 34 | -------------------------------------------------------------------------------- /src/mio/.gitignore: -------------------------------------------------------------------------------- 1 | test/** 2 | !test/test.cpp 3 | !test/example.cpp 4 | !test/CMakeLists.txt 5 | build/ 6 | -------------------------------------------------------------------------------- /src/mio/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 https://github.com/mandreyel/ 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/mio/cmake/WinApiLevels.cmake: -------------------------------------------------------------------------------- 1 | add_library(mio_full_winapi INTERFACE) 2 | target_link_libraries(mio_full_winapi 3 | INTERFACE mio_base 4 | ) 5 | add_library(mio::mio_full_winapi ALIAS mio_full_winapi) 6 | 7 | add_library(mio INTERFACE) 8 | target_link_libraries(mio 9 | INTERFACE mio_full_winapi 10 | ) 11 | target_compile_definitions(mio 12 | INTERFACE WIN32_LEAN_AND_MEAN NOMINMAX 13 | ) 14 | install(TARGETS mio_full_winapi EXPORT mioConfig) 15 | -------------------------------------------------------------------------------- /src/mio/include/mio/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # While not strictly necessary to specify header files as target sources, 3 | # doing so populates these files in the source listing when CMake is used 4 | # to generate XCode and Visual Studios projects 5 | # 6 | target_sources(mio_base INTERFACE 7 | $ 11 | $) 15 | 16 | add_subdirectory(detail) 17 | -------------------------------------------------------------------------------- /src/mio/include/mio/detail/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # iff mio is the highest level project, include the implementation 3 | # detail files in the source listing for CMake-generated IDE projects 4 | # 5 | if(NOT subproject) 6 | target_sources(mio_base INTERFACE 7 | $) 10 | endif() 11 | -------------------------------------------------------------------------------- /src/mio/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(mio.test test.cpp) 2 | target_link_libraries(mio.test PRIVATE mio::mio) 3 | add_test(NAME mio.test COMMAND mio.test) 4 | 5 | if(WIN32) 6 | add_executable(mio.unicode.test test.cpp) 7 | target_link_libraries(mio.unicode.test PRIVATE mio::mio) 8 | target_compile_definitions(mio.unicode.test PRIVATE UNICODE) 9 | add_test(NAME mio.unicode.test COMMAND mio.test) 10 | 11 | add_executable(mio.fullwinapi.test test.cpp) 12 | target_link_libraries(mio.fullwinapi.test 13 | PRIVATE mio::mio_full_winapi) 14 | add_test(NAME mio.fullwinapi.test COMMAND mio.test) 15 | endif() 16 | -------------------------------------------------------------------------------- /src/mio/third_party/LICENSE.md: -------------------------------------------------------------------------------- 1 | amalgamate.py - Amalgamate C source and header files 2 | Copyright (c) 2012, Erik Edlund 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of Erik Edlund, nor the names of its contributors may 15 | be used to endorse or promote products derived from this software without 16 | specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | -------------------------------------------------------------------------------- /src/mio/third_party/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "project": "Cross-platform C++11 header-only library for memory mapped file IO", 3 | "target": "../single_include/mio/mio.hpp", 4 | "sources": [ 5 | "../include/mio/mmap.hpp", 6 | "../include/mio/page.hpp", 7 | "../include/mio/shared_mmap.hpp" 8 | ], 9 | "include_paths": ["../include"] 10 | } 11 | 12 | -------------------------------------------------------------------------------- /src/spdlog/.gitattributes: -------------------------------------------------------------------------------- 1 | * text=false 2 | -------------------------------------------------------------------------------- /src/spdlog/.gitignore: -------------------------------------------------------------------------------- 1 | # Auto generated files 2 | build/* 3 | *.slo 4 | *.lo 5 | *.o 6 | *.obj 7 | *.suo 8 | *.tlog 9 | *.ilk 10 | *.log 11 | *.pdb 12 | *.idb 13 | *.iobj 14 | *.ipdb 15 | *.opensdf 16 | *.sdf 17 | 18 | # Compiled Dynamic libraries 19 | *.so 20 | *.dylib 21 | *.dll 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | 34 | # Codelite 35 | .codelite 36 | 37 | # KDevelop 38 | *.kdev4 39 | 40 | # .orig files 41 | *.orig 42 | 43 | # example files 44 | example/* 45 | !example/example.cpp 46 | !example/bench.cpp 47 | !example/utils.h 48 | !example/Makefile* 49 | !example/example.sln 50 | !example/example.vcxproj 51 | !example/CMakeLists.txt 52 | !example/meson.build 53 | !example/multisink.cpp 54 | !example/jni 55 | 56 | # generated files 57 | generated 58 | 59 | # Cmake 60 | CMakeCache.txt 61 | CMakeFiles 62 | CMakeScripts 63 | Makefile 64 | cmake_install.cmake 65 | install_manifest.txt 66 | /tests/tests.VC.VC.opendb 67 | /tests/tests.VC.db 68 | /tests/tests 69 | /tests/logs/* 70 | 71 | # idea 72 | .idea/ 73 | cmake-build-*/ 74 | *.db 75 | *.ipch 76 | *.filters 77 | *.db-wal 78 | *.opendb 79 | *.db-shm 80 | *.vcxproj 81 | *.tcl 82 | *.user 83 | *.sln 84 | -------------------------------------------------------------------------------- /src/spdlog/INSTALL: -------------------------------------------------------------------------------- 1 | Header only version: 2 | ================================================================== 3 | Just copy the files to your build tree and use a C++11 compiler. 4 | Or use CMake: 5 | add_executable(example_header_only example.cpp) 6 | target_link_libraries(example_header_only spdlog::spdlog_header_only) 7 | 8 | 9 | Compiled library version: 10 | ================================================================== 11 | CMake: 12 | add_executable(example example.cpp) 13 | target_link_libraries(example spdlog::spdlog) 14 | 15 | Or copy src/spdlog.cpp to your build tree and pass the -DSPDLOG_COMPILED_LIB to the compiler. 16 | 17 | Tested on: 18 | gcc 4.8.1 and above 19 | clang 3.5 20 | Visual Studio 2013 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/spdlog/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Gabi Melman. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | 23 | -- NOTE: Third party dependency used by this software -- 24 | This software depends on the fmt lib (MIT License), 25 | and users must comply to its license: https://github.com/fmtlib/fmt/blob/master/LICENSE.rst 26 | 27 | -------------------------------------------------------------------------------- /src/spdlog/appveyor.yml: -------------------------------------------------------------------------------- 1 | version: 1.0.{build} 2 | image: Visual Studio 2017 3 | environment: 4 | matrix: 5 | - GENERATOR: '"Visual Studio 14 2015"' 6 | BUILD_TYPE: Debug 7 | WCHAR: 'OFF' 8 | BUILD_SHARED: 'OFF' 9 | - GENERATOR: '"Visual Studio 14 2015"' 10 | BUILD_TYPE: Release 11 | WCHAR: 'ON' 12 | BUILD_SHARED: 'OFF' 13 | - GENERATOR: '"Visual Studio 14 2015 Win64"' 14 | BUILD_TYPE: Debug 15 | WCHAR: 'ON' 16 | BUILD_SHARED: 'OFF' 17 | - GENERATOR: '"Visual Studio 14 2015 Win64"' 18 | BUILD_TYPE: Release 19 | WCHAR: 'ON' 20 | BUILD_SHARED: 'OFF' 21 | - GENERATOR: '"Visual Studio 15 2017 Win64"' 22 | BUILD_TYPE: Debug 23 | WCHAR: 'ON' 24 | BUILD_SHARED: 'OFF' 25 | - GENERATOR: '"Visual Studio 15 2017 Win64"' 26 | BUILD_TYPE: Release 27 | WCHAR: 'OFF' 28 | BUILD_SHARED: 'OFF' 29 | - GENERATOR: '"Visual Studio 15 2017 Win64"' 30 | BUILD_TYPE: Release 31 | WCHAR: 'OFF' 32 | BUILD_SHARED: 'ON' 33 | build_script: 34 | - cmd: >- 35 | set 36 | 37 | mkdir build 38 | 39 | cd build 40 | 41 | set PATH=%PATH%;C:\Program Files\Git\usr\bin 42 | 43 | cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=%BUILD_TYPE% -DSPDLOG_WCHAR_SUPPORT=%WCHAR% -DSPDLOG_BUILD_SHARED=%BUILD_SHARED% -DSPDLOG_BUILD_EXAMPLE=ON -DSPDLOG_BUILD_EXAMPLE_HO=ON -DSPDLOG_BUILD_TESTS=ON -DSPDLOG_BUILD_TESTS_HO=OFF -DSPDLOG_BUILD_WARNINGS=ON 44 | 45 | cmake --build . --config %BUILD_TYPE% 46 | 47 | before_test: 48 | - set PATH=%PATH%;C:\projects\spdlog\build\%BUILD_TYPE% 49 | 50 | test_script: 51 | - C:\projects\spdlog\build\tests\%BUILD_TYPE%\spdlog-utests.exe 52 | -------------------------------------------------------------------------------- /src/spdlog/bench/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright(c) 2019 spdlog authors Distributed under the MIT License (http://opensource.org/licenses/MIT) 2 | 3 | cmake_minimum_required(VERSION 3.10) 4 | project(spdlog_bench CXX) 5 | 6 | if(NOT TARGET spdlog) 7 | # Stand-alone build 8 | find_package(spdlog CONFIG REQUIRED) 9 | endif() 10 | 11 | find_package(Threads REQUIRED) 12 | find_package(benchmark CONFIG) 13 | if (NOT benchmark_FOUND) 14 | message(STATUS "Using CMake Version ${CMAKE_VERSION}") 15 | if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.11.0") 16 | # User can fetch googlebenchmark 17 | message(STATUS "Downloading GoogleBenchmark") 18 | include(FetchContent) 19 | set(BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE INTERNAL "") 20 | # Do not build and run googlebenchmark tests 21 | FetchContent_Declare(googlebenchmark 22 | GIT_REPOSITORY https://github.com/google/benchmark.git 23 | GIT_TAG v1.5.2) 24 | 25 | FetchContent_MakeAvailable(googlebenchmark) 26 | else() 27 | message(FATAL_ERROR "GoogleBenchmark is missing. Use CMake >= 3.11 or download it") 28 | endif() 29 | endif() 30 | 31 | add_executable(bench bench.cpp) 32 | spdlog_enable_warnings(bench) 33 | target_link_libraries(bench PRIVATE spdlog::spdlog) 34 | 35 | add_executable(async_bench async_bench.cpp) 36 | target_link_libraries(async_bench PRIVATE spdlog::spdlog) 37 | 38 | add_executable(latency latency.cpp) 39 | target_link_libraries(latency PRIVATE benchmark::benchmark spdlog::spdlog) 40 | 41 | add_executable(formatter-bench formatter-bench.cpp) 42 | target_link_libraries(formatter-bench PRIVATE benchmark::benchmark spdlog::spdlog) 43 | -------------------------------------------------------------------------------- /src/spdlog/bench/utils.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | namespace utils { 13 | 14 | template 15 | inline std::string format(const T &value) 16 | { 17 | static std::locale loc(""); 18 | std::stringstream ss; 19 | ss.imbue(loc); 20 | ss << value; 21 | return ss.str(); 22 | } 23 | 24 | template<> 25 | inline std::string format(const double &value) 26 | { 27 | static std::locale loc(""); 28 | std::stringstream ss; 29 | ss.imbue(loc); 30 | ss << std::fixed << std::setprecision(1) << value; 31 | return ss.str(); 32 | } 33 | 34 | } // namespace utils 35 | -------------------------------------------------------------------------------- /src/spdlog/cmake/ide.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------------------- 2 | # IDE support for headers 3 | # --------------------------------------------------------------------------------------- 4 | set(SPDLOG_HEADERS_DIR "${CMAKE_CURRENT_LIST_DIR}/../include") 5 | 6 | file(GLOB SPDLOG_TOP_HEADERS "${SPDLOG_HEADERS_DIR}/spdlog/*.h") 7 | file(GLOB SPDLOG_DETAILS_HEADERS "${SPDLOG_HEADERS_DIR}/spdlog/details/*.h") 8 | file(GLOB SPDLOG_SINKS_HEADERS "${SPDLOG_HEADERS_DIR}/spdlog/sinks/*.h") 9 | file(GLOB SPDLOG_FMT_HEADERS "${SPDLOG_HEADERS_DIR}/spdlog/fmt/*.h") 10 | file(GLOB SPDLOG_FMT_BUNDELED_HEADERS "${SPDLOG_HEADERS_DIR}/spdlog/fmt/bundled/*.h") 11 | set(SPDLOG_ALL_HEADERS ${SPDLOG_TOP_HEADERS} ${SPDLOG_DETAILS_HEADERS} ${SPDLOG_SINKS_HEADERS} ${SPDLOG_FMT_HEADERS} 12 | ${SPDLOG_FMT_BUNDELED_HEADERS}) 13 | 14 | source_group("Header Files\\spdlog" FILES ${SPDLOG_TOP_HEADERS}) 15 | source_group("Header Files\\spdlog\\details" FILES ${SPDLOG_DETAILS_HEADERS}) 16 | source_group("Header Files\\spdlog\\sinks" FILES ${SPDLOG_SINKS_HEADERS}) 17 | source_group("Header Files\\spdlog\\fmt" FILES ${SPDLOG_FMT_HEADERS}) 18 | source_group("Header Files\\spdlog\\fmt\\bundled\\" FILES ${SPDLOG_FMT_BUNDELED_HEADERS}) 19 | -------------------------------------------------------------------------------- /src/spdlog/cmake/spdlog.pc.in: -------------------------------------------------------------------------------- 1 | prefix=@CMAKE_INSTALL_PREFIX@ 2 | exec_prefix=${prefix} 3 | includedir=${prefix}/include 4 | libdir=${exec_prefix}/@CMAKE_INSTALL_LIBDIR@ 5 | 6 | Name: lib@PROJECT_NAME@ 7 | Description: Fast C++ logging library. 8 | URL: https://github.com/gabime/@PROJECT_NAME@ 9 | Version: @SPDLOG_VERSION@ 10 | CFlags: -I${includedir} @PKG_CONFIG_DEFINES@ 11 | Libs: -L${libdir} -lspdlog -pthread 12 | Requires: @PKG_CONFIG_REQUIRES@ 13 | 14 | -------------------------------------------------------------------------------- /src/spdlog/cmake/spdlogConfig.cmake.in: -------------------------------------------------------------------------------- 1 | # Copyright(c) 2019 spdlog authors 2 | # Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | find_package(Threads REQUIRED) 5 | 6 | set(SPDLOG_FMT_EXTERNAL @SPDLOG_FMT_EXTERNAL@) 7 | set(config_targets_file @config_targets_file@) 8 | 9 | if(SPDLOG_FMT_EXTERNAL) 10 | include(CMakeFindDependencyMacro) 11 | find_dependency(fmt CONFIG) 12 | endif() 13 | 14 | 15 | include("${CMAKE_CURRENT_LIST_DIR}/${config_targets_file}") 16 | -------------------------------------------------------------------------------- /src/spdlog/cmake/version.rc.in: -------------------------------------------------------------------------------- 1 | #define APSTUDIO_READONLY_SYMBOLS 2 | #include 3 | #undef APSTUDIO_READONLY_SYMBOLS 4 | 5 | LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US 6 | 7 | 8 | VS_VERSION_INFO VERSIONINFO 9 | FILEVERSION @SPDLOG_VERSION_MAJOR@,@SPDLOG_VERSION_MINOR@,@SPDLOG_VERSION_PATCH@,0 10 | PRODUCTVERSION @SPDLOG_VERSION_MAJOR@,@SPDLOG_VERSION_MINOR@,@SPDLOG_VERSION_PATCH@,0 11 | FILEFLAGSMASK 0x3fL 12 | #ifdef _DEBUG 13 | FILEFLAGS 0x1L 14 | #else 15 | FILEFLAGS 0x0L 16 | #endif 17 | FILEOS 0x40004L 18 | FILETYPE 0x2L 19 | FILESUBTYPE 0x0L 20 | BEGIN 21 | BLOCK "StringFileInfo" 22 | BEGIN 23 | BLOCK "040904b0" 24 | BEGIN 25 | VALUE "FileDescription", "spdlog dll\0" 26 | VALUE "FileVersion", "@SPDLOG_VERSION@.0\0" 27 | VALUE "InternalName", "spdlog.dll\0" 28 | VALUE "LegalCopyright", "Copyright (C) spdlog\0" 29 | VALUE "ProductName", "spdlog\0" 30 | VALUE "ProductVersion", "@SPDLOG_VERSION@.0\0" 31 | END 32 | END 33 | BLOCK "VarFileInfo" 34 | BEGIN 35 | VALUE "Translation", 0x409, 1200 36 | END 37 | END 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /src/spdlog/example/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright(c) 2019 spdlog authors Distributed under the MIT License (http://opensource.org/licenses/MIT) 2 | 3 | cmake_minimum_required(VERSION 3.10) 4 | project(spdlog_examples CXX) 5 | 6 | if(NOT TARGET spdlog) 7 | # Stand-alone build 8 | find_package(spdlog REQUIRED) 9 | endif() 10 | 11 | # --------------------------------------------------------------------------------------- 12 | # Example of using pre-compiled library 13 | # --------------------------------------------------------------------------------------- 14 | add_executable(example example.cpp) 15 | target_link_libraries(example PRIVATE spdlog::spdlog) 16 | 17 | # --------------------------------------------------------------------------------------- 18 | # Example of using header-only library 19 | # --------------------------------------------------------------------------------------- 20 | if(SPDLOG_BUILD_EXAMPLE_HO) 21 | add_executable(example_header_only example.cpp) 22 | target_link_libraries(example_header_only PRIVATE spdlog::spdlog_header_only) 23 | endif() 24 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/cfg/argv.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | #include 6 | #include 7 | 8 | // 9 | // Init log levels using each argv entry that starts with "SPDLOG_LEVEL=" 10 | // 11 | // set all loggers to debug level: 12 | // example.exe "SPDLOG_LEVEL=debug" 13 | 14 | // set logger1 to trace level 15 | // example.exe "SPDLOG_LEVEL=logger1=trace" 16 | 17 | // turn off all logging except for logger1 and logger2: 18 | // example.exe "SPDLOG_LEVEL=off,logger1=debug,logger2=info" 19 | 20 | namespace spdlog { 21 | namespace cfg { 22 | 23 | // search for SPDLOG_LEVEL= in the args and use it to init the levels 24 | inline void load_argv_levels(int argc, const char **argv) 25 | { 26 | const std::string spdlog_level_prefix = "SPDLOG_LEVEL="; 27 | for (int i = 1; i < argc; i++) 28 | { 29 | std::string arg = argv[i]; 30 | if (arg.find(spdlog_level_prefix) == 0) 31 | { 32 | auto levels_string = arg.substr(spdlog_level_prefix.size()); 33 | helpers::load_levels(levels_string); 34 | } 35 | } 36 | } 37 | 38 | inline void load_argv_levels(int argc, char **argv) 39 | { 40 | load_argv_levels(argc, const_cast(argv)); 41 | } 42 | 43 | } // namespace cfg 44 | } // namespace spdlog 45 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/cfg/env.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | #include 6 | #include 7 | #include 8 | 9 | // 10 | // Init levels and patterns from env variables SPDLOG_LEVEL 11 | // Inspired from Rust's "env_logger" crate (https://crates.io/crates/env_logger). 12 | // Note - fallback to "info" level on unrecognized levels 13 | // 14 | // Examples: 15 | // 16 | // set global level to debug: 17 | // export SPDLOG_LEVEL=debug 18 | // 19 | // turn off all logging except for logger1: 20 | // export SPDLOG_LEVEL="*=off,logger1=debug" 21 | // 22 | 23 | // turn off all logging except for logger1 and logger2: 24 | // export SPDLOG_LEVEL="off,logger1=debug,logger2=info" 25 | 26 | namespace spdlog { 27 | namespace cfg { 28 | inline void load_env_levels() 29 | { 30 | auto env_val = details::os::getenv("SPDLOG_LEVEL"); 31 | if (!env_val.empty()) 32 | { 33 | helpers::load_levels(env_val); 34 | } 35 | } 36 | 37 | } // namespace cfg 38 | } // namespace spdlog 39 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/cfg/helpers.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | namespace spdlog { 10 | namespace cfg { 11 | namespace helpers { 12 | // 13 | // Init levels from given string 14 | // 15 | // Examples: 16 | // 17 | // set global level to debug: "debug" 18 | // turn off all logging except for logger1: "off,logger1=debug" 19 | // turn off all logging except for logger1 and logger2: "off,logger1=debug,logger2=info" 20 | // 21 | SPDLOG_API void load_levels(const std::string &txt); 22 | } // namespace helpers 23 | 24 | } // namespace cfg 25 | } // namespace spdlog 26 | 27 | #ifdef SPDLOG_HEADER_ONLY 28 | #include "helpers-inl.h" 29 | #endif // SPDLOG_HEADER_ONLY 30 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/details/backtracer.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | // Store log messages in circular buffer. 14 | // Useful for storing debug data in case of error/warning happens. 15 | 16 | namespace spdlog { 17 | namespace details { 18 | class SPDLOG_API backtracer 19 | { 20 | mutable std::mutex mutex_; 21 | std::atomic enabled_{false}; 22 | circular_q messages_; 23 | 24 | public: 25 | backtracer() = default; 26 | backtracer(const backtracer &other); 27 | 28 | backtracer(backtracer &&other) SPDLOG_NOEXCEPT; 29 | backtracer &operator=(backtracer other); 30 | 31 | void enable(size_t size); 32 | void disable(); 33 | bool enabled() const; 34 | void push_back(const log_msg &msg); 35 | 36 | // pop all items in the q and apply the given fun on each of them. 37 | void foreach_pop(std::function fun); 38 | }; 39 | 40 | } // namespace details 41 | } // namespace spdlog 42 | 43 | #ifdef SPDLOG_HEADER_ONLY 44 | #include "backtracer-inl.h" 45 | #endif -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/details/console_globals.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | namespace spdlog { 10 | namespace details { 11 | 12 | struct console_mutex 13 | { 14 | using mutex_t = std::mutex; 15 | static mutex_t &mutex() 16 | { 17 | static mutex_t s_mutex; 18 | return s_mutex; 19 | } 20 | }; 21 | 22 | struct console_nullmutex 23 | { 24 | using mutex_t = null_mutex; 25 | static mutex_t &mutex() 26 | { 27 | static mutex_t s_mutex; 28 | return s_mutex; 29 | } 30 | }; 31 | } // namespace details 32 | } // namespace spdlog 33 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/details/log_msg-inl.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | #ifndef SPDLOG_HEADER_ONLY 7 | #include 8 | #endif 9 | 10 | #include 11 | 12 | namespace spdlog { 13 | namespace details { 14 | 15 | SPDLOG_INLINE log_msg::log_msg(spdlog::log_clock::time_point log_time, spdlog::source_loc loc, string_view_t a_logger_name, 16 | spdlog::level::level_enum lvl, spdlog::string_view_t msg) 17 | : logger_name(a_logger_name) 18 | , level(lvl) 19 | , time(log_time) 20 | #ifndef SPDLOG_NO_THREAD_ID 21 | , thread_id(os::thread_id()) 22 | #endif 23 | , source(loc) 24 | , payload(msg) 25 | {} 26 | 27 | SPDLOG_INLINE log_msg::log_msg( 28 | spdlog::source_loc loc, string_view_t a_logger_name, spdlog::level::level_enum lvl, spdlog::string_view_t msg) 29 | : log_msg(os::now(), loc, a_logger_name, lvl, msg) 30 | {} 31 | 32 | SPDLOG_INLINE log_msg::log_msg(string_view_t a_logger_name, spdlog::level::level_enum lvl, spdlog::string_view_t msg) 33 | : log_msg(os::now(), source_loc{}, a_logger_name, lvl, msg) 34 | {} 35 | 36 | } // namespace details 37 | } // namespace spdlog 38 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/details/log_msg.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | namespace spdlog { 10 | namespace details { 11 | struct SPDLOG_API log_msg 12 | { 13 | log_msg() = default; 14 | log_msg(log_clock::time_point log_time, source_loc loc, string_view_t logger_name, level::level_enum lvl, string_view_t msg); 15 | log_msg(source_loc loc, string_view_t logger_name, level::level_enum lvl, string_view_t msg); 16 | log_msg(string_view_t logger_name, level::level_enum lvl, string_view_t msg); 17 | log_msg(const log_msg &other) = default; 18 | 19 | string_view_t logger_name; 20 | level::level_enum level{level::off}; 21 | log_clock::time_point time; 22 | size_t thread_id{0}; 23 | 24 | // wrapping the formatted text with color (updated by pattern_formatter). 25 | mutable size_t color_range_start{0}; 26 | mutable size_t color_range_end{0}; 27 | 28 | source_loc source; 29 | string_view_t payload; 30 | }; 31 | } // namespace details 32 | } // namespace spdlog 33 | 34 | #ifdef SPDLOG_HEADER_ONLY 35 | #include "log_msg-inl.h" 36 | #endif 37 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/details/log_msg_buffer.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | #include 7 | 8 | namespace spdlog { 9 | namespace details { 10 | 11 | // Extend log_msg with internal buffer to store its payload. 12 | // This is needed since log_msg holds string_views that points to stack data. 13 | 14 | class SPDLOG_API log_msg_buffer : public log_msg 15 | { 16 | memory_buf_t buffer; 17 | void update_string_views(); 18 | 19 | public: 20 | log_msg_buffer() = default; 21 | explicit log_msg_buffer(const log_msg &orig_msg); 22 | log_msg_buffer(const log_msg_buffer &other); 23 | log_msg_buffer(log_msg_buffer &&other) SPDLOG_NOEXCEPT; 24 | log_msg_buffer &operator=(const log_msg_buffer &other); 25 | log_msg_buffer &operator=(log_msg_buffer &&other) SPDLOG_NOEXCEPT; 26 | }; 27 | 28 | } // namespace details 29 | } // namespace spdlog 30 | 31 | #ifdef SPDLOG_HEADER_ONLY 32 | #include "log_msg_buffer-inl.h" 33 | #endif 34 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/details/null_mutex.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | #include 7 | #include 8 | // null, no cost dummy "mutex" and dummy "atomic" int 9 | 10 | namespace spdlog { 11 | namespace details { 12 | struct null_mutex 13 | { 14 | void lock() const {} 15 | void unlock() const {} 16 | bool try_lock() const 17 | { 18 | return true; 19 | } 20 | }; 21 | 22 | struct null_atomic_int 23 | { 24 | int value; 25 | null_atomic_int() = default; 26 | 27 | explicit null_atomic_int(int new_value) 28 | : value(new_value) 29 | {} 30 | 31 | int load(std::memory_order = std::memory_order_relaxed) const 32 | { 33 | return value; 34 | } 35 | 36 | void store(int new_value, std::memory_order = std::memory_order_relaxed) 37 | { 38 | value = new_value; 39 | } 40 | 41 | int exchange(int new_value, std::memory_order = std::memory_order_relaxed) 42 | { 43 | std::swap(new_value, value); 44 | return new_value; // return value before the call 45 | } 46 | }; 47 | 48 | } // namespace details 49 | } // namespace spdlog 50 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/details/periodic_worker-inl.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | #ifndef SPDLOG_HEADER_ONLY 7 | #include 8 | #endif 9 | 10 | namespace spdlog { 11 | namespace details { 12 | 13 | SPDLOG_INLINE periodic_worker::periodic_worker(const std::function &callback_fun, std::chrono::seconds interval) 14 | { 15 | active_ = (interval > std::chrono::seconds::zero()); 16 | if (!active_) 17 | { 18 | return; 19 | } 20 | 21 | worker_thread_ = std::thread([this, callback_fun, interval]() { 22 | for (;;) 23 | { 24 | std::unique_lock lock(this->mutex_); 25 | if (this->cv_.wait_for(lock, interval, [this] { return !this->active_; })) 26 | { 27 | return; // active_ == false, so exit this thread 28 | } 29 | callback_fun(); 30 | } 31 | }); 32 | } 33 | 34 | // stop the worker thread and join it 35 | SPDLOG_INLINE periodic_worker::~periodic_worker() 36 | { 37 | if (worker_thread_.joinable()) 38 | { 39 | { 40 | std::lock_guard lock(mutex_); 41 | active_ = false; 42 | } 43 | cv_.notify_one(); 44 | worker_thread_.join(); 45 | } 46 | } 47 | 48 | } // namespace details 49 | } // namespace spdlog 50 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/details/periodic_worker.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | // periodic worker thread - periodically executes the given callback function. 7 | // 8 | // RAII over the owned thread: 9 | // creates the thread on construction. 10 | // stops and joins the thread on destruction (if the thread is executing a callback, wait for it to finish first). 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | namespace spdlog { 18 | namespace details { 19 | 20 | class SPDLOG_API periodic_worker 21 | { 22 | public: 23 | periodic_worker(const std::function &callback_fun, std::chrono::seconds interval); 24 | periodic_worker(const periodic_worker &) = delete; 25 | periodic_worker &operator=(const periodic_worker &) = delete; 26 | // stop the worker thread and join it 27 | ~periodic_worker(); 28 | 29 | private: 30 | bool active_; 31 | std::thread worker_thread_; 32 | std::mutex mutex_; 33 | std::condition_variable cv_; 34 | }; 35 | } // namespace details 36 | } // namespace spdlog 37 | 38 | #ifdef SPDLOG_HEADER_ONLY 39 | #include "periodic_worker-inl.h" 40 | #endif 41 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/details/synchronous_factory.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | #include "registry.h" 7 | 8 | namespace spdlog { 9 | 10 | // Default logger factory- creates synchronous loggers 11 | class logger; 12 | 13 | struct synchronous_factory 14 | { 15 | template 16 | static std::shared_ptr create(std::string logger_name, SinkArgs &&...args) 17 | { 18 | auto sink = std::make_shared(std::forward(args)...); 19 | auto new_logger = std::make_shared(std::move(logger_name), std::move(sink)); 20 | details::registry::instance().initialize_logger(new_logger); 21 | return new_logger; 22 | } 23 | }; 24 | } // namespace spdlog -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/details/windows_include.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifndef NOMINMAX 4 | #define NOMINMAX // prevent windows redefining min/max 5 | #endif 6 | 7 | #ifndef WIN32_LEAN_AND_MEAN 8 | #define WIN32_LEAN_AND_MEAN 9 | #endif 10 | 11 | #include 12 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/fmt/bundled/LICENSE.rst: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 - present, Victor Zverovich 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | 22 | --- Optional exception to the license --- 23 | 24 | As an exception, if, as a result of your compiling your source code, portions 25 | of this Software are embedded into a machine-executable object form of such 26 | source code, you may redistribute such embedded portions in such object form 27 | without including the above copyright and permission notices. 28 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/fmt/bundled/posix.h: -------------------------------------------------------------------------------- 1 | #include "os.h" 2 | #warning "fmt/posix.h is deprecated; use fmt/os.h instead" 3 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/fmt/chrono.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2016 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | // 8 | // include bundled or external copy of fmtlib's chrono support 9 | // 10 | 11 | #if !defined(SPDLOG_FMT_EXTERNAL) 12 | #ifdef SPDLOG_HEADER_ONLY 13 | #ifndef FMT_HEADER_ONLY 14 | #define FMT_HEADER_ONLY 15 | #endif 16 | #endif 17 | #include 18 | #else 19 | #include 20 | #endif 21 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/fmt/fmt.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2016-2018 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | // 9 | // Include a bundled header-only copy of fmtlib or an external one. 10 | // By default spdlog include its own copy. 11 | // 12 | 13 | #if !defined(SPDLOG_FMT_EXTERNAL) 14 | #if !defined(SPDLOG_COMPILED_LIB) && !defined(FMT_HEADER_ONLY) 15 | #define FMT_HEADER_ONLY 16 | #endif 17 | #ifndef FMT_USE_WINDOWS_H 18 | #define FMT_USE_WINDOWS_H 0 19 | #endif 20 | // enable the 'n' flag in for backward compatibility with fmt 6.x 21 | #define FMT_DEPRECATED_N_SPECIFIER 22 | #include 23 | #include 24 | #else // SPDLOG_FMT_EXTERNAL is defined - use external fmtlib 25 | #include 26 | #include 27 | #endif -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/fmt/ostr.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2016 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | // 8 | // include bundled or external copy of fmtlib's ostream support 9 | // 10 | 11 | #if !defined(SPDLOG_FMT_EXTERNAL) 12 | #ifdef SPDLOG_HEADER_ONLY 13 | #ifndef FMT_HEADER_ONLY 14 | #define FMT_HEADER_ONLY 15 | #endif 16 | #endif 17 | #include 18 | #else 19 | #include 20 | #endif 21 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/formatter.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | namespace spdlog { 10 | 11 | class formatter 12 | { 13 | public: 14 | virtual ~formatter() = default; 15 | virtual void format(const details::log_msg &msg, memory_buf_t &dest) = 0; 16 | virtual std::unique_ptr clone() const = 0; 17 | }; 18 | } // namespace spdlog 19 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/fwd.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | namespace spdlog { 7 | class logger; 8 | class formatter; 9 | 10 | namespace sinks { 11 | class sink; 12 | } 13 | 14 | } // namespace spdlog 15 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/sinks/basic_file_sink-inl.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | #ifndef SPDLOG_HEADER_ONLY 7 | #include 8 | #endif 9 | 10 | #include 11 | #include 12 | 13 | namespace spdlog { 14 | namespace sinks { 15 | 16 | template 17 | SPDLOG_INLINE basic_file_sink::basic_file_sink(const filename_t &filename, bool truncate) 18 | { 19 | file_helper_.open(filename, truncate); 20 | } 21 | 22 | template 23 | SPDLOG_INLINE const filename_t &basic_file_sink::filename() const 24 | { 25 | return file_helper_.filename(); 26 | } 27 | 28 | template 29 | SPDLOG_INLINE void basic_file_sink::sink_it_(const details::log_msg &msg) 30 | { 31 | memory_buf_t formatted; 32 | base_sink::formatter_->format(msg, formatted); 33 | file_helper_.write(formatted); 34 | } 35 | 36 | template 37 | SPDLOG_INLINE void basic_file_sink::flush_() 38 | { 39 | file_helper_.flush(); 40 | } 41 | 42 | } // namespace sinks 43 | } // namespace spdlog 44 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/sinks/msvc_sink.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2016 Alexander Dalshov. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | #if defined(_WIN32) 7 | 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | 14 | 15 | // Avoid including windows.h (https://stackoverflow.com/a/30741042) 16 | extern "C" __declspec(dllimport) void __stdcall OutputDebugStringA(const char *lpOutputString); 17 | 18 | namespace spdlog { 19 | namespace sinks { 20 | /* 21 | * MSVC sink (logging using OutputDebugStringA) 22 | */ 23 | template 24 | class msvc_sink : public base_sink 25 | { 26 | public: 27 | msvc_sink() = default; 28 | 29 | protected: 30 | void sink_it_(const details::log_msg &msg) override 31 | { 32 | memory_buf_t formatted; 33 | base_sink::formatter_->format(msg, formatted); 34 | OutputDebugStringA(fmt::to_string(formatted).c_str()); 35 | } 36 | 37 | void flush_() override {} 38 | }; 39 | 40 | using msvc_sink_mt = msvc_sink; 41 | using msvc_sink_st = msvc_sink; 42 | 43 | using windebug_sink_mt = msvc_sink_mt; 44 | using windebug_sink_st = msvc_sink_st; 45 | 46 | } // namespace sinks 47 | } // namespace spdlog 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/sinks/null_sink.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | namespace spdlog { 13 | namespace sinks { 14 | 15 | template 16 | class null_sink : public base_sink 17 | { 18 | protected: 19 | void sink_it_(const details::log_msg &) override {} 20 | void flush_() override {} 21 | }; 22 | 23 | using null_sink_mt = null_sink; 24 | using null_sink_st = null_sink; 25 | 26 | } // namespace sinks 27 | 28 | template 29 | inline std::shared_ptr null_logger_mt(const std::string &logger_name) 30 | { 31 | auto null_logger = Factory::template create(logger_name); 32 | null_logger->set_level(level::off); 33 | return null_logger; 34 | } 35 | 36 | template 37 | inline std::shared_ptr null_logger_st(const std::string &logger_name) 38 | { 39 | auto null_logger = Factory::template create(logger_name); 40 | null_logger->set_level(level::off); 41 | return null_logger; 42 | } 43 | 44 | } // namespace spdlog 45 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/sinks/ostream_sink.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | 12 | namespace spdlog { 13 | namespace sinks { 14 | template 15 | class ostream_sink final : public base_sink 16 | { 17 | public: 18 | explicit ostream_sink(std::ostream &os, bool force_flush = false) 19 | : ostream_(os) 20 | , force_flush_(force_flush) 21 | {} 22 | ostream_sink(const ostream_sink &) = delete; 23 | ostream_sink &operator=(const ostream_sink &) = delete; 24 | 25 | protected: 26 | void sink_it_(const details::log_msg &msg) override 27 | { 28 | memory_buf_t formatted; 29 | base_sink::formatter_->format(msg, formatted); 30 | ostream_.write(formatted.data(), static_cast(formatted.size())); 31 | if (force_flush_) 32 | { 33 | ostream_.flush(); 34 | } 35 | } 36 | 37 | void flush_() override 38 | { 39 | ostream_.flush(); 40 | } 41 | 42 | std::ostream &ostream_; 43 | bool force_flush_; 44 | }; 45 | 46 | using ostream_sink_mt = ostream_sink; 47 | using ostream_sink_st = ostream_sink; 48 | 49 | } // namespace sinks 50 | } // namespace spdlog 51 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/sinks/sink-inl.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | #ifndef SPDLOG_HEADER_ONLY 7 | #include 8 | #endif 9 | 10 | #include 11 | 12 | SPDLOG_INLINE bool spdlog::sinks::sink::should_log(spdlog::level::level_enum msg_level) const 13 | { 14 | return msg_level >= level_.load(std::memory_order_relaxed); 15 | } 16 | 17 | SPDLOG_INLINE void spdlog::sinks::sink::set_level(level::level_enum log_level) 18 | { 19 | level_.store(log_level, std::memory_order_relaxed); 20 | } 21 | 22 | SPDLOG_INLINE spdlog::level::level_enum spdlog::sinks::sink::level() const 23 | { 24 | return static_cast(level_.load(std::memory_order_relaxed)); 25 | } 26 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/sinks/sink.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | namespace spdlog { 10 | 11 | namespace sinks { 12 | class SPDLOG_API sink 13 | { 14 | public: 15 | virtual ~sink() = default; 16 | virtual void log(const details::log_msg &msg) = 0; 17 | virtual void flush() = 0; 18 | virtual void set_pattern(const std::string &pattern) = 0; 19 | virtual void set_formatter(std::unique_ptr sink_formatter) = 0; 20 | 21 | void set_level(level::level_enum log_level); 22 | level::level_enum level() const; 23 | bool should_log(level::level_enum msg_level) const; 24 | 25 | protected: 26 | // sink log level - default is all 27 | level_t level_{level::trace}; 28 | }; 29 | 30 | } // namespace sinks 31 | } // namespace spdlog 32 | 33 | #ifdef SPDLOG_HEADER_ONLY 34 | #include "sink-inl.h" 35 | #endif 36 | -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/sinks/stdout_color_sinks-inl.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | #ifndef SPDLOG_HEADER_ONLY 7 | #include 8 | #endif 9 | 10 | #include 11 | #include 12 | 13 | namespace spdlog { 14 | 15 | template 16 | SPDLOG_INLINE std::shared_ptr stdout_color_mt(const std::string &logger_name, color_mode mode) 17 | { 18 | return Factory::template create(logger_name, mode); 19 | } 20 | 21 | template 22 | SPDLOG_INLINE std::shared_ptr stdout_color_st(const std::string &logger_name, color_mode mode) 23 | { 24 | return Factory::template create(logger_name, mode); 25 | } 26 | 27 | template 28 | SPDLOG_INLINE std::shared_ptr stderr_color_mt(const std::string &logger_name, color_mode mode) 29 | { 30 | return Factory::template create(logger_name, mode); 31 | } 32 | 33 | template 34 | SPDLOG_INLINE std::shared_ptr stderr_color_st(const std::string &logger_name, color_mode mode) 35 | { 36 | return Factory::template create(logger_name, mode); 37 | } 38 | } // namespace spdlog -------------------------------------------------------------------------------- /src/spdlog/include/spdlog/version.h: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #pragma once 5 | 6 | #define SPDLOG_VER_MAJOR 1 7 | #define SPDLOG_VER_MINOR 8 8 | #define SPDLOG_VER_PATCH 2 9 | 10 | #define SPDLOG_VERSION (SPDLOG_VER_MAJOR * 10000 + SPDLOG_VER_MINOR * 100 + SPDLOG_VER_PATCH) 11 | -------------------------------------------------------------------------------- /src/spdlog/scripts/extract_version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import re 5 | 6 | base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) 7 | config_h = os.path.join(base_path, 'include', 'spdlog', 'version.h') 8 | data = {'MAJOR': 0, 'MINOR': 0, 'PATCH': 0} 9 | reg = re.compile(r'^\s*#define\s+SPDLOG_VER_([A-Z]+)\s+([0-9]+).*$') 10 | 11 | with open(config_h, 'r') as fp: 12 | for l in fp: 13 | m = reg.match(l) 14 | if m: 15 | data[m.group(1)] = int(m.group(2)) 16 | 17 | print('{}.{}.{}'.format(data['MAJOR'], data['MINOR'], data['PATCH'])) 18 | -------------------------------------------------------------------------------- /src/spdlog/scripts/format.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd "$(dirname "$0")"/.. 4 | pwd 5 | echo -n "Running dos2unix " 6 | find . -name "*\.h" -o -name "*\.cpp"|grep -v bundled|xargs -I {} sh -c "dos2unix '{}' 2>/dev/null; echo -n '.'" 7 | echo 8 | echo -n "Running clang-format " 9 | find . -name "*\.h" -o -name "*\.cpp"|grep -v bundled|xargs -I {} sh -c "clang-format -i {}; echo -n '.'" 10 | echo 11 | echo -n "Running cmake-format " 12 | find . -name "CMakeLists.txt" -o -name "*\.cmake"|grep -v bundled|xargs -I {} sh -c "cmake-format --line-width 120 --tab-size 4 --max-subgroups-hwrap 4 -i {}; echo -n '.'" 13 | echo 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/spdlog/src/async.cpp: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #ifndef SPDLOG_COMPILED_LIB 5 | #error Please define SPDLOG_COMPILED_LIB to compile this file. 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | template class SPDLOG_API spdlog::details::mpmc_blocking_queue; -------------------------------------------------------------------------------- /src/spdlog/src/cfg.cpp: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #ifndef SPDLOG_COMPILED_LIB 5 | #error Please define SPDLOG_COMPILED_LIB to compile this file. 6 | #endif 7 | 8 | #include -------------------------------------------------------------------------------- /src/spdlog/src/file_sinks.cpp: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #ifndef SPDLOG_COMPILED_LIB 5 | #error Please define SPDLOG_COMPILED_LIB to compile this file. 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include 14 | 15 | template class SPDLOG_API spdlog::sinks::basic_file_sink; 16 | template class SPDLOG_API spdlog::sinks::basic_file_sink; 17 | 18 | #include 19 | template class SPDLOG_API spdlog::sinks::rotating_file_sink; 20 | template class SPDLOG_API spdlog::sinks::rotating_file_sink; -------------------------------------------------------------------------------- /src/spdlog/src/spdlog.cpp: -------------------------------------------------------------------------------- 1 | // Copyright(c) 2015-present, Gabi Melman & spdlog contributors. 2 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 3 | 4 | #ifndef SPDLOG_COMPILED_LIB 5 | #error Please define SPDLOG_COMPILED_LIB to compile this file. 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include 22 | 23 | // template instantiate logger constructor with sinks init list 24 | template SPDLOG_API spdlog::logger::logger(std::string name, sinks_init_list::iterator begin, sinks_init_list::iterator end); 25 | template class SPDLOG_API spdlog::sinks::base_sink; 26 | template class SPDLOG_API spdlog::sinks::base_sink; -------------------------------------------------------------------------------- /src/spdlog/tests/catch.license: -------------------------------------------------------------------------------- 1 | Boost Software License - Version 1.0 - August 17th, 2003 2 | 3 | Permission is hereby granted, free of charge, to any person or organization 4 | obtaining a copy of the software and accompanying documentation covered by 5 | this license (the "Software") to use, reproduce, display, distribute, 6 | execute, and transmit the Software, and to prepare derivative works of the 7 | Software, and to permit third-parties to whom the Software is furnished to 8 | do so, all subject to the following: 9 | 10 | The copyright notices in the Software and this entire statement, including 11 | the above license grant, this restriction and the following disclaimer, 12 | must be included in all copies of the Software, in whole or in part, and 13 | all derivative works of the Software, unless such copies or derivative 14 | works are solely in the form of machine-executable object code generated by 15 | a source language processor. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT 20 | SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE 21 | FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, 22 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 23 | DEALINGS IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /src/spdlog/tests/includes.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "catch.hpp" 4 | #include "utils.h" 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #define SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_DEBUG 17 | 18 | #include "spdlog/spdlog.h" 19 | #include "spdlog/async.h" 20 | #include "spdlog/sinks/basic_file_sink.h" 21 | #include "spdlog/sinks/daily_file_sink.h" 22 | #include "spdlog/sinks/null_sink.h" 23 | #include "spdlog/sinks/ostream_sink.h" 24 | #include "spdlog/sinks/rotating_file_sink.h" 25 | #include "spdlog/sinks/stdout_color_sinks.h" 26 | #include "spdlog/pattern_formatter.h" -------------------------------------------------------------------------------- /src/spdlog/tests/main.cpp: -------------------------------------------------------------------------------- 1 | #define CATCH_CONFIG_MAIN 2 | #include "catch.hpp" -------------------------------------------------------------------------------- /src/spdlog/tests/test_stopwatch.cpp: -------------------------------------------------------------------------------- 1 | #include "includes.h" 2 | #include "test_sink.h" 3 | #include "spdlog/stopwatch.h" 4 | 5 | TEST_CASE("stopwatch1", "[stopwatch]") 6 | { 7 | using std::chrono::milliseconds; 8 | milliseconds wait_ms(250); 9 | milliseconds tolerance_ms(250); 10 | 11 | spdlog::stopwatch sw; 12 | std::this_thread::sleep_for(wait_ms); 13 | REQUIRE(sw.elapsed() >= wait_ms); 14 | REQUIRE(sw.elapsed() <= wait_ms + tolerance_ms); 15 | } 16 | 17 | TEST_CASE("stopwatch2", "[stopwatch]") 18 | { 19 | using spdlog::sinks::test_sink_st; 20 | 21 | std::chrono::duration wait_duration(0.250); 22 | std::chrono::duration tolerance_duration(0.250); 23 | 24 | auto test_sink = std::make_shared(); 25 | 26 | spdlog::stopwatch sw; 27 | spdlog::logger logger("test-stopwatch", test_sink); 28 | logger.set_pattern("%v"); 29 | std::this_thread::sleep_for(wait_duration); 30 | logger.info("{}", sw); 31 | auto val = std::stod(test_sink->lines()[0]); 32 | 33 | REQUIRE(val >= wait_duration.count()); 34 | REQUIRE(val <= (wait_duration + tolerance_duration).count()); 35 | } 36 | -------------------------------------------------------------------------------- /src/spdlog/tests/test_systemd.cpp: -------------------------------------------------------------------------------- 1 | #include "includes.h" 2 | #include "spdlog/sinks/systemd_sink.h" 3 | 4 | TEST_CASE("systemd", "[all]") 5 | { 6 | auto systemd_sink = std::make_shared(); 7 | spdlog::logger logger("spdlog_systemd_test", systemd_sink); 8 | logger.set_level(spdlog::level::trace); 9 | logger.trace("test spdlog trace"); 10 | logger.debug("test spdlog debug"); 11 | SPDLOG_LOGGER_INFO((&logger), "test spdlog info"); 12 | SPDLOG_LOGGER_WARN((&logger), "test spdlog warn"); 13 | SPDLOG_LOGGER_ERROR((&logger), "test spdlog error"); 14 | SPDLOG_LOGGER_CRITICAL((&logger), "test spdlog critical"); 15 | } 16 | -------------------------------------------------------------------------------- /src/spdlog/tests/test_time_point.cpp: -------------------------------------------------------------------------------- 1 | #include "includes.h" 2 | #include "test_sink.h" 3 | #include "spdlog/async.h" 4 | 5 | TEST_CASE("time_point1", "[time_point log_msg]") 6 | { 7 | std::shared_ptr test_sink(new spdlog::sinks::test_sink_st); 8 | spdlog::logger logger("test-time_point", test_sink); 9 | 10 | spdlog::source_loc source{}; 11 | std::chrono::system_clock::time_point tp{std::chrono::system_clock::now()}; 12 | test_sink->set_pattern("%T.%F"); // interested in the time_point 13 | 14 | // all the following should have the same time 15 | test_sink->set_delay(std::chrono::milliseconds(10)); 16 | for (int i = 0; i < 5; i++) 17 | { 18 | spdlog::details::log_msg msg{tp, source, "test_logger", spdlog::level::info, "message"}; 19 | test_sink->log(msg); 20 | } 21 | 22 | logger.log(tp, source, spdlog::level::info, "formatted message"); 23 | logger.log(tp, source, spdlog::level::info, "formatted message"); 24 | logger.log(tp, source, spdlog::level::info, "formatted message"); 25 | logger.log(tp, source, spdlog::level::info, "formatted message"); 26 | logger.log(source, spdlog::level::info, "formatted message"); // last line has different time_point 27 | 28 | // now the real test... that the times are the same. 29 | std::vector lines = test_sink->lines(); 30 | REQUIRE(lines[0] == lines[1]); 31 | REQUIRE(lines[2] == lines[3]); 32 | REQUIRE(lines[4] == lines[5]); 33 | REQUIRE(lines[6] == lines[7]); 34 | REQUIRE(lines[8] != lines[9]); 35 | spdlog::drop_all(); 36 | } 37 | -------------------------------------------------------------------------------- /src/spdlog/tests/utils.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | std::size_t count_files(const std::string &folder); 7 | 8 | void prepare_logdir(); 9 | 10 | std::string file_contents(const std::string &filename); 11 | 12 | std::size_t count_lines(const std::string &filename); 13 | 14 | void require_message_count(const std::string &filename, const std::size_t messages); 15 | 16 | std::size_t get_filesize(const std::string &filename); 17 | 18 | bool ends_with(std::string const &value, std::string const &ending); -------------------------------------------------------------------------------- /src/vroom_chr.cc: -------------------------------------------------------------------------------- 1 | #include "vroom_chr.h" 2 | 3 | SEXP check_na(SEXP na, SEXP val) { 4 | for (R_xlen_t i = 0; i < Rf_xlength(na); ++i) { 5 | SEXP v = STRING_ELT(na, i); 6 | // We can just compare the addresses directly because they should now 7 | // both be in the global string cache. 8 | if (v == val) { 9 | return NA_STRING; 10 | } 11 | } 12 | return val; 13 | } 14 | 15 | cpp11::strings read_chr(vroom_vec_info* info) { 16 | 17 | R_xlen_t n = info->column->size(); 18 | 19 | cpp11::writable::strings out(n); 20 | 21 | SEXP nas = *info->na; 22 | 23 | cpp11::unwind_protect([&] { 24 | auto i = 0; 25 | auto col = info->column; 26 | for (auto b = col->begin(), e = col->end(); b != e; ++b) { 27 | auto str = *b; 28 | auto val = info->locale->encoder_.makeSEXP(str.begin(), str.end(), true); 29 | if (Rf_xlength(val) < str.end() - str.begin()) { 30 | info->errors->add_error( 31 | b.index(), col->get_index(), "", "embedded null", b.filename()); 32 | } 33 | 34 | SET_STRING_ELT(out, i++, check_na(nas, val)); 35 | } 36 | }); 37 | 38 | info->errors->warn_for_errors(); 39 | 40 | return out; 41 | } 42 | 43 | #ifdef HAS_ALTREP 44 | 45 | R_altrep_class_t vroom_chr::class_t; 46 | 47 | void init_vroom_chr(DllInfo* dll) { vroom_chr::Init(dll); } 48 | 49 | #else 50 | void init_vroom_chr(DllInfo* dll) {} 51 | #endif 52 | -------------------------------------------------------------------------------- /src/vroom_errors.cpp: -------------------------------------------------------------------------------- 1 | #include "vroom_errors.h" 2 | #include 3 | #include 4 | 5 | [[cpp11::register]] cpp11::data_frame 6 | vroom_errors_(cpp11::external_pointer> errors) { 7 | return (*errors)->error_table(); 8 | } 9 | -------------------------------------------------------------------------------- /src/vroom_int.cc: -------------------------------------------------------------------------------- 1 | #include "vroom_int.h" 2 | #include "parallel.h" 3 | 4 | // A version of strtoi that doesn't need null terminated strings, to avoid 5 | // needing to copy the data 6 | int strtoi(const char* begin, const char* end) { 7 | double val = 0; 8 | bool is_neg = false; 9 | 10 | if (begin == end) { 11 | return NA_INTEGER; 12 | } 13 | 14 | if (begin != end && *begin == '-') { 15 | is_neg = true; 16 | ++begin; 17 | } 18 | 19 | while (begin != end && isdigit(*begin)) { 20 | val = val * 10 + ((*begin++) - '0'); 21 | } 22 | 23 | // If there is more than digits, return NA 24 | if (begin != end) { 25 | return NA_INTEGER; 26 | } 27 | 28 | if (val > INT_MAX) { 29 | return NA_INTEGER; 30 | } 31 | 32 | return is_neg ? -val : val; 33 | } 34 | 35 | // Normal reading of integer vectors 36 | cpp11::integers read_int(vroom_vec_info* info) { 37 | 38 | R_xlen_t n = info->column->size(); 39 | 40 | cpp11::writable::integers out(n); 41 | 42 | parallel_for( 43 | n, 44 | [&](size_t start, size_t end, size_t) { 45 | R_xlen_t i = start; 46 | auto col = info->column->slice(start, end); 47 | for (auto b = col->begin(), e = col->end(); b != e; ++b) { 48 | out[i++] = parse_value( 49 | b, col, strtoi, info->errors, "an integer", *info->na); 50 | } 51 | }, 52 | info->num_threads); 53 | 54 | info->errors->warn_for_errors(); 55 | 56 | return out; 57 | } 58 | 59 | #ifdef HAS_ALTREP 60 | 61 | R_altrep_class_t vroom_int::class_t; 62 | 63 | void init_vroom_int(DllInfo* dll) { vroom_int::Init(dll); } 64 | 65 | #else 66 | void init_vroom_int(DllInfo* dll) {} 67 | #endif 68 | -------------------------------------------------------------------------------- /src/vroom_rle.cc: -------------------------------------------------------------------------------- 1 | #include "vroom_rle.h" 2 | 3 | #ifdef HAS_ALTREP 4 | 5 | R_altrep_class_t vroom_rle::class_t; 6 | 7 | void init_vroom_rle(DllInfo* dll) { vroom_rle::Init(dll); } 8 | 9 | #else 10 | void init_vroom_rle(DllInfo* dll) {} 11 | #endif 12 | -------------------------------------------------------------------------------- /src/vroom_types.h: -------------------------------------------------------------------------------- 1 | #include "vroom_errors.h" 2 | -------------------------------------------------------------------------------- /standalone/.gitignore: -------------------------------------------------------------------------------- 1 | vroom 2 | -------------------------------------------------------------------------------- /standalone/README.md: -------------------------------------------------------------------------------- 1 | ## Build a standalone version of vroom 2 | 3 | This is a simple example of a standalone C++ version using the vroom library. 4 | It simply returns the total number of fields in a delimited file. 5 | 6 | ## Run the fuzz tester 7 | 8 | Running the following with compile the standalone vroom and run afl-fuzz on it with 8 workers. 9 | 10 | If you are running on macOS you should probably use setup a ram disk so the heavy 11 | writing doesn't destroy your SSD, otherwise point `FUZZ_DIR` at the directory 12 | you want to run the fuzzing in. 13 | 14 | ```shell 15 | make clean 16 | FUZZ_DIR=/Volumns/ram_disk make fuzz 17 | ``` 18 | -------------------------------------------------------------------------------- /standalone/generate_tests.R: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env Rscript 2 | 3 | library(vroom) 4 | 5 | set.seed(42) 6 | 7 | out_dir <- commandArgs(TRUE)[[1]] 8 | 9 | # Simple file with all column types 10 | data <- gen_tbl(5, col_types = "difcDtT") 11 | vroom_write(data, file.path(out_dir, "00.tsv")) 12 | 13 | # File with quoting and a bom 14 | vroom_write(data, file.path(out_dir, "01.tsv"), bom = TRUE, quote = "all") 15 | 16 | 17 | # Add a field with quotes 18 | data[[4]][[1]] <- paste0('"f\to\no"') 19 | vroom_write(data, file.path(out_dir, "02.tsv"), bom = TRUE, quote = "all") 20 | 21 | # Add a header and blank lines to the start 22 | writeLines(c("", "# foo\t, bar", vroom_format(data)), file.path(out_dir, "03.tsv")) 23 | 24 | # Write a fixed width file 25 | data[] <- lapply(data, format) 26 | res <- apply(data, MARGIN = 1, paste0, collapse = " ") 27 | writeLines(res, file.path(out_dir, "04.tsv")) 28 | -------------------------------------------------------------------------------- /tests/spelling.R: -------------------------------------------------------------------------------- 1 | if(requireNamespace('spelling', quietly = TRUE)) 2 | spelling::spell_check_test(vignettes = TRUE, error = FALSE, 3 | skip_on_cran = TRUE) 4 | -------------------------------------------------------------------------------- /tests/testthat.R: -------------------------------------------------------------------------------- 1 | library(testthat) 2 | library(vroom) 3 | 4 | test_check("vroom") 5 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/col_types.md: -------------------------------------------------------------------------------- 1 | # all col_types can be reported with color 2 | 3 | Code 4 | spec(dat) 5 | Output 6 | cols( 7 | skip = col_skip(), 8 | guess = col_character(), 9 | character = col_character(), 10 | factor = col_factor(levels = NULL, ordered = FALSE, include_na = FALSE), 11 | logical = col_logical(), 12 | double = col_double(), 13 | integer = col_integer(), 14 | big_integer = col_big_integer(), 15 | number = col_number(), 16 | date = col_date(format = ""), 17 | datetime = col_datetime(format = ""), 18 | .delim = "," 19 | ) 20 | 21 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/path.md: -------------------------------------------------------------------------------- 1 | # vroom() informs user to use I() for literal data (or not) 2 | 3 | Code 4 | x <- vroom("a,b,c,d\n1,2,3,4", show_col_types = FALSE) 5 | Condition 6 | Warning: 7 | The `file` argument of `vroom()` must use `I()` for literal data as of vroom 1.5.0. 8 | 9 | # Bad: 10 | vroom("X,Y\n1.5,2.3\n") 11 | 12 | # Good: 13 | vroom(I("X,Y\n1.5,2.3\n")) 14 | 15 | # standardise_path() errors for a mix of connection and not connection 16 | 17 | Code 18 | f(list(file, conn)) 19 | Condition 20 | Error in `f()`: 21 | ! `some_arg_name` cannot be a mix of connection and non-connection inputs 22 | 23 | # standardise_path() errors for invalid input 24 | 25 | Code 26 | f(as.list(files)) 27 | Condition 28 | Error in `f()`: 29 | ! `some_arg_name` is not one of the supported inputs: 30 | * A filepath or character vector of filepaths 31 | * A connection or list of connections 32 | * Literal or raw input 33 | 34 | -------------------------------------------------------------------------------- /tests/testthat/_snaps/problems.md: -------------------------------------------------------------------------------- 1 | # problems returns a detailed warning message 2 | 3 | Code 4 | vroom(I("a,b,c\nx,y,z,,"), altrep = FALSE, col_types = "ccc") 5 | Condition 6 | Warning: 7 | One or more parsing issues, call `problems()` on your data frame for details, e.g.: 8 | dat <- vroom(...) 9 | problems(dat) 10 | Output 11 | # A tibble: 1 x 3 12 | a b c 13 | 14 | 1 x y z,, 15 | 16 | # emits an error message if provided incorrect input 17 | 18 | Code 19 | problems(a_vector) 20 | Condition 21 | Error in `problems()`: 22 | ! The `x` argument of `vroom::problems()` must be a data frame created by vroom: 23 | x `x` has class 24 | 25 | --- 26 | 27 | Code 28 | problems(a_tibble) 29 | Condition 30 | Error in `problems()`: 31 | ! The `x` argument of `vroom::problems()` must be a data frame created by vroom: 32 | x `x` seems to have been created with something else, maybe readr? 33 | 34 | -------------------------------------------------------------------------------- /tests/testthat/empty-file: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidyverse/vroom/968a3312d00f6f9de2a0c9cd3b1f2fa50a3e353e/tests/testthat/empty-file -------------------------------------------------------------------------------- /tests/testthat/enc-iso-8859-1.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidyverse/vroom/968a3312d00f6f9de2a0c9cd3b1f2fa50a3e353e/tests/testthat/enc-iso-8859-1.txt -------------------------------------------------------------------------------- /tests/testthat/fwf-trailing-crlf.txt: -------------------------------------------------------------------------------- 1 | 123 123 2 | 123 123 3 | -------------------------------------------------------------------------------- /tests/testthat/fwf-trailing.txt: -------------------------------------------------------------------------------- 1 | 123 123 2 | 123 123 3 | -------------------------------------------------------------------------------- /tests/testthat/multi-byte-ascii.txt: -------------------------------------------------------------------------------- 1 | id||name||age 2 | 1||ed||36 3 | 2||leigh||NA 4 | 3||nathan||14 5 | -------------------------------------------------------------------------------- /tests/testthat/multi-byte-unicode.txt: -------------------------------------------------------------------------------- 1 | id❤name❤age 2 | 1❤ed❤36 3 | 2❤leigh❤NA 4 | 3❤nathan❤14 5 | -------------------------------------------------------------------------------- /tests/testthat/multi-file/bar: -------------------------------------------------------------------------------- 1 | C,D 2 | 3,4 3 | -------------------------------------------------------------------------------- /tests/testthat/multi-file/baz: -------------------------------------------------------------------------------- 1 | A,B,C 2 | 1,2,3 3 | -------------------------------------------------------------------------------- /tests/testthat/multi-file/foo: -------------------------------------------------------------------------------- 1 | A,B 2 | 1,2 3 | -------------------------------------------------------------------------------- /tests/testthat/multi-file/qux: -------------------------------------------------------------------------------- 1 | A,B 2 | -------------------------------------------------------------------------------- /tests/testthat/raw.csv: -------------------------------------------------------------------------------- 1 | abc,def 2 | abc,def 3 | -------------------------------------------------------------------------------- /tests/testthat/test-chr.R: -------------------------------------------------------------------------------- 1 | # Encoding ---------------------------------------------------------------- 2 | 3 | test_that("locale encoding affects parsing", { 4 | x <- c("ao\u00FBt", "\u00E9l\u00E8ve", "\u00E7a va") 5 | #expect_equal(Encoding(x), rep("UTF-8", 3)) 6 | 7 | y <- iconv(paste0(x, collapse = "\n"), "UTF-8", "latin1") 8 | #expect_equal(Encoding(y), "latin1") 9 | 10 | fr <- locale("fr", encoding = "latin1") 11 | z <- vroom(I(y), delim = ",", locale = fr, col_names = FALSE, col_types = list()) 12 | # expect_equal(Encoding(z[[1]]), rep("UTF-8", 3)) 13 | 14 | # identical coerces encodings to match, so need to compare raw values 15 | as_raw <- function(x) lapply(x, charToRaw) 16 | expect_identical(as_raw(x), as_raw(z[[1]])) 17 | }) 18 | 19 | test_that("encodings are respected", { 20 | loc <- locale(encoding = "ISO-8859-1") 21 | expected <- c("fran\u00e7ais", "\u00e9l\u00e8ve") 22 | 23 | x <- vroom(test_path("enc-iso-8859-1.txt"), delim = "\n", locale = loc, col_names = FALSE, col_types = list()) 24 | expect_equal(x[[1]], expected) 25 | }) 26 | -------------------------------------------------------------------------------- /tests/testthat/test-col_types.R: -------------------------------------------------------------------------------- 1 | test_that("You can use !! in cols and cols_only", { 2 | var <- "xyz" 3 | expect_equal( 4 | cols(!!var := col_character()), 5 | cols(xyz = col_character()) 6 | ) 7 | 8 | expect_equal( 9 | cols_only(!!var := col_character()), 10 | cols_only(xyz = col_character()) 11 | ) 12 | }) 13 | 14 | test_that("format(col_spec) contains the delimiter if specified", { 15 | expect_match(fixed = TRUE, 16 | format(cols(.delim = "\t")), 17 | '.delim = "\\t"' 18 | ) 19 | }) 20 | 21 | test_that("col_types are truncated if you pass too many (#355)", { 22 | res <- vroom(I("a,b,c,d\n1,2,3,4"), col_types = "cccccccc") 23 | expect_equal(res, tibble::tibble(a = "1", b = "2", c = "3", d = "4")) 24 | }) 25 | 26 | test_that("all col_types can be reported with color", { 27 | local_reproducible_output(crayon = TRUE) 28 | dat <- vroom( 29 | I(glue::glue(" 30 | skip,guess,character,factor,logical,double,integer,big_integer,\\ 31 | number,date,datetime 32 | skip,a,b,c,TRUE,1.3,5,10,\"1,234.56\",2023-01-20,2018-01-01 10:01:01")), 33 | col_types = "_?cfldiInDT" 34 | ) 35 | expect_snapshot(spec(dat)) 36 | }) 37 | -------------------------------------------------------------------------------- /tests/testthat/test-int.R: -------------------------------------------------------------------------------- 1 | test_that("integers are returned correctly", { 2 | # empty fields are returned as NAs 3 | test_vroom("foo,bar,baz\n1,,3\n", col_types = list(.default = "i"), delim = ",", 4 | equals = tibble::tibble(foo = 1L, bar = NA_integer_, baz = 3L) 5 | ) 6 | 7 | # numbers which are not integers are returned as NAs 8 | test_vroom("foo,bar,baz\n1,1.5,3\n", col_types = list(.default = "i"), delim = ",", 9 | equals = tibble::tibble(foo = 1L, bar = NA_integer_, baz = 3L) 10 | ) 11 | 12 | # fields with non-digits are returned as NAs 13 | test_vroom("foo,bar,baz\n1,32xyz,3\n", col_types = list(.default = "i"), delim = ",", 14 | equals = tibble::tibble(foo = 1L, bar = NA_integer_, baz = 3L) 15 | ) 16 | 17 | # 2^31 - 1 is the maximum representable integer with 32 bit ints 18 | test_vroom("foo,bar,baz\n1,2147483647,3\n", col_types = list(.default = "i"), delim = ",", 19 | equals = tibble::tibble(foo = 1L, bar = 2147483647L, baz = 3L) 20 | ) 21 | 22 | test_vroom("foo,bar,baz\n1,2147483648,3\n", col_types = list(.default = "i"), delim = ",", 23 | equals = tibble::tibble(foo = 1L, bar = NA_integer_, baz = 3L) 24 | ) 25 | }) 26 | 27 | test_that("NA can be a int value", { 28 | test_vroom(I("x\n1\n2\n"), delim = ",", col_types = "i", na = "1", 29 | equals = tibble::tibble(x = c(NA_integer_, 2L))) 30 | }) 31 | -------------------------------------------------------------------------------- /tests/testthat/test-logical.R: -------------------------------------------------------------------------------- 1 | test_that("TRUE and FALSE parsed", { 2 | test_vroom("TRUE\nFALSE\n", col_names = FALSE, equals = tibble::tibble(X1 = c(TRUE, FALSE))) 3 | }) 4 | 5 | test_that("true and false parsed", { 6 | test_vroom("true\nfalse\n", col_names = FALSE, equals = tibble::tibble(X1 = c(TRUE, FALSE))) 7 | }) 8 | 9 | test_that("True and False parsed", { 10 | test_vroom("True\nFalse\n", col_names = FALSE, equals = tibble::tibble(X1 = c(TRUE, FALSE))) 11 | }) 12 | 13 | test_that("T and F parsed", { 14 | test_vroom("T\nF\n", col_names = FALSE, equals = tibble::tibble(X1 = c(TRUE, FALSE))) 15 | }) 16 | 17 | test_that("t and f parsed", { 18 | test_vroom("t\nf\n", col_names = FALSE, equals = tibble::tibble(X1 = c(TRUE, FALSE))) 19 | }) 20 | 21 | test_that("1 and 0 parsed", { 22 | # 1 and 0 are never guessed as logical, but they can be parsed as such if you 23 | # explicitly set the column type. 24 | test_vroom("1\n0\n", col_types = "l", col_names = FALSE, equals = tibble::tibble(X1 = c(TRUE, FALSE))) 25 | }) 26 | 27 | test_that("NA can be a logical value", { 28 | test_vroom("1\n0\n", col_types = "l", col_names = FALSE, na = "1", equals = tibble::tibble(X1 = c(NA, FALSE))) 29 | }) 30 | -------------------------------------------------------------------------------- /tests/testthat/test-multi-byte.R: -------------------------------------------------------------------------------- 1 | test_that("multi-byte reading works with unicode delimiters and UTF-8 encoding", { 2 | test_vroom(test_path("multi-byte-ascii.txt"), delim = "||", 3 | equals = tibble::tibble(id = 1:3, name = c("ed", "leigh", "nathan"), age = c(36, NA, 14)) 4 | ) 5 | }) 6 | 7 | test_that("multi-byte reading works with unicode delimiters and UTF-8 encoding", { 8 | skip_on_os("solaris") 9 | 10 | test_vroom(test_path("multi-byte-unicode.txt"), delim = "\U2764", 11 | equals = tibble::tibble(id = 1:3, name = c("ed", "leigh", "nathan"), age = c(36, NA, 14)) 12 | ) 13 | }) 14 | -------------------------------------------------------------------------------- /tests/testthat/test-num.R: -------------------------------------------------------------------------------- 1 | # Flexible number parsing ------------------------------------------------- 2 | test_that("col_number only takes first number", { 3 | test_parse_number("XYZ 123,000 BLAH 456", 123000) 4 | }) 5 | 6 | test_that("col_number helps with currency", { 7 | test_parse_number("$1,000,000.00", 1e6) 8 | 9 | es_MX <- locale("es", decimal_mark = ",") 10 | test_parse_number("$1.000.000,00", locale = es_MX, 1e6) 11 | }) 12 | 13 | test_that("invalid numbers don't parse", { 14 | test_parse_number(c("..", "--", "3.3.3", "4-1"), c(NA, NA, 3.3, 4.0)) 15 | }) 16 | -------------------------------------------------------------------------------- /vignettes/.gitignore: -------------------------------------------------------------------------------- 1 | *.html 2 | *.R 3 | -------------------------------------------------------------------------------- /vroom.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | 3 | RestoreWorkspace: Default 4 | SaveWorkspace: Default 5 | AlwaysSaveHistory: Default 6 | 7 | EnableCodeIndexing: Yes 8 | UseSpacesForTab: Yes 9 | NumSpacesForTab: 2 10 | Encoding: UTF-8 11 | 12 | RnwWeave: Sweave 13 | LaTeX: pdfLaTeX 14 | 15 | AutoAppendNewline: Yes 16 | StripTrailingWhitespace: Yes 17 | 18 | BuildType: Package 19 | PackageUseDevtools: Yes 20 | PackageInstallArgs: --no-multiarch --with-keep.source 21 | PackageRoxygenize: rd,collate,namespace 22 | --------------------------------------------------------------------------------