├── .github
├── ISSUE_TEMPLATE.md
├── PULL_REQUEST_TEMPLATE.md
└── workflows
│ └── ci.yml
├── .gitignore
├── .rakeTasks
├── .rubocop.yml
├── CONTRIBUTING.md
├── Gemfile
├── LICENSE
├── README.md
├── Rakefile
├── docs
├── CNAME
├── Gemfile
├── _config.yml
├── _layouts
│ ├── default.html
│ └── favicon.ico
├── config.md
├── copy_files.md
├── extensions.md
├── formats.md
├── images
│ └── rocket
│ │ ├── rocket-icon-128x128.png
│ │ ├── rocket-icon-2.psd
│ │ ├── rocket-icon-256.png
│ │ ├── rocket-icon-256x256.png
│ │ ├── rocket-icon-3.png
│ │ ├── rocket-icon-3.tif
│ │ ├── rocket-icon-3.xcf
│ │ ├── rocket-icon-3b.tif
│ │ ├── rocket-icon-4.png
│ │ ├── rocket-icon-4.tif
│ │ ├── rocket-icon-512x512.png
│ │ ├── rocket-icon-600x1200.png
│ │ └── rocket-icon-64x64.png
├── index.md
├── javascripts
│ └── main.js
├── path.md
├── pgp.md
├── streams.md
├── stylesheets
│ ├── github-light.css
│ ├── normalize.css
│ ├── print.css
│ ├── pygment_trac.css
│ ├── stylesheet.css
│ └── table.css
└── tutorial.md
├── iostreams.gemspec
├── lib
├── io_streams
│ ├── builder.rb
│ ├── bzip2
│ │ ├── reader.rb
│ │ └── writer.rb
│ ├── deprecated.rb
│ ├── encode
│ │ ├── reader.rb
│ │ └── writer.rb
│ ├── errors.rb
│ ├── gzip
│ │ ├── reader.rb
│ │ └── writer.rb
│ ├── io_streams.rb
│ ├── line
│ │ ├── reader.rb
│ │ └── writer.rb
│ ├── path.rb
│ ├── paths
│ │ ├── file.rb
│ │ ├── http.rb
│ │ ├── matcher.rb
│ │ ├── s3.rb
│ │ └── sftp.rb
│ ├── pgp.rb
│ ├── pgp
│ │ ├── reader.rb
│ │ └── writer.rb
│ ├── reader.rb
│ ├── record
│ │ ├── reader.rb
│ │ └── writer.rb
│ ├── row
│ │ ├── reader.rb
│ │ └── writer.rb
│ ├── stream.rb
│ ├── symmetric_encryption
│ │ ├── reader.rb
│ │ └── writer.rb
│ ├── tabular.rb
│ ├── tabular
│ │ ├── header.rb
│ │ ├── parser
│ │ │ ├── array.rb
│ │ │ ├── base.rb
│ │ │ ├── csv.rb
│ │ │ ├── fixed.rb
│ │ │ ├── hash.rb
│ │ │ ├── json.rb
│ │ │ └── psv.rb
│ │ └── utility
│ │ │ └── csv_row.rb
│ ├── utils.rb
│ ├── version.rb
│ ├── writer.rb
│ ├── xlsx
│ │ └── reader.rb
│ └── zip
│ │ ├── reader.rb
│ │ └── writer.rb
└── iostreams.rb
└── test
├── builder_test.rb
├── bzip2_reader_test.rb
├── bzip2_writer_test.rb
├── deprecated_test.rb
├── encode_reader_test.rb
├── encode_writer_test.rb
├── files
├── embedded_lines_test.csv
├── multiple_files.zip
├── spreadsheet.xlsx
├── test.csv
├── test.json
├── test.psv
├── text file.txt
├── text.txt
├── text.txt.bz2
├── text.txt.gz
├── text.txt.gz.zip
├── text.zip
├── text.zip.gz
├── unclosed_quote_large_test.csv
├── unclosed_quote_test.csv
└── unclosed_quote_test2.csv
├── gzip_reader_test.rb
├── gzip_writer_test.rb
├── io_streams_test.rb
├── line_reader_test.rb
├── line_writer_test.rb
├── minimal_file_reader.rb
├── path_test.rb
├── paths
├── file_test.rb
├── http_test.rb
├── matcher_test.rb
├── s3_test.rb
└── sftp_test.rb
├── pgp_reader_test.rb
├── pgp_test.rb
├── pgp_writer_test.rb
├── record_reader_test.rb
├── record_writer_test.rb
├── row_reader_test.rb
├── row_writer_test.rb
├── stream_test.rb
├── tabular_test.rb
├── test_helper.rb
├── utils_test.rb
├── xlsx_reader_test.rb
├── zip_reader_test.rb
└── zip_writer_test.rb
/.github/ISSUE_TEMPLATE.md:
--------------------------------------------------------------------------------
1 | ### Environment
2 |
3 | Provide at least:
4 | * Ruby Version.
5 | * IOStreams Version.
6 | * Application/framework names and versions (e.g. Rails, Sinatra, Puma, etc.).
7 | * Full Stack Trace, if an exception is being raised.
8 |
9 | ### Expected Behavior
10 |
11 | * Describe your expectation of how IOStreams should behave.
12 | * Ideally, provide a standalone Ruby script or a link to an example repository that helps reproduce the issue.
13 |
14 | ### Actual Behavior
15 |
16 | * Describe or show the actual behavior.
17 | * Provide text or screen capture showing the behavior.
18 |
19 | ### Pull Request
20 |
21 | * Consider submitting a Pull Request with a fix for the issue.
22 | * Or, even a Pull request that only includes a test that reproduces the problem.
23 |
--------------------------------------------------------------------------------
/.github/PULL_REQUEST_TEMPLATE.md:
--------------------------------------------------------------------------------
1 | ### Issue # (if available)
2 |
3 |
4 | ### Description of changes
5 |
6 |
7 | By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
8 |
--------------------------------------------------------------------------------
/.github/workflows/ci.yml:
--------------------------------------------------------------------------------
1 | name: build
2 |
3 | on:
4 | - push
5 | - pull_request
6 |
7 | jobs:
8 | test:
9 | name: "Test: Ruby ${{ matrix.ruby }}"
10 | runs-on: ubuntu-latest
11 | strategy:
12 | fail-fast: false
13 | matrix:
14 | ruby: [2.5, 2.6, 2.7, "3.0", jruby]
15 | steps:
16 | - uses: actions/checkout@v2
17 | - name: Set up Ruby
18 | uses: ruby/setup-ruby@v1
19 | with:
20 | ruby-version: ${{ matrix.ruby }}
21 | # runs 'bundle install' and caches installed gems automatically
22 | bundler-cache: true
23 | - name: Ruby Version
24 | run: ruby --version
25 | - name: Run Tests
26 | run: bundle exec rake
27 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *.gem
2 | *.rbc
3 | *.log
4 | /.config
5 | /.idea
6 | /coverage/
7 | /InstalledFiles
8 | /pkg/
9 | /spec/reports/
10 | /test/tmp/
11 | /test/version_tmp/
12 | /tmp/
13 |
14 | ## Documentation cache and generated files:
15 | /.yardoc/
16 | /_yardoc/
17 | /doc/
18 | /rdoc/
19 |
20 | ## Environment normalisation:
21 | /.bundle/
22 | /lib/bundler/man/
23 |
24 | # for a library or gem, you might want to ignore these files since the code is
25 | # intended to run in multiple environments; otherwise, check them in:
26 | Gemfile.lock
27 |
28 | /docs/_site
29 | .rakeTasks
30 |
--------------------------------------------------------------------------------
/.rakeTasks:
--------------------------------------------------------------------------------
1 |
2 |
8 |
--------------------------------------------------------------------------------
/.rubocop.yml:
--------------------------------------------------------------------------------
1 | AllCops:
2 | Exclude:
3 | - ".git/**/*"
4 | - "docs/**/*"
5 | - "gemfiles/*"
6 | - "lib/io_streams/deprecated.rb"
7 | - "lib/io_streams/tabular/utility/csv_row.rb"
8 | - "test/files/*"
9 | - "docs/stylesheets/*"
10 | NewCops: enable
11 | TargetRubyVersion: 2.5
12 |
13 | #
14 | # RuboCop built-in settings.
15 | # For documentation on all settings see: https://docs.rubocop.org/en/stable
16 | #
17 |
18 | # Trailing periods.
19 | Layout/DotPosition:
20 | EnforcedStyle: trailing
21 |
22 | # Turn on auto-correction of equals alignment.
23 | Layout/EndAlignment:
24 | AutoCorrect: true
25 |
26 | # Prevent accidental windows line endings
27 | Layout/EndOfLine:
28 | EnforcedStyle: lf
29 |
30 | # Use a table layout for hashes
31 | Layout/HashAlignment:
32 | EnforcedHashRocketStyle: table
33 | EnforcedColonStyle: table
34 |
35 | # Soften limits
36 | Layout/LineLength:
37 | Max: 128
38 | Exclude:
39 | - "**/test/**/*"
40 |
41 | # Match existing layout
42 | Layout/SpaceInsideHashLiteralBraces:
43 | EnforcedStyle: no_space
44 |
45 | # TODO: Soften Limits for phase 1 only
46 | Metrics/AbcSize:
47 | Max: 40
48 |
49 | # Support long block lengths for tests
50 | Metrics/BlockLength:
51 | Exclude:
52 | - "test/**/*"
53 | - "**/*/cli.rb"
54 | AllowedMethods:
55 | - "aasm"
56 | - "included"
57 |
58 | # Soften limits
59 | Metrics/ClassLength:
60 | Max: 250
61 | Exclude:
62 | - "test/**/*"
63 |
64 | # TODO: Soften Limits for phase 1 only
65 | Metrics/CyclomaticComplexity:
66 | Max: 15
67 |
68 | # Soften limits
69 | Metrics/MethodLength:
70 | Max: 50
71 |
72 | # Soften limits
73 | Metrics/ModuleLength:
74 | Max: 250
75 |
76 | Metrics/ParameterLists:
77 | CountKeywordArgs: false
78 |
79 | # TODO: Soften Limits for phase 1 only
80 | Metrics/PerceivedComplexity:
81 | Max: 21
82 |
83 | # Initialization Vector abbreviation
84 | Naming/MethodParameterName:
85 | AllowedNames: [ 'iv', '_', 'io', 'ap' ]
86 |
87 | # Does not allow Symbols to load
88 | Security/YAMLLoad:
89 | AutoCorrect: false
90 |
91 | # Needed for testing DateTime
92 | Style/DateTime:
93 | Exclude: [ "test/**/*" ]
94 |
95 | # TODO: Soften Limits for phase 1 only
96 | Style/Documentation:
97 | Enabled: false
98 |
99 | # One line methods
100 | Style/EmptyMethod:
101 | EnforcedStyle: expanded
102 |
103 | # Ruby 3 compatibility feature
104 | Style/FrozenStringLiteralComment:
105 | Enabled: false
106 |
107 | Style/NumericPredicate:
108 | AutoCorrect: true
109 |
110 | # Incorrectly changes job.fail to job.raise
111 | Style/SignalException:
112 | Enabled: false
113 |
114 | # Since English may not be loaded, cannot force its use.
115 | Style/SpecialGlobalVars:
116 | Enabled: false
117 |
118 | # Make it easier for developers to move between Elixir and Ruby.
119 | Style/StringLiterals:
120 | EnforcedStyle: double_quotes
121 |
--------------------------------------------------------------------------------
/Gemfile:
--------------------------------------------------------------------------------
1 | source "http://rubygems.org"
2 |
3 | gemspec
4 |
5 | gem "amazing_print"
6 | gem "minitest"
7 | gem "rake"
8 |
9 | gem "aws-sdk-s3"
10 | gem "bzip2-ffi"
11 | gem "creek"
12 | gem "nokogiri"
13 | # Rubyzip v2.2 blows up with some zip files
14 | gem "rubyzip", "~> 1.3"
15 | gem "symmetric-encryption"
16 | gem "zip_tricks"
17 |
18 | group :development do
19 | gem "rubocop"
20 | end
21 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # IOStreams
2 | [](https://rubygems.org/gems/iostreams) [](https://rubygems.org/gems/iostreams) [](http://opensource.org/licenses/Apache-2.0) 
3 |
4 | IOStreams is an incredibly powerful streaming library that makes changes to file formats, compression, encryption,
5 | or storage mechanism transparent to the application.
6 |
7 | ## Project Status
8 |
9 | Production Ready, heavily used in production environments, many as part of Rocket Job.
10 |
11 | ## Documentation
12 |
13 | Start with the [IOStreams tutorial](https://iostreams.rocketjob.io/tutorial) to get a great introduction to IOStreams.
14 |
15 | Next, checkout the remaining [IOStreams documentation](https://iostreams.rocketjob.io/)
16 |
17 | ## Upgrading to v1.6
18 |
19 | The old, deprecated api's are no longer loaded by default with v1.6. To add back the deprecated api support, add
20 | the following line to your code:
21 |
22 | ~~~ruby
23 | IOStreams.include(IOStreams::Deprecated)
24 | ~~~
25 |
26 | It is important to move any of the old deprecated apis over to the new api, since they will be removed in a future
27 | release.
28 |
29 | ## Versioning
30 |
31 | This project adheres to [Semantic Versioning](http://semver.org/).
32 |
33 | ## Author
34 |
35 | [Reid Morrison](https://github.com/reidmorrison)
36 |
37 | ## License
38 |
39 | Copyright 2020 Reid Morrison
40 |
41 | Licensed under the Apache License, Version 2.0 (the "License");
42 | you may not use this file except in compliance with the License.
43 | You may obtain a copy of the License at
44 |
45 | http://www.apache.org/licenses/LICENSE-2.0
46 |
47 | Unless required by applicable law or agreed to in writing, software
48 | distributed under the License is distributed on an "AS IS" BASIS,
49 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
50 | See the License for the specific language governing permissions and
51 | limitations under the License.
52 |
--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
1 | require "rake/testtask"
2 | require_relative "lib/io_streams/version"
3 |
4 | task :gem do
5 | system "gem build iostreams.gemspec"
6 | end
7 |
8 | task publish: :gem do
9 | system "git tag -a v#{IOStreams::VERSION} -m 'Tagging #{IOStreams::VERSION}'"
10 | system "git push --tags"
11 | system "gem push iostreams-#{IOStreams::VERSION}.gem"
12 | system "rm iostreams-#{IOStreams::VERSION}.gem"
13 | end
14 |
15 | Rake::TestTask.new(:test) do |t|
16 | t.pattern = "test/**/*_test.rb"
17 | t.verbose = true
18 | t.warning = true
19 | end
20 |
21 | task default: :test
22 |
--------------------------------------------------------------------------------
/docs/CNAME:
--------------------------------------------------------------------------------
1 | iostreams.rocketjob.io
--------------------------------------------------------------------------------
/docs/Gemfile:
--------------------------------------------------------------------------------
1 | source 'https://rubygems.org'
2 | gem 'github-pages'
3 |
--------------------------------------------------------------------------------
/docs/_config.yml:
--------------------------------------------------------------------------------
1 | markdown: kramdown
2 |
--------------------------------------------------------------------------------
/docs/_layouts/default.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | IOStreams: Input and Output streaming for Ruby
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
40 |
41 |
44 |
45 |
46 |
47 |
--------------------------------------------------------------------------------
/docs/_layouts/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/docs/_layouts/favicon.ico
--------------------------------------------------------------------------------
/docs/config.md:
--------------------------------------------------------------------------------
1 | ---
2 | layout: default
3 | ---
4 |
5 | ## Configuring IOStreams
6 |
7 | ### add_root
8 |
9 | When using `IOStreams.join` it uses a default root from which to join the remainder of the path.
10 |
11 | Set the default root for this environment
12 | ~~~ruby
13 | IOStreams.add_root(:default, "/var/my_app/files")
14 | ~~~
15 |
16 | Now the default root path is available:
17 | ~~~ruby
18 | IOStreams.root
19 | # => #
20 |
21 | IOStreams.root.to_s
22 | # => "/var/my_app/files"
23 | ~~~
24 |
25 | Comparing the final path using `path` and then `join` that uses a root path:
26 | ~~~ruby
27 | IOStreams.path("/var/my_app/files", "my_test_file.txt").to_s
28 | # => "/var/my_app/files/my_test_file.txt"
29 |
30 | IOStreams.join("my_test_file.txt").to_s
31 | # => "/var/my_app/files/my_test_file.txt"
32 | ~~~
33 |
34 |
35 | Using `path`:
36 | ~~~ruby
37 | IOStreams.path("/var/my_app/files", "my_test_file.txt").write("Hello World")
38 | ~~~
39 |
40 | With the default root path configured the above code can be simplified by using `join` since it resolves to the same path.
41 | ~~~ruby
42 | IOStreams.join("my_test_file.txt").write("Hello World")
43 | ~~~
44 |
45 | Other root paths can be added for special purposes.
46 |
47 | For example add special paths for `downloads` and `uploads`.
48 | ~~~ruby
49 | IOStreams.add_root(:downloads, "/var/my_app/downloads")
50 | IOStreams.add_root(:uploads, "/var/my_app/uploads")
51 | ~~~
52 |
53 | An example that writes a file into the `/var/my_app/downloads` directory:
54 | ~~~ruby
55 | IOStreams.join("my_test_file.txt", root: :downloads).write("Hello World")
56 | ~~~
57 |
58 | The other benefit is that the root paths used in an application are externalized from the code base. That way the
59 | roots can be changed to different locations depending on the environment.
60 |
61 | We can also change the storage mechanism by changing the root:
62 | ~~~ruby
63 | IOStreams.add_root(:downloads, "s3://my-app-bucket-name/downloads")
64 | IOStreams.add_root(:uploads, "s3://my-app-bucket-name/uploads")
65 | ~~~
66 |
67 | Now the application will write to S3 and the code does not change at all.
68 | ~~~ruby
69 | IOStreams.join("my_test_file.txt", root: :downloads).write("Hello World")
70 | ~~~
71 |
72 | To use or query a configured root path:
73 | ~~~ruby
74 | IOStreams.root(:downloads).to_s
75 | # => "s3://my-app-bucket-name/downloads"
76 | ~~~
77 |
78 | ## temp_dir
79 |
80 | When working with large files the standard temp file system location can be too small to handle downloading large
81 | files. For example to decrypt a pgp file from S3, because GnuPG is not streaming capable and only operates on local filess.
82 |
83 | By default IOStreams looks up the location to store temp files in the following order:
84 | * `ENV['TMPDIR']`
85 | * `ENV['TMP']`
86 | * `ENV['TEMP']`
87 | * `Etc.systmpdir`
88 | * `/tmp` (if it exists)
89 | * Otherwise `.`
90 |
91 | To explicity set the temp file location the following config option can be used:
92 |
93 | ~~~ruby
94 | IOStreams.temp_dir = "/var/really_big_temp"
95 | ~~~
96 |
--------------------------------------------------------------------------------
/docs/copy_files.md:
--------------------------------------------------------------------------------
1 | ---
2 | layout: default
3 | ---
4 |
5 | ## Copying between files
6 |
7 | File copying can be used to:
8 | * copy from one storage location to another.
9 | * create a decrypted / encrypted copy of an existing file.
10 | * create a decompressed / compressed copy of an existing file.
11 |
12 | ### Examples
13 |
14 | Decompress `example.csv.gz` into `example.csv`:
15 |
16 | ~~~ruby
17 | source = IOStreams.path("example.csv.gz")
18 | IOStreams.path("example.csv").copy_from(source)
19 | ~~~
20 |
21 | Decrypt a file encrypted with Symmetric Encryption:
22 |
23 | ~~~ruby
24 | source = IOStreams.path("example.csv.enc")
25 | IOStreams.path("example.csv").copy_from(source)
26 | ~~~
27 |
28 | Encrypt a file using PGP encryption so that it can only be read by `receiver@example.org`.
29 |
30 | ~~~ruby
31 | source = IOStreams.path("example.csv")
32 | target = IOStreams.path("example.csv.pgp")
33 | target.option(:pgp, recipient: "receiver@example.org")
34 | target.copy_from(source)
35 | ~~~
36 |
37 | When the file name does not have file extensions that would allow IOStreams to infer what streams to apply,
38 | the streams can be explicitly set using `stream`.
39 |
40 | In this example, the file `CUSTOMER_DATA`
41 |
42 | Decrypt the contents of file that was encrypted with Symmetric Encryption
43 | PGP encrypt the output file and write it to `xyz.csv.pgp` using the pgp key that was imported for `a@a.com`.
44 |
45 | ~~~ruby
46 | input = IOStreams.path("CUSTOMER_DATA").stream(:enc)
47 | IOStreams.path("xyz.csv.pgp").option(:pgp, recipient: "receiver@example.org").copy_from(input)
48 | ~~~
49 |
50 | To copy a file _without_ performing any conversions (ignore file extensions), set `convert` to `false`:
51 |
52 | ~~~ruby
53 | input = IOStreams.path("sample.json.zip")
54 | IOStreams.path("sample.copy").copy_from(input, convert: false)
55 | ~~~
56 |
--------------------------------------------------------------------------------
/docs/extensions.md:
--------------------------------------------------------------------------------
1 | ---
2 | layout: default
3 | ---
4 |
5 | #### File Extensions
6 |
7 | * Zip
8 | * Gzip
9 | * BZip2
10 | * PGP (Requires GnuPG)
11 | * Xlsx (Reading)
12 | * Encryption using [Symmetric Encryption](https://encryption.rocketjob.io/)
13 |
--------------------------------------------------------------------------------
/docs/formats.md:
--------------------------------------------------------------------------------
1 | ---
2 | layout: default
3 | ---
4 |
5 | #### File formats
6 |
7 | * CSV
8 | * Fixed width formats
9 | * JSON
10 | * PSV
11 |
--------------------------------------------------------------------------------
/docs/images/rocket/rocket-icon-128x128.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/docs/images/rocket/rocket-icon-128x128.png
--------------------------------------------------------------------------------
/docs/images/rocket/rocket-icon-2.psd:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/docs/images/rocket/rocket-icon-2.psd
--------------------------------------------------------------------------------
/docs/images/rocket/rocket-icon-256.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/docs/images/rocket/rocket-icon-256.png
--------------------------------------------------------------------------------
/docs/images/rocket/rocket-icon-256x256.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/docs/images/rocket/rocket-icon-256x256.png
--------------------------------------------------------------------------------
/docs/images/rocket/rocket-icon-3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/docs/images/rocket/rocket-icon-3.png
--------------------------------------------------------------------------------
/docs/images/rocket/rocket-icon-3.tif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/docs/images/rocket/rocket-icon-3.tif
--------------------------------------------------------------------------------
/docs/images/rocket/rocket-icon-3.xcf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/docs/images/rocket/rocket-icon-3.xcf
--------------------------------------------------------------------------------
/docs/images/rocket/rocket-icon-3b.tif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/docs/images/rocket/rocket-icon-3b.tif
--------------------------------------------------------------------------------
/docs/images/rocket/rocket-icon-4.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/docs/images/rocket/rocket-icon-4.png
--------------------------------------------------------------------------------
/docs/images/rocket/rocket-icon-4.tif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/docs/images/rocket/rocket-icon-4.tif
--------------------------------------------------------------------------------
/docs/images/rocket/rocket-icon-512x512.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/docs/images/rocket/rocket-icon-512x512.png
--------------------------------------------------------------------------------
/docs/images/rocket/rocket-icon-600x1200.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/docs/images/rocket/rocket-icon-600x1200.png
--------------------------------------------------------------------------------
/docs/images/rocket/rocket-icon-64x64.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/docs/images/rocket/rocket-icon-64x64.png
--------------------------------------------------------------------------------
/docs/javascripts/main.js:
--------------------------------------------------------------------------------
1 | console.log('This would be the main JS file.');
2 |
--------------------------------------------------------------------------------
/docs/stylesheets/github-light.css:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2014 GitHub Inc.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 |
16 | */
17 |
18 | .pl-c /* comment */ {
19 | color: #969896;
20 | }
21 |
22 | .pl-c1 /* constant, markup.raw, meta.diff.header, meta.module-reference, meta.property-name, support, support.constant, support.variable, variable.other.constant */,
23 | .pl-s .pl-v /* string variable */ {
24 | color: #0086b3;
25 | }
26 |
27 | .pl-e /* entity */,
28 | .pl-en /* entity.name */ {
29 | color: #795da3;
30 | }
31 |
32 | .pl-s .pl-s1 /* string source */,
33 | .pl-smi /* storage.modifier.import, storage.modifier.package, storage.type.java, variable.other, variable.parameter.function */ {
34 | color: #333;
35 | }
36 |
37 | .pl-ent /* entity.name.tag */ {
38 | color: #63a35c;
39 | }
40 |
41 | .pl-k /* keyword, storage, storage.type */ {
42 | color: #a71d5d;
43 | }
44 |
45 | .pl-pds /* punctuation.definition.string, string.regexp.character-class */,
46 | .pl-s /* string */,
47 | .pl-s .pl-pse .pl-s1 /* string punctuation.section.embedded source */,
48 | .pl-sr /* string.regexp */,
49 | .pl-sr .pl-cce /* string.regexp constant.character.escape */,
50 | .pl-sr .pl-sra /* string.regexp string.regexp.arbitrary-repitition */,
51 | .pl-sr .pl-sre /* string.regexp source.ruby.embedded */ {
52 | color: #183691;
53 | }
54 |
55 | .pl-v /* variable */ {
56 | color: #ed6a43;
57 | }
58 |
59 | .pl-id /* invalid.deprecated */ {
60 | color: #b52a1d;
61 | }
62 |
63 | .pl-ii /* invalid.illegal */ {
64 | background-color: #b52a1d;
65 | color: #f8f8f8;
66 | }
67 |
68 | .pl-sr .pl-cce /* string.regexp constant.character.escape */ {
69 | color: #63a35c;
70 | font-weight: bold;
71 | }
72 |
73 | .pl-ml /* markup.list */ {
74 | color: #693a17;
75 | }
76 |
77 | .pl-mh /* markup.heading */,
78 | .pl-mh .pl-en /* markup.heading entity.name */,
79 | .pl-ms /* meta.separator */ {
80 | color: #1d3e81;
81 | font-weight: bold;
82 | }
83 |
84 | .pl-mq /* markup.quote */ {
85 | color: #008080;
86 | }
87 |
88 | .pl-mi /* markup.italic */ {
89 | color: #333;
90 | font-style: italic;
91 | }
92 |
93 | .pl-mb /* markup.bold */ {
94 | color: #333;
95 | font-weight: bold;
96 | }
97 |
98 | .pl-md /* markup.deleted, meta.diff.header.from-file */ {
99 | background-color: #ffecec;
100 | color: #bd2c00;
101 | }
102 |
103 | .pl-mi1 /* markup.inserted, meta.diff.header.to-file */ {
104 | background-color: #eaffea;
105 | color: #55a532;
106 | }
107 |
108 | .pl-mdr /* meta.diff.range */ {
109 | color: #795da3;
110 | font-weight: bold;
111 | }
112 |
113 | .pl-mo /* meta.output */ {
114 | color: #1d3e81;
115 | }
116 |
117 |
--------------------------------------------------------------------------------
/docs/stylesheets/print.css:
--------------------------------------------------------------------------------
1 | html, body, div, span, applet, object, iframe,
2 | h1, h2, h3, h4, h5, h6, p, blockquote, pre,
3 | a, abbr, acronym, address, big, cite, code,
4 | del, dfn, em, img, ins, kbd, q, s, samp,
5 | small, strike, strong, sub, sup, tt, var,
6 | b, u, i, center,
7 | dl, dt, dd, ol, ul, li,
8 | fieldset, form, label, legend,
9 | table, caption, tbody, tfoot, thead, tr, th, td,
10 | article, aside, canvas, details, embed,
11 | figure, figcaption, footer, header, hgroup,
12 | menu, nav, output, ruby, section, summary,
13 | time, mark, audio, video {
14 | margin: 0;
15 | padding: 0;
16 | border: 0;
17 | font-size: 100%;
18 | font: inherit;
19 | vertical-align: baseline;
20 | }
21 | /* HTML5 display-role reset for older browsers */
22 | article, aside, details, figcaption, figure,
23 | footer, header, hgroup, menu, nav, section {
24 | display: block;
25 | }
26 | body {
27 | line-height: 1;
28 | }
29 | ol, ul {
30 | list-style: none;
31 | }
32 | blockquote, q {
33 | quotes: none;
34 | }
35 | blockquote:before, blockquote:after,
36 | q:before, q:after {
37 | content: '';
38 | content: none;
39 | }
40 | table {
41 | border-collapse: collapse;
42 | border-spacing: 0;
43 | }
44 | body {
45 | font-size: 13px;
46 | line-height: 1.5;
47 | font-family: 'Helvetica Neue', Helvetica, Arial, serif;
48 | color: #000;
49 | }
50 |
51 | a {
52 | color: #d5000d;
53 | font-weight: bold;
54 | }
55 |
56 | header {
57 | padding-top: 35px;
58 | padding-bottom: 10px;
59 | }
60 |
61 | header h1 {
62 | font-weight: bold;
63 | letter-spacing: -1px;
64 | font-size: 48px;
65 | color: #303030;
66 | line-height: 1.2;
67 | }
68 |
69 | header h2 {
70 | letter-spacing: -1px;
71 | font-size: 24px;
72 | color: #aaa;
73 | font-weight: normal;
74 | line-height: 1.3;
75 | }
76 | #downloads {
77 | display: none;
78 | }
79 | #main_content {
80 | padding-top: 20px;
81 | }
82 |
83 | code, pre {
84 | font-family: Monaco, "Bitstream Vera Sans Mono", "Lucida Console", Terminal;
85 | color: #222;
86 | margin-bottom: 30px;
87 | font-size: 12px;
88 | }
89 |
90 | code {
91 | padding: 0 3px;
92 | }
93 |
94 | pre {
95 | border: solid 1px #ddd;
96 | padding: 20px;
97 | overflow: auto;
98 | }
99 | pre code {
100 | padding: 0;
101 | }
102 |
103 | ul, ol, dl {
104 | margin-bottom: 20px;
105 | }
106 |
107 |
108 | /* COMMON STYLES */
109 |
110 | table {
111 | width: 100%;
112 | border: 1px solid #ebebeb;
113 | }
114 |
115 | th {
116 | font-weight: 500;
117 | }
118 |
119 | td {
120 | border: 1px solid #ebebeb;
121 | text-align: center;
122 | font-weight: 300;
123 | }
124 |
125 | form {
126 | background: #f2f2f2;
127 | padding: 20px;
128 |
129 | }
130 |
131 |
132 | /* GENERAL ELEMENT TYPE STYLES */
133 |
134 | h1 {
135 | font-size: 2.8em;
136 | }
137 |
138 | h2 {
139 | font-size: 22px;
140 | font-weight: bold;
141 | color: #303030;
142 | margin-bottom: 8px;
143 | }
144 |
145 | h3 {
146 | color: #d5000d;
147 | font-size: 18px;
148 | font-weight: bold;
149 | margin-bottom: 8px;
150 | }
151 |
152 | h4 {
153 | font-size: 16px;
154 | color: #303030;
155 | font-weight: bold;
156 | }
157 |
158 | h5 {
159 | font-size: 1em;
160 | color: #303030;
161 | }
162 |
163 | h6 {
164 | font-size: .8em;
165 | color: #303030;
166 | }
167 |
168 | p {
169 | font-weight: 300;
170 | margin-bottom: 20px;
171 | }
172 |
173 | a {
174 | text-decoration: none;
175 | }
176 |
177 | p a {
178 | font-weight: 400;
179 | }
180 |
181 | blockquote {
182 | font-size: 1.6em;
183 | border-left: 10px solid #e9e9e9;
184 | margin-bottom: 20px;
185 | padding: 0 0 0 30px;
186 | }
187 |
188 | ul li {
189 | list-style: disc inside;
190 | padding-left: 20px;
191 | }
192 |
193 | ol li {
194 | list-style: decimal inside;
195 | padding-left: 3px;
196 | }
197 |
198 | dl dd {
199 | font-style: italic;
200 | font-weight: 100;
201 | }
202 |
203 | footer {
204 | margin-top: 40px;
205 | padding-top: 20px;
206 | padding-bottom: 30px;
207 | font-size: 13px;
208 | color: #aaa;
209 | }
210 |
211 | footer a {
212 | color: #666;
213 | }
214 |
215 | /* MISC */
216 | .clearfix:after {
217 | clear: both;
218 | content: '.';
219 | display: block;
220 | visibility: hidden;
221 | height: 0;
222 | }
223 |
224 | .clearfix {display: inline-block;}
225 | * html .clearfix {height: 1%;}
226 | .clearfix {display: block;}
--------------------------------------------------------------------------------
/docs/stylesheets/pygment_trac.css:
--------------------------------------------------------------------------------
1 | .highlight { background: #ffffff; }
2 | .highlight .c { color: #999988; font-style: italic } /* Comment */
3 | .highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */
4 | .highlight .k { font-weight: bold } /* Keyword */
5 | .highlight .o { font-weight: bold } /* Operator */
6 | .highlight .cm { color: #999988; font-style: italic } /* Comment.Multiline */
7 | .highlight .cp { color: #999999; font-weight: bold } /* Comment.Preproc */
8 | .highlight .c1 { color: #999988; font-style: italic } /* Comment.Single */
9 | .highlight .cs { color: #999999; font-weight: bold; font-style: italic } /* Comment.Special */
10 | .highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */
11 | .highlight .gd .x { color: #000000; background-color: #ffaaaa } /* Generic.Deleted.Specific */
12 | .highlight .ge { font-style: italic } /* Generic.Emph */
13 | .highlight .gr { color: #aa0000 } /* Generic.Error */
14 | .highlight .gh { color: #999999 } /* Generic.Heading */
15 | .highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */
16 | .highlight .gi .x { color: #000000; background-color: #aaffaa } /* Generic.Inserted.Specific */
17 | .highlight .go { color: #888888 } /* Generic.Output */
18 | .highlight .gp { color: #555555 } /* Generic.Prompt */
19 | .highlight .gs { font-weight: bold } /* Generic.Strong */
20 | .highlight .gu { color: #800080; font-weight: bold; } /* Generic.Subheading */
21 | .highlight .gt { color: #aa0000 } /* Generic.Traceback */
22 | .highlight .kc { font-weight: bold } /* Keyword.Constant */
23 | .highlight .kd { font-weight: bold } /* Keyword.Declaration */
24 | .highlight .kn { font-weight: bold } /* Keyword.Namespace */
25 | .highlight .kp { font-weight: bold } /* Keyword.Pseudo */
26 | .highlight .kr { font-weight: bold } /* Keyword.Reserved */
27 | .highlight .kt { color: #445588; font-weight: bold } /* Keyword.Type */
28 | .highlight .m { color: #009999 } /* Literal.Number */
29 | .highlight .s { color: #d14 } /* Literal.String */
30 | .highlight .na { color: #008080 } /* Name.Attribute */
31 | .highlight .nb { color: #0086B3 } /* Name.Builtin */
32 | .highlight .nc { color: #445588; font-weight: bold } /* Name.Class */
33 | .highlight .no { color: #008080 } /* Name.Constant */
34 | .highlight .ni { color: #800080 } /* Name.Entity */
35 | .highlight .ne { color: #990000; font-weight: bold } /* Name.Exception */
36 | .highlight .nf { color: #990000; font-weight: bold } /* Name.Function */
37 | .highlight .nn { color: #555555 } /* Name.Namespace */
38 | .highlight .nt { color: #000080 } /* Name.Tag */
39 | .highlight .nv { color: #008080 } /* Name.Variable */
40 | .highlight .ow { font-weight: bold } /* Operator.Word */
41 | .highlight .w { color: #bbbbbb } /* Text.Whitespace */
42 | .highlight .mf { color: #009999 } /* Literal.Number.Float */
43 | .highlight .mh { color: #009999 } /* Literal.Number.Hex */
44 | .highlight .mi { color: #009999 } /* Literal.Number.Integer */
45 | .highlight .mo { color: #009999 } /* Literal.Number.Oct */
46 | .highlight .sb { color: #d14 } /* Literal.String.Backtick */
47 | .highlight .sc { color: #d14 } /* Literal.String.Char */
48 | .highlight .sd { color: #d14 } /* Literal.String.Doc */
49 | .highlight .s2 { color: #d14 } /* Literal.String.Double */
50 | .highlight .se { color: #d14 } /* Literal.String.Escape */
51 | .highlight .sh { color: #d14 } /* Literal.String.Heredoc */
52 | .highlight .si { color: #d14 } /* Literal.String.Interpol */
53 | .highlight .sx { color: #d14 } /* Literal.String.Other */
54 | .highlight .sr { color: #009926 } /* Literal.String.Regex */
55 | .highlight .s1 { color: #d14 } /* Literal.String.Single */
56 | .highlight .ss { color: #990073 } /* Literal.String.Symbol */
57 | .highlight .bp { color: #999999 } /* Name.Builtin.Pseudo */
58 | .highlight .vc { color: #008080 } /* Name.Variable.Class */
59 | .highlight .vg { color: #008080 } /* Name.Variable.Global */
60 | .highlight .vi { color: #008080 } /* Name.Variable.Instance */
61 | .highlight .il { color: #009999 } /* Literal.Number.Integer.Long */
62 |
63 | .type-csharp .highlight .k { color: #0000FF }
64 | .type-csharp .highlight .kt { color: #0000FF }
65 | .type-csharp .highlight .nf { color: #000000; font-weight: normal }
66 | .type-csharp .highlight .nc { color: #2B91AF }
67 | .type-csharp .highlight .nn { color: #000000 }
68 | .type-csharp .highlight .s { color: #A31515 }
69 | .type-csharp .highlight .sc { color: #A31515 }
70 |
--------------------------------------------------------------------------------
/docs/stylesheets/stylesheet.css:
--------------------------------------------------------------------------------
1 | * {
2 | box-sizing: border-box; }
3 |
4 | body {
5 | padding: 0;
6 | margin: 0;
7 | font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
8 | font-size: 16px;
9 | line-height: 1.5;
10 | color: #606c71; }
11 |
12 | a {
13 | color: #1e6bb8;
14 | text-decoration: none; }
15 | a:hover {
16 | text-decoration: underline; }
17 |
18 | .btn {
19 | display: inline-block;
20 | margin-bottom: 0.4rem;
21 | color: rgba(255, 255, 255, 0.7);
22 | background-color: rgba(255, 255, 255, 0.08);
23 | border-color: rgba(255, 255, 255, 0.2);
24 | border-style: solid;
25 | border-width: 1px;
26 | border-radius: 0.3rem;
27 | transition: color 0.2s, background-color 0.2s, border-color 0.2s; }
28 | .btn + .btn {
29 | margin-left: 0.2rem; }
30 |
31 | .btn:hover {
32 | color: rgba(255, 255, 255, 0.8);
33 | text-decoration: none;
34 | background-color: rgba(255, 255, 255, 0.2);
35 | border-color: rgba(255, 255, 255, 0.3); }
36 |
37 | @media screen and (min-width: 64em) {
38 | .btn {
39 | padding: 0.2rem 0.4rem;
40 | font-size: 0.9rem;
41 | } }
42 |
43 | @media screen and (min-width: 42em) and (max-width: 64em) {
44 | .btn {
45 | padding: 0.2rem 0.4rem;
46 | font-size: 0.9rem; } }
47 |
48 | @media screen and (max-width: 42em) {
49 | .btn {
50 | display: block;
51 | width: 100%;
52 | padding: 0.2rem;
53 | font-size: 0.9rem; }
54 | .btn + .btn {
55 | margin-top: 0.4rem;
56 | margin-left: 0; } }
57 |
58 | .page-header {
59 | color: #fff;
60 | text-align: center;
61 | background-color: #363899;
62 | background-image: linear-gradient(90deg, #000000, #99120f); }
63 |
64 | .header-image {
65 | align: middle;
66 | height: 140px;
67 | }
68 |
69 | @media screen and (min-width: 64em) {
70 | .page-header {
71 | padding: 1rem 1rem; } }
72 |
73 | @media screen and (min-width: 42em) and (max-width: 64em) {
74 | .page-header {
75 | padding: 1rem 1rem; } }
76 |
77 | @media screen and (max-width: 42em) {
78 | .page-header {
79 | padding: 0rem 0rem; } }
80 |
81 | .project-name {
82 | margin-top: 0;
83 | margin-bottom: 0.1rem; }
84 |
85 | @media screen and (min-width: 64em) {
86 | .project-name {
87 | font-size: 3.25rem; } }
88 |
89 | @media screen and (min-width: 42em) and (max-width: 64em) {
90 | .project-name {
91 | font-size: 2.25rem; } }
92 |
93 | @media screen and (max-width: 42em) {
94 | .project-name {
95 | font-size: 1.75rem; } }
96 |
97 | .project-tagline {
98 | margin-bottom: 0rem;
99 | font-weight: normal;
100 | opacity: 0.7; }
101 |
102 | @media screen and (min-width: 64em) {
103 | .project-tagline {
104 | font-size: 1.25rem; } }
105 |
106 | @media screen and (min-width: 42em) and (max-width: 64em) {
107 | .project-tagline {
108 | font-size: 1.15rem; } }
109 |
110 | @media screen and (max-width: 42em) {
111 | .project-tagline {
112 | font-size: 1rem; } }
113 |
114 | .main-content :first-child {
115 | margin-top: 0; }
116 | .main-content img {
117 | max-width: 100%; }
118 | .main-content h1, .main-content h2, .main-content h3, .main-content h4, .main-content h5, .main-content h6 {
119 | margin-top: 2rem;
120 | margin-bottom: 1rem;
121 | font-weight: normal;
122 | color: #159957; }
123 | .main-content p {
124 | margin-bottom: 1em; }
125 | .main-content code {
126 | padding: 2px 4px;
127 | font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace;
128 | font-size: 0.9rem;
129 | color: #383e41;
130 | background-color: #f3f6fa;
131 | border-radius: 0.3rem; }
132 | .main-content pre {
133 | padding: 0.8rem;
134 | margin-top: 0;
135 | margin-bottom: 1rem;
136 | font: 1rem Consolas, "Liberation Mono", Menlo, Courier, monospace;
137 | color: #567482;
138 | word-wrap: normal;
139 | background-color: #f3f6fa;
140 | border: solid 1px #dce6f0;
141 | border-radius: 0.3rem; }
142 | .main-content pre > code {
143 | padding: 0;
144 | margin: 0;
145 | font-size: 0.9rem;
146 | color: #567482;
147 | word-break: normal;
148 | white-space: pre;
149 | background: transparent;
150 | border: 0; }
151 | .main-content .highlight {
152 | margin-bottom: 1rem; }
153 | .main-content .highlight pre {
154 | margin-bottom: 0;
155 | word-break: normal; }
156 | .main-content .highlight pre, .main-content pre {
157 | padding: 0.8rem;
158 | overflow: auto;
159 | font-size: 0.9rem;
160 | line-height: 1.45;
161 | border-radius: 0.3rem; }
162 | .main-content pre code, .main-content pre tt {
163 | display: inline;
164 | max-width: initial;
165 | padding: 0;
166 | margin: 0;
167 | overflow: initial;
168 | line-height: inherit;
169 | word-wrap: normal;
170 | background-color: transparent;
171 | border: 0; }
172 | .main-content pre code:before, .main-content pre code:after, .main-content pre tt:before, .main-content pre tt:after {
173 | content: normal; }
174 | .main-content ul, .main-content ol {
175 | margin-top: 0; }
176 | .main-content blockquote {
177 | padding: 0 1rem;
178 | margin-left: 0;
179 | color: #819198;
180 | border-left: 0.3rem solid #dce6f0; }
181 | .main-content blockquote > :first-child {
182 | margin-top: 0; }
183 | .main-content blockquote > :last-child {
184 | margin-bottom: 0; }
185 | .main-content table {
186 | display: block;
187 | width: 100%;
188 | overflow: auto;
189 | word-break: normal;
190 | word-break: keep-all; }
191 | .main-content table th {
192 | font-weight: bold; }
193 | .main-content table th, .main-content table td {
194 | padding: 0.5rem 1rem;
195 | border: 1px solid #e9ebec; }
196 | .main-content dl {
197 | padding: 0; }
198 | .main-content dl dt {
199 | padding: 0;
200 | margin-top: 1rem;
201 | font-size: 1rem;
202 | font-weight: bold; }
203 | .main-content dl dd {
204 | padding: 0;
205 | margin-bottom: 1rem; }
206 | .main-content hr {
207 | height: 2px;
208 | padding: 0;
209 | margin: 1rem 0;
210 | background-color: #eff0f1;
211 | border: 0; }
212 |
213 | @media screen and (min-width: 64em) {
214 | .main-content {
215 | padding: 2rem 6rem;
216 | margin: 0 auto;
217 | font-size: 1.1rem; } }
218 |
219 | @media screen and (min-width: 42em) and (max-width: 64em) {
220 | .main-content {
221 | padding: 2rem 4rem;
222 | font-size: 1.1rem; } }
223 |
224 | @media screen and (max-width: 42em) {
225 | .main-content {
226 | padding: 2rem 1rem;
227 | font-size: 1rem; } }
228 |
229 | .site-footer {
230 | padding-top: 2rem;
231 | margin-top: 2rem;
232 | border-top: solid 1px #eff0f1; }
233 |
234 | .site-footer-owner {
235 | display: block;
236 | font-weight: bold; }
237 |
238 | .site-footer-credits {
239 | color: #819198; }
240 |
241 | @media screen and (min-width: 64em) {
242 | .site-footer {
243 | font-size: 1rem; } }
244 |
245 | @media screen and (min-width: 42em) and (max-width: 64em) {
246 | .site-footer {
247 | font-size: 1rem; } }
248 |
249 | @media screen and (max-width: 42em) {
250 | .site-footer {
251 | font-size: 0.9rem; } }
252 |
--------------------------------------------------------------------------------
/docs/stylesheets/table.css:
--------------------------------------------------------------------------------
1 | table#compare tr:nth-child(even) {
2 | background-color: #eee;
3 | }
4 | table#compare tr:nth-child(odd) {
5 | background-color: #fff;
6 | }
7 | table#compare th {
8 | color: white;
9 | background-color: black;
10 | }
11 |
--------------------------------------------------------------------------------
/iostreams.gemspec:
--------------------------------------------------------------------------------
1 | lib = File.expand_path("lib", __dir__)
2 | $:.unshift lib unless $:.include?(lib)
3 |
4 | # Maintain your gem's version:
5 | require "io_streams/version"
6 |
7 | # Describe your gem and declare its dependencies:
8 | Gem::Specification.new do |s|
9 | s.name = "iostreams"
10 | s.version = IOStreams::VERSION
11 | s.platform = Gem::Platform::RUBY
12 | s.authors = ["Reid Morrison"]
13 | s.homepage = "https://iostreams.rocketjob.io"
14 | s.summary = "Input and Output streaming for Ruby."
15 | s.files = Dir["lib/**/*", "bin/*", "LICENSE", "Rakefile", "README.md"]
16 | s.test_files = Dir["test/**/*"]
17 | s.license = "Apache-2.0"
18 | s.required_ruby_version = ">= 2.5"
19 | end
20 |
--------------------------------------------------------------------------------
/lib/io_streams/builder.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | # Build the streams that need to be applied to a path during reading or writing.
3 | class Builder
4 | attr_accessor :file_name, :format_options
5 | attr_reader :streams, :options
6 |
7 | def initialize(file_name = nil)
8 | @file_name = file_name
9 | @streams = nil
10 | @options = nil
11 | @format = nil
12 | @format_option = nil
13 | end
14 |
15 | # Supply an option that is only applied once the file name extensions have been parsed.
16 | # Note:
17 | # - Cannot set both `stream` and `option`
18 | def option(stream, **options)
19 | stream = stream.to_sym unless stream.is_a?(Symbol)
20 | raise(ArgumentError, "Invalid stream: #{stream.inspect}") unless IOStreams.extensions.include?(stream)
21 | raise(ArgumentError, "Cannot call both #option and #stream on the same streams instance}") if @streams
22 | raise(ArgumentError, "Cannot call #option unless the `file_name` was already set}") unless file_name
23 |
24 | @options ||= {}
25 | if (opts = @options[stream])
26 | opts.merge!(options)
27 | else
28 | @options[stream] = options.dup
29 | end
30 | self
31 | end
32 |
33 | def stream(stream, **options)
34 | stream = stream.to_sym unless stream.is_a?(Symbol)
35 | raise(ArgumentError, "Cannot call both #option and #stream on the same streams instance}") if @options
36 |
37 | # To prevent any streams from being applied supply a stream named `:none`
38 | if stream == :none
39 | @streams = {}
40 | return self
41 | end
42 | raise(ArgumentError, "Invalid stream: #{stream.inspect}") unless IOStreams.extensions.include?(stream)
43 |
44 | @streams ||= {}
45 | if (opts = @streams[stream])
46 | opts.merge!(options)
47 | else
48 | @streams[stream] = options.dup
49 | end
50 | self
51 | end
52 |
53 | def option_or_stream(stream, **options)
54 | if streams
55 | stream(stream, **options)
56 | elsif file_name
57 | option(stream, **options)
58 | else
59 | stream(stream, **options)
60 | end
61 | end
62 |
63 | # Return the options set for either a stream or option.
64 | def setting(stream)
65 | return streams[stream] if streams
66 |
67 | options[stream] if options
68 | end
69 |
70 | def reader(io_stream, &block)
71 | execute(:reader, pipeline, io_stream, &block)
72 | end
73 |
74 | def writer(io_stream, &block)
75 | execute(:writer, pipeline, io_stream, &block)
76 | end
77 |
78 | # Returns [Hash] the pipeline of streams
79 | # with their options that will be applied when the reader or writer is invoked.
80 | def pipeline
81 | return streams.dup.freeze if streams
82 |
83 | build_pipeline.freeze
84 | end
85 |
86 | # Removes the named stream from the current pipeline.
87 | # If the stream pipeline has not yet been built it will be built from the file_name if present.
88 | # Note: Any options must be set _before_ calling this method.
89 | def remove_from_pipeline(stream_name)
90 | @streams ||= build_pipeline
91 | @streams.delete(stream_name.to_sym)
92 | end
93 |
94 | # Returns the tabular format if set, otherwise tries to autodetect the format if the file_name has been set
95 | # Returns [nil] if no format is set, or if it cannot be determined from the file_name
96 | def format
97 | @format ||= file_name ? Tabular.format_from_file_name(file_name) : nil
98 | end
99 |
100 | def format=(format)
101 | unless format.nil? || IOStreams::Tabular.registered_formats.include?(format)
102 | raise(ArgumentError, "Invalid format: #{format.inspect}")
103 | end
104 |
105 | @format = format
106 | end
107 |
108 | private
109 |
110 | def build_pipeline
111 | return {} unless file_name
112 |
113 | built_streams = {}
114 | # Encode stream is always first
115 | built_streams[:encode] = options[:encode] if options&.key?(:encode)
116 |
117 | opts = options || {}
118 | parse_extensions.each { |stream| built_streams[stream] = opts[stream] || {} }
119 | built_streams
120 | end
121 |
122 | def class_for_stream(type, stream)
123 | ext = IOStreams.extensions[stream.nil? ? nil : stream.to_sym] ||
124 | raise(ArgumentError, "Unknown Stream type: #{stream.inspect}")
125 | ext.send("#{type}_class") || raise(ArgumentError, "No #{type} registered for Stream type: #{stream.inspect}")
126 | end
127 |
128 | # Returns the streams for the supplied file_name
129 | def parse_extensions
130 | parts = ::File.basename(file_name).split(".")
131 | extensions = []
132 | while (extension = parts.pop)
133 | sym = extension.downcase.to_sym
134 | break unless IOStreams.extensions[sym]
135 |
136 | extensions.unshift(sym)
137 | end
138 | extensions
139 | end
140 |
141 | # Executes the streams that need to be executed.
142 | def execute(type, pipeline, io_stream, &block)
143 | raise(ArgumentError, "IOStreams call is missing mandatory block") if block.nil?
144 |
145 | if pipeline.empty?
146 | block.call(io_stream)
147 | elsif pipeline.size == 1
148 | stream, opts = pipeline.first
149 | class_for_stream(type, stream).open(io_stream, **opts, &block)
150 | else
151 | # Daisy chain multiple streams together
152 | last = pipeline.keys.inject(block) do |inner, stream_sym|
153 | ->(io) { class_for_stream(type, stream_sym).open(io, **pipeline[stream_sym], &inner) }
154 | end
155 | last.call(io_stream)
156 | end
157 | end
158 | end
159 | end
160 |
--------------------------------------------------------------------------------
/lib/io_streams/bzip2/reader.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | module Bzip2
3 | class Reader < IOStreams::Reader
4 | # Read from a Bzip2 stream, decompressing the contents as it is read
5 | def self.stream(input_stream, **args)
6 | Utils.load_soft_dependency("bzip2-ffi", "Bzip2", "bzip2/ffi") unless defined?(::Bzip2::FFI)
7 |
8 | begin
9 | io = ::Bzip2::FFI::Reader.new(input_stream, args)
10 | yield io
11 | ensure
12 | io&.close
13 | end
14 | end
15 | end
16 | end
17 | end
18 |
--------------------------------------------------------------------------------
/lib/io_streams/bzip2/writer.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | module Bzip2
3 | class Writer < IOStreams::Writer
4 | # Write to a stream, compressing with Bzip2
5 | def self.stream(input_stream, original_file_name: nil, **args)
6 | Utils.load_soft_dependency("bzip2-ffi", "Bzip2", "bzip2/ffi") unless defined?(::Bzip2::FFI)
7 |
8 | begin
9 | io = ::Bzip2::FFI::Writer.new(input_stream, args)
10 | yield io
11 | ensure
12 | io&.close
13 | end
14 | end
15 | end
16 | end
17 | end
18 |
--------------------------------------------------------------------------------
/lib/io_streams/encode/reader.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | module Encode
3 | class Reader < IOStreams::Reader
4 | attr_reader :encoding, :cleaner
5 |
6 | NOT_PRINTABLE = Regexp.compile(/[^[:print:]|\r|\n]/).freeze
7 | # Builtin strip options to apply after encoding the read data.
8 | CLEANSE_RULES = {
9 | # Strips all non printable characters
10 | printable: ->(data, _) { data.gsub!(NOT_PRINTABLE, "") || data },
11 | # Replaces non printable characters with the value specified in the `replace` option.
12 | replace_non_printable: ->(data, replace) { data.gsub!(NOT_PRINTABLE, replace || "") || data }
13 | }.freeze
14 |
15 | # Read a line at a time from a file or stream
16 | def self.stream(input_stream, original_file_name: nil, **args)
17 | yield new(input_stream, **args)
18 | end
19 |
20 | # Apply encoding conversion when reading a stream.
21 | #
22 | # Parameters
23 | # input_stream
24 | # The input stream that implements #read
25 | #
26 | # encoding: [String|Encoding]
27 | # Encode returned data with this encoding.
28 | # 'US-ASCII': Original 7 bit ASCII Format
29 | # 'ASCII-8BIT': 8-bit ASCII Format
30 | # 'UTF-8': UTF-8 Format
31 | # Etc.
32 | # Default: 'UTF-8'
33 | #
34 | # replace: [String]
35 | # The character to replace with when a character cannot be converted to the target encoding.
36 | # nil: Don't replace any invalid characters. Encoding::UndefinedConversionError is raised.
37 | # Default: nil
38 | #
39 | # cleaner: [nil|symbol|Proc]
40 | # Cleanse data read from the input stream.
41 | # nil: No cleansing
42 | # :printable Cleanse all non-printable characters except \r and \n
43 | # Proc/lambda Proc to call after every read to cleanse the data
44 | # Default: nil
45 | def initialize(input_stream, encoding: "UTF-8", cleaner: nil, replace: nil)
46 | super(input_stream)
47 |
48 | @cleaner = self.class.extract_cleaner(cleaner)
49 | @encoding = encoding.nil? || encoding.is_a?(Encoding) ? encoding : Encoding.find(encoding)
50 | @encoding_options = replace.nil? ? {} : {invalid: :replace, undef: :replace, replace: replace}
51 | @replace = replace
52 |
53 | # More efficient read buffering only supported when the input stream `#read` method supports it.
54 | @read_cache_buffer = ("".encode(@encoding) if replace.nil? && !@input_stream.method(:read).arity.between?(0, 1))
55 | end
56 |
57 | # Returns [String] data returned from the input stream.
58 | # Returns [nil] if end of file and no further data was read.
59 | def read(size = nil)
60 | block =
61 | if @read_cache_buffer
62 | begin
63 | @input_stream.read(size, @read_cache_buffer)
64 | rescue ArgumentError
65 | # Handle arity of -1 when just 0..1
66 | @read_cache_buffer = nil
67 | @input_stream.read(size)
68 | end
69 | else
70 | @input_stream.read(size)
71 | end
72 |
73 | # EOF reached?
74 | return unless block
75 |
76 | block = block.encode(@encoding, **@encoding_options) unless block.encoding == @encoding
77 | block = @cleaner.call(block, @replace) if @cleaner
78 | block
79 | end
80 |
81 | def self.extract_cleaner(cleaner)
82 | return if cleaner.nil?
83 |
84 | case cleaner
85 | when Symbol
86 | proc = CLEANSE_RULES[cleaner]
87 | raise(ArgumentError, "Invalid cleansing rule #{cleaner.inspect}") unless proc
88 |
89 | proc
90 | when Proc
91 | cleaner
92 | end
93 | end
94 | end
95 | end
96 | end
97 |
--------------------------------------------------------------------------------
/lib/io_streams/encode/writer.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | module Encode
3 | class Writer < IOStreams::Writer
4 | attr_reader :encoding, :cleaner
5 |
6 | # Write a line at a time to a file or stream
7 | def self.stream(input_stream, original_file_name: nil, **args)
8 | yield new(input_stream, **args)
9 | end
10 |
11 | # A delimited stream writer that will write to the supplied output stream
12 | # Written data is encoded prior to writing.
13 | #
14 | # Parameters
15 | # output_stream
16 | # The output stream that implements #write
17 | #
18 | # encoding: [String|Encoding]
19 | # Encode returned data with this encoding.
20 | # 'US-ASCII': Original 7 bit ASCII Format
21 | # 'ASCII-8BIT': 8-bit ASCII Format
22 | # 'UTF-8': UTF-8 Format
23 | # Etc.
24 | # Default: 'UTF-8'
25 | #
26 | # replace: [String]
27 | # The character to replace with when a character cannot be converted to the target encoding.
28 | # nil: Don't replace any invalid characters. Encoding::UndefinedConversionError is raised.
29 | # Default: nil
30 | #
31 | # cleaner: [nil|symbol|Proc]
32 | # Cleanse data read from the input stream.
33 | # nil: No cleansing
34 | # :printable Cleanse all non-printable characters except \r and \n
35 | # Proc/lambda Proc to call after every read to cleanse the data
36 | # Default: nil
37 | def initialize(output_stream, encoding: "UTF-8", cleaner: nil, replace: nil)
38 | super(output_stream)
39 |
40 | @cleaner = ::IOStreams::Encode::Reader.send(:extract_cleaner, cleaner)
41 | @encoding = encoding.nil? || encoding.is_a?(Encoding) ? encoding : Encoding.find(encoding)
42 | @encoding_options = replace.nil? ? {} : {invalid: :replace, undef: :replace, replace: replace}
43 | @replace = replace
44 | end
45 |
46 | # Write a line to the output stream
47 | #
48 | # Example:
49 | # IOStreams.writer('a.txt', encoding: 'UTF-8') do |stream|
50 | # stream << 'first line' << 'second line'
51 | # end
52 | def <<(record)
53 | write(record)
54 | self
55 | end
56 |
57 | # Write a line to the output stream followed by the delimiter.
58 | # Returns [Integer] the number of bytes written.
59 | #
60 | # Example:
61 | # IOStreams.writer('a.txt', encoding: 'UTF-8') do |stream|
62 | # count = stream.write('first line')
63 | # puts "Wrote #{count} bytes to the output file, including the delimiter"
64 | # end
65 | def write(data)
66 | return 0 if data.nil?
67 |
68 | data = data.to_s
69 | block = data.encoding == @encoding ? data : data.encode(@encoding, **@encoding_options)
70 | block = @cleaner.call(block, @replace) if @cleaner
71 | @output_stream.write(block)
72 | end
73 | end
74 | end
75 | end
76 |
--------------------------------------------------------------------------------
/lib/io_streams/errors.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | module Errors
3 | class Error < StandardError
4 | end
5 |
6 | class InvalidHeader < Error
7 | end
8 |
9 | class MissingHeader < Error
10 | end
11 |
12 | class UnknownFormat < Error
13 | end
14 |
15 | class TypeMismatch < Error
16 | end
17 |
18 | class CommunicationsFailure < Error
19 | end
20 |
21 | # When the specified delimiter is not found in the supplied stream / file
22 | class DelimiterNotFound < Error
23 | end
24 |
25 | # Fixed length line has the wrong length
26 | class InvalidLineLength < Error
27 | end
28 |
29 | class ValueTooLong < Error
30 | end
31 |
32 | class MalformedDataError < RuntimeError
33 | attr_reader :line_number
34 |
35 | def initialize(message, line_number)
36 | @line_number = line_number
37 | super("#{message} on line #{line_number}.")
38 | end
39 | end
40 |
41 | class InvalidLayout < Error
42 | end
43 | end
44 | end
45 |
--------------------------------------------------------------------------------
/lib/io_streams/gzip/reader.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | module Gzip
3 | class Reader < IOStreams::Reader
4 | # Read from a gzip stream, decompressing the contents as it is read
5 | def self.stream(input_stream, original_file_name: nil)
6 | io = ::Zlib::GzipReader.new(input_stream)
7 | yield io
8 | ensure
9 | io&.close
10 | end
11 | end
12 | end
13 | end
14 |
--------------------------------------------------------------------------------
/lib/io_streams/gzip/writer.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | module Gzip
3 | class Writer < IOStreams::Writer
4 | # Write to a stream, compressing with GZip
5 | def self.stream(input_stream, original_file_name: nil, &block)
6 | io = ::Zlib::GzipWriter.new(input_stream)
7 | block.call(io)
8 | ensure
9 | io&.close
10 | end
11 | end
12 | end
13 | end
14 |
--------------------------------------------------------------------------------
/lib/io_streams/line/reader.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | module Line
3 | class Reader < IOStreams::Reader
4 | attr_reader :delimiter, :buffer_size, :line_number
5 |
6 | # Prevent denial of service when a delimiter is not found before this number * `buffer_size` characters are read.
7 | MAX_BLOCKS_MULTIPLIER = 100
8 |
9 | LINEFEED_REGEXP = Regexp.compile(/\r\n|\n|\r/).freeze
10 |
11 | # Read a line at a time from a stream
12 | def self.stream(input_stream, **args)
13 | # Pass-through if already a line reader
14 | return yield(input_stream) if input_stream.is_a?(self.class)
15 |
16 | yield new(input_stream, **args)
17 | end
18 |
19 | # Create a delimited stream reader from the supplied input stream.
20 | #
21 | # Lines returned will be in the encoding of the input stream.
22 | # To change the encoding of returned lines, use IOStreams::Encode::Reader.
23 | #
24 | # Parameters
25 | # input_stream
26 | # The input stream that implements #read
27 | #
28 | # delimiter: [String]
29 | # Line / Record delimiter to use to break the stream up into records
30 | # Any string to break the stream up by.
31 | # This delimiter is removed from each line when `#each` or `#readline` is called.
32 | # Default: nil
33 | # Automatically detect line endings and break up by line
34 | # Searches for the first "\r\n" or "\n" and then uses that as the
35 | # delimiter for all subsequent records.
36 | #
37 | # buffer_size: [Integer]
38 | # Size of blocks to read from the input stream at a time.
39 | # Default: 65536 ( 64K )
40 | #
41 | # embedded_within: [String]
42 | # Supports CSV files where a line may contain an embedded newline.
43 | # For CSV files set `embedded_within: '"'`
44 | #
45 | # Note:
46 | # * When using a line reader and the file_name ends with ".csv" then embedded_within is automatically set to `"`
47 | def initialize(input_stream, delimiter: nil, buffer_size: 65_536, embedded_within: nil, original_file_name: nil)
48 | super(input_stream)
49 |
50 | @embedded_within = embedded_within
51 | @buffer_size = buffer_size
52 |
53 | # More efficient read buffering only supported when the input stream `#read` method supports it.
54 | @use_read_cache_buffer = !@input_stream.method(:read).arity.between?(0, 1)
55 |
56 | @line_number = 0
57 | @eof = false
58 | @read_cache_buffer = nil
59 | @buffer = nil
60 | @delimiter = delimiter
61 |
62 | read_block
63 | # Auto-detect windows/linux line endings if not supplied. \n or \r\n
64 | @delimiter ||= auto_detect_line_endings
65 |
66 | return unless @buffer
67 |
68 | # Change the delimiters encoding to match that of the input stream
69 | @delimiter = @delimiter.encode(@buffer.encoding)
70 | @delimiter_size = @delimiter.size
71 | end
72 |
73 | # Iterate over every line in the file/stream passing each line to supplied block in turn.
74 | # Returns [Integer] the number of lines read from the file/stream.
75 | # Note:
76 | # * The line delimiter is _not_ returned.
77 | def each
78 | line_count = 0
79 | until eof?
80 | line = readline
81 | unless line.nil?
82 | yield(line)
83 | line_count += 1
84 | end
85 | end
86 | line_count
87 | end
88 |
89 | # Reads each line per the `delimeter`.
90 | # Accounts for lines that contain the `delimiter` when the `delimeter` is within the `embedded_within` delimiter.
91 | # For Example, CSV files can contain newlines embedded within double quotes.
92 | def readline
93 | line = _readline
94 | if line && @embedded_within
95 | initial_line_number = @line_number
96 | while line.count(@embedded_within).odd?
97 | if eof? || line.length > @buffer_size * 10
98 | raise(Errors::MalformedDataError.new(
99 | "Unbalanced delimited field, delimiter: #{@embedded_within}",
100 | initial_line_number
101 | ))
102 | end
103 | line << @delimiter
104 | next_line = _readline
105 | if next_line.nil?
106 | raise(Errors::MalformedDataError.new(
107 | "Unbalanced delimited field, delimiter: #{@embedded_within}",
108 | initial_line_number
109 | ))
110 | end
111 | line << next_line
112 | end
113 | end
114 | line
115 | end
116 |
117 | # Returns whether the end of file has been reached for this stream
118 | def eof?
119 | @eof && (@buffer.nil? || @buffer.empty?)
120 | end
121 |
122 | private
123 |
124 | def _readline
125 | return if eof?
126 |
127 | # Keep reading until it finds the delimiter
128 | while (index = @buffer.index(@delimiter)).nil? && read_block
129 | end
130 |
131 | # Delimiter found?
132 | if index
133 | data = @buffer.slice(0, index)
134 | @buffer = @buffer.slice(index + @delimiter_size, @buffer.size)
135 | @line_number += 1
136 | elsif @eof && @buffer.empty?
137 | data = nil
138 | @buffer = nil
139 | else
140 | # Last line without delimiter
141 | data = @buffer
142 | @buffer = nil
143 | @line_number += 1
144 | end
145 |
146 | data
147 | end
148 |
149 | # Returns whether more data is available to read
150 | # Returns false on EOF
151 | def read_block
152 | return false if @eof
153 |
154 | block =
155 | if @read_cache_buffer
156 | begin
157 | @input_stream.read(@buffer_size, @read_cache_buffer)
158 | rescue ArgumentError
159 | # Handle arity of -1 when just 0..1
160 | @read_cache_buffer = nil
161 | @use_read_cache_buffer = false
162 | @input_stream.read(@buffer_size)
163 | end
164 | else
165 | @input_stream.read(@buffer_size)
166 | end
167 |
168 | # EOF reached?
169 | if block.nil?
170 | @eof = true
171 | return false
172 | end
173 |
174 | if @buffer
175 | @buffer << block
176 | else
177 | # Take on the encoding from the input stream
178 | @buffer = block.dup
179 | # Take on the encoding from the first block that was read.
180 | @read_cache_buffer = "".encode(block.encoding) if @use_read_cache_buffer
181 | end
182 |
183 | if @buffer.size > MAX_BLOCKS_MULTIPLIER * @buffer_size
184 | raise(
185 | Errors::DelimiterNotFound,
186 | "Delimiter: #{@delimiter.inspect} not found after reading #{@buffer.size} bytes."
187 | )
188 | end
189 |
190 | true
191 | end
192 |
193 | # Auto-detect windows/linux line endings: \n, \r or \r\n
194 | def auto_detect_line_endings
195 | return "\n" if @buffer.nil? && !read_block
196 |
197 | # Could be "\r\n" broken in half by the block size
198 | read_block if @buffer[-1] == "\r"
199 |
200 | # Delimiter takes on the encoding from @buffer
201 | delimiter = @buffer.slice(LINEFEED_REGEXP)
202 | return delimiter if delimiter
203 |
204 | while read_block
205 | # Could be "\r\n" broken in half by the block size
206 | read_block if @buffer[-1] == "\r"
207 |
208 | # Delimiter takes on the encoding from @buffer
209 | delimiter = @buffer.slice(LINEFEED_REGEXP)
210 | return delimiter if delimiter
211 | end
212 |
213 | # One line files with no delimiter
214 | "\n"
215 | end
216 | end
217 | end
218 | end
219 |
--------------------------------------------------------------------------------
/lib/io_streams/line/writer.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | module Line
3 | class Writer < IOStreams::Writer
4 | attr_reader :delimiter
5 |
6 | # Write a line at a time to a stream.
7 | def self.stream(output_stream, **args)
8 | # Pass-through if already a line writer
9 | return yield(output_stream) if output_stream.is_a?(self.class)
10 |
11 | yield new(output_stream, **args)
12 | end
13 |
14 | # A delimited stream writer that will write to the supplied output stream.
15 | #
16 | # The output stream will have the encoding of data written to it.
17 | # To change the output encoding, use IOStreams::Encode::Writer.
18 | #
19 | # Parameters
20 | # output_stream
21 | # The output stream that implements #write
22 | #
23 | # delimiter: [String]
24 | # Add the specified delimiter after every record when writing it
25 | # to the output stream
26 | # Default: OS Specific. Linux: "\n"
27 | def initialize(output_stream, delimiter: $/, original_file_name: nil)
28 | super(output_stream)
29 | @delimiter = delimiter
30 | end
31 |
32 | # Write a line to the output stream
33 | #
34 | # Example:
35 | # IOStreams.path('a.txt').writer(:line) do |stream|
36 | # stream << 'first line' << 'second line'
37 | # end
38 | def <<(data)
39 | write(data)
40 | self
41 | end
42 |
43 | # Write a line to the output stream followed by the delimiter.
44 | # Returns [Integer] the number of bytes written.
45 | #
46 | # Example:
47 | # IOStreams.path('a.txt').writer(:line) do |stream|
48 | # count = stream.write('first line')
49 | # puts "Wrote #{count} bytes to the output file, including the delimiter"
50 | # end
51 | def write(data)
52 | output_stream.write(data.to_s + delimiter)
53 | end
54 | end
55 | end
56 | end
57 |
--------------------------------------------------------------------------------
/lib/io_streams/path.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | class Path < IOStreams::Stream
3 | attr_accessor :path
4 |
5 | def initialize(path)
6 | raise(ArgumentError, "Path cannot be nil") if path.nil?
7 | raise(ArgumentError, "Path must be a string: #{path.inspect}, class: #{path.class}") unless path.is_a?(String)
8 |
9 | @path = path.frozen? ? path : path.dup.freeze
10 | @io_stream = nil
11 | @builder = nil
12 | end
13 |
14 | # If elements already contains the current path then it is used as is without
15 | # adding the current path for a second time
16 | def join(*elements)
17 | return self if elements.empty?
18 |
19 | elements = elements.collect(&:to_s)
20 | relative = ::File.join(*elements)
21 |
22 | new_path = dup
23 | new_path.builder = nil
24 | new_path.path = relative.start_with?(path) ? relative : ::File.join(path, relative)
25 | new_path
26 | end
27 |
28 | def relative?
29 | !absolute?
30 | end
31 |
32 | def absolute?
33 | !!(path.strip =~ %r{\A/})
34 | end
35 |
36 | # By default realpath just returns self.
37 | def realpath
38 | self
39 | end
40 |
41 | # Runs the pattern from the current path, returning the complete path for located files.
42 | #
43 | # See IOStreams::Paths::File.each for arguments.
44 | def each_child(pattern = "*", **args, &block)
45 | raise NotImplementedError
46 | end
47 |
48 | # Returns [Array] of child files based on the supplied pattern
49 | def children(*args, **kargs)
50 | paths = []
51 | each_child(*args, **kargs) { |path| paths << path }
52 | paths
53 | end
54 |
55 | # Returns [String] the current path.
56 | def to_s
57 | path
58 | end
59 |
60 | # Removes the last element of the path, the file name, before creating the entire path.
61 | # Returns self
62 | def mkpath
63 | raise NotImplementedError
64 | end
65 |
66 | # Assumes the current path does not include a file name, and creates all elements in the path.
67 | # Returns self
68 | #
69 | # Note: Do not call this method if the path contains a file name, see `#mkpath`
70 | def mkdir
71 | raise NotImplementedError
72 | end
73 |
74 | # Returns [true|false] whether the file exists
75 | def exist?
76 | raise NotImplementedError
77 | end
78 |
79 | # Returns [Integer] size of the file
80 | def size
81 | raise NotImplementedError
82 | end
83 |
84 | # Cleanup an incomplete write to the target "file" if the copy fails.
85 | # rubocop:disable Lint/SuppressedException
86 | def copy_from(source, **args)
87 | super(source, **args)
88 | rescue StandardError => e
89 | begin
90 | delete
91 | rescue NotImplementedError
92 | end
93 | raise(e)
94 | end
95 | # rubocop:enable Lint/SuppressedException
96 |
97 | # Moves the file by copying it to the new path and then deleting the current path.
98 | # Returns [IOStreams::Path] the target path.
99 | #
100 | # Notes:
101 | # - Currently only supports moving individual files, not directories.
102 | def move_to(target_path)
103 | target = IOStreams.new(target_path)
104 | target.mkpath
105 | target.copy_from(self, convert: false)
106 | delete
107 | target
108 | end
109 |
110 | # Returns [IOStreams::Path] the directory for this file.
111 | #
112 | # If `path` does not include a directory name then "." is returned.
113 | #
114 | # IOStreams.path("test.rb").directory #=> "."
115 | # IOStreams.path("a/b/d/test.rb").directory #=> "a/b/d"
116 | # IOStreams.path(".a/b/d/test.rb").directory #=> ".a/b/d"
117 | # IOStreams.path("foo.").directory #=> "."
118 | # IOStreams.path("test").directory #=> "."
119 | # IOStreams.path(".profile").directory #=> "."
120 | def directory
121 | new_path = dup
122 | new_path.builder = nil
123 | new_path.path = ::File.dirname(path)
124 | new_path
125 | end
126 |
127 | # When path is a file, deletes this file.
128 | # When path is a directory, attempts to delete this directory. If the directory contains
129 | # any children it will fail.
130 | #
131 | # Returns self
132 | #
133 | # Notes:
134 | # * No error is raised if the file or directory is not present.
135 | # * Only the file is removed, not any of the parent paths.
136 | def delete
137 | raise NotImplementedError
138 | end
139 |
140 | # When path is a directory ,deletes this directory and all its children.
141 | # When path is a file ,deletes this file.
142 | #
143 | # Returns self
144 | #
145 | # Notes:
146 | # * No error is raised if the file is not present.
147 | # * Only the file is removed, not any of the parent paths.
148 | # * All children paths and files will be removed.
149 | def delete_all
150 | raise NotImplementedError
151 | end
152 |
153 | # Returns [true|false] whether the file is compressed based on its file extensions.
154 | def compressed?
155 | # TODO: Look at streams?
156 | !(path =~ /\.(zip|gz|gzip|xlsx|xlsm|bz2)\z/i).nil?
157 | end
158 |
159 | # Returns [true|false] whether the file is encrypted based on its file extensions.
160 | def encrypted?
161 | # TODO: Look at streams?
162 | !(path =~ /\.(enc|pgp|gpg)\z/i).nil?
163 | end
164 |
165 | # Returns [true|false] whether partially created files are visible on this path.
166 | #
167 | # With local file systems a file that is still being written to is visbile.
168 | # On AWS S3 a file is not visible until it is completely written to the bucket.
169 | def partial_files_visible?
170 | true
171 | end
172 |
173 | # TODO: Other possible methods:
174 | # - rename - File.rename
175 | # - rmtree - delete everything under this path - FileUtils.rm_r
176 | # - directory?
177 | # - file?
178 | # - empty?
179 | # - find(ignore_error: true) - Find.find
180 |
181 | # Paths are sortable by name
182 | def <=>(other)
183 | path <=> other.path
184 | end
185 |
186 | # Compare by path name, ignore streams
187 | def ==(other)
188 | path == other.path
189 | end
190 |
191 | def inspect
192 | str = "#<#{self.class.name}:#{path}"
193 | str << " @builder=#{builder.streams.inspect}" if builder.streams
194 | str << " @options=#{builder.options.inspect}" if builder.options
195 | str << " pipeline=#{pipeline.inspect}>"
196 | end
197 |
198 | private
199 |
200 | def builder
201 | @builder ||= IOStreams::Builder.new(path)
202 | end
203 | end
204 | end
205 |
--------------------------------------------------------------------------------
/lib/io_streams/paths/file.rb:
--------------------------------------------------------------------------------
1 | require "fileutils"
2 |
3 | module IOStreams
4 | module Paths
5 | class File < IOStreams::Path
6 | attr_accessor :create_path
7 |
8 | def initialize(file_name, create_path: true)
9 | @create_path = create_path
10 | super(file_name)
11 | end
12 |
13 | # Yields Paths within the current path.
14 | #
15 | # Examples:
16 | #
17 | # # Case Insensitive file name lookup:
18 | # IOStreams.path("ruby").glob("r*.md") { |name| puts name }
19 | #
20 | # # Case Sensitive file name lookup:
21 | # IOStreams.path("ruby").each("R*.md", case_sensitive: true) { |name| puts name }
22 | #
23 | # # Also return the names of directories found during the search:
24 | # IOStreams.path("ruby").each("R*.md", directories: true) { |name| puts name }
25 | #
26 | # # Case Insensitive recursive file name lookup:
27 | # IOStreams.path("ruby").glob("**/*.md") { |name| puts name }
28 | #
29 | # Parameters:
30 | # pattern [String]
31 | # The pattern is not a regexp, it is a string that may contain the following metacharacters:
32 | # `*` Matches all regular files.
33 | # `c*` Matches all regular files beginning with `c`.
34 | # `*c` Matches all regular files ending with `c`.
35 | # `*c*` Matches all regular files that have `c` in them.
36 | #
37 | # `**` Matches recursively into subdirectories.
38 | #
39 | # `?` Matches any one character.
40 | #
41 | # `[set]` Matches any one character in the supplied `set`.
42 | # `[^set]` Does not matches any one character in the supplied `set`.
43 | #
44 | # `\` Escapes the next metacharacter.
45 | #
46 | # `{a,b}` Matches on either pattern `a` or pattern `b`.
47 | #
48 | # case_sensitive [true|false]
49 | # Whether the pattern is case-sensitive.
50 | #
51 | # directories [true|false]
52 | # Whether to yield directory names.
53 | #
54 | # hidden [true|false]
55 | # Whether to yield hidden paths.
56 | #
57 | # Examples:
58 | #
59 | # Pattern: File name: match? Reason Options
60 | # =========== ================ ====== ============================= ===========================
61 | # "cat" "cat" true # Match entire string
62 | # "cat" "category" false # Only match partial string
63 | #
64 | # "c{at,ub}s" "cats" true # { } is supported
65 | #
66 | # "c?t" "cat" true # "?" match only 1 character
67 | # "c??t" "cat" false # ditto
68 | # "c*" "cats" true # "*" match 0 or more characters
69 | # "c*t" "c/a/b/t" true # ditto
70 | # "ca[a-z]" "cat" true # inclusive bracket expression
71 | # "ca[^t]" "cat" false # exclusive bracket expression ("^" or "!")
72 | #
73 | # "cat" "CAT" false # case sensitive {case_sensitive: false}
74 | # "cat" "CAT" true # case insensitive
75 | #
76 | # "\?" "?" true # escaped wildcard becomes ordinary
77 | # "\a" "a" true # escaped ordinary remains ordinary
78 | # "[\?]" "?" true # can escape inside bracket expression
79 | #
80 | # "*" ".profile" false # wildcard doesn't match leading
81 | # "*" ".profile" true # period by default.
82 | # ".*" ".profile" true {hidden: true}
83 | #
84 | # "**/*.rb" "main.rb" false
85 | # "**/*.rb" "./main.rb" false
86 | # "**/*.rb" "lib/song.rb" true
87 | # "**.rb" "main.rb" true
88 | # "**.rb" "./main.rb" false
89 | # "**.rb" "lib/song.rb" true
90 | # "*" "dave/.profile" true
91 | def each_child(pattern = "*", case_sensitive: false, directories: false, hidden: false)
92 | unless block_given?
93 | return to_enum(__method__, pattern,
94 | case_sensitive: case_sensitive, directories: directories, hidden: hidden)
95 | end
96 |
97 | flags = 0
98 | flags |= ::File::FNM_CASEFOLD unless case_sensitive
99 | flags |= ::File::FNM_DOTMATCH if hidden
100 |
101 | # Dir.each_child("testdir") {|x| puts "Got #{x}" }
102 | Dir.glob(::File.join(path, pattern), flags) do |full_path|
103 | next if !directories && ::File.directory?(full_path)
104 |
105 | yield(self.class.new(full_path))
106 | end
107 | end
108 |
109 | # Moves this file to the `target_path` by copying it to the new name and then deleting the current file.
110 | #
111 | # Notes:
112 | # - Can copy across buckets.
113 | def move_to(target_path)
114 | target = IOStreams.new(target_path)
115 | return super(target) unless target.is_a?(self.class)
116 |
117 | target.mkpath
118 | # In case the file is being moved across partitions
119 | FileUtils.move(path, target.to_s)
120 | target
121 | end
122 |
123 | def mkpath
124 | dir = ::File.dirname(path)
125 | FileUtils.mkdir_p(dir) unless ::File.exist?(dir)
126 | self
127 | end
128 |
129 | def mkdir
130 | FileUtils.mkdir_p(path) unless ::File.exist?(path)
131 | self
132 | end
133 |
134 | def exist?
135 | ::File.exist?(path)
136 | end
137 |
138 | def size
139 | ::File.size(path)
140 | end
141 |
142 | def delete
143 | return self unless exist?
144 |
145 | ::File.directory?(path) ? Dir.delete(path) : ::File.unlink(path)
146 | self
147 | end
148 |
149 | def delete_all
150 | return self unless exist?
151 |
152 | ::File.directory?(path) ? FileUtils.remove_dir(path) : ::File.unlink(path)
153 | self
154 | end
155 |
156 | # Returns the real path by stripping `.`, `..` and expands any symlinks.
157 | def realpath
158 | self.class.new(::File.realpath(path))
159 | end
160 |
161 | private
162 |
163 | # Read from file
164 | def stream_reader(&block)
165 | ::File.open(path, "rb") { |io| builder.reader(io, &block) }
166 | end
167 |
168 | # Write to file
169 | #
170 | # Note:
171 | # If an exception is raised whilst the file is being written to the file is removed to
172 | # prevent incomplete / partial files from being created.
173 | def stream_writer(&block)
174 | mkpath if create_path
175 | begin
176 | ::File.open(path, "wb") { |io| builder.writer(io, &block) }
177 | rescue StandardError => e
178 | ::File.unlink(path) if ::File.exist?(path)
179 | raise(e)
180 | end
181 | end
182 | end
183 | end
184 | end
185 |
--------------------------------------------------------------------------------
/lib/io_streams/paths/http.rb:
--------------------------------------------------------------------------------
1 | require "net/http"
2 | require "uri"
3 | require "cgi"
4 | module IOStreams
5 | module Paths
6 | class HTTP < IOStreams::Path
7 | attr_reader :username, :password, :http_redirect_count, :url
8 |
9 | # Stream to/from a remote file over http(s).
10 | #
11 | # Parameters:
12 | # url: [String]
13 | # URI of the file to download.
14 | # Example:
15 | # https://www5.fdic.gov/idasp/Offices2.zip
16 | # http://hostname/path/file_name
17 | #
18 | # Full url showing all the optional elements that can be set via the url:
19 | # https://username:password@hostname/path/file_name
20 | #
21 | # username: [String]
22 | # When supplied, basic authentication is used with the username and password.
23 | #
24 | # password: [String]
25 | # Password to use use with basic authentication when the username is supplied.
26 | #
27 | # http_redirect_count: [Integer]
28 | # Maximum number of http redirects to follow.
29 | def initialize(url, username: nil, password: nil, http_redirect_count: 10, parameters: nil)
30 | uri = URI.parse(url)
31 | unless %w[http https].include?(uri.scheme)
32 | raise(
33 | ArgumentError,
34 | "Invalid URL. Required Format: 'http:///', or 'https:///'"
35 | )
36 | end
37 |
38 | @username = username || uri.user
39 | @password = password || uri.password
40 | @http_redirect_count = http_redirect_count
41 | @url = parameters ? "#{url}?#{URI.encode_www_form(parameters)}" : url
42 | super(uri.path)
43 | end
44 |
45 | # Does not support relative file names since there is no concept of current working directory
46 | def relative?
47 | false
48 | end
49 |
50 | def to_s
51 | url
52 | end
53 |
54 | private
55 |
56 | # Read a file using an http get.
57 | #
58 | # For example:
59 | # IOStreams.path('https://www5.fdic.gov/idasp/Offices2.zip').reader {|file| puts file.read}
60 | #
61 | # Read the file without unzipping and streaming the first file in the zip:
62 | # IOStreams.path('https://www5.fdic.gov/idasp/Offices2.zip').stream(:none).reader {|file| puts file.read}
63 | #
64 | # Notes:
65 | # * Since Net::HTTP download only supports a push stream, the data is streamed into a tempfile first.
66 | def stream_reader(&block)
67 | handle_redirects(url, http_redirect_count, &block)
68 | end
69 |
70 | def handle_redirects(uri, http_redirect_count, &block)
71 | uri = URI.parse(uri) unless uri.is_a?(URI)
72 | result = nil
73 | raise(IOStreams::Errors::CommunicationsFailure, "Too many redirects") if http_redirect_count < 1
74 |
75 | Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https") do |http|
76 | request = Net::HTTP::Get.new(uri)
77 | request.basic_auth(username, password) if username
78 |
79 | http.request(request) do |response|
80 | raise(IOStreams::Errors::CommunicationsFailure, "Invalid URL: #{uri}") if response.is_a?(Net::HTTPNotFound)
81 | if response.is_a?(Net::HTTPUnauthorized)
82 | raise(IOStreams::Errors::CommunicationsFailure, "Authorization Required: Invalid :username or :password.")
83 | end
84 |
85 | if response.is_a?(Net::HTTPRedirection)
86 | new_uri = response["location"]
87 | return handle_redirects(new_uri, http_redirect_count - 1, &block)
88 | end
89 |
90 | unless response.is_a?(Net::HTTPSuccess)
91 | raise(IOStreams::Errors::CommunicationsFailure, "Invalid response code: #{response.code}")
92 | end
93 |
94 | # Since Net::HTTP download only supports a push stream, write it to a tempfile first.
95 | Utils.temp_file_name("iostreams_http") do |file_name|
96 | ::File.open(file_name, "wb") { |io| response.read_body { |chunk| io.write(chunk) } }
97 | # Return a read stream
98 | result = ::File.open(file_name, "rb") { |io| builder.reader(io, &block) }
99 | end
100 | end
101 | end
102 | result
103 | end
104 | end
105 | end
106 | end
107 |
--------------------------------------------------------------------------------
/lib/io_streams/paths/matcher.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | module Paths
3 | # Implement fnmatch logic for any path iterator
4 | class Matcher
5 | # Characters indicating that pattern matching is required
6 | MATCH_START_CHARS = /[*?\[{]/.freeze
7 |
8 | attr_reader :path, :pattern, :flags
9 |
10 | # If the supplied pattern contains sub-directories without wildcards, navigate down to that directory
11 | # first before applying wildcard lookups from that point on.
12 | #
13 | # Examples: If the current path is "/path/work"
14 | # "a/b/c/**/*" => "/path/work/a/b/c"
15 | # "a/b/c?/**/*" => "/path/work/a/b"
16 | # "**/*" => "/path/work"
17 | #
18 | # Note: Absolute paths in the pattern are not supported.
19 | def initialize(path, pattern, case_sensitive: false, hidden: false)
20 | extract_optimized_path(path, pattern)
21 |
22 | @flags = ::File::FNM_EXTGLOB | ::File::FNM_PATHNAME
23 | @flags |= ::File::FNM_CASEFOLD unless case_sensitive
24 | @flags |= ::File::FNM_DOTMATCH if hidden
25 | end
26 |
27 | # Returns whether the relative `file_name` matches
28 | def match?(file_name)
29 | relative_file_name = file_name.sub(path.to_s, "").sub(%r{\A/}, "")
30 | ::File.fnmatch?(pattern, relative_file_name, flags)
31 | end
32 |
33 | # Whether this pattern includes a recursive match.
34 | # I.e. Includes `**` anywhere in the path
35 | def recursive?
36 | @recursive ||= pattern.nil? ? false : pattern.include?("**")
37 | end
38 |
39 | private
40 |
41 | def extract_optimized_path(path, pattern)
42 | elements = pattern.split("/")
43 | index = elements.find_index { |e| e.match(MATCH_START_CHARS) }
44 | if index.nil?
45 | # No index means it has no pattern.
46 | @path = path.nil? ? IOStreams.path(pattern) : path.join(pattern)
47 | @pattern = nil
48 | elsif index.zero?
49 | # Cannot optimize path since the very first entry contains a wildcard
50 | @path = path || IOStreams.path
51 | @pattern = pattern
52 | else
53 | new_path = elements[0..index - 1].join("/")
54 | @path = path.nil? ? IOStreams.path(new_path) : path.join(new_path)
55 | @pattern = elements[index..-1].join("/")
56 | end
57 | end
58 | end
59 | end
60 | end
61 |
--------------------------------------------------------------------------------
/lib/io_streams/pgp/reader.rb:
--------------------------------------------------------------------------------
1 | require "open3"
2 |
3 | module IOStreams
4 | module Pgp
5 | class Reader < IOStreams::Reader
6 | # Passphrase to use to open the private key to decrypt the received file
7 | class << self
8 | attr_writer :default_passphrase
9 |
10 | private
11 |
12 | attr_reader :default_passphrase
13 |
14 | @default_passphrase = nil
15 | end
16 |
17 | # Read from a PGP / GPG file , decompressing the contents as it is read.
18 | #
19 | # file_name: [String]
20 | # Name of file to read from
21 | #
22 | # passphrase: [String]
23 | # Pass phrase for private key to decrypt the file with
24 | def self.file(file_name, passphrase: nil)
25 | # Cannot use `passphrase: self.default_passphrase` since it is considered private
26 | passphrase ||= default_passphrase
27 | raise(ArgumentError, "Missing both passphrase and IOStreams::Pgp::Reader.default_passphrase") unless passphrase
28 |
29 | loopback = IOStreams::Pgp.pgp_version.to_f >= 2.1 ? "--pinentry-mode loopback" : ""
30 | command = "#{IOStreams::Pgp.executable} #{loopback} --batch --no-tty --yes --decrypt --passphrase-fd 0 #{file_name}"
31 | IOStreams::Pgp.logger&.debug { "IOStreams::Pgp::Reader.open: #{command}" }
32 |
33 | # Read decrypted contents from stdout
34 | Open3.popen3(command) do |stdin, stdout, stderr, waith_thr|
35 | stdin.puts(passphrase) if passphrase
36 | stdin.close
37 | result =
38 | begin
39 | stdout.binmode
40 | yield(stdout)
41 | rescue Errno::EPIPE
42 | # Ignore broken pipe because gpg terminates early due to an error
43 | raise(Pgp::Failure, "GPG Failed reading from encrypted file: #{file_name}: #{stderr.read.chomp}")
44 | end
45 | raise(Pgp::Failure, "GPG Failed to decrypt file: #{file_name}: #{stderr.read.chomp}") unless waith_thr.value.success?
46 |
47 | result
48 | end
49 | end
50 | end
51 | end
52 | end
53 |
--------------------------------------------------------------------------------
/lib/io_streams/pgp/writer.rb:
--------------------------------------------------------------------------------
1 | require "open3"
2 |
3 | module IOStreams
4 | module Pgp
5 | class Writer < IOStreams::Writer
6 | class << self
7 | # Sign all encrypted files with this users key.
8 | # Default: Do not sign encrypted files.
9 | attr_writer :default_signer
10 |
11 | # Passphrase to use to open the private key when signing the file.
12 | # Default: None.
13 | attr_writer :default_signer_passphrase
14 |
15 | # Encrypt all pgp output files with this recipient for audit purposes.
16 | # Allows the generated pgp files to be decrypted with this email address.
17 | # Useful for audit or problem resolution purposes.
18 | attr_accessor :audit_recipient
19 |
20 | private
21 |
22 | attr_reader :default_signer_passphrase, :default_signer
23 |
24 | @default_signer_passphrase = nil
25 | @default_signer = nil
26 | @audit_recipient = nil
27 | end
28 |
29 | # Write to a PGP / GPG file, encrypting the contents as it is written.
30 | #
31 | # file_name: [String]
32 | # Name of file to write to.
33 | #
34 | # recipient: [String|Array]
35 | # One or more emails of users for which to encrypt the file.
36 | #
37 | # import_and_trust_key: [String|Array]
38 | # One or more pgp keys to import and then use to encrypt the file.
39 | # Note: Ascii Keys can contain multiple keys, only the last one in the file is used.
40 | #
41 | # signer: [String]
42 | # Name of user with which to sign the encypted file.
43 | # Default: default_signer or do not sign.
44 | #
45 | # signer_passphrase: [String]
46 | # Passphrase to use to open the private key when signing the file.
47 | # Default: default_signer_passphrase
48 | #
49 | # compress: [:none|:zip|:zlib|:bzip2]
50 | # Note: Standard PGP only supports :zip.
51 | # :zlib is better than zip.
52 | # :bzip2 is best, but uses a lot of memory and is much slower.
53 | # Default: :zip
54 | #
55 | # compress_level: [Integer]
56 | # Compression level
57 | # Default: 6
58 | def self.file(file_name,
59 | recipient: nil,
60 | import_and_trust_key: nil,
61 | signer: default_signer,
62 | signer_passphrase: default_signer_passphrase,
63 | compress: :zip,
64 | compression: nil, # Deprecated
65 | compress_level: 6,
66 | original_file_name: nil)
67 |
68 | raise(ArgumentError, "Requires either :recipient or :import_and_trust_key") unless recipient || import_and_trust_key
69 |
70 | # Backward compatibility
71 | compress = compression if compression
72 |
73 | compress_level = 0 if compress == :none
74 |
75 | recipients = Array(recipient)
76 | recipients << audit_recipient if audit_recipient
77 |
78 | Array(import_and_trust_key).each do |key|
79 | recipients << IOStreams::Pgp.import_and_trust(key: key)
80 | end
81 |
82 | # Write to stdin, with encrypted contents being written to the file
83 | command = "#{IOStreams::Pgp.executable} --batch --no-tty --yes --encrypt"
84 | command << " --sign --local-user \"#{signer}\"" if signer
85 | if signer_passphrase
86 | command << " --pinentry-mode loopback" if IOStreams::Pgp.pgp_version.to_f >= 2.1
87 | command << " --passphrase \"#{signer_passphrase}\""
88 | end
89 | command << " -z #{compress_level}" if compress_level != 6
90 | command << " --compress-algo #{compress}" unless compress == :none
91 | recipients.each { |address| command << " --recipient \"#{address}\"" }
92 | command << " -o \"#{file_name}\""
93 |
94 | IOStreams::Pgp.logger&.debug { "IOStreams::Pgp::Writer.open: #{command}" }
95 |
96 | result = nil
97 | Open3.popen2e(command) do |stdin, out, waith_thr|
98 | begin
99 | stdin.binmode
100 | result = yield(stdin)
101 | stdin.close
102 | rescue Errno::EPIPE
103 | # Ignore broken pipe because gpg terminates early due to an error
104 | ::File.delete(file_name) if ::File.exist?(file_name)
105 | raise(Pgp::Failure, "GPG Failed writing to encrypted file: #{file_name}: #{out.read.chomp}")
106 | end
107 | unless waith_thr.value.success?
108 | ::File.delete(file_name) if ::File.exist?(file_name)
109 | raise(Pgp::Failure, "GPG Failed to create encrypted file: #{file_name}: #{out.read.chomp}")
110 | end
111 | end
112 | result
113 | end
114 | end
115 | end
116 | end
117 |
--------------------------------------------------------------------------------
/lib/io_streams/reader.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | class Reader
3 | # When a Reader does not support streams, we copy the stream to a local temp file
4 | # and then pass that filename in for this reader.
5 | def self.stream(input_stream, **args, &block)
6 | Utils.temp_file_name("iostreams_reader") do |file_name|
7 | ::File.open(file_name, "wb") { |target| ::IO.copy_stream(input_stream, target) }
8 | file(file_name, **args, &block)
9 | end
10 | end
11 |
12 | # When a Writer supports streams, also allow it to simply support a file
13 | def self.file(file_name, original_file_name: file_name, **args, &block)
14 | ::File.open(file_name, "rb") { |file| stream(file, original_file_name: original_file_name, **args, &block) }
15 | end
16 |
17 | # For processing by either a file name or an open IO stream.
18 | def self.open(file_name_or_io, **args, &block)
19 | file_name_or_io.is_a?(String) ? file(file_name_or_io, **args, &block) : stream(file_name_or_io, **args, &block)
20 | end
21 |
22 | attr_reader :input_stream
23 |
24 | def initialize(input_stream)
25 | @input_stream = input_stream
26 | end
27 | end
28 | end
29 |
--------------------------------------------------------------------------------
/lib/io_streams/record/reader.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | module Record
3 | # Converts each line of an input stream into hash for every row
4 | class Reader < IOStreams::Reader
5 | include Enumerable
6 |
7 | # Read a record at a time from a line stream
8 | # Note:
9 | # - The supplied stream _must_ already be a line stream, or a stream that responds to :each
10 | def self.stream(line_reader, **args)
11 | # Pass-through if already a record reader
12 | return yield(line_reader) if line_reader.is_a?(self.class)
13 |
14 | yield new(line_reader, **args)
15 | end
16 |
17 | # When reading from a file also add the line reader stream
18 | def self.file(file_name, original_file_name: file_name, delimiter: $/, **args)
19 | IOStreams::Line::Reader.file(file_name, original_file_name: original_file_name, delimiter: delimiter) do |io|
20 | yield new(io, original_file_name: original_file_name, **args)
21 | end
22 | end
23 |
24 | # Create a Tabular reader to return the stream as Hash records
25 | # Parse a delimited data source.
26 | #
27 | # Parameters
28 | # format: [Symbol]
29 | # :csv, :hash, :array, :json, :psv, :fixed
30 | #
31 | # file_name: [String]
32 | # When `:format` is not supplied the file name can be used to infer the required format.
33 | # Optional. Default: nil
34 | #
35 | # format_options: [Hash]
36 | # Any specialized format specific options. For example, `:fixed` format requires the file definition.
37 | #
38 | # columns [Array]
39 | # The header columns when the file does not include a header row.
40 | # Note:
41 | # It is recommended to keep all columns as strings to avoid any issues when persistence
42 | # with MongoDB when it converts symbol keys to strings.
43 | #
44 | # allowed_columns [Array]
45 | # List of columns to allow.
46 | # Default: nil ( Allow all columns )
47 | # Note:
48 | # When supplied any columns that are rejected will be returned in the cleansed columns
49 | # as nil so that they can be ignored during processing.
50 | #
51 | # required_columns [Array]
52 | # List of columns that must be present, otherwise an Exception is raised.
53 | #
54 | # skip_unknown [true|false]
55 | # true:
56 | # Skip columns not present in the `allowed_columns` by cleansing them to nil.
57 | # #as_hash will skip these additional columns entirely as if they were not in the file at all.
58 | # false:
59 | # Raises Tabular::InvalidHeader when a column is supplied that is not in the whitelist.
60 | def initialize(line_reader, cleanse_header: true, original_file_name: nil, **args)
61 | unless line_reader.respond_to?(:each)
62 | raise(ArgumentError, "Stream must be a IOStreams::Line::Reader or implement #each")
63 | end
64 |
65 | @tabular = IOStreams::Tabular.new(file_name: original_file_name, **args)
66 | @line_reader = line_reader
67 | @cleanse_header = cleanse_header
68 | end
69 |
70 | def each
71 | @line_reader.each do |line|
72 | if @tabular.header?
73 | @tabular.parse_header(line)
74 | @tabular.cleanse_header! if @cleanse_header
75 | else
76 | yield @tabular.record_parse(line)
77 | end
78 | end
79 | end
80 | end
81 | end
82 | end
83 |
--------------------------------------------------------------------------------
/lib/io_streams/record/writer.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | module Record
3 | # Example, implied header from first record:
4 | # IOStreams.path('file.csv').writer(:hash) do |stream|
5 | # stream << {name: 'Jack', address: 'Somewhere', zipcode: 12345}
6 | # stream << {name: 'Joe', address: 'Lost', zipcode: 32443, age: 23}
7 | # end
8 | class Writer < IOStreams::Writer
9 | # Write a record as a Hash at a time to a stream.
10 | # Note:
11 | # - The supplied stream _must_ already be a line stream, or a stream that responds to :<<
12 | def self.stream(line_writer, **args)
13 | # Pass-through if already a record writer
14 | return yield(line_writer) if line_writer.is_a?(self.class)
15 |
16 | yield new(line_writer, **args)
17 | end
18 |
19 | # When writing to a file also add the line writer stream
20 | def self.file(file_name, original_file_name: file_name, delimiter: $/, **args, &block)
21 | IOStreams::Line::Writer.file(file_name, original_file_name: original_file_name, delimiter: delimiter) do |io|
22 | yield new(io, original_file_name: original_file_name, **args, &block)
23 | end
24 | end
25 |
26 | # Create a Tabular writer that takes individual
27 | # Parse a delimited data source.
28 | #
29 | # Parameters
30 | # format: [Symbol]
31 | # :csv, :hash, :array, :json, :psv, :fixed
32 | #
33 | # file_name: [String]
34 | # When `:format` is not supplied the file name can be used to infer the required format.
35 | # Optional. Default: nil
36 | #
37 | # format_options: [Hash]
38 | # Any specialized format specific options. For example, `:fixed` format requires the file definition.
39 | #
40 | # columns [Array]
41 | # The header columns when the file does not include a header row.
42 | # Note:
43 | # It is recommended to keep all columns as strings to avoid any issues when persistence
44 | # with MongoDB when it converts symbol keys to strings.
45 | #
46 | # allowed_columns [Array]
47 | # List of columns to allow.
48 | # Default: nil ( Allow all columns )
49 | # Note:
50 | # When supplied any columns that are rejected will be returned in the cleansed columns
51 | # as nil so that they can be ignored during processing.
52 | #
53 | # required_columns [Array]
54 | # List of columns that must be present, otherwise an Exception is raised.
55 | #
56 | # skip_unknown [true|false]
57 | # true:
58 | # Skip columns not present in the `allowed_columns` by cleansing them to nil.
59 | # #as_hash will skip these additional columns entirely as if they were not in the file at all.
60 | # false:
61 | # Raises Tabular::InvalidHeader when a column is supplied that is not in the whitelist.
62 | def initialize(line_writer, columns: nil, original_file_name: nil, **args)
63 | raise(ArgumentError, "Stream must be a IOStreams::Line::Writer or implement #<<") unless line_writer.respond_to?(:<<)
64 |
65 | @tabular = IOStreams::Tabular.new(columns: columns, file_name: original_file_name, **args)
66 | @line_writer = line_writer
67 |
68 | # Render header line when `columns` is supplied.
69 | @line_writer << @tabular.render_header if columns && @tabular.requires_header?
70 | end
71 |
72 | def <<(hash)
73 | raise(ArgumentError, "#<< only accepts a Hash argument") unless hash.is_a?(Hash)
74 |
75 | if @tabular.header?
76 | # Extract header from the keys from the first row when not supplied above.
77 | @tabular.header.columns = hash.keys
78 | @line_writer << @tabular.render_header
79 | end
80 | @line_writer << @tabular.render(hash)
81 | end
82 | end
83 | end
84 | end
85 |
--------------------------------------------------------------------------------
/lib/io_streams/row/reader.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | module Row
3 | # Converts each line of an input stream into an array for every line
4 | class Reader < IOStreams::Reader
5 | # Read a line as an Array at a time from a stream.
6 | # Note:
7 | # - The supplied stream _must_ already be a line stream, or a stream that responds to :each
8 | def self.stream(line_reader, **args)
9 | # Pass-through if already a row reader
10 | return yield(line_reader) if line_reader.is_a?(self.class)
11 |
12 | yield new(line_reader, **args)
13 | end
14 |
15 | # When reading from a file also add the line reader stream
16 | def self.file(file_name, original_file_name: file_name, delimiter: $/, **args)
17 | IOStreams::Line::Reader.file(file_name, original_file_name: original_file_name, delimiter: delimiter) do |io|
18 | yield new(io, original_file_name: original_file_name, **args)
19 | end
20 | end
21 |
22 | # Create a Tabular reader to return the stream rows as arrays.
23 | #
24 | # Parameters
25 | # delimited: [#each]
26 | # Anything that returns one line / record at a time when #each is called on it.
27 | #
28 | # format: [Symbol]
29 | # :csv, :hash, :array, :json, :psv, :fixed
30 | #
31 | # For all other parameters, see Tabular::Header.new
32 | def initialize(line_reader, cleanse_header: true, original_file_name: nil, **args)
33 | unless line_reader.respond_to?(:each)
34 | raise(ArgumentError, "Stream must be a IOStreams::Line::Reader or implement #each")
35 | end
36 |
37 | @tabular = IOStreams::Tabular.new(file_name: original_file_name, **args)
38 | @line_reader = line_reader
39 | @cleanse_header = cleanse_header
40 | end
41 |
42 | def each
43 | @line_reader.each do |line|
44 | if @tabular.header?
45 | columns = @tabular.parse_header(line)
46 | @tabular.cleanse_header! if @cleanse_header
47 | yield columns
48 | else
49 | yield @tabular.row_parse(line)
50 | end
51 | end
52 | end
53 | end
54 | end
55 | end
56 |
--------------------------------------------------------------------------------
/lib/io_streams/row/writer.rb:
--------------------------------------------------------------------------------
1 | require "csv"
2 | module IOStreams
3 | module Row
4 | # Example:
5 | # IOStreams.path("file.csv").writer(:array) do |stream|
6 | # stream << ['name', 'address', 'zipcode']
7 | # stream << ['Jack', 'Somewhere', 12345]
8 | # stream << ['Joe', 'Lost', 32443]
9 | # end
10 | class Writer < IOStreams::Writer
11 | # Write a record from an Array at a time to a stream.
12 | #
13 | # Note:
14 | # - The supplied stream _must_ already be a line stream, or a stream that responds to :<<
15 | def self.stream(line_writer, **args)
16 | # Pass-through if already a row writer
17 | return yield(line_writer) if line_writer.is_a?(self.class)
18 |
19 | yield new(line_writer, **args)
20 | end
21 |
22 | # When writing to a file also add the line writer stream
23 | def self.file(file_name, original_file_name: file_name, delimiter: $/, **args, &block)
24 | IOStreams::Line::Writer.file(file_name, original_file_name: original_file_name, delimiter: delimiter) do |io|
25 | yield new(io, original_file_name: original_file_name, **args, &block)
26 | end
27 | end
28 |
29 | # Create a Tabular writer that takes individual rows as arrays.
30 | #
31 | # Parameters
32 | # line_writer: [#<<]
33 | # Anything that accepts a line / record at a time when #<< is called on it.
34 | #
35 | # format: [Symbol]
36 | # :csv, :hash, :array, :json, :psv, :fixed
37 | #
38 | # For all other parameters, see Tabular::Header.new
39 | def initialize(line_writer, columns: nil, original_file_name: nil, **args)
40 | raise(ArgumentError, "Stream must be a IOStreams::Line::Writer or implement #<<") unless line_writer.respond_to?(:<<)
41 |
42 | @tabular = IOStreams::Tabular.new(columns: columns, file_name: original_file_name, **args)
43 | @line_writer = line_writer
44 |
45 | # Render header line when `columns` is supplied.
46 | line_writer << @tabular.render_header if columns && @tabular.requires_header?
47 | end
48 |
49 | # Supply a hash or an array to render
50 | def <<(array)
51 | raise(ArgumentError, "Must supply an Array") unless array.is_a?(Array)
52 |
53 | if @tabular.header?
54 | # If header (columns) was not supplied as an argument, assume first line is the header.
55 | @tabular.header.columns = array
56 | @line_writer << @tabular.render_header
57 | else
58 | @line_writer << @tabular.render(array)
59 | end
60 | end
61 | end
62 | end
63 | end
64 |
--------------------------------------------------------------------------------
/lib/io_streams/symmetric_encryption/reader.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | module SymmetricEncryption
3 | class Reader < IOStreams::Reader
4 | # read from a file/stream using Symmetric Encryption
5 | def self.stream(input_stream, **args, &block)
6 | Utils.load_soft_dependency("symmetric-encryption", ".enc streaming") unless defined?(SymmetricEncryption)
7 |
8 | ::SymmetricEncryption::Reader.open(input_stream, **args, &block)
9 | end
10 | end
11 | end
12 | end
13 |
--------------------------------------------------------------------------------
/lib/io_streams/symmetric_encryption/writer.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | module SymmetricEncryption
3 | class Writer < IOStreams::Writer
4 | # Write to stream using Symmetric Encryption
5 | # By default the output stream is compressed.
6 | # If the input_stream is already compressed consider setting compress: false.
7 | def self.stream(input_stream, compress: true, **args, &block)
8 | Utils.load_soft_dependency("symmetric-encryption", ".enc streaming") unless defined?(SymmetricEncryption)
9 |
10 | ::SymmetricEncryption::Writer.open(input_stream, compress: compress, **args, &block)
11 | end
12 |
13 | # Write to stream using Symmetric Encryption
14 | # By default the output stream is compressed unless the file_name extension indicates the file is already compressed.
15 | def self.file(file_name, compress: nil, **args, &block)
16 | Utils.load_soft_dependency("symmetric-encryption", ".enc streaming") unless defined?(SymmetricEncryption)
17 |
18 | ::SymmetricEncryption::Writer.open(file_name, compress: compress, **args, &block)
19 | end
20 | end
21 | end
22 | end
23 |
--------------------------------------------------------------------------------
/lib/io_streams/tabular/header.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | class Tabular
3 | # Process files / streams that start with a header.
4 | class Header
5 | # Column names that begin with this prefix have been rejected and should be ignored.
6 | IGNORE_PREFIX = "__rejected__".freeze
7 |
8 | attr_accessor :columns, :allowed_columns, :required_columns, :skip_unknown
9 |
10 | # Header
11 | #
12 | # Parameters
13 | # columns [Array]
14 | # Columns in this header.
15 | # Note:
16 | # It is recommended to keep all columns as strings to avoid any issues when persistence
17 | # with MongoDB when it converts symbol keys to strings.
18 | #
19 | # allowed_columns [Array]
20 | # List of columns to allow.
21 | # Default: nil ( Allow all columns )
22 | # Note:
23 | # * So that rejected columns can be identified in subsequent steps, they will be prefixed with `__rejected__`.
24 | # For example, `Unknown Column` would be cleansed as `__rejected__Unknown Column`.
25 | #
26 | # required_columns [Array]
27 | # List of columns that must be present, otherwise an Exception is raised.
28 | #
29 | # skip_unknown [true|false]
30 | # true:
31 | # Skip columns not present in the whitelist by cleansing them to nil.
32 | # #as_hash will skip these additional columns entirely as if they were not in the file at all.
33 | # false:
34 | # Raises Tabular::InvalidHeader when a column is supplied that is not in the whitelist.
35 | def initialize(columns: nil, allowed_columns: nil, required_columns: nil, skip_unknown: true)
36 | @columns = columns
37 | @required_columns = required_columns
38 | @allowed_columns = allowed_columns
39 | @skip_unknown = skip_unknown
40 | end
41 |
42 | # Returns [Array] list columns that were ignored during cleansing.
43 | #
44 | # Each column is cleansed as follows:
45 | # - Leading and trailing whitespace is stripped.
46 | # - All characters converted to lower case.
47 | # - Spaces and '-' are converted to '_'.
48 | # - All characters except for letters, digits, and '_' are stripped.
49 | #
50 | # Notes:
51 | # * So that rejected columns can be identified in subsequent steps, they will be prefixed with `__rejected__`.
52 | # For example, `Unknown Column` would be cleansed as `__rejected__Unknown Column`.
53 | # * Raises Tabular::InvalidHeader when there are no rejected columns left after cleansing.
54 | def cleanse!
55 | return [] if columns.nil? || columns.empty?
56 |
57 | ignored_columns = []
58 | self.columns = columns.collect do |column|
59 | cleansed = cleanse_column(column)
60 | if allowed_columns.nil? || allowed_columns.include?(cleansed)
61 | cleansed
62 | else
63 | ignored_columns << column
64 | "#{IGNORE_PREFIX}#{column}"
65 | end
66 | end
67 |
68 | if !skip_unknown && !ignored_columns.empty?
69 | raise(IOStreams::Errors::InvalidHeader, "Unknown columns after cleansing: #{ignored_columns.join(',')}")
70 | end
71 |
72 | if ignored_columns.size == columns.size
73 | raise(IOStreams::Errors::InvalidHeader, "All columns are unknown after cleansing: #{ignored_columns.join(',')}")
74 | end
75 |
76 | if required_columns
77 | missing_columns = required_columns - columns
78 | unless missing_columns.empty?
79 | raise(IOStreams::Errors::InvalidHeader, "Missing columns after cleansing: #{missing_columns.join(',')}")
80 | end
81 | end
82 |
83 | ignored_columns
84 | end
85 |
86 | # Marshal to Hash from Array or Hash by applying this header
87 | #
88 | # Parameters:
89 | # cleanse [true|false]
90 | # Whether to cleanse and narrow the supplied hash to just those columns in this header.
91 | # Only Applies to when the hash is already a Hash.
92 | # Useful to turn off narrowing when the input data is already trusted.
93 | def to_hash(row, cleanse = true)
94 | return if IOStreams::Utils.blank?(row)
95 |
96 | case row
97 | when Array
98 | unless columns
99 | raise(IOStreams::Errors::InvalidHeader, "Missing mandatory header when trying to convert a row into a hash")
100 | end
101 |
102 | array_to_hash(row)
103 | when Hash
104 | cleanse && columns ? cleanse_hash(row) : row
105 | else
106 | raise(IOStreams::Errors::TypeMismatch, "Don't know how to convert #{row.class.name} to a Hash")
107 | end
108 | end
109 |
110 | def to_array(row, cleanse = true)
111 | if row.is_a?(Hash) && columns
112 | row = cleanse_hash(row) if cleanse
113 | row = columns.collect { |column| row[column] }
114 | end
115 |
116 | unless row.is_a?(Array)
117 | raise(
118 | IOStreams::Errors::TypeMismatch,
119 | "Don't know how to convert #{row.class.name} to an Array without the header columns being set."
120 | )
121 | end
122 |
123 | row
124 | end
125 |
126 | private
127 |
128 | def array_to_hash(row)
129 | h = {}
130 | columns.each_with_index { |col, i| h[col] = row[i] unless IOStreams::Utils.blank?(col) || col.start_with?(IGNORE_PREFIX) }
131 | h
132 | end
133 |
134 | # Perform cleansing on returned Hash keys during the narrowing process.
135 | # For example, avoids issues with case etc.
136 | def cleanse_hash(hash)
137 | unmatched = columns - hash.keys
138 | unless unmatched.empty?
139 | hash = hash.dup
140 | unmatched.each { |name| hash[cleanse_column(name)] = hash.delete(name) }
141 | end
142 | hash.slice(*columns)
143 | end
144 |
145 | def cleanse_column(name)
146 | cleansed = name.to_s.strip.downcase
147 | cleansed.gsub!(/\s+/, "_")
148 | cleansed.gsub!(/-+/, "_")
149 | cleansed.gsub!(/\W+/, "")
150 | cleansed
151 | end
152 | end
153 | end
154 | end
155 |
--------------------------------------------------------------------------------
/lib/io_streams/tabular/parser/array.rb:
--------------------------------------------------------------------------------
1 | require "json"
2 | module IOStreams
3 | class Tabular
4 | module Parser
5 | class Array < Base
6 | # Returns [Array] the header row.
7 | # Returns nil if the row is blank.
8 | def parse_header(row)
9 | unless row.is_a?(::Array)
10 | raise(IOStreams::Errors::InvalidHeader, "Format is :array. Invalid input header: #{row.class.name}")
11 | end
12 |
13 | row
14 | end
15 |
16 | # Returns Array
17 | def parse(row)
18 | raise(IOStreams::Errors::TypeMismatch, "Format is :array. Invalid input: #{row.class.name}") unless row.is_a?(::Array)
19 |
20 | row
21 | end
22 |
23 | def render(row, header)
24 | header.to_array(row)
25 | end
26 | end
27 | end
28 | end
29 | end
30 |
--------------------------------------------------------------------------------
/lib/io_streams/tabular/parser/base.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | class Tabular
3 | module Parser
4 | class Base
5 | # Returns [true|false] whether a header row is required for this format.
6 | def requires_header?
7 | true
8 | end
9 | end
10 | end
11 | end
12 | end
13 |
--------------------------------------------------------------------------------
/lib/io_streams/tabular/parser/csv.rb:
--------------------------------------------------------------------------------
1 | require "csv"
2 | module IOStreams
3 | class Tabular
4 | module Parser
5 | class Csv < Base
6 | attr_reader :csv_parser
7 |
8 | unless RUBY_VERSION.to_f >= 2.6
9 | def initialize
10 | @csv_parser = Utility::CSVRow.new
11 | end
12 | end
13 |
14 | # Returns [Array] the header row.
15 | # Returns nil if the row is blank.
16 | def parse_header(row)
17 | return row if row.is_a?(::Array)
18 |
19 | unless row.is_a?(String)
20 | raise(IOStreams::Errors::InvalidHeader, "Format is :csv. Invalid input header: #{row.class.name}")
21 | end
22 |
23 | parse_line(row)
24 | end
25 |
26 | # Returns [Array] the parsed CSV line
27 | def parse(row)
28 | return row if row.is_a?(::Array)
29 |
30 | raise(IOStreams::Errors::TypeMismatch, "Format is :csv. Invalid input: #{row.class.name}") unless row.is_a?(String)
31 |
32 | parse_line(row)
33 | end
34 |
35 | # Return the supplied array as a single line CSV string.
36 | def render(row, header)
37 | array = header.to_array(row)
38 | render_array(array)
39 | end
40 |
41 | private
42 |
43 | if RUBY_VERSION.to_f >= 2.6
44 | # About 10 times slower than the approach used in Ruby 2.5 and earlier,
45 | # but at least it works on Ruby 2.6 and above.
46 | def parse_line(line)
47 | return if IOStreams::Utils.blank?(line)
48 |
49 | CSV.parse_line(line)
50 | end
51 |
52 | def render_array(array)
53 | CSV.generate_line(array, encoding: "UTF-8", row_sep: "")
54 | end
55 | else
56 | def parse_line(line)
57 | csv_parser.parse(line)
58 | end
59 |
60 | def render_array(array)
61 | csv_parser.to_csv(array)
62 | end
63 | end
64 | end
65 | end
66 | end
67 | end
68 |
--------------------------------------------------------------------------------
/lib/io_streams/tabular/parser/fixed.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | class Tabular
3 | module Parser
4 | # Parsing and rendering fixed length data
5 | class Fixed < Base
6 | attr_reader :layout, :truncate
7 |
8 | # Returns [IOStreams::Tabular::Parser]
9 | #
10 | # Parameters:
11 | # layout: [Array]
12 | # [
13 | # {size: 23, key: "name"},
14 | # {size: 40, key: "address"},
15 | # {size: 2},
16 | # {size: 5, key: "zip"},
17 | # {size: 8, key: "age", type: :integer},
18 | # {size: 10, key: "weight", type: :float, decimals: 2}
19 | # ]
20 | #
21 | # Notes:
22 | # * Leave out the name of the key to ignore that column during parsing,
23 | # and to space fill when rendering. For example as a filler.
24 | #
25 | # Types:
26 | # :string
27 | # This is the default type.
28 | # Applies space padding and the value is left justified.
29 | # Returns value as a String
30 | # :integer
31 | # Applies zero padding to the left.
32 | # Returns value as an Integer
33 | # Raises Errors::ValueTooLong when the supplied value cannot be rendered in `size` characters.
34 | # :float
35 | # Applies zero padding to the left.
36 | # Returns value as a float.
37 | # The :size is the total size of this field including the `.` and the decimals.
38 | # Number of :decimals
39 | # Raises Errors::ValueTooLong when the supplied value cannot be rendered in `size` characters.
40 | #
41 | # In some circumstances the length of the last column is variable.
42 | # layout: [Array]
43 | # [
44 | # {size: 23, key: "name"},
45 | # {size: :remainder, key: "rest"}
46 | # ]
47 | # By setting a size of `:remainder` it will take the rest of the line as the value for that column.
48 | #
49 | # A size of `:remainder` and no `:key` will discard the remainder of the line without validating the length.
50 | # layout: [Array]
51 | # [
52 | # {size: 23, key: "name"},
53 | # {size: :remainder}
54 | # ]
55 | #
56 | def initialize(layout:, truncate: true)
57 | @layout = Layout.new(layout)
58 | @truncate = truncate
59 | end
60 |
61 | # The required line length for every fixed length line
62 | def line_length
63 | layout.length
64 | end
65 |
66 | # Returns [String] fixed layout values extracted from the supplied hash.
67 | #
68 | # Notes:
69 | # * A nil value is considered an empty string
70 | # * When a supplied value exceeds the column size it is truncated.
71 | def render(row, header)
72 | hash = header.to_hash(row)
73 |
74 | result = ""
75 | layout.columns.each do |column|
76 | result << column.render(hash[column.key], truncate)
77 | end
78 | result
79 | end
80 |
81 | # Returns [Hash] fixed layout values extracted from the supplied line.
82 | # String will be encoded to `encoding`
83 | def parse(line)
84 | unless line.is_a?(String)
85 | raise(Errors::TypeMismatch, "Line must be a String when format is :fixed. Actual: #{line.class.name}")
86 | end
87 |
88 | if layout.length.positive? && (line.length != layout.length)
89 | raise(Errors::InvalidLineLength, "Expected line length: #{layout.length}, actual line length: #{line.length}")
90 | end
91 |
92 | hash = {}
93 | index = 0
94 | layout.columns.each do |column|
95 | if column.size == -1
96 | hash[column.key] = column.parse(line[index..-1]) if column.key
97 | break
98 | end
99 |
100 | # Ignore "columns" that have no keys. E.g. Fillers
101 | hash[column.key] = column.parse(line[index, column.size]) if column.key
102 | index += column.size
103 | end
104 | hash
105 | end
106 |
107 | # The header is required as an argument and cannot be supplied in the file itself.
108 | def requires_header?
109 | false
110 | end
111 |
112 | class Layout
113 | attr_reader :columns, :length
114 |
115 | # Returns [Array] the layout for this fixed width file.
116 | # Also validates values
117 | def initialize(layout)
118 | @length = 0
119 | @columns = parse_layout(layout)
120 | end
121 |
122 | private
123 |
124 | def parse_layout(layout)
125 | @length = 0
126 | layout.collect do |hash|
127 | raise(Errors::InvalidLayout, "Missing required :size in: #{hash.inspect}") unless hash.key?(:size)
128 |
129 | column = Column.new(**hash)
130 | if column.size == -1
131 | if @length == -1
132 | raise(Errors::InvalidLayout, "Only the last :size can be '-1' or :remainder in: #{hash.inspect}")
133 | end
134 |
135 | @length = -1
136 | else
137 | @length += column.size
138 | end
139 | column
140 | end
141 | end
142 | end
143 |
144 | class Column
145 | TYPES = %i[string integer float].freeze
146 |
147 | attr_reader :key, :size, :type, :decimals
148 |
149 | def initialize(size:, key: nil, type: :string, decimals: 2)
150 | @key = key
151 | @size = (size == :remainder || size == "remainder") ? -1 : size.to_i
152 | @type = type.to_sym
153 | @decimals = decimals
154 |
155 | unless @size.positive? || (@size == -1)
156 | raise(Errors::InvalidLayout, "Size #{size.inspect} must be positive or :remainder")
157 | end
158 | raise(Errors::InvalidLayout, "Unknown type: #{type.inspect}") unless TYPES.include?(@type)
159 | end
160 |
161 | def parse(value)
162 | return if value.nil?
163 |
164 | stripped_value = value.to_s.strip
165 |
166 | case type
167 | when :string
168 | stripped_value
169 | when :integer
170 | stripped_value.length.zero? ? nil : value.to_i
171 | when :float
172 | stripped_value.length.zero? ? nil : value.to_f
173 | else
174 | raise(Errors::InvalidLayout, "Unsupported type: #{type.inspect}")
175 | end
176 | end
177 |
178 | def render(value, truncate)
179 | formatted =
180 | case type
181 | when :string
182 | value = value.to_s
183 | return value if size == -1
184 |
185 | format(truncate ? "%-#{size}.#{size}s" : "%-#{size}s", value)
186 | when :integer
187 | return value.to_i.to_s if size == -1
188 |
189 | truncate = false
190 | format("%0#{size}d", value.to_i)
191 | when :float
192 | return value.to_f.to_s if size == -1
193 |
194 | truncate = false
195 | format("%0#{size}.#{decimals}f", value.to_f)
196 | else
197 | raise(Errors::InvalidLayout, "Unsupported type: #{type.inspect}")
198 | end
199 |
200 | if !truncate && formatted.length > size
201 | raise(Errors::ValueTooLong, "Value: #{value} is too large to fit into column:#{key} of size:#{size}")
202 | end
203 |
204 | formatted
205 | end
206 | end
207 | end
208 | end
209 | end
210 | end
211 |
--------------------------------------------------------------------------------
/lib/io_streams/tabular/parser/hash.rb:
--------------------------------------------------------------------------------
1 | require "json"
2 | module IOStreams
3 | class Tabular
4 | module Parser
5 | class Hash < Base
6 | def parse(row)
7 | raise(IOStreams::Errors::TypeMismatch, "Format is :hash. Invalid input: #{row.class.name}") unless row.is_a?(::Hash)
8 |
9 | row
10 | end
11 |
12 | def render(row, header)
13 | header.to_hash(row)
14 | end
15 |
16 | def requires_header?
17 | false
18 | end
19 | end
20 | end
21 | end
22 | end
23 |
--------------------------------------------------------------------------------
/lib/io_streams/tabular/parser/json.rb:
--------------------------------------------------------------------------------
1 | require "json"
2 | module IOStreams
3 | class Tabular
4 | module Parser
5 | # For parsing a single line of JSON at a time
6 | class Json < Base
7 | def parse(row)
8 | return row if row.is_a?(::Hash)
9 |
10 | raise(IOStreams::Errors::TypeMismatch, "Format is :json. Invalid input: #{row.class.name}") unless row.is_a?(String)
11 |
12 | JSON.parse(row)
13 | end
14 |
15 | # Return the supplied array as a single line JSON string.
16 | def render(row, header)
17 | hash = header.to_hash(row)
18 | hash.to_json
19 | end
20 |
21 | def requires_header?
22 | false
23 | end
24 | end
25 | end
26 | end
27 | end
28 |
--------------------------------------------------------------------------------
/lib/io_streams/tabular/parser/psv.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | class Tabular
3 | module Parser
4 | # For parsing a single line of Pipe-separated values
5 | class Psv < Base
6 | # Returns [Array] the header row.
7 | # Returns nil if the row is blank.
8 | def parse_header(row)
9 | return row if row.is_a?(::Array)
10 |
11 | unless row.is_a?(String)
12 | raise(IOStreams::Errors::InvalidHeader, "Format is :psv. Invalid input header: #{row.class.name}")
13 | end
14 |
15 | row.split("|")
16 | end
17 |
18 | # Returns [Array] the parsed PSV line
19 | def parse(row)
20 | return row if row.is_a?(::Array)
21 |
22 | raise(IOStreams::Errors::TypeMismatch, "Format is :psv. Invalid input: #{row.class.name}") unless row.is_a?(String)
23 |
24 | row.split("|")
25 | end
26 |
27 | # Return the supplied array as a single line JSON string.
28 | def render(row, header)
29 | array = header.to_array(row)
30 | cleansed_array = array.collect do |i|
31 | i.is_a?(String) ? i.tr("|", ":") : i
32 | end
33 | cleansed_array.join("|")
34 | end
35 | end
36 | end
37 | end
38 | end
39 |
--------------------------------------------------------------------------------
/lib/io_streams/tabular/utility/csv_row.rb:
--------------------------------------------------------------------------------
1 | require "csv"
2 | module IOStreams
3 | class Tabular
4 | module Utility
5 | # For parsing a single line of CSV at a time
6 | # 2 to 3 times better performance than CSV.parse_line and considerably less
7 | # garbage collection required.
8 | #
9 | # Note: Only used prior to Ruby 2.6
10 | class CSVRow < ::CSV
11 | UTF8_ENCODING = Encoding.find("UTF-8").freeze
12 |
13 | def initialize(encoding = UTF8_ENCODING)
14 | @io = StringIO.new("".force_encoding(encoding))
15 | super(@io, row_sep: "")
16 | end
17 |
18 | # Parse a single line of CSV data
19 | # Parameters
20 | # line [String]
21 | # A single line of CSV data without any line terminators
22 | def parse(line)
23 | return if IOStreams::Utils.blank?(line)
24 | return if @skip_lines&.match(line)
25 |
26 | in_extended_col = false
27 | csv = []
28 | parts = line.split(@col_sep, -1)
29 | csv << nil if parts.empty?
30 |
31 | # This loop is the hot path of csv parsing. Some things may be non-dry
32 | # for a reason. Make sure to benchmark when refactoring.
33 | parts.each do |part|
34 | if in_extended_col
35 | # If we are continuing a previous column
36 | if part[-1] == @quote_char && part.count(@quote_char).odd?
37 | # extended column ends
38 | csv.last << part[0..-2]
39 | raise MalformedCSVError, "Missing or stray quote in line #{lineno + 1}" if csv.last =~ @parsers[:stray_quote]
40 |
41 | csv.last.gsub!(@quote_char * 2, @quote_char)
42 | in_extended_col = false
43 | else
44 | csv.last << part
45 | csv.last << @col_sep
46 | end
47 | elsif part[0] == @quote_char
48 | # If we are starting a new quoted column
49 | if part[-1] != @quote_char || part.count(@quote_char).odd?
50 | # start an extended column
51 | csv << part[1..-1]
52 | csv.last << @col_sep
53 | in_extended_col = true
54 | else
55 | # regular quoted column
56 | csv << part[1..-2]
57 | raise MalformedCSVError, "Missing or stray quote in line #{lineno + 1}" if csv.last =~ @parsers[:stray_quote]
58 |
59 | csv.last.gsub!(@quote_char * 2, @quote_char)
60 | end
61 | elsif part =~ @parsers[:quote_or_nl]
62 | # Unquoted field with bad characters.
63 | if part =~ @parsers[:nl_or_lf]
64 | raise MalformedCSVError, "Unquoted fields do not allow \\r or \\n (line #{lineno + 1})."
65 | else
66 | raise MalformedCSVError, "Illegal quoting in line #{lineno + 1}."
67 | end
68 | else
69 | # Regular ole unquoted field.
70 | csv << (part.empty? ? nil : part)
71 | end
72 | end
73 |
74 | # Replace tacked on @col_sep with @row_sep if we are still in an extended
75 | # column.
76 | csv[-1][-1] = @row_sep if in_extended_col
77 |
78 | raise MalformedCSVError, "Unclosed quoted field on line #{lineno + 1}." if in_extended_col
79 |
80 | @lineno += 1
81 |
82 | # save fields unconverted fields, if needed...
83 | unconverted = csv.dup if @unconverted_fields
84 |
85 | # convert fields, if needed...
86 | csv = convert_fields(csv) unless @use_headers || @converters.empty?
87 | # parse out header rows and handle CSV::Row conversions...
88 | csv = parse_headers(csv) if @use_headers
89 |
90 | # inject unconverted fields and accessor, if requested...
91 | add_unconverted_fields(csv, unconverted) if @unconverted_fields && (!csv.respond_to? :unconverted_fields)
92 |
93 | csv
94 | end
95 |
96 | # Return the supplied array as a single line CSV string.
97 | def render(row)
98 | row.map(&@quote).join(@col_sep) + @row_sep # quote and separate
99 | end
100 |
101 | alias to_csv render
102 | end
103 | end
104 | end
105 | end
106 |
--------------------------------------------------------------------------------
/lib/io_streams/utils.rb:
--------------------------------------------------------------------------------
1 | require "uri"
2 | require "tmpdir"
3 | module IOStreams
4 | module Utils
5 | MAX_TEMP_FILE_NAME_ATTEMPTS = 5
6 |
7 | # Lazy load dependent gem so that it remains a soft dependency.
8 | def self.load_soft_dependency(gem_name, stream_type, require_name = gem_name)
9 | require require_name
10 | rescue LoadError => e
11 | raise(LoadError, "Please install the gem '#{gem_name}' to support #{stream_type}. #{e.message}")
12 | end
13 |
14 | # Helper method: Returns [true|false] if a value is blank?
15 | def self.blank?(value)
16 | return true if value.nil?
17 | return value !~ /\S/ if value.is_a?(String)
18 |
19 | value.respond_to?(:empty?) ? value.empty? : !value
20 | end
21 |
22 | # Yields the path to a temporary file_name.
23 | #
24 | # File is deleted upon completion if present.
25 | def self.temp_file_name(basename, extension = "")
26 | result = nil
27 | ::Dir::Tmpname.create([basename, extension], IOStreams.temp_dir, max_try: MAX_TEMP_FILE_NAME_ATTEMPTS) do |tmpname|
28 | result = yield(tmpname)
29 | ensure
30 | ::File.unlink(tmpname) if ::File.exist?(tmpname)
31 | end
32 | result
33 | end
34 |
35 | class URI
36 | attr_reader :scheme, :hostname, :path, :user, :password, :port, :query
37 |
38 | def initialize(url)
39 | url = url.gsub(" ", "%20")
40 | uri = ::URI.parse(url)
41 | @scheme = uri.scheme
42 | @hostname = uri.hostname
43 | @path = CGI.unescape(uri.path)
44 | @user = uri.user
45 | @password = uri.password
46 | @port = uri.port
47 | return unless uri.query
48 |
49 | @query = {}
50 | ::URI.decode_www_form(uri.query).each { |key, value| @query[key] = value }
51 | end
52 | end
53 | end
54 | end
55 |
--------------------------------------------------------------------------------
/lib/io_streams/version.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | VERSION = "1.10.3".freeze
3 | end
4 |
--------------------------------------------------------------------------------
/lib/io_streams/writer.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | class Writer
3 | # When a Writer does not support streams, we copy the stream to a local temp file
4 | # and then pass that filename in for this reader.
5 | def self.stream(output_stream, original_file_name: nil, **args, &block)
6 | Utils.temp_file_name("iostreams_writer") do |file_name|
7 | count = file(file_name, original_file_name: original_file_name, **args, &block)
8 | ::File.open(file_name, "rb") { |source| ::IO.copy_stream(source, output_stream) }
9 | count
10 | end
11 | end
12 |
13 | # When a Writer supports streams, also allow it to simply support a file
14 | def self.file(file_name, original_file_name: file_name, **args, &block)
15 | ::File.open(file_name, "wb") { |file| stream(file, original_file_name: original_file_name, **args, &block) }
16 | end
17 |
18 | # For processing by either a file name or an open IO stream.
19 | def self.open(file_name_or_io, **args, &block)
20 | file_name_or_io.is_a?(String) ? file(file_name_or_io, **args, &block) : stream(file_name_or_io, **args, &block)
21 | end
22 |
23 | attr_reader :output_stream
24 |
25 | def initialize(output_stream)
26 | @output_stream = output_stream
27 | end
28 | end
29 | end
30 |
--------------------------------------------------------------------------------
/lib/io_streams/xlsx/reader.rb:
--------------------------------------------------------------------------------
1 | require "csv"
2 |
3 | module IOStreams
4 | module Xlsx
5 | class Reader < IOStreams::Reader
6 | # Convert a xlsx, or xlsm file into CSV format.
7 | def self.file(file_name, original_file_name: file_name, &block)
8 | # Stream into a temp file as csv
9 | Utils.temp_file_name("iostreams_csv") do |temp_file_name|
10 | ::File.open(temp_file_name, "wb") { |io| new(file_name).each { |lines| io << lines.to_csv } }
11 | ::File.open(temp_file_name, "rb", &block)
12 | end
13 | end
14 |
15 | def initialize(file_name)
16 | begin
17 | require "creek" unless defined?(Creek::Book)
18 | rescue LoadError => e
19 | raise(LoadError, "Please install the 'creek' gem for xlsx streaming support. #{e.message}")
20 | end
21 |
22 | workbook = Creek::Book.new(file_name, check_file_extension: false)
23 | @worksheet = workbook.sheets[0]
24 | end
25 |
26 | # Returns each [Array] row from the spreadsheet
27 | def each
28 | @worksheet.rows.each { |row| yield row.values }
29 | end
30 | end
31 | end
32 | end
33 |
--------------------------------------------------------------------------------
/lib/io_streams/zip/reader.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | module Zip
3 | class Reader < IOStreams::Reader
4 | # Read from a zip file or stream, decompressing the contents as it is read
5 | # The input stream from the first file found in the zip file is passed
6 | # to the supplied block.
7 | #
8 | # Parameters:
9 | # entry_file_name: [String]
10 | # Name of the file within the Zip file to read.
11 | # Default: Read the first file found in the zip file.
12 | #
13 | # Example:
14 | # IOStreams::Zip::Reader.open('abc.zip') do |io_stream|
15 | # # Read 256 bytes at a time
16 | # while data = io_stream.read(256)
17 | # puts data
18 | # end
19 | # end
20 | if defined?(JRuby)
21 | # Java has built-in support for Zip files
22 | def self.file(file_name, entry_file_name: nil)
23 | fin = Java::JavaIo::FileInputStream.new(file_name)
24 | zin = Java::JavaUtilZip::ZipInputStream.new(fin)
25 |
26 | get_entry(zin, entry_file_name) ||
27 | raise(Java::JavaUtilZip::ZipException, "File #{entry_file_name} not found within zip file.")
28 |
29 | yield(zin.to_io)
30 | ensure
31 | zin&.close
32 | fin&.close
33 | end
34 |
35 | def self.get_entry(zin, entry_file_name)
36 | if entry_file_name.nil?
37 | zin.get_next_entry
38 | return true
39 | end
40 |
41 | while (entry = zin.get_next_entry)
42 | return true if entry.name == entry_file_name
43 | end
44 | false
45 | end
46 | else
47 | # Read from a zip file or stream, decompressing the contents as it is read
48 | # The input stream from the first file found in the zip file is passed
49 | # to the supplied block
50 | def self.file(file_name, entry_file_name: nil, &block)
51 | Utils.load_soft_dependency("rubyzip v1.x", "Read Zip", "zip") unless defined?(::Zip)
52 |
53 | ::Zip::File.open(file_name) do |zip_file|
54 | if entry_file_name
55 | zip_file.get_input_stream(entry_file_name, &block)
56 | else
57 | result = nil
58 | # Return the first file
59 | zip_file.each do |entry|
60 | result = entry.get_input_stream(&block)
61 | break
62 | end
63 | result
64 | end
65 | end
66 | end
67 | end
68 | end
69 | end
70 | end
71 |
--------------------------------------------------------------------------------
/lib/io_streams/zip/writer.rb:
--------------------------------------------------------------------------------
1 | module IOStreams
2 | module Zip
3 | class Writer < IOStreams::Writer
4 | # Write a single file in Zip format to the supplied output stream
5 | #
6 | # Parameters
7 | # output_stream [IO]
8 | # Output stream to write to
9 | #
10 | # original_file_name [String]
11 | # Since this is a stream the original file name is used to create the entry_file_name if not supplied
12 | #
13 | # entry_file_name: [String]
14 | # Name of the file entry within the Zip file.
15 | #
16 | # The stream supplied to the block only responds to #write
17 | def self.stream(output_stream, original_file_name: nil, zip_file_name: nil, entry_file_name: zip_file_name)
18 | # Default the name of the file within the zip to the supplied file_name without the zip extension
19 | if entry_file_name.nil? && original_file_name && (original_file_name =~ /\.(zip)\z/i)
20 | entry_file_name = original_file_name.to_s[0..-5]
21 | end
22 | entry_file_name ||= "file"
23 |
24 | Utils.load_soft_dependency("zip_tricks", "Zip") unless defined?(ZipTricks::Streamer)
25 |
26 | result = nil
27 | ZipTricks::Streamer.open(output_stream) do |zip|
28 | zip.write_deflated_file(entry_file_name) { |io| result = yield(io) }
29 | end
30 | result
31 | end
32 | end
33 | end
34 | end
35 |
--------------------------------------------------------------------------------
/lib/iostreams.rb:
--------------------------------------------------------------------------------
1 | require "io_streams/version"
2 | # @formatter:off
3 | module IOStreams
4 | autoload :Builder, "io_streams/builder"
5 | autoload :Errors, "io_streams/errors"
6 | autoload :Path, "io_streams/path"
7 | autoload :Pgp, "io_streams/pgp"
8 | autoload :Reader, "io_streams/reader"
9 | autoload :Stream, "io_streams/stream"
10 | autoload :Tabular, "io_streams/tabular"
11 | autoload :Utils, "io_streams/utils"
12 | autoload :Writer, "io_streams/writer"
13 |
14 | module Paths
15 | autoload :File, "io_streams/paths/file"
16 | autoload :HTTP, "io_streams/paths/http"
17 | autoload :Matcher, "io_streams/paths/matcher"
18 | autoload :S3, "io_streams/paths/s3"
19 | autoload :SFTP, "io_streams/paths/sftp"
20 | end
21 |
22 | module Bzip2
23 | autoload :Reader, "io_streams/bzip2/reader"
24 | autoload :Writer, "io_streams/bzip2/writer"
25 | end
26 |
27 | module Encode
28 | autoload :Reader, "io_streams/encode/reader"
29 | autoload :Writer, "io_streams/encode/writer"
30 | end
31 |
32 | module Gzip
33 | autoload :Reader, "io_streams/gzip/reader"
34 | autoload :Writer, "io_streams/gzip/writer"
35 | end
36 |
37 | module Line
38 | autoload :Reader, "io_streams/line/reader"
39 | autoload :Writer, "io_streams/line/writer"
40 | end
41 |
42 | module Record
43 | autoload :Reader, "io_streams/record/reader"
44 | autoload :Writer, "io_streams/record/writer"
45 | end
46 |
47 | module Row
48 | autoload :Reader, "io_streams/row/reader"
49 | autoload :Writer, "io_streams/row/writer"
50 | end
51 |
52 | module SymmetricEncryption
53 | autoload :Reader, "io_streams/symmetric_encryption/reader"
54 | autoload :Writer, "io_streams/symmetric_encryption/writer"
55 | end
56 |
57 | module Xlsx
58 | autoload :Reader, "io_streams/xlsx/reader"
59 | end
60 |
61 | module Zip
62 | autoload :Reader, "io_streams/zip/reader"
63 | autoload :Writer, "io_streams/zip/writer"
64 | end
65 | end
66 | require "io_streams/deprecated"
67 | require "io_streams/io_streams"
68 |
--------------------------------------------------------------------------------
/test/bzip2_reader_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 |
3 | class Bzip2ReaderTest < Minitest::Test
4 | describe IOStreams::Bzip2::Reader do
5 | let :file_name do
6 | File.join(File.dirname(__FILE__), "files", "text.txt.bz2")
7 | end
8 |
9 | let :decompressed do
10 | File.read(File.join(File.dirname(__FILE__), "files", "text.txt"))
11 | end
12 |
13 | describe ".file" do
14 | it "file" do
15 | result = IOStreams::Bzip2::Reader.file(file_name, &:read)
16 | assert_equal decompressed, result
17 | end
18 |
19 | it "stream" do
20 | result = File.open(file_name) do |file|
21 | IOStreams::Bzip2::Reader.stream(file, &:read)
22 | end
23 | assert_equal decompressed, result
24 | end
25 | end
26 | end
27 | end
28 |
--------------------------------------------------------------------------------
/test/bzip2_writer_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 |
3 | class Bzip2WriterTest < Minitest::Test
4 | describe IOStreams::Bzip2::Writer do
5 | let :temp_file do
6 | Tempfile.new("iostreams")
7 | end
8 |
9 | let :file_name do
10 | temp_file.path
11 | end
12 |
13 | let :decompressed do
14 | File.read(File.join(File.dirname(__FILE__), "files", "text.txt"))
15 | end
16 |
17 | after do
18 | temp_file.delete
19 | end
20 |
21 | describe ".file" do
22 | it "file" do
23 | result =
24 | IOStreams::Bzip2::Writer.file(file_name) do |io|
25 | io.write(decompressed)
26 | io.write(decompressed)
27 | 53534
28 | end
29 | assert_equal 53534, result
30 |
31 | File.open(file_name, "rb") do |file|
32 | io = ::Bzip2::FFI::Reader.new(file)
33 | result = io.read
34 | temp_file.delete
35 | assert_equal decompressed + decompressed, result
36 | end
37 | end
38 |
39 | it "stream" do
40 | io_string = StringIO.new("".b)
41 | result =
42 | IOStreams::Bzip2::Writer.stream(io_string) do |io|
43 | io.write(decompressed)
44 | io.write(decompressed)
45 | 53534
46 | end
47 | assert_equal 53534, result
48 |
49 | io = StringIO.new(io_string.string)
50 | rbzip2 = ::Bzip2::FFI::Reader.new(io)
51 | data = rbzip2.read
52 | assert_equal decompressed + decompressed, data
53 | end
54 | end
55 | end
56 | end
57 |
--------------------------------------------------------------------------------
/test/deprecated_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 |
3 | IOStreams.include(IOStreams::Deprecated)
4 |
5 | # Test deprecated api
6 | class DeprecatedTest < Minitest::Test
7 | describe IOStreams do
8 | let :source_file_name do
9 | File.join(__dir__, "files", "text.txt")
10 | end
11 |
12 | let :data do
13 | File.read(source_file_name)
14 | end
15 |
16 | let :temp_file do
17 | Tempfile.new("iostreams")
18 | end
19 |
20 | let :target_file_name do
21 | temp_file.path
22 | end
23 |
24 | let :bad_data do
25 | [
26 | "New M\xE9xico,NE".b,
27 | "good line",
28 | "New M\xE9xico,\x07SF".b
29 | ].join("\n").encode("BINARY")
30 | end
31 |
32 | let :stripped_data do
33 | bad_data.gsub("\xE9".b, "").gsub("\x07", "")
34 | end
35 |
36 | let :multiple_zip_file_name do
37 | File.join(File.dirname(__FILE__), "files", "multiple_files.zip")
38 | end
39 |
40 | let :zip_gz_file_name do
41 | File.join(File.dirname(__FILE__), "files", "text.zip.gz")
42 | end
43 |
44 | let :contents_test_txt do
45 | File.read(File.join(File.dirname(__FILE__), "files", "text.txt"))
46 | end
47 |
48 | let :contents_test_json do
49 | File.read(File.join(File.dirname(__FILE__), "files", "test.json"))
50 | end
51 |
52 | after do
53 | temp_file.delete
54 | end
55 |
56 | describe ".copy" do
57 | it "streams" do
58 | size = IOStreams.reader(source_file_name) do |source_stream|
59 | IOStreams.writer(target_file_name) do |target_stream|
60 | IOStreams.copy(source_stream, target_stream)
61 | end
62 | end
63 | actual = File.read(target_file_name)
64 |
65 | assert_equal actual, data
66 | assert_equal actual.size, size
67 | end
68 |
69 | it "IO stream" do
70 | size = File.open(source_file_name) do |source_stream|
71 | IOStreams.writer(target_file_name) do |target_stream|
72 | IOStreams.copy(source_stream, target_stream)
73 | end
74 | end
75 | actual = File.read(target_file_name)
76 |
77 | assert_equal actual, data
78 | assert_equal actual.size, size
79 | end
80 |
81 | it "files" do
82 | size = IOStreams.copy(source_file_name, target_file_name)
83 | actual = File.read(target_file_name)
84 |
85 | assert_equal actual, data
86 | assert_equal actual.size, size
87 | end
88 | end
89 |
90 | describe ".each_line" do
91 | it "returns a line at a time" do
92 | lines = []
93 | count = IOStreams.each_line(source_file_name) { |line| lines << line }
94 | assert_equal data.lines.map(&:strip), lines
95 | assert_equal data.lines.count, count
96 | end
97 |
98 | it "strips non-printable characters" do
99 | input = StringIO.new(bad_data)
100 | lines = []
101 | count = IOStreams.each_line(input, encoding: "UTF-8", encode_cleaner: :printable, encode_replace: "") do |line|
102 | lines << line
103 | end
104 | assert_equal stripped_data.lines.map(&:strip), lines
105 | assert_equal stripped_data.lines.count, count
106 | end
107 | end
108 |
109 | describe ".reader" do
110 | it "reads a zip file" do
111 | result = IOStreams.reader(multiple_zip_file_name, streams: {zip: {entry_file_name: "test.json"}}, &:read)
112 | assert_equal contents_test_json, result
113 | end
114 |
115 | it "reads a zip file from within a gz file" do
116 | result = IOStreams.reader(zip_gz_file_name, &:read)
117 | assert_equal contents_test_txt, result
118 | end
119 | end
120 | end
121 | end
122 |
--------------------------------------------------------------------------------
/test/encode_reader_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 |
3 | class EncodeReaderTest < Minitest::Test
4 | describe IOStreams::Encode::Reader do
5 | let :bad_data do
6 | [
7 | "New M\xE9xico,NE".b,
8 | "good line",
9 | "New M\xE9xico,\x07SF".b
10 | ].join("\n").encode("BINARY")
11 | end
12 |
13 | let :cleansed_data do
14 | bad_data.gsub("\xE9".b, "")
15 | end
16 |
17 | let :stripped_data do
18 | cleansed_data.gsub("\x07", "")
19 | end
20 |
21 | describe "#read" do
22 | describe "replacement" do
23 | it "does not strip invalid characters" do
24 | skip "Does not raise on JRuby" if defined?(JRuby)
25 | input = StringIO.new(bad_data)
26 | IOStreams::Encode::Reader.stream(input, encoding: "UTF-8") do |io|
27 | assert_raises ::Encoding::UndefinedConversionError do
28 | io.read.encoding
29 | end
30 | end
31 | end
32 |
33 | it "strips invalid characters" do
34 | input = StringIO.new(bad_data)
35 | data =
36 | IOStreams::Encode::Reader.stream(input, encoding: "UTF-8", replace: "", &:read)
37 | assert_equal cleansed_data, data
38 | end
39 | end
40 |
41 | describe "printable" do
42 | it "strips non-printable characters" do
43 | input = StringIO.new(bad_data)
44 | data =
45 | IOStreams::Encode::Reader.stream(input, encoding: "UTF-8", cleaner: :printable, replace: "", &:read)
46 | assert_equal stripped_data, data
47 | end
48 | end
49 | end
50 | end
51 | end
52 |
--------------------------------------------------------------------------------
/test/encode_writer_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 |
3 | class EncodeWriterTest < Minitest::Test
4 | describe IOStreams::Encode::Writer do
5 | let :bad_data do
6 | [
7 | "New M\xE9xico,NE".b,
8 | "good line",
9 | "New M\xE9xico,\x07SF".b
10 | ].join("\n").encode("BINARY")
11 | end
12 |
13 | let :cleansed_data do
14 | bad_data.gsub("\xE9".b, "?")
15 | end
16 |
17 | let :stripped_data do
18 | cleansed_data.gsub("\x07", "")
19 | end
20 |
21 | describe "#<<" do
22 | it "file" do
23 | temp_file = Tempfile.new("rocket_job")
24 | file_name = temp_file.to_path
25 | result =
26 | IOStreams::Encode::Writer.file(file_name, encoding: "ASCII-8BIT") do |io|
27 | io << bad_data
28 | 53534
29 | end
30 | assert_equal 53534, result
31 | result = File.read(file_name, mode: "rb")
32 | assert_equal bad_data, result
33 | end
34 |
35 | it "stream" do
36 | io = StringIO.new("".b)
37 | result =
38 | IOStreams::Encode::Writer.stream(io, encoding: "ASCII-8BIT") do |encoded|
39 | encoded << bad_data
40 | 53534
41 | end
42 | assert_equal 53534, result
43 | assert_equal "ASCII-8BIT", io.string.encoding.to_s
44 | assert_equal bad_data, io.string
45 | end
46 |
47 | it "stream as utf-8" do
48 | io = StringIO.new("")
49 | assert_raises Encoding::UndefinedConversionError do
50 | IOStreams::Encode::Writer.stream(io, encoding: "UTF-8") do |encoded|
51 | encoded << bad_data
52 | end
53 | end
54 | end
55 |
56 | it "stream as utf-8 with replacement" do
57 | io = StringIO.new("")
58 | IOStreams::Encode::Writer.stream(io, encoding: "UTF-8", replace: "?") do |encoded|
59 | encoded << bad_data
60 | end
61 | assert_equal "UTF-8", io.string.encoding.to_s
62 | assert_equal cleansed_data, io.string
63 | end
64 |
65 | it "stream as utf-8 with replacement and printable cleansing" do
66 | io = StringIO.new("")
67 | IOStreams::Encode::Writer.stream(io, encoding: "UTF-8", replace: "?", cleaner: :printable) do |encoded|
68 | encoded << bad_data
69 | end
70 | assert_equal "UTF-8", io.string.encoding.to_s
71 | assert_equal stripped_data, io.string
72 | end
73 | end
74 |
75 | describe ".write" do
76 | it "returns byte count" do
77 | io_string = StringIO.new("".b)
78 | count = 0
79 | result =
80 | IOStreams::Encode::Writer.stream(io_string, encoding: "ASCII-8BIT") do |io|
81 | count += io.write(bad_data)
82 | 53534
83 | end
84 | assert_equal 53534, result
85 | assert_equal bad_data, io_string.string
86 | assert_equal bad_data.size, count
87 | end
88 | end
89 | end
90 | end
91 |
--------------------------------------------------------------------------------
/test/files/embedded_lines_test.csv:
--------------------------------------------------------------------------------
1 | name, description, zip
2 | "
3 | Jack","Firstname is Jack","234567"
4 | "John","Firstname
5 | is John","234568"
6 | "Zack","Firstname is Zack","234568
7 | "
--------------------------------------------------------------------------------
/test/files/multiple_files.zip:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/test/files/multiple_files.zip
--------------------------------------------------------------------------------
/test/files/spreadsheet.xlsx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/test/files/spreadsheet.xlsx
--------------------------------------------------------------------------------
/test/files/test.csv:
--------------------------------------------------------------------------------
1 | name, address, state, zip
2 | Joe Black, 23 Where Rd, FL, 12452
3 | Who is this, 42 Able Rd, GA, 23432
4 | Another's Person, 42 and 2nd St, NY, 17449
5 |
--------------------------------------------------------------------------------
/test/files/test.json:
--------------------------------------------------------------------------------
1 | {"name":"Joe Black"," address":" 23 Where Rd"," state":" FL"," zip":" 12452"}
2 | {"name":"Who is this"," address":" 42 Able Rd"," state":" GA"," zip":" 23432"}
3 | {"name":"Another's Person"," address":" 42 and 2nd St"," state":" NY"," zip":" 17449"}
4 |
--------------------------------------------------------------------------------
/test/files/test.psv:
--------------------------------------------------------------------------------
1 | name|address|state|zip
2 | Joe Black|23 Where Rd|FL|12452
3 | Who is this|42 Able Rd|GA|23432
4 | Another's Person|42 and 2nd St|NY|17449
5 |
--------------------------------------------------------------------------------
/test/files/text file.txt:
--------------------------------------------------------------------------------
1 | Hello World
2 | Line2
3 | Line3
4 |
--------------------------------------------------------------------------------
/test/files/text.txt:
--------------------------------------------------------------------------------
1 | Hello World
2 | Line2
3 | Line3
4 |
--------------------------------------------------------------------------------
/test/files/text.txt.bz2:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/test/files/text.txt.bz2
--------------------------------------------------------------------------------
/test/files/text.txt.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/test/files/text.txt.gz
--------------------------------------------------------------------------------
/test/files/text.txt.gz.zip:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/test/files/text.txt.gz.zip
--------------------------------------------------------------------------------
/test/files/text.zip:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/test/files/text.zip
--------------------------------------------------------------------------------
/test/files/text.zip.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reidmorrison/iostreams/8188e924eaa3b8904c133b94391d4152cbefddde/test/files/text.zip.gz
--------------------------------------------------------------------------------
/test/files/unclosed_quote_test.csv:
--------------------------------------------------------------------------------
1 | name, description, zip
2 | "Jack","Firstn"ame is Jack","234567"
3 | "John","Firstname is John","234568"
4 | "Zack","Firstname is Zack","234568"
--------------------------------------------------------------------------------
/test/files/unclosed_quote_test2.csv:
--------------------------------------------------------------------------------
1 | name, description, zip
2 | Jack,Firstn"ame is Jack,234567
3 | John,Firstname is John,234568
4 |
--------------------------------------------------------------------------------
/test/gzip_reader_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 |
3 | class GzipReaderTest < Minitest::Test
4 | describe IOStreams::Gzip::Reader do
5 | let :file_name do
6 | File.join(File.dirname(__FILE__), "files", "text.txt.gz")
7 | end
8 |
9 | let :decompressed do
10 | Zlib::GzipReader.open(file_name, &:read)
11 | end
12 |
13 | describe ".open" do
14 | it "file" do
15 | result = IOStreams::Gzip::Reader.file(file_name, &:read)
16 | assert_equal decompressed, result
17 | end
18 |
19 | it "stream" do
20 | result = File.open(file_name) do |file|
21 | IOStreams::Gzip::Reader.stream(file, &:read)
22 | end
23 | assert_equal decompressed, result
24 | end
25 | end
26 | end
27 | end
28 |
--------------------------------------------------------------------------------
/test/gzip_writer_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 |
3 | class GzipWriterTest < Minitest::Test
4 | describe IOStreams::Gzip::Writer do
5 | let :temp_file do
6 | Tempfile.new("iostreams")
7 | end
8 |
9 | let :file_name do
10 | temp_file.path
11 | end
12 |
13 | let :decompressed do
14 | File.read(File.join(File.dirname(__FILE__), "files", "text.txt"))
15 | end
16 |
17 | after do
18 | temp_file.delete
19 | end
20 |
21 | describe ".file" do
22 | it "file" do
23 | result =
24 | IOStreams::Gzip::Writer.file(file_name) do |io|
25 | io.write(decompressed)
26 | 53534
27 | end
28 | assert_equal 53534, result
29 |
30 | result = Zlib::GzipReader.open(file_name, &:read)
31 | temp_file.delete
32 | assert_equal decompressed, result
33 | end
34 |
35 | it "stream" do
36 | io_string = StringIO.new("".b)
37 | result =
38 | IOStreams::Gzip::Writer.stream(io_string) do |io|
39 | io.write(decompressed)
40 | 53534
41 | end
42 | assert_equal 53534, result
43 |
44 | io = StringIO.new(io_string.string)
45 | gz = Zlib::GzipReader.new(io)
46 | data = gz.read
47 | gz.close
48 | assert_equal decompressed, data
49 | end
50 | end
51 | end
52 | end
53 |
--------------------------------------------------------------------------------
/test/io_streams_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 | require "json"
3 |
4 | module IOStreams
5 | class PathTest < Minitest::Test
6 | describe IOStreams do
7 | let :records do
8 | [
9 | {"name" => "Jack Jones", "login" => "jjones"},
10 | {"name" => "Jill Smith", "login" => "jsmith"}
11 | ]
12 | end
13 |
14 | let :expected_json do
15 | records.collect(&:to_json).join("\n") + "\n"
16 | end
17 |
18 | let :json_file_name do
19 | "/tmp/iostreams_abc.json"
20 | end
21 |
22 | describe ".root" do
23 | it "return default path" do
24 | path = ::File.expand_path(::File.join(__dir__, "../tmp/default"))
25 | assert_equal path, IOStreams.root.to_s
26 | end
27 |
28 | it "return downloads path" do
29 | path = ::File.expand_path(::File.join(__dir__, "../tmp/downloads"))
30 | assert_equal path, IOStreams.root(:downloads).to_s
31 | end
32 | end
33 |
34 | describe ".join" do
35 | it "returns path" do
36 | assert_equal IOStreams.root.to_s, IOStreams.join.to_s
37 | end
38 |
39 | it "adds path to root" do
40 | assert_equal ::File.join(IOStreams.root.to_s, "test"), IOStreams.join("test").to_s
41 | end
42 |
43 | it "adds paths to root" do
44 | assert_equal ::File.join(IOStreams.root.to_s, "test", "second", "third"), IOStreams.join("test", "second", "third").to_s
45 | end
46 |
47 | it "returns path and filename" do
48 | path = ::File.join(IOStreams.root.to_s, "file.xls")
49 | assert_equal path, IOStreams.join("file.xls").to_s
50 | end
51 |
52 | it "adds path to root and filename" do
53 | path = ::File.join(IOStreams.root.to_s, "test", "file.xls")
54 | assert_equal path, IOStreams.join("test", "file.xls").to_s
55 | end
56 |
57 | it "adds paths to root" do
58 | path = ::File.join(IOStreams.root.to_s, "test", "second", "third", "file.xls")
59 | assert_equal path, IOStreams.join("test", "second", "third", "file.xls").to_s
60 | end
61 |
62 | it "return path as sent in when full path" do
63 | path = ::File.join(IOStreams.root.to_s, "file.xls")
64 | assert_equal path, IOStreams.join(path).to_s
65 | end
66 | end
67 |
68 | describe ".path" do
69 | it "default" do
70 | path = IOStreams.path("a.xyz")
71 | assert path.is_a?(IOStreams::Paths::File), path
72 | end
73 |
74 | it "s3" do
75 | skip "TODO"
76 | IOStreams.path("s3://a.xyz")
77 | assert_equal :s3, path
78 | end
79 |
80 | it "hash writer detects json format from file name" do
81 | path = IOStreams.path("/tmp/io_streams/abc.json")
82 | path.writer(:hash) do |io|
83 | records.each { |hash| io << hash }
84 | end
85 | actual = path.read
86 | path.delete
87 | assert_equal expected_json, actual
88 | end
89 |
90 | it "hash reader detects json format from file name" do
91 | ::File.open(json_file_name, "wb") { |file| file.write(expected_json) }
92 | rows = []
93 | path = IOStreams.path(json_file_name)
94 | path.each(:hash) do |row|
95 | rows << row
96 | end
97 | actual = rows.collect(&:to_json).join("\n") + "\n"
98 | path.delete
99 | assert_equal expected_json, actual
100 | end
101 |
102 | it "array writer detects json format from file name" do
103 | path = IOStreams.path("/tmp/io_streams/abc.json")
104 | path.writer(:array, columns: %w[name login]) do |io|
105 | io << ["Jack Jones", "jjones"]
106 | io << ["Jill Smith", "jsmith"]
107 | end
108 | actual = path.read
109 | path.delete
110 | assert_equal expected_json, actual
111 | end
112 | end
113 |
114 | describe ".temp_file" do
115 | it "returns value from block" do
116 | result = IOStreams.temp_file("base", ".ext") { |_path| 257 }
117 | assert_equal 257, result
118 | end
119 |
120 | it "supplies new temp file_name" do
121 | path1 = nil
122 | path2 = nil
123 | IOStreams.temp_file("base", ".ext") { |path| path1 = path }
124 | IOStreams.temp_file("base", ".ext") { |path| path2 = path }
125 | refute_equal path1.to_s, path2.to_s
126 | assert path1.is_a?(IOStreams::Paths::File), path1
127 | assert path2.is_a?(IOStreams::Paths::File), path2
128 | end
129 | end
130 | end
131 | end
132 | end
133 |
--------------------------------------------------------------------------------
/test/line_writer_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 |
3 | class DelimitedWriterTest < Minitest::Test
4 | describe IOStreams::Line::Writer do
5 | let :file_name do
6 | File.join(File.dirname(__FILE__), "files", "text.txt")
7 | end
8 |
9 | let :raw do
10 | File.read(file_name)
11 | end
12 |
13 | let :lines do
14 | raw.lines.map(&:strip)
15 | end
16 |
17 | describe "#<<" do
18 | it "file" do
19 | temp_file = Tempfile.new("rocket_job")
20 | file_name = temp_file.to_path
21 | result =
22 | IOStreams::Line::Writer.file(file_name) do |io|
23 | lines.each { |line| io << line }
24 | 53534
25 | end
26 | assert_equal 53534, result
27 |
28 | result = File.read(file_name)
29 | assert_equal raw, result
30 | end
31 |
32 | it "stream" do
33 | io_string = StringIO.new
34 | result =
35 | IOStreams::Line::Writer.stream(io_string) do |io|
36 | lines.each { |line| io << line }
37 | 53534
38 | end
39 | assert_equal 53534, result
40 | assert_equal raw, io_string.string
41 | end
42 | end
43 |
44 | describe ".write" do
45 | it "returns byte count" do
46 | io_string = StringIO.new
47 | count = 0
48 | result =
49 | IOStreams::Line::Writer.stream(io_string) do |io|
50 | lines.each { |line| count += io.write(line) }
51 | 53534
52 | end
53 | assert_equal 53534, result
54 | assert_equal raw, io_string.string
55 | assert_equal raw.size, count
56 | end
57 | end
58 | end
59 | end
60 |
--------------------------------------------------------------------------------
/test/minimal_file_reader.rb:
--------------------------------------------------------------------------------
1 | # The miminum methods that any IOStreams Reader must implement.
2 | class MinimalFileReader
3 | def self.open(file_name)
4 | io = new(file_name)
5 | yield(io)
6 | ensure
7 | io&.close
8 | end
9 |
10 | def initialize(file_name)
11 | @file = File.open(file_name)
12 | end
13 |
14 | def read(size = nil, outbuf = nil)
15 | @file.read(size, outbuf)
16 | end
17 |
18 | def close
19 | @file.close
20 | end
21 |
22 | def closed?
23 | @file.closed
24 | end
25 | end
26 |
--------------------------------------------------------------------------------
/test/path_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 |
3 | module IOStreams
4 | class PathTest < Minitest::Test
5 | describe IOStreams::Path do
6 | describe ".join" do
7 | let(:path) { IOStreams::Path.new("some_path") }
8 |
9 | it "returns self when no elements" do
10 | assert_equal path.object_id, path.join.object_id
11 | end
12 |
13 | it "adds element to path" do
14 | assert_equal ::File.join("some_path", "test"), path.join("test").to_s
15 | end
16 |
17 | it "adds paths to root" do
18 | assert_equal ::File.join("some_path", "test", "second", "third"), path.join("test", "second", "third").to_s
19 | end
20 |
21 | it "returns path and filename" do
22 | assert_equal ::File.join("some_path", "file.xls"), path.join("file.xls").to_s
23 | end
24 |
25 | it "adds elements to path" do
26 | assert_equal ::File.join("some_path", "test", "second", "third", "file.xls"), path.join("test", "second", "third", "file.xls").to_s
27 | end
28 |
29 | it "return path as sent in when full path" do
30 | assert_equal ::File.join("some_path", "test", "second", "third", "file.xls"), path.join("some_path", "test", "second", "third", "file.xls").to_s
31 | end
32 | end
33 |
34 | describe "#absolute?" do
35 | it "true on absolute" do
36 | assert_equal true, IOStreams::Path.new("/a/b/c/d").absolute?
37 | end
38 |
39 | it "false when not absolute" do
40 | assert_equal false, IOStreams::Path.new("a/b/c/d").absolute?
41 | end
42 | end
43 |
44 | describe "#relatve?" do
45 | it "true on relative" do
46 | assert_equal true, IOStreams::Path.new("a/b/c/d").relative?
47 | end
48 |
49 | it "false on absolute" do
50 | assert_equal false, IOStreams::Path.new("/a/b/c/d").relative?
51 | end
52 | end
53 | end
54 | end
55 | end
56 |
--------------------------------------------------------------------------------
/test/paths/file_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "../test_helper"
2 |
3 | module Paths
4 | class FileTest < Minitest::Test
5 | describe IOStreams::Paths::File do
6 | let(:root) { IOStreams::Paths::File.new("/tmp/iostreams").delete_all }
7 | let(:directory) { root.join("/some_test_dir") }
8 | let(:data) { "Hello World\nHow are you doing?\nOn this fine day" }
9 | let(:file_path) do
10 | path = root.join("some_test_dir/test_file.txt")
11 | path.writer { |io| io << data }
12 | path
13 | end
14 | let(:file_path2) do
15 | path = root.join("some_test_dir/test_file2.txt")
16 | path.writer { |io| io << "Hello World2" }
17 | path
18 | end
19 |
20 | describe "#each" do
21 | it "reads lines" do
22 | records = []
23 | count = file_path.each { |line| records << line }
24 | assert_equal count, data.lines.size
25 | assert_equal data.lines.collect(&:strip), records
26 | end
27 | end
28 |
29 | describe "#each_child" do
30 | it "iterates an empty path" do
31 | none = nil
32 | directory.join("does_not_exist").mkdir.each_child { |path| none = path }
33 | assert_nil none
34 | end
35 |
36 | it "iterates a non-existant path" do
37 | none = nil
38 | directory.join("does_not_exist").each_child { |path| none = path }
39 | assert_nil none
40 | end
41 |
42 | it "find all files" do
43 | expected = [file_path.to_s, file_path2.to_s]
44 | actual = root.children("**/*").collect(&:to_s)
45 | assert_equal expected.sort, actual.sort
46 | end
47 |
48 | it "find matches case-insensitive" do
49 | expected = [file_path.to_s, file_path2.to_s]
50 | actual = root.children("**/Test*.TXT").collect(&:to_s)
51 | assert_equal expected, actual.sort
52 | end
53 |
54 | it "find matches case-sensitive" do
55 | skip "TODO"
56 | expected = [file_path.to_s, file_path2.to_s]
57 | actual = root.children("**/Test*.TXT", case_sensitive: true).collect(&:to_s)
58 | refute_equal expected, actual.sort
59 | end
60 |
61 | it "with no block returns enumerator" do
62 | expected = [file_path.to_s, file_path2.to_s]
63 | actual = root.each_child("**/*").first(100).collect(&:to_s)
64 | assert_equal expected.sort, actual.sort
65 | end
66 | end
67 |
68 | describe "#mkpath" do
69 | it "makes path skipping file_name" do
70 | new_path = directory.join("test_mkpath.xls").mkpath
71 | assert ::File.exist?(directory.to_s)
72 | refute ::File.exist?(new_path.to_s)
73 | end
74 | end
75 |
76 | describe "#mkdir" do
77 | it "makes entire path that does not have a file name" do
78 | new_path = directory.join("more_path").mkdir
79 | assert ::File.exist?(directory.to_s)
80 | assert ::File.exist?(new_path.to_s)
81 | end
82 | end
83 |
84 | describe "#exist?" do
85 | it "true on existing file or directory" do
86 | assert ::File.exist?(file_path.to_s)
87 | assert ::File.exist?(directory.to_s)
88 |
89 | assert directory.exist?
90 | assert file_path.exist?
91 | end
92 |
93 | it "false when not found" do
94 | non_existant_directory = directory.join("oh_no")
95 | refute ::File.exist?(non_existant_directory.to_s)
96 |
97 | non_existant_file_path = directory.join("abc.txt")
98 | refute ::File.exist?(non_existant_file_path.to_s)
99 |
100 | refute non_existant_directory.exist?
101 | refute non_existant_file_path.exist?
102 | end
103 | end
104 |
105 | describe "#size" do
106 | it "of file" do
107 | assert_equal data.size, file_path.size
108 | end
109 | end
110 |
111 | describe "#realpath" do
112 | it "already a real path" do
113 | path = ::File.expand_path(__dir__, "../files/test.csv")
114 | assert_equal path, IOStreams::Paths::File.new(path).realpath.to_s
115 | end
116 |
117 | it "removes .." do
118 | path = ::File.join(__dir__, "../files/test.csv")
119 | realpath = ::File.realpath(path)
120 | assert_equal realpath, IOStreams::Paths::File.new(path).realpath.to_s
121 | end
122 | end
123 |
124 | describe "#move_to" do
125 | it "move_to existing file" do
126 | IOStreams.temp_file("iostreams_move_test", ".txt") do |temp_file|
127 | temp_file.write("Hello World")
128 | begin
129 | target = temp_file.directory.join("move_test.txt")
130 | response = temp_file.move_to(target)
131 | assert_equal target, response
132 | assert target.exist?
133 | refute temp_file.exist?
134 | assert_equal "Hello World", response.read
135 | assert_equal target.to_s, response.to_s
136 | ensure
137 | target&.delete
138 | end
139 | end
140 | end
141 |
142 | it "missing source file" do
143 | IOStreams.temp_file("iostreams_move_test", ".txt") do |temp_file|
144 | refute temp_file.exist?
145 | target = temp_file.directory.join("move_test.txt")
146 | assert_raises Errno::ENOENT do
147 | temp_file.move_to(target)
148 | end
149 | refute target.exist?
150 | refute temp_file.exist?
151 | end
152 | end
153 |
154 | it "missing target directories" do
155 | IOStreams.temp_file("iostreams_move_test", ".txt") do |temp_file|
156 | temp_file.write("Hello World")
157 | begin
158 | target = temp_file.directory.join("a/b/c/move_test.txt")
159 | response = temp_file.move_to(target)
160 | assert_equal target, response
161 | assert target.exist?
162 | refute temp_file.exist?
163 | assert_equal "Hello World", response.read
164 | assert_equal target.to_s, response.to_s
165 | ensure
166 | temp_file.directory.join("a").delete_all
167 | end
168 | end
169 | end
170 | end
171 |
172 | describe "#delete" do
173 | it "deletes existing file" do
174 | assert ::File.exist?(file_path.to_s)
175 | file_path.delete
176 | refute ::File.exist?(file_path.to_s)
177 | end
178 |
179 | it "ignores missing file" do
180 | file_path.delete
181 | file_path.delete
182 | end
183 | end
184 |
185 | describe "reader" do
186 | it "reads file" do
187 | assert_equal data, file_path.read
188 | end
189 | end
190 |
191 | describe "writer" do
192 | it "creates file" do
193 | new_file_path = directory.join("new.txt")
194 | refute ::File.exist?(new_file_path.to_s)
195 | new_file_path.writer { |io| io << data }
196 | assert ::File.exist?(new_file_path.to_s)
197 | assert_equal data.size, new_file_path.size
198 | end
199 | end
200 | end
201 | end
202 | end
203 |
--------------------------------------------------------------------------------
/test/paths/http_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "../test_helper"
2 |
3 | module Paths
4 | class HTTPTest < Minitest::Test
5 | describe IOStreams::Paths::HTTP do
6 | let :url do
7 | "http://example.com/index.html?count=10"
8 | end
9 |
10 | let :ssl_url do
11 | "https://example.com/index.html?count=10"
12 | end
13 |
14 | describe ".open" do
15 | it "reads http" do
16 | result = IOStreams::Paths::HTTP.new(url).read
17 | assert_includes result, ""
18 | end
19 |
20 | it "reads https" do
21 | result = IOStreams::Paths::HTTP.new(ssl_url).read
22 | assert_includes result, ""
23 | end
24 |
25 | it "does not support streams" do
26 | assert_raises URI::InvalidURIError do
27 | io = StringIO.new
28 | IOStreams::Paths::HTTP.new(io)
29 | end
30 | end
31 | end
32 | end
33 | end
34 | end
35 |
--------------------------------------------------------------------------------
/test/paths/matcher_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "../test_helper"
2 |
3 | module Paths
4 | class MatcherTest < Minitest::Test
5 | describe IOStreams::Paths::Matcher do
6 | let :cases do
7 | [
8 | {
9 | path: "/path/work",
10 | pattern: "a/b/c/**/*",
11 | expected_path: "/path/work/a/b/c",
12 | expected_pattern: "**/*",
13 | recursive: true,
14 | matches: %w[/path/work/a/b/c/any/file /path/work/a/b/c/other/file],
15 | not_matches: %w[/path/work/a/b/c/.profile /path/work/a/b/c/sub/.name]
16 | },
17 | {path: "/path/work", pattern: "a/b/c?/**", expected_path: "/path/work/a/b", expected_pattern: "c?/**", recursive: true},
18 | {path: "/path/work", pattern: "**", expected_path: "/path/work", expected_pattern: "**", recursive: true},
19 | # Case-insensitive exists that returns the actual file name.
20 | {path: "/path/work", pattern: "a/b/file.txt", expected_path: "/path/work/a/b/file.txt", expected_pattern: nil, recursive: false},
21 | {
22 | path: "/path/work",
23 | pattern: "a/b/file*{zip,gz}",
24 | expected_path: "/path/work/a/b",
25 | expected_pattern: "file*{zip,gz}",
26 | recursive: false,
27 | matches: %w[/path/work/a/b/file.GZ /path/work/a/b/FILE.ZIP /path/work/a/b/file123.zIp],
28 | not_matches: %w[/path/work/a/b/.profile /path/work/a/b/filter.zip /path/work/a/b/outgoing/filter.zip],
29 | case_sensitive: false
30 | },
31 | {
32 | path: "/path/work",
33 | pattern: "a/b/*",
34 | expected_path: "/path/work/a/b",
35 | expected_pattern: "*",
36 | recursive: false,
37 | matches: %w[/path/work/a/b/file.GZ /path/work/a/b/FILE.ZIP /path/work/a/b/file123.zIp],
38 | not_matches: %w[/path/work/a/b/.profile /path/work/a/b/my/filter.zip /path/work/a/b/outgoing/filter.zip],
39 | case_sensitive: false
40 | },
41 | {
42 | path: "/path/work",
43 | pattern: "a/b/file*{zip,gz}",
44 | expected_path: "/path/work/a/b",
45 | expected_pattern: "file*{zip,gz}",
46 | recursive: false,
47 | matches: %w[/path/work/a/b/file.gz /path/work/a/b/file.zip],
48 | not_matches: %w[/path/work/a/b/file.GZ /path/work/a/b/FILE.ZIP],
49 | case_sensitive: true
50 | },
51 | {path: "/path/work", pattern: "file.txt", expected_path: "/path/work/file.txt", expected_pattern: nil, recursive: false},
52 | {path: "/path/work", pattern: "*", expected_path: "/path/work", expected_pattern: "*", recursive: false}
53 | ]
54 | end
55 | # , case_sensitive: false, hidden: false
56 |
57 | describe "#recursive?" do
58 | it "identifies recursive paths correctly" do
59 | cases.each do |test_case|
60 | path = IOStreams.path(test_case[:path])
61 | matcher = IOStreams::Paths::Matcher.new(path, test_case[:pattern])
62 | assert_equal test_case[:recursive], matcher.recursive?, test_case
63 | end
64 | end
65 | end
66 |
67 | describe "#path?" do
68 | it "optimizes path correctly" do
69 | cases.each do |test_case|
70 | path = IOStreams.path(test_case[:path])
71 | matcher = IOStreams::Paths::Matcher.new(path, test_case[:pattern])
72 | assert_equal test_case[:expected_path], matcher.path.to_s, test_case
73 | end
74 | end
75 | end
76 |
77 | describe "#pattern" do
78 | it "optimizes pattern correctly" do
79 | cases.each do |test_case|
80 | path = IOStreams.path(test_case[:path])
81 | matcher = IOStreams::Paths::Matcher.new(path, test_case[:pattern])
82 | if test_case[:expected_pattern].nil?
83 | assert_nil matcher.pattern, test_case
84 | else
85 | assert_equal test_case[:expected_pattern], matcher.pattern, test_case
86 | end
87 | end
88 | end
89 | end
90 |
91 | describe "#match?" do
92 | it "matches" do
93 | cases.each do |test_case|
94 | path = IOStreams.path(test_case[:path])
95 | case_sensitive = test_case.fetch(:case_sensitive, false)
96 | matcher = IOStreams::Paths::Matcher.new(path, test_case[:pattern], case_sensitive: case_sensitive)
97 | next unless test_case[:matches]
98 |
99 | test_case[:matches].each do |file_name|
100 | assert matcher.match?(file_name), test_case.merge(file_name: file_name)
101 | end
102 | end
103 | end
104 |
105 | it "should not match" do
106 | cases.each_with_index do |test_case, index|
107 | path = IOStreams.path(test_case[:path])
108 | case_sensitive = test_case.key?(:case_sensitive) ? test_case[:case_sensitive] : false
109 | matcher = IOStreams::Paths::Matcher.new(path, test_case[:pattern], case_sensitive: case_sensitive)
110 | next unless test_case[:not_matches]
111 |
112 | test_case[:not_matches].each do |file_name|
113 | refute matcher.match?(file_name), -> { {case_sensitive: case_sensitive, test_case_number: index + 1, failed_file_name: file_name, test_case: test_case}.ai }
114 | end
115 | end
116 | end
117 | end
118 | end
119 | end
120 | end
121 |
--------------------------------------------------------------------------------
/test/paths/s3_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "../test_helper"
2 |
3 | module Paths
4 | class S3Test < Minitest::Test
5 | describe IOStreams::Paths::S3 do
6 | before do
7 | skip "Supply 'S3_BUCKET_NAME' environment variable with S3 bucket name to test S3 paths" unless ENV["S3_BUCKET_NAME"]
8 | end
9 |
10 | let :file_name do
11 | File.join(File.dirname(__FILE__), "..", "files", "text file.txt")
12 | end
13 |
14 | let :raw do
15 | File.read(file_name)
16 | end
17 |
18 | let(:root_path) { IOStreams::Paths::S3.new("s3://#{ENV['S3_BUCKET_NAME']}/iostreams_test") }
19 |
20 | let :existing_path do
21 | path = root_path.join("test.txt")
22 | path.write(raw) unless path.exist?
23 | path
24 | end
25 |
26 | let :missing_path do
27 | root_path.join("unknown.txt")
28 | end
29 |
30 | let :write_path do
31 | root_path.join("writer_test.txt").delete
32 | end
33 |
34 | describe "#delete" do
35 | it "existing file" do
36 | assert existing_path.delete.is_a?(IOStreams::Paths::S3)
37 | end
38 |
39 | it "missing file" do
40 | assert missing_path.delete.is_a?(IOStreams::Paths::S3)
41 | end
42 | end
43 |
44 | describe "#exist?" do
45 | it "existing file" do
46 | assert existing_path.exist?
47 | end
48 |
49 | it "missing file" do
50 | refute missing_path.exist?
51 | end
52 | end
53 |
54 | describe "#mkpath" do
55 | it "returns self for non-existant path" do
56 | assert existing_path.mkpath.is_a?(IOStreams::Paths::S3)
57 | end
58 |
59 | it "checks for lack of existence" do
60 | assert missing_path.mkpath.is_a?(IOStreams::Paths::S3)
61 | end
62 | end
63 |
64 | describe "#mkdir" do
65 | it "returns self for non-existant path" do
66 | assert existing_path.mkdir.is_a?(IOStreams::Paths::S3)
67 | end
68 |
69 | it "checks for lack of existence" do
70 | assert missing_path.mkdir.is_a?(IOStreams::Paths::S3)
71 | end
72 | end
73 |
74 | describe "#reader" do
75 | it "reads" do
76 | assert_equal raw, existing_path.read
77 | end
78 | end
79 |
80 | describe "#size" do
81 | it "existing file" do
82 | assert_equal raw.size, existing_path.size
83 | end
84 |
85 | it "missing file" do
86 | assert_nil missing_path.size
87 | end
88 | end
89 |
90 | describe "#writer" do
91 | it "writes" do
92 | assert_equal(raw.size, write_path.writer { |io| io.write(raw) })
93 | assert write_path.exist?
94 | assert_equal raw, write_path.read
95 | end
96 | end
97 |
98 | describe "#each_line" do
99 | it "reads line by line" do
100 | lines = []
101 | existing_path.each(:line) { |line| lines << line }
102 | assert_equal raw.lines.collect(&:chomp), lines
103 | end
104 | end
105 |
106 | describe "#each_child" do
107 | # TODO: case_sensitive: false, directories: false, hidden: false
108 | let(:abd_file_names) { %w[abd/test1.txt abd/test5.file abd/extra/file.csv] }
109 | let(:files_for_test) { abd_file_names + %w[xyz/test2.csv xyz/another.csv] }
110 |
111 | let :each_root do
112 | root_path.join("each_child_test")
113 | end
114 |
115 | let :multiple_paths do
116 | files_for_test.collect { |file_name| each_root.join(file_name) }
117 | end
118 |
119 | let :write_raw_data do
120 | multiple_paths.each { |path| path.write(raw) unless path.exist? }
121 | end
122 |
123 | it "existing file returns just the file itself" do
124 | # Glorified exists call
125 | existing_path
126 | assert_equal root_path.join("test.txt").to_s, root_path.children("test.txt").first.to_s
127 | end
128 |
129 | it "missing file does nothing" do
130 | # Glorified exists call
131 | assert_equal [], missing_path.children("readme").collect(&:to_s)
132 | end
133 |
134 | it "returns all the children" do
135 | write_raw_data
136 | assert_equal multiple_paths.collect(&:to_s).sort, each_root.children("**/*").collect(&:to_s).sort
137 | end
138 |
139 | it "returns all the children under a sub-dir" do
140 | write_raw_data
141 | expected = %w[abd/test1.txt abd/test5.file].collect { |file_name| each_root.join(file_name) }
142 | assert_equal expected.collect(&:to_s).sort, each_root.children("abd/*").collect(&:to_s).sort
143 | end
144 |
145 | it "missing path" do
146 | count = 0
147 | missing_path.each_child { |_| count += 1 }
148 | assert_equal 0, count
149 | end
150 |
151 | # Test is here since all the test artifacts have been created already in S3.
152 | describe "IOStreams.each_child" do
153 | it "returns all the children" do
154 | write_raw_data
155 | children = []
156 | IOStreams.each_child(each_root.join("**/*").to_s) { |child| children << child }
157 | assert_equal multiple_paths.collect(&:to_s).sort, children.collect(&:to_s).sort
158 | end
159 | end
160 | end
161 |
162 | describe "#move_to" do
163 | it "moves existing file" do
164 | source = root_path.join("move_test_source.txt")
165 | begin
166 | source.write("Hello World")
167 | target = source.directory.join("move_test_target.txt")
168 | response = source.move_to(target)
169 | assert_equal target, response
170 | assert target.exist?
171 | refute source.exist?
172 | assert_equal "Hello World", response.read
173 | assert_equal target.to_s, response.to_s
174 | ensure
175 | source&.delete
176 | target&.delete
177 | end
178 | end
179 |
180 | it "missing source file" do
181 | source = root_path.join("move_test_source.txt")
182 | refute source.exist?
183 | begin
184 | target = source.directory.join("move_test_target.txt")
185 | assert_raises Aws::S3::Errors::NoSuchKey do
186 | source.move_to(target)
187 | end
188 | refute target.exist?
189 | ensure
190 | source&.delete
191 | target&.delete
192 | end
193 | end
194 |
195 | it "missing target directories" do
196 | source = root_path.join("move_test_source.txt")
197 | begin
198 | source.write("Hello World")
199 | target = source.directory.join("a/b/c/move_test_target.txt")
200 | response = source.move_to(target)
201 | assert_equal target, response
202 | assert target.exist?
203 | refute source.exist?
204 | assert_equal "Hello World", response.read
205 | assert_equal target.to_s, response.to_s
206 | ensure
207 | source&.delete
208 | target&.delete
209 | end
210 | end
211 | end
212 |
213 | describe "#partial_files_visible?" do
214 | it "visible only after upload" do
215 | refute root_path.partial_files_visible?
216 | end
217 | end
218 | end
219 | end
220 | end
221 |
--------------------------------------------------------------------------------
/test/paths/sftp_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "../test_helper"
2 |
3 | module Paths
4 | class SFTPTest < Minitest::Test
5 | describe IOStreams::Paths::SFTP do
6 | before do
7 | unless ENV["SFTP_HOSTNAME"]
8 | skip "Supply environment variables to test SFTP paths: SFTP_HOSTNAME, SFTP_USERNAME, SFTP_PASSWORD, and optional SFTP_DIR, SFTP_IDENTITY_FILE"
9 | end
10 | end
11 |
12 | let(:host_name) { ENV["SFTP_HOSTNAME"] }
13 | let(:username) { ENV["SFTP_USERNAME"] }
14 | let(:password) { ENV["SFTP_PASSWORD"] }
15 | let(:ftp_dir) { ENV["SFTP_DIR"] || "iostreams_test" }
16 | let(:identity_username) { ENV["SFTP_IDENTITY_USERNAME"] || username }
17 |
18 | let(:url) { File.join("sftp://", host_name, ftp_dir) }
19 |
20 | let(:file_name) { File.join(File.dirname(__FILE__), "..", "files", "text file.txt") }
21 | let(:raw) { File.read(file_name) }
22 |
23 | let(:root_path) do
24 | if ENV["SFTP_HOST_KEY"]
25 | IOStreams::Paths::SFTP.new(url, username: username, password: password, ssh_options: {"HostKey" => ENV["SFTP_HOST_KEY"]})
26 | else
27 | IOStreams::Paths::SFTP.new(url, username: username, password: password)
28 | end
29 | end
30 |
31 | let :existing_path do
32 | path = root_path.join("test.txt")
33 | path.write(raw)
34 | path
35 | end
36 |
37 | let :missing_path do
38 | root_path.join("unknown_path", "test_file.txt")
39 | end
40 |
41 | let :missing_file_path do
42 | root_path.join("test_file.txt")
43 | end
44 |
45 | let :write_path do
46 | root_path.join("writer_test.txt")
47 | end
48 |
49 | describe "#reader" do
50 | it "reads" do
51 | assert_equal raw, existing_path.read
52 | end
53 |
54 | it "fails when the file does not exist" do
55 | assert_raises IOStreams::Errors::CommunicationsFailure do
56 | missing_file_path.read
57 | end
58 | end
59 |
60 | it "fails when the directory does not exist" do
61 | assert_raises IOStreams::Errors::CommunicationsFailure do
62 | missing_path.read
63 | end
64 | end
65 | end
66 |
67 | describe "#writer" do
68 | it "writes" do
69 | assert_equal(raw.size, write_path.writer { |io| io.write(raw) })
70 | assert_equal raw, write_path.read
71 | end
72 |
73 | it "fails when the directory does not exist" do
74 | assert_raises IOStreams::Errors::CommunicationsFailure do
75 | missing_path.write("Bad path")
76 | end
77 | end
78 |
79 | describe "use identity file instead of password" do
80 | let :root_path do
81 | IOStreams::Paths::SFTP.new(url, username: identity_username, ssh_options: {"IdentityFile" => ENV["SFTP_IDENTITY_FILE"]})
82 | end
83 |
84 | it "writes" do
85 | skip "No identity file env var set: SFTP_IDENTITY_FILE" unless ENV["SFTP_IDENTITY_FILE"]
86 | assert_equal(raw.size, write_path.writer { |io| io.write(raw) })
87 | assert_equal raw, write_path.read
88 | end
89 | end
90 |
91 | describe "use identity key instead of password" do
92 | let :root_path do
93 | key = File.open(ENV["SFTP_IDENTITY_FILE"], "rb", &:read)
94 | IOStreams::Paths::SFTP.new(url, username: identity_username, ssh_options: {"IdentityKey" => key})
95 | end
96 |
97 | it "writes" do
98 | skip "No identity file env var set: SFTP_IDENTITY_FILE" unless ENV["SFTP_IDENTITY_FILE"]
99 | assert_equal(raw.size, write_path.writer { |io| io.write(raw) })
100 | assert_equal raw, write_path.read
101 | end
102 | end
103 | end
104 | end
105 | end
106 | end
107 |
--------------------------------------------------------------------------------
/test/pgp_reader_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 |
3 | class PgpReaderTest < Minitest::Test
4 | describe IOStreams::Pgp::Reader do
5 | let :temp_file do
6 | Tempfile.new("iostreams")
7 | end
8 |
9 | let :decrypted do
10 | file_name = File.join(File.dirname(__FILE__), "files", "text.txt")
11 | File.read(file_name)
12 | end
13 |
14 | after do
15 | temp_file.delete
16 | end
17 |
18 | describe ".file" do
19 | it "reads encrypted file" do
20 | IOStreams::Pgp::Writer.file(temp_file.path, recipient: "receiver@example.org") do |io|
21 | io.write(decrypted)
22 | end
23 |
24 | result = IOStreams::Pgp::Reader.file(temp_file.path, passphrase: "receiver_passphrase", &:read)
25 | assert_equal decrypted, result
26 | end
27 |
28 | it "fails with bad passphrase" do
29 | assert_raises IOStreams::Pgp::Failure do
30 | IOStreams::Pgp::Reader.file(temp_file.path, passphrase: "BAD", &:read)
31 | end
32 | end
33 |
34 | it "streams input" do
35 | io_string = StringIO.new("".b)
36 | IOStreams::Pgp::Writer.stream(io_string, recipient: "receiver@example.org", signer: "sender@example.org", signer_passphrase: "sender_passphrase") do |io|
37 | io.write(decrypted)
38 | end
39 |
40 | io = StringIO.new(io_string.string)
41 | result = IOStreams::Pgp::Reader.stream(io, passphrase: "receiver_passphrase", &:read)
42 | assert_equal decrypted, result
43 | end
44 | end
45 | end
46 | end
47 |
--------------------------------------------------------------------------------
/test/pgp_writer_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 |
3 | class PgpWriterTest < Minitest::Test
4 | describe IOStreams::Pgp::Writer do
5 | let :temp_file do
6 | Tempfile.new("iostreams")
7 | end
8 |
9 | let :file_name do
10 | temp_file.path
11 | end
12 |
13 | let :decrypted do
14 | file_name = File.join(File.dirname(__FILE__), "files", "text.txt")
15 | File.read(file_name)
16 | end
17 |
18 | after do
19 | temp_file.delete
20 | end
21 |
22 | describe ".file" do
23 | it "writes encrypted text file" do
24 | result =
25 | IOStreams::Pgp::Writer.file(file_name, recipient: "receiver@example.org") do |io|
26 | io.write(decrypted)
27 | 53534
28 | end
29 | assert_equal 53534, result
30 |
31 | result = IOStreams::Pgp::Reader.file(file_name, passphrase: "receiver_passphrase", &:read)
32 | assert_equal decrypted, result
33 | end
34 |
35 | it "writes encrypted binary file" do
36 | binary_file_name = File.join(File.dirname(__FILE__), "files", "spreadsheet.xlsx")
37 | binary_data = File.open(binary_file_name, "rb", &:read)
38 |
39 | File.open(binary_file_name, "rb") do |input|
40 | result =
41 | IOStreams::Pgp::Writer.file(file_name, recipient: "receiver@example.org") do |output|
42 | IO.copy_stream(input, output)
43 | 53534
44 | end
45 | assert_equal 53534, result
46 | end
47 |
48 | result = IOStreams::Pgp::Reader.file(file_name, passphrase: "receiver_passphrase", &:read)
49 | assert_equal binary_data, result
50 | end
51 |
52 | it "writes and signs encrypted file" do
53 | IOStreams::Pgp::Writer.file(file_name, recipient: "receiver@example.org", signer: "sender@example.org", signer_passphrase: "sender_passphrase") do |io|
54 | io.write(decrypted)
55 | end
56 |
57 | result = IOStreams::Pgp::Reader.file(file_name, passphrase: "receiver_passphrase", &:read)
58 | assert_equal decrypted, result
59 | end
60 |
61 | it "supports multiple recipients" do
62 | IOStreams::Pgp::Writer.file(file_name, recipient: %w[receiver@example.org receiver2@example.org], signer: "sender@example.org", signer_passphrase: "sender_passphrase") do |io|
63 | io.write(decrypted)
64 | end
65 |
66 | result = IOStreams::Pgp::Reader.file(file_name, passphrase: "receiver_passphrase", &:read)
67 | assert_equal decrypted, result
68 |
69 | result = IOStreams::Pgp::Reader.file(file_name, passphrase: "receiver2_passphrase", &:read)
70 | assert_equal decrypted, result
71 | end
72 |
73 | it "encrypts for recipient and audit recipient" do
74 | IOStreams::Pgp::Writer.stub(:audit_recipient, "receiver2@example.org") do
75 | IOStreams::Pgp::Writer.file(file_name, recipient: "receiver@example.org", signer: "sender@example.org", signer_passphrase: "sender_passphrase") do |io|
76 | io.write(decrypted)
77 | end
78 | end
79 |
80 | result = IOStreams::Pgp::Reader.file(file_name, passphrase: "receiver_passphrase", &:read)
81 | assert_equal decrypted, result
82 |
83 | result = IOStreams::Pgp::Reader.file(file_name, passphrase: "receiver2_passphrase", &:read)
84 | assert_equal decrypted, result
85 | end
86 |
87 | it "fails with bad signer passphrase" do
88 | skip "GnuPG v2.1 and above passes when it should not" if IOStreams::Pgp.pgp_version.to_f >= 2.1
89 | assert_raises IOStreams::Pgp::Failure do
90 | IOStreams::Pgp::Writer.file(file_name, recipient: "receiver@example.org", signer: "sender@example.org", signer_passphrase: "BAD") do |io|
91 | io.write(decrypted)
92 | end
93 | end
94 | end
95 |
96 | it "fails with bad recipient" do
97 | assert_raises IOStreams::Pgp::Failure do
98 | IOStreams::Pgp::Writer.file(file_name, recipient: "BAD@example.org", signer: "sender@example.org", signer_passphrase: "sender_passphrase") do |io|
99 | io.write(decrypted)
100 | # Allow process to terminate
101 | sleep 1
102 | io.write(decrypted)
103 | end
104 | end
105 | end
106 |
107 | it "fails with bad signer" do
108 | assert_raises IOStreams::Pgp::Failure do
109 | IOStreams::Pgp::Writer.file(file_name, recipient: "receiver@example.org", signer: "BAD@example.org", signer_passphrase: "sender_passphrase") do |io|
110 | io.write(decrypted)
111 | end
112 | end
113 | end
114 |
115 | it "writes to a stream" do
116 | io_string = StringIO.new("".b)
117 | result =
118 | IOStreams::Pgp::Writer.stream(io_string, recipient: "receiver@example.org", signer: "sender@example.org", signer_passphrase: "sender_passphrase") do |io|
119 | io.write(decrypted)
120 | 53534
121 | end
122 | assert_equal 53534, result
123 |
124 | io = StringIO.new(io_string.string)
125 | result = IOStreams::Pgp::Reader.stream(io, passphrase: "receiver_passphrase", &:read)
126 | assert_equal decrypted, result
127 | end
128 | end
129 | end
130 | end
131 |
--------------------------------------------------------------------------------
/test/record_reader_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 |
3 | class RecordReaderTest < Minitest::Test
4 | describe IOStreams::Record::Reader do
5 | let :file_name do
6 | File.join(File.dirname(__FILE__), "files", "test.csv")
7 | end
8 |
9 | let :json_file_name do
10 | File.join(File.dirname(__FILE__), "files", "test.json")
11 | end
12 |
13 | let :csv_rows do
14 | CSV.read(file_name)
15 | end
16 |
17 | let :expected do
18 | rows = csv_rows.dup
19 | header = rows.shift
20 | rows.collect { |row| header.zip(row).to_h }
21 | end
22 |
23 | describe "#each" do
24 | it "csv file" do
25 | records = []
26 | IOStreams::Record::Reader.file(file_name, cleanse_header: false) do |io|
27 | io.each { |row| records << row }
28 | end
29 | assert_equal expected, records
30 | end
31 |
32 | it "json file" do
33 | records = []
34 | IOStreams::Record::Reader.file(json_file_name, cleanse_header: false, format: :json) do |input|
35 | input.each { |row| records << row }
36 | end
37 | assert_equal expected, records
38 | end
39 |
40 | it "stream" do
41 | rows = []
42 | IOStreams::Line::Reader.file(file_name) do |file|
43 | IOStreams::Record::Reader.stream(file, cleanse_header: false) do |io|
44 | io.each { |row| rows << row }
45 | end
46 | end
47 | assert_equal expected, rows
48 | end
49 | end
50 |
51 | describe "#collect" do
52 | it "json file" do
53 | records = IOStreams::Record::Reader.file(json_file_name, format: :json) do |input|
54 | input.collect { |record| record["state"] }
55 | end
56 | assert_equal expected.collect { |record| record["state"] }, records
57 | end
58 | end
59 | end
60 | end
61 |
--------------------------------------------------------------------------------
/test/record_writer_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 | require "csv"
3 |
4 | class RecordWriterTest < Minitest::Test
5 | describe IOStreams::Record::Writer do
6 | let :csv_file_name do
7 | File.join(File.dirname(__FILE__), "files", "test.csv")
8 | end
9 |
10 | let :json_file_name do
11 | File.join(File.dirname(__FILE__), "files", "test.json")
12 | end
13 |
14 | let :raw_csv_data do
15 | File.read(csv_file_name)
16 | end
17 |
18 | let :raw_json_data do
19 | File.read(json_file_name)
20 | end
21 |
22 | let :csv_rows do
23 | CSV.read(csv_file_name)
24 | end
25 |
26 | let :inputs do
27 | rows = csv_rows.dup
28 | header = rows.shift
29 | rows.collect { |row| header.zip(row).to_h }
30 | end
31 |
32 | let :temp_file do
33 | Tempfile.new("iostreams")
34 | end
35 |
36 | let :file_name do
37 | temp_file.path
38 | end
39 |
40 | after do
41 | temp_file.delete
42 | end
43 |
44 | describe "#<<" do
45 | it "file" do
46 | result =
47 | IOStreams::Record::Writer.file(file_name) do |io|
48 | inputs.each { |hash| io << hash }
49 | 53534
50 | end
51 | assert_equal 53534, result
52 | result = File.read(file_name)
53 | assert_equal raw_csv_data, result
54 | end
55 |
56 | it "json file" do
57 | result =
58 | IOStreams::Record::Writer.file(file_name, file_name: "abc.json") do |io|
59 | inputs.each { |hash| io << hash }
60 | 53534
61 | end
62 | assert_equal 53534, result
63 |
64 | result = File.read(file_name)
65 | assert_equal raw_json_data, result
66 | end
67 |
68 | it "stream" do
69 | io_string = StringIO.new
70 | result =
71 | IOStreams::Line::Writer.stream(io_string) do |io|
72 | IOStreams::Record::Writer.stream(io) do |stream|
73 | inputs.each { |row| stream << row }
74 | 53534
75 | end
76 | end
77 | assert_equal 53534, result
78 | assert_equal raw_csv_data, io_string.string
79 | end
80 | end
81 | end
82 | end
83 |
--------------------------------------------------------------------------------
/test/row_reader_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 |
3 | class RowReaderTest < Minitest::Test
4 | describe IOStreams::Row::Reader do
5 | let :file_name do
6 | File.join(File.dirname(__FILE__), "files", "test.csv")
7 | end
8 |
9 | let :expected do
10 | CSV.read(file_name)
11 | end
12 |
13 | describe "#each" do
14 | it "file" do
15 | rows = []
16 | count = IOStreams::Row::Reader.file(file_name) do |io|
17 | io.each { |row| rows << row }
18 | end
19 | assert_equal expected, rows
20 | assert_equal expected.size, count
21 | end
22 |
23 | it "stream" do
24 | rows = []
25 | count = IOStreams::Line::Reader.file(file_name) do |file|
26 | IOStreams::Row::Reader.stream(file) do |io|
27 | io.each { |row| rows << row }
28 | end
29 | end
30 | assert_equal expected, rows
31 | assert_equal expected.size, count
32 | end
33 | end
34 | end
35 | end
36 |
--------------------------------------------------------------------------------
/test/row_writer_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 | require "csv"
3 |
4 | class RowWriterTest < Minitest::Test
5 | describe IOStreams::Row::Writer do
6 | let :csv_file_name do
7 | File.join(File.dirname(__FILE__), "files", "test.csv")
8 | end
9 |
10 | let :raw_csv_data do
11 | File.read(csv_file_name)
12 | end
13 |
14 | let :csv_rows do
15 | CSV.read(csv_file_name)
16 | end
17 |
18 | let :temp_file do
19 | Tempfile.new("iostreams")
20 | end
21 |
22 | let :file_name do
23 | temp_file.path
24 | end
25 |
26 | after do
27 | temp_file.delete
28 | end
29 |
30 | describe ".stream" do
31 | it "file" do
32 | result =
33 | IOStreams::Row::Writer.file(file_name) do |io|
34 | csv_rows.each { |array| io << array }
35 | 53534
36 | end
37 | assert_equal 53534, result
38 | result = ::File.read(file_name)
39 | assert_equal raw_csv_data, result
40 | end
41 |
42 | it "streams" do
43 | io_string = StringIO.new
44 | result =
45 | IOStreams::Line::Writer.stream(io_string) do |io|
46 | IOStreams::Row::Writer.stream(io) do |stream|
47 | csv_rows.each { |array| stream << array }
48 | 53534
49 | end
50 | end
51 | assert_equal 53534, result
52 | assert_equal raw_csv_data, io_string.string
53 | end
54 | end
55 | end
56 | end
57 |
--------------------------------------------------------------------------------
/test/test_helper.rb:
--------------------------------------------------------------------------------
1 | $LOAD_PATH.unshift File.dirname(__FILE__) + "/../lib"
2 |
3 | require "yaml"
4 | require "minitest/autorun"
5 | require "iostreams"
6 | require "amazing_print"
7 | require "symmetric-encryption"
8 |
9 | # Since PGP libraries use UTC for Dates
10 | ENV["TZ"] = "UTC"
11 |
12 | # Test cipher
13 | SymmetricEncryption.cipher = SymmetricEncryption::Cipher.new(
14 | cipher_name: "aes-128-cbc",
15 | key: "1234567890ABCDEF1234567890ABCDEF",
16 | iv: "1234567890ABCDEF",
17 | encoding: :base64strict
18 | )
19 |
20 | # IOStreams::Pgp.logger = Logger.new($stdout)
21 | # IOStreams::Pgp.executable = 'gpg1'
22 |
23 | # Test PGP Keys
24 | unless IOStreams::Pgp.key?(email: "sender@example.org")
25 | puts "Generating test PGP key: sender@example.org"
26 | IOStreams::Pgp.generate_key(name: "Sender", email: "sender@example.org", passphrase: "sender_passphrase", key_length: 2048)
27 | end
28 | unless IOStreams::Pgp.key?(email: "receiver@example.org")
29 | puts "Generating test PGP key: receiver@example.org"
30 | IOStreams::Pgp.generate_key(name: "Receiver", email: "receiver@example.org", passphrase: "receiver_passphrase", key_length: 2048)
31 | end
32 | unless IOStreams::Pgp.key?(email: "receiver2@example.org")
33 | puts "Generating test PGP key: receiver2@example.org"
34 | IOStreams::Pgp.generate_key(name: "Receiver2", email: "receiver2@example.org", passphrase: "receiver2_passphrase", key_length: 2048)
35 | end
36 |
37 | # Test paths
38 | root = File.expand_path(File.join(__dir__, "../tmp"))
39 | IOStreams.add_root(:default, File.join(root, "default"))
40 | IOStreams.add_root(:downloads, File.join(root, "downloads"))
41 |
--------------------------------------------------------------------------------
/test/utils_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 |
3 | class UtilsTest < Minitest::Test
4 | describe IOStreams::Utils do
5 | describe ".temp_file_name" do
6 | it "returns value from block" do
7 | result = IOStreams::Utils.temp_file_name("base", ".ext") { |_name| 257 }
8 | assert_equal 257, result
9 | end
10 |
11 | it "supplies new temp file_name" do
12 | file_name = nil
13 | file_name2 = nil
14 | IOStreams::Utils.temp_file_name("base", ".ext") { |name| file_name = name }
15 | IOStreams::Utils.temp_file_name("base", ".ext") { |name| file_name2 = name }
16 | refute_equal file_name, file_name2
17 | end
18 | end
19 | end
20 | end
21 |
--------------------------------------------------------------------------------
/test/xlsx_reader_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 | require "csv"
3 |
4 | class XlsxReaderTest
5 | describe IOStreams::Xlsx::Reader do
6 | let :file_name do
7 | File.join(File.dirname(__FILE__), "files", "spreadsheet.xlsx")
8 | end
9 |
10 | let :xlsx_contents do
11 | [
12 | ["first column", "second column", "third column"],
13 | ["data 1", "data 2", "more data"]
14 | ]
15 | end
16 |
17 | describe ".file" do
18 | describe "with a file path" do
19 | it "returns the contents of the file" do
20 | csv = IOStreams::Xlsx::Reader.file(file_name, &:read)
21 | assert_equal xlsx_contents, CSV.parse(csv)
22 | end
23 | end
24 |
25 | describe "with a file stream" do
26 | it "returns the contents of the file" do
27 | csv = ""
28 | File.open(file_name, "rb") do |file|
29 | csv = IOStreams::Xlsx::Reader.stream(file, &:read)
30 | end
31 |
32 | assert_equal xlsx_contents, CSV.parse(csv)
33 | end
34 | end
35 | end
36 | end
37 | end
38 |
--------------------------------------------------------------------------------
/test/zip_reader_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 | require_relative "minimal_file_reader"
3 | require "zip"
4 |
5 | class ZipReaderTest < Minitest::Test
6 | describe IOStreams::Zip::Reader do
7 | let :file_name do
8 | File.join(File.dirname(__FILE__), "files", "text.zip")
9 | end
10 |
11 | let :multiple_zip_file_name do
12 | File.join(File.dirname(__FILE__), "files", "multiple_files.zip")
13 | end
14 |
15 | let :contents_test_txt do
16 | File.read(File.join(File.dirname(__FILE__), "files", "text.txt"))
17 | end
18 |
19 | let :contents_test_json do
20 | File.read(File.join(File.dirname(__FILE__), "files", "test.json"))
21 | end
22 |
23 | let :decompressed do
24 | Zip::File.open(file_name) { |zip_file| zip_file.first.get_input_stream.read }
25 | end
26 |
27 | describe ".file" do
28 | it "reads the first file" do
29 | result = IOStreams::Zip::Reader.file(file_name, &:read)
30 | assert_equal decompressed, result
31 | end
32 |
33 | it "reads entry within zip file" do
34 | result = IOStreams::Zip::Reader.file(multiple_zip_file_name, entry_file_name: "text.txt", &:read)
35 | assert_equal contents_test_txt, result
36 | end
37 |
38 | it "reads another entry within zip file" do
39 | result = IOStreams::Zip::Reader.file(multiple_zip_file_name, entry_file_name: "test.json", &:read)
40 | assert_equal contents_test_json, result
41 | end
42 |
43 | # it 'reads from a stream' do
44 | # result = MinimalFileReader.open(file_name) do |file|
45 | # IOStreams::Zip::Reader.stream(file) do |io|
46 | # io.read
47 | # end
48 | # end
49 | # assert_equal decompressed, result
50 | # end
51 | end
52 | end
53 | end
54 |
--------------------------------------------------------------------------------
/test/zip_writer_test.rb:
--------------------------------------------------------------------------------
1 | require_relative "test_helper"
2 | require "zip"
3 |
4 | class ZipWriterTest < Minitest::Test
5 | describe IOStreams::Zip::Writer do
6 | let :temp_file do
7 | Tempfile.new("iostreams")
8 | end
9 |
10 | let :file_name do
11 | temp_file.path
12 | end
13 |
14 | let :decompressed do
15 | ::File.read(File.join(File.dirname(__FILE__), "files", "text.txt"))
16 | end
17 |
18 | after do
19 | temp_file.delete
20 | end
21 |
22 | describe ".file" do
23 | it "file" do
24 | result =
25 | IOStreams::Zip::Writer.file(file_name, entry_file_name: "text.txt") do |io|
26 | io.write(decompressed)
27 | 53534
28 | end
29 | assert_equal 53534, result
30 | result = IOStreams::Zip::Reader.file(file_name, &:read)
31 | assert_equal decompressed, result
32 | end
33 |
34 | it "stream" do
35 | io_string = StringIO.new("".b)
36 | result =
37 | IOStreams::Zip::Writer.stream(io_string) do |io|
38 | io.write(decompressed)
39 | 53534
40 | end
41 | assert_equal 53534, result
42 | io = StringIO.new(io_string.string)
43 | result = IOStreams::Zip::Reader.stream(io, &:read)
44 | assert_equal decompressed, result
45 | end
46 | end
47 | end
48 | end
49 |
--------------------------------------------------------------------------------
23 |
24 |