├── .gitignore ├── .mailmap ├── .travis.yml ├── CHANGES ├── CONTRIBUTING.md ├── LICENSE ├── Makefile.PL ├── README.mkdn ├── bin ├── yfrom ├── ygrok ├── yq ├── ysql ├── yto └── yts ├── cpanfile ├── dist.ini ├── eg └── transform.pl ├── lib └── ETL │ ├── Yertl.pm │ └── Yertl │ ├── Adapter │ ├── graphite.pm │ └── influxdb.pm │ ├── Command │ ├── ygrok.pm │ ├── ysql.pm │ └── yts.pm │ ├── Format.pm │ ├── Format │ ├── csv.pm │ ├── json.pm │ └── yaml.pm │ ├── FormatStream.pm │ ├── Help │ ├── Cookbook.pod │ └── ysql.pod │ ├── InputSeries.pm │ ├── LineStream.pm │ ├── Transform.pm │ ├── Transform │ └── Yq.pm │ └── Util.pm ├── site.yml ├── site ├── blog │ ├── 2015 │ │ ├── 11 │ │ │ ├── 24 │ │ │ │ └── release-v0-026 │ │ │ │ │ └── index.markdown │ │ │ └── 28 │ │ │ │ └── release-v0-027 │ │ │ │ └── index.markdown │ │ ├── 01 │ │ │ ├── 11 │ │ │ │ ├── initial-release.markdown │ │ │ │ └── release-v0.017.markdown │ │ │ └── 21 │ │ │ │ └── release-v0.019.markdown │ │ ├── 02 │ │ │ ├── 26 │ │ │ │ └── release-v0.023 │ │ │ │ │ └── index.markdown │ │ │ └── 01 │ │ │ │ ├── release-v0.020.markdown │ │ │ │ ├── release-v0.021.markdown │ │ │ │ └── release-v0.022.markdown │ │ ├── 05 │ │ │ └── 31 │ │ │ │ └── release-v0-024 │ │ │ │ └── index.markdown │ │ └── 08 │ │ │ └── 23 │ │ │ └── release-v0-025 │ │ │ └── index.markdown │ ├── 2016 │ │ └── 10 │ │ │ └── 09 │ │ │ └── release-v0-028 │ │ │ └── index.markdown │ └── 2017 │ │ ├── 10 │ │ ├── 18 │ │ │ └── release-v0-035 │ │ │ │ └── index.markdown │ │ ├── 25 │ │ │ └── release-v0-036 │ │ │ │ └── index.markdown │ │ └── 27 │ │ │ └── release-v0-037 │ │ │ └── index.markdown │ │ └── 08 │ │ ├── 16 │ │ └── release-v0-029 │ │ │ └── index.markdown │ │ └── 25 │ │ └── release-v0-032 │ │ └── index.markdown ├── index.markdown ├── release-blog.pl └── theme │ ├── blog │ ├── index.atom.ep │ ├── index.html.ep │ ├── index.rss.ep │ └── post.html.ep │ ├── css │ ├── normalize.css │ ├── skeleton.css │ └── statocles-default.css │ ├── layout │ ├── default.html.ep │ └── full-width.html.ep │ ├── perldoc │ ├── pod.html.ep │ └── source.html.ep │ ├── plugin │ └── highlight │ │ ├── default.css │ │ ├── solarized-dark.css │ │ └── solarized-light.css │ └── site │ ├── footer.html.ep │ ├── head_after.html.ep │ ├── header_after.html.ep │ ├── navbar_extra.html.ep │ ├── robots.txt.ep │ ├── sidebar_before.html.ep │ ├── sitemap.xml.ep │ └── style.html ├── t ├── adapter │ ├── graphite.t │ └── influxdb.t ├── bin │ ├── yfrom.t │ ├── ygrok.t │ ├── yq.t │ ├── ysql.t │ ├── yto.t │ └── yts.t ├── command │ ├── ygrok │ │ ├── edit_patterns.t │ │ └── pattern │ │ │ ├── http_combined.t │ │ │ ├── http_common.t │ │ │ ├── linux_proc.t │ │ │ ├── ls-l.t │ │ │ ├── ps-u.t │ │ │ ├── ps-x.t │ │ │ ├── ps.t │ │ │ └── syslog.t │ └── yq │ │ ├── combinator.t │ │ ├── comparison.t │ │ ├── conditional.t │ │ ├── filter.t │ │ ├── functions.t │ │ ├── regression.t │ │ └── values.t ├── format.t ├── format │ ├── csv.t │ ├── json.t │ └── yaml.t ├── format_stream.t ├── helpers.t ├── input_series.t ├── integration │ └── parse_apache_log.t ├── lib │ ├── ETL │ │ └── Yertl │ │ │ └── Adapter │ │ │ └── test.pm │ ├── Local │ │ └── AddHello.pm │ └── LowVersion.pm ├── share │ ├── command │ │ ├── ymask │ │ │ └── in.yaml │ │ ├── ysql │ │ │ ├── .yertl │ │ │ │ └── ysql.yml │ │ │ ├── deep.yml │ │ │ └── write.yml │ │ └── yts │ │ │ ├── write-short.yml │ │ │ └── write.yml │ ├── csv │ │ ├── test-colon.csv │ │ └── test.csv │ ├── json │ │ └── test.json │ ├── lines │ │ ├── custom_category.txt │ │ ├── custom_category_edit.txt │ │ ├── custom_override.txt │ │ ├── custom_override_edit.txt │ │ ├── custom_plain.txt │ │ ├── custom_plain_edit.txt │ │ ├── http_combined_log.txt │ │ ├── http_common_log.txt │ │ ├── irc.txt │ │ ├── linux │ │ │ ├── proc_loadavg.txt │ │ │ └── proc_uptime.txt │ │ ├── ls-l.txt │ │ ├── macosx │ │ │ ├── ps-u.txt │ │ │ ├── ps-x.txt │ │ │ └── ps.txt │ │ ├── openbsd │ │ │ ├── ps-u.txt │ │ │ ├── ps-x.txt │ │ │ └── ps.txt │ │ ├── rhel5 │ │ │ ├── ps-u.txt │ │ │ ├── ps-x.txt │ │ │ └── ps.txt │ │ └── syslog.txt │ └── yaml │ │ ├── foo.yaml │ │ ├── group_by.yaml │ │ ├── noseperator.yaml │ │ └── test.yaml └── transform.t └── weaver.ini /.gitignore: -------------------------------------------------------------------------------- 1 | .build 2 | *.swp 3 | *.tar.gz 4 | /ETL-Yertl-* 5 | site/_build 6 | .statocles 7 | synthesize-master 8 | Vagrantfile 9 | .vagrant 10 | MYMETA.json 11 | MYMETA.yml 12 | blib 13 | pm_to_blib 14 | Makefile 15 | -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- 1 | Doug Bell 2 | Doug Bell 3 | Doug Bell 4 | Doug Bell 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: perl 2 | perl: 3 | - "5.10.1" # 3-part version to ensure core-only Perl 4 | - "5.12" 5 | - "5.14" 6 | - "5.16" 7 | - "5.18" 8 | - "5.20" 9 | - "5.22" 10 | - "5.24.0" # 3-part version to ensure core-only Perl 11 | - "blead" 12 | 13 | matrix: 14 | include: 15 | - perl: "5.20" 16 | env: COVERAGE=1 17 | allow_failures: 18 | - perl: "blead" 19 | 20 | before_install: 21 | - git clone git://github.com/travis-perl/helpers ~/travis-perl-helpers 22 | - source ~/travis-perl-helpers/init 23 | - build-perl 24 | - perl -V 25 | - build-dist 26 | - cd $BUILD_DIR 27 | 28 | install: 29 | - cpan-install --deps 30 | - cpan-install --coverage 31 | 32 | before_script: 33 | - coverage-setup 34 | 35 | script: 36 | - perl Makefile.PL 37 | - make 38 | - prove -bl -s -j$(test-jobs) $(test-files) 39 | 40 | after_success: 41 | - coverage-report 42 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # CONTRIBUTING 2 | 3 | If you're not sure about anything, feel free to open an issue and ask! 4 | 5 | ## What to Contribute 6 | 7 | ### Fixes 8 | 9 | For fixes, simply fork and send a pull request. Be sure to add yourself 10 | to the dist.ini as an author! 11 | 12 | Fixes to anything, documentation, code, tests, are equally welcome, 13 | appreciated, and addressed! 14 | 15 | ### Features 16 | 17 | All contributions are welcome if they fit the scope of this project. If 18 | you're not sure if your feature fits, open an issue and ask. If it doesn't 19 | fit, we will try to find a way to enable you to add your feature in a 20 | related project (if it means changes in this project). 21 | 22 | ## Before you Contribute 23 | 24 | ### Copyright and License 25 | 26 | All contributions are copyright their respective owners, so make sure you 27 | agree with the project license (found in the LICENSE file) before 28 | contributing. 29 | 30 | Make sure to add yourself as an author to either the AUTHORS file or 31 | the dist.ini file so you get your proper copyright attribution. 32 | 33 | ### Formatting and Syntax 34 | 35 | I don't worry too much about this, yet. I'm sure I'll fill this section 36 | in a bit more. For now, try to match as best you can the code that 37 | already exists in this project. 38 | 39 | ## Developing on this Project 40 | 41 | This project uses Dist::Zilla for its releases, but you aren't required 42 | to use it for contributing. 43 | 44 | ### Using Build.PL 45 | 46 | This is the easiest way that requires the fewest dependencies. 47 | 48 | Install the project's dependencies and run the tests by doing: 49 | 50 | ``` 51 | perl Build.PL 52 | ./Build installdeps 53 | ./Build test 54 | ``` 55 | 56 | ### Using Makefile.PL 57 | 58 | This is the older standard way. If you can install CPAN modules, you can 59 | probably do this. It requires `make` and maybe a C compiler. 60 | 61 | Run the tests by doing: 62 | 63 | ``` 64 | perl Makefile.PL 65 | make test 66 | ``` 67 | 68 | Install the module's dependencies by doing: 69 | 70 | ``` 71 | cpanm . 72 | ``` 73 | 74 | ### Using Dist::Zilla 75 | 76 | Once you have installed Dist::Zilla, you can get this distributions's 77 | dependencies by doing: 78 | 79 | ``` 80 | dzil listdeps --author --missing | cpanm 81 | ``` 82 | 83 | Once all that is done, testing is as easy as: 84 | 85 | ``` 86 | dzil test 87 | ``` 88 | -------------------------------------------------------------------------------- /Makefile.PL: -------------------------------------------------------------------------------- 1 | # This file was automatically generated by Dist::Zilla::Plugin::MakeMaker v6.012. 2 | use strict; 3 | use warnings; 4 | 5 | use 5.010; 6 | 7 | use ExtUtils::MakeMaker; 8 | 9 | my %WriteMakefileArgs = ( 10 | "ABSTRACT" => "ETL with a Shell", 11 | "AUTHOR" => "Doug Bell ", 12 | "CONFIGURE_REQUIRES" => { 13 | "ExtUtils::MakeMaker" => 0 14 | }, 15 | "DISTNAME" => "ETL-Yertl", 16 | "EXE_FILES" => [ 17 | "bin/yfrom", 18 | "bin/ygrok", 19 | "bin/yq", 20 | "bin/ysql", 21 | "bin/yto", 22 | "bin/yts" 23 | ], 24 | "LICENSE" => "perl", 25 | "MIN_PERL_VERSION" => "5.010", 26 | "NAME" => "ETL::Yertl", 27 | "PREREQ_PM" => { 28 | "File::HomeDir" => 0, 29 | "File::Spec" => 0, 30 | "Getopt::Long" => "2.36", 31 | "Hash::Merge::Simple" => 0, 32 | "IO::Async" => "0.77", 33 | "IO::Interactive" => 0, 34 | "Import::Base" => "0.010", 35 | "JSON::MaybeXS" => 0, 36 | "List::Util" => 0, 37 | "Module::Runtime" => 0, 38 | "Net::Async::HTTP" => 0, 39 | "Path::Tiny" => "0.072", 40 | "Pod::Usage::Return" => 0, 41 | "Regexp::Common" => "2013031301", 42 | "SQL::Abstract" => 0, 43 | "Time::Local" => 0, 44 | "Time::Piece" => 0, 45 | "URI" => 0, 46 | "YAML::Tiny" => 0, 47 | "boolean" => 0 48 | }, 49 | "TEST_REQUIRES" => { 50 | "Capture::Tiny" => 0, 51 | "Dir::Self" => 0, 52 | "ExtUtils::MakeMaker" => 0, 53 | "File::Spec" => 0, 54 | "Future" => 0, 55 | "IO::Handle" => 0, 56 | "IPC::Open3" => 0, 57 | "JSON::PP" => 0, 58 | "Mock::MonkeyPatch" => 0, 59 | "Test::Deep" => 0, 60 | "Test::Differences" => 0, 61 | "Test::Exception" => 0, 62 | "Test::Lib" => 0, 63 | "Test::More" => "1.001005" 64 | }, 65 | "VERSION" => "0.045", 66 | "test" => { 67 | "TESTS" => "t/*.t t/adapter/*.t t/bin/*.t t/command/ygrok/*.t t/command/ygrok/pattern/*.t t/command/yq/*.t t/format/*.t t/integration/*.t" 68 | } 69 | ); 70 | 71 | 72 | my %FallbackPrereqs = ( 73 | "Capture::Tiny" => 0, 74 | "Dir::Self" => 0, 75 | "ExtUtils::MakeMaker" => 0, 76 | "File::HomeDir" => 0, 77 | "File::Spec" => 0, 78 | "Future" => 0, 79 | "Getopt::Long" => "2.36", 80 | "Hash::Merge::Simple" => 0, 81 | "IO::Async" => "0.77", 82 | "IO::Handle" => 0, 83 | "IO::Interactive" => 0, 84 | "IPC::Open3" => 0, 85 | "Import::Base" => "0.010", 86 | "JSON::MaybeXS" => 0, 87 | "JSON::PP" => 0, 88 | "List::Util" => 0, 89 | "Mock::MonkeyPatch" => 0, 90 | "Module::Runtime" => 0, 91 | "Net::Async::HTTP" => 0, 92 | "Path::Tiny" => "0.072", 93 | "Pod::Usage::Return" => 0, 94 | "Regexp::Common" => "2013031301", 95 | "SQL::Abstract" => 0, 96 | "Test::Deep" => 0, 97 | "Test::Differences" => 0, 98 | "Test::Exception" => 0, 99 | "Test::Lib" => 0, 100 | "Test::More" => "1.001005", 101 | "Time::Local" => 0, 102 | "Time::Piece" => 0, 103 | "URI" => 0, 104 | "YAML::Tiny" => 0, 105 | "boolean" => 0 106 | ); 107 | 108 | 109 | unless ( eval { ExtUtils::MakeMaker->VERSION(6.63_03) } ) { 110 | delete $WriteMakefileArgs{TEST_REQUIRES}; 111 | delete $WriteMakefileArgs{BUILD_REQUIRES}; 112 | $WriteMakefileArgs{PREREQ_PM} = \%FallbackPrereqs; 113 | } 114 | 115 | delete $WriteMakefileArgs{CONFIGURE_REQUIRES} 116 | unless eval { ExtUtils::MakeMaker->VERSION(6.52) }; 117 | 118 | WriteMakefile(%WriteMakefileArgs); 119 | -------------------------------------------------------------------------------- /bin/yfrom: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | package yfrom; 3 | our $VERSION = '0.045'; 4 | # ABSTRACT: Build YAML from another format (like JSON or CSV) 5 | 6 | use ETL::Yertl; 7 | use Pod::Usage::Return qw( pod2usage ); 8 | use Getopt::Long qw( GetOptionsFromArray ); 9 | use ETL::Yertl::Format; 10 | use ETL::Yertl::FormatStream; 11 | use ETL::Yertl::InputSeries; 12 | use IO::Async::Loop; 13 | 14 | $|++; # no buffering 15 | 16 | sub main { 17 | my ( $class, @argv ) = @_; 18 | my %opt; 19 | GetOptionsFromArray( \@argv, \%opt, 20 | 'help|h', 21 | 'version', 22 | 'delimiter|d=s', 23 | ); 24 | return pod2usage(0) if $opt{help}; 25 | if ( $opt{version} ) { 26 | print "yfrom version $yfrom::VERSION (Perl $^V)\n"; 27 | return 0; 28 | } 29 | 30 | my $format = shift @argv; 31 | # Check for - (STDIN) and stringify for sanity 32 | my @files = map { $_ eq '-' ? \*STDIN : "$_" } @argv; 33 | push @files, \*STDIN unless @files; 34 | 35 | if ( !$format ) { 36 | return pod2usage( "ERROR: Must give a format" ); 37 | } 38 | 39 | my $in_format = eval { ETL::Yertl::Format->get( $format, %opt ) }; 40 | if ( $@ ) { 41 | warn "ERROR: $@\n"; 42 | return 1; 43 | } 44 | 45 | my $out = ETL::Yertl::FormatStream->new_for_stdout( 46 | autoflush => 1, 47 | ); 48 | 49 | my $series = ETL::Yertl::InputSeries->new( 50 | streams => \@files, 51 | format => $in_format, 52 | on_doc => sub { 53 | my ( $self, $doc, $eof ) = @_; 54 | $out->write( $doc )->await; 55 | }, 56 | on_read_eof => sub { shift->loop->stop }, 57 | ); 58 | 59 | my $loop = IO::Async::Loop->new; 60 | $loop->add( $out ); 61 | $loop->add( $series ); 62 | $loop->run; 63 | 64 | return 0; 65 | } 66 | 67 | exit __PACKAGE__->main( @ARGV ) unless caller(0); 68 | 69 | __END__ 70 | 71 | =head1 SYNOPSIS 72 | 73 | yfrom [...] 74 | 75 | yfrom csv [-d ] [...] 76 | 77 | yfrom -h|--help|--version 78 | 79 | =head1 DESCRIPTION 80 | 81 | This program takes a stream of documents in the given format (on STDIN or file arguments), 82 | and prints them as YAML. 83 | 84 | =head1 ARGUMENTS 85 | 86 | =head2 format 87 | 88 | The format to read. Currently supported formats: JSON, CSV 89 | 90 | =head2 91 | 92 | A file to read. The special file "-" refers to STDIN. If no files are 93 | specified, read STDIN. 94 | 95 | =head1 OPTIONS 96 | 97 | =head2 -d | --delimiter 98 | 99 | The delimiter to use for the C format. Defaults to C<,>. 100 | 101 | =head2 -h | --help 102 | 103 | Show this help document. 104 | 105 | =head2 --version 106 | 107 | Print the current yfrom and Perl versions. 108 | 109 | =head1 ENVIRONMENT VARIABLES 110 | 111 | =over 4 112 | 113 | =item YERTL_FORMAT 114 | 115 | Specify the default format Yertl uses between commands. Defaults to C. Can be 116 | set to C for interoperability with other programs. 117 | 118 | =back 119 | 120 | -------------------------------------------------------------------------------- /bin/ygrok: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | package ygrok; 3 | our $VERSION = '0.045'; 4 | # ABSTRACT: Build YAML by parsing lines of plain text 5 | 6 | use ETL::Yertl; 7 | use Pod::Usage::Return qw( pod2usage ); 8 | use Getopt::Long qw( GetOptionsFromArray :config pass_through ); 9 | use ETL::Yertl::Command::ygrok; 10 | 11 | $|++; # no buffering 12 | 13 | sub main { 14 | my ( $class, @argv ) = @_; 15 | my %opt; 16 | GetOptionsFromArray( \@argv, \%opt, 17 | 'loose|l', 18 | 'help|h', 19 | 'version', 20 | ); 21 | return pod2usage(0) if $opt{help}; 22 | if ( $opt{version} ) { 23 | print "ygrok version $ygrok::VERSION (Perl $^V)\n"; 24 | return 0; 25 | } 26 | 27 | eval { 28 | ETL::Yertl::Command::ygrok->main( @argv, \%opt ); 29 | }; 30 | if ( $@ ) { 31 | return pod2usage( "ERROR: $@" ); 32 | } 33 | return 0; 34 | } 35 | 36 | exit __PACKAGE__->main( @ARGV ) unless caller(0); 37 | 38 | __END__ 39 | 40 | =head1 SYNOPSIS 41 | 42 | ygrok [-l|--loose] [...] 43 | ygrok --pattern [ []] 44 | yfrom -h|--help|--version 45 | 46 | =head1 DESCRIPTION 47 | 48 | This program takes lines of plain text and converts them into documents. 49 | 50 | =head1 ARGUMENTS 51 | 52 | =head2 pattern 53 | 54 | The pattern to match with. Any line that does not match the pattern will be ignored. 55 | 56 | See the full documentation for pattern syntax. 57 | 58 | =head2 file 59 | 60 | A file to read. The special file "-" refers to STDIN. If no files are 61 | specified, read STDIN. 62 | 63 | =head1 OPTIONS 64 | 65 | =head2 -l|--loose 66 | 67 | Match anywhere in the line. Normally, the pattern must match the full line. 68 | Setting this allows the pattern to match anywhere in the line (but still only once). 69 | 70 | =head2 --pattern 71 | 72 | View, add, and edit patterns. With no arguments, shows all the patterns. With 73 | C, shows the specific pattern or pattern category. With C, 74 | adds a custom pattern that can then be used in future patterns. 75 | 76 | # Show all patterns 77 | ygrok --pattern 78 | 79 | # Show all "NET" patterns 80 | ygrok --pattern NET 81 | 82 | # Show the "NET.HOSTNAME" pattern 83 | ygrok --pattern NET.HOSTNAME 84 | 85 | # Add a new pattern 86 | ygrok --pattern HOSTS_LINE '%{NET.HOSTNAME:host} %{NET.IPV4:ip}' 87 | 88 | # Use the new pattern 89 | ygrok '%{HOSTS_LINE}' < /etc/hosts 90 | 91 | =head2 -h | --help 92 | 93 | Show this help document. 94 | 95 | =head2 --version 96 | 97 | Print the current ygrok and Perl versions. 98 | 99 | =head1 PATTERNS 100 | 101 | A pattern is a match for the entire line, splitting the line into fields. 102 | 103 | A named ygrok match has the format: C<%{PATTERN_NAME:field_name}>. The 104 | C is one of the available patterns, listed below. The 105 | C is the field to put the matched data. 106 | 107 | Additionally, the pattern is a Perl regular expression, so any regular 108 | expression syntax will work. Any named captures 109 | (C<(?Efield_nameEPATTERN)>) will be part of the document. 110 | 111 | =head2 BUILT-IN PATTERNS 112 | 113 | The built-in patterns are common patterns that are always available. 114 | 115 | =over 4 116 | 117 | =item Simple Patterns 118 | 119 | =over 4 120 | 121 | =item WORD 122 | 123 | A single word, C<\b\w+\b>. 124 | 125 | =item DATA 126 | 127 | A non-slurpy section of data, C<.*?>. 128 | 129 | =item INT 130 | 131 | An integer, positive or negative. 132 | 133 | =item NUM 134 | 135 | A floating-point number, positive or negative, with optional exponent. 136 | 137 | =back 138 | 139 | =item Date/Time Patterns 140 | 141 | =over 4 142 | 143 | =item DATE.MONTH 144 | 145 | A full or abbreviated month name for the "C" locale (January (Jan), February 146 | (Feb), etc...) 147 | 148 | =item DATE.ISO8601 149 | 150 | An ISO8601 date/time 151 | 152 | =item DATE.HTTP 153 | 154 | An RFC822 date/time, used by HTTP. 155 | 156 | =item DATE.SYSLOG 157 | 158 | A syslog date, like "Jan 01 01:23:45" 159 | 160 | =back 161 | 162 | =item Operating System Patterns 163 | 164 | =over 4 165 | 166 | =item OS.USER 167 | 168 | A username. 169 | 170 | =item OS.PROCNAME 171 | 172 | A process name 173 | 174 | =back 175 | 176 | =item Networking Patterns 177 | 178 | =over 4 179 | 180 | =item NET.IPV4 181 | 182 | An IPv4 address. 183 | 184 | =item NET.IPV6 185 | 186 | An IPv6 address. 187 | 188 | =item NET.HOSTNAME 189 | 190 | A network host name. Either an RFC1101 domain or an IPv4 or IPv6 address. 191 | 192 | =back 193 | 194 | =item URL Patterns 195 | 196 | =over 4 197 | 198 | =item URL 199 | 200 | A full URL with scheme 201 | 202 | =item URL.PATH 203 | 204 | The path part of a URL 205 | 206 | =back 207 | 208 | =item Log File Patterns 209 | 210 | =over 4 211 | 212 | =item LOG.HTTP_COMMON 213 | 214 | The Apache Common Log Format. 215 | 216 | =item LOG.HTTP_COMBINED 217 | 218 | The Apache Combined Log Format. 219 | 220 | =item LOG.SYSLOG 221 | 222 | The syslog format (RFC 3164) 223 | 224 | =back 225 | 226 | =item POSIX Command Output Patterns 227 | 228 | =over 4 229 | 230 | =item POSIX.LS 231 | 232 | Parse the output of C 233 | 234 | =item POSIX.PS 235 | 236 | Parse the output of C 237 | 238 | =item POSIX.PSX 239 | 240 | Parse the output of C and C 241 | 242 | =item POSIX.PSU 243 | 244 | Parse the output of C 245 | 246 | =back 247 | 248 | =back 249 | 250 | =head1 ENVIRONMENT VARIABLES 251 | 252 | =over 4 253 | 254 | =item YERTL_FORMAT 255 | 256 | Specify the default format Yertl uses between commands. Defaults to C. Can be 257 | set to C for interoperability with other programs. 258 | 259 | =back 260 | 261 | -------------------------------------------------------------------------------- /bin/yto: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | package yto; 3 | our $VERSION = '0.045'; 4 | # ABSTRACT: Change YAML to another format (like JSON) 5 | 6 | use ETL::Yertl; 7 | use Pod::Usage::Return qw( pod2usage ); 8 | use Getopt::Long qw( GetOptionsFromArray ); 9 | use ETL::Yertl::Format; 10 | use ETL::Yertl::FormatStream; 11 | use ETL::Yertl::InputSeries; 12 | use IO::Async::Loop; 13 | 14 | $|++; # no buffering 15 | 16 | sub main { 17 | my ( $class, @argv ) = @_; 18 | my %opt; 19 | GetOptionsFromArray( \@argv, \%opt, 20 | 'help|h', 21 | 'version', 22 | 'delimiter|d=s', 23 | ); 24 | return pod2usage(0) if $opt{help}; 25 | if ( $opt{version} ) { 26 | print "yto version $yto::VERSION (Perl $^V)\n"; 27 | return 0; 28 | } 29 | 30 | my $format = shift @argv; 31 | # Check for - (STDIN) and stringify for sanity 32 | my @files = map { $_ eq '-' ? \*STDIN : "$_" } @argv; 33 | push @files, \*STDIN unless @files; 34 | 35 | if ( !$format ) { 36 | return pod2usage( "ERROR: Must give a format" ); 37 | } 38 | 39 | my $out_format = eval { ETL::Yertl::Format->get( $format, %opt ) }; 40 | if ( $@ ) { 41 | warn "ERROR: $@\n"; 42 | return 1; 43 | } 44 | 45 | my $out = ETL::Yertl::FormatStream->new_for_stdout( 46 | format => $out_format, 47 | autoflush => 1, 48 | ); 49 | 50 | my $series = ETL::Yertl::InputSeries->new( 51 | streams => \@files, 52 | on_doc => sub { 53 | my ( $self, $doc, $eof ) = @_; 54 | $out->write( $doc ); 55 | }, 56 | on_read_eof => sub { shift->loop->stop }, 57 | ); 58 | 59 | my $loop = IO::Async::Loop->new; 60 | $loop->add( $out ); 61 | $loop->add( $series ); 62 | $loop->run; 63 | 64 | return 0; 65 | } 66 | 67 | exit __PACKAGE__->main( @ARGV ) unless caller(0); 68 | 69 | __END__ 70 | 71 | =head1 SYNOPSIS 72 | 73 | yto [...] 74 | 75 | yto csv [-d ] [...] 76 | 77 | yto -h|--help|--version 78 | 79 | =head1 DESCRIPTION 80 | 81 | This program takes a stream of YAML documents (on STDIN or file arguments), 82 | and prints them in the desired format. 83 | 84 | =head1 ARGUMENTS 85 | 86 | =head2 format 87 | 88 | The format to output. Currently supported formats: JSON 89 | 90 | =head2 91 | 92 | A YAML file to read. The special file "-" refers to STDIN. If no files are 93 | specified, read STDIN. 94 | 95 | =head1 OPTIONS 96 | 97 | =head2 -d | --delimiter 98 | 99 | The delimiter to use for the C format. Defaults to C<,>. 100 | 101 | =head2 -h | --help 102 | 103 | Show this help document. 104 | 105 | =head2 --version 106 | 107 | Print the current yto and Perl versions. 108 | 109 | =head1 ENVIRONMENT VARIABLES 110 | 111 | =over 4 112 | 113 | =item YERTL_FORMAT 114 | 115 | Specify the default format Yertl uses between commands. Defaults to C. Can be 116 | set to C for interoperability with other programs. 117 | 118 | =back 119 | 120 | -------------------------------------------------------------------------------- /bin/yts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | package yts; 3 | our $VERSION = '0.045'; 4 | # ABSTRACT: Read/write time series data 5 | 6 | use ETL::Yertl; 7 | use Pod::Usage::Return qw( pod2usage ); 8 | use Getopt::Long qw( GetOptionsFromArray :config pass_through ); 9 | use ETL::Yertl::Command::yts; 10 | 11 | $|++; # no buffering 12 | 13 | sub main { 14 | my ( $class, @argv ) = @_; 15 | my %opt; 16 | GetOptionsFromArray( \@argv, \%opt, 17 | 'help|h', 18 | 'version', 19 | ); 20 | return pod2usage(0) if $opt{help}; 21 | if ( $opt{version} ) { 22 | print "yts version $yts::VERSION (Perl $^V)\n"; 23 | return 0; 24 | } 25 | 26 | eval { 27 | ETL::Yertl::Command::yts->main( @argv, \%opt ); 28 | }; 29 | if ( $@ ) { 30 | return pod2usage( "ERROR: $@" ); 31 | } 32 | return 0; 33 | } 34 | 35 | exit __PACKAGE__->main( @ARGV ) unless caller(0); 36 | 37 | __END__ 38 | 39 | =head1 SYNOPSIS 40 | 41 | # Read a time series 42 | yts [-s] [--start ] [--end ] \ 43 | [--tag =] 44 | 45 | # Write a time series 46 | yts [-s] [] [--tag =] 47 | 48 | yts -h|--help|--version 49 | 50 | =head1 DESCRIPTION 51 | 52 | This program reads and writes time series data. Time series are lists of 53 | date/time and value pairs, optionally with additional metadata tags 54 | attached to them. Time series are stored in databases as metrics. 55 | 56 | Since time series databases have a vast array of features, some of these 57 | arguments/options will not make sense for your database. Any database 58 | that does not support a feature you've requested will cause C to 59 | exit with an error message. 60 | 61 | =head2 Reading Time Series 62 | 63 | To read a time series, ask for a metric. This will display a set of YAML 64 | documents describing the values in the series. 65 | 66 | $ yts influx://localhost:8086 mydb.cpu_load_1m 67 | --- 68 | timestamp: 2017-01-01T00:00:00 69 | metric: mydb.cpu_load_1m 70 | value: 1.23 71 | tags: 72 | server: 10.0.0.1 73 | --- 74 | timestamp: 2017-01-01T00:00:00 75 | metric: mydb.cpu_load_1m 76 | value: 0.61 77 | tags: 78 | server: 10.0.0.2 79 | 80 | To show only the C and C of a single metric, use the 81 | C<--short> format: 82 | 83 | $ yts -s influx://localhost:8086 mydb.cpu_load_1m 84 | --- 85 | '2017-01-01T00:00:00': 1.23 86 | '2017-01-01T00:00:10': 1.26 87 | '2017-01-01T00:00:20': 1.13 88 | 89 | If multiple series are being shown with duplicate timestamps, some 90 | values will be missing. 91 | 92 | =head2 Writing Time Series 93 | 94 | If C has input to read, it will write new values to the time series 95 | database. These values are YAML documents in one of the following 96 | formats: 97 | 98 | =head3 Long format 99 | 100 | This format is a series of YAML documents that describe each point 101 | fully. Use this format if you want to write to many different metrics 102 | with different tags for each metric. The timestamp can be any ISO8601 103 | date/time format or an epoch timestamp. The value can be anything the 104 | time series database accepts. 105 | 106 | --- 107 | timestamp: 2017-01-01T00:00:00 108 | metric: mydb.cpu_load_1m.value 109 | value: 1.23 110 | tags: 111 | server: 10.0.0.1 112 | --- 113 | timestamp: 2017-01-01T00:00:10 114 | metric: mydb.cpu_load_1m.value 115 | value: 1.26 116 | tags: 117 | server: 10.0.0.1 118 | 119 | If your documents have tags, but the database does not support per-value 120 | tagging, C will exit with an error message. 121 | 122 | =head3 Short format 123 | 124 | If you pass in the C<--short> option, you can use the short format. 125 | This format is a set of date/value pairs. The date can be any ISO8601 126 | date/time format or an epoch timestamp. The value can be anything the 127 | time series database accepts. 128 | 129 | --- 130 | '2017-01-01T00:00:00': 1.23 131 | '2017-01-01T00:00:10': 1.26 132 | '2017-01-01T00:00:20': 1.13 133 | 134 | This format can only usefully handle one metric. The metric to write is 135 | taken from the command line. Any tags specified on the command line will 136 | be attached to every point as it is written. 137 | 138 | =head1 ARGUMENTS 139 | 140 | =head2 C 141 | 142 | The database to connect to, in the format C<< ://: >>. 143 | 144 | These adapters come with Yertl: 145 | 146 | =over 147 | 148 | =item * 149 | 150 | L 151 | 152 | =back 153 | 154 | =head2 C 155 | 156 | The metric to read/write. Databases may specify metrics in different ways. 157 | See the adapter documentation for more information. 158 | 159 | =head1 OPTIONS 160 | 161 | =head2 --start 162 | 163 | When reading, only get points starting at or after the given date/time. 164 | Date/time are accepted in any ISO8601 format (C, 165 | C) or as a UNIX epoch timestamp. 166 | 167 | =head2 --end 168 | 169 | When reading, only get points ending at or before the given date/time. 170 | Date/time are accepted in any ISO8601 format (C, 171 | C) or as a UNIX epoch timestamp. 172 | 173 | =head2 --tag = 174 | 175 | When reading, only read values with the given tag name equal to the 176 | given value. When writing, write the given tag name and value to every 177 | point, unless the point specifies a different value for the tag. 178 | 179 | If the time series database does not support tagging of individual values, 180 | this option will cause C to exit with an error. 181 | 182 | =head2 -s | --short 183 | 184 | Read/write the time series in the "short" format. See L for 185 | more information. 186 | 187 | =head2 -h | --help 188 | 189 | Show this help document. 190 | 191 | =head2 --version 192 | 193 | Print the current ysql and Perl versions. 194 | 195 | =head1 ENVIRONMENT VARIABLES 196 | 197 | =over 4 198 | 199 | =item YERTL_FORMAT 200 | 201 | Specify the default format Yertl uses between commands. Defaults to C. Can be 202 | set to C for interoperability with other programs. 203 | 204 | =back 205 | 206 | -------------------------------------------------------------------------------- /cpanfile: -------------------------------------------------------------------------------- 1 | requires "File::HomeDir" => "0"; 2 | requires "File::Spec" => "0"; 3 | requires "Getopt::Long" => "2.36"; 4 | requires "Hash::Merge::Simple" => "0"; 5 | requires "IO::Async" => "0.77"; 6 | requires "IO::Interactive" => "0"; 7 | requires "Import::Base" => "0.010"; 8 | requires "JSON::MaybeXS" => "0"; 9 | requires "List::Util" => "0"; 10 | requires "Module::Runtime" => "0"; 11 | requires "Net::Async::HTTP" => "0"; 12 | requires "Path::Tiny" => "0.072"; 13 | requires "Pod::Usage::Return" => "0"; 14 | requires "Regexp::Common" => "2013031301"; 15 | requires "SQL::Abstract" => "0"; 16 | requires "Time::Local" => "0"; 17 | requires "Time::Piece" => "0"; 18 | requires "URI" => "0"; 19 | requires "YAML::Tiny" => "0"; 20 | requires "boolean" => "0"; 21 | requires "perl" => "5.010"; 22 | recommends "DBD::SQLite" => "0"; 23 | recommends "DBI" => "0"; 24 | recommends "JSON::PP" => "0"; 25 | recommends "JSON::XS" => "0"; 26 | recommends "Text::CSV" => "0"; 27 | recommends "Text::CSV_XS" => "0"; 28 | recommends "YAML::Syck" => "0"; 29 | recommends "YAML::XS" => "0"; 30 | 31 | on 'test' => sub { 32 | requires "Capture::Tiny" => "0"; 33 | requires "Dir::Self" => "0"; 34 | requires "ExtUtils::MakeMaker" => "0"; 35 | requires "File::Spec" => "0"; 36 | requires "Future" => "0"; 37 | requires "IO::Handle" => "0"; 38 | requires "IPC::Open3" => "0"; 39 | requires "JSON::PP" => "0"; 40 | requires "Mock::MonkeyPatch" => "0"; 41 | requires "Test::Deep" => "0"; 42 | requires "Test::Differences" => "0"; 43 | requires "Test::Exception" => "0"; 44 | requires "Test::Lib" => "0"; 45 | requires "Test::More" => "1.001005"; 46 | }; 47 | 48 | on 'test' => sub { 49 | recommends "CPAN::Meta" => "2.120900"; 50 | }; 51 | 52 | on 'configure' => sub { 53 | requires "ExtUtils::MakeMaker" => "0"; 54 | }; 55 | -------------------------------------------------------------------------------- /dist.ini: -------------------------------------------------------------------------------- 1 | name = ETL-Yertl 2 | author = Doug Bell 3 | license = Perl_5 4 | main_module = lib/ETL/Yertl.pm 5 | copyright_holder = Doug Bell 6 | copyright_year = 2018 7 | 8 | [Authority] 9 | authority = cpan:PREACTION 10 | do_munging = 0 11 | 12 | [GithubMeta] 13 | issues = 1 14 | [MetaResources] 15 | IRC = irc://irc.perl.org/#yertl 16 | 17 | ; --- Module management 18 | [@Filter] 19 | -bundle = @Basic 20 | ; GatherDir must be configured separately 21 | -remove = GatherDir 22 | -remove = Readme 23 | 24 | [Readme::Brief] 25 | [ReadmeAnyFromPod] 26 | location = root 27 | filename = README.mkdn 28 | type = markdown 29 | 30 | [PodWeaver] 31 | replacer = replace_with_comment 32 | post_code_replacer = replace_with_nothing 33 | [RewriteVersion] 34 | [CPANFile] 35 | [MetaJSON] 36 | [MetaProvides::Package] 37 | [MetaNoIndex] 38 | directory = t 39 | directory = xt 40 | directory = inc 41 | directory = share 42 | directory = eg 43 | directory = examples 44 | [Git::Contributors] 45 | ; authordep Pod::Weaver::Section::Contributors 46 | [Test::ReportPrereqs] 47 | [Test::Compile] 48 | 49 | ; --- Repository management 50 | [Git::GatherDir] 51 | include_dotfiles = 1 52 | prune_directory = ^eg 53 | ; Exclude dotfiles in the root directory 54 | exclude_match = ^\.[^/]+$ 55 | ; Exclude ini files in the root directory 56 | exclude_match = ^[^/]+\.ini$ 57 | ; Exclude generated root content, which is included by the various plugins 58 | ; Without this, we get an error about duplicate content 59 | exclude_filename = cpanfile 60 | exclude_filename = LICENSE 61 | exclude_filename = README.mkdn 62 | exclude_filename = Makefile.PL 63 | 64 | [CopyFilesFromBuild] 65 | ; Copy generated content to the repository root so users without Dist::Zilla 66 | ; can use it 67 | copy = cpanfile 68 | copy = LICENSE 69 | copy = Makefile.PL 70 | 71 | [Run::AfterBuild] 72 | ; Add travis and coveralls badges to README.mkdn 73 | run = perl -pi -e 's{(# SYNOPSIS)}{# STATUS\n\nCoverage Status\n\n$1}' README.mkdn 74 | 75 | ; --- Git management 76 | [CheckChangesHasContent] 77 | changelog = CHANGES 78 | [Git::Check] 79 | ; Automatically commit these files during release 80 | allow_dirty_match = README.* 81 | allow_dirty_match = .*[.]PL 82 | allow_dirty = cpanfile 83 | allow_dirty = LICENSE 84 | allow_dirty = CHANGES 85 | 86 | [Git::Commit / Commit_Dirty_Files] 87 | ; Automatically commit with release version and changelog 88 | changelog = CHANGES 89 | commit_msg = release v%v%n%n%c 90 | allow_dirty_match = README.* 91 | allow_dirty_match = .*[.]PL 92 | allow_dirty = cpanfile 93 | allow_dirty = LICENSE 94 | allow_dirty = CHANGES 95 | add_files_in = . 96 | [Git::Tag] 97 | changelog = CHANGES 98 | tag_message = %N v%v - %{yyyy-MM-dd}d%n%n%c ; Tag annotations show up in github release list 99 | 100 | ; NextRelease acts *during* pre-release to write $VERSION and 101 | ; timestamp to Changes and *after* release to add a new {{$NEXT}} 102 | ; section, so to act at the right time after release, it must actually 103 | ; come after Commit_Dirty_Files but before Commit_Changes in the 104 | ; dist.ini. It will still act during pre-release as usual 105 | 106 | [NextRelease] 107 | filename = CHANGES 108 | [BumpVersionAfterRelease] 109 | 110 | [Git::Commit / Commit_Changes] 111 | commit_msg = incrementing version after release 112 | allow_dirty = CHANGES 113 | allow_dirty_match = ^lib/.*\.pm$ 114 | allow_dirty_match = .*[.]PL 115 | allow_dirty_match = ^bin/ 116 | [Git::Push] 117 | 118 | ; --- Project-specific directives 119 | [Prereqs] 120 | perl = 5.010 ; Recursive regexp: (?R) and defined-or 121 | Time::Piece = 0 122 | Time::Local = 0 123 | YAML::Tiny = 0 124 | Import::Base = 0.010 125 | Getopt::Long = 2.36 ; First version with GetOptionsFromArray 126 | Pod::Usage::Return = 0 127 | File::Spec = 0 128 | boolean = 0 129 | Regexp::Common = 2013031301 ; Added IPv6 pattern 130 | Module::Runtime = 0 131 | List::Util = 0 132 | Path::Tiny = 0.072 ; Fixes issues with File::Path 133 | File::HomeDir = 0 134 | Hash::Merge::Simple = 0 135 | SQL::Abstract = 0 136 | Net::Async::HTTP = 0 137 | URI = 0 138 | JSON::MaybeXS = 0 139 | IO::Interactive = 0 140 | IO::Async = 0.77 141 | 142 | [Prereqs / Recommends] 143 | ; Formatter modules 144 | JSON::XS = 0 145 | JSON::PP = 0 146 | YAML::XS = 0 147 | YAML::Syck = 0 148 | Text::CSV = 0 149 | Text::CSV_XS = 0 150 | ; Optional, compiled modules 151 | DBI = 0 152 | DBD::SQLite = 0 153 | 154 | [Prereqs / TestRequires] 155 | Test::More = 1.001005 156 | Test::Exception = 0 157 | Test::Deep = 0 158 | Test::Differences = 0 159 | Test::Lib = 0 160 | Capture::Tiny = 0 161 | JSON::PP = 0 162 | Dir::Self = 0 163 | Mock::MonkeyPatch = 0 164 | Future = 0 165 | -------------------------------------------------------------------------------- /eg/transform.pl: -------------------------------------------------------------------------------- 1 | 2 | package Local::Dump; 3 | BEGIN { $INC{ 'Local/Dump.pm' } = __FILE__ }; 4 | use ETL::Yertl; 5 | use Data::Dumper; 6 | use base 'ETL::Yertl::Transform'; 7 | 8 | sub transform_doc { 9 | local $Data::Dumper::Terse = 1; 10 | local $Data::Dumper::Indent = 0; 11 | say STDERR "# DUMPER: " . Dumper( $_ ); 12 | return $_; 13 | } 14 | 15 | package Local::AddHello; 16 | BEGIN { $INC{ 'Local/AddHello.pm' } = __FILE__ }; 17 | use ETL::Yertl; 18 | use base 'ETL::Yertl::Transform'; 19 | 20 | sub transform_doc { 21 | my ( $self, $doc ) = @_; 22 | if ( ref $doc eq 'HASH' ) { 23 | $doc->{__HELLO__} = "World"; 24 | } 25 | $self->write( $doc ); 26 | } 27 | 28 | package main; 29 | use ETL::Yertl; 30 | use Local::Dump; 31 | use Local::AddHello; 32 | 33 | my $xform 34 | = stdin( format => 'json' ) 35 | | transform( "Local::Dump" ) 36 | | transform( "Local::AddHello" ) >> stdout() 37 | | transform( 38 | sub { 39 | say STDERR "# Hey"; 40 | # Return instead of write 41 | ; say "Returning a doc"; 42 | return $_; 43 | }, 44 | ) >> file( '>', 'output.yaml' ) 45 | ; 46 | 47 | # loop()->run 48 | $xform->run; 49 | 50 | -------------------------------------------------------------------------------- /lib/ETL/Yertl/Command/yts.pm: -------------------------------------------------------------------------------- 1 | package ETL::Yertl::Command::yts; 2 | our $VERSION = '0.045'; 3 | # ABSTRACT: Read/Write time series data 4 | 5 | =head1 SYNOPSIS 6 | 7 | =head1 DESCRIPTION 8 | 9 | =head1 SEE ALSO 10 | 11 | L 12 | 13 | =cut 14 | 15 | use ETL::Yertl; 16 | use ETL::Yertl::Util qw( load_module ); 17 | use Getopt::Long qw( GetOptionsFromArray :config pass_through ); 18 | use IO::Interactive qw( is_interactive ); 19 | use ETL::Yertl::Format; 20 | use ETL::Yertl::FormatStream; 21 | use IO::Async::Loop; 22 | 23 | sub main { 24 | my $class = shift; 25 | 26 | my %opt; 27 | if ( ref $_[-1] eq 'HASH' ) { 28 | %opt = %{ pop @_ }; 29 | } 30 | 31 | my @args = @_; 32 | GetOptionsFromArray( \@args, \%opt, 33 | 'start=s', 34 | 'end=s', 35 | 'short|s', 36 | 'tags=s%', 37 | ); 38 | #; use Data::Dumper; 39 | #; say Dumper \@args; 40 | #; say Dumper \%opt; 41 | 42 | my ( $db_spec, $metric ) = @args; 43 | 44 | die "Must give a database\n" unless $db_spec; 45 | 46 | my ( $db_type ) = $db_spec =~ m{^([^:]+):}; 47 | 48 | my $db = load_module( adapter => $db_type )->new( $db_spec ); 49 | 50 | # Write metrics 51 | if ( !is_interactive( \*STDIN ) ) { 52 | if ( $opt{short} ) { 53 | die "Must give a metric\n" unless $metric; 54 | } 55 | 56 | my $count = 0; 57 | my $loop = IO::Async::Loop->new; 58 | my $in = ETL::Yertl::FormatStream->new_for_stdin( 59 | on_doc => sub { 60 | my ( $self, $doc, $eof ) = @_; 61 | return unless $doc; 62 | #; use Data::Dumper 63 | #; say "Got doc: " . Dumper $doc; 64 | if ( $opt{short} ) { 65 | my @docs; 66 | for my $stamp ( sort keys %$doc ) { 67 | push @docs, { 68 | timestamp => $stamp, 69 | metric => $metric, 70 | value => $doc->{ $stamp }, 71 | ( $opt{tags} ? ( tags => $opt{tags} ) : () ), 72 | }; 73 | } 74 | #; use Data::Dumper; 75 | #; print Dumper \@docs; 76 | $db->write_ts( @docs ); 77 | $count += @docs; 78 | } 79 | else { 80 | $doc->{metric} ||= $metric; 81 | $doc->{tags} ||= $opt{tags} if $opt{tags}; 82 | $db->write_ts( $doc ); 83 | $count++; 84 | } 85 | }, 86 | on_read_eof => sub { $loop->stop }, 87 | ); 88 | $loop->add( $in ); 89 | $loop->run; 90 | #; say "Wrote $count points"; 91 | } 92 | # Read metrics 93 | else { 94 | die "Must give a metric\n" unless $metric; 95 | my $out_fmt = ETL::Yertl::Format->get_default; 96 | my @points = $db->read_ts( { 97 | metric => $metric, 98 | tags => $opt{tags}, 99 | start => $opt{start}, 100 | end => $opt{end}, 101 | } ); 102 | if ( $opt{short} ) { 103 | my %ts = map { $_->{timestamp} => $_->{value} } @points; 104 | print $out_fmt->format( \%ts ); 105 | } 106 | else { 107 | print $out_fmt->format( $_ ) for @points; 108 | } 109 | } 110 | 111 | return 0; 112 | } 113 | 114 | 1; 115 | __END__ 116 | 117 | -------------------------------------------------------------------------------- /lib/ETL/Yertl/Format.pm: -------------------------------------------------------------------------------- 1 | package ETL::Yertl::Format; 2 | our $VERSION = '0.045'; 3 | # ABSTRACT: Base class for input/output formats 4 | 5 | =head1 SYNOPSIS 6 | 7 | use ETL::Yertl::Format; 8 | my $json_format = ETL::Yertl::Format->get( "json" ); 9 | my $default_format = ETL::Yertl::Format->get_default; 10 | 11 | =head1 DESCRIPTION 12 | 13 | Formatters handle parsing input strings into document hashes and 14 | formatting document hashes into output strings. 15 | 16 | Formatter objects are given to L objects. 17 | 18 | =head1 SEE ALSO 19 | 20 | L 21 | 22 | =cut 23 | 24 | use ETL::Yertl; 25 | use ETL::Yertl::Util qw( load_module ); 26 | use Module::Runtime qw( use_module ); 27 | 28 | sub new { 29 | my ( $class, %opt ) = @_; 30 | $opt{formatter_class} ||= $class->_find_formatter_class; 31 | return bless \%opt, $class; 32 | } 33 | 34 | =method get 35 | 36 | my $format = ETL::Yertl::Format->get( $name, %args ); 37 | 38 | Get the formatter with the given name. C<$name> should be the last word 39 | in the C subclass (like C for 40 | C). C<%args> will be passed-in to the 41 | formatter constructor. 42 | 43 | =cut 44 | 45 | sub get { 46 | my ( $class, $type, @args ) = @_; 47 | load_module( format => $type )->new( @args ); 48 | } 49 | 50 | =method get_default 51 | 52 | my $format = ETL::Yertl::Format->get_default; 53 | 54 | Get the default format for Yertl programs to communicate with each 55 | other. By default, this is C, but it can be set to C by 56 | setting the C environment variable to C<"json">. 57 | 58 | Setting the default format to something besides YAML can help 59 | interoperate with other programs like 60 | L or L. 61 | 62 | =cut 63 | 64 | sub get_default { 65 | my ( $class ) = @_; 66 | my $format = $ENV{YERTL_FORMAT} || 'yaml'; 67 | return $class->get( $format ); 68 | } 69 | 70 | sub _formatter_classes { 71 | die '_formatter_classes must be overridden'; 72 | } 73 | 74 | sub _find_formatter_class { 75 | my ( $class ) = @_; 76 | my @class_versions = $class->_formatter_classes; 77 | for my $class_version ( @class_versions ) { 78 | eval { 79 | # Prototypes on use_module() make @$class_version not work correctly 80 | use_module( $class_version->[0], $class_version->[1] ); 81 | }; 82 | if ( !$@ ) { 83 | return $class_version->[0]; 84 | } 85 | } 86 | die "Could not load a formatter for $class. Please install one of the following modules:\n" 87 | . join( "", 88 | map { sprintf "\t%s (%s)", $_->[0], $_->[1] ? "version $_->[1]" : "Any version" } 89 | @class_versions 90 | ) 91 | . "\n"; 92 | } 93 | 94 | 1; 95 | -------------------------------------------------------------------------------- /lib/ETL/Yertl/Format/csv.pm: -------------------------------------------------------------------------------- 1 | package ETL::Yertl::Format::csv; 2 | our $VERSION = '0.045'; 3 | # ABSTRACT: CSV read/write support for Yertl 4 | 5 | =head1 SYNOPSIS 6 | 7 | =head1 DESCRIPTION 8 | 9 | =head1 SEE ALSO 10 | 11 | L 12 | 13 | =cut 14 | 15 | use ETL::Yertl; 16 | use base 'ETL::Yertl::Format'; 17 | 18 | sub new { 19 | my ( $class, %opt ) = @_; 20 | $opt{delimiter} ||= ','; 21 | return $class->SUPER::new( %opt ); 22 | } 23 | 24 | sub _formatter_classes { 25 | return ( 26 | [ 'Text::CSV_XS' => 0 ], 27 | [ 'Text::CSV' => 0 ], 28 | ); 29 | } 30 | 31 | sub _formatter { 32 | my ( $self ) = @_; 33 | return $self->{formatter_class}->new( { sep_char => $self->{delimiter} } ); 34 | } 35 | 36 | sub read_buffer { 37 | my ( $self, $buffref, $eof ) = @_; 38 | my $csv = $self->_formatter; 39 | my $names = $self->{_field_names} ||= []; 40 | my @docs; 41 | while ( $$buffref =~ s/^(.*\n)// ) { 42 | my $line = $1; 43 | if ( !@$names ) { 44 | $csv->parse( $line ); 45 | @$names = $csv->fields; 46 | next; 47 | } 48 | 49 | my $status = $csv->parse( $line ); 50 | my @fields = $csv->fields; 51 | my $doc = { 52 | map {; $names->[ $_ ] => $fields[ $_ ] } 53 | 0..$#fields 54 | }; 55 | push @docs, $doc; 56 | } 57 | return @docs; 58 | } 59 | 60 | sub format { 61 | my ( $self, $doc ) = @_; 62 | my $csv = $self->_formatter; 63 | 64 | my $names = $self->{_field_names} ||= []; 65 | if ( !@$names ) { 66 | @$names = sort keys %$doc; 67 | } 68 | 69 | my $str = ''; 70 | if ( !$self->{_wrote_header} ) { 71 | $csv->combine( @$names ); 72 | $str .= $csv->string . $/; 73 | $self->{_wrote_header} = 1; 74 | } 75 | 76 | $csv->combine( map { $doc->{ $_ } } @$names ); 77 | $str .= $csv->string . $/; 78 | 79 | return $str; 80 | } 81 | 82 | 1; 83 | -------------------------------------------------------------------------------- /lib/ETL/Yertl/Format/json.pm: -------------------------------------------------------------------------------- 1 | package ETL::Yertl::Format::json; 2 | our $VERSION = '0.045'; 3 | # ABSTRACT: JSON read/write support for Yertl 4 | 5 | =head1 SYNOPSIS 6 | 7 | =head1 DESCRIPTION 8 | 9 | =head1 SEE ALSO 10 | 11 | L 12 | 13 | =cut 14 | 15 | use ETL::Yertl; 16 | use base 'ETL::Yertl::Format'; 17 | 18 | sub _formatter_classes { 19 | return ( 20 | [ 'JSON::XS' => 0 ], 21 | [ 'JSON::PP' => 0 ], 22 | ); 23 | } 24 | 25 | sub _json_writer { 26 | my ( $self ) = @_; 27 | $self->{_json_writer} ||= do { 28 | my $json = $self->{formatter_class}->new->canonical->pretty->allow_nonref; 29 | if ( $self->{formatter_class} ne 'JSON::XS' ) { 30 | $json->indent_length(3); 31 | } 32 | $json; 33 | }; 34 | } 35 | 36 | sub read_buffer { 37 | my ( $self, $buffref, $eof ) = @_; 38 | my $json = $self->{_json_reader} ||= $self->{formatter_class}->new->relaxed; 39 | my @docs; 40 | 41 | # Work around a bug in JSON::PP: incr_parse() only returns the 42 | # first item, see: https://github.com/makamaka/JSON-PP/pull/7 43 | # Adapted from IO::Async::JSONStream by Paul Evans 44 | $json->incr_parse( $$buffref ); 45 | $$buffref = ''; 46 | PARSE_ONE: { 47 | my $doc; 48 | 49 | my $fail = not eval { 50 | $doc = $json->incr_parse; 51 | 1 52 | }; 53 | chomp( my $e = $@ ); 54 | 55 | if ( $doc ) { 56 | #; use Data::Dumper; 57 | #; say STDERR "## Got document " . Dumper $doc; 58 | push @docs, $doc; 59 | redo PARSE_ONE; 60 | } 61 | elsif ( $fail ) { 62 | # XXX: Parse failure 63 | $json->incr_skip; 64 | redo PARSE_ONE; 65 | } 66 | # else last 67 | } 68 | 69 | return @docs; 70 | } 71 | 72 | sub format { 73 | my ( $self, $doc ) = @_; 74 | my $json = $self->_json_writer; 75 | return $json->encode( $doc ); 76 | } 77 | 78 | 1; 79 | -------------------------------------------------------------------------------- /lib/ETL/Yertl/Format/yaml.pm: -------------------------------------------------------------------------------- 1 | package ETL::Yertl::Format::yaml; 2 | our $VERSION = '0.045'; 3 | # ABSTRACT: YAML read/write support for Yertl 4 | 5 | =head1 SYNOPSIS 6 | 7 | =head1 DESCRIPTION 8 | 9 | =head1 SEE ALSO 10 | 11 | L 12 | 13 | =cut 14 | 15 | use ETL::Yertl; 16 | use base 'ETL::Yertl::Format'; 17 | 18 | sub new { 19 | my ( $class, @args ) = @_; 20 | my $self = $class->SUPER::new( @args ); 21 | no strict 'refs'; 22 | $self->{_load} = \&{ $self->{formatter_class} . '::Load' }; 23 | $self->{_dump} = \&{ $self->{formatter_class} . '::Dump' }; 24 | return $self; 25 | } 26 | 27 | sub _formatter_classes { 28 | return ( 29 | [ 'YAML::XS' => 0 ], 30 | [ 'YAML::Syck' => 0 ], 31 | # [ 'YAML' => 0 ], # Disabled: YAML::Old changes have broke something here... 32 | [ 'YAML::Tiny' => 0 ], 33 | ); 34 | } 35 | 36 | sub read_buffer { 37 | my ( $self, $buffref, $eof ) = @_; 38 | my @docs; 39 | $self->{_doc_buf} ||= ''; 40 | while ( $$buffref =~ s/^(.*\n)// ) { 41 | my $line = $1; 42 | if ( $line =~ /^---/ && $self->{_doc_buf} ) { 43 | #; say STDERR "## Got document\n$self->{_doc_buf}"; 44 | push @docs, $self->{_load}->( $self->{_doc_buf} ); 45 | $self->{_doc_buf} = ''; 46 | } 47 | else { 48 | $self->{_doc_buf} .= $line; 49 | } 50 | } 51 | if ( $eof && $self->{_doc_buf} ) { 52 | #; say STDERR "## Got document\n$self->{_doc_buf}"; 53 | push @docs, $self->{_load}->( $self->{_doc_buf} ); 54 | } 55 | return @docs; 56 | } 57 | 58 | sub format { 59 | my ( $self, $doc ) = @_; 60 | return $self->{_dump}->( $doc ); 61 | } 62 | 63 | 1; 64 | -------------------------------------------------------------------------------- /lib/ETL/Yertl/FormatStream.pm: -------------------------------------------------------------------------------- 1 | package ETL::Yertl::FormatStream; 2 | our $VERSION = '0.045'; 3 | # ABSTRACT: Read/write I/O stream with Yertl formatters 4 | 5 | =head1 SYNOPSIS 6 | 7 | use ETL::Yertl; 8 | use ETL::Yertl::FormatStream; 9 | use ETL::Yertl::Format; 10 | use IO::Async::Loop; 11 | 12 | my $loop = IO::Async::Loop->new; 13 | my $format = ETL::Yertl::Format->get( "json" ); 14 | 15 | my $input = ETL::Yertl::FormatStream->new( 16 | read_handle => \*STDIN, 17 | format => $format, 18 | on_doc => sub { 19 | my ( $self, $doc, $eof ) = @_; 20 | 21 | # ... do something with $doc 22 | 23 | if ( $eof ) { 24 | $loop->stop; 25 | } 26 | }, 27 | ); 28 | 29 | $loop->add( $input ); 30 | $loop->run; 31 | 32 | =head1 DESCRIPTION 33 | 34 | =head1 SEE ALSO 35 | 36 | L 37 | 38 | =cut 39 | 40 | use ETL::Yertl; 41 | use base 'IO::Async::Stream'; 42 | use ETL::Yertl::Format; 43 | use Carp qw( croak ); 44 | use Fcntl; 45 | 46 | sub configure { 47 | my ( $self, %args ) = @_; 48 | 49 | if ( exists $args{format} ) { 50 | $self->{format} = delete $args{format}; 51 | } 52 | elsif ( !$self->{format} ) { 53 | $self->{format} = ETL::Yertl::Format->get_default; 54 | } 55 | 56 | for my $event ( qw( on_doc ) ) { 57 | $self->{ $event } = delete $args{ $event } if exists $args{ $event }; 58 | } 59 | if ( $self->read_handle ) { 60 | $self->can_event( "on_doc" ) 61 | or croak "Expected either an on_doc callback or to be able to ->on_doc"; 62 | } 63 | 64 | if ( $args{autoflush} && $args{write_handle} ) { 65 | my $flags = fcntl( $args{write_handle}, F_GETFL, 0 ); 66 | fcntl( $args{write_handle}, F_SETFL, $flags | O_NONBLOCK ); 67 | } 68 | 69 | $self->SUPER::configure( %args ); 70 | } 71 | 72 | sub on_read { 73 | my ( $self, $buffref, $eof ) = @_; 74 | my @docs = $self->{format}->read_buffer( $buffref, $eof ); 75 | for my $doc ( @docs ) { 76 | $self->invoke_event( on_doc => $doc, $eof ); 77 | } 78 | return 0; 79 | } 80 | 81 | sub write { 82 | my ( $self, $doc, @args ) = @_; 83 | my $str = $self->{format}->format( $doc ); 84 | return $self->SUPER::write( $str, @args ); 85 | } 86 | 87 | 1; 88 | -------------------------------------------------------------------------------- /lib/ETL/Yertl/Help/Cookbook.pod: -------------------------------------------------------------------------------- 1 | # PODNAME: ETL::Yertl::Help::Cookbook 2 | # ABSTRACT: Some recipies for common ETL tasks 3 | 4 | =head1 DESCRIPTION 5 | 6 | This document describes how to do some common ETL tasks 7 | 8 | =head1 EMBEDDED DOCUMENTS 9 | 10 | When you've got serialized JSON in a database field, you can use C to deserialize 11 | it: 12 | 13 | ysql TESTDB 'SELECT json_field FROM table' # Query the database 14 | | yto json # Convert to JSON for jq 15 | | js '.json_field | fromjson' # Inflate the JSON 16 | | yfrom json # Convert back to Yertl 17 | -------------------------------------------------------------------------------- /lib/ETL/Yertl/InputSeries.pm: -------------------------------------------------------------------------------- 1 | package ETL::Yertl::InputSeries; 2 | our $VERSION = '0.045'; 3 | # ABSTRACT: Read a series of input streams 4 | 5 | =head1 SYNOPSIS 6 | 7 | use ETL::Yertl; 8 | use ETL::Yertl::InputSeries; 9 | my $series = ETL::Yertl::InputSeries->new( 10 | streams => [ \*STDIN, "/path/to/file.yaml" ], 11 | on_doc => sub { 12 | my ( $self, $doc, $eof ) = @_; 13 | # ... do something with $doc 14 | }, 15 | on_child_eof => sub { 16 | my ( $self ) = @_; 17 | say STDERR "Switching stream"; 18 | }, 19 | on_read_eof => sub { 20 | my ( $self ) = @_; 21 | # All streams have been exhausted 22 | $self->loop->stop; 23 | }, 24 | ); 25 | 26 | use IO::Async::Loop; 27 | my $loop = IO::Async::Loop->new; 28 | $loop->add( $series ); 29 | $loop->run; 30 | 31 | =head1 DESCRIPTION 32 | 33 | This module reads a series of input streams in the default format 34 | (determined by L). Input streams can be 35 | filehandles (like C) or paths to files. 36 | 37 | =head1 SEE ALSO 38 | 39 | L 40 | 41 | =cut 42 | 43 | use ETL::Yertl; 44 | use base 'IO::Async::Notifier'; 45 | use Carp qw( croak ); 46 | use Scalar::Util qw( weaken ); 47 | 48 | sub configure { 49 | my ( $self, %args ) = @_; 50 | 51 | # Args to pass to streams as we create them 52 | # XXX: This is why I would prefer already having the streams 53 | # created! 54 | for my $arg ( qw( format ) ) { 55 | $self->{stream_args}{ $arg } = delete $args{ $arg } if $args{ $arg }; 56 | } 57 | 58 | if ( my $streams = delete $args{streams} ) { 59 | # TODO: Support any kind of Yertl input stream. Stream must not 60 | # yet be added to a loop, otherwise it will start producing 61 | # events we're not ready to handle yet (I think) 62 | if ( grep { ref $_ && ref $_ ne 'GLOB' } @$streams ) { 63 | croak "InputSeries streams must be file paths or filehandles"; 64 | } 65 | $self->{streams} = $streams; 66 | } 67 | for my $event ( qw( on_doc on_read_eof on_child_read_eof ) ) { 68 | $self->{ $event } = delete $args{ $event } if exists $args{ $event }; 69 | } 70 | 71 | return $self->SUPER::configure( %args ); 72 | } 73 | 74 | sub _add_to_loop { 75 | my ( $self ) = @_; 76 | $self->can_event( "on_doc" ) 77 | or croak "Expected either an on_doc callback or to be able to ->on_doc"; 78 | $self->_shift_stream; 79 | } 80 | 81 | sub _shift_stream { 82 | my ( $self ) = @_; 83 | weaken $self; 84 | my $stream = shift @{ $self->{streams} }; 85 | my $fh; 86 | if ( !ref $stream ) { 87 | open $fh, '<', $stream or die "Could not open $stream for reading: $!"; 88 | } 89 | elsif ( ref $stream eq 'GLOB' ) { 90 | $fh = $stream; 91 | } 92 | else { 93 | die "Unknown stream type '$stream': Should be path or filehandle"; 94 | } 95 | 96 | my $current_stream = $self->{current_stream} = ETL::Yertl::FormatStream->new( 97 | %{ $self->{stream_args} }, 98 | read_handle => $fh, 99 | on_doc => sub { 100 | my ( undef, $doc, $eof ) = @_; 101 | $self->invoke_event( on_doc => $doc, $eof ); 102 | }, 103 | on_read_eof => sub { $self->_on_child_read_eof }, 104 | ); 105 | $self->add_child( $current_stream ); 106 | } 107 | 108 | sub _on_child_read_eof { 109 | my ( $self ) = @_; 110 | $self->remove_child( $self->{current_stream} ); 111 | $self->maybe_invoke_event( 'on_child_read_eof' ); 112 | if ( !@{ $self->{streams} } ) { 113 | $self->maybe_invoke_event( 'on_read_eof' ); 114 | return; 115 | } 116 | $self->_shift_stream; 117 | } 118 | 119 | 1; 120 | -------------------------------------------------------------------------------- /lib/ETL/Yertl/LineStream.pm: -------------------------------------------------------------------------------- 1 | package ETL::Yertl::LineStream; 2 | our $VERSION = '0.045'; 3 | # ABSTRACT: Read/write I/O streams in lines 4 | 5 | =head1 SYNOPSIS 6 | 7 | use ETL::Yertl; 8 | use ETL::Yertl::LineStream; 9 | use IO::Async::Loop; 10 | 11 | my $loop = IO::Async::Loop->new; 12 | my $input = ETL::Yertl::LineStream->new( 13 | read_handle => \*STDIN, 14 | on_line => sub { 15 | my ( $self, $doc, $eof ) = @_; 16 | 17 | # ... do something with $doc 18 | 19 | if ( $eof ) { 20 | $loop->stop; 21 | } 22 | }, 23 | ); 24 | 25 | $loop->add( $input ); 26 | $loop->run; 27 | 28 | =head1 DESCRIPTION 29 | 30 | This is an unformatted I/O stream. Use this to write simple scalars to 31 | the output or to read lines from the input. 32 | 33 | =head1 SEE ALSO 34 | 35 | L 36 | 37 | =cut 38 | 39 | use ETL::Yertl; 40 | use base 'IO::Async::Stream'; 41 | use Fcntl; 42 | 43 | sub configure { 44 | my ( $self, %args ) = @_; 45 | if ( $args{autoflush} && $args{write_handle} ) { 46 | my $flags = fcntl( $args{write_handle}, F_GETFL, 0 ); 47 | fcntl( $args{write_handle}, F_SETFL, $flags | O_NONBLOCK ); 48 | } 49 | $self->SUPER::configure( %args ); 50 | } 51 | 52 | sub on_read { 53 | my ( $self, $buffref, $eof ) = @_; 54 | my @lines = $$buffref =~ s{\g(.+$/)}{}g; 55 | for my $line ( @lines ) { 56 | $self->invoke_event( on_line => $line, $eof ); 57 | } 58 | return 0; 59 | } 60 | 61 | sub write { 62 | my ( $self, $line, %args ) = @_; 63 | return unless $line; 64 | $line .= "\n" unless $line =~ /\n$/; 65 | return $self->SUPER::write( $line, %args ); 66 | } 67 | 68 | 1; 69 | 70 | -------------------------------------------------------------------------------- /lib/ETL/Yertl/Util.pm: -------------------------------------------------------------------------------- 1 | package ETL::Yertl::Util; 2 | our $VERSION = '0.045'; 3 | # ABSTRACT: Utility functions for Yertl modules 4 | 5 | =head1 SYNOPSIS 6 | 7 | =head1 DESCRIPTION 8 | 9 | =head1 SEE ALSO 10 | 11 | =cut 12 | 13 | use ETL::Yertl; 14 | use Exporter qw( import ); 15 | use Module::Runtime qw( use_module compose_module_name ); 16 | 17 | our @EXPORT_OK = qw( 18 | load_module firstidx 19 | docs_from_string 20 | ); 21 | 22 | sub docs_from_string { 23 | my ( $format, $string ); 24 | if ( @_ > 1 ) { 25 | $format = ETL::Yertl::Format->get( $_[0] ); 26 | $string = $_[1]; 27 | } 28 | else { 29 | $format = ETL::Yertl::Format->get_default; 30 | $string = $_[0]; 31 | } 32 | my @docs = $format->read_buffer( \$string, 1 ); 33 | return @docs; 34 | } 35 | 36 | =sub load_module 37 | 38 | $class = load_module( format => $format ); 39 | $class = load_module( protocol => $proto ); 40 | $class = load_module( database => $db ); 41 | 42 | Load a module of the given type with the given name. Throws an exception if the 43 | module is not found or the module cannot be loaded. 44 | 45 | This function should be used to load modules that the user requests. The error 46 | messages are suitable for user consumption. 47 | 48 | =cut 49 | 50 | sub load_module { 51 | my ( $type, $name ) = @_; 52 | 53 | die "$type is required\n" unless $name; 54 | my $class = eval { compose_module_name( 'ETL::Yertl::' . ucfirst $type, $name ) }; 55 | if ( $@ ) { 56 | die "Unknown $type '$name'\n"; 57 | } 58 | 59 | eval { 60 | use_module( $class ); 61 | }; 62 | if ( $@ ) { 63 | if ( $@ =~ /^Can't locate \S+ in \@INC/ ) { 64 | die "Unknown $type '$name'\n"; 65 | } 66 | die "Could not load $type '$name': $@"; 67 | } 68 | 69 | return $class; 70 | } 71 | 72 | =sub firstidx 73 | 74 | my $i = firstidx { ... } @array; 75 | 76 | Return the index of the first item that matches the code block, or C<-1> if 77 | none match 78 | 79 | =cut 80 | 81 | # This duplicates List::Util firstidx, but this is not included in Perl 5.10 82 | sub firstidx(&@) { 83 | my $code = shift; 84 | for my $i ( 0 .. @_ ) { 85 | local $_ = $_[ $i ]; 86 | return $i if $code->(); 87 | } 88 | return -1; 89 | } 90 | 91 | 1; 92 | -------------------------------------------------------------------------------- /site.yml: -------------------------------------------------------------------------------- 1 | theme: 2 | class: Statocles::Theme 3 | args: 4 | store: site/theme 5 | 6 | gh_pages: 7 | class: Statocles::Deploy::Git 8 | args: 9 | path: '.' 10 | branch: gh-pages 11 | 12 | personal: 13 | class: Statocles::Deploy::Git 14 | args: 15 | path: '.' 16 | branch: deploy 17 | remote: www 18 | 19 | site: 20 | class: Statocles::Site 21 | args: 22 | title: Yertl 23 | base_url: http://preaction.me/yertl/ 24 | plugins: 25 | link_check: 26 | $class: Statocles::Plugin::LinkCheck 27 | nav: 28 | main: 29 | - title: Blog 30 | href: /blog 31 | - title: Docs 32 | href: /pod 33 | - title: Code 34 | href: 'http://github.com/preaction/ETL-Yertl' 35 | - title: Bugs 36 | href: 'http://github.com/preaction/ETL-Yertl/issues' 37 | - title: CPAN 38 | href: 'http://metacpan.org/pod/ETL::Yertl' 39 | - title: IRC 40 | href: 'https://chat.mibbit.com/?channel=%23yertl&server=irc.perl.org' 41 | deploy: 42 | $ref: personal 43 | theme: 44 | $ref: theme 45 | apps: 46 | 47 | page: 48 | $class: Statocles::App::Basic 49 | $args: 50 | url_root: / 51 | store: site/ 52 | 53 | pod: 54 | $class: Statocles::App::Perldoc 55 | $args: 56 | url_root: /pod 57 | inc: 58 | - lib/ 59 | - bin/ 60 | modules: 61 | - 'ETL::Yertl' 62 | - 'ETL::Yertl::' 63 | - 'yfrom' 64 | - 'yto' 65 | - 'ysql' 66 | - 'yq' 67 | - 'ygrok' 68 | index_module: 'ETL::Yertl' 69 | weave: 1 70 | 71 | blog: 72 | $class: Statocles::App::Blog 73 | $args: 74 | store: site/blog 75 | url_root: /blog 76 | data: 77 | google_analytics_id: 'UA-61295159-4' 78 | -------------------------------------------------------------------------------- /site/blog/2015/01/11/initial-release.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | author: preaction 3 | last_modified: 2015-01-11 21:47:13 4 | tags: release 5 | title: Initial Release 6 | --- 7 | This is the first release with Yertl's new website! Yertl is an ETL (Extract, 8 | Transform, Load) for the shell, with tools designed to be quickly assembled 9 | together into useful ETL processes. 10 | 11 | This release includes: 12 | 13 | * Convert to and from YAML, JSON, and CSV (yto and yfrom) 14 | * Read/write to/from SQL databases (ysql) 15 | * Simple mask-style filtering (ymask) 16 | * The beginning of a rich transform language (yq) 17 | 18 | Future plans include: 19 | 20 | * Job distribution using ZeroMQ send/receive 21 | * Perl API for adding Yertl workflows to larger Perl scripts 22 | * API for ORMs (DBIx::Class) 23 | * API for document and key/value stores (MongoDB, Redis, Memcached) 24 | * API for downloading and scraping data from websites (curl, Web::Query, Mojo::DOM) 25 | * More options for filtering and transforming data (LINQ, XPath) 26 | 27 | See [the Github feature tickets](https://github.com/preaction/ETL-Yertl/labels/feature) 28 | for more future plans. 29 | -------------------------------------------------------------------------------- /site/blog/2015/01/11/release-v0.017.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | author: preaction 3 | last_modified: 2015-01-11 23:33:47 4 | tags: release 5 | title: Release v0.017 6 | --- 7 | 8 | This release fixes a bunch of issues with `ysql`, and allows displaying of database 9 | configuration. 10 | 11 | Full changelog below: 12 | 13 | --- 14 | 15 | * [remove docs about trim option](https://github.com/preaction/ETL-Yertl/commit/bec7ae0df7a91d8c8b285e26209692aa5212fa9d) 16 | * [add validation for database driver](https://github.com/preaction/ETL-Yertl/commit/2cac631f018e2ba12efeeb712f24d7ec017753d8) ([#76](https://github.com/preaction/ETL-Yertl/issues/76)) 17 | * [add drivers ysql command to list database drivers](https://github.com/preaction/ETL-Yertl/commit/62a9176890b641d26ec843bb157304e7b2476bf5) ([#75](https://github.com/preaction/ETL-Yertl/issues/75)) 18 | * [document new ysql config read commands](https://github.com/preaction/ETL-Yertl/commit/2b801e0238f510df3c4849f47620cf9d8d9ee571) 19 | * [show error if a database key does not exist](https://github.com/preaction/ETL-Yertl/commit/b0552cc674de31341dfe0d2da9643d4bfce6a689) ([#74](https://github.com/preaction/ETL-Yertl/issues/74)) 20 | * [add read config commands to ysql](https://github.com/preaction/ETL-Yertl/commit/90850e278ff311233a7ade276b7b60b2c2381767) ([#73](https://github.com/preaction/ETL-Yertl/issues/73)) 21 | * [add documentation for ymask](https://github.com/preaction/ETL-Yertl/commit/0b9a46c2d14cc0fc7fa5c9fb5affb93bf859cf64) ([#53](https://github.com/preaction/ETL-Yertl/issues/53)) 22 | * [make DBI completely optional](https://github.com/preaction/ETL-Yertl/commit/cb42b499ef1755eb8798a37c7d757c24e1f8c53b) ([#77](https://github.com/preaction/ETL-Yertl/issues/77)) 23 | * [add cookbook with embedded json recipe](https://github.com/preaction/ETL-Yertl/commit/aff2829abd5c7d6c328fc8c737a7164b0735c5c1) ([#82](https://github.com/preaction/ETL-Yertl/issues/82)) 24 | -------------------------------------------------------------------------------- /site/blog/2015/01/21/release-v0.019.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | last_modified: 2015-01-21 22:06:32 3 | tags: release 4 | title: Release v0.019 5 | --- 6 | 7 | This release fixes a few bugs and inconsistencies in the `ysql` command, in 8 | anticipation of a blog post explaining how it works. 9 | 10 | We removed the "write" command in favor of making the "query" command accept 11 | input on `STDIN` and handle placeholders. So now you can run "SELECT" queries 12 | for each document as it comes in, perhaps to look up users as something 13 | else write's YAML with their ID in it. 14 | 15 | Full changelog below: 16 | 17 | --- 18 | 19 | * [add missing ABSTRACTs to Command modules](https://github.com/preaction/ETL-Yertl/commit/f8c9a102a691037659ad5a89fe615f076d7984fc) 20 | * [add error checking when preparing or executing SQL](https://github.com/preaction/ETL-Yertl/commit/5a2783d5fafa6473bf422308cec6763c7375902d) ([#92](https://github.com/preaction/ETL-Yertl/issues/92)) 21 | * [allow using "--dsn" to edit config](https://github.com/preaction/ETL-Yertl/commit/749060f26b576947ad290a8b9ec8683637f16165) ([#91](https://github.com/preaction/ETL-Yertl/issues/91)) 22 | * [combine write and query in ysql](https://github.com/preaction/ETL-Yertl/commit/9ff860bb6a96d963a11e1126fe364fd8bf158900) 23 | * [fix test failure on DBD::SQLite 1.33](https://github.com/preaction/ETL-Yertl/commit/42845367e54f3402fcb61de91c8b6d681e6fb7ec) ([#86](https://github.com/preaction/ETL-Yertl/issues/86)) 24 | -------------------------------------------------------------------------------- /site/blog/2015/02/01/release-v0.020.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | last_modified: 2015-02-01 23:01:43 3 | tags: release 4 | title: Release v0.020 5 | --- 6 | 7 | This release adds the first initial development of `ygrok`, the Yertl program 8 | for parsing lines of plain text input into documents. 9 | 10 | `ygrok` is perfect for parsing log files and command output. By using simple 11 | patterns or named regex captures, we can parse a line of input into a document, 12 | which we can then write to a database using `ysql`, or convert to CSV or JSON 13 | with `yto`. 14 | 15 | Lots more development on `ygrok` to come! 16 | 17 | Full changelog below: 18 | 19 | --- 20 | 21 | * [add short descriptions of all the Yertl tools](https://github.com/preaction/ETL-Yertl/commit/43ae7601fe059316e08da0c0736220435f6f8cb4) 22 | * [add ysql to main page synopsis](https://github.com/preaction/ETL-Yertl/commit/60b11d2bbcf0d3a27f3ee4bf0159307188ace100) ([#98](https://github.com/preaction/ETL-Yertl/issues/98)) 23 | * [add links to yertl homepage](https://github.com/preaction/ETL-Yertl/commit/11f5d1def364c186f4f65d8b71eff7b087b7c8d1) ([#99](https://github.com/preaction/ETL-Yertl/issues/99)) 24 | * [exclude the website from the cpan dist](https://github.com/preaction/ETL-Yertl/commit/75abbd5884a52f2d71ad6db8c77928817da76435) ([#104](https://github.com/preaction/ETL-Yertl/issues/104)) 25 | * [fix "prototype mismatch" from Parse::RecDescent](https://github.com/preaction/ETL-Yertl/commit/e1da406e1ec7b166764bed9099c5385291b7d406) ([#103](https://github.com/preaction/ETL-Yertl/issues/103)) 26 | * [add initial ygrok script](https://github.com/preaction/ETL-Yertl/commit/af37bfa9e7b28df47b15a431717c74f0b446fdb5) ([#105](https://github.com/preaction/ETL-Yertl/issues/105)) 27 | * [add dzil plugin for prereqs and compile tests](https://github.com/preaction/ETL-Yertl/commit/b18fcb7d80e657151dfa90ebac647cf5de64bd93) 28 | -------------------------------------------------------------------------------- /site/blog/2015/02/01/release-v0.021.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | last_modified: 2015-02-01 23:20:52 3 | tags: release 4 | title: Release v0.021 5 | --- 6 | 7 | A breaking change in this release: `ysql` no longer needs the word `query` to 8 | run a query. With the earlier removal of the `write` command, the `query` 9 | command was the only way to run a query. Instead of requiring the additional 10 | command, now we can save a little bit of typing. 11 | 12 | This release has plenty of additions to the `ygrok` script, allowing it to 13 | easily parse logs from Apache HTTPD and Syslog, the output of `ls -l` and `ps`. 14 | There's also a new `-l` flag to allow loose matching of input lines. 15 | 16 | This release also allows you to add your own `ygrok` patterns. In addition to 17 | adding your own custom patterns, you can list all the existing patterns by 18 | doing `ygrok --pattern`. See [the ygrok documentation](/pod/ygrok) for more 19 | details. 20 | 21 | With all the possibilities, `ygrok` patterns can now be organized into 22 | namespaces. This will hopefully make it easier to manage as we add more 23 | patterns to the Yertl core. 24 | 25 | Full changelog below: 26 | 27 | --- 28 | 29 | * [add --loose flag to ygrok to match partial lines](https://github.com/preaction/ETL-Yertl/commit/b1302b5d17781c6fe17fcd6b6ecccab4011b2409) ([#120](https://github.com/preaction/ETL-Yertl/issues/120)) 30 | * [add ygrok patterns for ps, ps u, and ps x](https://github.com/preaction/ETL-Yertl/commit/11e99aa926db9b1b4aa657940103e7371db86aa4) 31 | * [add ygrok pattern for POSIX `ls -l`](https://github.com/preaction/ETL-Yertl/commit/d086c452ecaed0e281ad5fe89f8d6c8d5353d271) ([#116](https://github.com/preaction/ETL-Yertl/issues/116)) 32 | * [add ygrok to the main page synopsis](https://github.com/preaction/ETL-Yertl/commit/97369b3847b8cd66a677d0b3de1de1b88dd6e5e3) ([#112](https://github.com/preaction/ETL-Yertl/issues/112)) 33 | * [add syslog pattern to ygrok](https://github.com/preaction/ETL-Yertl/commit/eae9af551a6273579e13c6cb562e73c6ed95764f) ([#108](https://github.com/preaction/ETL-Yertl/issues/108)) 34 | * [allow adding, editing, and listing ygrok patterns](https://github.com/preaction/ETL-Yertl/commit/ab84ebb175099d1727ba3bd5c2cb09c075235c0e) ([#107](https://github.com/preaction/ETL-Yertl/issues/107)) 35 | * [fix multiply-nested patterns](https://github.com/preaction/ETL-Yertl/commit/e05f872c571cee75521e1fa1b52aee3b5c133d7b) 36 | * [document all existing ygrok patterns](https://github.com/preaction/ETL-Yertl/commit/282433dba939c0f4f66e06888c59c370f1dc42ee) ([#111](https://github.com/preaction/ETL-Yertl/issues/111)) 37 | * [add pattern categories to ygrok](https://github.com/preaction/ETL-Yertl/commit/ca49f9b0e2951c90875bafc186ffe2bde9fc49ac) ([#109](https://github.com/preaction/ETL-Yertl/issues/109)) 38 | * [add ysql help guide](https://github.com/preaction/ETL-Yertl/commit/beab13b66ce946f638dece748127917718f278ef) 39 | * [remove need for "query" when using ysql](https://github.com/preaction/ETL-Yertl/commit/970b3f477c2da5e201fd42f11b2dea04320b1170) ([#119](https://github.com/preaction/ETL-Yertl/issues/119)) 40 | * [add combined log format pattern for grok](https://github.com/preaction/ETL-Yertl/commit/e2d390c4fa8c25c055195ad4be3b4a980f102c0c) 41 | * [allow recursive grok patterns](https://github.com/preaction/ETL-Yertl/commit/295305fb31e73584a7b0113494ace2ab63af571e) 42 | * [add ygrok patterns to parse http common log format](https://github.com/preaction/ETL-Yertl/commit/ba5ba63f6a183845ab095d30b806be35c692b9b4) 43 | * [fix warnings when an unknown grok pattern is used](https://github.com/preaction/ETL-Yertl/commit/f2b146afe89b725dfcaa3dec493f892d3e566c3f) 44 | * [fix spurious contributor](https://github.com/preaction/ETL-Yertl/commit/aa22a8a0b876c2796e38de48570f034a36ec0413) ([#106](https://github.com/preaction/ETL-Yertl/issues/106)) 45 | -------------------------------------------------------------------------------- /site/blog/2015/02/01/release-v0.022.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | last_modified: 2015-02-01 23:51:48 3 | tags: release 4 | title: Release v0.022 5 | --- 6 | 7 | A quick release to fix up some incorrect documentation. 8 | 9 | Full changelog below: 10 | 11 | --- 12 | 13 | * [add guides to the site main page](https://github.com/preaction/ETL-Yertl/commit/79e70dc98c5a71d8b9daa8c4d55c44e0b88fe38a) 14 | * [fix ysql synopsis missing -- before config option](https://github.com/preaction/ETL-Yertl/commit/e031c6b4a59f9bb545ffae10fc91c2ff78bdb21e) 15 | * [add site intro and fix links to internal docs](https://github.com/preaction/ETL-Yertl/commit/f4389b5fc87aee971b81f08b99b82d75927789d0) 16 | * [fix link to Yertl on metacpan.org](https://github.com/preaction/ETL-Yertl/commit/e0472dc184c687ac9f2b2487208d2ac3edb8ef21) 17 | * [update statocles default theme](https://github.com/preaction/ETL-Yertl/commit/0d3f794188deae46a406a1ec3bf00f5ffddc2b82) 18 | -------------------------------------------------------------------------------- /site/blog/2015/02/26/release-v0.023/index.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | last_modified: 2015-02-26 12:43:58 3 | tags: release 4 | title: Release v0.023 5 | --- 6 | 7 | A quick release to fix a bug in `yq` that would cause a confusing error. Since `yq` 8 | was using YAML exclusively, but other commands were using other YAML modules, there 9 | was a possibility for one command to write output that another command could not 10 | parse. 11 | 12 | Now this is fixed and `yq` uses the same formatting routines as everything else. 13 | 14 | Full changelog below: 15 | 16 | --- 17 | 18 | * [make yq use formatter modules](https://github.com/preaction/ETL-Yertl/commit/966c355fa059af8b3b23fcfaf7ec6034b19534c1) 19 | -------------------------------------------------------------------------------- /site/blog/2015/05/31/release-v0-024/index.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | tags: 3 | - release 4 | title: Release v0.024 5 | --- 6 | 7 | A small release to fix the `ps` grok patterns for GNU procutils and to add 8 | the `yq -x` option to make it a lot easier to pipe the output of yq into 9 | another program. 10 | 11 | Full changelog below 12 | 13 | --- 14 | 15 | * [add xargs mode to yq](https://github.com/preaction/ETL-Yertl/commit/3eb42192089b91e545ac57dbc0aa76b84d0f1b28) ([#52](https://github.com/preaction/ETL-Yertl/issues/52)) 16 | * [remove ModuleBuild to prevent toolchain confusion](https://github.com/preaction/ETL-Yertl/commit/eb41b737f2f14dcaf5f0058be4e0528c75f0dc2f) 17 | * [fix ps grok patterns to work on RHEL 5](https://github.com/preaction/ETL-Yertl/commit/8295b200d38d5fc01c30afc5bc785405684db010) ([#127](https://github.com/preaction/ETL-Yertl/issues/127)) 18 | -------------------------------------------------------------------------------- /site/blog/2015/08/23/release-v0-025/index.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | tags: 3 | - release 4 | title: Release v0.025 5 | --- 6 | 7 | Some minor bugfixes in this release. One that would make Yertl 8 | fail its tests (and therefore fail to install). Two that caused some 9 | very confusing error messages to be printed when using 10 | [ysql](/pod/ysql). And a final one that made it hard to use the 11 | HTTP log pattern with [ygrok](/pod/ygrok). 12 | 13 | Full changelog below... 14 | 15 | --- 16 | 17 | * [fix SQL error using --dsn printing error twice](https://github.com/preaction/ETL-Yertl/commit/72ddcae4449deb0eac3aa38b59eee6d9195e82dc) 18 | * [handle error in connect with useful error message](https://github.com/preaction/ETL-Yertl/commit/0218234274315114acda42691ce81012f787bab6) ([#131](https://github.com/preaction/ETL-Yertl/issues/131)) 19 | * [update Regexp::Common to 2013031301](https://github.com/preaction/ETL-Yertl/commit/1fe4dce56b49c2675745701dd10d42e4b1e0375c) ([#124](https://github.com/preaction/ETL-Yertl/issues/124)) 20 | * [make http log patterns more lenient](https://github.com/preaction/ETL-Yertl/commit/29dded382f2a410b0cd036c0b78c513fc8111de1) ([#129](https://github.com/preaction/ETL-Yertl/issues/129)) 21 | -------------------------------------------------------------------------------- /site/blog/2015/11/24/release-v0-026/index.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | tags: release 3 | title: Release v0.026 4 | --- 5 | 6 | This release adds some SQL helper options to [the ysql 7 | command](/pod/ysql): 8 | 9 | * `--select ` creates a simple `SELECT * FROM
` query 10 | * Add `--where ` and `--order-by ` to modify your select 11 | query 12 | * `--insert
` creates a simple `INSERT INTO
` query with 13 | the documents on `STDIN` 14 | 15 | Additionally, a new flag (`-e` or `--edit`) lets you edit saved SQL 16 | queries in your text editor. This is useful if you've got a long query 17 | with a bunch of joins and don't want to mess about on the command line. 18 | 19 | Full changelog below... 20 | 21 | --- 22 | 23 | * [add --order-by helper for ysql](https://github.com/preaction/ETL-Yertl/commit/bdf3fc77b2b254b034ddb648e763ddc40a382cde) ([#136](https://github.com/preaction/ETL-Yertl/issues/136)) 24 | * [add sortable field to ysql test](https://github.com/preaction/ETL-Yertl/commit/5cd76f70adfa6d424e0dccc7eee2d2f623c52a94) 25 | * [add --where helper for select queries](https://github.com/preaction/ETL-Yertl/commit/965f4591b7dc615348bfa8429aed5a19679f165f) ([#136](https://github.com/preaction/ETL-Yertl/issues/136)) 26 | * [add --select and --insert query helpers to ysql](https://github.com/preaction/ETL-Yertl/commit/41dcbf6f59c298f96577386387522e643a46b8cf) ([#136](https://github.com/preaction/ETL-Yertl/issues/136)) 27 | * [add flag to edit SQL query in text editor](https://github.com/preaction/ETL-Yertl/commit/c3d8f1ebc57d2c471f2d0ee907d59cab74fedade) ([#135](https://github.com/preaction/ETL-Yertl/issues/135)) 28 | * [update docs to new website at preaction.me](https://github.com/preaction/ETL-Yertl/commit/c367d70fe8fe0d2e69e73c66f9cc31b98b3b145e) 29 | -------------------------------------------------------------------------------- /site/blog/2015/11/28/release-v0-027/index.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | tags: release 3 | title: Release v0.027 4 | --- 5 | 6 | Only some minor fixes, but one bugfix that could cause the test to fail. 7 | 8 | * Updated the version of Path::Tiny we rely on to fix a potential test 9 | failure. 10 | * Add the `--delete
` option to [the ysql utility](/pod/ysql). 11 | This allows you to delete from a table, optionally adding a where 12 | clause option (`--where `). 13 | * Add IPv4 and IPv6 patterns to [the ygrok utility](/pod/ygrok). These 14 | were already provided by [the Regexp::Common 15 | module](http://metacpan.org/pod/Regexp::Common), so best to use the 16 | tested regular expressions 17 | 18 | Full changelog below. 19 | 20 | --- 21 | 22 | * [use ipv4 and ipv6 patterns from Regexp::Common](https://github.com/preaction/ETL-Yertl/commit/0e2d7f60d7f3994945654df6852708b085aed8b5) 23 | * [update Path::Tiny version to fix File::Path issue](https://github.com/preaction/ETL-Yertl/commit/f2b55f5560483161417cab514d844b1fb86f87d6) ([#137](https://github.com/preaction/ETL-Yertl/issues/137)) 24 | * [expand sql helpers docs to link related helpers](https://github.com/preaction/ETL-Yertl/commit/29003751b447d663bcc2f7dc800e25ff141b4580) ([#136](https://github.com/preaction/ETL-Yertl/issues/136)) 25 | * [add the --delete helper option to ysql](https://github.com/preaction/ETL-Yertl/commit/1f6597d9640463e2b613b560f2c57e5317b77f2a) ([#136](https://github.com/preaction/ETL-Yertl/issues/136)) 26 | -------------------------------------------------------------------------------- /site/blog/2016/10/09/release-v0-028/index.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | tags: release 3 | title: Release v0.028 4 | --- 5 | 6 | These changes have been pending for quite a while, but they're finally 7 | here! Though it may not show, I actually love this project, and every 8 | excuse someone gives me to work on it I take with both hands. 9 | 10 | In this release: 11 | 12 | ## Added 13 | 14 | * The YAML::Tiny module is now officially supported 15 | * The YERTL_FORMAT environment variable will set the default format 16 | that Yertl uses to communicate. Set it to `json` for Yertl to 17 | seamlessly interoperate with other JSON-based tools like `jq` and 18 | `recs`. 19 | * CSV conversion (`yto`/`yfrom`) now allows setting a delimiter so 20 | we can parse colon-separated values or esoteric formats 21 | 22 | ## Fixed 23 | 24 | * `yq -x` now checks for definedness before trying to print. This 25 | silences warnings from Perl about "Uninitialized value in print". 26 | In the future, we may re-enable warnings through a `-w` or `-v` 27 | switch. 28 | * If one filter in a pipe returns empty, that empty result is 29 | propagated through further filters. This makes the output of 30 | something like `select( .foo == 1 ) | .bar` more intuitive: If 31 | a document doesn't pass the first part, the result of the second 32 | part is `empty`, not `undef` (`--- ~`). 33 | * The `ysql` helpers and placeholders are now better documented in 34 | the ysql guide. 35 | 36 | ## Other 37 | 38 | * ygrok tests are now split into more maintainable chunks based on 39 | the patterns they are testing. 40 | -------------------------------------------------------------------------------- /site/blog/2017/08/16/release-v0-029/index.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | tags: release 3 | title: Release v0.029 4 | --- 5 | 6 | In this release: 7 | 8 | ## Added 9 | 10 | * yq's filter language now allows `$.` to refer to the initial document 11 | even if the current document (`.`) is something else. This way one can 12 | traverse the document using `|` filters and still get to the original. 13 | * yq's filter language now allows assigning values to filters, so you 14 | can modify documents in-place. 15 | * yq's filter language now has an `each()` function to iterate over 16 | hashes/arrays 17 | * yq's filter language now allows function calls on either side of 18 | binary operators (assignment operator and comparison operators). This 19 | means we can filter based on the output of a function (like find all 20 | arrays with more than 5 elements) or assign the output of a function 21 | to a place in the document (like replace an array with its length). 22 | 23 | ## Fixed 24 | 25 | * We now give better error messages when accidentally trying to insert 26 | hashrefs or arrayrefs into a column using the `ysql` `--insert` 27 | helper. SQL::Abstract does not allow this and instead gives an 28 | untrappable warning (instead of throwing an exception, which is 29 | potentially earmarked for SQL::Abstract v2) 30 | 31 | [More information on ETL::Yertl 0.029 on MetaCPAN](https://metacpan.org/release/PREACTION/ETL-Yertl-0.029) 32 | -------------------------------------------------------------------------------- /site/blog/2017/08/25/release-v0-032/index.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | status: published 3 | tags: 4 | - release 5 | title: Release v0.032 6 | --- 7 | 8 | In this release, we've added a command to read/write time series data: 9 | `yts`. It only works with [InfluxDB](http://influxdata.com) for now, but 10 | writing new interfaces to time series databases is easy! 11 | 12 | Along with the new `ysql` `--count` option, it's easy to create metrics 13 | from data in your database: 14 | 15 | # Count the number of test reports 16 | ysql cpantesters --count test_reports \\ 17 | | yts influxdb://localhost/telegraf cpantesters report_count 18 | 19 | # Count the number of processed reports 20 | ysql cpantesters --count cpanstats \\ 21 | | yts influxdb://localhost/telegraf cpantesters processed_count 22 | 23 | # Build metrics for Minion job worker 24 | ysql cpantesters --count minion_jobs \\ 25 | | yts influxdb://localhost/telegraf minion total_jobs 26 | ysql cpantesters --count minion_jobs --where 'status="inactive"' \\ 27 | | yts influxdb://localhost/telegraf minion inactive_jobs 28 | ysql cpantesters --count minion_jobs --where 'status="finished"' \\ 29 | | yts influxdb://localhost/telegraf minion finished_jobs 30 | 31 | [More information about ETL-Yertl v0.032 on MetaCPAN](http://metacpan.org/release/PREACTION/ETL-Yertl-0.032) 32 | -------------------------------------------------------------------------------- /site/blog/2017/10/18/release-v0-035/index.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | tags: release 3 | title: Release v0.035 4 | --- 5 | 6 | In this release: 7 | 8 | [More information about ETL-Yertl v0.035 on MetaCPAN](http://metacpan.org/release/PREACTION/ETL-Yertl-0.035) 9 | 10 | ## Added 11 | 12 | * Added Graphite time series adapter. Now `yts` can read/write time 13 | series from the Graphite system and related subsystems. The write 14 | function is compatible with the "line" protocol, so any databases that 15 | understand Graphite's line protocol can be written to. 16 | 17 | * Added --start and --end options to `yts` command to constrain the time 18 | series returned to a specific date/time range. 19 | 20 | ## Fixed 21 | 22 | * Removed 'Z' time zone indicator from `yts` input/output and time series 23 | adapters. Time zone does not enter in to any of the things we're doing 24 | with time series. If users need to do things with time zones, they'll 25 | have to do it themselves (poor souls). 26 | 27 | * Fixed a race condition that could cause `yts` to read the data and not 28 | write the data passed-in on `STDIN`. This requires a new dependency, 29 | IO::Interactive. 30 | 31 | * Removed dependencies on some modules to reduce the memory footprint 32 | and improve startup times: 33 | 34 | * DateTime 35 | * Moo 36 | * Type::Tiny 37 | 38 | * Removed some unused modules that are not going to be developed 39 | anymore. 40 | 41 | * Parse::RecDescent versions of the `yq` language parser. Perl 5.10 42 | regex grammars work just fine. 43 | 44 | * Fixed YAML.pm (YAML::Old) causing tests to fail. This has been removed 45 | (for now) as a supported YAML module. The default supported YAML 46 | module is now YAML::Tiny. 47 | 48 | * Fixed the `yts` time series format. Now there is only the "metric" to 49 | identify the time series, which for some time series databases may 50 | contain abstractions like "database" and "column". 51 | 52 | * Fixed nulls appearing in Graphite time series. 53 | -------------------------------------------------------------------------------- /site/blog/2017/10/25/release-v0-036/index.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | tags: release 3 | title: Release v0.036 4 | --- 5 | 6 | In this release: 7 | 8 | ## Added 9 | 10 | * Added `parse_time` function to `yq`. This function parses a date/time 11 | string into a UNIX epoch. 12 | * Added epoch timestamps as an allowed format for `yts` time series 13 | timestamps. 14 | * Add default timestamps of "now" in `yts`. This makes all time series 15 | databases the same and makes it easier to add metrics. 16 | * Added 'LINUX.PROC.LOADAVG' and 'LINUX.PROC.UPTIME' patterns to `ygrok` 17 | to parse the `/proc/loadavg` and `/proc/uptime` files on Linux 18 | systems. 19 | 20 | ## Fixed 21 | 22 | * Reduced version requirement for List::Util. This continues our quest 23 | to be installable on a core Perl 5.10 without a compiler. 24 | * Removed unused prereq (Text::Trim). 25 | 26 | ## Removed 27 | 28 | * Removed the `ymask` command and associated prereqs. This command just 29 | wasn't very useful, and depended on a module that requires a compiler. 30 | 31 | [More information about ETL-Yertl v0.036 on 32 | MetaCPAN](http://metacpan.org/release/PREACTION/ETL-Yertl-0.036) 33 | -------------------------------------------------------------------------------- /site/blog/2017/10/27/release-v0-037/index.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | tags: release 3 | title: Release v0.037 4 | --- 5 | 6 | In this release: 7 | 8 | ## Fixed 9 | 10 | * Fixed test that depended on YAML.pm (which we removed as 11 | a dependency). Now the test uses YAML::Tiny instead. Thanks 12 | [@ltriant](http://github.com/ltriant)! [\[Github 13 | #155\]](http://github.com/preaction/ETL-Yertl/issues/155) 14 | 15 | [More information about ETL-Yertl v0.037 on 16 | MetaCPAN](http://metacpan.org/release/PREACTION/ETL-Yertl-0.037) 17 | -------------------------------------------------------------------------------- /site/index.markdown: -------------------------------------------------------------------------------- 1 | --- 2 | author: preaction 3 | title: Yertl -- ETL with a Shell 4 | --- 5 | 6 | # Yertl - ETL with a Shell 7 | 8 | Yertl is an ETL ([Extract, Transform, 9 | Load](https://en.wikipedia.org/wiki/Extract,_transform,_load)) system 10 | for working with structured data with a command-line. 11 | 12 | ## Getting Started 13 | 14 | Install the latest version of Yertl from CPAN. 15 | 16 | cpan ETL::Yertl 17 | 18 | ## Documentation 19 | 20 | Check out the documentation for Yertl: 21 | 22 | ### Guides 23 | 24 | Completely new to Yertl? Start here with the guides: 25 | 26 | * [ysql - Work with SQL databases](/pod/ETL/Yertl/Help/ysql) 27 | 28 | ### Input/Output 29 | 30 | * [yfrom - Convert formatted data (JSON, CSV) to a Yertl feed (YAML)](/pod/yfrom) 31 | * [yto - Convert Yertl feed (YAML) to an output format (JSON, CSV)](/pod/yto) 32 | * [ygrok - Parse lines of plain text into documents](/pod/ygrok) 33 | * [ysql - Read/write to/from a SQL database](/pod/ysql) 34 | 35 | ### Filter/Transform 36 | 37 | * [yq - A transform mini-language](/pod/yq) 38 | 39 | ## Other Resources 40 | 41 | ### Filter/Transform 42 | 43 | * [jq - A transform mini-language for JSON](http://stedolan.github.io/jq/) 44 | -------------------------------------------------------------------------------- /site/release-blog.pl: -------------------------------------------------------------------------------- 1 | package release_blog; 2 | 3 | use strict; 4 | use warnings; 5 | use v5.10; 6 | use CPAN::Changes; 7 | use Getopt::Long qw( GetOptionsFromArray ); 8 | use Pod::Usage::Return; 9 | 10 | my $GITHUB_ROOT = 'https://github.com/preaction/ETL-Yertl'; 11 | my $DIST = 'ETL-Yertl'; 12 | my $CHANGES_FILE = 'CHANGES'; 13 | my $NEXT_TOKEN = qr/\{\{\s*\$NEXT\s*\}\}/; 14 | 15 | sub main { 16 | my ( $class, @args ) = @_; 17 | 18 | my %opt; 19 | GetOptionsFromArray( \@args, \%opt, 20 | 'help|h', 21 | ); 22 | return pod2usage(0) if $opt{help}; 23 | 24 | my $changes = CPAN::Changes->load( 25 | $CHANGES_FILE, 26 | next_token => $NEXT_TOKEN, 27 | ); 28 | 29 | my $release; 30 | if ( my $version = shift @args ) { 31 | $release = $changes->release( $version ) || die "Could not find version $version in $CHANGES_FILE\n"; 32 | } 33 | else { 34 | my @releases = $changes->releases; 35 | if ( $releases[-1]->version !~ $NEXT_TOKEN ) { 36 | $release = $releases[-1]; 37 | } 38 | else { 39 | $release = $releases[-2]; 40 | } 41 | } 42 | 43 | my $version = $release->version; 44 | 45 | say "---"; 46 | say "title: Release v$version"; 47 | say "tags: release"; 48 | say "---"; 49 | say ""; 50 | say "In this release:"; 51 | say ""; 52 | 53 | for my $group ( $release->groups ) { 54 | say "## $group"; 55 | say ""; 56 | for my $change ( @{ $release->changes( $group ) } ) { 57 | $change =~ s{[\[(]Github \#(\d+)[\])]}{[\[Github #$1\]]($GITHUB_ROOT/issues/$1)}g; 58 | $change =~ s{\@(\w+)}{[\@$1](http://github.com/$1)}g; 59 | say "* " . $change; 60 | } 61 | say ""; 62 | } 63 | 64 | say "[More information about $DIST v$version on MetaCPAN](http://metacpan.org/release/PREACTION/$DIST-$version)"; 65 | 66 | return 0; 67 | } 68 | 69 | exit release_blog->main( @ARGV ) unless caller; 70 | 71 | 1; 72 | __END__ 73 | 74 | =head1 NAME 75 | 76 | release-blog.pl - Prepare a release blog entry for this project 77 | 78 | =head1 SYNOPSIS 79 | 80 | release-blog.pl [] 81 | release-blog.pl --help|-h 82 | 83 | =head1 DESCRIPTION 84 | 85 | This script prepares the list of changes from the CHANGES file and adds 86 | links to any tickets referenced in the commit. 87 | 88 | =head1 ARGUMENTS 89 | 90 | =head2 version 91 | 92 | Optional. The version to use. Defaults to the latest release. 93 | 94 | =head1 OPTIONS 95 | 96 | =head2 --help|-h 97 | 98 | See the help for this command. 99 | 100 | -------------------------------------------------------------------------------- /site/theme/blog/index.atom.ep: -------------------------------------------------------------------------------- 1 | 2 | 3 | <%= $site->url( $self->links( 'alternate' )->href ) %> 4 | <%= $site->title %> 5 | <%= $self->date->strftime('%Y-%m-%dT%H:%M:%SZ') %> 6 | 7 | 8 | Statocles 9 | % for my $p ( @$pages ) { 10 | 11 | <%= $site->url( $p->path ) %> 12 | <%== $p->title %> 13 | % if ( $p->author ) { 14 | <%= $p->author %> 15 | % } 16 | 17 | sections; 19 | <%= $sections[0] %> 20 | % if ( $p->links( 'crosspost' ) ) { 21 | 31 | % } 32 | % else { 33 |

Continue reading...

34 | % } 35 |

Tags: 36 | % for my $tag ( $p->tags ) { 37 | <%= $tag->text %> 38 | % } 39 |

40 | ]]> 41 | <%= $p->date->strftime('%Y-%m-%dT%H:%M:%SZ') %> 42 | % for my $t ( $p->tags ) { 43 | 44 | % } 45 | 46 | % } 47 | 48 | -------------------------------------------------------------------------------- /site/theme/blog/index.html.ep: -------------------------------------------------------------------------------- 1 | 2 | % if ( my $tag_text = $self->data->{ tag_text } ) { 3 | <%= markdown $tag_text %> 4 | % } 5 | 6 | % for my $page ( @$pages ) { 7 |
8 |
9 |

<%== $page->title %>

10 | 11 |

Tags: 12 | % for my $tag ( $page->tags ) { 13 | 14 | % } 15 |

16 | 17 | 29 | 30 | % if ( $page->links( 'crosspost' ) ) { 31 |

Originally posted as: 32 | % for my $link ( $page->links( 'crosspost' ) ) { 33 | 34 | <%= $page->title %> on <%= $link->title %>. 35 | 36 | % } 37 |

38 | % } 39 |
40 | 41 | % my @sections = $page->sections; 42 | <%= $sections[0] %> 43 | 44 | % if ( @sections > 1 ) { 45 |

Continue reading <%= $page->title %>...

46 | % } 47 | 48 |
49 | % } 50 | 51 | 67 | 68 | % if ( $site->data->{disqus}{shortname} ) { 69 | 78 | % } 79 | 80 | % content tags => begin 81 | 89 | % end 90 | 91 | % if ( my @links = $self->links( 'feed' ) ) { 92 | % content feeds => begin 93 |

Feeds

94 | 103 | % end 104 | % } 105 | 106 | -------------------------------------------------------------------------------- /site/theme/blog/index.rss.ep: -------------------------------------------------------------------------------- 1 | %# RSS requires date/time in the 'C' locale as per RFC822. strftime() is one of 2 | %# the few things that actually cares about locale. 3 | % use POSIX qw( locale_h ); 4 | % my $current_locale = setlocale( LC_TIME ); 5 | % setlocale( LC_TIME, 'C' ); 6 | 7 | 8 | 9 | <%= $site->title %> 10 | <%= $site->url( $self->links( 'alternate' )->href ) %> 11 | 12 | Blog feed of <%= $site->title %> 13 | Statocles <%= $Statocles::VERSION %> 14 | % for my $p ( @$pages ) { 15 | 16 | <%== $p->title %> 17 | <%= $site->url( $p->path ) %> 18 | <%= $site->url( $p->path ) %> 19 | sections; 21 | <%= $sections[0] %> 22 | % if ( $p->links( 'crosspost' ) ) { 23 | 33 | % } 34 | % else { 35 |

Continue reading...

36 | % } 37 |

Tags: 38 | % for my $tag ( $p->tags ) { 39 | <%= $tag->text %> 40 | % } 41 |

42 | ]]> 43 | 44 | <%= $p->date->strftime('%a, %d %b %Y %H:%M:%S ') . sprintf q{%+05d}, $p->date->offset / 3600 %> 45 | 46 | 47 | % } 48 | 49 | 50 | % setlocale( LC_TIME, $current_locale ); 51 | -------------------------------------------------------------------------------- /site/theme/blog/post.html.ep: -------------------------------------------------------------------------------- 1 |
2 |

<%== $self->title %>

3 |

Tags: 4 | % for my $tag ( $self->tags ) { 5 | 6 | % } 7 |

8 | 9 | 18 | 19 | % if ( $self->links( 'crosspost' ) ) { 20 |

Originally posted as: 21 | % for my $link ( $self->links( 'crosspost' ) ) { 22 | 23 | <%= $self->title %> on <%= $link->title %>. 24 | 25 | % } 26 |

27 | % } 28 | 29 |
30 | % my @sections = $self->sections; 31 | % for my $i ( 0..$#sections ) { 32 |
33 | <%= $sections[$i] %> 34 |
35 | % } 36 | 37 | % if ( $site->data->{disqus}{shortname} ) { 38 |

Comments

39 |
40 | 50 | 51 | % } 52 | 53 | % content tags => begin 54 | 62 | % end 63 | -------------------------------------------------------------------------------- /site/theme/css/statocles-default.css: -------------------------------------------------------------------------------- 1 | 2 | /************************************************ 3 | * Basic style 4 | ************************************************/ 5 | 6 | body { 7 | /* Raleway looks terrible, and in some cases is hard to read */ 8 | font-family: "HelveticaNeue", "Helvetica Neue", Helvetica, Arial, sans-serif; 9 | } 10 | 11 | h1 { 12 | font-size: 3.5em; 13 | } 14 | 15 | h2 { 16 | font-size: 2.75em; 17 | } 18 | 19 | h3 { 20 | font-size: 2.25em; 21 | } 22 | 23 | h4 { 24 | font-size: 1.75em; 25 | } 26 | 27 | h5 { 28 | font-size: 1.3em; 29 | } 30 | 31 | code { 32 | /* Only pre should set nowrap */ 33 | white-space: normal; 34 | } 35 | 36 | /* Fix

inside

  • leaving the list bullet on its own line in Firefox */ 37 | ul { 38 | list-style: circle outside; 39 | margin-left: 1.4em; 40 | } 41 | ol { 42 | list-style: decimal outside; 43 | margin-left: 1.4em; 44 | } 45 | 46 | ul ul, ul ol, ol ul, ol ol { 47 | /* Fix each successive layer of lists getting smaller and smaller. 48 | * After three layers of lists, it's too small to read well 49 | */ 50 | font-size: 100%; 51 | } 52 | 53 | .button.disabled { 54 | cursor: not-allowed; 55 | } 56 | 57 | main header h1 { 58 | border-bottom: 1px solid #045578; 59 | } 60 | 61 | main header h1 a { 62 | text-decoration: none; 63 | } 64 | 65 | header aside { 66 | margin-top: 0; 67 | font-size: 12pt; 68 | } 69 | 70 | /** Add a line when using the "Continue reading..." link */ 71 | section:target { 72 | padding-top: 0.5em; 73 | border-top: 2px dashed #777; 74 | } 75 | 76 | /** Don't allow 
     to push too much */
 77 | pre {
 78 |     max-width: 100%;
 79 |     overflow-x: scroll;
 80 | }
 81 | 
 82 | /**
 83 |  * Don't allow main content images to push things around
 84 |  */
 85 | main img {
 86 |     max-width: 100%;
 87 | }
 88 | 
 89 | a[rel=external]::after {
 90 |     content: '\f08e';
 91 |     padding-left: 0.2em;
 92 |     font-family: "FontAwesome";
 93 |     display: inline-block;
 94 |     font-size: smaller;
 95 |     vertical-align: top;
 96 | }
 97 | 
 98 | /************************************************
 99 |  * Classes
100 |  ************************************************/
101 | 
102 | ol.bare, ul.bare {
103 |     list-style-type: none;
104 |     margin-left: 0;
105 | }
106 | 
107 | ol.list-inline, ul.list-inline {
108 |     margin: 0;
109 | }
110 | 
111 | ol.list-inline li, ul.list-inline li {
112 |     display: inline-block;
113 |     margin: 0;
114 | }
115 | 
116 | img.u-pull-left, img.u-pull-right {
117 |     /* Give some padding so the text isn't too close */
118 |     padding: 0.2em;
119 | }
120 | 
121 | /************************************************
122 |  * Grid
123 |  ************************************************/
124 | 
125 | .column, .columns {
126 |     margin-left: 2%;
127 |     padding-left: 1%;
128 |     padding-right: 1%;
129 | }
130 | 
131 | /************************************************
132 |  * Widgets
133 |  ************************************************/
134 | 
135 | /** Navbar widget */
136 | .navbar {
137 |     overflow: hidden;
138 |     border-bottom: 1px solid #bbb;
139 |     color: #777;
140 |     background-color: white;
141 |     margin-bottom: 1em;
142 | }
143 | 
144 | .navbar a {
145 |     color: #777;
146 |     text-decoration: none;
147 | }
148 | 
149 | .navbar a:hover {
150 |     color: #1EAEDB;
151 | }
152 | 
153 | .navbar .brand {
154 |     font-size: 18px;
155 |     line-height: 20px;
156 |     margin-right: 0.5em;
157 | }
158 | 
159 | .navbar ul {
160 |     list-style: none;
161 |     display: inline-block;
162 |     margin: 1em 0;
163 | }
164 | 
165 | .navbar li {
166 |     display: inline-block;
167 |     padding: 0 1em;
168 |     margin: 0;
169 | }
170 | 
171 | /** Sidebar widget */
172 | .sidebar h1 {
173 |     font-size: 2.25em;
174 | }
175 | 
176 | .sidebar h2 {
177 |     font-size: 1.75em;
178 | }
179 | 
180 | .sidebar h3 {
181 |     font-size: 1.3em;
182 | }
183 | 
184 | .sidebar h4 {
185 |     font-size: 1.1em;
186 | }
187 | 
188 | .sidebar h5 {
189 |     font-size: 1em;
190 | }
191 | 
192 | .sidebar {
193 |     font-size: 0.8em;
194 | }
195 | 
196 | /** Pager widget */
197 | ul.pager {
198 |     list-style: none;
199 | }
200 | 
201 | .pager .prev {
202 |     float: left;
203 | }
204 | 
205 | .pager .next {
206 |     float: right;
207 | }
208 | 
209 | /** Footer */
210 | footer {
211 |     margin-top: 1em;
212 |     padding-bottom: 0.5em;
213 |     font-size: 0.7em;
214 | }
215 | 
216 | footer .tagline {
217 |     font-size: smaller;
218 |     line-height: 1.3;
219 |     text-align: right;
220 | }
221 | 
222 | /** Crumbtrail widget */
223 | 
224 | .crumbtrail ul {
225 |     display: inline-block;
226 |     list-style: none;
227 |     margin: 0;
228 | }
229 | 
230 | .crumbtrail li {
231 |     display: inline-block;
232 | }
233 | 
234 | .crumbtrail li::before {
235 |     content: '::';
236 |     display: inline-block;
237 | }
238 | 
239 | .crumbtrail li:first-child::before {
240 |     content: '';
241 | }
242 | 
243 | /************************************************
244 |  * Applications
245 |  */
246 | 
247 | /** Blog */
248 | .tags {
249 |     float: right;
250 | }
251 | 
252 | 


--------------------------------------------------------------------------------
/site/theme/layout/default.html.ep:
--------------------------------------------------------------------------------
 1 | 
 2 | 
 3 |     
 4 |         
 5 |         
 6 |         
 7 |         
 8 |         
 9 |         <%== $self->title ? $self->title . ' - ' : '' %><%== $site->title %>
10 |         
11 |         % for my $link ( $self->links( 'feed' ) ) {
12 |         
13 |         % }
14 |         % if ( my $img = $site->images->{ 'icon' } ) {
15 |         
16 |         % }
17 |         %= include 'site/head_after.html.ep'
18 |         % for my $link ( $site->links( 'stylesheet' ) ) {
19 |         
20 |         % }
21 |         % for my $link ( $self->links( 'stylesheet' ) ) {
22 |         
23 |         % }
24 |         % for my $link ( $site->links( 'script' ) ) {
25 |         
26 |         % }
27 |         % for my $link ( $self->links( 'script' ) ) {
28 |         
29 |         % }
30 |     
31 |     
32 |         
    
33 |             
48 |             %= include 'site/header_after.html.ep'
49 |         
    
50 |         
    
51 |             
    
52 |                 
    
53 |                     
    
54 |                         <%= content %>
55 |                     
    
56 |                 
    
57 | 
58 |                 
    
59 |                     %= include 'site/sidebar_before.html.ep';
60 |                     %= content 'tags';
61 |                     %= content 'feeds';
62 |                 
    
63 |             
    
64 |         
    
65 |         
72 |     
73 | 
74 | 


--------------------------------------------------------------------------------
/site/theme/layout/full-width.html.ep:
--------------------------------------------------------------------------------
 1 | 
 2 | 
 3 |     
 4 |         
 5 |         
 6 |         
 7 |         
 8 |         
 9 |         <%== $self->title ? $self->title . ' - ' : '' %><%== $site->title %>
10 |         
11 |         % for my $link ( $self->links( 'feed' ) ) {
12 |         
13 |         % }
14 |         % if ( my $img = $site->images->{ 'icon' } ) {
15 |         
16 |         % }
17 |         %= include 'site/head_after.html.ep'
18 |         % for my $link ( $site->links( 'stylesheet' ) ) {
19 |         
20 |         % }
21 |         % for my $link ( $self->links( 'stylesheet' ) ) {
22 |         
23 |         % }
24 |         % for my $link ( $site->links( 'script' ) ) {
25 |         
26 |         % }
27 |         % for my $link ( $self->links( 'script' ) ) {
28 |         
29 |         % }
30 |     
31 |     
32 |         
    
33 |             
48 |             %= include 'site/header_after.html.ep'
49 |         
    
50 |         
    
51 |             
    
52 |                 
    
53 |                     
    
54 |                         <%= content %>
55 |                     
    
56 |                 
    
57 |             
    
58 |         
    
59 |         
66 |     
67 | 
68 | 


--------------------------------------------------------------------------------
/site/theme/perldoc/pod.html.ep:
--------------------------------------------------------------------------------
 1 | 
    
 2 | 
      
 3 | % for my $trail ( @{ $self->data->{ crumbtrail } } ) {
 4 |     
    • 
 5 |         % if ( $trail->{href} ) {
 6 |         <%= $trail->{text} %>
 7 |         % }
 8 |         % else {
 9 |         <%= $trail->{text} %>
10 |         % }
11 |     
      • 
12 | % }
13 | 
    
14 | (source)
15 | 
    
16 | <%= $content %>
17 | 


--------------------------------------------------------------------------------
/site/theme/perldoc/source.html.ep:
--------------------------------------------------------------------------------
1 | 
2 |     Back to documentation
3 | 
4 | 
    <%== $content %>
    
5 | 


--------------------------------------------------------------------------------
/site/theme/plugin/highlight/default.css:
--------------------------------------------------------------------------------
 1 | /*
 2 | 
 3 | Original highlight.js style (c) Ivan Sagalaev 
 4 | Modified for Statocles
 5 | 
 6 | */
 7 | 
 8 | .hljs {
 9 |   overflow-x: auto;
10 |   background: #F0F0F0;
11 | }
12 | 
13 | .hljs,
14 | .hljs-subst {
15 |   color: #444;
16 | }
17 | 
18 | .hljs-keyword,
19 | .hljs-attribute,
20 | .hljs-selector-tag,
21 | .hljs-meta-keyword,
22 | .hljs-doctag,
23 | .hljs-name {
24 |   font-weight: bold;
25 | }
26 | 
27 | .hljs-built_in,
28 | .hljs-literal,
29 | .hljs-bullet,
30 | .hljs-code,
31 | .hljs-addition {
32 |   color: #1F811F;
33 | }
34 | 
35 | .hljs-regexp,
36 | .hljs-symbol,
37 | .hljs-variable,
38 | .hljs-template-variable,
39 | .hljs-link,
40 | .hljs-selector-attr,
41 | .hljs-selector-pseudo {
42 |   color: #BC6060;
43 | }
44 | 
45 | .hljs-type,
46 | .hljs-string,
47 | .hljs-number,
48 | .hljs-selector-id,
49 | .hljs-selector-class,
50 | .hljs-quote,
51 | .hljs-template-tag,
52 | .hljs-deletion {
53 |   color: #880000;
54 | }
55 | 
56 | .hljs-title,
57 | .hljs-section {
58 |   color: #880000;
59 |   font-weight: bold;
60 | }
61 | 
62 | .hljs-comment {
63 |   color: #888888;
64 | }
65 | 
66 | .hljs-meta {
67 |   color: #2B6EA1;
68 | }
69 | 
70 | .hljs-emphasis {
71 |   font-style: italic;
72 | }
73 | 
74 | .hljs-strong {
75 |   font-weight: bold;
76 | }
77 | 


--------------------------------------------------------------------------------
/site/theme/plugin/highlight/solarized-dark.css:
--------------------------------------------------------------------------------
 1 | /*
 2 | 
 3 | Orginal Style from ethanschoonover.com/solarized (c) Jeremy Hull 
 4 | Modified for Statocles
 5 | 
 6 | */
 7 | 
 8 | .hljs {
 9 |   overflow-x: auto;
10 |   background: #002b36;
11 |   color: #839496;
12 | }
13 | 
14 | .hljs-comment,
15 | .hljs-quote {
16 |   color: #586e75;
17 | }
18 | 
19 | /* Solarized Green */
20 | .hljs-keyword,
21 | .hljs-selector-tag,
22 | .hljs-addition {
23 |   color: #859900;
24 | }
25 | 
26 | /* Solarized Cyan */
27 | .hljs-number,
28 | .hljs-string,
29 | .hljs-meta .hljs-meta-string,
30 | .hljs-literal,
31 | .hljs-doctag,
32 | .hljs-regexp {
33 |   color: #2aa198;
34 | }
35 | 
36 | /* Solarized Blue */
37 | .hljs-title,
38 | .hljs-section,
39 | .hljs-name,
40 | .hljs-selector-id,
41 | .hljs-selector-class {
42 |   color: #268bd2;
43 | }
44 | 
45 | /* Solarized Yellow */
46 | .hljs-attribute,
47 | .hljs-attr,
48 | .hljs-variable,
49 | .hljs-template-variable,
50 | .hljs-class .hljs-title,
51 | .hljs-type {
52 |   color: #b58900;
53 | }
54 | 
55 | /* Solarized Orange */
56 | .hljs-symbol,
57 | .hljs-bullet,
58 | .hljs-subst,
59 | .hljs-meta,
60 | .hljs-meta .hljs-keyword,
61 | .hljs-selector-attr,
62 | .hljs-selector-pseudo,
63 | .hljs-link {
64 |   color: #cb4b16;
65 | }
66 | 
67 | /* Solarized Red */
68 | .hljs-built_in,
69 | .hljs-deletion {
70 |   color: #dc322f;
71 | }
72 | 
73 | .hljs-formula {
74 |   background: #073642;
75 | }
76 | 
77 | .hljs-emphasis {
78 |   font-style: italic;
79 | }
80 | 
81 | .hljs-strong {
82 |   font-weight: bold;
83 | }
84 | 


--------------------------------------------------------------------------------
/site/theme/plugin/highlight/solarized-light.css:
--------------------------------------------------------------------------------
 1 | /*
 2 | 
 3 | Orginal Style from ethanschoonover.com/solarized (c) Jeremy Hull 
 4 | Modified for Statocles
 5 | 
 6 | */
 7 | 
 8 | .hljs {
 9 |   overflow-x: auto;
10 |   background: #fdf6e3;
11 |   color: #657b83;
12 | }
13 | 
14 | .hljs-comment,
15 | .hljs-quote {
16 |   color: #93a1a1;
17 | }
18 | 
19 | /* Solarized Green */
20 | .hljs-keyword,
21 | .hljs-selector-tag,
22 | .hljs-addition {
23 |   color: #859900;
24 | }
25 | 
26 | /* Solarized Cyan */
27 | .hljs-number,
28 | .hljs-string,
29 | .hljs-meta .hljs-meta-string,
30 | .hljs-literal,
31 | .hljs-doctag,
32 | .hljs-regexp {
33 |   color: #2aa198;
34 | }
35 | 
36 | /* Solarized Blue */
37 | .hljs-title,
38 | .hljs-section,
39 | .hljs-name,
40 | .hljs-selector-id,
41 | .hljs-selector-class {
42 |   color: #268bd2;
43 | }
44 | 
45 | /* Solarized Yellow */
46 | .hljs-attribute,
47 | .hljs-attr,
48 | .hljs-variable,
49 | .hljs-template-variable,
50 | .hljs-class .hljs-title,
51 | .hljs-type {
52 |   color: #b58900;
53 | }
54 | 
55 | /* Solarized Orange */
56 | .hljs-symbol,
57 | .hljs-bullet,
58 | .hljs-subst,
59 | .hljs-meta,
60 | .hljs-meta .hljs-keyword,
61 | .hljs-selector-attr,
62 | .hljs-selector-pseudo,
63 | .hljs-link {
64 |   color: #cb4b16;
65 | }
66 | 
67 | /* Solarized Red */
68 | .hljs-built_in,
69 | .hljs-deletion {
70 |   color: #dc322f;
71 | }
72 | 
73 | .hljs-formula {
74 |   background: #eee8d5;
75 | }
76 | 
77 | .hljs-emphasis {
78 |   font-style: italic;
79 | }
80 | 
81 | .hljs-strong {
82 |   font-weight: bold;
83 | }
84 | 


--------------------------------------------------------------------------------
/site/theme/site/footer.html.ep:
--------------------------------------------------------------------------------
 1 | % if ( $site->data->{google_analytics_id} ) {
 2 | 
12 | % }
13 | 


--------------------------------------------------------------------------------
/site/theme/site/head_after.html.ep:
--------------------------------------------------------------------------------
1 | 
2 | 
7 | 


--------------------------------------------------------------------------------
/site/theme/site/header_after.html.ep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/preaction/ETL-Yertl/61b1f01b203a5a9f1bf4cc6885e368a169ecae48/site/theme/site/header_after.html.ep


--------------------------------------------------------------------------------
/site/theme/site/navbar_extra.html.ep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/preaction/ETL-Yertl/61b1f01b203a5a9f1bf4cc6885e368a169ecae48/site/theme/site/navbar_extra.html.ep


--------------------------------------------------------------------------------
/site/theme/site/robots.txt.ep:
--------------------------------------------------------------------------------
1 | Sitemap: <%= $site->url( 'sitemap.xml' ) %>
2 | User-Agent: *
3 | Disallow:
4 | 
5 | 


--------------------------------------------------------------------------------
/site/theme/site/sidebar_before.html.ep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/preaction/ETL-Yertl/61b1f01b203a5a9f1bf4cc6885e368a169ecae48/site/theme/site/sidebar_before.html.ep


--------------------------------------------------------------------------------
/site/theme/site/sitemap.xml.ep:
--------------------------------------------------------------------------------
 1 | 
 2 | 
 3 | % for my $page ( @$pages ) {
 4 |     
 5 |         <%= $site->url( $page->path ) %>
 6 |         <%= $page->search_change_frequency %>
 7 |         <%= $page->search_priority %>
 8 |         <%= $page->date->strftime( '%Y-%m-%d' ) %>
 9 |     
10 | % }
11 | 
12 | 


--------------------------------------------------------------------------------
/t/adapter/graphite.t:
--------------------------------------------------------------------------------
  1 | 
  2 | =head1 DESCRIPTION
  3 | 
  4 | This test ensures that the Graphite adapter (L)
  5 | is able to read/write time series data
  6 | 
  7 | =head1 SEE ALSO
  8 | 
  9 | L
 10 | 
 11 | =cut
 12 | 
 13 | use ETL::Yertl 'Test';
 14 | use IO::Async::Loop;
 15 | use IO::Async::Test qw( wait_for testing_loop );
 16 | use Mock::MonkeyPatch;
 17 | use Future;
 18 | use HTTP::Response;
 19 | use JSON::PP qw( encode_json );
 20 | use ETL::Yertl::Adapter::graphite;
 21 | 
 22 | my $loop = IO::Async::Loop->new();
 23 | testing_loop( $loop );
 24 | 
 25 | subtest 'constructor spec' => sub {
 26 |     subtest 'success' => sub {
 27 |         my $db = ETL::Yertl::Adapter::graphite->new( 'graphite://localhost:2003' );
 28 |         is $db->{host}, 'localhost', 'host is correct';
 29 |         is $db->{write_port}, 2003, 'write_port is correct';
 30 |         is $db->{http_port}, 2003, 'http_port is correct';
 31 |     };
 32 | 
 33 |     subtest 'port defaults to 8086' => sub {
 34 |         my $db = ETL::Yertl::Adapter::graphite->new( 'graphite://localhost' );
 35 |         is $db->{host}, 'localhost', 'host is correct';
 36 |         is $db->{write_port}, 2003, 'write_port is correct';
 37 |         is $db->{http_port}, 8080, 'http_port is correct';
 38 |     };
 39 | 
 40 |     subtest 'host is required' => sub {
 41 |         dies_ok { ETL::Yertl::Adapter::graphite->new( 'graphite://:8086' ) };
 42 |     };
 43 | 
 44 | };
 45 | 
 46 | subtest 'read ts' => sub {
 47 |     my $db = ETL::Yertl::Adapter::graphite->new(
 48 |         _loop => $loop,
 49 |         host => 'localhost',
 50 |     );
 51 | 
 52 |     my $content = encode_json(
 53 |         [{
 54 |           target => "cpu_load.1m",
 55 |           datapoints => [
 56 |             [1.0, 1311836008],
 57 |             [2.0, 1311836009],
 58 |             [3.0, 1311836010],
 59 |           ]
 60 |         }]
 61 |     );
 62 |     my $mock = Mock::MonkeyPatch->patch(
 63 |         'Net::Async::HTTP::GET' => sub {
 64 |             return Future->done(
 65 |                 HTTP::Response->new(
 66 |                     200 => 'OK',
 67 |                     [
 68 |                         'Content-Length' => length $content,
 69 |                         'Content-Type' => 'text/plain',
 70 |                     ],
 71 |                     $content,
 72 |                 ),
 73 |             );
 74 |         },
 75 |     );
 76 | 
 77 |     my @points = $db->read_ts( { metric => 'cpu_load.1m' } );
 78 |     cmp_deeply \@points, [
 79 |         {
 80 |             timestamp => '2011-07-28T06:53:28',
 81 |             metric => 'cpu_load.1m',
 82 |             value => 1.0,
 83 |         },
 84 |         {
 85 |             timestamp => '2011-07-28T06:53:29',
 86 |             metric => 'cpu_load.1m',
 87 |             value => 2.0,
 88 |         },
 89 |         {
 90 |             timestamp => '2011-07-28T06:53:30',
 91 |             metric => 'cpu_load.1m',
 92 |             value => 3.0,
 93 |         },
 94 |     ];
 95 | 
 96 |     ok $mock->called, 'mock GET called';
 97 |     my $args = $mock->method_arguments;
 98 |     my $url = URI->new( $args->[0] );
 99 |     is $url->host_port, 'localhost:8080', 'host/port is correct';
100 |     cmp_deeply { $url->query_form }, {
101 |         target => 'cpu_load.1m',
102 |         format => 'json',
103 |         noNullPoints => 'true',
104 |     }, 'query params correct';
105 | };
106 | 
107 | subtest 'write ts' => sub {
108 |     my $empty = 0;
109 |     my ( $reader, $writer ) = IO::Async::OS->pipepair;
110 |     $_->blocking( 0 ) for $reader, $writer;
111 | 
112 |     my $stream = IO::Async::Stream->new(
113 |         write_handle => $writer,
114 |         on_outgoing_empty => sub { $empty++ },
115 |     );
116 |     $loop->add( $stream );
117 | 
118 |     my $db = ETL::Yertl::Adapter::graphite->new(
119 |         _loop => $loop,
120 |         host => 'localhost',
121 |         write_client => $stream,
122 |     );
123 | 
124 |     my @points = (
125 |         {
126 |             timestamp => 1483228800,
127 |             metric => 'mydb.cpu_load.5m',
128 |             value => 1.23,
129 |         },
130 |         {
131 |             timestamp => '2017-01-01T00:05:00',
132 |             metric => 'mydb.cpu_load.1m',
133 |             value => 1.26,
134 |         },
135 |         {
136 |             metric => 'mydb.cpu_load.1m',
137 |             value => 1.31,
138 |         },
139 |     );
140 | 
141 |     my $time = time;
142 |     no warnings 'once';
143 |     local *CORE::GLOBAL::time = sub { $time };
144 |     use warnings;
145 |     $db->write_ts( @points );
146 |     wait_for { $empty };
147 | 
148 |     my @lines = (
149 |         "mydb.cpu_load.5m 1.23 1483228800",
150 |         "mydb.cpu_load.1m 1.26 1483229100",
151 |         "mydb.cpu_load.1m 1.31 " . $time,
152 |         ""
153 |     );
154 |     my $buffer;
155 |     $reader->sysread( $buffer, 8192 );
156 |     is $buffer, join( "\n", @lines ), 'graphite plaintext protocol points correct';
157 | };
158 | 
159 | done_testing;
160 | 


--------------------------------------------------------------------------------
/t/adapter/influxdb.t:
--------------------------------------------------------------------------------
  1 | 
  2 | =head1 DESCRIPTION
  3 | 
  4 | This test ensures that the InfluxDB adapter (L)
  5 | is able to read/write time series data
  6 | 
  7 | =head1 SEE ALSO
  8 | 
  9 | L
 10 | 
 11 | =cut
 12 | 
 13 | use ETL::Yertl 'Test';
 14 | use IO::Async::Loop;
 15 | use IO::Async::Test;
 16 | use Mock::MonkeyPatch;
 17 | use Future;
 18 | use HTTP::Response;
 19 | use JSON::PP qw( encode_json );
 20 | use ETL::Yertl::Adapter::influxdb;
 21 | 
 22 | my $loop = IO::Async::Loop->new();
 23 | testing_loop( $loop );
 24 | 
 25 | subtest 'constructor spec' => sub {
 26 |     subtest 'success' => sub {
 27 |         my $db = ETL::Yertl::Adapter::influxdb->new( 'influxdb://localhost:8086' );
 28 |         is $db->{host}, 'localhost', 'host is correct';
 29 |         is $db->{port}, 8086, 'port is correct';
 30 |     };
 31 | 
 32 |     subtest 'port defaults to 8086' => sub {
 33 |         my $db = ETL::Yertl::Adapter::influxdb->new( 'influxdb://localhost' );
 34 |         is $db->{host}, 'localhost', 'host is correct';
 35 |         is $db->{port}, 8086, 'port is correct';
 36 |     };
 37 | 
 38 |     subtest 'host is required' => sub {
 39 |         dies_ok { ETL::Yertl::Adapter::influxdb->new( 'influxdb://:8086' ) };
 40 |     };
 41 | 
 42 | };
 43 | 
 44 | subtest 'read ts' => sub {
 45 |     my $db = ETL::Yertl::Adapter::influxdb->new(
 46 |         _loop => $loop,
 47 |         host => 'localhost',
 48 |     );
 49 | 
 50 |     my $content = encode_json(
 51 |         {
 52 |             results => [
 53 |                 {
 54 |                     statement_id => 0,
 55 |                     series => [
 56 |                         {
 57 |                             name => "cpu_load_1m",
 58 |                             columns => [qw( time value )],
 59 |                             values => [
 60 |                                 [
 61 |                                     "2017-01-01T00:00:00",
 62 |                                     1.23
 63 |                                 ],
 64 |                                 [
 65 |                                     "2017-01-01T00:00:10",
 66 |                                     1.26
 67 |                                 ]
 68 |                             ]
 69 |                         }
 70 |                     ]
 71 |                 }
 72 |             ]
 73 |         },
 74 |     );
 75 |     my $mock = Mock::MonkeyPatch->patch(
 76 |         'Net::Async::HTTP::GET' => sub {
 77 |             return Future->done(
 78 |                 HTTP::Response->new(
 79 |                     200 => 'OK',
 80 |                     [
 81 |                         'Content-Length' => length $content,
 82 |                         'Content-Type' => 'text/plain',
 83 |                     ],
 84 |                     $content,
 85 |                 ),
 86 |             );
 87 |         },
 88 |     );
 89 | 
 90 |     my @points = $db->read_ts( { metric => 'mydb.cpu_load_1m.value' } );
 91 |     cmp_deeply \@points, [
 92 |         {
 93 |             timestamp => '2017-01-01T00:00:00',
 94 |             metric => 'mydb.cpu_load_1m',
 95 |             value => 1.23,
 96 |         },
 97 |         {
 98 |             timestamp => '2017-01-01T00:00:10',
 99 |             metric => 'mydb.cpu_load_1m',
100 |             value => 1.26,
101 |         },
102 |     ];
103 | 
104 |     ok $mock->called, 'mock GET called';
105 |     my $args = $mock->method_arguments;
106 |     my $url = URI->new( $args->[0] );
107 |     is $url->host_port, 'localhost:8086', 'host/port is correct';
108 |     cmp_deeply { $url->query_form }, {
109 |         db => 'mydb',
110 |         q => 'SELECT "value" FROM "cpu_load_1m"',
111 |     }, 'query params correct';
112 | };
113 | 
114 | subtest 'write ts' => sub {
115 |     my $db = ETL::Yertl::Adapter::influxdb->new(
116 |         _loop => $loop,
117 |         host => 'localhost',
118 |     );
119 | 
120 |     my $mock = Mock::MonkeyPatch->patch(
121 |         'Net::Async::HTTP::POST' => sub {
122 |             return Future->done(
123 |                 HTTP::Response->new(
124 |                     204 => 'No Content',
125 |                     [
126 |                         'Content-Length' => 0,
127 |                     ],
128 |                 ),
129 |             );
130 |         },
131 |     );
132 | 
133 |     my @points = (
134 |         {
135 |             timestamp => 1483228800,
136 |             metric => 'mydb.cpu_load.5m',
137 |             value => 1.23,
138 |         },
139 |         {
140 |             timestamp => '2017-01-01T00:05:00',
141 |             metric => 'mydb.cpu_load.5m',
142 |             value => 1.25,
143 |         },
144 |         {
145 |             metric => 'mydb.cpu_load.1m',
146 |             value => 1.26,
147 |         },
148 |     );
149 | 
150 |     my $time = time;
151 |     no warnings 'once';
152 |     local *CORE::GLOBAL::time = sub { $time };
153 |     use warnings;
154 |     $db->write_ts( @points );
155 | 
156 |     ok $mock->called, 'mock POST called';
157 |     my $args = $mock->method_arguments;
158 |     is $args->[0], 'http://localhost:8086/write?db=mydb', 'POST URL correct';
159 |     my @lines = (
160 |         "cpu_load 5m=1.23 1483228800000000000",
161 |         "cpu_load 5m=1.25 1483229100000000000",
162 |         "cpu_load 1m=1.26 " . ( $time * 10**9 ),
163 |     );
164 |     is $args->[1], join( "\n", @lines ), 'influxdb line protocol points correct';
165 |     cmp_deeply { @{ $args }[ 2..$#$args ] },
166 |         { content_type => 'text/plain' },
167 |         'additional options are correct';
168 | };
169 | 
170 | done_testing;
171 | 


--------------------------------------------------------------------------------
/t/bin/yfrom.t:
--------------------------------------------------------------------------------
 1 | 
 2 | use ETL::Yertl 'Test';
 3 | use Capture::Tiny qw( capture );
 4 | use ETL::Yertl::FormatStream;
 5 | my $SHARE_DIR = path( __DIR__, '..', 'share' );
 6 | 
 7 | my $script = "$FindBin::Bin/../../bin/yfrom";
 8 | require $script;
 9 | $0 = $script; # So pod2usage finds the right file
10 | 
11 | subtest 'error checking' => sub {
12 |     subtest 'no arguments' => sub {
13 |         my ( $stdout, $stderr, $exit ) = capture { yfrom->main() };
14 |         isnt $exit, 0, 'error status';
15 |         like $stderr, qr{ERROR: Must give a format}, 'contains error message';
16 |     };
17 |     subtest 'unknown format' => sub {
18 |         my ( $stdout, $stderr, $exit ) = capture { yfrom->main( "thisisabadformat" ) };
19 |         isnt $exit, 0, 'error status';
20 |         like $stderr, qr{ERROR: Unknown format 'thisisabadformat'}, 'contains error message';
21 |     };
22 | };
23 | 
24 | subtest 'JSON -> DOC' => sub {
25 |     my $json_fn = $SHARE_DIR->child( json => 'test.json' );
26 |     my @expect = (
27 |         { baz => 'buzz', foo => 'bar' },
28 |         { flip => [qw( flop blip )] },
29 |         [qw( foo bar baz )],
30 |     );
31 | 
32 |     subtest 'filename' => sub {
33 |         my ( $stdout, $stderr, $exit ) = capture { yfrom->main( 'json', $json_fn ) };
34 |         is $exit, 0, 'exit 0';
35 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
36 |         cmp_deeply [ docs_from_string( $stdout ) ], \@expect;
37 |     };
38 | 
39 |     subtest 'stdin' => sub {
40 |         local *STDIN = $json_fn->openr;
41 |         my ( $stdout, $stderr, $exit ) = capture { yfrom->main( 'json' ) };
42 |         is $exit, 0, 'exit 0';
43 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
44 |         cmp_deeply [ docs_from_string( $stdout ) ], \@expect;
45 |     };
46 | };
47 | 
48 | done_testing;
49 | 


--------------------------------------------------------------------------------
/t/bin/ygrok.t:
--------------------------------------------------------------------------------
 1 | 
 2 | use ETL::Yertl 'Test';
 3 | use Capture::Tiny qw( capture );
 4 | use ETL::Yertl::Format::yaml;
 5 | my $SHARE_DIR = path( __DIR__, '..', 'share' );
 6 | 
 7 | my $script = "$FindBin::Bin/../../bin/ygrok";
 8 | require $script;
 9 | $0 = $script; # So pod2usage finds the right file
10 | 
11 | subtest 'error checking' => sub {
12 |     subtest 'no arguments' => sub {
13 |         my ( $stdout, $stderr, $exit ) = capture { ygrok->main() };
14 |         isnt $exit, 0, 'error status';
15 |         like $stderr, qr{ERROR: Must give a pattern}, 'contains error message';
16 |     };
17 | };
18 | 
19 | sub test_ygrok {
20 |     my ( $file, $pattern, $expect, $args ) = @_;
21 | 
22 |     $args ||= [];
23 | 
24 |     subtest 'filename' => sub {
25 |         my ( $stdout, $stderr, $exit ) = capture { ygrok->main( @$args, $pattern, $file ) };
26 |         is $exit, 0, 'exit 0';
27 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
28 |         my @docs = docs_from_string( $stdout );
29 |         cmp_deeply \@docs, $expect or diag explain \@docs;;
30 |     };
31 | 
32 |     subtest 'stdin' => sub {
33 |         local *STDIN = $file->openr;
34 |         my ( $stdout, $stderr, $exit ) = capture { ygrok->main( @$args, $pattern ) };
35 |         is $exit, 0, 'exit 0';
36 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
37 |         my @docs = docs_from_string( $stdout );
38 |         cmp_deeply \@docs, $expect or diag explain \@docs;;
39 |     };
40 | }
41 | 
42 | subtest 'parse lines' => sub {
43 |     my $file = $SHARE_DIR->child( lines => 'irc.txt' );
44 |     my $pattern = '%{DATE.ISO8601:timestamp} %{WORD:user}@%{NET.IPV4:ip}> %{DATA:text}';
45 |     my @expect = (
46 |         {
47 |             timestamp => '2014-01-01T00:00:00Z',
48 |             user => 'preaction',
49 |             ip => '127.0.0.1',
50 |             text => 'Hello, world!',
51 |         },
52 |         {
53 |             timestamp => '2014-01-01T00:15:24Z',
54 |             user => 'jberger',
55 |             ip => '127.0.1.1',
56 |             text => 'Hello, preaction!',
57 |         },
58 |         {
59 |             timestamp => '2014-01-01T01:14:51Z',
60 |             user => 'preaction',
61 |             ip => '127.0.0.1',
62 |             text => 'Hello, jberger!',
63 |         },
64 |     );
65 | 
66 |     test_ygrok( $file, $pattern, \@expect );
67 | 
68 |     subtest 'loose parsing' => sub {
69 |         my $file = $SHARE_DIR->child( lines => 'irc.txt' );
70 |         my $pattern = '> %{DATA:text}$';
71 |         my @expect = (
72 |             {
73 |                 text => 'Hello, world!',
74 |             },
75 |             {
76 |                 text => 'Hello, preaction!',
77 |             },
78 |             {
79 |                 text => 'Hello, jberger!',
80 |             },
81 |         );
82 | 
83 |         test_ygrok( $file, $pattern, \@expect, [ '-l' ] );
84 |     };
85 | };
86 | 
87 | done_testing;
88 | 


--------------------------------------------------------------------------------
/t/bin/yq.t:
--------------------------------------------------------------------------------
  1 | 
  2 | use ETL::Yertl 'Test';
  3 | use YAML::Tiny;
  4 | use Capture::Tiny qw( capture );
  5 | use File::Spec;
  6 | my $SHARE_DIR = path( __DIR__, '..', 'share' );
  7 | 
  8 | my $script = "$FindBin::Bin/../../bin/yq";
  9 | require $script;
 10 | $0 = $script; # So pod2usage finds the right file
 11 | 
 12 | subtest 'help options' => sub {
 13 |     my ( $out, $err, $exit ) = capture { yq::main( 'yq', '-h' ) };
 14 |     is $exit, 0, 'successfully showed the help';
 15 |     ok !$err, 'requested help is on stdout';
 16 |     like $out, qr{Usage:}, 'synopsis is included';
 17 |     like $out, qr{Arguments:}, 'arguments are included';
 18 |     like $out, qr{Options:}, 'options are included';
 19 | };
 20 | 
 21 | subtest 'must provide a filter' => sub {
 22 |     my ( $out, $err, $exit ) = capture { yq::main() };
 23 |     is $exit, 2, 'fatal error';
 24 |     ok !$out, 'errors are on stderr';
 25 |     like $err, qr{ERROR: Must give a filter};
 26 |     like $err, qr{Usage:};
 27 | };
 28 | 
 29 | subtest 'empty does not print' => sub {
 30 |     local *STDIN = $SHARE_DIR->child( yaml => 'noseperator.yaml' )->openr;
 31 |     my $filter = 'if .foo eq bar then .foo else empty';
 32 |     my ( $output, $stderr ) = capture { yq::main( 'yq', $filter ) };
 33 |     ok !$stderr, 'stderr is empty' or diag "STDERR: $stderr";
 34 |     my @got = @{ YAML::Tiny->read_string( $output ) };
 35 |     cmp_deeply \@got, [ 'bar' ] or diag explain \@got;
 36 | };
 37 | 
 38 | subtest 'single document with no --- separator' => sub {
 39 |     local *STDIN = $SHARE_DIR->child( yaml => 'noseperator.yaml' )->openr;
 40 |     my $filter = '.flip.[0]';
 41 |     my ( $output, $stderr ) = capture { yq::main( 'yq', $filter ) };
 42 |     ok !$stderr, 'stderr is empty' or diag "STDERR: $stderr";
 43 |     my @got = @{ YAML::Tiny->read_string( $output ) };
 44 |     cmp_deeply \@got, [ 'flop' ] or diag explain \@got;
 45 | };
 46 | 
 47 | subtest 'file in ARGV' => sub {
 48 |     my $file = $SHARE_DIR->child( yaml => 'foo.yaml' );
 49 |     my $filter = '.foo';
 50 |     my ( $output, $stderr ) = capture { yq::main( 'yq', $filter, "$file" ) };
 51 |     ok !$stderr, 'stderr is empty' or diag "STDERR: $stderr";
 52 |     my @got = @{ YAML::Tiny->read_string( $output ) };
 53 |     cmp_deeply \@got, [ 'bar' ] or diag explain \@got;
 54 | 
 55 |     subtest 'multiple files with no seperators' => sub {
 56 |         my $file = $SHARE_DIR->child( yaml => 'noseperator.yaml' );
 57 |         my $filter = '.foo';
 58 |         my ( $output, $stderr ) = capture { yq::main( 'yq', $filter, "$file", "$file" ) };
 59 |         ok !$stderr, 'stderr is empty' or diag "STDERR: $stderr";
 60 |         my @got = @{ YAML::Tiny->read_string( $output ) };
 61 |         cmp_deeply \@got, [ 'bar', 'bar' ] or diag explain \@got;
 62 |     };
 63 | };
 64 | 
 65 | subtest 'multiple documents print properly' => sub {
 66 |     local *STDIN = $SHARE_DIR->child( yaml => 'noseperator.yaml' )->openr;
 67 |     my $filter = '.foo, .baz';
 68 |     my ( $output, $stderr ) = capture { yq::main( 'yq', $filter ) };
 69 |     ok !$stderr, 'stderr is empty' or diag "STDERR: $stderr";
 70 |     my @got = @{ YAML::Tiny->read_string( $output ) };
 71 |     cmp_deeply \@got, [ 'bar', 'buzz' ] or diag explain \@got;
 72 | };
 73 | 
 74 | subtest 'finish() gets called' => sub {
 75 |     local *STDIN = $SHARE_DIR->child( yaml => 'group_by.yaml' )->openr;
 76 |     my $filter = 'group_by( .foo )';
 77 |     my ( $output, $stderr ) = capture { yq::main( 'yq', $filter ) };
 78 |     ok !$stderr, 'stderr is empty' or diag "STDERR: $stderr";
 79 |     my @got = docs_from_string( $output );
 80 | 
 81 |     cmp_deeply \@got, [
 82 |         {
 83 |             bar => [
 84 |                 {
 85 |                     foo => 'bar',
 86 |                     baz => 1,
 87 |                 },
 88 |                 {
 89 |                     foo => 'bar',
 90 |                     baz => 2,
 91 |                 },
 92 |             ],
 93 |             baz => [
 94 |                 {
 95 |                     foo => 'baz',
 96 |                     baz => 3,
 97 |                 },
 98 |             ],
 99 |         },
100 |     ] or diag explain \@got;
101 | };
102 | 
103 | subtest 'xargs (-x) output' => sub {
104 |     local *STDIN = $SHARE_DIR->child( yaml => 'noseperator.yaml' )->openr;
105 |     my $filter = '.foo, .baz';
106 |     my ( $output, $stderr ) = capture { yq::main( 'yq', $filter, "-x" ) };
107 |     ok !$stderr, 'stderr is empty' or diag "STDERR: $stderr";
108 |     eq_or_diff $output, "bar\nbuzz\n";
109 | 
110 |     subtest 'missing fields' => sub {
111 |         local *STDIN = $SHARE_DIR->child( yaml => 'group_by.yaml' )->openr;
112 |         my $filter = '.foo, .bar, .baz';
113 |         my ( $output, $stderr ) = capture { yq::main( 'yq', $filter, "-x" ) };
114 |         ok !$stderr, 'stderr is empty' or diag "STDERR: $stderr";
115 |         eq_or_diff $output, "bar\n1\nbar\n2\nbaz\n3\n";
116 |     };
117 | };
118 | 
119 | subtest 'version check' => sub {
120 |     local $yq::VERSION = '1.00';
121 |     my ( $output, $stderr, $exit ) = capture { yq::main( 'yq', '--version' ) };
122 |     is $exit, 0;
123 |     ok !$stderr, 'stderr is empty' or diag "STDERR: $stderr";
124 |     is $output, "yq version 1.00 (Perl $^V)\n";
125 | };
126 | 
127 | done_testing;
128 | 


--------------------------------------------------------------------------------
/t/bin/yto.t:
--------------------------------------------------------------------------------
 1 | 
 2 | use ETL::Yertl 'Test';
 3 | use Capture::Tiny qw( capture );
 4 | my $SHARE_DIR = path( __DIR__, '..', 'share' );
 5 | 
 6 | my $script = "$FindBin::Bin/../../bin/yto";
 7 | require $script;
 8 | $0 = $script; # So pod2usage finds the right file
 9 | 
10 | my $doc_fn = $SHARE_DIR->child( yaml => 'test.yaml' );
11 | 
12 | subtest 'error checking' => sub {
13 |     subtest 'no arguments' => sub {
14 |         my ( $stdout, $stderr, $exit ) = capture { yto->main() };
15 |         isnt $exit, 0, 'error status';
16 |         like $stderr, qr{ERROR: Must give a format}, 'contains error message';
17 |     };
18 |     subtest 'unknown format' => sub {
19 |         my ( $stdout, $stderr, $exit ) = capture { yto->main( "$doc_fn" ) };
20 |         isnt $exit, 0, 'error status';
21 |         like $stderr, qr{ERROR: Unknown format '$doc_fn'}, 'contains error message';
22 |     };
23 | };
24 | 
25 | subtest 'DOC -> JSON' => sub {
26 |     my @expect = docs_from_string( yaml => $doc_fn->slurp );
27 | 
28 |     subtest 'filename' => sub {
29 |         my ( $stdout, $stderr, $exit ) = capture { yto->main( 'json', $doc_fn ) };
30 |         is $exit, 0, 'exit 0';
31 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
32 |         cmp_deeply [ docs_from_string( json => $stdout ) ], \@expect;
33 |     };
34 | 
35 |     subtest 'stdin' => sub {
36 |         local *STDIN = $doc_fn->openr;
37 | 
38 |         my ( $stdout, $stderr, $exit ) = capture { yto->main( 'json' ) };
39 |         is $exit, 0, 'exit 0';
40 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
41 | 
42 |         cmp_deeply [ docs_from_string( json => $stdout ) ], \@expect;
43 |     };
44 | };
45 | 
46 | done_testing;
47 | 


--------------------------------------------------------------------------------
/t/bin/yts.t:
--------------------------------------------------------------------------------
  1 | 
  2 | use ETL::Yertl 'Test';
  3 | use Capture::Tiny qw( capture );
  4 | use Test::Lib;
  5 | use ETL::Yertl::Format::yaml;
  6 | use ETL::Yertl::Adapter::test;
  7 | my $SHARE_DIR = path( __DIR__, '..', 'share' );
  8 | 
  9 | my $script = "$FindBin::Bin/../../bin/yts";
 10 | require $script;
 11 | $0 = $script; # So pod2usage finds the right file
 12 | 
 13 | subtest 'error checking' => sub {
 14 |     subtest 'no arguments' => sub {
 15 |         my ( $stdout, $stderr, $exit ) = capture { yts->main() };
 16 |         isnt $exit, 0, 'error status';
 17 |         like $stderr, qr{ERROR: Must give a database}, 'contains error message';
 18 |     };
 19 | };
 20 | 
 21 | subtest 'read' => sub {
 22 | 
 23 |     local @ETL::Yertl::Adapter::test::READ_TS = my @ts = (
 24 |         {
 25 |             timestamp => '2017-01-01 00:00:00',
 26 |             metric => 'cpu_load_1m',
 27 |             value => 1.23,
 28 |         },
 29 |         {
 30 |             timestamp => '2017-01-01 00:01:00',
 31 |             metric => 'cpu_load_1m',
 32 |             value => 1.26,
 33 |         },
 34 |     );
 35 | 
 36 |     subtest 'read metric' => sub {
 37 |         my ( $stdout, $stderr, $exit ) = capture {
 38 |             yts->main( 'test://localhost', 'cpu_load_1m' );
 39 |         };
 40 |         is $exit, 0;
 41 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
 42 | 
 43 |         cmp_deeply [ docs_from_string( $stdout ) ], \@ts;
 44 |     };
 45 | 
 46 |     subtest 'read metric -- short' => sub {
 47 |         my ( $stdout, $stderr, $exit ) = capture {
 48 |             yts->main( 'test://localhost', 'cpu_load_1m', '--short' );
 49 |         };
 50 |         is $exit, 0;
 51 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
 52 | 
 53 |         cmp_deeply
 54 |             [ docs_from_string( $stdout ) ],
 55 |             [ { map {; $_->{timestamp}, $_->{value} } @ts } ];
 56 |     };
 57 | 
 58 |     subtest 'read metric -- start/end' => sub {
 59 |         local @ETL::Yertl::Adapter::test::LAST_READ_TS_ARGS;
 60 |         my ( $stdout, $stderr, $exit ) = capture {
 61 |             yts->main( 'test://localhost', 'cpu_load_1m', '--start', '2017-01-01', '--end', '2017-01-02' );
 62 |         };
 63 |         is $exit, 0;
 64 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
 65 | 
 66 |         cmp_deeply \@ETL::Yertl::Adapter::test::LAST_READ_TS_ARGS,
 67 |             [ {
 68 |                 metric => 'cpu_load_1m',
 69 |                 start => '2017-01-01',
 70 |                 end => '2017-01-02',
 71 |                 tags => undef,
 72 |             } ],
 73 |             'read_ts args correct'
 74 |                 or diag explain \@ETL::Yertl::Adapter::test::LAST_READ_TS_ARGS;
 75 | 
 76 |         cmp_deeply [ docs_from_string( $stdout ) ], \@ts;
 77 |     };
 78 | 
 79 |     subtest 'error: no metric' => sub {
 80 |         my ( $stdout, $stderr, $exit ) = capture { yts->main( 'test://localhost' ) };
 81 |         isnt $exit, 0, 'error status';
 82 |         like $stderr, qr{ERROR: Must give a metric}, 'contains error message';
 83 |     };
 84 | };
 85 | 
 86 | subtest 'write' => sub {
 87 | 
 88 |     my @ts = (
 89 |         {
 90 |             timestamp => '2017-01-01T00:00:00',
 91 |             metric => 'cpu_load_1m',
 92 |             value => 1.23,
 93 |         },
 94 |         {
 95 |             timestamp => '2017-01-01T00:01:00',
 96 |             metric => 'cpu_load_1m',
 97 |             value => 1.26,
 98 |         },
 99 |     );
100 | 
101 |     subtest 'write metric' => sub {
102 |         local @ETL::Yertl::Adapter::test::WRITE_TS = ();
103 |         local *STDIN = $SHARE_DIR->child( 'command', 'yts', 'write.yml' )->openr;
104 |         my ( $stdout, $stderr, $exit ) = capture {
105 |             yts->main( 'test://localhost' );
106 |         };
107 |         is $exit, 0;
108 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
109 |         ok !$stdout, 'nothing on stdout' or diag $stdout;
110 | 
111 |         cmp_deeply \@ETL::Yertl::Adapter::test::WRITE_TS, \@ts
112 |             or diag explain \@ETL::Yertl::Adapter::test::WRITE_TS;
113 |     };
114 | 
115 |     subtest 'write metric -- short' => sub {
116 |         local @ETL::Yertl::Adapter::test::WRITE_TS = ();
117 |         local *STDIN = $SHARE_DIR->child( 'command', 'yts', 'write-short.yml' )->openr;
118 |         my ( $stdout, $stderr, $exit ) = capture {
119 |             yts->main( 'test://localhost', '--short', 'cpu_load_1m' );
120 |         };
121 |         is $exit, 0;
122 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
123 | 
124 |         cmp_deeply \@ETL::Yertl::Adapter::test::WRITE_TS, \@ts;
125 |     };
126 | };
127 | 
128 | done_testing;
129 | 


--------------------------------------------------------------------------------
/t/command/ygrok/pattern/http_combined.t:
--------------------------------------------------------------------------------
  1 | 
  2 | use ETL::Yertl 'Test';
  3 | use Capture::Tiny qw( capture );
  4 | use ETL::Yertl::Command::ygrok;
  5 | my $SHARE_DIR = path( __DIR__, '..', '..', '..', 'share' );
  6 | 
  7 | sub test_ygrok {
  8 |     my ( $file, $pattern, $expect, $args ) = @_;
  9 | 
 10 |     $args ||= [];
 11 | 
 12 |     subtest 'filename' => sub {
 13 |         my ( $stdout, $stderr, $exit ) = capture {
 14 |             ETL::Yertl::Command::ygrok->main( @$args, $pattern, $file );
 15 |         };
 16 |         ok !$exit, 'nothing returned';
 17 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
 18 |         my @docs = docs_from_string( $stdout );
 19 |         cmp_deeply \@docs, $expect or diag explain \@docs;;
 20 |     };
 21 | 
 22 |     subtest 'stdin' => sub {
 23 |         local *STDIN = $file->openr;
 24 |         my ( $stdout, $stderr, $exit ) = capture {
 25 |             ETL::Yertl::Command::ygrok->main( @$args, $pattern );
 26 |         };
 27 |         ok !$exit, 'nothing returned';
 28 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
 29 |         my @docs = docs_from_string( $stdout );
 30 |         cmp_deeply \@docs, $expect or diag explain \@docs;
 31 |     };
 32 | }
 33 | 
 34 | subtest 'http combined log format' => sub {
 35 |     my $file = $SHARE_DIR->child( lines => 'http_combined_log.txt' );
 36 |     my $pattern = join " ", '%{LOG.HTTP_COMMON}',
 37 |                             '"%{URL:referer}"', '"%{DATA:user_agent}"',
 38 |                             ;
 39 | 
 40 |     my @expect = (
 41 |         {
 42 |             remote_addr => 'www.preaction.me',
 43 |             ident => 'doug',
 44 |             user => 'preaction',
 45 |             timestamp => '10/Oct/2000:13:55:36 -0700',
 46 |             method => 'GET',
 47 |             path => '/',
 48 |             http_version => '1.0',
 49 |             status => '200',
 50 |             content_length => 2326,
 51 |             referer => 'http://example.com/referer',
 52 |             user_agent => 'Mozilla/5.0 And A Lot of Other Stuff',
 53 |         },
 54 | 
 55 |         {
 56 |             remote_addr => '127.0.1.1',
 57 |             ident => '-',
 58 |             user => 'jberger',
 59 |             timestamp => '21/Nov/2001:18:02:42 -0800',
 60 |             method => 'GET',
 61 |             path => '/foo/bar/baz?fizz=no&buzz=yes',
 62 |             http_version => '1.1',
 63 |             status => '200',
 64 |             content_length => 236,
 65 |             referer => '-',
 66 |             user_agent => 'Mojolicious/5.1',
 67 |         },
 68 | 
 69 |         {
 70 |             remote_addr => '127.0.0.1',
 71 |             ident => '-',
 72 |             user => 'preaction',
 73 |             timestamp => '01/Jan/2002:13:55:36 +0500',
 74 |             method => 'POST',
 75 |             path => '/blog/edit',
 76 |             http_version => '1.1',
 77 |             status => '200',
 78 |             content_length => 326,
 79 |             referer => '/blog',
 80 |             user_agent => 'Mozilla/4.2 (Compatible; MSIE 5.0)',
 81 |         },
 82 | 
 83 |         {
 84 |             remote_addr => '127.1.0.1',
 85 |             ident => '-',
 86 |             user => 'murphy',
 87 |             timestamp => '24/Jun/2015:06:12:36 -0000',
 88 |             method => 'GET',
 89 |             path => '/NOT_FOUND',
 90 |             http_version => '1.0',
 91 |             status => '404',
 92 |             content_length => 124,
 93 |             referer => '-',
 94 |             user_agent => 'GoogleBot/1.0',
 95 |         },
 96 | 
 97 |         {
 98 |             remote_addr => 'bonzi.example.com',
 99 |             ident => '-',
100 |             user => 'morty',
101 |             timestamp => '04/Mar/2015:12:24:38 -0600',
102 |             method => 'read',
103 |             path => '/ping',
104 |             http_version => '1.16.2',
105 |             status => '200',
106 |             content_length => '-',
107 |             referer => '-',
108 |             user_agent => 'Github/1.2 Something/2.4 Other/0.0',
109 |         },
110 |     );
111 | 
112 |     test_ygrok( $file, $pattern, \@expect );
113 |     test_ygrok( $file, "%{LOG.HTTP_COMBINED}", \@expect )
114 | };
115 | 
116 | done_testing;
117 | 


--------------------------------------------------------------------------------
/t/command/ygrok/pattern/http_common.t:
--------------------------------------------------------------------------------
  1 | 
  2 | use ETL::Yertl 'Test';
  3 | use Capture::Tiny qw( capture );
  4 | use ETL::Yertl::Command::ygrok;
  5 | my $SHARE_DIR = path( __DIR__, '..', '..', '..', 'share' );
  6 | 
  7 | sub test_ygrok {
  8 |     my ( $file, $pattern, $expect, $args ) = @_;
  9 | 
 10 |     $args ||= [];
 11 | 
 12 |     subtest 'filename' => sub {
 13 |         my ( $stdout, $stderr, $exit ) = capture {
 14 |             ETL::Yertl::Command::ygrok->main( @$args, $pattern, $file );
 15 |         };
 16 |         ok !$exit, 'nothing returned';
 17 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
 18 |         my @docs = docs_from_string( $stdout );
 19 |         cmp_deeply \@docs, $expect or diag explain \@docs;;
 20 |     };
 21 | 
 22 |     subtest 'stdin' => sub {
 23 |         local *STDIN = $file->openr;
 24 |         my ( $stdout, $stderr, $exit ) = capture {
 25 |             ETL::Yertl::Command::ygrok->main( @$args, $pattern );
 26 |         };
 27 |         ok !$exit, 'nothing returned';
 28 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
 29 |         my @docs = docs_from_string( $stdout );
 30 |         cmp_deeply \@docs, $expect or diag explain \@docs;
 31 |     };
 32 | }
 33 | 
 34 | subtest 'http common log format' => sub {
 35 |     my $file = $SHARE_DIR->child( lines => 'http_common_log.txt' );
 36 |     my $pattern = join " ", '%{NET.HOSTNAME:remote_addr}', '%{OS.USER:ident}', '%{OS.USER:user}',
 37 |                             '\[%{DATE.HTTP:timestamp}]',
 38 |                             '"%{WORD:method} %{URL.PATH:path} [^/]+/%{VERSION:http_version}"',
 39 |                             '%{INT:status}', '(?\d+|-)',
 40 |                             ;
 41 | 
 42 |     my @expect = (
 43 |         {
 44 |             remote_addr => 'www.preaction.me',
 45 |             ident => 'doug',
 46 |             user => 'preaction',
 47 |             timestamp => '10/Oct/2000:13:55:36 -0700',
 48 |             method => 'GET',
 49 |             path => '/',
 50 |             http_version => '1.0',
 51 |             status => '200',
 52 |             content_length => 2326,
 53 |         },
 54 | 
 55 |         {
 56 |             remote_addr => '127.0.1.1',
 57 |             ident => '-',
 58 |             user => 'jberger',
 59 |             timestamp => '21/Nov/2001:18:02:42 -0800',
 60 |             method => 'GET',
 61 |             path => '/foo/bar/baz?fizz=no&buzz=yes',
 62 |             http_version => '1.1',
 63 |             status => '200',
 64 |             content_length => 236,
 65 |         },
 66 | 
 67 |         {
 68 |             remote_addr => '127.0.0.1',
 69 |             ident => '-',
 70 |             user => 'preaction',
 71 |             timestamp => '01/Jan/2002:13:55:36 +0500',
 72 |             method => 'POST',
 73 |             path => '/blog/edit',
 74 |             http_version => '1.1',
 75 |             status => '200',
 76 |             content_length => 326,
 77 |         },
 78 | 
 79 |         {
 80 |             remote_addr => '127.1.0.1',
 81 |             ident => '-',
 82 |             user => 'murphy',
 83 |             timestamp => '24/Jun/2015:06:12:36 -0000',
 84 |             method => 'GET',
 85 |             path => '/NOT_FOUND',
 86 |             http_version => '1.0',
 87 |             status => '404',
 88 |             content_length => 124,
 89 |         },
 90 | 
 91 |         {
 92 |             remote_addr => 'bonzi.example.com',
 93 |             ident => '-',
 94 |             user => 'morty',
 95 |             timestamp => '04/Mar/2015:12:24:38 -0600',
 96 |             method => 'read',
 97 |             path => '/ping',
 98 |             http_version => '1.16.2',
 99 |             status => '200',
100 |             content_length => '-',
101 |         },
102 |     );
103 | 
104 |     test_ygrok( $file, $pattern, \@expect );
105 |     test_ygrok( $file, "%{LOG.HTTP_COMMON}", \@expect )
106 | };
107 | 
108 | done_testing;
109 | 


--------------------------------------------------------------------------------
/t/command/ygrok/pattern/linux_proc.t:
--------------------------------------------------------------------------------
 1 | 
 2 | use ETL::Yertl 'Test';
 3 | use Capture::Tiny qw( capture );
 4 | use ETL::Yertl::Command::ygrok;
 5 | my $SHARE_DIR = path( __DIR__, '..', '..', '..', 'share' );
 6 | 
 7 | sub test_ygrok {
 8 |     my ( $file, $pattern, $expect, $args ) = @_;
 9 | 
10 |     $args ||= [];
11 | 
12 |     subtest 'filename' => sub {
13 |         my ( $stdout, $stderr, $exit ) = capture {
14 |             ETL::Yertl::Command::ygrok->main( @$args, $pattern, $file );
15 |         };
16 |         ok !$exit, 'nothing returned';
17 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
18 |         my @docs = docs_from_string( $stdout );
19 |         cmp_deeply \@docs, $expect or diag explain \@docs;;
20 |     };
21 | 
22 |     subtest 'stdin' => sub {
23 |         local *STDIN = $file->openr;
24 |         my ( $stdout, $stderr, $exit ) = capture {
25 |             ETL::Yertl::Command::ygrok->main( @$args, $pattern );
26 |         };
27 |         ok !$exit, 'nothing returned';
28 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
29 |         my @docs = docs_from_string( $stdout );
30 |         cmp_deeply \@docs, $expect or diag explain \@docs;
31 |     };
32 | }
33 | 
34 | subtest '/proc/loadavg' => sub {
35 |     my $file = $SHARE_DIR->child( lines => linux => 'proc_loadavg.txt' );
36 |     my $pattern = '%{NUM:load1m}\s+%{NUM:load5m}\s+%{NUM:load15m}\s+%{INT:running}/%{INT:total}\s+%{INT:lastpid}';
37 | 
38 |     my @expect = (
39 |         {
40 |             load1m => '0.00',
41 |             load5m => 0.01,
42 |             load15m => 0.05,
43 |             running => 1,
44 |             total => 72,
45 |             lastpid => 9774,
46 |         },
47 |     );
48 | 
49 |     test_ygrok( $file, $pattern, \@expect );
50 |     test_ygrok( $file, "%{LINUX.PROC.LOADAVG}", \@expect )
51 | };
52 | 
53 | subtest '/proc/uptime' => sub {
54 |     my $file = $SHARE_DIR->child( lines => linux => 'proc_uptime.txt' );
55 |     my $pattern = '%{NUM:uptime}\s+%{NUM:idletime}';
56 | 
57 |     my @expect = (
58 |         {
59 |             uptime => 4040.03,
60 |             idletime => '3667.00',
61 |         },
62 |     );
63 | 
64 |     test_ygrok( $file, $pattern, \@expect );
65 |     test_ygrok( $file, "%{LINUX.PROC.UPTIME}", \@expect )
66 | };
67 | done_testing;
68 | 


--------------------------------------------------------------------------------
/t/command/ygrok/pattern/ls-l.t:
--------------------------------------------------------------------------------
 1 | 
 2 | use ETL::Yertl 'Test';
 3 | use Capture::Tiny qw( capture );
 4 | use ETL::Yertl::Command::ygrok;
 5 | my $SHARE_DIR = path( __DIR__, '..', '..', '..', 'share' );
 6 | 
 7 | sub test_ygrok {
 8 |     my ( $file, $pattern, $expect, $args ) = @_;
 9 | 
10 |     $args ||= [];
11 | 
12 |     subtest 'filename' => sub {
13 |         my ( $stdout, $stderr, $exit ) = capture {
14 |             ETL::Yertl::Command::ygrok->main( @$args, $pattern, $file );
15 |         };
16 |         ok !$exit, 'nothing returned';
17 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
18 |         my @docs = docs_from_string( $stdout );
19 |         cmp_deeply \@docs, $expect or diag explain \@docs;;
20 |     };
21 | 
22 |     subtest 'stdin' => sub {
23 |         local *STDIN = $file->openr;
24 |         my ( $stdout, $stderr, $exit ) = capture {
25 |             ETL::Yertl::Command::ygrok->main( @$args, $pattern );
26 |         };
27 |         ok !$exit, 'nothing returned';
28 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
29 |         my @docs = docs_from_string( $stdout );
30 |         cmp_deeply \@docs, $expect or diag explain \@docs;
31 |     };
32 | }
33 | 
34 | subtest 'ls -l' => sub {
35 |     my $file = $SHARE_DIR->child( lines => 'ls-l.txt' );
36 |     my $pattern = join( " +",
37 |         '(?[bcdlsp-][rwxSsTt-]{9})',
38 |         '%{INT:links}',
39 |         '%{OS.USER:owner}',
40 |         '%{OS.USER:group}',
41 |         '%{INT:bytes}',
42 |         '(?%{DATE.MONTH} +\d+ +\d+(?::\d+)?)',
43 |         '%{DATA:name}',
44 |     );
45 | 
46 |     my @expect = (
47 |         {
48 |             mode => 'drwxr-xr-x',
49 |             links => 15,
50 |             owner => 'doug',
51 |             group => 'staff',
52 |             bytes => 510,
53 |             modified => 'Jan 28 19:37',
54 |             name => 'ETL-Yertl-0.021',
55 |         },
56 |         {
57 |             mode => '-rw-r--r--',
58 |             links => 1,
59 |             owner => 'doug',
60 |             group => 'staff',
61 |             bytes => 43666,
62 |             modified => 'Jan 28 19:37',
63 |             name => 'ETL-Yertl-0.021.tar.gz',
64 |         },
65 |         {
66 |             mode => 'drwxr-xr-x',
67 |             links => 9,
68 |             owner => 'doug',
69 |             group => 'staff',
70 |             bytes => 306,
71 |             modified => 'Feb  1 19:44',
72 |             name => 'bin',
73 |         },
74 |         {
75 |             mode => '-rw-r--r--',
76 |             links => 1,
77 |             owner => 'doug',
78 |             group => 'staff',
79 |             bytes => 3654,
80 |             modified => 'Feb  1  2012',
81 |             name => 'dist.ini',
82 |         },
83 |     );
84 | 
85 |     test_ygrok( $file, $pattern, \@expect );
86 |     test_ygrok( $file, "%{POSIX.LS}", \@expect )
87 | };
88 | 
89 | done_testing;
90 | 


--------------------------------------------------------------------------------
/t/command/ygrok/pattern/ps-u.t:
--------------------------------------------------------------------------------
  1 | 
  2 | use ETL::Yertl 'Test';
  3 | use Capture::Tiny qw( capture );
  4 | use ETL::Yertl::Command::ygrok;
  5 | my $SHARE_DIR = path( __DIR__, '..', '..', '..', 'share' );
  6 | 
  7 | sub test_ygrok {
  8 |     my ( $file, $pattern, $expect, $args ) = @_;
  9 | 
 10 |     $args ||= [];
 11 | 
 12 |     subtest 'filename' => sub {
 13 |         my ( $stdout, $stderr, $exit ) = capture {
 14 |             ETL::Yertl::Command::ygrok->main( @$args, $pattern, $file );
 15 |         };
 16 |         ok !$exit, 'nothing returned';
 17 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
 18 |         my @docs = docs_from_string( $stdout );
 19 |         cmp_deeply \@docs, $expect or diag explain \@docs;;
 20 |     };
 21 | 
 22 |     subtest 'stdin' => sub {
 23 |         local *STDIN = $file->openr;
 24 |         my ( $stdout, $stderr, $exit ) = capture {
 25 |             ETL::Yertl::Command::ygrok->main( @$args, $pattern );
 26 |         };
 27 |         ok !$exit, 'nothing returned';
 28 |         ok !$stderr, 'nothing on stderr' or diag $stderr;
 29 |         my @docs = docs_from_string( $stdout );
 30 |         cmp_deeply \@docs, $expect or diag explain \@docs;
 31 |     };
 32 | }
 33 | 
 34 | subtest 'ps -u' => sub {
 35 |     my $pattern = join( " +",
 36 |         '%{OS.USER:user}',
 37 |         '%{INT:pid}',
 38 |         '%{NUM:cpu}',
 39 |         '%{NUM:mem}',
 40 |         '%{INT:vsz}',
 41 |         '%{INT:rss}',
 42 |         '(?[\w?/]+)',
 43 |         '(?(?:[\w+]+))?',
 44 |         '(?[\w:]+)',
 45 |         '(?