├── .circleci └── config.yml ├── .coveralls.yml ├── .github ├── CODEOWNERS └── workflows │ └── codeql-analysis.yml ├── .gitignore ├── ChangeLog ├── Gemfile ├── README.rdoc ├── Rakefile ├── SECURITY.md ├── appveyor.yml ├── bin └── td ├── contrib └── completion │ ├── _td │ └── td-completion.bash ├── data ├── sample_apache.json └── sample_apache_gen.rb ├── dist ├── build_osx.sh ├── build_win81.sh ├── exe.rake ├── pkg.rake └── resources │ ├── exe │ ├── td │ ├── td-cmd.bat │ ├── td.bat │ └── td.iss │ └── pkg │ ├── Distribution.erb │ ├── PackageInfo.erb │ ├── postinstall │ └── td ├── java └── logging.properties ├── lib ├── td.rb └── td │ ├── command │ ├── account.rb │ ├── apikey.rb │ ├── bulk_import.rb │ ├── common.rb │ ├── connector.rb │ ├── db.rb │ ├── export.rb │ ├── help.rb │ ├── import.rb │ ├── job.rb │ ├── list.rb │ ├── options.rb │ ├── password.rb │ ├── query.rb │ ├── result.rb │ ├── runner.rb │ ├── sample.rb │ ├── sched.rb │ ├── schema.rb │ ├── server.rb │ ├── status.rb │ ├── table.rb │ ├── update.rb │ ├── user.rb │ └── workflow.rb │ ├── compact_format_yamler.rb │ ├── compat_core.rb │ ├── compat_gzip_reader.rb │ ├── config.rb │ ├── connector_config_normalizer.rb │ ├── distribution.rb │ ├── file_reader.rb │ ├── helpers.rb │ ├── updater.rb │ └── version.rb ├── spec ├── file_reader │ ├── filter_spec.rb │ ├── io_filter_spec.rb │ ├── line_reader_spec.rb │ ├── parsing_reader_spec.rb │ └── shared_context.rb ├── file_reader_spec.rb ├── spec_helper.rb └── td │ ├── command │ ├── account_spec.rb │ ├── connector_spec.rb │ ├── export_spec.rb │ ├── import_spec.rb │ ├── job_spec.rb │ ├── query_spec.rb │ ├── sched_spec.rb │ ├── schema_spec.rb │ ├── table_spec.rb │ └── workflow_spec.rb │ ├── common_spec.rb │ ├── compact_format_yamler_spec.rb │ ├── config_spec.rb │ ├── connector_config_normalizer_spec.rb │ ├── fixture │ ├── bulk_load.yml │ ├── ca.cert │ ├── server.cert │ ├── server.key │ ├── testRootCA.crt │ ├── testServer.crt │ ├── testServer.key │ └── tmp.zip │ ├── helpers_spec.rb │ ├── updater_spec.rb │ └── version_spec.rb └── td.gemspec /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | 4 | orbs: 5 | ruby: circleci/ruby@2.0.0 6 | win: circleci/windows@5.0.0 7 | 8 | 9 | commands: 10 | install_windows_requirements: 11 | description: "Install windows requirements" 12 | steps: 13 | - run: 14 | name: "Install MSYS2" 15 | command: choco install msys2 -y 16 | - run: 17 | name: "Install Ruby devkit" 18 | command: ridk install 2 3 19 | bundle-install: 20 | description: "Install dependencies" 21 | steps: 22 | - run: 23 | name: Which bundler? 24 | command: ruby -v; bundle -v 25 | - run: 26 | name: Bundle install 27 | command: bundle install 28 | run-tests: 29 | description: "Run tests" 30 | steps: 31 | - run: 32 | name: Run tests 33 | command: bundle exec rake spec SPEC_OPTS="-fd" 34 | run-tests-flow: 35 | description: "Single flow for running tests" 36 | steps: 37 | - checkout 38 | - bundle-install 39 | - run-tests 40 | 41 | 42 | jobs: 43 | 44 | ruby_27: 45 | docker: 46 | - image: cimg/ruby:2.7-browsers 47 | steps: 48 | - run-tests-flow 49 | 50 | ruby_30: 51 | docker: 52 | - image: cimg/ruby:3.0-browsers 53 | steps: 54 | - run-tests-flow 55 | 56 | ruby_31: 57 | docker: 58 | - image: cimg/ruby:3.1-browsers 59 | steps: 60 | - run-tests-flow 61 | 62 | ruby_32: 63 | docker: 64 | - image: cimg/ruby:3.2-browsers 65 | steps: 66 | - run-tests-flow 67 | 68 | win_ruby: 69 | executor: 70 | name: win/default 71 | shell: powershell.exe 72 | steps: 73 | - install_windows_requirements 74 | - run: 75 | name: "Install bundler" 76 | shell: powershell.exe 77 | command: gem install bundler 78 | - run-tests-flow 79 | 80 | 81 | workflows: 82 | tests: 83 | jobs: 84 | - ruby_27 85 | - ruby_30 86 | - ruby_31 87 | - ruby_32 88 | - win_ruby 89 | -------------------------------------------------------------------------------- /.coveralls.yml: -------------------------------------------------------------------------------- 1 | service_name: travis-ci 2 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @treasure-data/integrations 2 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '29 4 * * 5' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'ruby' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v3 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v2 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | 52 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 53 | # queries: security-extended,security-and-quality 54 | 55 | 56 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 57 | # If this step fails, then you should remove it and run the build manually (see below) 58 | - name: Autobuild 59 | uses: github/codeql-action/autobuild@v2 60 | 61 | # ℹ️ Command-line programs to run using the OS shell. 62 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 63 | 64 | # If the Autobuild fails above, remove it and uncomment the following three lines. 65 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 66 | 67 | # - run: | 68 | # echo "Run, Build Application using script" 69 | # ./location_of_script_within_repo/buildscript.sh 70 | 71 | - name: Perform CodeQL Analysis 72 | uses: github/codeql-action/analyze@v2 73 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .bundle 2 | .DS_Store 3 | bundler 4 | build/td-import-java 5 | Gemfile.lock 6 | vendor/* 7 | /pkg/ 8 | build/* 9 | coverage/ 10 | dist/resources/pkg/ruby*.pkg 11 | *~ 12 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | #gem 'td-client', :git => 'https://github.com/treasure-data/td-client-ruby.git' 4 | 5 | gemspec 6 | -------------------------------------------------------------------------------- /README.rdoc: -------------------------------------------------------------------------------- 1 | = Treasure Data command line tool 2 | {Build Status}[https://circleci.com/gh/treasure-data/td.svg?style=svg] 3 | {appveyor}[https://ci.appveyor.com/project/treasure-data/td/branch/master] 4 | {Coverage Status}[https://coveralls.io/github/treasure-data/td?branch=master] 5 | 6 | This CUI utility wraps the {Ruby Client Library td-client-ruby}[https://github.com/treasure-data/td-client-ruby] 7 | to interact with the REST API in managing databases and jobs on the Treasure Data Cloud. 8 | 9 | For more info about Treasure Data, see . 10 | 11 | For full documentation see . 12 | 13 | = Getting Started 14 | 15 | Install td command as a gem. 16 | 17 | > gem install td 18 | 19 | See help message for details. 20 | 21 | > td 22 | 23 | You need to authorize the account, before executing any other commands. 24 | 25 | > td account 26 | 27 | = Sample Workflow 28 | 29 | > td account -f # authorize an account 30 | user: k@treasure-data.com 31 | password: ********** 32 | > td database:create mydb # create a database 33 | > td table:create mydb www_access # create a table 34 | 35 | = Packaging 36 | 37 | == Mac OS X 38 | 39 | Disable RVM or rbenv and use ruby.pkg's ruby (/usr/local/td/ruby/bin/ruby). 40 | And then run following commands: 41 | 42 | $ /usr/local/td/ruby/bin/gem install bundler rubyzip 43 | $ /usr/local/td/ruby/bin/bundle install 44 | $ /usr/local/td/ruby/bin/rake pkg:build 45 | 46 | It uses https://github.com/treasure-data/ruby-osx-packager 47 | 48 | == Windows 49 | 50 | Install following binary packages: 51 | 52 | * MinGW with MSYS Basic System and using mingw-get-inst 53 | * Git for Windows, with Windows Command Prompt support 54 | * Ruby ruby-1.9.3p327 using RubyInstaller for Windows, with PATH update 55 | * Inno Setup 5 56 | 57 | Then run following commands on MinGW Shell: 58 | 59 | $ mingw-get install msys-vim 60 | $ mingw-get install msys-wget 61 | $ gem install bundler rubyzip 62 | $ bundle install # don't use "--path" option 63 | $ rake exe:build # don't use "bundle exec" 64 | 65 | == Bulk Import 66 | 67 | Some of the +td+ commands prefixed with +td+ +import+ leverages the {Java Bulk Import CLI td-import-java}[https://github.com/treasure-data/td-import-java] 68 | to process and Bulk load data in the Treasure Data Cloud. 69 | 70 | The Bulk Import CLI is downloaded automatically at the first call to any of the command that require it; the use will 71 | need internet connectivity in order to fetch the Bulk Import CLI JAR file from the 72 | {Central Maven repository}[https://repo1.maven.org/maven2/com/treasuredata/td-import/] 73 | and take advantage of these advanced features. If you need to setup a proxy, please consult this 74 | {documentation}[https://docs.treasuredata.com/display/public/INT/Legacy+Bulk+Import+Tips+and+Tricks#LegacyBulkImportTipsandTricks-UsingaProxyServer] page. 75 | 76 | The log levels and properties of the Bulk Import CLI can be configured in a +logging.properties+ file. A default 77 | configuration is provided in a file within the gem or Toolbelt folder root, in the +java/+ folder. If you wish to 78 | customize it, please make a copy of this file and store it in the: 79 | 80 | ~/.td/java folder on Mac OSX or Linux 81 | %USERPROFILE%\.td\java folder on Windows 82 | 83 | == Testing Hooks 84 | 85 | The CLI implements several hooks to enable/disable/trigger special behaviors. 86 | These hooks are expressed as environment variables and can therefore be provided in several ways: 87 | 88 | === How to Use 89 | 90 | * Unix / Linux / MacOSX 91 | * environment variable export in the shell the command is executed. The setting remains active until the shell is closed. E.g.: 92 | 93 | $ export TD_TOOLBELT_DEBUG=1 94 | 95 | * in the shell configuration file, to be active in any new shell that is opened. E.g.: add 96 | 97 | export TD_TOOLBELT_DEBUG=1 98 | 99 | to ~/.bashrc or equivalent shell configuration file. 100 | To make the setting active in the current shell, source the configuration file, e.g.: 101 | 102 | $ source ~/.bashrc 103 | 104 | * on the command line at runtime (active only for the duration of the command). E.g.: 105 | 106 | $ TD_TOOLBELT_DEBUG=1 td .... 107 | 108 | * as alias on in the current shell. The setting remains active until the shell is closed. E.g.: 109 | 110 | $ alias td='TD_TOOLBELT_DEBUG=1 td' 111 | 112 | * as alias in configuration file, to be active in any new shell that is opened. E.g.: 113 | 114 | alias td='TD_TOOLBELT_DEBUG=1 td'` 115 | 116 | to ~/.bashrc or equivalent shell configuration file. 117 | To make the setting active in the current shell, source the configuration file, e.g.: 118 | 119 | $ source ~/.bashrc 120 | 121 | * Windows 122 | * in the command prompt the command is executed. The setting remains active until the command prompt window is closed. E.g.: 123 | 124 | cmd> set TD_TOOLBELT_DEBUG=1 125 | 126 | * as a global environment variable in the system settings. It will be active for all new command prompt windows. 127 | 128 | These are the available hooks: 129 | 130 | * Enable debugging mode: 131 | 132 | $ TD_TOOLBELT_DEBUG=1 133 | 134 | * JAR auto update (enabled by default is not specified). This setting does not affect import:jar_update: 135 | * Enable: 136 | 137 | $ TD_TOOLBELT_JAR_UPDATE=1 138 | 139 | * Disable: 140 | 141 | $ TD_TOOLBELT_JAR_UPDATE=0 142 | 143 | * Specify an alternative endpoint to use updating the toolbelt (default: http://toolbelt.treasuredata.com): 144 | 145 | $ TD_TOOLBELT_UPDATE_ROOT="http://toolbelt.treasuredata.com" 146 | 147 | * Specify an alternative endpoint to use updating the JAR file (default: https://repo1.maven.org): 148 | 149 | $ TD_TOOLBELT_JARUPDATE_ROOT="https://repo1.maven.org" 150 | 151 | 152 | = Copyright 153 | 154 | Copyright:: Copyright (c) 2015 Treasure Data Inc. 155 | License:: Apache License, Version 2.0 156 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bundler' 3 | require 'zip/zip' 4 | Bundler::GemHelper.install_tasks 5 | 6 | task :default => :build 7 | 8 | # common methods for package build scripts 9 | require 'fileutils' 10 | require "erb" 11 | 12 | def version 13 | require project_root_path('lib/td/version') 14 | TreasureData::TOOLBELT_VERSION 15 | end 16 | 17 | def project_root_path(path) 18 | "#{PROJECT_ROOT}/#{path}" 19 | end 20 | 21 | PROJECT_ROOT = File.expand_path(File.dirname(__FILE__)) 22 | USE_GEMS = ["#{PROJECT_ROOT}/pkg/td-#{version}.gem"] 23 | 24 | def install_use_gems(target_dir) 25 | unless ENV['GEM_HOME'].to_s.empty? 26 | puts "**" 27 | puts "** WARNING" 28 | puts "**" 29 | puts "** GEM_HOME is already set. Created package might be broken." 30 | puts "** RVM surely breaks the package. Use rbenv instead." 31 | puts "**" 32 | end 33 | 34 | require 'rubygems/gem_runner' 35 | require 'rubygems/rdoc' # avoid `Gem.finish_resolve` raises error 36 | 37 | # system(env, cmd) doesn't work with ruby 1.8 38 | ENV['GEM_HOME'] = target_dir 39 | ENV['GEM_PATH'] = '' 40 | USE_GEMS.each {|gem| 41 | begin 42 | # this is a hack to have the dependency handling for the 'td' gem 43 | # pick up a local gem for 'td-client' so as to be able to build 44 | # and test the 'toolbelt' package without publishing the 'td-client' 45 | # gem on rubygems.com 46 | unless ENV['TD_TOOLBELT_LOCAL_CLIENT_GEM'].nil? 47 | unless File.exists? ENV['TD_TOOLBELT_LOCAL_CLIENT_GEM'] 48 | raise "Cannot find gem file with path #{ENV['TD_TOOLBELT_LOCAL_CLIENT_GEM']}" 49 | end 50 | puts "Copy local gem #{ENV['TD_TOOLBELT_LOCAL_CLIENT_GEM']} to #{Dir.pwd}" 51 | FileUtils.cp File.expand_path(ENV['TD_TOOLBELT_LOCAL_CLIENT_GEM']), Dir.pwd 52 | end 53 | Gem::GemRunner.new.run ["install", gem, "--no-document"] # Use one of two commands bellow. Gem::GemRunner on ruby 3.0.5 has some weird bug 54 | # sh "gem install \"#{gem}" --no-document" 55 | # sh "/usr/local/td/ruby/bin/gem install \"#{gem}\" --no-document" # On OSX 56 | rescue Gem::SystemExitException => e 57 | unless e.exit_code.zero? 58 | puts e, e.backtrace.join("\n") 59 | raise e 60 | end 61 | end 62 | } 63 | FileUtils.mv Dir.glob("#{target_dir}/gems/*"), target_dir 64 | FileUtils.rm_f Dir.glob("#{target_dir}/*.gem") 65 | %W(bin cache doc gems specifications build_info).each { |dir| 66 | FileUtils.remove_dir("#{target_dir}/#{dir}", true) 67 | } 68 | end 69 | 70 | def resource_path(path) 71 | project_root_path("dist/resources/#{path}") 72 | end 73 | 74 | def install_resource(resource_name, target_path, mode) 75 | FileUtils.mkdir_p File.dirname(target_path) 76 | FileUtils.cp resource_path(resource_name), target_path 77 | File.chmod(mode, target_path) 78 | end 79 | 80 | def install_erb_resource(resource_name, target_path, mode, variables) 81 | FileUtils.mkdir_p File.dirname(target_path) 82 | erb_raw = File.read resource_path(resource_name) 83 | 84 | ctx = Object.new 85 | variables.each_pair {|k,v| 86 | # ctx.define_singleton_method(k) { v } doesn't work with ruby 1.8 87 | (class<subcommand' && ret=0 99 | 100 | if (( CURRENT == 1 )); then 101 | _describe -t subcommands "td subcommands" subcommands 102 | return 103 | fi 104 | 105 | local -a _subcommand_args 106 | case "$words[1]" in 107 | account) 108 | _subcommand_args=( 109 | '(-f|--force)'{-f,--force}'[overwrite current account setting]' \ 110 | ) 111 | ;; 112 | apikey:set) 113 | _subcommand_args=( 114 | '(-f|--force)'{-f,--force}'[overwrite current account setting]' \ 115 | ) 116 | ;; 117 | bulk_import:create) 118 | _subcommand_args=( 119 | '(-g|--org)'{-g,--org}'[create the bukl import session under this organization]' \ 120 | ) 121 | ;; 122 | bulk_import:upload_parts) 123 | _subcommand_args=( 124 | '(-P|--prefix)'{-P,--prefix}'[add prefix to parts name]' \ 125 | '(-s|--use-suffix)'{-s,--use-suffix}'[use COUNT number of . (dots) in the source file name to the parts name]' \ 126 | '--auto-perform[perform bulk import job automatically]' \ 127 | '--parallel[perform uploading in parallel (default: 2; max 8)]' \ 128 | ) 129 | ;; 130 | bulk_import:perform) 131 | _subcommand_args=( 132 | '(-w|--wait)'{-w,--wait}'[wait for finishing the job]' \ 133 | '(-f|--force)'{-f,--force}'[force start performing]' \ 134 | ) 135 | ;; 136 | bulk_import:prepare_parts) 137 | _subcommand_args=( 138 | '(-s|--split-size)'{-s,--split-size}'[size of each parts]' \ 139 | '(-o|--output)'{-o,--output}'[output directory]' \ 140 | ) 141 | ;; 142 | db:create) 143 | _subcommand_args=( 144 | '(-g|--org)'{-g,--org}'[create the database under this organization]' \ 145 | ) 146 | ;; 147 | db:delete) 148 | _subcommand_args=( 149 | '(-f|--force)'{-f,--force}'[clear tables and delete the database]' \ 150 | ) 151 | ;; 152 | job:list) 153 | _subcommand_args=( 154 | '(-p|--page)'{-p,--page}'[skip N pages]' \ 155 | '(-s|--skip)'{-s,--skip}'[skip N jobs]' \ 156 | '(-R|--running)'{-R,--running}'[show only running jobs]' \ 157 | '(-S|--success)'{-S,--success}'[show only succeeded jobs]' \ 158 | '(-E|--error)'{-E,--error}'[show only failed jobs]' \ 159 | '--show[show slow queries (default threshold: 3600 seconds)]' \ 160 | ) 161 | ;; 162 | job:show) 163 | _subcommand_args=( 164 | '(-v|--verbose)'{-v,--verbose}'[show logs]' \ 165 | '(-w|--wait)'{-w,--wait}'[wait for finishing the job]' \ 166 | '(-G|--vertical)'{-G,--vertical}'[use vertical table to show results]' \ 167 | '(-o|--output)'{-o,--output}'[write result to the file]' \ 168 | '(-f|--format)'{-f,--format}'[format of the result to write to the file (tsv, csv, json or msgpack)]' 169 | ) 170 | ;; 171 | query) 172 | _subcommand_args=( 173 | '(-g|--org)'{-g,--org}'[issue the query under this organization]' \ 174 | '(-d|--database)'{-d,--database}'[use the database (required)]' \ 175 | '(-w|--wait)'{-w,--wait}'[wait for finishing the job]' \ 176 | '(-G|--vertical)'{-G,--vertical}'[use vertical table to show results]' \ 177 | '(-o|--output)'{-o,--output}'[write result to the file]' \ 178 | '(-f|--format)'{-f,--format}'[format of the result to write to the file (tsv, csv, json or msgpack)]' 179 | '(-r|--result)'{-r,--result}'[write result to the URL (see also result:create subcommand)]' \ 180 | '(-u|--user)'{-u,--user}'[set user name for the result URL]' \ 181 | '(-p|--password)'{-p,--password}'[ask password for the result URL]' \ 182 | '(-P|--priority)'{-P,--priority}'[set priority]' \ 183 | '(-R|--retry)'{-R,--retry}'[automatic retrying count]' \ 184 | '(-q|--query)'{-q,--query}'[use file instead of inline query]' \ 185 | '--sampling[enable random sampling to reduce records 1/DENOMINATOR]' \ 186 | ) 187 | ;; 188 | result:create) 189 | _subcommand_args=( 190 | '(-g|--org)'{-g,--org}'[create the result under this organization]' \ 191 | '(-u|--user)'{-u,--user}'[set user name for authentication]' \ 192 | '(-p|--password)'{-p,--password}'[ask password for authentication]' \ 193 | ) 194 | ;; 195 | sched:create) 196 | _subcommand_args=( 197 | '(-g|--org)'{-g,--org}'[create the schedule under this organization]' \ 198 | '(-d|--database)'{-d,--database}'[use the database (required)]' \ 199 | '(-t|--timezone)'{-t,--timezone}'[name of the timezone (like Asia/Tokyo)]' \ 200 | '(-D|--delay)'{-D,--delay}'[delay time of the schedule]' \ 201 | '(-o|--output)'{-o,--output}'[write result to the file]' \ 202 | '(-r|--result)'{-r,--result}'[write result to the URL (see also result:create subcommand)]' \ 203 | '(-u|--user)'{-u,--user}'[set user name for the result URL]' \ 204 | '(-p|--password)'{-p,--password}'[ask password for the result URL]' \ 205 | '(-P|--priority)'{-P,--priority}'[set priority]' \ 206 | '(-R|--retry)'{-R,--retry}'[automatic retrying count]' \ 207 | ) 208 | ;; 209 | sched:update) 210 | _subcommand_args=( 211 | '(-s|--schedule)'{-s,--schedule}'[change the schedule]' \ 212 | '(-q|--query)'{-q,--query}'[change the query]' \ 213 | '(-d|--database)'{-d,--database}'[change the database]' \ 214 | '(-r|--result)'{-r,--result}'[change the result table]' \ 215 | '(-t|--timezone)'{-t,--timezone}'[change the name of the timezone]' \ 216 | '(-D|--delay)'{-D,--delay}'[change the delay time of the schedule]' \ 217 | '(-P|--priority)'{-P,--priority}'[set priority]' \ 218 | '(-R|--retry)'{-R,--retry}'[automatic retrying count]' \ 219 | ) 220 | ;; 221 | sched:histroy) 222 | _subcommand_args=( 223 | '(-p|--page)'{-p,--page}'[skip N pages]' \ 224 | '(-s|--skip)'{-s,--skip}'[skip N jobs]' \ 225 | ) 226 | ;; 227 | sched:run) 228 | _subcommand_args=( 229 | '(-n|--num)'{-n,--num}'[number of jobs to run]' \ 230 | ) 231 | ;; 232 | table:delete) 233 | _subcommand_args=( 234 | '(-f|--force)'{-f,--force}'[never prompt]' \ 235 | ) 236 | ;; 237 | table:list) 238 | _subcommand_args=( 239 | '(-n|--num_threads)'{-num,--num_threads}'[number of threads to get list in parallel]' \ 240 | '--show-bytes[show estimated table in bytes]' \ 241 | ) 242 | ;; 243 | table:tail) 244 | _subcommand_args=( 245 | '(-t|--to)'{-t,--to}'[end time of logs to get]' \ 246 | '(-f|--from)'{-f,--from}'[start time of logs to get]' \ 247 | '(-c|--count)'{-c,--count}'[number of logs to get]' \ 248 | '(-P|--pretty)'{-P,--pretty}'[pretty print]' \ 249 | ) 250 | ;; 251 | table:import) 252 | _subcommand_args=( 253 | '--format[file format (default: apache)]' \ 254 | '--apache[same as --format apache; apache common log format]' \ 255 | '--syslog[same as --format syslog; syslog]' \ 256 | '--msgpack[same as --format msgpack; msgpack stream format]' \ 257 | '--json[same as --format json; LF-separated json format]' \ 258 | '(-t|--time-key)'{-t,--time-key}'[time key name for json and msgpack format (e.g. created_at)]' \ 259 | "--auto-create-table[create table and database if doesn't exist]" \ 260 | ) 261 | ;; 262 | table:export) 263 | _subcommand_args=( 264 | '(-g|--org)'{-g,--org}'[export the data under this organization]' \ 265 | '(-t|--to)'{-t,--to}'[export data which is older than the TIME]' \ 266 | '(-f|--from)'{-f,--from}'[export data which is newer than or same with the TIME]' \ 267 | '(-b|--bucket)'{-b,--bucket}'[name of the destination S3 bucket (required)]' \ 268 | '(-k|--aws-key-id)'{-k,--aws-key-id}'[AWS access key id to export data (required)]' \ 269 | '(-s|--aws-secret-key)'{-s,--aws-secret-key}'[AWS secret key to export data (required)]' \ 270 | ) 271 | ;; 272 | esac 273 | 274 | _arguments \ 275 | $_subcommand_args \ 276 | && return 0 277 | } 278 | 279 | _td 280 | -------------------------------------------------------------------------------- /contrib/completion/td-completion.bash: -------------------------------------------------------------------------------- 1 | # bash completion for td commands 2 | 3 | _td() 4 | { 5 | COMP_WORDBREAKS=${COMP_WORDBREAKS//:} 6 | local cur="${COMP_WORDS[COMP_CWORD]}" 7 | local prev="${COMP_WORDS[COMP_CWORD - 1]}" 8 | local list="`td help:all | awk '{print $1}' | grep '^[a-z]' | xargs`" 9 | 10 | # echo "cur=$cur, prev=$prev" 11 | 12 | if [[ "$prev" == "td" ]]; then 13 | if [[ "$cur" == "" ]]; then 14 | COMPREPLY=($list) 15 | else 16 | COMPREPLY=($(compgen -W "$list" -- "$cur")) 17 | fi 18 | fi 19 | } 20 | complete -F _td td 21 | 22 | # Local variables: 23 | # # mode: shell-script 24 | # # sh-basic-offset: 4 25 | # # sh-indent-comment: t 26 | # # indent-tabs-mode: nil 27 | # # End: 28 | # # ex: ts=4 sw=4 et filetype=sh 29 | -------------------------------------------------------------------------------- /data/sample_apache_gen.rb: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | require 'json' 3 | require 'msgpack' 4 | require 'digest/md5' 5 | require 'cgi' 6 | require 'uuidtools' 7 | require 'fileutils' 8 | require 'zlib' 9 | require 'parallel' 10 | 11 | RECORDS = 5000 12 | HOSTS = RECORDS/4 13 | PAGES = RECORDS/4 14 | 15 | AGENT_LIST_STRING = < host.ip, 182 | 'user' => '-', 183 | 'method' => page.method, 184 | 'path' => page.path, 185 | 'code' => grand(10000) == 0 ? 500 : page.code, 186 | 'referer' => (grand(2) == 0 ? @pages[grand(@pages.size)].path : page.referer) || '-', 187 | 'size' => page.size, 188 | 'agent' => host.agent, 189 | } 190 | puts record.to_json 191 | #puts %[#{record['host']} - #{record['user']} [#{Time.at(now).strftime('%d/%b/%Y:%H:%M:%S %z')}] "#{record['method']} #{record['path']} HTTP/1.1" #{record['code']} #{record['size']} "#{record['referer']}" "#{record['agent']}"] 192 | end 193 | 194 | -------------------------------------------------------------------------------- /dist/build_osx.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -u 2 | set -e 3 | 4 | PACKAGER="/usr/local/td/ruby/bin" 5 | 6 | git pull 7 | ${PACKAGER}/gem install bundler rubyzip --no-document 8 | rbenv rehash 9 | ${PACKAGER}/bundle install 10 | ${PACKAGER}/rake pkg:clean 11 | ${PACKAGER}/rake pkg:build 12 | -------------------------------------------------------------------------------- /dist/build_win81.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash -u 2 | 3 | if [ $# -ne 1 ]; then 4 | echo "$0 TD_TOOLBELT_LOCAL_CLIENT_GEM" 1>&2 5 | exit 1 6 | fi 7 | 8 | bundle install 9 | rake exe:clean 10 | TD_TOOLBELT_LOCAL_CLIENT_GEM=$1 rake exe:build 11 | -------------------------------------------------------------------------------- /dist/exe.rake: -------------------------------------------------------------------------------- 1 | namespace 'exe' do 2 | desc "build Windows exe package" 3 | task 'build' => '^build' do 4 | create_build_dir('exe') do |dir| 5 | install_ruby_version = '3.0.5' 6 | # create ./installers/ 7 | FileUtils.mkdir_p "installers" 8 | installer_path = project_root_path("dist/resources/exe/rubyinstaller-#{install_ruby_version}-1-x64.exe") 9 | FileUtils.cp installer_path, "installers/rubyinstaller.exe" 10 | 11 | variables = { 12 | :version => version, 13 | :basename => "td-#{version}", 14 | :outdir => ".", 15 | :install_ruby_version => install_ruby_version, 16 | } 17 | 18 | # create ./td/ 19 | mkchdir("td") do 20 | mkchdir('vendor/gems') do 21 | install_use_gems(Dir.pwd) 22 | end 23 | install_resource 'exe/td', 'bin/td', 0755 24 | install_erb_resource 'exe/td.bat', 'bin/td.bat', 0755, variables 25 | install_resource 'exe/td-cmd.bat', 'td-cmd.bat', 0755 26 | end 27 | 28 | zip_files(project_root_path("pkg/td-update-exe-#{version}.zip"), 'td') 29 | 30 | # create td.iss and run Inno Setup 31 | install_erb_resource 'exe/td.iss', 'td.iss', 0644, variables 32 | 33 | inno_dir = ENV["INNO_DIR"] || 'C:/Program Files (x86)/Inno Setup 5' 34 | inno_bin = ENV["INNO_BIN"] || "#{inno_dir}/Compil32.exe" 35 | puts "INNO_BIN: #{inno_bin}" 36 | 37 | sh "\"#{inno_bin}\" /cc \"td.iss\"" 38 | FileUtils.cp "td-#{version}.exe", project_root_path("pkg/td-#{version}.exe") 39 | end 40 | end 41 | 42 | desc "clean Windows exe package" 43 | task "clean" do 44 | FileUtils.rm_rf build_dir_path('exe') 45 | FileUtils.rm_rf project_root_path("pkg/td-#{version}.exe") 46 | FileUtils.rm_rf project_root_path("pkg/td-update-exe-#{version}.zip") 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /dist/pkg.rake: -------------------------------------------------------------------------------- 1 | namespace 'pkg' do 2 | desc "build Mac OS X pkg package" 3 | task 'build' => '^build' do 4 | create_build_dir('pkg') do |dir| 5 | FileUtils.mkdir_p "bundle" 6 | FileUtils.mkdir_p "bundle/Resources" 7 | FileUtils.mkdir_p "bundle/td-client.pkg" 8 | 9 | # create ./bundle/td-client.pkg/Payload 10 | mkchdir('td-client.build') do 11 | mkchdir('vendor/gems') do 12 | install_use_gems(Dir.pwd) 13 | end 14 | install_resource 'pkg/td', 'bin/td', 0755 15 | sh "pax -wz -x cpio . > ../bundle/td-client.pkg/Payload" 16 | end 17 | 18 | zip_files(project_root_path("pkg/td-update-pkg-#{version}.zip"), 'td-client.build') 19 | 20 | # create ./bundle/td-client.pkg/Bom 21 | sh "mkbom -s td-client.build bundle/td-client.pkg/Bom" 22 | 23 | # create ./bundle/td-client.pkg/Scripts/ 24 | install_resource 'pkg/postinstall', 'bundle/td-client.pkg/Scripts/postinstall', 0755 25 | 26 | variables = { 27 | :version => version, 28 | :kbytes => `du -ks td-client.build | cut -f 1`.strip.to_i, 29 | :num_files => `find td-client.build | wc -l`, 30 | } 31 | 32 | # create ./bundle/td-client.pkg/PackageInfo 33 | install_erb_resource('pkg/PackageInfo.erb', 'bundle/td-client.pkg/PackageInfo', 0644, variables) 34 | 35 | # create ./bundle/Distribution 36 | install_erb_resource('pkg/Distribution.erb', 'bundle/Distribution', 0644, variables) 37 | 38 | ruby_version = '3.0.5' 39 | sh "pkgutil --expand #{project_root_path("dist/resources/pkg/ruby-#{ruby_version}.pkg")} ruby" 40 | mv "ruby/ruby-#{ruby_version}.pkg", "bundle/ruby.pkg" 41 | 42 | # create td-a.b.c.pkg 43 | sh "pkgutil --flatten bundle td-#{version}.pkg" 44 | FileUtils.cp "td-#{version}.pkg", project_root_path("pkg/td-#{version}.pkg") 45 | puts "Finished building pkg/td-#{version}.pkg" 46 | end 47 | end 48 | 49 | desc "clean Mac OS X pkg package" 50 | task "clean" do 51 | FileUtils.rm_rf build_dir_path('pkg') 52 | FileUtils.rm_rf project_root_path("pkg/td-#{version}.pkg") 53 | FileUtils.rm_rf project_root_path("pkg/td-update-pkg-#{version}.zip") 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /dist/resources/exe/td: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # -*- coding: utf-8 -*- 3 | 4 | # avoid conflicts with rvm 5 | ENV.delete 'GEM_HOME' 6 | ENV.delete 'GEM_PATH' 7 | 8 | # attempt to load rubygems 9 | begin 10 | require "rubygems" 11 | rescue LoadError 12 | end 13 | 14 | # resolve bin path, ignoring symlinks 15 | require "pathname" 16 | here = File.dirname(Pathname.new(__FILE__).realpath) 17 | 18 | # add locally installed gems to libpath 19 | gem_dir = File.expand_path("../vendor/gems", here) 20 | Dir["#{gem_dir}/**/lib"].each do |libdir| 21 | $:.unshift libdir 22 | end 23 | 24 | # inject any code in ~/.td/updated/vendor/gems over top 25 | require 'td/updater' 26 | TreasureData::Updater.inject_libpath 27 | 28 | # start up the CLI 29 | require 'td/command/runner' 30 | exit TreasureData::Command::Runner.new.run ARGV 31 | -------------------------------------------------------------------------------- /dist/resources/exe/td-cmd.bat: -------------------------------------------------------------------------------- 1 | @cd %USERPROFILE% 2 | @cmd /k td 3 | -------------------------------------------------------------------------------- /dist/resources/exe/td.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | :: determine if this is an NT operating system 4 | if not "%~f0" == "~f0" goto WinNT 5 | goto Win9x 6 | 7 | :Win9x 8 | @"%~dp0\..\ruby-<%= install_ruby_version %>\bin\ruby.exe" -Eutf-8 "td" %1 %2 %3 %4 %5 %6 %7 %8 %9 9 | goto :EOF 10 | 11 | :WinNT 12 | @"%~dp0\..\ruby-<%= install_ruby_version %>\bin\ruby.exe" -Eutf-8 "%~dpn0" %* 13 | goto :EOF 14 | -------------------------------------------------------------------------------- /dist/resources/exe/td.iss: -------------------------------------------------------------------------------- 1 | [Setup] 2 | AppName=Treasure Data 3 | AppVersion=<%= version %> 4 | DefaultDirName={pf}\Treasure Data 5 | DefaultGroupName=Treasure Data 6 | Compression=lzma2 7 | SolidCompression=yes 8 | OutputBaseFilename=<%= basename %> 9 | OutputDir=<%= outdir %> 10 | ChangesEnvironment=yes 11 | UsePreviousSetupType=no 12 | AlwaysShowComponentsList=no 13 | 14 | ; For Ruby expansion ~ 32MB (installed) - 12MB (installer) 15 | ExtraDiskSpaceRequired=20971520 16 | 17 | [Types] 18 | Name: client; Description: "Full Installation"; 19 | Name: custom; Description: "Custom Installation"; flags: iscustom 20 | 21 | [Components] 22 | Name: "toolbelt"; Description: "Treasure Data Toolbelt"; Types: "client custom" 23 | Name: "toolbelt/client"; Description: "Treasure Data Client"; Types: "client custom"; Flags: fixed 24 | 25 | [InstallDelete] 26 | Type: filesandordirs; Name: "{app}\vendor" 27 | 28 | [Files] 29 | Source: "td\*.*"; DestDir: "{app}"; Flags: recursesubdirs; Components: "toolbelt/client" 30 | Source: "installers\rubyinstaller.exe"; DestDir: "{tmp}"; Components: "toolbelt/client" 31 | 32 | [Icons] 33 | Name: "{group}\Treasure Data command prompt"; Filename: "{app}\td-cmd.bat" 34 | 35 | [Registry] 36 | Root: HKLM; Subkey: "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"; ValueType: "expandsz"; ValueName: "Path"; \ 37 | ValueData: "{olddata};{app}\bin"; Check: NeedsAddPath(ExpandConstant('{app}\bin')) 38 | Root: HKCU; Subkey: "Environment"; ValueType: "expandsz"; ValueName: "HOME"; \ 39 | ValueData: "%USERPROFILE%"; Flags: createvalueifdoesntexist 40 | 41 | [Run] 42 | Filename: "{tmp}\rubyinstaller.exe"; Parameters: "/verysilent /noreboot /nocancel /noicons /dir=""{app}/ruby-<%= install_ruby_version %>"""; \ 43 | Flags: shellexec waituntilterminated; StatusMsg: "Installing Ruby"; Components: "toolbelt/client" 44 | ; Filename: "{app}\td-cmd.bat"; Description: "Run command prompt"; Flags: postinstall 45 | 46 | [Code] 47 | 48 | function NeedsAddPath(Param: string): boolean; 49 | var 50 | OrigPath: string; 51 | begin 52 | if not RegQueryStringValue(HKEY_LOCAL_MACHINE, 53 | 'SYSTEM\CurrentControlSet\Control\Session Manager\Environment', 54 | 'Path', OrigPath) 55 | then begin 56 | Result := True; 57 | exit; 58 | end; 59 | // look for the path with leading and trailing semicolon 60 | // Pos() returns 0 if not found 61 | Result := Pos(';' + Param + ';', ';' + OrigPath + ';') = 0; 62 | end; 63 | 64 | function IsProgramInstalled(Name: string): boolean; 65 | var 66 | ResultCode: integer; 67 | begin 68 | Result := Exec(Name, 'version', '', SW_HIDE, ewWaitUntilTerminated, ResultCode); 69 | end; 70 | -------------------------------------------------------------------------------- /dist/resources/pkg/Distribution.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | TreasureData Client 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | #td-client.pkg 14 | #ruby.pkg 15 | 16 | -------------------------------------------------------------------------------- /dist/resources/pkg/PackageInfo.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /dist/resources/pkg/postinstall: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | mkdir -p /usr/local/bin 3 | ln -sf /usr/local/td/bin/td /usr/local/bin/td 4 | osascript -e ' 5 | tell application "Terminal" 6 | activate 7 | do script "td" 8 | end tell' 9 | -------------------------------------------------------------------------------- /dist/resources/pkg/td: -------------------------------------------------------------------------------- 1 | #!/usr/local/td/ruby/bin/ruby 2 | # -*- coding: utf-8 -*- 3 | 4 | # avoid conflicts with rvm 5 | ENV.delete 'GEM_HOME' 6 | ENV.delete 'GEM_PATH' 7 | 8 | # attempt to load rubygems 9 | begin 10 | require "rubygems" 11 | rescue LoadError 12 | end 13 | 14 | # resolve bin path, ignoring symlinks 15 | require "pathname" 16 | here = File.dirname(Pathname.new(__FILE__).realpath) 17 | 18 | # add locally installed gems to libpath 19 | gem_dir = File.expand_path("../vendor/gems", here) 20 | Dir["#{gem_dir}/**/lib"].each do |libdir| 21 | $:.unshift libdir 22 | end 23 | 24 | # inject any code in ~/.td/updated/vendor/gems over top 25 | require 'td/updater' 26 | TreasureData::Updater.inject_libpath 27 | 28 | # start up the CLI 29 | require 'td/command/runner' 30 | exit TreasureData::Command::Runner.new.run ARGV 31 | -------------------------------------------------------------------------------- /java/logging.properties: -------------------------------------------------------------------------------- 1 | ############################################################ 2 | # Treasure Data BulkImport Logging Configuration File 3 | # 4 | # You can use a different file by specifying a filename 5 | # with the java.util.logging.config.file system property. 6 | # For example java -Djava.util.logging.config.file=myfile 7 | ############################################################ 8 | 9 | ############################################################ 10 | # Global properties 11 | ############################################################ 12 | 13 | # "handlers" specifies a comma separated list of log Handler 14 | # classes. These handlers will be installed during VM startup. 15 | # Note that these classes must be on the system classpath. 16 | # By default we only configure a ConsoleHandler, which will only 17 | # show messages at the INFO and above levels. 18 | handlers= java.util.logging.FileHandler 19 | 20 | # To also add the FileHandler, use the following line instead. 21 | #handlers= java.util.logging.FileHandler, java.util.logging.ConsoleHandler 22 | 23 | # Default global logging level. 24 | # This specifies which kinds of events are logged across 25 | # all loggers. For any given facility this global level 26 | # can be overriden by a facility specific level 27 | # Note that the ConsoleHandler also has a separate level 28 | # setting to limit messages printed to the console. 29 | .level= INFO 30 | 31 | ############################################################ 32 | # Handler specific properties. 33 | # Describes specific configuration info for Handlers. 34 | ############################################################ 35 | 36 | java.util.logging.FileHandler.level = INFO 37 | java.util.logging.FileHandler.pattern=td-bulk-import.log 38 | java.util.logging.FileHandler.append=true 39 | java.util.logging.FileHandler.limit = 50000 40 | java.util.logging.FileHandler.count = 1 41 | java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter 42 | 43 | # Limit the message that are printed on the console to INFO and above. 44 | java.util.logging.ConsoleHandler.level = INFO 45 | java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter 46 | 47 | ############################################################ 48 | # Facility specific properties. 49 | # Provides extra control for each logger. 50 | ############################################################ 51 | 52 | # For example, set the com.xyz.foo logger to only log SEVERE 53 | # messages: 54 | com.xyz.foo.level = SEVERE 55 | -------------------------------------------------------------------------------- /lib/td.rb: -------------------------------------------------------------------------------- 1 | require 'td-logger' 2 | -------------------------------------------------------------------------------- /lib/td/command/account.rb: -------------------------------------------------------------------------------- 1 | require 'td/helpers' 2 | 3 | module TreasureData 4 | module Command 5 | 6 | def account(op) 7 | op.banner << "\noptions:\n" 8 | 9 | force = false 10 | op.on('-f', '--force', 'overwrite current account setting', TrueClass) {|b| 11 | force = true 12 | } 13 | 14 | user_name = op.cmd_parse 15 | 16 | endpoint = nil 17 | # user may be calling 'td account' with the -e / --endpoint 18 | # option, which we want to preserve and save 19 | begin 20 | endpoint = Config.endpoint 21 | rescue ConfigNotFoundError => e 22 | # the endpoint is neither stored in the config file 23 | # nor passed as option on the command line 24 | end 25 | 26 | conf = nil 27 | begin 28 | conf = Config.read 29 | rescue ConfigError 30 | end 31 | 32 | if conf && conf['account.apikey'] 33 | unless force 34 | if conf['account.user'] 35 | $stderr.puts "Account is already configured with '#{conf['account.user']}' account." 36 | else 37 | $stderr.puts "Account is already configured." 38 | end 39 | $stderr.puts "Add '-f' option to overwrite." 40 | exit 0 41 | end 42 | end 43 | 44 | $stdout.puts "Enter your Treasure Data credentials. For Google SSO user, please see https://docs.treasuredata.com/display/public/PD/Configuring+Authentication+for+TD+Using+the+TD+Toolbelt#ConfiguringAuthenticationforTDUsingtheTDToolbelt-SettingUpGoogleSSOUsers" 45 | unless user_name 46 | begin 47 | $stdout.print "Email: " 48 | line = STDIN.gets || "" 49 | user_name = line.strip 50 | rescue Interrupt 51 | $stderr.puts "\ncanceled." 52 | exit 1 53 | end 54 | end 55 | 56 | if user_name.empty? 57 | $stderr.puts "canceled." 58 | exit 0 59 | end 60 | 61 | client = nil 62 | 63 | 3.times do 64 | begin 65 | $stdout.print "Password (typing will be hidden): " 66 | password = get_password 67 | rescue Interrupt 68 | $stderr.print "\ncanceled." 69 | exit 1 70 | ensure 71 | system "stty echo" # TODO termios 72 | $stdout.print "\n" 73 | end 74 | 75 | if password.empty? 76 | $stderr.puts "canceled." 77 | exit 0 78 | end 79 | 80 | begin 81 | # enalbe SSL for the authentication 82 | opts = {} 83 | opts[:ssl] = true 84 | opts[:endpoint] = endpoint if endpoint 85 | client = Client.authenticate(user_name, password, opts) 86 | rescue TreasureData::AuthError 87 | $stderr.puts "User name or password mismatched." 88 | end 89 | 90 | break if client 91 | end 92 | return unless client 93 | 94 | $stdout.puts "Authenticated successfully." 95 | 96 | conf ||= Config.new 97 | conf["account.user"] = user_name 98 | conf["account.apikey"] = client.apikey 99 | conf['account.endpoint'] = endpoint if endpoint 100 | conf.save 101 | 102 | $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "db:create ' to create a database." 103 | end 104 | 105 | def account_usage(op) 106 | op.cmd_parse 107 | 108 | client = get_client 109 | a = client.account 110 | 111 | $stderr.puts "Storage: #{a.storage_size_string}" 112 | end 113 | 114 | private 115 | if Helpers.on_windows? 116 | require 'fiddle' 117 | 118 | def get_char 119 | msvcrt = Fiddle.dlopen('msvcrt') 120 | getch = Fiddle::Function.new(msvcrt['_getch'], [], Fiddle::TYPE_CHAR) 121 | getch.call() 122 | rescue Exception 123 | crtdll = Fiddle.dlopen('crtdll') 124 | getch = Fiddle::Function.new(crtdll['_getch'], [], Fiddle::TYPE_CHAR) 125 | getch.call() 126 | end 127 | 128 | def get_password 129 | password = '' 130 | 131 | while c = get_char 132 | break if c == 13 || c == 10 # CR or NL 133 | if c == 127 || c == 8 # 128: backspace, 8: delete 134 | password.slice!(-1, 1) 135 | else 136 | password << c.chr 137 | end 138 | end 139 | 140 | password 141 | end 142 | else 143 | def get_password 144 | system "stty -echo" # TODO termios 145 | password = STDIN.gets || "" 146 | password[0..-2] # strip \n 147 | end 148 | end 149 | end 150 | end 151 | 152 | -------------------------------------------------------------------------------- /lib/td/command/apikey.rb: -------------------------------------------------------------------------------- 1 | 2 | module TreasureData 3 | module Command 4 | 5 | def apikey_show(op) 6 | if Config.apikey 7 | $stdout.puts Config.apikey 8 | return 9 | end 10 | 11 | conf = nil 12 | begin 13 | conf = Config.read 14 | rescue ConfigError 15 | end 16 | 17 | if !conf || !conf['account.apikey'] 18 | $stderr.puts "Account is not configured yet." 19 | $stderr.puts "Use '#{$prog} apikey:set' or '#{$prog} account' first." 20 | exit 1 21 | end 22 | 23 | $stdout.puts conf['account.apikey'] 24 | end 25 | 26 | def apikey_set(op) 27 | op.banner << "\noptions:\n" 28 | 29 | force = false 30 | op.on('-f', '--force', 'overwrite current account setting', TrueClass) {|b| 31 | force = true 32 | } 33 | 34 | apikey = op.cmd_parse 35 | 36 | conf = nil 37 | begin 38 | conf = Config.read 39 | rescue ConfigError 40 | end 41 | if conf && conf['account.apikey'] 42 | unless force 43 | if conf['account.user'] 44 | $stderr.puts "Account is already configured with '#{conf['account.user']}' account." 45 | else 46 | $stderr.puts "Account is already configured." 47 | end 48 | $stderr.puts "Add '-f' option to overwrite." 49 | exit 0 50 | end 51 | end 52 | 53 | conf ||= Config.new 54 | conf.delete("account.user") 55 | conf["account.apikey"] = apikey 56 | conf.save 57 | 58 | $stdout.puts "API key is set." 59 | $stdout.puts "Use '#{$prog} db:create ' to create a database." 60 | end 61 | 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /lib/td/command/db.rb: -------------------------------------------------------------------------------- 1 | 2 | module TreasureData 3 | module Command 4 | 5 | def db_show(op) 6 | set_render_format_option(op) 7 | 8 | db_name = op.cmd_parse 9 | 10 | client = get_client 11 | 12 | db = get_database(client, db_name) 13 | 14 | rows = [] 15 | db.tables.each {|table| 16 | pschema = table.schema.fields.map {|f| 17 | "#{f.name}:#{f.type}" 18 | }.join(', ') 19 | rows << {:Table => table.name, :Type => table.type.to_s, :Count => table.count.to_s, :Schema=>pschema.to_s} 20 | } 21 | rows = rows.sort_by {|map| 22 | [map[:Type].size, map[:Table]] 23 | } 24 | 25 | $stdout.puts cmd_render_table(rows, :fields => [:Table, :Type, :Count, :Schema], :render_format => op.render_format) 26 | end 27 | 28 | def db_list(op) 29 | set_render_format_option(op) 30 | 31 | op.cmd_parse 32 | 33 | client = get_client 34 | dbs = client.databases 35 | 36 | rows = [] 37 | dbs.each {|db| 38 | rows << {:Name=>db.name, :Count=>db.count} 39 | } 40 | $stdout.puts cmd_render_table(rows, :fields => [:Name, :Count], :render_format => op.render_format) 41 | 42 | if dbs.empty? 43 | $stderr.puts "There are no databases." 44 | $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "db:create ' to create a database." 45 | end 46 | end 47 | 48 | def db_create(op) 49 | db_name = op.cmd_parse 50 | 51 | API.validate_database_name(db_name) 52 | 53 | client = get_client 54 | 55 | opts = {} 56 | begin 57 | client.create_database(db_name, opts) 58 | rescue AlreadyExistsError 59 | $stderr.puts "Database '#{db_name}' already exists." 60 | exit 1 61 | end 62 | 63 | $stderr.puts "Database '#{db_name}' is created." 64 | $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "table:create #{db_name} ' to create a table." 65 | end 66 | 67 | def db_delete(op) 68 | force = false 69 | op.on('-f', '--force', 'clear tables and delete the database', TrueClass) {|b| 70 | force = true 71 | } 72 | 73 | db_name = op.cmd_parse 74 | 75 | client = get_client 76 | 77 | begin 78 | db = client.database(db_name) 79 | 80 | if !force && !db.tables.empty? 81 | $stderr.puts "Database '#{db_name}' is not empty. Use '-f' option or drop tables first." 82 | exit 1 83 | end 84 | 85 | db.delete 86 | rescue NotFoundError 87 | $stderr.puts "Database '#{db_name}' does not exist." 88 | exit 1 89 | end 90 | 91 | $stderr.puts "Database '#{db_name}' is deleted." 92 | end 93 | 94 | end 95 | end 96 | 97 | -------------------------------------------------------------------------------- /lib/td/command/export.rb: -------------------------------------------------------------------------------- 1 | require 'td/command/job' 2 | 3 | module TreasureData 4 | module Command 5 | SUPPORTED_FORMATS = %W[json.gz line-json.gz tsv.gz jsonl.gz] 6 | SUPPORTED_ENCRYPT_METHOD = %W[s3] 7 | 8 | def export_result(op) 9 | wait = false 10 | priority = nil 11 | retry_limit = nil 12 | 13 | op.on('-w', '--wait', 'wait until the job is completed', TrueClass) {|b| 14 | wait = b 15 | } 16 | op.on('-P', '--priority PRIORITY', 'set priority') {|s| 17 | priority = job_priority_id_of(s) 18 | unless priority 19 | raise "unknown priority #{s.inspect} should be -2 (very-low), -1 (low), 0 (normal), 1 (high) or 2 (very-high)" 20 | end 21 | } 22 | op.on('-R', '--retry COUNT', 'automatic retrying count', Integer) {|i| 23 | retry_limit = i 24 | } 25 | 26 | target_job_id, result = op.cmd_parse 27 | 28 | client = get_ssl_client 29 | 30 | opts = { 31 | result: result, 32 | retry_limit: retry_limit, 33 | priority: priority, 34 | } 35 | if wait 36 | job = client.job(target_job_id) 37 | if !job.finished? 38 | $stderr.puts "target job #{target_job_id} is still running..." 39 | wait_job(job) 40 | end 41 | end 42 | 43 | job = client.result_export(target_job_id, opts) 44 | $stderr.puts "result export job #{job.job_id} is queued." 45 | $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job.job_id}' to show the status." 46 | 47 | if wait 48 | wait_job(job) 49 | $stdout.puts "status : #{job.status}" 50 | end 51 | end 52 | 53 | def export_table(op) 54 | table_export(op) 55 | end 56 | 57 | def table_export(op) 58 | from = nil 59 | to = nil 60 | s3_bucket = nil 61 | wait = false 62 | aws_access_key_id = nil 63 | aws_secret_access_key = nil 64 | file_prefix = nil 65 | file_format = "json.gz" # default 66 | pool_name = nil 67 | encryption = nil 68 | assume_role = nil 69 | 70 | # Increase the summary_width value from default 32 into 36, to show the options help message properly. 71 | op.summary_width = 36 72 | 73 | op.on('-w', '--wait', 'wait until the job is completed', TrueClass) {|b| 74 | wait = b 75 | } 76 | op.on('-f', '--from TIME', 'export data which is newer than or same with the TIME') {|s| 77 | from = export_parse_time(s) 78 | } 79 | op.on('-t', '--to TIME', 'export data which is older than the TIME') {|s| 80 | to = export_parse_time(s) 81 | } 82 | op.on('-b', '--s3-bucket NAME', 'name of the destination S3 bucket (required)') {|s| 83 | s3_bucket = s 84 | } 85 | op.on('-p', '--prefix PATH', 'path prefix of the file on S3') {|s| 86 | file_prefix = s 87 | } 88 | op.on('-k', '--aws-key-id KEY_ID', 'AWS access key id to export data (required)') {|s| 89 | aws_access_key_id = s 90 | } 91 | op.on('-s', '--aws-secret-key SECRET_KEY', 'AWS secret access key to export data (required)') {|s| 92 | aws_secret_access_key = s 93 | } 94 | op.on('-F', '--file-format FILE_FORMAT', 95 | 'file format for exported data.', 96 | 'Available formats are tsv.gz (tab-separated values per line) and jsonl.gz (JSON record per line).', 97 | 'The json.gz and line-json.gz formats are default and still available but only for backward compatibility purpose;', 98 | ' use is discouraged because they have far lower performance.') { |s| 99 | raise ArgumentError, "#{s} is not a supported file format" unless SUPPORTED_FORMATS.include?(s) 100 | file_format = s 101 | } 102 | op.on('-O', '--pool-name NAME', 'specify resource pool by name') {|s| 103 | pool_name = s 104 | } 105 | op.on('-e', '--encryption ENCRYPT_METHOD', 'export with server side encryption with the ENCRYPT_METHOD') {|s| 106 | raise ArgumentError, "#{s} is not a supported encryption method" unless SUPPORTED_ENCRYPT_METHOD.include?(s) 107 | encryption = s 108 | } 109 | op.on('-a', '--assume-role ASSUME_ROLE_ARN', 'export with assume role with ASSUME_ROLE_ARN as role arn') {|s| 110 | assume_role = s 111 | } 112 | 113 | db_name, table_name = op.cmd_parse 114 | 115 | unless s3_bucket 116 | $stderr.puts "-b, --s3-bucket NAME option is required." 117 | exit 1 118 | end 119 | 120 | unless aws_access_key_id 121 | $stderr.puts "-k, --aws-key-id KEY_ID option is required." 122 | exit 1 123 | end 124 | 125 | unless aws_secret_access_key 126 | $stderr.puts "-s, --aws-secret-key SECRET_KEY option is required." 127 | exit 1 128 | end 129 | 130 | client = get_client 131 | 132 | get_table(client, db_name, table_name) 133 | 134 | client = get_ssl_client 135 | 136 | s3_opts = {} 137 | s3_opts['from'] = from.to_s if from 138 | s3_opts['to'] = to.to_s if to 139 | s3_opts['file_prefix'] = file_prefix if file_prefix 140 | s3_opts['file_format'] = file_format 141 | s3_opts['bucket'] = s3_bucket 142 | s3_opts['access_key_id'] = aws_access_key_id 143 | s3_opts['secret_access_key'] = aws_secret_access_key 144 | s3_opts['pool_name'] = pool_name if pool_name 145 | s3_opts['encryption'] = encryption if encryption 146 | s3_opts['assume_role'] = assume_role if assume_role 147 | 148 | job = client.export(db_name, table_name, "s3", s3_opts) 149 | 150 | $stderr.puts "Export job #{job.job_id} is queued." 151 | $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job.job_id}' to show the status." 152 | 153 | if wait 154 | wait_job(job) 155 | $stdout.puts "Status : #{job.status}" 156 | end 157 | end 158 | 159 | private 160 | def export_parse_time(time) 161 | if time.to_i.to_s == time.to_s 162 | # UNIX time 163 | return time.to_i 164 | else 165 | require 'time' 166 | begin 167 | return Time.parse(time).to_i 168 | rescue 169 | $stderr.puts "invalid time format: #{time}" 170 | exit 1 171 | end 172 | end 173 | end 174 | 175 | end 176 | end 177 | 178 | -------------------------------------------------------------------------------- /lib/td/command/help.rb: -------------------------------------------------------------------------------- 1 | 2 | module TreasureData 3 | module Command 4 | 5 | def help(op) 6 | cmd = op.cmd_parse 7 | 8 | c = List.get_option(cmd) 9 | if c == nil 10 | $stderr.puts "'#{cmd}' is not a td command. Run '#{$prog}' to show the list." 11 | List.show_guess(cmd) 12 | exit 1 13 | 14 | elsif c.name != cmd && c.group == cmd 15 | # group command 16 | $stdout.puts List.cmd_usage(cmd) 17 | exit 1 18 | 19 | else 20 | method, cmd_req_connectivity = List.get_method(cmd) 21 | method.call(['--help']) 22 | end 23 | end 24 | 25 | def help_all(op) 26 | cmd = op.cmd_parse 27 | 28 | TreasureData::Command::List.show_help(op.summary_indent) 29 | $stdout.puts "" 30 | $stdout.puts "Type '#{$prog} help COMMAND' for more information on a specific command." 31 | end 32 | 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/td/command/options.rb: -------------------------------------------------------------------------------- 1 | module TreasureData 2 | module Command 3 | module Options 4 | 5 | def job_show_options(op) 6 | opts = {} 7 | opts[:verbose] = nil 8 | opts[:wait] = false 9 | opts[:output] = nil 10 | opts[:format] = nil 11 | opts[:render_opts] = {:header => false} 12 | opts[:limit] = nil 13 | opts[:exclude] = false 14 | 15 | op.on('-v', '--verbose', 'show logs', TrueClass) {|b| 16 | opts[:verbose] = b 17 | } 18 | op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b| 19 | opts[:wait] = b 20 | } 21 | op.on('-G', '--vertical', 'use vertical table to show results', TrueClass) {|b| 22 | opts[:render_opts][:vertical] = b 23 | } 24 | op.on('-o', '--output PATH', 'write result to the file') {|s| 25 | unless Dir.exist?(File.dirname(s)) 26 | s = File.expand_path(s) 27 | end 28 | opts[:output] = s 29 | opts[:format] ||= 'tsv' 30 | } 31 | op.on('-l', '--limit ROWS', 'limit the number of result rows shown when not outputting to file') {|s| 32 | unless s.to_i > 0 33 | raise "Invalid limit number. Must be a positive integer" 34 | end 35 | opts[:limit] = s.to_i 36 | } 37 | op.on('-c', '--column-header', 'output of the columns\' header when the schema is available', 38 | ' for the table (only applies to tsv and csv formats)', TrueClass) {|b| 39 | opts[:render_opts][:header] = b; 40 | } 41 | op.on('-x', '--exclude', 'do not automatically retrieve the job result', TrueClass) {|b| 42 | opts[:exclude] = b 43 | } 44 | 45 | op.on('--null STRING', "null expression in csv or tsv") {|s| 46 | opts[:render_opts][:null_expr] = s.to_s 47 | } 48 | 49 | write_format_option(op) {|s| opts[:format] = s } 50 | 51 | # CAUTION: this opts is filled by op.cmd_parse 52 | opts 53 | end 54 | 55 | def write_format_option(op) 56 | op.on('-f', '--format FORMAT', 'format of the result to write to the file (tsv, csv, json, msgpack, and msgpack.gz)') {|s| 57 | unless ['tsv', 'csv', 'json', 'msgpack', 'msgpack.gz'].include?(s) 58 | raise "Unknown format #{s.dump}. Supported formats are: tsv, csv, json, msgpack, and msgpack.gz" 59 | end 60 | 61 | yield(s) 62 | } 63 | end 64 | module_function :write_format_option 65 | end # module Options 66 | end # module Command 67 | end # module TreasureData 68 | -------------------------------------------------------------------------------- /lib/td/command/password.rb: -------------------------------------------------------------------------------- 1 | 2 | module TreasureData 3 | module Command 4 | 5 | def password_change(op) 6 | op.cmd_parse 7 | 8 | old_password = nil 9 | password = nil 10 | 11 | begin 12 | system "stty -echo" # TODO termios 13 | $stdout.print "Old password (typing will be hidden): " 14 | old_password = STDIN.gets || "" 15 | old_password = old_password[0..-2] # strip \n 16 | rescue Interrupt 17 | $stderr.print "\ncanceled." 18 | exit 1 19 | ensure 20 | system "stty echo" # TODO termios 21 | $stdout.print "\n" 22 | end 23 | 24 | if old_password.empty? 25 | $stderr.puts "canceled." 26 | exit 0 27 | end 28 | 29 | 3.times do 30 | begin 31 | system "stty -echo" # TODO termios 32 | $stdout.print "New password (typing will be hidden): " 33 | password = STDIN.gets || "" 34 | password = password[0..-2] # strip \n 35 | rescue Interrupt 36 | $stderr.print "\ncanceled." 37 | exit 1 38 | ensure 39 | system "stty echo" # TODO termios 40 | $stdout.print "\n" 41 | end 42 | 43 | if password.empty? 44 | $stderr.puts "canceled." 45 | exit 0 46 | end 47 | 48 | begin 49 | system "stty -echo" # TODO termios 50 | $stdout.print "Retype new password: " 51 | password2 = STDIN.gets || "" 52 | password2 = password2[0..-2] # strip \n 53 | rescue Interrupt 54 | $stderr.print "\ncanceled." 55 | exit 1 56 | ensure 57 | system "stty echo" # TODO termios 58 | $stdout.print "\n" 59 | end 60 | 61 | if password == password2 62 | break 63 | end 64 | 65 | $stdout.puts "Doesn't match." 66 | end 67 | 68 | client = get_client(:ssl => true) 69 | 70 | client.change_my_password(old_password, password) 71 | 72 | $stderr.puts "Password changed." 73 | end 74 | 75 | end 76 | end 77 | 78 | -------------------------------------------------------------------------------- /lib/td/command/query.rb: -------------------------------------------------------------------------------- 1 | require 'td/command/job' 2 | require 'td/command/options' 3 | 4 | module TreasureData 5 | module Command 6 | 7 | def query(op) 8 | db_name = nil 9 | wait = false 10 | output = nil 11 | format = nil 12 | render_opts = {:header => false} 13 | result_url = nil 14 | result_user = nil 15 | result_ask_password = false 16 | priority = nil 17 | retry_limit = nil 18 | query = nil 19 | type = nil 20 | limit = nil 21 | exclude = false 22 | pool_name = nil 23 | domain_key = nil 24 | engine_version = nil 25 | 26 | op.on('-d', '--database DB_NAME', 'use the database (required)') {|s| 27 | db_name = s 28 | } 29 | op.on('-w', '--wait[=SECONDS]', 'wait for finishing the job (for seconds)', Integer) {|b| 30 | wait = b 31 | } 32 | op.on('-G', '--vertical', 'use vertical table to show results', TrueClass) {|b| 33 | render_opts[:vertical] = b 34 | } 35 | op.on('-o', '--output PATH', 'write result to the file') {|s| 36 | unless Dir.exist?(File.dirname(s)) 37 | s = File.expand_path(s) 38 | end 39 | output = s 40 | format = 'tsv' if format.nil? 41 | } 42 | 43 | TreasureData::Command::Options.write_format_option(op) {|s| format = s } 44 | 45 | op.on('-r', '--result RESULT_URL', 'write result to the URL (see also result:create subcommand)', 46 | ' It is suggested for this option to be used with the -x / --exclude option to suppress printing', 47 | ' of the query result to stdout or -o / --output to dump the query result into a file.') {|s| 48 | result_url = s 49 | } 50 | op.on('-u', '--user NAME', 'set user name for the result URL') {|s| 51 | result_user = s 52 | } 53 | op.on('-p', '--password', 'ask password for the result URL') {|s| 54 | result_ask_password = true 55 | } 56 | op.on('-P', '--priority PRIORITY', 'set priority') {|s| 57 | priority = job_priority_id_of(s) 58 | unless priority 59 | raise "unknown priority #{s.inspect} should be -2 (very-low), -1 (low), 0 (normal), 1 (high) or 2 (very-high)" 60 | end 61 | } 62 | op.on('-R', '--retry COUNT', 'automatic retrying count', Integer) {|i| 63 | retry_limit = i 64 | } 65 | op.on('-q', '--query PATH', 'use file instead of inline query') {|s| 66 | query = File.open(s) { |f| f.read.strip } 67 | } 68 | op.on('-T', '--type TYPE', 'set query type (hive, presto)') {|s| 69 | type = s.to_sym 70 | } 71 | op.on('--sampling DENOMINATOR', 'OBSOLETE - enable random sampling to reduce records 1/DENOMINATOR', Integer) {|i| 72 | $stdout.puts "WARNING: the random sampling feature enabled through the '--sampling' option was removed and does no longer" 73 | $stdout.puts " have any effect. It is left for backwards compatibility with older scripts using 'td'." 74 | $stdout.puts 75 | } 76 | op.on('-l', '--limit ROWS', 'limit the number of result rows shown when not outputting to file') {|s| 77 | unless s.to_i > 0 78 | raise "Invalid limit number. Must be a positive integer" 79 | end 80 | limit = s.to_i 81 | } 82 | op.on('-c', '--column-header', 'output of the columns\' header when the schema is available for the table (only applies to json, tsv and csv formats)', TrueClass) {|b| 83 | render_opts[:header] = b 84 | } 85 | op.on('-x', '--exclude', 'do not automatically retrieve the job result', TrueClass) {|b| 86 | exclude = b 87 | } 88 | op.on('-O', '--pool-name NAME', 'specify resource pool by name') {|s| 89 | pool_name = s 90 | } 91 | op.on('--domain-key DOMAIN_KEY', 'optional user-provided unique ID. You can include this ID with your `create` request to ensure idempotence') {|s| 92 | domain_key = s 93 | } 94 | op.on('--engine-version ENGINE_VERSION', 'EXPERIMENTAL: specify query engine version by name') {|s| 95 | engine_version = s 96 | } 97 | 98 | sql = op.cmd_parse 99 | 100 | # required parameters 101 | 102 | unless db_name 103 | raise ParameterConfigurationError, 104 | "-d / --database DB_NAME option is required." 105 | end 106 | 107 | if sql == '-' 108 | sql = STDIN.read 109 | elsif sql.nil? 110 | sql = query 111 | end 112 | unless sql 113 | raise ParameterConfigurationError, 114 | " argument or -q / --query PATH option is required." 115 | end 116 | 117 | # parameter concurrency validation 118 | 119 | if output.nil? && format 120 | unless ['tsv', 'csv', 'json'].include?(format) 121 | raise ParameterConfigurationError, 122 | "Supported formats are only tsv, csv and json without --output option" 123 | end 124 | end 125 | 126 | if render_opts[:header] 127 | unless ['tsv', 'csv', 'json'].include?(format) 128 | raise ParameterConfigurationError, 129 | "Option -c / --column-header is only supported with json, tsv and csv formats" 130 | end 131 | end 132 | 133 | if result_url 134 | require 'td/command/result' 135 | result_url = build_result_url(result_url, result_user, result_ask_password) 136 | end 137 | 138 | client = get_client 139 | 140 | # local existence check 141 | get_database(client, db_name) 142 | 143 | opts = {} 144 | opts['type'] = type if type 145 | opts['pool_name'] = pool_name if pool_name 146 | opts['domain_key'] = domain_key if domain_key 147 | opts['engine_version'] = engine_version if engine_version 148 | job = client.query(db_name, sql, result_url, priority, retry_limit, opts) 149 | 150 | $stdout.puts "Job #{job.job_id} is queued." 151 | $stdout.puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job.job_id}' to show the status." 152 | #puts "See #{job.url} to see the progress." 153 | 154 | if wait != false # `wait==nil` means `--wait` is specified 155 | wait_job(job, true, wait) 156 | $stdout.puts "Status : #{job.status}" 157 | if job.success? && !exclude 158 | begin 159 | show_result_with_retry(job, output, limit, format, render_opts) 160 | rescue TreasureData::NotFoundError => e 161 | end 162 | end 163 | elsif output 164 | $stdout.puts "The output file won't be generated without additional `-w` or `--wait` option." 165 | end 166 | end 167 | 168 | require 'td/command/job' # wait_job, job_priority_id_of 169 | end 170 | end 171 | -------------------------------------------------------------------------------- /lib/td/command/result.rb: -------------------------------------------------------------------------------- 1 | 2 | module TreasureData 3 | module Command 4 | 5 | def result_show(op) 6 | name = op.cmd_parse 7 | client = get_client 8 | 9 | rs = client.results 10 | r = rs.find {|r| name == r.name } 11 | 12 | unless r 13 | $stderr.puts "Result URL '#{name}' does not exist." 14 | $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "result:create #{name} ' to create the URL." 15 | exit 1 16 | end 17 | 18 | $stdout.puts "Name : #{r.name}" 19 | $stdout.puts "URL : #{r.url}" 20 | end 21 | 22 | def result_list(op) 23 | set_render_format_option(op) 24 | 25 | op.cmd_parse 26 | 27 | client = get_client 28 | 29 | rs = client.results 30 | 31 | rows = [] 32 | rs.each {|r| 33 | rows << {:Name => r.name, :URL => r.url} 34 | } 35 | rows = rows.sort_by {|map| 36 | map[:Name] 37 | } 38 | 39 | $stdout.puts cmd_render_table(rows, :fields => [:Name, :URL], :render_format => op.render_format) 40 | 41 | if rs.empty? 42 | $stderr.puts "There are no result URLs." 43 | $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "result:create ' to create a result URL." 44 | end 45 | end 46 | 47 | def result_create(op) 48 | result_user = nil 49 | result_ask_password = false 50 | 51 | op.on('-u', '--user NAME', 'set user name for authentication') {|s| 52 | result_user = s 53 | } 54 | op.on('-p', '--password', 'ask password for authentication') {|s| 55 | result_ask_password = true 56 | } 57 | 58 | name, url = op.cmd_parse 59 | API.validate_result_set_name(name) 60 | 61 | client = get_client 62 | 63 | url = build_result_url(url, result_user, result_ask_password) 64 | 65 | opts = {} 66 | begin 67 | client.create_result(name, url, opts) 68 | rescue AlreadyExistsError 69 | $stderr.puts "Result URL '#{name}' already exists." 70 | exit 1 71 | end 72 | 73 | $stderr.puts "Result URL '#{name}' is created." 74 | end 75 | 76 | def result_delete(op) 77 | name = op.cmd_parse 78 | 79 | client = get_client 80 | 81 | begin 82 | client.delete_result(name) 83 | rescue NotFoundError 84 | $stderr.puts "Result URL '#{name}' does not exist." 85 | exit 1 86 | end 87 | 88 | $stderr.puts "Result URL '#{name}' is deleted." 89 | end 90 | 91 | private 92 | def build_result_url(url, user, ask_password) 93 | if ask_password 94 | begin 95 | system "stty -echo" # TODO termios 96 | $stdout.print "Password (typing will be hidden): " 97 | password = STDIN.gets || "" 98 | password = password[0..-2] # strip \n 99 | rescue Interrupt 100 | $stderr.print "\ncanceled." 101 | exit 1 102 | ensure 103 | system "stty echo" # TODO termios 104 | $stdout.print "\n" 105 | end 106 | end 107 | 108 | ups = nil 109 | if user && password 110 | require 'cgi' 111 | ups = "#{CGI.escape(user)}:#{CGI.escape(password)}@" 112 | elsif user 113 | require 'cgi' 114 | ups = "#{CGI.escape(user)}@" 115 | elsif password 116 | require 'cgi' 117 | ups = ":#{CGI.escape(password)}@" 118 | end 119 | if ups 120 | url = url.sub(/\A([\w]+:(?:\/\/)?)/, "\\1#{ups}") 121 | end 122 | 123 | url 124 | end 125 | 126 | # DEPRECATED: relying on API server side validation which will return 127 | # immediately after query submission with error code 422. 128 | private 129 | def validate_td_result_url(url) 130 | re = /td:\/\/[^@]*@\/(.*)\/([^?]+)/ 131 | match = re.match(url) 132 | if match.nil? 133 | raise ParameterConfigurationError, "Treasure Data result output invalid URL format" 134 | end 135 | dbs = match[1] 136 | tbl = match[2] 137 | begin 138 | API.validate_name("Treasure Data result output destination database", 3, 256, dbs) 139 | API.validate_name("Treasure Data result output destination table", 3, 256, tbl) 140 | rescue ParameterValidationError => e 141 | raise ParameterConfigurationError, e 142 | end 143 | end 144 | end 145 | end 146 | 147 | -------------------------------------------------------------------------------- /lib/td/command/runner.rb: -------------------------------------------------------------------------------- 1 | module TreasureData 2 | module Command 3 | 4 | class Runner 5 | def initialize 6 | @config_path = nil 7 | @apikey = nil 8 | @endpoint = nil 9 | @import_endpoint = nil 10 | @prog_name = nil 11 | @insecure = false 12 | end 13 | 14 | attr_accessor :apikey, :endpoint, :import_endpoint, :config_path, :prog_name, :insecure 15 | 16 | def run(argv=ARGV) 17 | require 'td/version' 18 | require 'td/compat_core' 19 | require 'optparse' 20 | 21 | $prog = @prog_name || File.basename($0) 22 | 23 | op = OptionParser.new 24 | op.version = TOOLBELT_VERSION 25 | op.banner = < e 191 | # known exceptions are rendered as simple error messages unless the 192 | # TD_TOOLBELT_DEBUG variable is set or the -v / --verbose option is used. 193 | # List of known exceptions: 194 | # => ParameterConfigurationError 195 | # => BulkImportExecutionError 196 | # => UpUpdateError 197 | # => ImportError 198 | require 'td/client/api' 199 | # => APIError 200 | # => ForbiddenError 201 | # => NotFoundError 202 | # => AuthError 203 | if ![ParameterConfigurationError, BulkImportExecutionError, UpdateError, ImportError, 204 | APIError, ForbiddenError, NotFoundError, AuthError, AlreadyExistsError, WorkflowError].include?(e.class) || 205 | !ENV['TD_TOOLBELT_DEBUG'].nil? || $verbose 206 | show_backtrace "Error #{$!.class}: backtrace:", $!.backtrace 207 | end 208 | 209 | if $!.respond_to?(:api_backtrace) && $!.api_backtrace 210 | show_backtrace "Error backtrace from server:", $!.api_backtrace.split("\n") 211 | end 212 | 213 | $stdout.print "Error: " 214 | if [ForbiddenError, NotFoundError, AuthError].include?(e.class) 215 | $stdout.print "#{e.class} - " 216 | end 217 | $stdout.puts $!.to_s 218 | 219 | require 'socket' 220 | if e.is_a?(::SocketError) 221 | $stderr.puts < --json #{fname}' to import this file." 29 | end 30 | 31 | end 32 | end 33 | 34 | -------------------------------------------------------------------------------- /lib/td/command/schema.rb: -------------------------------------------------------------------------------- 1 | module TreasureData::Command 2 | 3 | def schema_show(op) 4 | db_name, table_name = op.cmd_parse 5 | 6 | client = get_client 7 | table = get_table(client, db_name, table_name) 8 | 9 | $stdout.puts "#{db_name}.#{table_name} (" 10 | table.schema.fields.each {|f| 11 | if f.sql_alias 12 | $stdout.puts " #{f.name}:#{f.type}@#{f.sql_alias}" 13 | else 14 | $stdout.puts " #{f.name}:#{f.type}" 15 | end 16 | } 17 | $stdout.puts ")" 18 | end 19 | 20 | def schema_set(op) 21 | db_name, table_name, *columns = op.cmd_parse 22 | 23 | client = get_client 24 | table = get_table(client, db_name, table_name) 25 | 26 | schema = TreasureData::Schema.parse(columns) 27 | client.update_schema(db_name, table_name, schema) 28 | 29 | $stderr.puts "Schema is updated on #{db_name}.#{table_name} table." 30 | end 31 | 32 | def schema_add(op) 33 | db_name, table_name, *columns = op.cmd_parse 34 | 35 | client = get_client 36 | table = get_table(client, db_name, table_name) 37 | 38 | schema = table.schema.merge(TreasureData::Schema.parse(columns)) 39 | client.update_schema(db_name, table_name, schema) 40 | 41 | $stderr.puts "Schema is updated on #{db_name}.#{table_name} table." 42 | end 43 | 44 | def schema_remove(op) 45 | db_name, table_name, *columns = op.cmd_parse 46 | 47 | client = get_client 48 | table = get_table(client, db_name, table_name) 49 | 50 | schema = table.schema 51 | 52 | columns.each {|col| 53 | unless schema.fields.reject!{|f| f.name == col } 54 | $stderr.puts "Column name '#{col}' does not exist." 55 | exit 1 56 | end 57 | } 58 | 59 | client.update_schema(db_name, table_name, schema) 60 | 61 | $stderr.puts "Schema is updated on #{db_name}.#{table_name} table." 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /lib/td/command/server.rb: -------------------------------------------------------------------------------- 1 | require 'uri' 2 | 3 | module TreasureData 4 | module Command 5 | 6 | def server_status(op) 7 | op.cmd_parse 8 | 9 | $stdout.puts Client.server_status 10 | end 11 | 12 | def server_endpoint(op) 13 | endpoint = op.cmd_parse 14 | 15 | if Config.cl_endpoint and endpoint != Config.endpoint 16 | raise ParameterConfigurationError, 17 | "You specified the API server endpoint in the command options as well (-e / --endpoint " + 18 | "option) but it does not match the value provided to the 'server:endpoint' command. " + 19 | "Please remove the option or ensure the endpoints URLs match each other." 20 | end 21 | 22 | Command.validate_api_endpoint(endpoint) 23 | Command.test_api_endpoint(endpoint) 24 | 25 | conf = nil 26 | begin 27 | conf = Config.read 28 | rescue ConfigError 29 | conf = Config.new 30 | end 31 | conf["account.endpoint"] = endpoint 32 | conf.save 33 | end 34 | 35 | end 36 | end 37 | 38 | -------------------------------------------------------------------------------- /lib/td/command/status.rb: -------------------------------------------------------------------------------- 1 | 2 | module TreasureData 3 | module Command 4 | 5 | def status(op) 6 | op.cmd_parse 7 | 8 | client = get_client 9 | 10 | # +----------------+ 11 | # | scheds | 12 | # +----------------+ 13 | # +----------------+ 14 | # | jobs | 15 | # +----------------+ 16 | # +------+ +-------+ 17 | # |tables| |results| 18 | # +------+ +-------+ 19 | 20 | scheds = [] 21 | jobs = [] 22 | tables = [] 23 | results = [] 24 | 25 | s = client.schedules 26 | s.each {|sched| 27 | scheds << {:Name => sched.name, :Cron => sched.cron, :Result => sched.result_url, :Query => sched.query} 28 | } 29 | scheds = scheds.sort_by {|map| 30 | map[:Name] 31 | } 32 | x1, y1 = status_render(0, 0, "[Schedules]", scheds, :fields => [:Name, :Cron, :Result, :Query]) 33 | 34 | j = client.jobs(0, 4) 35 | j.each {|job| 36 | start = job.start_at 37 | elapsed = Command.humanize_elapsed_time(start, job.end_at) 38 | jobs << {:JobID => job.job_id, :Status => job.status, :Query => job.query.to_s, :Start => (start ? start.localtime : ''), :Elapsed => elapsed, :Result => job.result_url} 39 | } 40 | x2, y2 = status_render(0, 0, "[Jobs]", jobs, :fields => [:JobID, :Status, :Start, :Elapsed, :Result, :Query]) 41 | 42 | dbs = client.databases 43 | dbs.map {|db| 44 | db.tables.each {|table| 45 | tables << {:Database => db.name, :Table => table.name, :Count => table.count.to_s, :Size => table.estimated_storage_size_string} 46 | } 47 | } 48 | x3, y3 = status_render(0, 0, "[Tables]", tables, :fields => [:Database, :Table, :Count, :Size]) 49 | 50 | rs = client.results 51 | rs.each {|r| 52 | results << {:Name => r.name, :URL => r.url} 53 | } 54 | results = results.sort_by {|map| 55 | map[:Name] 56 | } 57 | x4, y4 = status_render(x3+2, y3, "[Results]", results, :fields => [:Name, :URL]) 58 | 59 | (y3-y4-1).times do 60 | $stdout.print "\eD" 61 | end 62 | $stdout.print "\eE" 63 | end 64 | 65 | private 66 | def status_render(movex, movey, msg, *args) 67 | lines = cmd_render_table(*args).split("\n") 68 | lines.pop # remove 'N rows in set' line 69 | lines.unshift(msg) 70 | #lines.unshift("") 71 | 72 | $stdout.print "\e[#{movey}A" if movey > 0 73 | 74 | max_width = 0 75 | height = 0 76 | lines.each {|line| 77 | $stdout.print "\e[#{movex}C" if movex > 0 78 | $stdout.puts line 79 | width = line.length 80 | max_width = width if max_width < width 81 | height += 1 82 | } 83 | 84 | return movex+max_width, height 85 | end 86 | 87 | end # module Command 88 | end # module TreasureData 89 | 90 | -------------------------------------------------------------------------------- /lib/td/command/update.rb: -------------------------------------------------------------------------------- 1 | require "td/updater" 2 | 3 | module TreasureData 4 | module Command 5 | 6 | def update(op) 7 | # for gem installation, this command is disallowed - 8 | # it only works for the toolbelt. 9 | if Updater.disable? 10 | $stderr.puts Updater.disable_message 11 | exit 12 | end 13 | 14 | start_time = Time.now 15 | $stdout.puts "Updating 'td' from #{TOOLBELT_VERSION}..." 16 | if new_version = Updater.update 17 | $stdout.puts "Successfully updated to #{new_version} in #{Command.humanize_time((Time.now - start_time).to_i)}." 18 | else 19 | $stdout.puts "Nothing to update." 20 | end 21 | end 22 | 23 | end # module Command 24 | end # module TreasureData 25 | 26 | -------------------------------------------------------------------------------- /lib/td/command/user.rb: -------------------------------------------------------------------------------- 1 | 2 | module TreasureData 3 | module Command 4 | 5 | def user_show(op) 6 | name = op.cmd_parse 7 | 8 | client = get_client 9 | 10 | users = client.users 11 | user = users.find {|user| name == user.name } 12 | unless user 13 | $stderr.puts "User '#{name}' does not exist." 14 | $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "user:create ' to create an user." 15 | exit 1 16 | end 17 | 18 | $stderr.puts "Name : #{user.name}" 19 | $stderr.puts "Email : #{user.email}" 20 | end 21 | 22 | def user_list(op) 23 | set_render_format_option(op) 24 | 25 | op.cmd_parse 26 | 27 | client = get_client 28 | 29 | users = client.users 30 | 31 | rows = [] 32 | users.each {|user| 33 | rows << {:Name => user.name, :Email => user.email} 34 | } 35 | 36 | $stdout.puts cmd_render_table(rows, :fields => [:Name, :Email], :render_format => op.render_format) 37 | 38 | if rows.empty? 39 | $stderr.puts "There are no users." 40 | $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "user:create ' to create an users." 41 | end 42 | end 43 | 44 | def user_create(op) 45 | email = nil 46 | random_password = false 47 | 48 | op.on('-e', '--email EMAIL', "Use this email address to identify the user") {|s| 49 | email = s 50 | } 51 | 52 | op.on('-R', '--random-password', "Generate random password", TrueClass) {|b| 53 | random_password = b 54 | } 55 | 56 | name = op.cmd_parse 57 | 58 | unless email 59 | $stderr.puts "-e, --email EMAIL option is required." 60 | exit 1 61 | end 62 | 63 | if random_password 64 | lower = ('a'..'z').to_a 65 | upper = ('A'..'Z').to_a 66 | digit = ('0'..'9').to_a 67 | symbol = %w[! # $ % - _ = + < >] 68 | 69 | r = [] 70 | 3.times { r << lower.sort_by{rand}.first } 71 | 3.times { r << upper.sort_by{rand}.first } 72 | 2.times { r << digit.sort_by{rand}.first } 73 | 1.times { r << symbol.sort_by{rand}.first } 74 | password = r.sort_by{rand}.join 75 | 76 | $stdout.puts "Password: #{password}" 77 | 78 | else 79 | 3.times do 80 | begin 81 | system "stty -echo" # TODO termios 82 | $stdout.print "Password (typing will be hidden): " 83 | password = STDIN.gets || "" 84 | password = password[0..-2] # strip \n 85 | rescue Interrupt 86 | $stderr.print "\ncanceled." 87 | exit 1 88 | ensure 89 | system "stty echo" # TODO termios 90 | $stdout.print "\n" 91 | end 92 | 93 | if password.empty? 94 | $stderr.puts "canceled." 95 | exit 0 96 | end 97 | 98 | begin 99 | system "stty -echo" # TODO termios 100 | $stdout.print "Retype password: " 101 | password2 = STDIN.gets || "" 102 | password2 = password2[0..-2] # strip \n 103 | rescue Interrupt 104 | $stderr.print "\ncanceled." 105 | exit 1 106 | ensure 107 | system "stty echo" # TODO termios 108 | $stdout.print "\n" 109 | end 110 | 111 | if password == password2 112 | break 113 | end 114 | 115 | $stdout.puts "Doesn't match." 116 | end 117 | end 118 | 119 | client = get_client(:ssl => true) 120 | client.add_user(name, nil, email, password) 121 | 122 | $stderr.puts "User '#{name}' is created." 123 | $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "user:apikeys #{name}' to show the API key." 124 | end 125 | 126 | def user_delete(op) 127 | name = op.cmd_parse 128 | 129 | client = get_client 130 | 131 | client.remove_user(name) 132 | 133 | $stderr.puts "User '#{name}' is deleted." 134 | end 135 | 136 | ## TODO user:email:change 137 | #def user_email_change(op) 138 | #end 139 | 140 | def user_apikey_add(op) 141 | name = op.cmd_parse 142 | 143 | client = get_client 144 | 145 | begin 146 | client.add_apikey(name) 147 | rescue TreasureData::NotFoundError 148 | $stderr.puts "User '#{name}' does not exist." 149 | $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "users' to show users." 150 | exit 1 151 | end 152 | 153 | $stderr.puts "Added an API key to user '#{name}'." 154 | $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "user:apikeys #{name}' to show the API key" 155 | end 156 | 157 | def user_apikey_remove(op) 158 | name, key = op.cmd_parse 159 | 160 | client = get_client 161 | 162 | begin 163 | client.remove_apikey(name, key) 164 | rescue TreasureData::NotFoundError 165 | $stderr.puts "User '#{name}' or API key '#{key}' does not exist." 166 | $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "users' to show users." 167 | $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "user:apikeys '#{key}' to show API keys" 168 | exit 1 169 | end 170 | 171 | $stderr.puts "Removed an an API key from user '#{name}'." 172 | end 173 | 174 | def user_apikey_list(op) 175 | set_render_format_option(op) 176 | 177 | name = op.cmd_parse 178 | 179 | client = get_client 180 | 181 | keys = client.list_apikeys(name) 182 | 183 | rows = [] 184 | keys.each {|key| 185 | rows << {:Key => key} 186 | } 187 | 188 | $stdout.puts cmd_render_table(rows, :fields => [:Key], :render_format => op.render_format) 189 | end 190 | 191 | def user_password_change(op) 192 | name = op.cmd_parse 193 | 194 | password = nil 195 | 196 | 3.times do 197 | begin 198 | system "stty -echo" # TODO termios 199 | $stdout.print "New password (typing will be hidden): " 200 | password = STDIN.gets || "" 201 | password = password[0..-2] # strip \n 202 | rescue Interrupt 203 | $stderr.print "\ncanceled." 204 | exit 1 205 | ensure 206 | system "stty echo" # TODO termios 207 | $stdout.print "\n" 208 | end 209 | 210 | if password.empty? 211 | $stderr.puts "canceled." 212 | exit 0 213 | end 214 | 215 | begin 216 | system "stty -echo" # TODO termios 217 | $stdout.print "Retype new password: " 218 | password2 = STDIN.gets || "" 219 | password2 = password2[0..-2] # strip \n 220 | rescue Interrupt 221 | $stderr.print "\ncanceled." 222 | exit 1 223 | ensure 224 | system "stty echo" # TODO termios 225 | $stdout.print "\n" 226 | end 227 | 228 | if password == password2 229 | break 230 | end 231 | 232 | $stdout.puts "Doesn't match." 233 | end 234 | 235 | client = get_client(:ssl => true) 236 | 237 | client.change_password(name, password) 238 | 239 | $stderr.puts "Password of user '#{name}' changed." 240 | end 241 | 242 | end 243 | end 244 | 245 | -------------------------------------------------------------------------------- /lib/td/compact_format_yamler.rb: -------------------------------------------------------------------------------- 1 | require 'psych' 2 | 3 | module TreasureData 4 | module CompactFormatYamler 5 | module Visitors 6 | class YAMLTree < Psych::Visitors::YAMLTree 7 | # NOTE support 2.0 following 8 | unless self.respond_to? :create 9 | class << self 10 | alias :create :new 11 | end 12 | end 13 | 14 | def visit_Hash o 15 | if o.class == ::Hash && o.values.all? {|v| v.kind_of?(Numeric) || v.kind_of?(String) || v.kind_of?(Symbol) } 16 | register(o, @emitter.start_mapping(nil, nil, true, Psych::Nodes::Mapping::FLOW)) 17 | 18 | o.each do |k,v| 19 | accept k 20 | accept v 21 | end 22 | @emitter.end_mapping 23 | else 24 | super 25 | end 26 | end 27 | end 28 | end 29 | 30 | def self.dump(o, io = nil, options = {}) 31 | if Hash === io 32 | options = io 33 | io = nil 34 | end 35 | 36 | visitor = ::TreasureData::CompactFormatYamler::Visitors::YAMLTree.create options 37 | visitor << o 38 | visitor.tree.yaml io, options 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/td/compat_core.rb: -------------------------------------------------------------------------------- 1 | 2 | # File#size 3 | methods = File.public_instance_methods.map {|m| m.to_sym } 4 | if !methods.include?(:size) 5 | class File 6 | def size 7 | lstat.size 8 | end 9 | end 10 | end 11 | 12 | -------------------------------------------------------------------------------- /lib/td/compat_gzip_reader.rb: -------------------------------------------------------------------------------- 1 | 2 | methods = Zlib::GzipReader.public_instance_methods 3 | if !methods.include?(:readpartial) && !methods.include?('readpartial') 4 | class Zlib::GzipReader 5 | def readpartial(size, out=nil) 6 | o = read(size) 7 | if o 8 | if out 9 | out.replace(o) 10 | return out 11 | else 12 | return o 13 | end 14 | end 15 | raise EOFError, "end of file reached" 16 | end 17 | end 18 | end 19 | 20 | -------------------------------------------------------------------------------- /lib/td/config.rb: -------------------------------------------------------------------------------- 1 | 2 | module TreasureData 3 | 4 | 5 | class ConfigError < StandardError 6 | end 7 | 8 | class ConfigNotFoundError < ConfigError 9 | end 10 | 11 | class ConfigParseError < ConfigError 12 | end 13 | 14 | 15 | class Config 16 | # class variables 17 | @@path = ENV['TREASURE_DATA_CONFIG_PATH'] || ENV['TD_CONFIG_PATH'] || File.join(ENV['HOME'], '.td', 'td.conf') 18 | @@apikey = ENV['TREASURE_DATA_API_KEY'] || ENV['TD_API_KEY'] 19 | @@apikey = nil if @@apikey == "" 20 | @@cl_apikey = false # flag to indicate whether an apikey has been provided through the command-line 21 | @@endpoint = ENV['TREASURE_DATA_API_SERVER'] || ENV['TD_API_SERVER'] 22 | @@endpoint = nil if @@endpoint == "" 23 | @@cl_endpoint = false # flag to indicate whether an endpoint has been provided through the command-line 24 | @@import_endpoint = ENV['TREASURE_DATA_API_IMPORT_SERVER'] || ENV['TD_API_IMPORT_SERVER'] 25 | @@import_endpoint = nil if @@endpoint == "" 26 | @@cl_import_endpoint = false # flag to indicate whether an endpoint has been provided through the command-line option 27 | @@secure = true 28 | @@retry_post_requests = false 29 | 30 | def initialize 31 | @path = nil 32 | @conf = {} # section.key = val 33 | end 34 | 35 | def self.read(path=Config.path, create=false) 36 | new.read(path) 37 | end 38 | 39 | def [](cate_key) 40 | @conf[cate_key] 41 | end 42 | 43 | def []=(cate_key, val) 44 | @conf[cate_key] = val 45 | end 46 | 47 | def delete(cate_key) 48 | @conf.delete(cate_key) 49 | end 50 | 51 | def read(path=@path) 52 | @path = path 53 | begin 54 | data = File.read(@path) 55 | rescue 56 | e = ConfigNotFoundError.new($!.to_s) 57 | e.set_backtrace($!.backtrace) 58 | raise e 59 | end 60 | 61 | section = "" 62 | 63 | data.each_line {|line| 64 | line.strip! 65 | case line 66 | when /^#/ 67 | next 68 | when /\[(.+)\]/ 69 | section = $~[1] 70 | when /^(\w+)\s*=\s*(.+?)\s*$/ 71 | key = $~[1] 72 | val = $~[2] 73 | @conf["#{section}.#{key}"] = val 74 | else 75 | raise ConfigParseError, "invalid config line '#{line}' at #{@path}" 76 | end 77 | } 78 | 79 | self 80 | end 81 | 82 | def save(path=@path||Config.path) 83 | @path = path 84 | write 85 | end 86 | 87 | private 88 | def write 89 | require 'fileutils' 90 | FileUtils.mkdir_p File.dirname(@path) 91 | File.open(@path, "w") {|f| 92 | @conf.keys.map {|cate_key| 93 | cate_key.split('.', 2) 94 | }.zip(@conf.values).group_by {|(section,key), val| 95 | section 96 | }.each {|section,cate_key_vals| 97 | f.puts "[#{section}]" 98 | cate_key_vals.each {|(section,key), val| 99 | f.puts " #{key} = #{val}" 100 | } 101 | } 102 | } 103 | end 104 | 105 | 106 | def self.path 107 | @@path 108 | end 109 | 110 | def self.path=(path) 111 | @@path = path 112 | end 113 | 114 | 115 | def self.secure 116 | @@secure 117 | end 118 | 119 | def self.secure=(secure) 120 | @@secure = secure 121 | end 122 | 123 | 124 | def self.retry_post_requests 125 | @@retry_post_requests 126 | end 127 | 128 | def self.retry_post_requests=(retry_post_requests) 129 | @@retry_post_requests = retry_post_requests 130 | end 131 | 132 | 133 | def self.apikey 134 | @@apikey || Config.read['account.apikey'] 135 | end 136 | 137 | def self.apikey=(apikey) 138 | @@apikey = apikey 139 | end 140 | 141 | def self.cl_apikey 142 | @@cl_apikey 143 | end 144 | 145 | def self.cl_apikey=(flag) 146 | @@cl_apikey = flag 147 | end 148 | 149 | 150 | def self.endpoint 151 | endpoint = @@endpoint || Config.read['account.endpoint'] 152 | endpoint.sub(/(\/)+$/, '') if endpoint 153 | end 154 | 155 | def self.endpoint=(endpoint) 156 | @@endpoint = endpoint 157 | end 158 | 159 | def self.endpoint_domain 160 | (self.endpoint || 'api.treasuredata.com').sub(%r[https?://], '') 161 | end 162 | 163 | def self.cl_endpoint 164 | @@cl_endpoint 165 | end 166 | 167 | def self.cl_endpoint=(flag) 168 | @@cl_endpoint = flag 169 | end 170 | 171 | def self.import_endpoint 172 | endpoint = @@import_endpoint || Config.read['account.import_endpoint'] 173 | endpoint.sub(/(\/)+$/, '') if endpoint 174 | end 175 | 176 | def self.import_endpoint=(endpoint) 177 | @@import_endpoint = endpoint 178 | end 179 | 180 | def self.cl_import_endpoint 181 | @@cl_import_endpoint 182 | end 183 | 184 | def self.cl_import_endpoint=(flag) 185 | @@cl_import_endpoint = flag 186 | end 187 | 188 | def self.workflow_endpoint 189 | case self.endpoint_domain 190 | when /\Aapi(-(?:staging|development))?(-[a-z0-9]+)?\.(connect\.)?((?:eu01|ap02|ap03)\.)?treasuredata\.(com|co\.jp)\z/i 191 | "https://api#{$1}-workflow#{$2}.#{$3}#{$4}treasuredata.#{$5}" 192 | else 193 | raise ConfigError, "Workflow is not supported for '#{self.endpoint}'" 194 | end 195 | end 196 | 197 | # renders the apikey and endpoint options as a string for the helper commands 198 | def self.cl_options_string 199 | string = "" 200 | string += "-k #{@@apikey} " if @@cl_apikey 201 | string += "-e #{@@endpoint} " if @@cl_endpoint 202 | string += "--import-endpoint #{@@import_endpoint} " if @@cl_import_endpoint 203 | string 204 | end 205 | 206 | end # class Config 207 | end # module TreasureData 208 | -------------------------------------------------------------------------------- /lib/td/connector_config_normalizer.rb: -------------------------------------------------------------------------------- 1 | module TreasureData 2 | class ConnectorConfigNormalizer 3 | def initialize(config) 4 | @config = config 5 | end 6 | 7 | def normalized_config 8 | case 9 | when @config['in'] 10 | { 11 | 'in' => @config['in'], 12 | 'out' => @config['out'] || {}, 13 | 'exec' => @config['exec'] || {}, 14 | 'filters' => @config['filters'] || [] 15 | } 16 | when @config['config'] 17 | if @config.size != 1 18 | raise "Setting #{(@config.keys - ['config']).inspect} keys in a configuration file is not supported. Please set options to the command line argument." 19 | end 20 | 21 | self.class.new(@config['config']).normalized_config 22 | else 23 | { 24 | 'in' => @config, 25 | 'out' => {}, 26 | 'exec' => {}, 27 | 'filters' => [] 28 | } 29 | end 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/td/distribution.rb: -------------------------------------------------------------------------------- 1 | module TreasureData 2 | module Distribution 3 | def self.files 4 | fs = Dir[File.expand_path("../../../{bin,data,lib}/**/*", __FILE__)].select do |file| 5 | File.file?(file) 6 | end 7 | fs << File.expand_path("../../../Gemfile", __FILE__) 8 | fs << File.expand_path("../../../td.gemspec", __FILE__) 9 | fs 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/td/helpers.rb: -------------------------------------------------------------------------------- 1 | module TreasureData 2 | module Helpers 3 | module_function 4 | 5 | def format_with_delimiter(number, delimiter = ',') 6 | number.to_s.gsub(/(\d)(?=(?:\d{3})+(?!\d))/, "\\1#{delimiter}") 7 | end 8 | 9 | def home_directory 10 | on_windows? ? ENV['USERPROFILE'].gsub("\\","/") : ENV['HOME'] 11 | end 12 | 13 | def on_windows? 14 | RUBY_PLATFORM =~ /mswin32|mingw32|mingw-ucrt/ 15 | end 16 | 17 | def on_mac? 18 | RUBY_PLATFORM =~ /-darwin\d/ 19 | end 20 | 21 | def on_64bit_os? 22 | if on_windows? 23 | if ENV.fetch('PROCESSOR_ARCHITECTURE', '').downcase.include? 'amd64' 24 | return true 25 | end 26 | return ENV.has_key?('PROCESSOR_ARCHITEW6432') 27 | else 28 | require 'open3' 29 | out, status = Open3.capture2('uname', '-m') 30 | raise 'Failed to detect OS bitness' unless status.success? 31 | return out.downcase.include? 'x86_64' 32 | end 33 | end 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /lib/td/version.rb: -------------------------------------------------------------------------------- 1 | module TreasureData 2 | TOOLBELT_VERSION = '0.18.0' 3 | end 4 | -------------------------------------------------------------------------------- /spec/file_reader/filter_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'file_reader/shared_context' 3 | 4 | require 'stringio' 5 | require 'td/file_reader' 6 | 7 | include TreasureData 8 | 9 | describe 'FileReader filters' do 10 | include_context 'error_proc' 11 | 12 | let :delimiter do 13 | "\t" 14 | end 15 | 16 | let :dataset do 17 | [ 18 | ['hoge', 12345, true, 'null', Time.now.to_s], 19 | ['foo', 34567, false, 'null', Time.now.to_s], 20 | ['piyo', 56789, true, nil, Time.now.to_s], 21 | ] 22 | end 23 | 24 | let :lines do 25 | dataset.map { |data| data.map(&:to_s).join(delimiter) } 26 | end 27 | 28 | let :parser do 29 | io = StringIO.new(lines.join("\n")) 30 | reader = FileReader::LineReader.new(io, error, {}) 31 | FileReader::DelimiterParser.new(reader, error, :delimiter_expr => delimiter) 32 | end 33 | 34 | describe FileReader::AutoTypeConvertParserFilter do 35 | let :options do 36 | { 37 | :null_expr => /\A(?:nil||\-|\\N)\z/i, 38 | :true_expr => /\A(?:true)\z/i, 39 | :false_expr => /\A(?:false)\z/i, 40 | } 41 | end 42 | 43 | it 'initialize' do 44 | filter = FileReader::AutoTypeConvertParserFilter.new(parser, error, options) 45 | expect(filter).not_to be_nil 46 | end 47 | 48 | context 'after initialization' do 49 | let :filter do 50 | FileReader::AutoTypeConvertParserFilter.new(parser, error, options) 51 | end 52 | 53 | it 'forward returns one converted line' do 54 | expect(filter.forward).to eq(dataset[0]) 55 | end 56 | 57 | it 'feeds all lines' do 58 | begin 59 | i = 0 60 | while line = filter.forward 61 | expect(line).to eq(dataset[i]) 62 | i += 1 63 | end 64 | rescue 65 | end 66 | end 67 | end 68 | end 69 | 70 | describe FileReader::HashBuilder do 71 | let :columns do 72 | ['str', 'num', 'bool', 'null', 'log_at'] 73 | end 74 | 75 | let :built_dataset do 76 | # [{"str" => "hoge", "num" => "12345", "bool" => "true" , "null" =>"null", "log_at" => "2012-12-26 05:14:09 +0900"}, ...] 77 | dataset.map { |data| Hash[columns.zip(data.map(&:to_s))]} 78 | end 79 | 80 | it 'initialize' do 81 | builder = FileReader::HashBuilder.new(parser, error, columns) 82 | expect(builder).not_to be_nil 83 | end 84 | 85 | context 'after initialization' do 86 | let :builder do 87 | FileReader::HashBuilder.new(parser, error, columns) 88 | end 89 | 90 | it 'forward returns one converted line' do 91 | expect(builder.forward).to eq(built_dataset[0]) 92 | end 93 | 94 | it 'feeds all lines' do 95 | begin 96 | i = 0 97 | while line = builder.forward 98 | expect(line).to eq(built_dataset[i]) 99 | i += 1 100 | end 101 | rescue 102 | end 103 | end 104 | 105 | describe FileReader::TimeParserFilter do 106 | it "can't be initialized without :time_column option" do 107 | expect { 108 | FileReader::TimeParserFilter.new(parser, error, {}) 109 | }.to raise_error(Exception, /--time-column/) 110 | end 111 | 112 | it 'initialize' do 113 | filter = FileReader::TimeParserFilter.new(builder, error, :time_column => 'log_at') 114 | expect(filter).not_to be_nil 115 | end 116 | 117 | context 'after initialization' do 118 | let :timed_dataset do 119 | require 'time' 120 | built_dataset.each { |data| data['time'] = Time.parse(data['log_at']).to_i } 121 | end 122 | 123 | let :filter do 124 | FileReader::TimeParserFilter.new(builder, error, :time_column => 'log_at') 125 | end 126 | 127 | it 'forward returns one parse line with parsed log_at' do 128 | expect(filter.forward).to eq(timed_dataset[0]) 129 | end 130 | 131 | it 'feeds all lines' do 132 | begin 133 | i = 0 134 | while line = filter.forward 135 | expect(line).to eq(timed_dataset[i]) 136 | i += 1 137 | end 138 | rescue 139 | end 140 | end 141 | 142 | context 'missing log_at column lines' do 143 | let :columns do 144 | ['str', 'num', 'bool', 'null', 'created_at'] 145 | end 146 | 147 | let :error_pattern do 148 | /^time column 'log_at' is missing/ 149 | end 150 | 151 | it 'feeds all lines' do 152 | i = 0 153 | begin 154 | while line = filter.forward 155 | i += 1 156 | end 157 | rescue RSpec::Expectations::ExpectationNotMetError => e 158 | fail 159 | rescue 160 | expect(i).to eq(0) 161 | end 162 | end 163 | end 164 | 165 | context 'invalid time format' do 166 | let :error_pattern do 167 | /^invalid time format/ 168 | end 169 | 170 | [{:time_column => 'log_at', :time_format => "%d"}, 171 | {:time_column => 'str'}].each { |options| 172 | let :filter do 173 | FileReader::TimeParserFilter.new(builder, error, options) 174 | end 175 | 176 | it 'feeds all lines' do 177 | i = 0 178 | begin 179 | while line = filter.forward 180 | i += 1 181 | end 182 | rescue RSpec::Expectations::ExpectationNotMetError => e 183 | fail 184 | rescue 185 | expect(i).to eq(0) 186 | end 187 | end 188 | } 189 | end 190 | end 191 | end 192 | 193 | describe FileReader::SetTimeParserFilter do 194 | it "can't be initialized without :time_value option" do 195 | expect { 196 | FileReader::SetTimeParserFilter.new(parser, error, {}) 197 | }.to raise_error(Exception, /--time-value/) 198 | end 199 | 200 | it 'initialize' do 201 | filter = FileReader::SetTimeParserFilter.new(builder, error, :time_value => Time.now.to_i) 202 | expect(filter).not_to be_nil 203 | end 204 | 205 | context 'after initialization' do 206 | let :time_value do 207 | Time.now.to_i 208 | end 209 | 210 | let :timed_dataset do 211 | built_dataset.each { |data| data['time'] = time_value } 212 | end 213 | 214 | let :filter do 215 | FileReader::SetTimeParserFilter.new(builder, error, :time_value => time_value) 216 | end 217 | 218 | it 'forward returns one converted line with time' do 219 | expect(filter.forward).to eq(timed_dataset[0]) 220 | end 221 | 222 | it 'feeds all lines' do 223 | begin 224 | i = 0 225 | while line = filter.forward 226 | expect(line).to eq(timed_dataset[i]) 227 | i += 1 228 | end 229 | rescue 230 | end 231 | end 232 | end 233 | end 234 | end 235 | end 236 | end 237 | -------------------------------------------------------------------------------- /spec/file_reader/io_filter_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'file_reader/shared_context' 3 | 4 | require 'stringio' 5 | require 'td/file_reader' 6 | 7 | include TreasureData 8 | 9 | describe 'FileReader io filters' do 10 | include_context 'error_proc' 11 | 12 | describe FileReader::DecompressIOFilter do 13 | let :lines do 14 | [ 15 | '{"host":"128.216.140.97","user":"-","method":"GET","path":"/item/sports/2511","code":200,"size":95,"time":1353541928}', 16 | '{"host":"224.225.147.72","user":"-","method":"GET","path":"/category/electronics","code":200,"size":43,"time":1353541927}', 17 | '{"host":"172.75.186.56","user":"-","method":"GET","path":"/category/jewelry","code":200,"size":79,"time":1353541925}', 18 | ] 19 | end 20 | 21 | let :io do 22 | StringIO.new(lines.join("\n")) 23 | end 24 | 25 | let :gzipped_io do 26 | require 'zlib' 27 | 28 | io = StringIO.new('', 'w+') 29 | gz = Zlib::GzipWriter.new(io) 30 | gz.write(lines.join("\n")) 31 | gz.close 32 | StringIO.new(io.string) 33 | end 34 | 35 | it "can't filter with unknown compression" do 36 | expect { 37 | FileReader::DecompressIOFilter.filter(io, error, :compress => 'oreore') 38 | }.to raise_error(Exception, /unknown compression/) 39 | end 40 | 41 | describe 'gzip' do 42 | it "can't be wrapped with un-gzipped io" do 43 | expect { 44 | FileReader::DecompressIOFilter.filter(io, error, :compress => 'gzip') 45 | }.to raise_error(Zlib::GzipFile::Error) 46 | end 47 | 48 | it 'returns Zlib::GzipReader with :gzip' do 49 | wrapped = FileReader::DecompressIOFilter.filter(gzipped_io, error, :compress => 'gzip') 50 | expect(wrapped).to be_an_instance_of(Zlib::GzipReader) 51 | end 52 | 53 | it 'returns Zlib::GzipReader with auto detection' do 54 | wrapped = FileReader::DecompressIOFilter.filter(gzipped_io, error, {}) 55 | expect(wrapped).to be_an_instance_of(Zlib::GzipReader) 56 | end 57 | 58 | context 'after initialization' do 59 | [{:compress => 'gzip'}, {}].each { |opts| 60 | let :reader do 61 | wrapped = FileReader::DecompressIOFilter.filter(gzipped_io, error, opts) 62 | FileReader::LineReader.new(wrapped, error, {}) 63 | end 64 | 65 | it 'forward_row returns one line' do 66 | expect(reader.forward_row).to eq(lines[0]) 67 | end 68 | 69 | it 'feeds all lines' do 70 | begin 71 | i = 0 72 | while line = reader.forward_row 73 | expect(line).to eq(lines[i]) 74 | i += 1 75 | end 76 | rescue 77 | expect(gzipped_io.eof?).to be_truthy 78 | end 79 | end 80 | } 81 | end 82 | end 83 | 84 | describe 'plain' do 85 | it 'returns passed io with :plain' do 86 | wrapped = FileReader::DecompressIOFilter.filter(io, error, :compress => 'plain') 87 | expect(wrapped).to be_an_instance_of(StringIO) 88 | end 89 | 90 | it 'returns passed io with auto detection' do 91 | wrapped = FileReader::DecompressIOFilter.filter(io, error, {}) 92 | expect(wrapped).to be_an_instance_of(StringIO) 93 | end 94 | end 95 | end 96 | end 97 | -------------------------------------------------------------------------------- /spec/file_reader/line_reader_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'file_reader/shared_context' 3 | 4 | require 'stringio' 5 | require 'td/file_reader' 6 | 7 | include TreasureData 8 | 9 | describe FileReader::LineReader do 10 | include_context 'error_proc' 11 | 12 | let :lines do 13 | [ 14 | '{"host":"128.216.140.97","user":"-","method":"GET","path":"/item/sports/2511","code":200,"referer":"http://www.google.com/search?ie=UTF-8&q=google&sclient=psy-ab&q=Sports+Electronics&oq=Sports+Electronics&aq=f&aqi=g-vL1&aql=&pbx=1&bav=on.2,or.r_gc.r_pw.r_qf.,cf.osb&biw=3994&bih=421","size":95,"agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7","time":1353541928}', 15 | '{"host":"224.225.147.72","user":"-","method":"GET","path":"/category/electronics","code":200,"referer":"-","size":43,"agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)","time":1353541927}', 16 | '{"host":"172.75.186.56","user":"-","method":"GET","path":"/category/jewelry","code":200,"referer":"-","size":79,"agent":"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)","time":1353541925}', 17 | ] 18 | end 19 | 20 | let :io do 21 | StringIO.new(lines.join("\n")) 22 | end 23 | 24 | it 'initialize' do 25 | if io.respond_to?(:external_encoding) 26 | ee = io.external_encoding 27 | end 28 | FileReader::LineReader.new(io, error, {}) 29 | if io.respond_to?(:external_encoding) 30 | expect(io.external_encoding).to eq(ee) 31 | end 32 | end 33 | 34 | it 'initialize with specifid encoding' do 35 | if io.respond_to?(:external_encoding) 36 | original_encoding = io.external_encoding 37 | end 38 | 39 | # when RUBY_VERSION >= 2.0, default encoding is utf-8. 40 | # ensure that external_encoding is differ from the original external_encoding(original_encoding). 41 | if original_encoding == Encoding.find('utf-8') 42 | specified_encoding = 'sjis' 43 | else 44 | specified_encoding = 'utf-8' 45 | end 46 | 47 | FileReader::LineReader.new(io, error, {:encoding => specified_encoding}) 48 | if io.respond_to?(:external_encoding) 49 | expect(io.external_encoding).not_to eq(original_encoding) 50 | expect(io.external_encoding).to eq(Encoding.find(specified_encoding)) 51 | end 52 | end 53 | 54 | context 'after initialization' do 55 | let :reader do 56 | FileReader::LineReader.new(io, error, {}) 57 | end 58 | 59 | it 'forward_row returns one line' do 60 | expect(reader.forward_row).to eq(lines[0]) 61 | end 62 | 63 | # TODO: integrate with following shared_examples_for 64 | it 'feeds all lines' do 65 | begin 66 | i = 0 67 | while line = reader.forward_row 68 | expect(line).to eq(lines[i]) 69 | i += 1 70 | end 71 | rescue RSpec::Expectations::ExpectationNotMetError => e 72 | fail 73 | rescue 74 | expect(io.eof?).to be_truthy 75 | end 76 | end 77 | 78 | shared_examples_for 'parser iterates all' do |step| 79 | step = step || 1 80 | 81 | it 'feeds all' do 82 | begin 83 | i = 0 84 | while line = parser.forward 85 | expect(line).to eq(get_expected.call(i)) 86 | i += step 87 | end 88 | rescue RSpec::Expectations::ExpectationNotMetError => e 89 | fail 90 | rescue 91 | expect(io.eof?).to be_truthy 92 | end 93 | end 94 | end 95 | 96 | describe FileReader::JSONParser do 97 | it 'initialize with LineReader' do 98 | parser = FileReader::JSONParser.new(reader, error, {}) 99 | expect(parser).not_to be_nil 100 | end 101 | 102 | context 'after initialization' do 103 | let :parser do 104 | FileReader::JSONParser.new(reader, error, {}) 105 | end 106 | 107 | it 'forward returns one line' do 108 | expect(parser.forward).to eq(JSON.parse(lines[0])) 109 | end 110 | 111 | let :get_expected do 112 | lambda { |i| JSON.parse(lines[i]) } 113 | end 114 | 115 | it_should_behave_like 'parser iterates all' 116 | 117 | context 'with broken line' do 118 | let :lines do 119 | [ 120 | '{"host":"128.216.140.97","user":"-","method":"GET","path":"/item/sports/2511","code":200,"referer":"http://www.google.com/search?ie=UTF-8&q=google&sclient=psy-ab&q=Sports+Electronics&oq=Sports+Electronics&aq=f&aqi=g-vL1&aql=&pbx=1&bav=on.2,or.r_gc.r_pw.r_qf.,cf.osb&biw=3994&bih=421","size":95,"agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7","time":1353541928}', 121 | '{This is invalid as a JSON}', 122 | '{"host":"172.75.186.56","user":"-","method":"GET","path":"/category/jewelry","code":200,"referer":"-","size":79,"agent":"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)","time":1353541925}', 123 | ] 124 | end 125 | 126 | let :error_pattern do 127 | /^invalid json format/ 128 | end 129 | 130 | it_should_behave_like 'parser iterates all', 2 131 | end 132 | end 133 | end 134 | 135 | [',', "\t"].each { |pattern| 136 | describe FileReader::DelimiterParser do 137 | let :lines do 138 | [ 139 | ['hoge', '12345', Time.now.to_s].join(pattern), 140 | ['foo', '34567', Time.now.to_s].join(pattern), 141 | ['piyo', '56789', Time.now.to_s].join(pattern), 142 | ] 143 | end 144 | 145 | it "initialize with LineReader and #{pattern} delimiter" do 146 | parser = FileReader::DelimiterParser.new(reader, error, {:delimiter_expr => Regexp.new(pattern)}) 147 | expect(parser).not_to be_nil 148 | end 149 | 150 | context 'after initialization' do 151 | let :parser do 152 | FileReader::DelimiterParser.new(reader, error, {:delimiter_expr => Regexp.new(pattern)}) 153 | end 154 | 155 | it 'forward returns one line' do 156 | expect(parser.forward).to eq(lines[0].split(pattern)) 157 | end 158 | 159 | let :get_expected do 160 | lambda { |i| lines[i].split(pattern) } 161 | end 162 | 163 | it_should_behave_like 'parser iterates all' 164 | end 165 | end 166 | } 167 | 168 | { 169 | FileReader::ApacheParser => [ 170 | [ 171 | '58.83.188.60 - - [23/Oct/2011:08:15:46 -0700] "HEAD / HTTP/1.0" 200 277 "-" "-"', 172 | '127.0.0.1 - - [23/Oct/2011:08:20:01 -0700] "GET / HTTP/1.0" 200 492 "-" "Wget/1.12 (linux-gnu)"', 173 | '68.64.37.100 - - [24/Oct/2011:01:48:54 -0700] "GET /phpMyAdmin/scripts/setup.php HTTP/1.1" 404 480 "-" "ZmEu"' 174 | ], 175 | [ 176 | ["58.83.188.60", "-", "23/Oct/2011:08:15:46 -0700", "HEAD", "/", "200", "277", "-", "-"], 177 | ["127.0.0.1", "-", "23/Oct/2011:08:20:01 -0700", "GET", "/", "200", "492", "-", "Wget/1.12 (linux-gnu)"], 178 | ["68.64.37.100", "-", "24/Oct/2011:01:48:54 -0700", "GET", "/phpMyAdmin/scripts/setup.php", "404", "480", "-", "ZmEu"], 179 | ] 180 | ], 181 | FileReader::SyslogParser => [ 182 | [ 183 | 'Dec 20 12:41:44 localhost kernel: [4843680.692840] e1000e: eth2 NIC Link is Down', 184 | 'Dec 20 12:41:44 localhost kernel: [4843680.734466] br0: port 1(eth2) entering disabled state', 185 | 'Dec 22 10:42:41 localhost kernel: [5009052.220155] zsh[25578]: segfault at 7fe849460260 ip 00007fe8474fd74d sp 00007fffe3bdf0e0 error 4 in libc-2.11.1.so[7fe847486000+17a000]', 186 | ], 187 | [ 188 | ["Dec 20 12:41:44", "localhost", "kernel", nil, "[4843680.692840] e1000e: eth2 NIC Link is Down"], 189 | ["Dec 20 12:41:44", "localhost", "kernel", nil, "[4843680.734466] br0: port 1(eth2) entering disabled state"], 190 | ["Dec 22 10:42:41", "localhost", "kernel", nil, "[5009052.220155] zsh[25578]: segfault at 7fe849460260 ip 00007fe8474fd74d sp 00007fffe3bdf0e0 error 4 in libc-2.11.1.so[7fe847486000+17a000]"], 191 | ] 192 | ] 193 | }.each_pair { |parser_class, (input, output)| 194 | describe parser_class do 195 | let :lines do 196 | input 197 | end 198 | 199 | it "initialize with LineReader" do 200 | parser = parser_class.new(reader, error, {}) 201 | expect(parser).not_to be_nil 202 | end 203 | 204 | context 'after initialization' do 205 | let :parser do 206 | parser_class.new(reader, error, {}) 207 | end 208 | 209 | it 'forward returns one line' do 210 | expect(parser.forward).to eq(output[0]) 211 | end 212 | 213 | let :get_expected do 214 | lambda { |i| output[i] } 215 | end 216 | 217 | it_should_behave_like 'parser iterates all' 218 | 219 | context 'with broken line' do 220 | let :lines do 221 | broken = input.dup 222 | broken[1] = "Raw text sometimes is broken!" 223 | broken 224 | end 225 | 226 | let :error_pattern do 227 | /^invalid #{parser.instance_variable_get(:@format)} format/ 228 | end 229 | 230 | it_should_behave_like 'parser iterates all', 2 231 | end 232 | end 233 | end 234 | } 235 | end 236 | end 237 | -------------------------------------------------------------------------------- /spec/file_reader/parsing_reader_spec.rb: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | require 'spec_helper' 3 | require 'file_reader/shared_context' 4 | 5 | require 'stringio' 6 | require 'msgpack' 7 | require 'td/file_reader' 8 | 9 | include TreasureData 10 | 11 | describe 'FileReader parsing readers' do 12 | include_context 'error_proc' 13 | 14 | shared_examples_for 'forward basics' do 15 | it 'forward returns one data' do 16 | expect(reader.forward).to eq(dataset[0]) 17 | end 18 | 19 | it 'feeds all dataset' do 20 | begin 21 | i = 0 22 | while line = reader.forward 23 | expect(line).to eq(dataset[i]) 24 | i += 1 25 | end 26 | rescue RSpec::Expectations::ExpectationNotMetError => e 27 | fail 28 | rescue => e 29 | expect(io.eof?).to be_truthy 30 | end 31 | end 32 | end 33 | 34 | describe FileReader::MessagePackParsingReader do 35 | let :dataset do 36 | [ 37 | {'name' => 'k', 'num' => 12345, 'time' => Time.now.to_i}, 38 | {'name' => 's', 'num' => 34567, 'time' => Time.now.to_i}, 39 | {'name' => 'n', 'num' => 56789, 'time' => Time.now.to_i}, 40 | ] 41 | end 42 | 43 | let :io do 44 | StringIO.new(dataset.map(&:to_msgpack).join("")) 45 | end 46 | 47 | it 'initialize' do 48 | reader = FileReader::MessagePackParsingReader.new(io, error, {}) 49 | expect(reader).not_to be_nil 50 | end 51 | 52 | context 'after initialization' do 53 | let :reader do 54 | FileReader::MessagePackParsingReader.new(io, error, {}) 55 | end 56 | 57 | it_should_behave_like 'forward basics' 58 | end 59 | end 60 | 61 | test_time = Time.now.to_i 62 | { 63 | 'csv' => [ 64 | {:delimiter_expr => ',', :quote_char => '"', :encoding => 'utf-8'}, 65 | [ 66 | %!k,123,"fo\no",true,#{test_time}!, 67 | %!s,456,"Ｔ,Ｄ",false,#{test_time}!, 68 | %!n,789,"ba""z",false,#{test_time}!, 69 | ], 70 | [ 71 | %W(k 123 fo\no true #{test_time}), 72 | %W(s 456 Ｔ,Ｄ false #{test_time}), 73 | %W(n 789 ba\"z false #{test_time}), 74 | ] 75 | ], 76 | 'tsv' => [ 77 | {:delimiter_expr => "\t"}, 78 | [ 79 | %!k\t123\t"fo\no"\ttrue\t#{test_time}!, 80 | %!s\t456\t"b,ar"\tfalse\t#{test_time}!, 81 | %!n\t789\t"ba\tz"\tfalse\t#{test_time}!, 82 | ], 83 | [ 84 | %W(k 123 fo\no true #{test_time}), 85 | %W(s 456 b,ar false #{test_time}), 86 | %W(n 789 ba\tz false #{test_time}), 87 | ] 88 | ] 89 | }.each_pair { |format, (opts, input, output)| 90 | describe FileReader::SeparatedValueParsingReader do 91 | let :dataset do 92 | output 93 | end 94 | 95 | let :lines do 96 | input 97 | end 98 | 99 | let :io do 100 | StringIO.new(lines.join($/)) 101 | end 102 | 103 | it "initialize #{format}" do 104 | reader = FileReader::SeparatedValueParsingReader.new(io, error, opts) 105 | expect(reader).not_to be_nil 106 | end 107 | 108 | context "after #{format} initialization" do 109 | let :reader do 110 | reader = FileReader::SeparatedValueParsingReader.new(io, error, opts) 111 | end 112 | 113 | it_should_behave_like 'forward basics' 114 | 115 | context "broken encodings" do 116 | end 117 | end 118 | end 119 | } 120 | end 121 | -------------------------------------------------------------------------------- /spec/file_reader/shared_context.rb: -------------------------------------------------------------------------------- 1 | require 'rspec' 2 | 3 | shared_context 'error_proc' do 4 | let :error do 5 | Proc.new { |reason, data| 6 | expect(reason).to match(error_pattern) 7 | } 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 2 | $LOAD_PATH.unshift(File.dirname(__FILE__)) 3 | 4 | require 'rspec' 5 | require 'json' 6 | 7 | if ENV['SIMPLE_COV'] 8 | # SimpleCov 9 | # https://github.com/colszowka/simplecov 10 | require 'simplecov' 11 | SimpleCov.start do 12 | add_filter 'spec/' 13 | add_filter 'pkg/' 14 | add_filter 'vendor/' 15 | end 16 | end 17 | 18 | # XXX skip coverage setting if run appveyor. Because, fail to push coveralls in appveyor. 19 | unless ENV['APPVEYOR'] 20 | require 'coveralls' 21 | Coveralls.wear!('rails') 22 | end 23 | 24 | RSpec.configure do |config| 25 | # This allows you to limit a spec run to individual examples or groups 26 | # you care about by tagging them with `:focus` metadata. When nothing 27 | # is tagged with `:focus`, all examples get run. RSpec also provides 28 | # aliases for `it`, `describe`, and `context` that include `:focus` 29 | # metadata: `fit`, `fdescribe` and `fcontext`, respectively. 30 | config.filter_run_when_matching :focus 31 | end 32 | 33 | require 'td/command/runner' 34 | 35 | def execute_td(command_line) 36 | args = command_line.split(" ") 37 | original_stdin, original_stderr, original_stdout = $stdin, $stderr, $stdout 38 | 39 | $stdin = captured_stdin = StringIO.new 40 | $stderr = captured_stderr = StringIO.new 41 | $stdout = captured_stdout = StringIO.new 42 | class << captured_stdout 43 | def tty? 44 | true 45 | end 46 | end 47 | 48 | begin 49 | runner = TreasureData::Command::Runner.new 50 | $0 = 'td' 51 | runner.run(args) 52 | rescue SystemExit 53 | ensure 54 | $stdin, $stderr, $stdout = original_stdin, original_stderr, original_stdout 55 | end 56 | 57 | [captured_stderr.string, captured_stdout.string] 58 | end 59 | 60 | class CallSystemExitError < RuntimeError; end 61 | 62 | shared_context 'quiet_out' do 63 | let(:stdout_io) { StringIO.new } 64 | let(:stderr_io) { StringIO.new } 65 | 66 | around do |example| 67 | out = $stdout.dup 68 | err= $stdout.dup 69 | begin 70 | $stdout = stdout_io 71 | $stderr = stderr_io 72 | example.call 73 | ensure 74 | $stdout = out 75 | $stderr = err 76 | end 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /spec/td/command/account_spec.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'spec_helper' 4 | require "tempfile" 5 | require 'td/command/common' 6 | require 'td/command/account' 7 | require 'td/command/list' 8 | require 'td/helpers' 9 | 10 | module TreasureData::Command 11 | describe 'account command' do 12 | let :command do 13 | Class.new { include TreasureData::Command }.new 14 | end 15 | let(:client) { double(:client) } 16 | let(:conf_file) { Tempfile.new("td.conf").tap {|s| s.close } } 17 | let(:conf_expect) { 18 | """[account] 19 | user = test@email.com 20 | apikey = 1/xxx 21 | """ 22 | } 23 | 24 | before do 25 | TreasureData::Config.path = conf_file.path 26 | end 27 | 28 | it 'is called without any option' do 29 | expect(STDIN).to receive(:gets).and_return('test@email.com') 30 | if TreasureData::Helpers.on_windows? 31 | 'password'.chars.each { |c| expect(command).to receive(:get_char).and_return(c) } 32 | expect(command).to receive(:get_char).and_return(nil) 33 | else 34 | expect(STDIN).to receive(:gets).and_return('password') 35 | end 36 | expect(TreasureData::Client).to receive(:authenticate).and_return(client) 37 | expect(client).to receive(:apikey).and_return("1/xxx") 38 | 39 | op = List::CommandParser.new("account", %w[], %w[], false, [], true) 40 | command.account(op) 41 | 42 | expect(File.read(conf_file.path)).to eq(conf_expect) 43 | end 44 | 45 | it 'is called with -f option' do 46 | if TreasureData::Helpers.on_windows? 47 | 'password'.chars.each { |c| expect(command).to receive(:get_char).and_return(c) } 48 | expect(command).to receive(:get_char).and_return(nil) 49 | else 50 | expect(STDIN).to receive(:gets).and_return('password') 51 | end 52 | expect(TreasureData::Client).to receive(:authenticate).and_return(client) 53 | expect(client).to receive(:apikey).and_return("1/xxx") 54 | 55 | op = List::CommandParser.new("account", %w[-f], %w[], false, ['test@email.com'], true) 56 | command.account(op) 57 | 58 | expect(File.read(conf_file.path)).to eq(conf_expect) 59 | end 60 | 61 | it 'is called with username password mismatched' do 62 | if TreasureData::Helpers.on_windows? 63 | 3.times do 64 | 'password'.chars.each { |c| expect(command).to receive(:get_char).and_return(c) } 65 | expect(command).to receive(:get_char).and_return(nil) 66 | end 67 | else 68 | expect(STDIN).to receive(:gets).and_return('password').thrice 69 | end 70 | expect(TreasureData::Client).to receive(:authenticate).thrice.and_raise(TreasureData::AuthError) 71 | expect(STDERR).to receive(:puts).with('User name or password mismatched.').thrice 72 | 73 | op = List::CommandParser.new("account", %w[-f], %w[], false, ['test@email.com'], true) 74 | command.account(op) 75 | 76 | expect(File.read(conf_file.path)).to eq("") 77 | end 78 | end 79 | end -------------------------------------------------------------------------------- /spec/td/command/export_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/command/common' 3 | require 'td/command/list' 4 | require 'td/command/export' 5 | 6 | module TreasureData::Command 7 | describe 'export commands' do 8 | let(:command) { 9 | Class.new { include TreasureData::Command }.new 10 | } 11 | let(:stdout_io) { StringIO.new } 12 | let(:stderr_io) { StringIO.new } 13 | 14 | around do |example| 15 | stdout = $stdout.dup 16 | stderr = $stderr.dup 17 | 18 | begin 19 | $stdout = stdout_io 20 | $stderr = stderr_io 21 | 22 | example.run 23 | ensure 24 | $stdout = stdout 25 | $stderr = stderr 26 | end 27 | end 28 | 29 | describe '#table_export' do 30 | let(:option) { 31 | List::CommandParser.new("table:export", ["db_name", "table_name"], [], nil, option_list, true) 32 | } 33 | let(:option_with_encryption) { 34 | ops = option_list 35 | ops.push "-e" 36 | ops.push encryption 37 | List::CommandParser.new("table:export", ["db_name", "table_name"], [], nil, ops, true) 38 | } 39 | let(:option_with_wrong_encryption) { 40 | ops = option_list 41 | ops.push "-e" 42 | ops.push wrong_encryption 43 | List::CommandParser.new("table:export", ["db_name", "table_name"], [], nil, ops, true) 44 | } 45 | let(:option_with_assume_role) { 46 | ops = option_list 47 | ops.push "-a" 48 | ops.push assume_role 49 | List::CommandParser.new("table:export", ["db_name", "table_name"], [], nil, ops, true) 50 | } 51 | let(:option_list) { [database, table, "-b", bucket, "-p", path, "-k", key, "-s", pass, "-F", format] } 52 | let(:database) { 'database' } 53 | let(:table) { 'table' } 54 | let(:bucket) { 'bucket' } 55 | let(:path) { 'path' } 56 | let(:key) { 'key' } 57 | let(:pass) { 'pass' } 58 | let(:format) { 'tsv.gz' } 59 | let(:encryption) { 's3' } 60 | let(:assume_role) { 'arn:aws:iam::000:role/assume' } 61 | let(:wrong_encryption) { 's3s3' } 62 | let(:job_id) { 111 } 63 | 64 | before do 65 | client = double(:client) 66 | job = double(:job, job_id: job_id) 67 | allow(client).to receive(:export).and_return(job) 68 | table = double(:table) 69 | 70 | allow(command).to receive(:get_client).and_return(client) 71 | allow(command).to receive(:get_table).and_return(table) 72 | end 73 | 74 | it 'export table without encryption' do 75 | expect { 76 | command.table_export(option) 77 | }.to_not raise_exception 78 | end 79 | 80 | it 'export table with encryption' do 81 | expect { 82 | command.table_export(option_with_encryption) 83 | }.to_not raise_exception 84 | end 85 | 86 | it 'fail to export table with wrong encryption' do 87 | expect { 88 | command.table_export(option_with_wrong_encryption) 89 | }.to raise_exception 90 | end 91 | 92 | it 'export table without assume role' do 93 | expect { 94 | command.table_export(option_with_assume_role) 95 | }.to_not raise_exception 96 | end 97 | 98 | it 'export table with assume role' do 99 | expect { 100 | command.table_export(option_with_assume_role) 101 | }.to_not raise_exception 102 | end 103 | end 104 | 105 | describe '#export_table' do 106 | let(:option) { 107 | List::CommandParser.new("export:table", ["db_name", "table_name"], [], nil, option_list, true) 108 | } 109 | let(:option_list) { [database, table, "-b", bucket, "-p", path, "-k", key, "-s", pass, "-F", format] } 110 | let(:database) { 'database' } 111 | let(:table) { 'table' } 112 | let(:bucket) { 'bucket' } 113 | let(:path) { 'path' } 114 | let(:key) { 'key' } 115 | let(:pass) { 'pass' } 116 | let(:format) { 'tsv.gz' } 117 | let(:job_id) { 111 } 118 | 119 | before do 120 | client = double(:client) 121 | job = double(:job, job_id: job_id) 122 | allow(client).to receive(:export).and_return(job) 123 | table = double(:table) 124 | 125 | allow(command).to receive(:get_client).and_return(client) 126 | allow(command).to receive(:get_table).and_return(table) 127 | end 128 | 129 | it 'export table successfully like #table_export' do 130 | expect { 131 | command.table_export(option) 132 | }.to_not raise_exception 133 | end 134 | end 135 | 136 | describe '#export_result' do 137 | let(:option) { 138 | List::CommandParser.new("export:result", ["target_job_id", "result_url"], [], nil, option_list, true) 139 | } 140 | let(:option_with_retry) { 141 | List::CommandParser.new("export:result", ["target_job_id", "result_url"], [], nil, ['-R', '3'] + option_list, true) 142 | } 143 | let(:option_with_priority) { 144 | List::CommandParser.new("export:result", ["target_job_id", "result_url"], [], nil, ['-P', '-2'] + option_list, true) 145 | } 146 | let(:option_with_wrong_priority) { 147 | List::CommandParser.new("export:result", ["target_job_id", "result_url"], [], nil, ['-P', '3'] + option_list, true) 148 | } 149 | let(:option_with_wait) { 150 | List::CommandParser.new("export:result", ["target_job_id", "result_url"], [], nil, ['-w'] + option_list, true) 151 | } 152 | let(:option_list) { [110, 'mysql://user:pass@host.com/database/table'] } 153 | let(:job_id) { 111 } 154 | let(:client) { double(:client) } 155 | let(:job) { double(:job, job_id: job_id) } 156 | 157 | before do 158 | allow(client).to receive(:result_export).and_return(job) 159 | 160 | allow(command).to receive(:get_client).and_return(client) 161 | end 162 | 163 | it 'export result successfully' do 164 | expect { 165 | command.export_result(option) 166 | }.not_to raise_exception 167 | end 168 | 169 | it 'works with retry option' do 170 | expect { 171 | command.export_result(option_with_retry) 172 | }.not_to raise_exception 173 | end 174 | 175 | it 'works with priority option' do 176 | expect { 177 | command.export_result(option_with_priority) 178 | }.not_to raise_exception 179 | end 180 | 181 | it 'detects wrong priority option' do 182 | expect { 183 | command.export_result(option_with_wrong_priority) 184 | }.to raise_exception 185 | end 186 | 187 | it 'detects wait option' do 188 | target_job = double('target_job') 189 | expect(client).to receive(:job).and_return(target_job) 190 | count_target_job_finished_p = 0 191 | expect(target_job).to receive(:finished?).and_return(false) 192 | allow(target_job).to receive(:wait) 193 | allow(target_job).to receive(:status) 194 | allow(job).to receive(:wait) 195 | allow(job).to receive(:status) 196 | command.export_result(option_with_wait) 197 | end 198 | end 199 | end 200 | end 201 | -------------------------------------------------------------------------------- /spec/td/command/import_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/command/common' 3 | require 'td/command/list' 4 | require 'td/command/import' 5 | 6 | module TreasureData::Command 7 | describe 'import commands' do 8 | let(:command) { 9 | Class.new { include TreasureData::Command }.new 10 | } 11 | let(:stdout_io) { StringIO.new } 12 | let(:stderr_io) { StringIO.new } 13 | 14 | around do |example| 15 | stdout = $stdout.dup 16 | stderr = $stderr.dup 17 | 18 | begin 19 | $stdout = stdout_io 20 | $stderr = stderr_io 21 | 22 | example.run 23 | ensure 24 | $stdout = stdout 25 | $stderr = stderr 26 | end 27 | end 28 | 29 | describe '#import_list' do 30 | let(:option) { 31 | List::CommandParser.new("import:list", [], [], nil, [], true) 32 | } 33 | let(:database) { 'database' } 34 | let(:table) { 'table' } 35 | let(:response) { 36 | [ 37 | double(:bulk_import, 38 | name: 'bulk_import', 39 | database: database, 40 | table: table, 41 | status: :committed, 42 | upload_frozen?: true, 43 | job_id: '123456', 44 | valid_parts: '', 45 | error_parts: '', 46 | valid_records: '', 47 | error_records: '', 48 | ) 49 | ] 50 | } 51 | 52 | before do 53 | client = double(:client, bulk_imports: response) 54 | allow(command).to receive(:get_client).and_return(client) 55 | 56 | command.import_list(option) 57 | end 58 | 59 | it 'show import list' do 60 | expect(stdout_io.string).to include [database, table].join('.') 61 | end 62 | end 63 | 64 | describe '#import_list' do 65 | let(:option) { 66 | List::CommandParser.new("import:list", [], [], nil, [], true) 67 | } 68 | let(:database) { 'database' } 69 | let(:table) { 'table' } 70 | let(:response) { 71 | [ 72 | double(:bulk_import, 73 | name: 'bulk_import', 74 | database: database, 75 | table: table, 76 | status: :committed, 77 | upload_frozen?: true, 78 | job_id: '123456', 79 | valid_parts: '', 80 | error_parts: '', 81 | valid_records: '', 82 | error_records: '', 83 | ) 84 | ] 85 | } 86 | 87 | before do 88 | client = double(:client, bulk_imports: response) 89 | allow(command).to receive(:get_client).and_return(client) 90 | 91 | command.import_list(option) 92 | end 93 | 94 | it 'show import list' do 95 | expect(stdout_io.string).to include [database, table].join('.') 96 | end 97 | end 98 | 99 | describe '#import_show' do 100 | let(:import_name) { 'bulk_import' } 101 | let(:option) { 102 | List::CommandParser.new("import:show", ['name'], [], nil, [import_name], true) 103 | } 104 | let(:client) { double(:client) } 105 | 106 | before do 107 | allow(command).to receive(:get_client).and_return(client) 108 | end 109 | 110 | context 'not exists import' do 111 | it 'should be error' do 112 | expect(client).to receive(:bulk_import).with(import_name).and_return(nil) 113 | expect(command).to receive(:exit).with(1) { raise CallSystemExitError } 114 | 115 | expect { 116 | command.import_show(option) 117 | }.to raise_error CallSystemExitError 118 | end 119 | end 120 | 121 | context 'exist import' do 122 | let(:import_name) { 'bulk_import' } 123 | let(:bulk_import) { 124 | double(:bulk_import, 125 | name: import_name, 126 | database: 'database', 127 | table: 'table', 128 | status: :committed, 129 | upload_frozen?: true, 130 | job_id: '123456', 131 | valid_parts: '', 132 | error_parts: '', 133 | valid_records: '', 134 | error_records: '', 135 | ) 136 | } 137 | let(:bulk_import_parts) { 138 | %w(part1 part2 part3) 139 | } 140 | 141 | before do 142 | expect(client).to receive(:bulk_import).with(import_name).and_return(bulk_import) 143 | expect(client).to receive(:list_bulk_import_parts).with(import_name).and_return(bulk_import_parts) 144 | 145 | command.import_show(option) 146 | end 147 | 148 | it 'stderr should be include import name' do 149 | expect(stderr_io.string).to include import_name 150 | end 151 | 152 | it 'stdout should be include import parts' do 153 | bulk_import_parts.each do |part| 154 | expect(stdout_io.string).to include part 155 | end 156 | end 157 | end 158 | end 159 | 160 | describe '#import_create' do 161 | let(:import_name) { 'bulk_import' } 162 | let(:database) { 'database' } 163 | let(:table) { 'table' } 164 | let(:option) { 165 | List::CommandParser.new("import:create", %w(name, db_name, table_name), [], nil, [import_name, database, table], true) 166 | } 167 | let(:client) { double(:client) } 168 | 169 | before do 170 | allow(command).to receive(:get_client).and_return(client) 171 | end 172 | 173 | it 'create bulk import' do 174 | expect(client).to receive(:create_bulk_import).with(import_name, database, table, {}) 175 | 176 | command.import_create(option) 177 | 178 | expect(stderr_io.string).to include import_name 179 | end 180 | end 181 | 182 | describe '#import_config' do 183 | include_context 'quiet_out' 184 | 185 | let :command do 186 | Class.new { include TreasureData::Command }.new 187 | end 188 | let :option do 189 | List::CommandParser.new("import:config", args, [], nil, arguments, true) 190 | end 191 | let :out_file do 192 | Tempfile.new("seed.yml").tap {|s| s.close } 193 | end 194 | let(:endpoint) { 'http://example.com' } 195 | let(:apikey) { '1234ABCDEFGHIJKLMN' } 196 | 197 | before do 198 | allow(TreasureData::Config).to receive(:endpoint).and_return(endpoint) 199 | allow(TreasureData::Config).to receive(:apikey).and_return(apikey) 200 | end 201 | 202 | context 'unknown format' do 203 | let(:args) { ['url'] } 204 | let(:arguments) { ['localhost', '--format', 'msgpack', '-o', out_file.path] } 205 | 206 | it 'exit command' do 207 | expect { 208 | command.import_config(option) 209 | }.to raise_error TreasureData::Command::ParameterConfigurationError 210 | end 211 | end 212 | 213 | context 'support format' do 214 | let(:td_output_config) { 215 | { 216 | 'type' => 'td', 217 | 'endpoint' => TreasureData::Config.endpoint, 218 | 'apikey' => TreasureData::Config.apikey, 219 | 'database' => '', 220 | 'table' => '', 221 | } 222 | } 223 | 224 | before do 225 | command.import_config(option) 226 | end 227 | 228 | subject(:generated_config) { YAML.load_file(out_file.path) } 229 | 230 | %w(csv tsv).each do |format| 231 | context "--format #{format}" do 232 | let(:args) { ['url'] } 233 | context 'use path' do 234 | let(:path_prefix) { 'path/to/prefix_' } 235 | let(:arguments) { ["#{path_prefix}*.#{format}", '--format', format, '-o', out_file.path] } 236 | 237 | it 'generate configuration file' do 238 | expect(generated_config).to eq({ 239 | 'in' => { 240 | 'type' => 'file', 241 | 'decorders' => [{'type' => 'gzip'}], 242 | 'path_prefix' => path_prefix, 243 | }, 244 | 'out' => td_output_config 245 | }) 246 | end 247 | end 248 | 249 | context 'use s3 scheme' do 250 | let(:s3_access_key) { 'ABCDEFGHIJKLMN' } 251 | let(:s3_secret_key) { '1234ABCDEFGHIJKLMN' } 252 | let(:buckt_name) { 'my_bucket' } 253 | let(:path_prefix) { 'path/to/prefix_' } 254 | let(:s3_url) { "s3://#{s3_access_key}:#{s3_secret_key}@/#{buckt_name}/#{path_prefix}*.#{format}" } 255 | let(:arguments) { [s3_url, '--format', format, '-o', out_file.path] } 256 | 257 | it 'generate configuration file' do 258 | expect(generated_config).to eq({ 259 | 'in' => { 260 | 'type' => 's3', 261 | 'access_key_id' => s3_access_key, 262 | 'secret_access_key' => s3_secret_key, 263 | 'bucket' => buckt_name, 264 | 'path_prefix' => path_prefix, 265 | }, 266 | 'out' => { 267 | 'mode' => 'append' 268 | } 269 | }) 270 | end 271 | end 272 | end 273 | end 274 | 275 | context 'format is mysql' do 276 | let(:format) { 'mysql' } 277 | let(:host) { 'localhost' } 278 | let(:database) { 'database' } 279 | let(:user) { 'my_user' } 280 | let(:password) { 'my_password' } 281 | let(:table) { 'my_table' } 282 | 283 | let(:expected_config) { 284 | { 285 | 'in' => { 286 | 'type' => 'mysql', 287 | 'host' => host, 288 | 'port' => port, 289 | 'database' => database, 290 | 'user' => user, 291 | 'password' => password, 292 | 'table' => table, 293 | 'select' => "*", 294 | }, 295 | 'out' => td_output_config 296 | } 297 | } 298 | 299 | context 'like import:prepare arguments' do 300 | let(:args) { ['url'] } 301 | let(:arguments) { [table, '--db-url', mysql_url, '--db-user', user, '--db-password', password, '--format', 'mysql', '-o', out_file.path] } 302 | 303 | context 'scheme is jdbc' do 304 | let(:port) { 3333 } 305 | let(:mysql_url) { "jdbc:mysql://#{host}:#{port}/#{database}" } 306 | 307 | it 'generate configuration file' do 308 | expect(generated_config).to eq expected_config 309 | end 310 | end 311 | 312 | context 'scheme is mysql' do 313 | context 'with port' do 314 | let(:port) { 3333 } 315 | let(:mysql_url) { "mysql://#{host}:#{port}/#{database}" } 316 | 317 | it 'generate configuration file' do 318 | expect(generated_config).to eq expected_config 319 | end 320 | end 321 | 322 | context 'without port' do 323 | let(:mysql_url) { "mysql://#{host}/#{database}" } 324 | let(:port) { 3306 } 325 | 326 | it 'generate configuration file' do 327 | expect(generated_config).to eq expected_config 328 | end 329 | end 330 | end 331 | end 332 | 333 | context 'like import:upload arguments' do 334 | let(:args) { ['session', 'url'] } 335 | let(:arguments) { ['session', table, '--db-url', mysql_url, '--db-user', user, '--db-password', password, '--format', 'mysql', '-o', out_file.path] } 336 | let(:mysql_url) { "jdbc:mysql://#{host}/#{database}" } 337 | let(:port) { 3306 } 338 | 339 | it 'generate configuration file' do 340 | expect(generated_config).to eq expected_config 341 | end 342 | end 343 | end 344 | end 345 | 346 | context 'not migrate options' do 347 | %w(--columns --column-header).each do |opt| 348 | context "with #{opt}" do 349 | let(:args) { ['url'] } 350 | let(:arguments) { ["path/to/prefix_*.csv", '--format', 'csv', opt, 'col1,col2', '-o', out_file.path] } 351 | 352 | it "#{opt} is not migrate" do 353 | expect { command.import_config(option) }.not_to raise_error 354 | expect(stderr_io.string).to include 'not migrate. Please, edit config file after execute guess commands.' 355 | end 356 | end 357 | end 358 | end 359 | end 360 | end 361 | end 362 | -------------------------------------------------------------------------------- /spec/td/command/query_spec.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'spec_helper' 4 | require 'td/command/common' 5 | require 'td/command/list' 6 | require 'td/command/query' 7 | require 'td/client' 8 | require 'tempfile' 9 | 10 | module TreasureData::Command 11 | describe 'query commands' do 12 | let :client do 13 | double('client') 14 | end 15 | let :job do 16 | double('job', job_id: 123) 17 | end 18 | let :command do 19 | Class.new { include TreasureData::Command }.new 20 | end 21 | before do 22 | allow(command).to receive(:get_client).and_return(client) 23 | expect(client).to receive(:database).with('sample_datasets') 24 | end 25 | 26 | describe 'domain key' do 27 | it 'accepts --domain-key' do 28 | expect(client).to receive(:query). 29 | with("sample_datasets", "SELECT 1;", nil, nil, nil, {"domain_key"=>"hoge"}). 30 | and_return(job) 31 | op = List::CommandParser.new("query", %w[query], %w[], nil, ['--domain-key=hoge', '-dsample_datasets', 'SELECT 1;'], false) 32 | command.query(op) 33 | end 34 | 35 | it 'raises error if --domain-key is duplicated' do 36 | expect(client).to receive(:query). 37 | with("sample_datasets", "SELECT 1;", nil, nil, nil, {"domain_key"=>"hoge"}). 38 | and_raise(::TreasureData::AlreadyExistsError.new('Query failed: domain_key has already been taken')) 39 | op = List::CommandParser.new("query", %w[query], %w[], nil, ['--domain-key=hoge', '-dsample_datasets', 'SELECT 1;'], false) 40 | expect{ command.query(op) }.to raise_error(TreasureData::AlreadyExistsError) 41 | end 42 | end 43 | 44 | describe 'wait' do 45 | let (:job_id){ 123 } 46 | let :job do 47 | obj = TreasureData::Job.new(client, job_id, 'presto', 'SELECT 1;') 48 | allow(obj).to receive(:debug).and_return(double.as_null_object) 49 | allow(obj).to receive(:sleep){|arg|@now += arg} 50 | obj 51 | end 52 | before do 53 | expect(client).to receive(:query). 54 | with("sample_datasets", "SELECT 1;", nil, nil, nil, {}). 55 | and_return(job) 56 | @now = 1_400_000_000 57 | allow(Time).to receive(:now){ @now } 58 | allow(command).to receive(:show_result_with_retry) 59 | end 60 | context 'success' do 61 | before do 62 | status = nil 63 | allow(client).to receive_message_chain(:api, :show_job).with(job_id) do 64 | if @now < 1_400_000_010 65 | status = TreasureData::Job::STATUS_RUNNING 66 | else 67 | status = TreasureData::Job::STATUS_SUCCESS 68 | end 69 | nil 70 | end 71 | allow(client).to receive(:job_status).with(job_id){ status } 72 | end 73 | it 'works with --wait' do 74 | op = List::CommandParser.new("query", %w[query], %w[], nil, ['--wait', '-dsample_datasets', 'SELECT 1;'], false) 75 | expect(command.query(op)).to be_nil 76 | end 77 | it 'works with --wait=360' do 78 | op = List::CommandParser.new("query", %w[query], %w[], nil, ['--wait=360', '-dsample_datasets', 'SELECT 1;'], false) 79 | expect(command.query(op)).to be_nil 80 | end 81 | end 82 | context 'temporary failure' do 83 | before do 84 | status = nil 85 | allow(client).to receive_message_chain(:api, :show_job).with(job_id) do 86 | if @now < 1_400_000_010 87 | status = TreasureData::Job::STATUS_RUNNING 88 | else 89 | status = TreasureData::Job::STATUS_SUCCESS 90 | end 91 | nil 92 | end 93 | allow(client).to receive(:job_status).with(job_id){ status } 94 | end 95 | it 'works with --wait' do 96 | op = List::CommandParser.new("query", %w[query], %w[], nil, ['--wait', '-dsample_datasets', 'SELECT 1;'], false) 97 | expect(command.query(op)).to be_nil 98 | end 99 | it 'works with --wait=360' do 100 | op = List::CommandParser.new("query", %w[query], %w[], nil, ['--wait=360', '-dsample_datasets', 'SELECT 1;'], false) 101 | expect(command.query(op)).to be_nil 102 | end 103 | end 104 | end 105 | 106 | describe 'engine version' do 107 | it 'accepts --engine-version' do 108 | expect(client).to receive(:query). 109 | with("sample_datasets", "SELECT 1;", nil, nil, nil, {"engine_version"=>"stable"}). 110 | and_return(job) 111 | op = List::CommandParser.new("query", %w[query], %w[], nil, ['--engine-version=stable', '-dsample_datasets', 'SELECT 1;'], false) 112 | command.query(op) 113 | end 114 | end 115 | end 116 | end 117 | -------------------------------------------------------------------------------- /spec/td/command/sched_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/command/common' 3 | require 'td/config' 4 | require 'td/command/list' 5 | require 'td/command/sched' 6 | require 'td/client/model' 7 | require 'time' 8 | 9 | module TreasureData::Command 10 | describe TreasureData::Command do 11 | let(:client) { Object.new } 12 | 13 | let :job_params do 14 | ['job_id', :type, 'query', 'status', nil, nil, time, time, 123, 456] 15 | end 16 | 17 | let(:job1) { TreasureData::ScheduledJob.new(client, '2015-02-17 13:22:52 +0900', *job_params) } 18 | let(:job2) { TreasureData::ScheduledJob.new(client, nil, *job_params) } 19 | let(:time) { Time.now.xmlschema } 20 | let(:command) { Class.new { include TreasureData::Command }.new } 21 | let(:argv) { [] } 22 | let(:schedules) { [] } 23 | let(:op) { List::CommandParser.new('sched:last_job', %w[], %w[], false, argv, []) } 24 | 25 | before do 26 | allow(client).to receive(:schedules).and_return(schedules) 27 | allow(command).to receive(:get_client).and_return(client) 28 | end 29 | 30 | describe 'sched_create' do 31 | before do 32 | expect(client).to receive(:database).with('sample_datasets') 33 | end 34 | 35 | describe 'engine version' do 36 | let(:op) { 37 | List::CommandParser.new( 38 | 'sched:create', 39 | %w[name cron sql], 40 | %w[], 41 | false, 42 | ['--engine-version=stable', '-dsample_datasets', 'sched1', '0 * * * *', 'select count(*) from table1'], 43 | false 44 | ) 45 | } 46 | 47 | it 'accepts --engine-version' do 48 | expect(client).to receive(:create_schedule). 49 | with( 50 | "sched1", 51 | { 52 | cron: '0 * * * *', 53 | query: 'select count(*) from table1', 54 | database: 'sample_datasets', 55 | result: nil, 56 | timezone: nil, 57 | delay: 0, 58 | priority: nil, 59 | retry_limit: nil, 60 | type: nil, 61 | engine_version: "stable" 62 | } 63 | ). 64 | and_return(nil) 65 | command.sched_create(op) 66 | end 67 | end 68 | end 69 | 70 | describe 'sched_update' do 71 | describe 'engine version' do 72 | let(:op) { List::CommandParser.new('sched:update', %w[name], %w[], false, ['--engine-version=stable', 'old_name'], false) } 73 | 74 | it 'accepts --engine-version' do 75 | expect(client).to receive(:update_schedule). 76 | with("old_name", {"engine_version"=>"stable"}). 77 | and_return(nil) 78 | command.sched_update(op) 79 | end 80 | end 81 | end 82 | 83 | describe 'sched_history' do 84 | before do 85 | allow(client).to receive(:history).and_return(history) 86 | end 87 | 88 | let(:history) { [job1, job2] } 89 | 90 | it 'runs' do 91 | expect { 92 | command.sched_history(op) 93 | }.to_not raise_exception 94 | end 95 | end 96 | 97 | describe 'sched_result' do 98 | subject { command.sched_result(op) } 99 | 100 | before do 101 | allow(command).to receive(:get_history).with(client, nil, (back_number - 1), back_number).and_return(history) 102 | end 103 | 104 | shared_examples_for("passing argv and job_id to job:show") do 105 | it "invoke 'job:show [original argv] [job id]'" do 106 | expect_any_instance_of(TreasureData::Command::Runner).to receive(:run).with(["job:show", *show_arg, job_id]) 107 | subject 108 | end 109 | end 110 | 111 | context "history exists" do 112 | 113 | let(:job_id) { history.first.job_id } 114 | 115 | context 'without --last option' do 116 | let(:history) { [job1] } 117 | let(:back_number) { 1 } 118 | 119 | let(:argv) { %w(--last --format csv) } 120 | let(:show_arg) { %w(--format csv) } 121 | it_behaves_like "passing argv and job_id to job:show" 122 | end 123 | 124 | context '--last witout Num' do 125 | let(:history) { [job1] } 126 | let(:back_number) { 1 } 127 | 128 | let(:argv) { %w(--last --format csv) } 129 | let(:show_arg) { %w(--format csv) } 130 | it_behaves_like "passing argv and job_id to job:show" 131 | end 132 | 133 | context '--last 1' do 134 | let(:history) { [job1] } 135 | let(:back_number) { 1 } 136 | 137 | let(:argv) { %w(--last 1 --format csv) } 138 | let(:show_arg) { %w(--format csv) } 139 | it_behaves_like "passing argv and job_id to job:show" 140 | end 141 | 142 | context '--last 1 and --limit 1' do 143 | let(:history) { [job1] } 144 | let(:back_number) { 1 } 145 | 146 | let(:argv) { %w(--last 1 --format csv --limit 1) } 147 | let(:show_arg) { %w(--format csv --limit 1) } 148 | it_behaves_like "passing argv and job_id to job:show" 149 | end 150 | 151 | context '--last 2 after format option' do 152 | let(:history) { [job2] } 153 | let(:back_number) { 2 } 154 | 155 | let(:argv) { %w(--format csv --last 2 ) } 156 | let(:show_arg) { %w(--format csv) } 157 | it_behaves_like "passing argv and job_id to job:show" 158 | end 159 | 160 | context '--last 3(too back over)' do 161 | let(:history) { [] } 162 | let(:back_number) { 3 } 163 | 164 | let(:argv) { %w(--last 3 --format csv) } 165 | it 'raise ' do 166 | expect { 167 | command.sched_result(op) 168 | }.to raise_exception, "No jobs available for this query. Refer to 'sched:history'" 169 | end 170 | end 171 | 172 | context '--last WRONGARG(not a number)' do 173 | let(:history) { [job1] } 174 | let(:back_number) { 1 } 175 | 176 | let(:argv) { %w(--last TEST --format csv) } 177 | 178 | it "exit with 1" do 179 | begin 180 | command.sched_result(op) 181 | rescue SystemExit => e 182 | expect(e.status).to eq 1 183 | end 184 | end 185 | end 186 | end 187 | 188 | context "history dose not exists" do 189 | let(:back_number) { 1 } 190 | let(:history) { [] } 191 | before { allow(client).to receive(:history) { raise TreasureData::NotFoundError } } 192 | 193 | it "exit with 1" do 194 | begin 195 | subject 196 | rescue SystemExit => e 197 | expect(e.status).to eq 1 198 | end 199 | end 200 | end 201 | end 202 | end 203 | end 204 | -------------------------------------------------------------------------------- /spec/td/command/schema_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/command/common' 3 | require 'td/config' 4 | require 'td/command/list' 5 | require 'td/command/schema' 6 | require 'td/client/model' 7 | require 'time' 8 | 9 | describe TreasureData::Command do 10 | let(:client){ double('client') } 11 | let(:table){ double('table', schema: schema) } 12 | let(:db_name){ "database" } 13 | let(:table_name){ "table" } 14 | let(:command) { Class.new { include TreasureData::Command }.new } 15 | let(:columns){ ["foo:int", "BAR\u3070\u30FC:string@bar", "baz:baz!:array@baz"] } 16 | let(:schema){ TreasureData::Schema.parse(columns) } 17 | let(:stdout){ StringIO.new } 18 | before do 19 | allow(command).to receive(:get_client).and_return(client) 20 | allow(command).to receive(:get_table).with(client, db_name, table_name).and_return(table) 21 | allow(table).to receive(:get_table).with(client, db_name, table_name).and_return(table) 22 | allow($stdout).to receive(:puts){|arg| stdout.puts(*arg) } 23 | end 24 | 25 | describe 'schema_show' do 26 | let(:op){ double('op', cmd_parse: [db_name, table_name]) } 27 | it 'puts $stdout the schema' do 28 | command.schema_show(op) 29 | expect(stdout.string).to eq <@baz 34 | ) 35 | eom 36 | end 37 | end 38 | 39 | describe 'schema_set' do 40 | let(:op){ double('op', cmd_parse: [db_name, table_name, *columns]) } 41 | it 'calls client.update_schema' do 42 | allow($stderr).to receive(:puts) 43 | expect(client).to receive(:update_schema) do |x, y, z| 44 | expect(x).to eq db_name 45 | expect(y).to eq table_name 46 | expect(z.to_json).to eq schema.to_json 47 | end 48 | command.schema_set(op) 49 | end 50 | end 51 | 52 | describe 'schema_add' do 53 | let(:columns2){ ["foo2:int", "BAR\u30702:string@bar2", "baz:baz!2:array@baz2"] } 54 | let(:schema2){ TreasureData::Schema.parse(columns+columns2) } 55 | let(:op){ double('op', cmd_parse: [db_name, table_name, *columns2]) } 56 | it 'calls client.update_schema' do 57 | allow($stderr).to receive(:puts) 58 | expect(client).to receive(:update_schema) do |x, y, z| 59 | expect(x).to eq db_name 60 | expect(y).to eq table_name 61 | expect(z.to_json).to eq schema2.to_json 62 | end 63 | command.schema_add(op) 64 | end 65 | end 66 | 67 | describe 'schema_remove' do 68 | let(:op){ double('op', cmd_parse: [db_name, table_name, "foo"]) } 69 | let(:columns2){ ["BAR\u3070\u30FC:string@bar", "baz:baz!:array@baz"] } 70 | let(:schema2){ TreasureData::Schema.parse(columns2) } 71 | it 'calls client.update_schema' do 72 | allow($stderr).to receive(:puts) 73 | expect(client).to receive(:update_schema) do |x, y, z| 74 | expect(x).to eq db_name 75 | expect(y).to eq table_name 76 | expect(z.to_json).to eq schema.to_json 77 | end 78 | command.schema_remove(op) 79 | end 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /spec/td/common_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/command/common' 3 | require 'td/client/api_error' 4 | 5 | module TreasureData::Command 6 | describe 'humanize_bytesize' do 7 | describe 'for values < 1024' do 8 | values = [0, 1, 10, 1023] 9 | values.each {|v| 10 | it "uses B as label and has no suffix (#{v})" do 11 | expect(TreasureData::Command::humanize_bytesize(v, 1)).to eq("#{v} B") 12 | end 13 | } 14 | end 15 | 16 | describe 'for 1024' do 17 | it 'uses kB and does not have a suffix' do 18 | expect(TreasureData::Command::humanize_bytesize(1024)).to eq("1 kB") 19 | end 20 | end 21 | describe 'for values between 1025 and (1024^2 - 1)' do 22 | base = 1024 23 | values = [ 24 | [base + 1, "1.0"], 25 | [base + 2, "1.0"], 26 | [base * 1024 - 1, "1023.9"] 27 | ] 28 | values.each {|val, exp| 29 | it "uses kB as label and has a suffix (#{val})" do 30 | result = TreasureData::Command::humanize_bytesize(val, 1) 31 | expect(result).to eq("#{exp} kB") 32 | end 33 | } 34 | end 35 | 36 | describe 'for 1024^2' do 37 | it 'uses MB and does not have a suffix' do 38 | expect(TreasureData::Command::humanize_bytesize(1024 ** 2)).to eq("1 MB") 39 | end 40 | end 41 | describe 'for values between (1024^2 + 1) and (1024^3 - 1)' do 42 | base = 1024 ** 2 43 | values = [ 44 | [base + 1, "1.0"], 45 | [base + 2, "1.0"], 46 | [base * 1024 - 1, "1023.9"] 47 | ] 48 | values.each {|val, exp| 49 | it "uses MB as label and has a suffix (#{val})" do 50 | result = TreasureData::Command::humanize_bytesize(val, 1) 51 | expect(result).to eq("#{exp} MB") 52 | end 53 | } 54 | end 55 | 56 | describe 'for 1024^3' do 57 | it 'uses GB and does not have a suffix' do 58 | expect(TreasureData::Command::humanize_bytesize(1024 ** 3)).to eq("1 GB") 59 | end 60 | end 61 | describe 'for values between (1024^3 + 1) and (1024^4 - 1)' do 62 | base = 1024 ** 3 63 | values = [ 64 | [base + 1, "1.0"], 65 | [base + 2, "1.0"], 66 | [base * 1024 - 1, "1023.9"] 67 | ] 68 | values.each {|val, exp| 69 | it "uses GB as label and has a suffix (#{val})" do 70 | result = TreasureData::Command::humanize_bytesize(val, 1) 71 | expect(result).to eq("#{exp} GB") 72 | end 73 | } 74 | end 75 | 76 | describe 'for 1024^4' do 77 | it 'uses TB and does not have a suffix' do 78 | expect(TreasureData::Command::humanize_bytesize(1024 ** 4)).to eq("1 TB") 79 | end 80 | end 81 | describe 'for values between (1024^4 + 1) and (1024^5 - 1)' do 82 | base = 1024 ** 4 83 | values = [ 84 | [base + 1, "1.0"], 85 | [base + 2, "1.0"], 86 | [base * 1024 - 1, "1023.9"] 87 | ] 88 | values.each {|val, exp| 89 | it "uses TB as label and has a suffix (#{val})" do 90 | result = TreasureData::Command::humanize_bytesize(val, 1) 91 | expect(result).to eq("#{exp} TB") 92 | end 93 | } 94 | end 95 | 96 | describe 'for 1024^5' do 97 | it 'uses TB and does not have a suffix' do 98 | expect(TreasureData::Command::humanize_bytesize(1024 ** 5)).to eq("1024 TB") 99 | end 100 | end 101 | describe 'for values between (1024^5 + 1) and (1024^6 - 1)' do 102 | base = 1024 ** 5 103 | values = [ 104 | [base + 1, "1024.0"], 105 | [base + 2, "1024.0"], 106 | [base * 1024 - 1, "1048575.9"] 107 | ] 108 | values.each {|val, exp| 109 | it "uses TB as label and has a suffix (#{val})" do 110 | result = TreasureData::Command::humanize_bytesize(val, 1) 111 | expect(result).to eq("#{exp} TB") 112 | end 113 | } 114 | end 115 | 116 | describe 'shows 1 digit' do 117 | it 'without second function argument' do 118 | values = [1024 + 1024 / 2, "1.5"] 119 | val, exp = values 120 | result = TreasureData::Command::humanize_bytesize(val) 121 | expect(result).to eq("#{exp} kB") 122 | end 123 | end 124 | describe 'shows the correct number of digits specified by the second argument' do 125 | (0...5).each {|i| 126 | it "when = #{i}" do 127 | val = 1024 + 1024 / 2 128 | if i == 0 129 | exp = 1.to_s 130 | else 131 | exp = sprintf "%.*f", i, 1.5 132 | end 133 | result = TreasureData::Command::humanize_bytesize(val, i) 134 | expect(result).to eq("#{exp} kB") 135 | end 136 | } 137 | end 138 | end 139 | 140 | describe 'SizeBasedDownloadProgressIndicator' do 141 | it "shows in 1% increments with default 'perc_step'" do 142 | size = 200 143 | indicator = TreasureData::Command::SizeBasedDownloadProgressIndicator.new("Downloading", size) 144 | size_increments = 2 145 | curr_size = 0 146 | while (curr_size += size_increments) < size do 147 | indicator.update(curr_size) 148 | sleep(0.05) 149 | end 150 | indicator.finish 151 | end 152 | 153 | it "shows in only current byte size if total is nil" do 154 | indicator = TreasureData::Command::SizeBasedDownloadProgressIndicator.new("Downloading", nil) 155 | size_increments = 2 156 | curr_size = 0 157 | while (curr_size += size_increments) < 200 do 158 | indicator.update(curr_size) 159 | sleep(0.05) 160 | end 161 | indicator.finish 162 | end 163 | 164 | it "shows in only current byte size if total is 0" do 165 | indicator = TreasureData::Command::SizeBasedDownloadProgressIndicator.new("Downloading", 0) 166 | size_increments = 2 167 | curr_size = 0 168 | while (curr_size += size_increments) < 200 do 169 | indicator.update(curr_size) 170 | sleep(0.05) 171 | end 172 | indicator.finish 173 | end 174 | end 175 | 176 | describe 'TimeBasedDownloadProgressIndicator' do 177 | it "increments about every 2 seconds with default 'periodicity'" do 178 | start_time = Time.now.to_i 179 | indicator = TreasureData::Command::TimeBasedDownloadProgressIndicator.new("Downloading", start_time) 180 | end_time = start_time + 10 181 | last_time = start_time 182 | while (curr_time = Time.now.to_i) < end_time do 183 | ret = indicator.update 184 | if ret == true 185 | diff = curr_time - last_time 186 | expect(diff).to be >= 2 187 | expect(diff).to be < 3 188 | last_time = curr_time 189 | end 190 | sleep(0.5) 191 | end 192 | indicator.finish 193 | end 194 | 195 | periodicities = [1, 2, 5] 196 | periodicities.each {|periodicity| 197 | it "increments about every #{periodicity} seconds with 'periodicity' = #{periodicity}" do 198 | start_time = Time.now.to_i 199 | indicator = TreasureData::Command::TimeBasedDownloadProgressIndicator.new("Downloading", start_time, periodicity) 200 | end_time = start_time + 10 201 | last_time = start_time 202 | while (curr_time = Time.now.to_i) < end_time do 203 | ret = indicator.update 204 | if ret == true 205 | expect(curr_time - last_time).to be >= periodicity 206 | expect(curr_time - last_time).to be < (periodicity + 1) 207 | last_time = curr_time 208 | end 209 | sleep(0.5) 210 | end 211 | indicator.finish 212 | end 213 | } 214 | end 215 | 216 | describe '#create_database_and_table_if_not_exist' do 217 | let(:command_class) { Class.new { include TreasureData::Command } } 218 | let(:command) { command_class.new } 219 | let(:client) { double(:client) } 220 | let(:database_name) { 'database' } 221 | let(:table_name) { 'table1' } 222 | 223 | let(:stderr_io) { StringIO.new } 224 | 225 | subject(:call_create_database_and_table_if_not_exist) { 226 | stderr = $stderr.dup 227 | begin 228 | $stderr = stderr_io 229 | 230 | command.__send__(:create_database_and_table_if_not_exist, client, database_name, table_name) 231 | ensure 232 | $stderr = stderr 233 | end 234 | } 235 | 236 | describe 'create database' do 237 | before do 238 | allow(client).to receive(:create_log_table) 239 | end 240 | 241 | context 'client.database success' do 242 | it 'not call client.create_database' do 243 | expect(client).to receive(:database).with(database_name) 244 | expect(client).not_to receive(:create_database) 245 | 246 | call_create_database_and_table_if_not_exist 247 | expect(stderr_io.string).not_to include "Database '#{database_name}'" 248 | end 249 | end 250 | 251 | context 'client.database raise NotFoundError' do 252 | before do 253 | expect(client).to receive(:database).with(database_name) { raise TreasureData::NotFoundError } 254 | end 255 | 256 | context 'craet_database success' do 257 | it 'call client.create_database' do 258 | expect(client).to receive(:create_database).with(database_name) 259 | 260 | call_create_database_and_table_if_not_exist 261 | expect(stderr_io.string).to include "Database '#{database_name}'" 262 | end 263 | end 264 | 265 | context 'craet_database raise AlreadyExistsError' do 266 | it 'resuce in method' do 267 | expect(client).to receive(:create_database).with(database_name) { raise TreasureData::AlreadyExistsError } 268 | 269 | expect { 270 | call_create_database_and_table_if_not_exist 271 | }.not_to raise_error 272 | 273 | expect(stderr_io.string).not_to include "Database '#{database_name}'" 274 | end 275 | end 276 | end 277 | 278 | context 'client.database raise ForbiddenError' do 279 | it 'not call client.create_database' do 280 | expect(client).to receive(:database).with(database_name) { raise TreasureData::ForbiddenError } 281 | expect(client).not_to receive(:create_database) 282 | 283 | call_create_database_and_table_if_not_exist 284 | expect(stderr_io.string).not_to include "Database '#{database_name}'" 285 | end 286 | end 287 | end 288 | 289 | describe 'create table' do 290 | before do 291 | allow(client).to receive(:database) 292 | end 293 | 294 | context 'create_log_table success' do 295 | it 'show message' do 296 | expect(client).to receive(:create_log_table).with(database_name, table_name) 297 | 298 | call_create_database_and_table_if_not_exist 299 | expect(stderr_io.string).to include "Table '#{database_name}.#{table_name}'" 300 | end 301 | end 302 | 303 | context 'create_log_table raise AlreadyExistsError' do 304 | it 'resuce in method' do 305 | expect(client).to receive(:create_log_table).with(database_name, table_name) { raise TreasureData::AlreadyExistsError } 306 | 307 | expect { 308 | call_create_database_and_table_if_not_exist 309 | }.not_to raise_error 310 | 311 | expect(stderr_io.string).not_to include "Table '#{database_name}.#{table_name}'" 312 | end 313 | end 314 | end 315 | end 316 | end 317 | -------------------------------------------------------------------------------- /spec/td/compact_format_yamler_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/compact_format_yamler' 3 | 4 | module TreasureData 5 | describe TreasureData::CompactFormatYamler do 6 | describe '.dump' do 7 | let(:data) { 8 | { 9 | 'a' => { 10 | 'b' => { 11 | 'c' => 1, 12 | 'd' => 'e' 13 | }, 14 | 'f' => [1, 2, 3] 15 | } 16 | } 17 | } 18 | 19 | let(:comapct_format_yaml) { 20 | <<-EOS 21 | --- 22 | a: 23 | b: {c: 1, d: e} 24 | f: 25 | - 1 26 | - 2 27 | - 3 28 | EOS 29 | } 30 | 31 | subject { TreasureData::CompactFormatYamler.dump data } 32 | 33 | it 'use compact format for deepest Hash' do 34 | expect(subject).to eq comapct_format_yaml 35 | end 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /spec/td/config_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'tempfile' 3 | require 'td/config' 4 | 5 | describe TreasureData::Config do 6 | context 'workflow_endpoint' do 7 | before { TreasureData::Config.endpoint = api_endpoint } 8 | subject { TreasureData::Config.workflow_endpoint } 9 | context 'api.treasuredata.com' do 10 | context 'works without http schema' do 11 | let(:api_endpoint){ 'api.treasuredata.com' } 12 | it { is_expected.to eq 'https://api-workflow.treasuredata.com' } 13 | end 14 | context 'works with http schema' do 15 | let(:api_endpoint){ 'http://api.treasuredata.com' } 16 | it { is_expected.to eq 'https://api-workflow.treasuredata.com' } 17 | end 18 | context 'works with https schema' do 19 | let(:api_endpoint){ 'https://api.treasuredata.com' } 20 | it { is_expected.to eq 'https://api-workflow.treasuredata.com' } 21 | end 22 | end 23 | context 'api.treasuredata.co.jp' do 24 | let(:api_endpoint){ 'api.treasuredata.co.jp' } 25 | it { is_expected.to eq 'https://api-workflow.treasuredata.co.jp' } 26 | end 27 | context 'api.eu01.treasuredata.com' do 28 | let(:api_endpoint){ 'api.eu01.treasuredata.com' } 29 | it { is_expected.to eq 'https://api-workflow.eu01.treasuredata.com' } 30 | end 31 | context 'api.ap02.treasuredata.com' do 32 | let(:api_endpoint){ 'api.ap02.treasuredata.com' } 33 | it { is_expected.to eq 'https://api-workflow.ap02.treasuredata.com' } 34 | end 35 | context 'api.ap03.treasuredata.com' do 36 | let(:api_endpoint){ 'api.ap03.treasuredata.com' } 37 | it { is_expected.to eq 'https://api-workflow.ap03.treasuredata.com' } 38 | end 39 | 40 | context 'api-hoge.connect.treasuredata.com' do 41 | let(:api_endpoint){ 'api-hoge.connect.treasuredata.com' } 42 | it { is_expected.to eq 'https://api-workflow-hoge.connect.treasuredata.com' } 43 | end 44 | context 'api-czc21f.connect.treasuredata.co.jp' do 45 | let(:api_endpoint){ 'api-czc21f.connect.treasuredata.co.jp' } 46 | it { is_expected.to eq 'https://api-workflow-czc21f.connect.treasuredata.co.jp' } 47 | end 48 | 49 | context 'api-staging.treasuredata.com' do 50 | let(:api_endpoint){ 'api-staging.treasuredata.com' } 51 | it { is_expected.to eq 'https://api-staging-workflow.treasuredata.com' } 52 | end 53 | context 'api-staging.treasuredata.co.jp' do 54 | let(:api_endpoint){ 'api-staging.treasuredata.co.jp' } 55 | it { is_expected.to eq 'https://api-staging-workflow.treasuredata.co.jp' } 56 | end 57 | context 'api-staging.eu01.treasuredata.com' do 58 | let(:api_endpoint){ 'api-staging.eu01.treasuredata.com' } 59 | it { is_expected.to eq 'https://api-staging-workflow.eu01.treasuredata.com' } 60 | end 61 | context 'api-staging.ap02.treasuredata.com' do 62 | let(:api_endpoint){ 'api-staging.ap02.treasuredata.com' } 63 | it { is_expected.to eq 'https://api-staging-workflow.ap02.treasuredata.com' } 64 | end 65 | context 'api-staging.ap03.treasuredata.com' do 66 | let(:api_endpoint){ 'api-staging.ap03.treasuredata.com' } 67 | it { is_expected.to eq 'https://api-staging-workflow.ap03.treasuredata.com' } 68 | end 69 | 70 | context 'api-development.treasuredata.com' do 71 | let(:api_endpoint){ 'api-development.treasuredata.com' } 72 | it { is_expected.to eq 'https://api-development-workflow.treasuredata.com' } 73 | end 74 | context 'api-development.treasuredata.co.jp' do 75 | let(:api_endpoint){ 'api-development.treasuredata.co.jp' } 76 | it { is_expected.to eq 'https://api-development-workflow.treasuredata.co.jp' } 77 | end 78 | context 'api-development.eu01.treasuredata.com' do 79 | let(:api_endpoint){ 'api-development.eu01.treasuredata.com' } 80 | it { is_expected.to eq 'https://api-development-workflow.eu01.treasuredata.com' } 81 | end 82 | context 'api-development.ap02.treasuredata.com' do 83 | let(:api_endpoint){ 'api-development.ap02.treasuredata.com' } 84 | it { is_expected.to eq 'https://api-development-workflow.ap02.treasuredata.com' } 85 | end 86 | context 'api-development.ap03.treasuredata.com' do 87 | let(:api_endpoint){ 'api-development.ap03.treasuredata.com' } 88 | it { is_expected.to eq 'https://api-development-workflow.ap03.treasuredata.com' } 89 | end 90 | 91 | context 'ybi.jp-east.idcfcloud.com' do 92 | let(:api_endpoint){ 'ybi.jp-east.idcfcloud.com' } 93 | it 'raise error' do 94 | expect { subject }.to raise_error(TreasureData::ConfigError) 95 | end 96 | end 97 | end 98 | 99 | describe 'allow endpoint with trailing slash' do 100 | context 'self.endpoint' do 101 | before { TreasureData::Config.endpoint = api_endpoint } 102 | subject { TreasureData::Config.endpoint } 103 | 104 | context 'api.treasuredata.com' do 105 | let(:api_endpoint) { 'https://api.treasuredata.com/' } 106 | it { is_expected.to eq 'https://api.treasuredata.com' } 107 | end 108 | context 'api.treasuredata.co.jp' do 109 | let(:api_endpoint) { 'https://api.treasuredata.co.jp/' } 110 | it { is_expected.to eq 'https://api.treasuredata.co.jp' } 111 | end 112 | context 'api.eu01.treasuredata.com' do 113 | let(:api_endpoint) { 'https://api.eu01.treasuredata.com/' } 114 | it { is_expected.to eq 'https://api.eu01.treasuredata.com' } 115 | end 116 | context 'api.ap02.treasuredata.com' do 117 | let(:api_endpoint) { 'https://api.ap02.treasuredata.com/' } 118 | it { is_expected.to eq 'https://api.ap02.treasuredata.com' } 119 | end 120 | context 'api.ap03.treasuredata.com' do 121 | let(:api_endpoint) { 'https://api.ap03.treasuredata.com/' } 122 | it { is_expected.to eq 'https://api.ap03.treasuredata.com' } 123 | end 124 | context 'api-hoge.connect.treasuredata.com' do 125 | let(:api_endpoint){ 'https://api-hoge.connect.treasuredata.com/' } 126 | it { is_expected.to eq 'https://api-hoge.connect.treasuredata.com' } 127 | end 128 | end 129 | 130 | context 'self.import_endpoint' do 131 | before { TreasureData::Config.import_endpoint = api_endpoint } 132 | subject { TreasureData::Config.import_endpoint } 133 | 134 | context 'api-import.treasuredata.com' do 135 | let(:api_endpoint) { 'https://api-import.treasuredata.com/' } 136 | it { is_expected.to eq 'https://api-import.treasuredata.com' } 137 | end 138 | context 'api-import.treasuredata.co.jp' do 139 | let(:api_endpoint) { 'https://api-import.treasuredata.co.jp/' } 140 | it { is_expected.to eq 'https://api-import.treasuredata.co.jp' } 141 | end 142 | context 'api-import.eu01.treasuredata.com' do 143 | let(:api_endpoint) { 'https://api-import.eu01.treasuredata.com/' } 144 | it { is_expected.to eq 'https://api-import.eu01.treasuredata.com' } 145 | end 146 | context 'api-import.ap02.treasuredata.com' do 147 | let(:api_endpoint) { 'https://api-import.ap02.treasuredata.com/' } 148 | it { is_expected.to eq 'https://api-import.ap02.treasuredata.com' } 149 | end 150 | context 'api-import.ap03.treasuredata.com' do 151 | let(:api_endpoint) { 'https://api-import.ap03.treasuredata.com/' } 152 | it { is_expected.to eq 'https://api-import.ap03.treasuredata.com' } 153 | end 154 | end 155 | end 156 | 157 | describe '#read' do 158 | it 'sets @conf' do 159 | Tempfile.create('td.conf') do |f| 160 | f << <<-EOF 161 | # This is comment 162 | [section1] 163 | # This is comment 164 | key=val 165 | foo=bar 166 | EOF 167 | 168 | f.close 169 | 170 | config = TreasureData::Config.new 171 | config.read(f.path) 172 | 173 | expect(config["section1.key"]).to eq "val" 174 | expect(config["section1.foo"]).to eq "bar" 175 | end 176 | end 177 | end 178 | end 179 | -------------------------------------------------------------------------------- /spec/td/connector_config_normalizer_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/connector_config_normalizer' 3 | require 'td/client/api_error' 4 | 5 | module TreasureData 6 | describe ConnectorConfigNormalizer do 7 | describe '#normalized_config' do 8 | subject { TreasureData::ConnectorConfigNormalizer.new(config).normalized_config } 9 | 10 | context 'has key :in' do 11 | context 'without :out, :exec' do 12 | let(:config) { {'in' => {'type' => 's3'}} } 13 | 14 | it { expect(subject).to eq config.merge('out' => {}, 'exec' => {}, 'filters' => []) } 15 | end 16 | 17 | context 'with :out, :exec, :filters' do 18 | let(:config) { 19 | { 20 | 'in' => {'type' => 's3'}, 21 | 'out' => {'mode' => 'append'}, 22 | 'exec' => {'guess_plugins' => ['json', 'query_string']}, 23 | 'filters' => [{'type' => 'speedometer'}] 24 | } 25 | } 26 | 27 | it { expect(subject).to eq config } 28 | end 29 | end 30 | 31 | context 'has key :config' do 32 | context 'with :in' do 33 | let(:config) { 34 | { 'config' => 35 | { 36 | 'in' => {'type' => 's3'}, 37 | 'out' => {'mode' => 'append'}, 38 | 'exec' => {'guess_plugins' => ['json', 'query_string']}, 39 | 'filters' => [{'type' => 'speedometer'}] 40 | } 41 | } 42 | } 43 | 44 | it { expect(subject).to eq config['config'] } 45 | 46 | end 47 | 48 | context 'without :in' do 49 | let(:config) { {'config' => {'type' => 's3'}} } 50 | 51 | it { expect(subject).to eq({'in' => config['config'], 'out' => {}, 'exec' => {}, 'filters' => []}) } 52 | end 53 | end 54 | 55 | context 'does not have key :in or :config' do 56 | let(:config) { {'type' => 's3'} } 57 | 58 | it { expect(subject).to eq({'in' => config, 'out' => {}, 'exec' => {}, 'filters' => []}) } 59 | end 60 | end 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /spec/td/fixture/bulk_load.yml: -------------------------------------------------------------------------------- 1 | --- 2 | config: 3 | type: s3 4 | access_key_id: ACCESS_KEY 5 | secret_access_key: SECRET_ACCESS 6 | bucket: test-bucket 7 | path_prefix: dummy.csv.gz 8 | parser: 9 | charset: ISO-8859-9 10 | newline: CRLF 11 | type: csv 12 | delimiter: "," 13 | quote: '' 14 | escape: '' 15 | skip_header_lines: 1 16 | columns: 17 | - name: foo 18 | type: long 19 | - name: bar 20 | type: long 21 | - name: baz 22 | type: long 23 | decoders: 24 | - type: gzip 25 | -------------------------------------------------------------------------------- /spec/td/fixture/ca.cert: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID0DCCArigAwIBAgIBADANBgkqhkiG9w0BAQUFADA8MQswCQYDVQQGDAJKUDES 3 | MBAGA1UECgwJSklOLkdSLkpQMQwwCgYDVQQLDANSUlIxCzAJBgNVBAMMAkNBMB4X 4 | DTA0MDEzMDAwNDIzMloXDTM2MDEyMjAwNDIzMlowPDELMAkGA1UEBgwCSlAxEjAQ 5 | BgNVBAoMCUpJTi5HUi5KUDEMMAoGA1UECwwDUlJSMQswCQYDVQQDDAJDQTCCASIw 6 | DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANbv0x42BTKFEQOE+KJ2XmiSdZpR 7 | wjzQLAkPLRnLB98tlzs4xo+y4RyY/rd5TT9UzBJTIhP8CJi5GbS1oXEerQXB3P0d 8 | L5oSSMwGGyuIzgZe5+vZ1kgzQxMEKMMKlzA73rbMd4Jx3u5+jdbP0EDrPYfXSvLY 9 | bS04n2aX7zrN3x5KdDrNBfwBio2/qeaaj4+9OxnwRvYP3WOvqdW0h329eMfHw0pi 10 | JI0drIVdsEqClUV4pebT/F+CPUPkEh/weySgo9wANockkYu5ujw2GbLFcO5LXxxm 11 | dEfcVr3r6t6zOA4bJwL0W/e6LBcrwiG/qPDFErhwtgTLYf6Er67SzLyA66UCAwEA 12 | AaOB3DCB2TAPBgNVHRMBAf8EBTADAQH/MDEGCWCGSAGG+EIBDQQkFiJSdWJ5L09w 13 | ZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRlMB0GA1UdDgQWBBRJ7Xd380KzBV7f 14 | USKIQ+O/vKbhDzAOBgNVHQ8BAf8EBAMCAQYwZAYDVR0jBF0wW4AUSe13d/NCswVe 15 | 31EiiEPjv7ym4Q+hQKQ+MDwxCzAJBgNVBAYMAkpQMRIwEAYDVQQKDAlKSU4uR1Iu 16 | SlAxDDAKBgNVBAsMA1JSUjELMAkGA1UEAwwCQ0GCAQAwDQYJKoZIhvcNAQEFBQAD 17 | ggEBAIu/mfiez5XN5tn2jScgShPgHEFJBR0BTJBZF6xCk0jyqNx/g9HMj2ELCuK+ 18 | r/Y7KFW5c5M3AQ+xWW0ZSc4kvzyTcV7yTVIwj2jZ9ddYMN3nupZFgBK1GB4Y05GY 19 | MJJFRkSu6d/Ph5ypzBVw2YMT/nsOo5VwMUGLgS7YVjU+u/HNWz80J3oO17mNZllj 20 | PvORJcnjwlroDnS58KoJ7GDgejv3ESWADvX1OHLE4cRkiQGeLoEU4pxdCxXRqX0U 21 | PbwIkZN9mXVcrmPHq8MWi4eC/V7hnbZETMHuWhUoiNdOEfsAXr3iP4KjyyRdwc7a 22 | d/xgcK06UVQRL/HbEYGiQL056mc= 23 | -----END CERTIFICATE----- 24 | -------------------------------------------------------------------------------- /spec/td/fixture/server.cert: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIC/zCCAeegAwIBAgIBATANBgkqhkiG9w0BAQUFADA/MQswCQYDVQQGDAJKUDES 3 | MBAGA1UECgwJSklOLkdSLkpQMQwwCgYDVQQLDANSUlIxDjAMBgNVBAMMBVN1YkNB 4 | MB4XDTA0MDEzMTAzMTMxNloXDTMzMDEyMzAzMTMxNlowQzELMAkGA1UEBgwCSlAx 5 | EjAQBgNVBAoMCUpJTi5HUi5KUDEMMAoGA1UECwwDUlJSMRIwEAYDVQQDDAlsb2Nh 6 | bGhvc3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANFJTxWqup3nV9dsJAku 7 | p+WaXnPNIzcpAA3qMGZDJTJsfa8Du7ZxTP0XJK5mETttBrn711cJxAuP3KjqnW9S 8 | vtZ9lY2sXJ6Zj62sN5LwG3VVe25dI28yR1EsbHjJ5Zjf9tmggMC6am52dxuHbt5/ 9 | vHo4ngJuKE/U+eeGRivMn6gFAgMBAAGjgYUwgYIwDAYDVR0TAQH/BAIwADAxBglg 10 | hkgBhvhCAQ0EJBYiUnVieS9PcGVuU1NMIEdlbmVyYXRlZCBDZXJ0aWZpY2F0ZTAd 11 | BgNVHQ4EFgQUpZIyygD9JxFYHHOTEuWOLbCKfckwCwYDVR0PBAQDAgWgMBMGA1Ud 12 | JQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3DQEBBQUAA4IBAQBwAIj5SaBHaA5X31IP 13 | CFCJiep96awfp7RANO0cuUj+ZpGoFn9d6FXY0g+Eg5wAkCNIzZU5NHN9xsdOpnUo 14 | zIBbyTfQEPrge1CMWMvL6uGaoEXytq84VTitF/xBTky4KtTn6+es4/e7jrrzeUXQ 15 | RC46gkHObmDT91RkOEGjHLyld2328jo3DIN/VTHIryDeVHDWjY5dENwpwdkhhm60 16 | DR9IrNBbXWEe9emtguNXeN0iu1ux0lG1Hc6pWGQxMlRKNvGh0yZB9u5EVe38tOV0 17 | jQaoNyL7qzcQoXD3Dmbi1p0iRmg/+HngISsz8K7k7MBNVsSclztwgCzTZOBiVtkM 18 | rRlQ 19 | -----END CERTIFICATE----- 20 | -------------------------------------------------------------------------------- /spec/td/fixture/server.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIICXQIBAAKBgQDRSU8Vqrqd51fXbCQJLqflml5zzSM3KQAN6jBmQyUybH2vA7u2 3 | cUz9FySuZhE7bQa5+9dXCcQLj9yo6p1vUr7WfZWNrFyemY+trDeS8Bt1VXtuXSNv 4 | MkdRLGx4yeWY3/bZoIDAumpudncbh27ef7x6OJ4CbihP1PnnhkYrzJ+oBQIDAQAB 5 | AoGBAIf4CstW2ltQO7+XYGoex7Hh8s9lTSW/G2vu5Hbr1LTHy3fzAvdq8MvVR12O 6 | rk9fa+lU9vhzPc0NMB0GIDZ9GcHuhW5hD1Wg9OSCbTOkZDoH3CAFqonjh4Qfwv5W 7 | IPAFn9KHukdqGXkwEMdErsUaPTy9A1V/aROVEaAY+HJgq/eZAkEA/BP1QMV04WEZ 8 | Oynzz7/lLizJGGxp2AOvEVtqMoycA/Qk+zdKP8ufE0wbmCE3Qd6GoynavsHb6aGK 9 | gQobb8zDZwJBANSK6MrXlrZTtEaeZuyOB4mAmRzGzOUVkUyULUjEx2GDT93ujAma 10 | qm/2d3E+wXAkNSeRpjUmlQXy/2oSqnGvYbMCQQDRM+cYyEcGPUVpWpnj0shrF/QU 11 | 9vSot/X1G775EMTyaw6+BtbyNxVgOIu2J+rqGbn3c+b85XqTXOPL0A2RLYkFAkAm 12 | syhSDtE9X55aoWsCNZY/vi+i4rvaFoQ/WleogVQAeGVpdo7/DK9t9YWoFBIqth0L 13 | mGSYFu9ZhvZkvQNV8eYrAkBJ+rOIaLDsmbrgkeDruH+B/9yrm4McDtQ/rgnOGYnH 14 | LjLpLLOrgUxqpzLWe++EwSLwK2//dHO+SPsQJ4xsyQJy 15 | -----END RSA PRIVATE KEY----- 16 | -------------------------------------------------------------------------------- /spec/td/fixture/testRootCA.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDBDCCAewCCQCQdq0wStTtITANBgkqhkiG9w0BAQsFADBEMR0wGwYDVQQDDBR3 3 | d3cudHJlYXN1cmVkYXRhLmNvbTELMAkGA1UEBhMCVVMxFjAUBgNVBAcMDVNhbiBG 4 | cmFuc2lzY28wHhcNMjIxMDE0MDQzNjA0WhcNMzIwNzEzMDQzNjA0WjBEMR0wGwYD 5 | VQQDDBR3d3cudHJlYXN1cmVkYXRhLmNvbTELMAkGA1UEBhMCVVMxFjAUBgNVBAcM 6 | DVNhbiBGcmFuc2lzY28wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCU 7 | 4RbAD5pop+AUYyJT0L4r/ZwOFz72TqQwyPh8EbpPlkvErWPP07kLO1gmfPHPH6zE 8 | 4sAd8Pl5HZ29Z3fF9zO4ZaOehO+kMHp3jp1nq6vURzMjxDj42CyP+M7DOLrov98m 9 | uwcZMdVjJlz9ZWt7P4x/F5cyqnzmFYxefi0xbgFFXKaWL1W3Fmq1dKMCgbCVWxlA 10 | cufRFQmmO+OOcKfJTpcZZXH+VKai81OLjjoBkEERqZU8ySv2zR3gLBMAU3x/VL6/ 11 | AxcAi6ZbxQ29O28VTojQ44dA0op5wDtbI3reH9cMgALI1Swo2DyxXejYoZJYKqk3 12 | AH2BiM0dbPnzOoufxjm/AgMBAAEwDQYJKoZIhvcNAQELBQADggEBACLeA1a/TU4E 13 | 8bKVEr8X9CMROG2j5lsmPLZRTDUQT50iu5MBZzCNQi2tzUC+QHWC8m/xrhsg4b4t 14 | r52dQuZlPSEGbzdiltrzQPODnSkA6tmgPeaea5wKG8abDgVQRbMBiamwgJruIi1s 15 | ynX2yD4qdo07fTAC9CyKd9kogMRMXaqVppfIrob+HL3KQJ4MQPRJb+IN3K14Bdq8 16 | /XL+8zH4zsWxPCkJyKlcUwGx3M0NHnYX0l8Enap65McTiaPIiCgr5TnkJmctDKBm 17 | Zrv+c1sb69oJtlayGKWNymxdU+1f79IVioQf4qCa7ZB1//ON60KrQ9cRsmObJtnE 18 | KD6Oett1Kto= 19 | -----END CERTIFICATE----- 20 | -------------------------------------------------------------------------------- /spec/td/fixture/testServer.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID2jCCAsKgAwIBAgIJALE6L1U9f459MA0GCSqGSIb3DQEBCwUAMEQxHTAbBgNV 3 | BAMMFHd3dy50cmVhc3VyZWRhdGEuY29tMQswCQYDVQQGEwJVUzEWMBQGA1UEBwwN 4 | U2FuIEZyYW5zaXNjbzAeFw0yMjEwMTQwOTU3NDdaFw0zMjEwMTEwOTU3NDdaMGwx 5 | CzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1TYW4g 6 | RnJhbnNpc2NvMRYwFAYDVQQKDA1UcmVhc3VyZSBEYXRhMQwwCgYDVQQLDANFbmcx 7 | CjAIBgNVBAMMASowggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC6TBzz 8 | TnMXEwQhmsF9ggvyjzhsVQRQnVafPjR45mKgN5+4QmkW+t1RUv93ysz2UOXM1XfN 9 | HptcrgbdsEmhDDWopsPgfJrSdN6O44oxplZOTusn20lJDG4Ogm0Kdn2k0P6Cj1bG 10 | PmFLgw4BeXqhMWI60rbQw/HtFDMx6MWU555wr//jJNPIPOVBzVOiGhmoWYJY8ARB 11 | WlplKeW0BkQ2J+650Q4dcqrrq4R6UbnyTjf9pqxSa2qP5y9ZQr+69COveX26ZSmA 12 | 76PRzGE1oohsB37OCcQMwU+CuIHZkS9CMEGgkYfmfKaGtrsGtG52eGGwiYoAwSGJ 13 | BJ/v53Pp3BGKWo9tAgMBAAGjgaYwgaMwXgYDVR0jBFcwVaFIpEYwRDEdMBsGA1UE 14 | AwwUd3d3LnRyZWFzdXJlZGF0YS5jb20xCzAJBgNVBAYTAlVTMRYwFAYDVQQHDA1T 15 | YW4gRnJhbnNpc2NvggkAkHatMErU7SEwCQYDVR0TBAIwADALBgNVHQ8EBAMCBPAw 16 | EwYDVR0lBAwwCgYIKwYBBQUHAwEwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqG 17 | SIb3DQEBCwUAA4IBAQBQkwlcEc98Pj9NB9NjhwZalgI36ptmj7vxyD8GXaJQWh7P 18 | BKnOTxwbqhY68B7QgQ+NTusltnWsiO2IBAZR3MZGeLh4rFUUi4daww7fCLbYQD0t 19 | YKOjUv8C4rlp8lRs/IMDMl1i2w1R6x78+RKSy1OhIvxT3qV9Zxx8RfqINboyDlbo 20 | 6lOEQlLYGGUJutIbkUoPCo3TO3/oaLfwlFhnxgEwf+lX/qHoqTkz16JRzil6yFOx 21 | qc5ugRF0S34lpnBd6qfbhV1QFY17WvSbbXmpBPONI2SLs0WSwwCTFVRg8Z+Y90oG 22 | oGH+JdWsxn+9e8JTB/UOpxr+vWTyQ63wLaElPOyC 23 | -----END CERTIFICATE----- 24 | -------------------------------------------------------------------------------- /spec/td/fixture/testServer.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEogIBAAKCAQEAukwc805zFxMEIZrBfYIL8o84bFUEUJ1Wnz40eOZioDefuEJp 3 | FvrdUVL/d8rM9lDlzNV3zR6bXK4G3bBJoQw1qKbD4Hya0nTejuOKMaZWTk7rJ9tJ 4 | SQxuDoJtCnZ9pND+go9Wxj5hS4MOAXl6oTFiOtK20MPx7RQzMejFlOeecK//4yTT 5 | yDzlQc1TohoZqFmCWPAEQVpaZSnltAZENifuudEOHXKq66uEelG58k43/aasUmtq 6 | j+cvWUK/uvQjr3l9umUpgO+j0cxhNaKIbAd+zgnEDMFPgriB2ZEvQjBBoJGH5nym 7 | hra7BrRudnhhsImKAMEhiQSf7+dz6dwRilqPbQIDAQABAoIBAF8IKpB2yUDRA398 8 | 6Qz0BNIz+u1QJQZWbHSJD81IgLEIDuK4hdEiITm14/mgqxNPSxpFHnq1DT2mzHvT 9 | zItppgmlIDBof7WxxkIPklQnbMk/erd3JhgsTgv6vlLjBM7JibriEbrI4WrarI9V 10 | /5cwkNI+4OD3w3ZTopXoDroZuPnz/v0EKhDz05E4jHijfiRL3OxaAvZ2zWVKifVh 11 | elilbCeLR2KgKG6jPrBzub2Yll9C7Xxt7emIMvyTg5jEOkmm7IWfAZ6uIraI7pAs 12 | 5MlwQDIb/Ypby4gHJ0CPWy2AhdybF2Y+hm5jXY1aD7xjlG8pjudidYtnrpWO1kPV 13 | bbqIvUUCgYEA3G5JQvcA9ZTiHculyW/6O++NXrOkznXI04JhhKN6JvdTdp3xC61k 14 | H+gnP0e2/FeqqZWF6KWlmPBuonofEQqyCwJdHRZESaixgw7iKqN2t11Z9EhWzoGq 15 | UGc1oMT90AIHxsGUBbUIZdlj1D8KDx/qymlsKp8ozTKwyCqNesiYet8CgYEA2FvQ 16 | 94zgOauOCwfOwDipwivrJ9nblykMfcp8NCC2jLxI3rjH30EhRWBZrc0V55erjfrz 17 | QgJYlj4kN/3GwLVq2Agr5EczjePScRlkpPNHdfAfXKvg7dHHsgrYf9lP6j2Adz8B 18 | mhyYs1gfo6ekeHLoqKBlSn08AosfSTpETuIdizMCgYBFATFmCTT/rA/tC+dmW+uV 19 | /7PdxZb+Gtk3fUVR5GtE73/tThw7b5g8dMx0ftrFvBvs4qX84n4olnvL2TcIerSp 20 | xZ+oj2PpOyn2wR4EAxAS7uJOGqcyFl1etjCPl5ttFnWgvtC7yKRMXfVmaCWZ/n/d 21 | xYra/OAk/I1i3A9WNJ2nOQKBgD83jLZYPkf7fXRxopJ9u/RVOs+ZE1V2lATJPkNI 22 | 763tcelJ2nS8JgmMXoeu7eCOa3z/v0YhQ1sa6yBFEWbLW12l/ZUkzMZ/s8SCI+si 23 | flXShIdiXUV/zzaRfrLUf0o1EC1HhqNOCbwVWqFJ4X+kK6DhxNbgAsHHfqu5z62w 24 | 2esLAoGAbIbd5Xd+lRXbuNTyIqAONf4KgH529hU4W9tw+kR5OFqadC557Jvs33nn 25 | iSqGgET+DZTxaOZk8KeCKZys8IcIxvWyhq22/Pd0BKl1ozWj4w/bkDlqO2V9I9NY 26 | 32DBk4iAHljJqkmxnJkSXgGypf4guPZlNGPbTijXw3oHaIpvPl0= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /spec/td/fixture/tmp.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/treasure-data/td/7da645b4e835615a31fd31bc3e76e710fd39ce25/spec/td/fixture/tmp.zip -------------------------------------------------------------------------------- /spec/td/helpers_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/helpers' 3 | require 'open3' 4 | 5 | module TreasureData 6 | 7 | describe 'format_with_delimiter' do 8 | it "delimits the number with ',' by default" do 9 | expect(Helpers.format_with_delimiter(0)).to eq("0") 10 | expect(Helpers.format_with_delimiter(10)).to eq("10") 11 | expect(Helpers.format_with_delimiter(100)).to eq("100") 12 | expect(Helpers.format_with_delimiter(1000)).to eq("1,000") 13 | expect(Helpers.format_with_delimiter(10000)).to eq("10,000") 14 | expect(Helpers.format_with_delimiter(100000)).to eq("100,000") 15 | expect(Helpers.format_with_delimiter(1000000)).to eq("1,000,000") 16 | end 17 | end 18 | 19 | describe 'on_64bit_os?' do 20 | 21 | def with_env(name, var) 22 | backup, ENV[name] = ENV[name], var 23 | begin 24 | yield 25 | ensure 26 | ENV[name] = backup 27 | end 28 | end 29 | 30 | it 'returns true for windows when PROCESSOR_ARCHITECTURE=amd64' do 31 | allow(Helpers).to receive(:on_windows?) {true} 32 | with_env('PROCESSOR_ARCHITECTURE', 'amd64') { 33 | expect(Helpers.on_64bit_os?).to be(true) 34 | } 35 | end 36 | 37 | it 'returns true for windows when PROCESSOR_ARCHITECTURE=x86 and PROCESSOR_ARCHITEW6432 is set' do 38 | allow(Helpers).to receive(:on_windows?) {true} 39 | with_env('PROCESSOR_ARCHITECTURE', 'x86') { 40 | with_env('PROCESSOR_ARCHITEW6432', '') { 41 | expect(Helpers.on_64bit_os?).to be(true) 42 | } 43 | } 44 | end 45 | 46 | it 'returns false for windows when PROCESSOR_ARCHITECTURE=x86 and PROCESSOR_ARCHITEW6432 is not set' do 47 | allow(Helpers).to receive(:on_windows?) {true} 48 | with_env('PROCESSOR_ARCHITECTURE', 'x86') { 49 | with_env('PROCESSOR_ARCHITEW6432', nil) { 50 | expect(Helpers.on_64bit_os?).to be(false) 51 | } 52 | } 53 | end 54 | 55 | it 'returns true for non-windows when uname -m prints x86_64' do 56 | allow(Helpers).to receive(:on_windows?) {false} 57 | allow(Open3).to receive(:capture2).with('uname', '-m') {['x86_64', double(:success? => true)]} 58 | expect(Helpers.on_64bit_os?).to be(true) 59 | expect(Open3).to have_received(:capture2).with('uname', '-m') 60 | end 61 | 62 | it 'returns false for non-windows when uname -m prints i686' do 63 | allow(Helpers).to receive(:on_windows?) {false} 64 | allow(Open3).to receive(:capture2).with('uname', '-m') {['i686', double(:success? => true)]} 65 | expect(Helpers.on_64bit_os?).to be(false) 66 | expect(Open3).to have_received(:capture2).with('uname', '-m') 67 | end 68 | 69 | end 70 | 71 | end 72 | -------------------------------------------------------------------------------- /spec/td/updater_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/command/common' 3 | require 'td/updater' 4 | require 'webrick' 5 | require 'webrick/https' 6 | require 'webrick/httpproxy' 7 | require 'logger' 8 | 9 | module TreasureData::Updater 10 | 11 | %w(x86_64-darwin14 x64-mingw32).each do |platform| 12 | describe "RUBY_PLATFORM is '#{platform}'" do 13 | before do 14 | stub_const('RUBY_PLATFORM', platform) 15 | end 16 | 17 | describe 'without the TD_TOOLBELT_UPDATE_ROOT environment variable defined' do 18 | let :default_toolbelt_url do 19 | "http://toolbelt.treasuredata.com" 20 | end 21 | 22 | describe 'endpoints methods' do 23 | it 'use the default root path' do 24 | expect(TreasureData::Updater.endpoint_root).to eq(default_toolbelt_url) 25 | expect(TreasureData::Updater.version_endpoint).to match(Regexp.new(default_toolbelt_url)) 26 | expect(TreasureData::Updater.update_package_endpoint).to match(Regexp.new(default_toolbelt_url)) 27 | end 28 | end 29 | end 30 | 31 | describe 'with the TD_TOOLBELT_UPDATE_ROOT environment variable defined' do 32 | before do 33 | ENV['TD_TOOLBELT_UPDATE_ROOT'] = 'https://0.0.0.0:5000/' 34 | end 35 | describe 'endpoints methods' do 36 | it 'use the custom root path' do 37 | expect(TreasureData::Updater.endpoint_root).to eq(ENV['TD_TOOLBELT_UPDATE_ROOT']) 38 | expect(TreasureData::Updater.version_endpoint).to match(Regexp.new(ENV['TD_TOOLBELT_UPDATE_ROOT'])) 39 | expect(TreasureData::Updater.update_package_endpoint).to match(Regexp.new(ENV['TD_TOOLBELT_UPDATE_ROOT'])) 40 | end 41 | end 42 | after do 43 | ENV.delete 'TD_TOOLBELT_UPDATE_ROOT' 44 | end 45 | end 46 | end 47 | end 48 | 49 | describe 'with a proxy' do 50 | before :each do 51 | setup_proxy_server 52 | setup_server 53 | end 54 | 55 | after :each do 56 | if @proxy_server 57 | @proxy_server.shutdown 58 | @proxy_server_thread.join 59 | end 60 | if @server 61 | @server.shutdown 62 | @server_thread.join 63 | end 64 | end 65 | 66 | class TestUpdater 67 | include TreasureData::Updater::ModuleDefinition 68 | 69 | def initialize(endpoint_root) 70 | @endpoint_root = endpoint_root 71 | end 72 | 73 | def updating_lock_path 74 | File.expand_path("updating_lock_path.lock", File.dirname(__FILE__)) 75 | end 76 | 77 | def on_windows? 78 | true 79 | end 80 | 81 | def endpoint_root 82 | @endpoint_root 83 | end 84 | 85 | def latest_local_version 86 | '0.11.5' 87 | end 88 | end 89 | 90 | class JarUpdateTester 91 | include TreasureData::Updater 92 | 93 | def kick 94 | jar_update 95 | end 96 | end 97 | 98 | it 'downloads tmp.zip via proxy and raise td version conflict' do 99 | with_proxy do 100 | expect { 101 | TestUpdater.new("https://localhost:#{@server.config[:Port]}").update 102 | }.to raise_error TreasureData::Command::UpdateError 103 | end 104 | end 105 | 106 | it 'works' do 107 | with_proxy do 108 | with_env('TD_TOOLBELT_JARUPDATE_ROOT', "https://localhost:#{@server.config[:Port]}") do 109 | expect { 110 | JarUpdateTester.new.kick 111 | }.not_to raise_error 112 | end 113 | end 114 | end 115 | 116 | describe "current working directory doesn't change after call `jar_update`" do 117 | shared_examples_for("jar_update behavior") do 118 | it "doesn't change cwd" do 119 | with_env('TD_TOOLBELT_JARUPDATE_ROOT', "https://localhost:#{@server.config[:Port]}") do 120 | pwd = Dir.pwd 121 | subject 122 | expect(Dir.pwd).to eq pwd 123 | end 124 | end 125 | 126 | it "don't exists td-import.jar.new" do 127 | with_env('TD_TOOLBELT_JARUPDATE_ROOT', "https://localhost:#{@server.config[:Port]}") do 128 | subject 129 | end 130 | tmpfile = File.join(TreasureData::Updater.jarfile_dest_path, 'td-import.jar.new') 131 | expect(File.exist?(tmpfile)).to eq false 132 | end 133 | end 134 | 135 | let(:updater) { JarUpdateTester.new } 136 | 137 | subject { updater.kick } 138 | 139 | context "not updated" do 140 | before { allow(updater).to receive(:existent_jar_updated_time).and_return(Time.now) } 141 | 142 | it_behaves_like "jar_update behavior" 143 | end 144 | 145 | context "updated" do 146 | before { allow(updater).to receive(:existent_jar_updated_time).and_return(Time.at(0)) } 147 | 148 | it_behaves_like "jar_update behavior" 149 | end 150 | end 151 | 152 | def with_proxy 153 | with_env('HTTP_PROXY', "http://localhost:#{@proxy_server.config[:Port]}") do 154 | yield 155 | end 156 | end 157 | 158 | def with_env(name, var) 159 | backup, ENV[name] = ENV[name], var 160 | begin 161 | yield 162 | ensure 163 | ENV[name] = backup 164 | end 165 | end 166 | 167 | def setup_proxy_server 168 | logger = Logger.new(STDERR) 169 | logger.progname = 'proxy' 170 | logger.level = Logger::Severity::FATAL # avoid logging 171 | @proxy_server = WEBrick::HTTPProxyServer.new( 172 | :BindAddress => "localhost", 173 | :Logger => logger, 174 | :Port => 1000 + rand(1000), 175 | :AccessLog => [] 176 | ) 177 | @proxy_server_thread = start_server_thread(@proxy_server) 178 | @proxy_server 179 | end 180 | 181 | def setup_server 182 | logger = Logger.new(STDERR) 183 | logger.progname = 'server' 184 | logger.level = Logger::Severity::FATAL # avoid logging 185 | @server = WEBrick::HTTPServer.new( 186 | :BindAddress => "localhost", 187 | :Logger => logger, 188 | :Port => 1000 + rand(1000), 189 | :AccessLog => [], 190 | :DocumentRoot => '.', 191 | :SSLEnable => true, 192 | :SSLCACertificateFile => fixture_file('testRootCA.crt'), 193 | :SSLCertificate => cert('testServer.crt'), 194 | :SSLPrivateKey => key('testServer.key') 195 | ) 196 | @serverport = @server.config[:Port] 197 | @server.mount( 198 | '/version.exe', 199 | WEBrick::HTTPServlet::ProcHandler.new(method(:version).to_proc) 200 | ) 201 | @server.mount( 202 | '/td-update-exe.zip', 203 | WEBrick::HTTPServlet::ProcHandler.new(method(:download).to_proc) 204 | ) 205 | @server.mount( 206 | '/maven2/com/treasuredata/td-import/maven-metadata.xml', 207 | WEBrick::HTTPServlet::ProcHandler.new(method(:metadata).to_proc) 208 | ) 209 | @server.mount( 210 | '/maven2/com/treasuredata/td-import/version/td-import-version-jar-with-dependencies.jar', 211 | WEBrick::HTTPServlet::ProcHandler.new(method(:jar).to_proc) 212 | ) 213 | @server_thread = start_server_thread(@server) 214 | @server 215 | end 216 | 217 | def version(req, res) 218 | res['content-type'] = 'text/plain' 219 | res.body = '0.11.6' 220 | end 221 | 222 | def download(req, res) 223 | res['content-type'] = 'application/octet-stream' 224 | res.body = File.read(fixture_file('tmp.zip')) 225 | end 226 | 227 | def metadata(req, res) 228 | res['content-type'] = 'application/xml' 229 | res.body = '20141204123456version' 230 | end 231 | 232 | def jar(req, res) 233 | res['content-type'] = 'application/octet-stream' 234 | res.body = File.read(fixture_file('tmp.zip')) 235 | end 236 | 237 | def start_server_thread(server) 238 | t = Thread.new { 239 | Thread.current.abort_on_exception = true 240 | server.start 241 | } 242 | while server.status != :Running 243 | sleep 0.1 244 | unless t.alive? 245 | t.join 246 | raise 247 | end 248 | end 249 | t 250 | end 251 | 252 | def cert(filename) 253 | OpenSSL::X509::Certificate.new(File.read(fixture_file(filename))) 254 | end 255 | 256 | def key(filename) 257 | OpenSSL::PKey::RSA.new(File.read(fixture_file(filename))) 258 | end 259 | 260 | def fixture_file(filename) 261 | File.expand_path(File.join('fixture', filename), File.dirname(__FILE__)) 262 | end 263 | end 264 | end 265 | -------------------------------------------------------------------------------- /spec/td/version_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | module TreasureData::Command 4 | describe '--version' do 5 | it "shows version" do 6 | stderr, stdout = execute_td("--version") 7 | expect(stderr).to eq("") 8 | expect(stdout).to eq <<-STDOUT 9 | #{TreasureData::TOOLBELT_VERSION} 10 | STDOUT 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /td.gemspec: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | $:.push File.expand_path('../lib', __FILE__) 3 | require 'td/version' 4 | 5 | Gem::Specification.new do |gem| 6 | gem.name = "td" 7 | gem.description = "CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing" 8 | gem.homepage = "http://treasure-data.com/" 9 | gem.summary = "CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing" 10 | gem.version = TreasureData::TOOLBELT_VERSION 11 | gem.authors = ["Treasure Data, Inc."] 12 | gem.email = "support@treasure-data.com" 13 | gem.files = `git ls-files`.split("\n").select { |f| !f.start_with?('dist') } 14 | gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") 15 | gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } 16 | gem.require_paths = ['lib'] 17 | gem.required_ruby_version = '>= 2.1' 18 | gem.licenses = ["Apache-2.0"] 19 | 20 | gem.add_dependency "msgpack" 21 | gem.add_dependency "rexml" 22 | gem.add_dependency "yajl-ruby", ">= 1.3.1", "< 2.0" 23 | gem.add_dependency "hirb", ">= 0.4.5" 24 | gem.add_dependency "parallel", "~> 1.20.0" 25 | gem.add_dependency "td-client", ">= 1.0.8", "< 3" 26 | gem.add_dependency "td-logger", ">= 0.3.21", "< 2" 27 | gem.add_dependency "rubyzip", "~> 1.3.0" 28 | gem.add_dependency "zip-zip", "~> 0.3" 29 | gem.add_dependency "ruby-progressbar", "~> 1.7" 30 | gem.add_development_dependency "rake" 31 | gem.add_development_dependency "rspec" 32 | gem.add_development_dependency 'webrick' 33 | gem.add_development_dependency "simplecov" 34 | gem.add_development_dependency 'coveralls' 35 | end 36 | --------------------------------------------------------------------------------