├── .bundle └── config ├── .circleci └── config.yml ├── .coveralls.yml ├── .github ├── dependabot.yml └── workflows │ └── codeql-analysis.yml ├── .gitignore ├── ChangeLog ├── Gemfile ├── README.md ├── Rakefile ├── SECURITY.md ├── VERSION ├── data └── ca-bundle.crt ├── examples └── walkthrough.rb ├── lib ├── td-client.rb └── td │ ├── client.rb │ ├── client │ ├── api.rb │ ├── api │ │ ├── account.rb │ │ ├── bulk_import.rb │ │ ├── bulk_load.rb │ │ ├── database.rb │ │ ├── export.rb │ │ ├── import.rb │ │ ├── job.rb │ │ ├── result.rb │ │ ├── schedule.rb │ │ ├── server_status.rb │ │ ├── table.rb │ │ └── user.rb │ ├── api_error.rb │ ├── compat_gzip_reader.rb │ ├── model.rb │ └── version.rb │ └── core_ext │ └── openssl │ └── ssl │ └── sslcontext │ └── set_params.rb ├── spec ├── spec_helper.rb └── td │ ├── client │ ├── account_api_spec.rb │ ├── api_error_spec.rb │ ├── api_spec.rb │ ├── api_ssl_connection_spec.rb │ ├── bulk_import_spec.rb │ ├── bulk_load_spec.rb │ ├── db_api_spec.rb │ ├── export_api_spec.rb │ ├── import_api_spec.rb │ ├── job_api_spec.rb │ ├── model_job_spec.rb │ ├── model_schedule_spec.rb │ ├── model_schema_spec.rb │ ├── result_api_spec.rb │ ├── sched_api_spec.rb │ ├── server_status_api_spec.rb │ ├── spec_resources.rb │ ├── table_api_spec.rb │ └── user_api_spec.rb │ ├── client_sched_spec.rb │ └── client_spec.rb └── td-client.gemspec /.bundle/config: -------------------------------------------------------------------------------- 1 | --- 2 | BUNDLE_JOBS: 4 3 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | 4 | orbs: 5 | ruby: circleci/ruby@2.0.0 6 | win: circleci/windows@5.1.0 7 | 8 | commands: 9 | install_windows_requirements: 10 | description: "Install windows requirements" 11 | parameters: 12 | ruby_version: 13 | type: string 14 | default: "latest" 15 | steps: 16 | - run: 17 | name: "Install MSYS2" 18 | shell: powershell.exe 19 | command: choco install msys2 -y 20 | - run: 21 | name: "Install Ruby devkit" 22 | shell: powershell.exe 23 | command: ridk install 2 3 24 | - run: 25 | name: "Install Ruby version" 26 | shell: powershell.exe 27 | command: choco install ruby --version=<> 28 | - run: 29 | name: "Install bundler" 30 | command: gem install bundler 31 | bundle-install: 32 | description: "Install dependencies" 33 | steps: 34 | - run: 35 | name: Which bundler? 36 | command: ruby -v; bundle -v 37 | - run: 38 | name: Bundle install 39 | command: bundle install 40 | pre-tests: 41 | description: "Prepare for tests" 42 | steps: 43 | - run: 44 | name: Echo test certs 45 | command: | 46 | echo $TEST_ROOT_CA | base64 -d > ./spec/td/client/testRootCA.crt 47 | echo $TEST_SERVER_CRT | base64 -d > ./spec/td/client/testServer.crt 48 | echo $TEST_SERVER_KEY | base64 -d > ./spec/td/client/testServer.key 49 | run-tests: 50 | description: "Run tests" 51 | steps: 52 | - run: 53 | name: Run tests 54 | command: bundle exec rake spec 55 | run-tests-flow: 56 | description: "Single flow for running tests" 57 | steps: 58 | - checkout 59 | - bundle-install 60 | - pre-tests 61 | - run-tests 62 | run-windows-tests-flow: 63 | description: "Single flow for running tests on Windows" 64 | steps: 65 | - checkout 66 | - run: 67 | name: Which bundler? 68 | shell: powershell.exe 69 | command: ruby -v; bundle -v 70 | - run: 71 | name: Bundle install 72 | shell: powershell.exe 73 | command: bundle install 74 | - pre-tests 75 | - run: 76 | name: Run tests 77 | shell: powershell.exe 78 | command: bundle exec rake spec 79 | 80 | jobs: 81 | 82 | ruby_27: 83 | docker: 84 | - image: cimg/ruby:2.7 85 | steps: 86 | - run-tests-flow 87 | 88 | ruby_30: 89 | docker: 90 | - image: cimg/ruby:3.0 91 | steps: 92 | - run-tests-flow 93 | 94 | ruby_31: 95 | docker: 96 | - image: cimg/ruby:3.1 97 | steps: 98 | - run-tests-flow 99 | 100 | ruby_32: 101 | docker: 102 | - image: cimg/ruby:3.2 103 | steps: 104 | - run-tests-flow 105 | 106 | ruby_33: 107 | docker: 108 | - image: cimg/ruby:3.3.8 109 | steps: 110 | - run-tests-flow 111 | 112 | ruby_34: 113 | docker: 114 | - image: cimg/ruby:3.4.4 115 | steps: 116 | - run-tests-flow 117 | 118 | win_ruby: 119 | executor: 120 | name: win/default 121 | shell: bash.exe 122 | steps: 123 | - install_windows_requirements: 124 | ruby_version: "3.4.4.2" 125 | - run-windows-tests-flow 126 | 127 | jruby_latest: 128 | docker: 129 | - image: circleci/jruby:latest 130 | steps: 131 | - run-tests-flow 132 | 133 | workflows: 134 | tests: 135 | jobs: 136 | - ruby_27 137 | - ruby_30 138 | - ruby_31 139 | - ruby_32 140 | - ruby_33 141 | - ruby_34 142 | - jruby_latest 143 | - win_ruby 144 | -------------------------------------------------------------------------------- /.coveralls.yml: -------------------------------------------------------------------------------- 1 | service_name: circleci 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "bundler" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '19 15 * * 3' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'ruby' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v3 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v2 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | 52 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 53 | # queries: security-extended,security-and-quality 54 | 55 | 56 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 57 | # If this step fails, then you should remove it and run the build manually (see below) 58 | - name: Autobuild 59 | uses: github/codeql-action/autobuild@v2 60 | 61 | # ℹ️ Command-line programs to run using the OS shell. 62 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 63 | 64 | # If the Autobuild fails above, remove it and uncomment the following three lines. 65 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 66 | 67 | # - run: | 68 | # echo "Run, Build Application using script" 69 | # ./location_of_script_within_repo/buildscript.sh 70 | 71 | - name: Perform CodeQL Analysis 72 | uses: github/codeql-action/analyze@v2 73 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .ruby-version 2 | Gemfile.lock 3 | coverage/* 4 | pkg/* 5 | doc/ 6 | .yardoc/ 7 | spec/td/client/testRootCA.crt 8 | spec/td/client/testServer.crt 9 | spec/td/client/testServer.key 10 | -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- 1 | == 2025-08-05 version 3.0.0 2 | 3 | * Remove partial_delete API (#153) 4 | 5 | * Support trino type query (#152) 6 | 7 | == 2023-07-11 version 2.0.0 8 | 9 | * v3 core utilization API removal 10 | 11 | * add custom cert and cert verification options: cert_path and skip_cert_verify 12 | 13 | * Support Ruby v3.2 14 | 15 | == 2021-03-03 version 1.0.8 16 | 17 | * Ignore elapsed_time #129 18 | 19 | * Handle client erros #127 20 | 21 | 22 | == 2019-06-04 version 1.0.7 23 | 24 | * Add TreasureData::Client#change_database #123 25 | 26 | * Ruby 1.8.7 compatibility was removed #121 27 | 28 | == 2018-03-29 version 1.0.6 29 | 30 | * Support Treasure Data's new result export #117 #120 31 | 32 | == 2018-01-18 version 1.0.5 33 | 34 | * Add TreasureData::Job#auto_update_status=false to prevent API call #115 35 | * Support include_v flag #114 36 | 37 | == 2017-09-21 version 1.0.4 38 | 39 | * Fix the bug in #112 #113 40 | 41 | == 2017-09-15 version 1.0.3 42 | 43 | * Handle the case HTTPClient doesn't yield block or body is nil #112 44 | 45 | == 2017-07-20 version 1.0.2 46 | 47 | * Allow name and sql_alias are equal #111 48 | * Organize HTTP exceptions #110 49 | 50 | == 2017-05-22 version 1.0.1 51 | 52 | * fix bug on result download if API returns 500 #109 53 | 54 | == 2017-01-20 version 1.0.0 55 | 56 | * Fix the bug to show the number of records as duration #107 57 | * Support Ruby 2.4.0 or later #108 58 | 59 | == 2016-11-29 version 0.8.85 60 | 61 | * Support JRuby 1.7 again #98 62 | * Add num records with job api #106 63 | * Add independent result export #105 64 | 65 | == 2016-10-17 version 0.8.84 66 | 67 | * fix schema validation #93 68 | * Job#wait rescues network error #94 69 | * Change default endpoint #39 70 | * Drop Ruby 2.0.0 support 71 | 72 | == 2016-09-23 version 0.8.83 73 | 74 | * Support TD table's SQL Column Alias (#87) 75 | * support updating bulk_load session settings (#80) 76 | * Drop Ruby 1.9 support (#90) 77 | 78 | == 2016-07-11 version 0.8.82 79 | 80 | * Add conflicts_with to AlreadyExistError (#84) 81 | 82 | == 2016-06-03 version 0.8.81 83 | 84 | * Rewrite job_result_download logic #85 85 | 86 | == 2016-05-30 version 0.8.80 87 | 88 | * Support resume on more APIs (#81) 89 | 90 | == 2016-04-25 version 0.8.79 91 | 92 | * Support dummy schedule #78 93 | * Resuming job result downloading #79 94 | 95 | == 2016-01-25 version 0.8.78 96 | 97 | * Implement Job#wait 98 | * add JRuby to supported list 99 | * Allow MessagePack 0.7 100 | 101 | == 2015-12-14 version 0.8.77 102 | 103 | * fix inflate post body 104 | * fix response byte size check for multibyte string 105 | * fix TD_CLIENT_DEBUG mode 106 | 107 | == 2015-10-29 version 0.8.76 108 | 109 | * fix BulkLoad#bulk_load_run with scheduled_time. 110 | * change url encode pattern. ' '(space) -> %20, '+' -> %2b, '.' -> %2E 111 | * fix some typos. 112 | 113 | == 2015-08-10 version 0.8.75 114 | 115 | * add APIError#api_backtrace. 116 | and exclude API Backtrace from APIError#message 117 | 118 | == 2015-08-03 version 0.8.74 119 | 120 | * fix Scheduled#run that call client#run_schedule 121 | 122 | == 2015-07-29 version 0.8.73 123 | 124 | * remove item table support. 125 | * raise TreasureData::IncompleteError if mismatched content-length and response body when GET request. 126 | It occures by combination OpenSSL raise EOFError and use MessagePack::Unpacker. 127 | 128 | == 2015-07-27 version 0.8.72 129 | 130 | * add new API Job#result_raw 131 | * update httpclient dependency version, because can't work under httpclient 2.5.1 132 | 133 | == 2015-07-10 version 0.8.71 134 | 135 | * Remove to/from parameter from table/tail API 136 | 137 | == 2015-05-19 version 0.8.70 138 | 139 | * [experimenta] remove client side validation from bulk_load API. 140 | * Add 'duration' property to Job model. 141 | 142 | == 2015-04-17 version 0.8.69 143 | 144 | * [experimental] bulk_load API support. The API is subject to change. 145 | * Use HTTPClient's wiredump debugging when TD_CLIENT_DEBUG is set. 146 | * YARD annotations. Thanks uu59! 147 | * Improved test coverage. Thanks to yotii23! 148 | 149 | == 2015-02-16 version 0.8.68 150 | 151 | * Fix non ASCII & UTF-8 filename issue in bulk_import:upload. API side 152 | requires part_name is to be in UTF-8 encoded so change CES at client side 153 | before sending. 154 | * Allow httpclient version ~= 2.5 155 | * Fix inappropriate error message for POST request. It doesn't retry but error 156 | message stated retry period. 157 | * (internal) Split API definition into several files per API group. 158 | * Fix resulting value of Command#history for ScheduledJobs. 159 | * Fix ScheduledJob crash when scheduled_at == '' 160 | 161 | == 2014-12-03 version 0.8.67 162 | 163 | * Reset HTTPClient afterward to avoid potential fd exhaustion by keep-alive 164 | connections 165 | 166 | == 2014-10-16 version 0.8.66 167 | 168 | * Disabled SSLv3 for the server connection as further follow up to the 169 | POODLE vulnerability for SSL v3. 170 | * Make maximum cumulative retry delay configurable through the API's constructor 171 | 172 | == 2014-10-14 version 0.8.65 173 | 174 | * Upgraded httpclient used by PUT operations. New v2.4.0 enables SSL/TLS 175 | version negotiation 176 | 177 | == 2014-09-25 version 0.8.64 178 | 179 | * Implemented retrying on all get REST APIs requests excepting 180 | job_result_format with IO, job_result_each and job_result_each_with_compr_size 181 | that lead to certain types of exception thrown: Errno::ECONNREFUSED, 182 | Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, 183 | SocketError and HTTP status code greater than 500 (Server errors). 184 | * Implemented retrying on post REST APIs requests if the client options contain 185 | the 'retry_post_requests' option under the same circumstances as the get 186 | requests 187 | 188 | == 2014-07-31 version 0.8.63 189 | 190 | * Added Job's model 'result_each_with_compr_size' method to expose the 191 | progressive result file compressed size read to the callee 192 | * Added 'ForbiddenError' exception class for HTTP error codes 403 193 | * Added 'permission' field to Database model and made it accessible from the 194 | Database and Table models 195 | * Converted the 'update_schedule' method to using POST instead of GET 196 | * Fixed tests 197 | 198 | == 2014-06-18 version 0.8.62 199 | 200 | * Fix JSON parsing issue when Pig queries don't alias column values outputted 201 | from Pig functions (e.g. COUNT, SUM) 202 | * Fix database, column, and table validation methods to use max length 255 203 | instead of 256 204 | * Use constants instead of hardcoded status names when handling jobs 205 | 206 | == 2014-05-22 version 0.8.61 207 | 208 | * Fix bug in raising error when response body is nil 209 | 210 | == 2014-04-28 version 0.8.60 211 | 212 | * Fix bug in fetching the query/job result schema 213 | 214 | == 2014-04-23 version 0.8.59 215 | 216 | * Improved client side validation methods for database, table, column, and 217 | result set 218 | * show GET response body in debug mode (enabled with TD_CLIENT_DEBUG environment 219 | variable) 220 | * optimized the job model self updating methods to avoid an update if the job 221 | is in a 'finished' state 222 | * propagate the job show and job list CPU time field 223 | 224 | == 2014-03-18 version 0.8.58 225 | 226 | * Improved visualization of item tables 227 | * Replace api.treasuredata.com with api-import.treasuredata.com during imports 228 | * Raise AuthError exceptions if the API key failed to authenticate 229 | 230 | == 2014-02-21 version 0.8.57 231 | 232 | * Remove Aggregation Schema 233 | * show_bulk_import uses new efficient endpoint 234 | * Update SSL certificate 235 | 236 | == 2013-11-14 version 0.8.56 237 | 238 | * Remove organization, role and ip_limit subcommands 239 | * Change item_table parameter 240 | * Now accept :header option to set custom header 241 | 242 | == 2013-09-13 version 0.8.55 243 | 244 | * Use httpclient gem for import and bulk_import upload 245 | * connect_timeout / read_timeout / send_timeout options are available. 246 | * these options affect only import and bulk_import upload. 247 | 248 | 249 | == 2013-08-23 version 0.8.54 250 | 251 | * Support table's expire_days API 252 | 253 | 254 | == 2013-07-22 version 0.8.53 255 | 256 | * Add normalized_msgpack method to serialize Bignum type 257 | 258 | 259 | == 2013-06-24 version 0.8.52 260 | 261 | * Add last_log_timestamp to Table model 262 | 263 | 264 | == 2013-06-17 version 0.8.51 265 | 266 | * Relax dependent gem versions 267 | 268 | 269 | == 2013-05-27 version 0.8.50 270 | 271 | * add_user now requires email and passowrd 272 | 273 | 274 | == 2013-05-06 version 0.8.49 275 | 276 | * Add User-Agent header 277 | * VERSION constant moved to under TreasureData::Client 278 | 279 | 280 | == 2013-04-22 version 0.8.48 281 | 282 | * create_schedule now takes :type option 283 | * Fix wrong error messages 284 | * Ues 'api-import' instead of 'api' on data import 285 | 286 | 287 | == 2013-04-09 version 0.8.47 288 | 289 | * Fix HTTP proxy handlig issue which is overwritten with ENV['HTTP_PROXY'] 290 | 291 | 292 | == 2013-03-29 version 0.8.46 293 | 294 | * Add IP limitation API 295 | 296 | 297 | == 2013-01-25 version 0.8.45 298 | 299 | * Re-implement Client#job_status using /v3/job/status/job_id 300 | instead of /v3/job/show/job_id to poll the progress of a job 301 | 302 | 303 | == 2013-01-23 version 0.8.44 304 | 305 | * Re-add json gem dependency 306 | 307 | 308 | == 2013-01-23 version 0.8.43 309 | 310 | * Add organization parameter support to create_database, query, 311 | partial_delete, create_bulk_import, create_result 312 | 313 | 314 | == 2013-01-16 version 0.8.42 315 | 316 | * Added retry_limit to job and schedule APIs 317 | * Increased table/database name limit from 32 to 256 318 | 319 | 320 | == 2013-01-10 version 0.8.41 321 | 322 | * Fix API#job_result_format to handle Content-Encoding properly 323 | 324 | 325 | == 2012-12-27 version 0.8.40 326 | 327 | * Add Table#last_import to use counter_updated_at column 328 | 329 | 330 | == 2012-12-05 version 0.8.39 331 | 332 | * Add conditions argument to Client#list_jobs for slow query listing 333 | 334 | 335 | == 2012-11-21 version 0.8.38 336 | 337 | * Add Account#created_at 338 | 339 | 340 | == 2012-11-16 version 0.8.37 341 | 342 | * Remove json gem dependency again (previous version has bug) 343 | 344 | 345 | == 2012-11-16 version 0.8.36 346 | 347 | * Remove json gem dependency 348 | 349 | 350 | == 2012-10-23 version 0.8.35 351 | 352 | * Added Account#account_id 353 | 354 | 355 | == 2012-10-16 version 0.8.34 356 | 357 | * Set Net::HTTP#open_timeout = 60 358 | 359 | 360 | == 2012-10-10 version 0.8.33 361 | 362 | * Supports import_with_id API 363 | * Supports deflate and gzip Content-Encodings and sends Accept-Encoding header 364 | 365 | 366 | == 2012-10-09 version 0.8.32 367 | 368 | * Added Client#swap_table 369 | 370 | 371 | == 2012-09-21 version 0.8.31 372 | 373 | * Added Job#db_name 374 | 375 | 376 | == 2012-09-21 version 0.8.30 377 | 378 | * Fixed Account#storage_size_string and Table#estimated_storage_size_string 379 | 380 | 381 | == 2012-09-17 version 0.8.29 382 | 383 | * Added Client#core_utilization method 384 | * Added Account#guaranteed_cores and #maximum_cores methods 385 | 386 | 387 | == 2012-09-17 version 0.8.27 388 | 389 | * Added Table#estimated_storage_size_string 390 | 391 | 392 | == 2012-09-13 version 0.8.26 393 | 394 | * Added Account model and Table#esetimated_storage_size method 395 | * Name length limit is changed from 32 characters to 256 characters 396 | 397 | 398 | == 2012-09-04 version 0.8.25 399 | 400 | * Added Client#change_my_password(old_password, password) 401 | 402 | 403 | == 2012-08-30 version 0.8.24 404 | 405 | * TreasureData::Client.new supports :http_proxy option 406 | 407 | 408 | == 2012-08-30 version 0.8.23 409 | 410 | * Supports HTTP_PROXY environment variable 411 | 412 | 413 | == 2012-08-20 version 0.8.22 414 | 415 | * Top-level resource models support org_name parameter 416 | 417 | 418 | == 2012-08-06 version 0.8.21 419 | 420 | * Added multiuser features: organizations, users, roles 421 | * Added access control 422 | 423 | 424 | == 2012-07-23 version 0.8.20 425 | 426 | * Implemented Zlib::GzipReader#readpartial for compatibility with ruby 1.8 427 | 428 | 429 | == 2012-07-03 version 0.8.19 430 | 431 | * Added Client#partial_delete 432 | * Client#query and Client#create_schedule support 'priority' option 433 | 434 | 435 | == 2012-06-26 version 0.8.18 436 | 437 | * Client#result_each(&block) uses streaming raed not to read all data into memory 438 | * Client#result_format(format, io=nil) supports second argument not to read 439 | all data into memory 440 | 441 | 442 | == 2012-06-11 version 0.8.17 443 | 444 | * Client#jobs supports status option 445 | 446 | 447 | == 2012-05-10 version 0.8.16 448 | 449 | * Added bulk import feature 450 | 451 | 452 | == 2012-04-26 version 0.8.15 453 | 454 | * Result model replaces ResultSet model 455 | * Removed methods related to ResultSet from Job and Schedule models 456 | * Added methods related to Result to Job and Schedule models 457 | 458 | 459 | == 2012-04-03 version 0.8.14 460 | 461 | * Added Database#count, #created_at and #updated_at 462 | * Added Table#created_at and #updated_at 463 | 464 | 465 | == 2012-04-03 version 0.8.13 466 | 467 | * Added Job#hive_result_schema 468 | 469 | 470 | == 2012-03-12 version 0.8.12 471 | 472 | * Client#run_schedule returns an array of ScheduledJob 473 | 474 | 475 | == 2012-03-01 version 0.8.11 476 | 477 | * Use data/ca-bundle.crt for SSL connections 478 | 479 | 480 | == 2012-02-22 version 0.8.10 481 | 482 | * Added Client#run_schedule and update_schedule 483 | * Added timezone, delay and next_time fields to the Schedule model 484 | * create_aggregation_schema accepts params argument 485 | 486 | 487 | == 2012-02-12 version 0.8.9 488 | 489 | * Added API#normalize_table_name and API#normalize_database_name 490 | 491 | 492 | == 2012-02-02 version 0.8.8 493 | 494 | * Fixed SSL support 495 | 496 | 497 | == 2012-02-02 version 0.8.7 498 | 499 | * Added SSL support 500 | 501 | 502 | == 2012-01-19 version 0.8.6 503 | 504 | * Check JSON format of HTTP responses 505 | 506 | 507 | == 2011-12-04 version 0.8.5 508 | 509 | * added new feature: ResultSet 510 | * added new feature: AggregationSchema 511 | * added Job#rset and Schedule#rset to get associated ResultSet 512 | 513 | 514 | == 2011-11-11 version 0.8.4 515 | 516 | * Added Model#client 517 | * Added Database#query 518 | * Added Table#import 519 | * Increased http.read_timeout on Client#import 520 | 521 | 522 | == 2011-10-03 version 0.8.3 523 | 524 | * Added Client#tail method 525 | 526 | 527 | == 2011-09-13 version 0.8.2 528 | 529 | * Added APIs for scheduled queries 530 | * Set 'Content-Length: 0' header to POST request if no parameters are 531 | provided 532 | 533 | 534 | == 2011-09-09 version 0.8.1 535 | 536 | * Added Client#kill method 537 | * Client.authenticate throws AuthError instead of APIError when 538 | status code is 400 539 | 540 | 541 | == 2011-08-21 version 0.8.0 542 | 543 | * First release 544 | 545 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Treasure Data API library for Ruby 2 | 3 | [Coverage Status](https://coveralls.io/github/treasure-data/td-client-ruby?branch=master) 4 | 5 | ## Getting Started 6 | 7 | > gem install td-client 8 | 9 | ### Running Tests 10 | 11 | > gem install jeweler 12 | > gem install webmock 13 | > rake spec 14 | 15 | ## Configuration 16 | 17 | The Client API library constructor supports a number of options that can 18 | be provided as part of the optional 'opts' hash map to these methods: 19 | 20 | - `initialize`(apikey, opts={}) (constructor) 21 | - `Client.authenticate`(user, password, opts={}) (class method) 22 | - `Client.server_status`(opts={}) (class method) 23 | 24 | ### Endpoint 25 | 26 | Add the `:endpoint` key to the opts to provide an alternative endpoint to make 27 | the API calls to. Examples are an alternate API endpoint or a static IP address 28 | provided by Treasure Data on a case-by-case basis. 29 | 30 | The default endpoint is: 31 | 32 | https://api.treasuredata.com 33 | 34 | and configure communication using SSL encryption over HTTPS; the same happens 35 | every time the provided endpoint is prefixed by 'https'. 36 | 37 | E.g. 38 | 39 | opts.merge({:endpoint => "https://api-alternate.treasuredata.com"}) 40 | 41 | The endpoint can alternatively be provided by setting the `TD_API_SERVER` 42 | environment variable. The `:endpoint` option takes precedence over the 43 | `TD_API_SERVER` environment variable setting. 44 | 45 | For communication through a Proxy, please see the Proxy option below. 46 | 47 | ### Connection, Read, and Send Timeouts 48 | 49 | The connection, read, and send timeouts can be provided via the 50 | `:connect_timeout`, `:read_timeout`, `:send_timeout` keys respectively. 51 | The values for these keys is the number of seconds. 52 | 53 | E.g. 54 | 55 | opts.merge({:connect_timeout => 60, 56 | :read_timeout => 60, 57 | :send_timeout => 60}) 58 | 59 | ### SSL 60 | 61 | The `:ssl` key specifies whether SSL communication ought to be used when 62 | communicating with the default or custom endpoint. 63 | 64 | This option is ignored if the endpoint (default or custom) URL specifies the 65 | scheme (e.g. the protocol, https or http) in which case SSL enabled/disabled is 66 | inferred directly from the URL scheme. 67 | 68 | E.g. 69 | 70 | # SSL is enabled as specified by the :ssl option 71 | opts.merge({:endpoint => "api.treasuredata.com", :ssl => true}) 72 | 73 | # the ssl option is ignored in this case 74 | opts.merge({:endpoint => "https://api.treasuredata.com", :ssl => false}) 75 | 76 | # You can disable verification 77 | opts.merge({:endpoint => "https://api.treasuredata.com", :verify => false}) 78 | 79 | # You can use your own certificate 80 | opts.merge({:endpoint => "https://api.treasuredata.com", :verify => "/path/to/cert"}) 81 | 82 | ### Proxy 83 | 84 | If your network requires accessing our endpoint through a proxy (anonymous or 85 | private), the proxy settings can be specified through the `:http_proxy` option. 86 | 87 | E.g. 88 | 89 | # anonymous proxies 90 | opts.merge({:http_proxy => "http://myproxy.com:1234"}) 91 | opts.merge({:http_proxy => "myproxy.com:1234"}) 92 | 93 | # private proxies 94 | opts.merge({:http_proxy => "https://username:password@myproxy.com:1234"}) 95 | opts.merge({:http_proxy => "username:password@myproxy.com:1234"}) 96 | 97 | The proxy settings can alternatively be provided by setting the `HTTP_PROXY` 98 | environment variable. The `:http_proxy` option takes precedence over the 99 | `HTTP_PROXY` environment variable setting. 100 | 101 | ### Additional Header(s) 102 | 103 | The Ruby client configures the communication with the Treasure Data REST API 104 | endpoints using the required HTTP Headers (including authorization, Date, 105 | User-Agent and Accept-Encoding, Content-Length, Content-Encoding where 106 | applicable) basing on what method call is made. 107 | 108 | The user can specify any additional HTTP Header using the `:headers` option. 109 | 110 | E.g. 111 | 112 | opts.merge({:headers => "MyHeader: myheadervalue;"}) 113 | 114 | To specify a custom User-Agent please see the option below. 115 | 116 | ### Additional User-Agent(s) 117 | 118 | Add the `:user_agent` key to the opts hash to provide an additional user agent 119 | for all the interactions with the APIs. 120 | The provided user agent string will be added to this default client library user 121 | agent `TD-Client-Ruby: X.Y.Z` where X.Y.Z is the version number of this Ruby 122 | Client library. 123 | 124 | E.g. 125 | 126 | opts.merge({:user_agent => "MyApp: A.B.C"}) 127 | 128 | which sets the user agent to: 129 | 130 | "MyApp: A.B.C; TD-Client-Ruby: X.Y.Z" 131 | 132 | ### Retry POST Requests 133 | 134 | Add the `:retry_post_requests` key to the opts hash to require that every 135 | failed POST request is retried for up to 10 minutes with an exponentially 136 | doubling backoff window just as it happens for GET requests by default. 137 | 138 | Note that this can lead to unwanted results since POST request as not always 139 | idempotent, that is retrying a POST request by lead to the creation of 140 | duplicated resources, such as the submission of 2 identical jobs on failures. 141 | 142 | As for GET requests, the retrying mechanism is triggered on 500+ HTTP error 143 | codes or the raising of exceptions during the communication with the remote 144 | server. 145 | 146 | E.g. 147 | 148 | opts.merge({:retry_post_requests => true}) 149 | 150 | to enable retrying for POST requests. 151 | 152 | ## Testing Hooks 153 | 154 | The client library implements several hooks to enable/disable/trigger special 155 | behaviors. These hooks are triggered using environment variables: 156 | 157 | - Enable debugging mode: 158 | 159 | `$ TD_CLIENT_DEBUG=1` 160 | 161 | Currently debugging mode consists of: 162 | 163 | - show request and response of `HTTP`/`HTTPS` `GET` REST API calls; 164 | - show request of `HTTP`/`HTTPS` `POST`/`PUT` REST API calls. 165 | 166 | ## More Information 167 | 168 | - Copyright: (c) 2011-2023 Treasure Data Inc. 169 | - License: Apache License, Version 2.0 170 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bundler' 3 | Bundler::GemHelper.install_tasks 4 | 5 | require 'rspec/core' 6 | require 'rspec/core/rake_task' 7 | 8 | RSpec::Core::RakeTask.new(:spec) do |spec| 9 | spec.pattern = FileList['spec/**/*_spec.rb'] 10 | end 11 | 12 | task :coverage do |t| 13 | ENV['SIMPLE_COV'] = '1' 14 | Rake::Task["spec"].invoke 15 | end 16 | 17 | task :default => :build 18 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Reporting a Vulnerability 4 | 5 | Treasure Data values the security of its customers and is committed to ensuring that the systems and products are secure. We invite all bug bounty researchers to join our efforts in identifying and reporting vulnerabilities in our systems. 6 | 7 | Submit your findings to our dedicated bug bounty email address [vulnerabilities@treasuredata.com](mailto:vulnerabilities@treasuredata.com) and help us keep Treasure Data secure. Let’s work together to make the Internet a safer place! 8 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 2.0.0 2 | -------------------------------------------------------------------------------- /examples/walkthrough.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib') 2 | 3 | require "msgpack" 4 | require "tempfile" 5 | require "stringio" 6 | require "zlib" 7 | require "td-client" 8 | 9 | include TreasureData 10 | 11 | class Example 12 | def initialize(api_key, opts={}) 13 | @client = Client.new(api_key, opts) 14 | end 15 | 16 | # Utils 17 | def put_title(title) 18 | puts title 19 | puts "-" * 3 20 | end 21 | 22 | def put_separator 23 | puts "*" * 50 24 | puts "" 25 | end 26 | 27 | def wait_job(job) 28 | # wait for job to be finished 29 | cmdout_lines = 0 30 | stderr_lines = 0 31 | job.wait(nil, detail: true, verbose: true) do 32 | cmdout = job.debug['cmdout'].to_s.split("\n")[cmdout_lines..-1] || [] 33 | stderr = job.debug['stderr'].to_s.split("\n")[stderr_lines..-1] || [] 34 | (cmdout + stderr).each {|line| 35 | $stdout.puts " "+line 36 | } 37 | cmdout_lines += cmdout.size 38 | stderr_lines += stderr.size 39 | end 40 | end 41 | # API 42 | def server_status 43 | begin 44 | server_status = @client.server_status 45 | puts "Server Status: #{server_status}" 46 | rescue StandardError => e 47 | puts e.message 48 | end 49 | end 50 | 51 | def account_info 52 | put_title "Account" 53 | begin 54 | account = @client.account 55 | puts "ID: #{account.account_id}" 56 | puts "Plan: #{account.plan}" 57 | puts "Storage Size: #{account.storage_size}" 58 | puts "Guaranteed cores: #{account.guaranteed_cores}" 59 | puts "Maximum cores: #{account.maximum_cores}" 60 | puts "Created at: #{account.created_at}" 61 | rescue StandardError => e 62 | puts e.message 63 | end 64 | end 65 | 66 | def list_databases 67 | put_title "Databases" 68 | begin 69 | databases = @client.databases 70 | for db in databases 71 | puts "Name: #{db.name}" 72 | puts "Created at: #{db.created_at}" 73 | end 74 | rescue StandardError => e 75 | puts e.message 76 | end 77 | end 78 | 79 | def create_database(name) 80 | put_title "Create database" 81 | begin 82 | flag = @client.create_database(name) 83 | puts "Created database #{name} successfully!" 84 | rescue StandardError => e 85 | puts e.message 86 | end 87 | end 88 | 89 | def delete_database(name) 90 | put_title "Delete database" 91 | begin 92 | flag = @client.delete_database(name) 93 | puts "Deleted database #{name} successfully!" 94 | rescue StandardError => e 95 | puts e.message 96 | end 97 | end 98 | 99 | def create_log_table(db, name) 100 | put_title "Create log table" 101 | begin 102 | flag = @client.create_log_table(db, name) 103 | puts "Created log table #{name} in database #{db}!" 104 | rescue StandardError => e 105 | puts e.message 106 | end 107 | end 108 | 109 | def update_schema(db, table, schema) 110 | put_title "Update schema" 111 | begin 112 | flag = @client.update_schema(db, table, schema) 113 | puts "Updated schema for table #{table} in database #{db}!" 114 | rescue StandardError => e 115 | puts e.message 116 | end 117 | end 118 | 119 | def import_data(db, table) 120 | put_title "Import data" 121 | 122 | sample_data = { 123 | "time" => "1", 124 | "col1" => "value1", 125 | "col2" => "value2" 126 | } 127 | 128 | out = Tempfile.new("td-import") 129 | out.binmode if out.respond_to?(:binmode) 130 | 131 | writer = Zlib::GzipWriter.new(out) 132 | 133 | begin 134 | writer.write sample_data.to_msgpack 135 | writer.finish 136 | 137 | size = out.pos 138 | out.pos = 0 139 | 140 | time = @client.import(db, table, "msgpack.gz", out, size) 141 | puts "Importing data is done in #{time}" 142 | rescue StandardError => e 143 | puts e.message 144 | ensure 145 | out.close 146 | writer.close 147 | end 148 | end 149 | 150 | def delete_bulk_import(name) 151 | begin 152 | @client.delete_bulk_import(name) 153 | rescue StandardError => e 154 | puts e.message 155 | end 156 | 157 | end 158 | 159 | def create_bulk_import(name, db, table) 160 | begin 161 | @client.create_bulk_import(name, db, table) 162 | rescue StandardError => e 163 | puts e.message 164 | end 165 | end 166 | 167 | def perform_bulk_import(name) 168 | put_title "Bulk import upload part" 169 | 170 | str_io = StringIO.new 171 | writer = Zlib::GzipWriter.new(str_io) 172 | writer.sync = true 173 | 174 | packer = MessagePack::Packer.new(writer) 175 | 176 | begin 177 | (1..100).each { |i| 178 | obj = { 179 | "time" => "#{i}", 180 | "col1" => "value#{i}", 181 | "col2" => "value_2_#{i}" 182 | } 183 | packer.write obj 184 | } 185 | packer.flush 186 | writer.flush(Zlib::FINISH) 187 | 188 | @client.bulk_import_upload_part(name, "part_1", str_io, str_io.size) 189 | job = @client.perform_bulk_import(name) 190 | puts "Performing bulk import, job ID: #{job.job_id}" 191 | 192 | wait_job(job) 193 | 194 | result = @client.bulk_import(name) 195 | puts "Bulk Status: #{result.status}" 196 | rescue StandardError => e 197 | puts e.message 198 | ensure 199 | writer.close 200 | end 201 | end 202 | 203 | def query_example(db_name, q) 204 | put_title "Querying data" 205 | job = @client.query(db_name, q) 206 | wait_job(job) 207 | end 208 | 209 | def run 210 | bulk_name = "td_client_ruby_bulk_import" 211 | db_name = "client_ruby_test" 212 | table = "log1" 213 | 214 | #server_status 215 | 216 | #account_info 217 | 218 | #list_databases 219 | 220 | #create_database(db_name) 221 | 222 | #create_log_table(db_name, table) 223 | 224 | ## Update table schema with specific fields 225 | #field1 = TreasureData::Schema::Field.new("col1", "string") 226 | #field2 = TreasureData::Schema::Field.new("col2", "string") 227 | #schema = TreasureData::Schema.new([field1, field2]) 228 | #update_schema(db_name, table, schema) 229 | 230 | ## Or from json 231 | #schema = TreasureData::Schema.new 232 | #schema.from_json({ 233 | #col1: "string", 234 | #col2: "string" 235 | #}) 236 | #update_schema(db_name, table, schema) 237 | 238 | #import_data(db_name, table) 239 | 240 | #query_example(db_name, "select * from log1") 241 | #create_bulk_import(bulk_name, db_name, table) 242 | #perform_bulk_import(bulk_name) 243 | 244 | # delete_database(db_name) 245 | # delete_bulk_import(bulk_name) 246 | end 247 | end 248 | 249 | options = {:verify => ENV["MY_CUSTOM_CERT"]} 250 | api_key = ENV["TD_API_KEY"] ||= "" 251 | ex = Example.new(api_key, opt=options) 252 | ex.server_status 253 | -------------------------------------------------------------------------------- /lib/td-client.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'td', 'client') 2 | -------------------------------------------------------------------------------- /lib/td/client.rb: -------------------------------------------------------------------------------- 1 | module TreasureData 2 | 3 | require 'td/client/api' 4 | require 'td/client/model' 5 | 6 | 7 | class Client 8 | # @param [String] user TreasureData username 9 | # @param [String] password TreasureData password 10 | # @param [Hash] opts options for API 11 | # @return [Client] instance of this class 12 | def self.authenticate(user, password, opts={}) 13 | api = API.new(nil, opts) 14 | apikey = api.authenticate(user, password) 15 | new(apikey) 16 | end 17 | 18 | # @param [Hash] opts options for API 19 | # @return [String] HTTP status code of server returns 20 | def self.server_status(opts={}) 21 | api = API.new(nil, opts) 22 | api.server_status 23 | end 24 | 25 | # @param [String] apikey TreasureData API key 26 | # @param [Hash] opts options for API 27 | def initialize(apikey, opts={}) 28 | @api = API.new(apikey, opts) 29 | end 30 | 31 | # @!attribute [r] api 32 | attr_reader :api 33 | 34 | # @return [String] API key 35 | def apikey 36 | @api.apikey 37 | end 38 | 39 | # @return [String] HTTP status code of server returns 40 | def server_status 41 | @api.server_status 42 | end 43 | 44 | # @param [String] db_name 45 | # @param [Hash] opts 46 | # @return [true] 47 | def create_database(db_name, opts={}) 48 | @api.create_database(db_name, opts) 49 | end 50 | 51 | # @param [String] db_name 52 | # @return [Symbol] 53 | def delete_database(db_name) 54 | @api.delete_database(db_name) 55 | end 56 | 57 | # @return [Account] 58 | def account 59 | account_id, plan, storage, guaranteed_cores, maximum_cores, created_at = @api.show_account 60 | return Account.new(self, account_id, plan, storage, guaranteed_cores, maximum_cores, created_at) 61 | end 62 | 63 | # @return [Array] databases 64 | def databases 65 | m = @api.list_databases 66 | m.map {|db_name,(count, created_at, updated_at, org, permission)| 67 | Database.new(self, db_name, nil, count, created_at, updated_at, org, permission) 68 | } 69 | end 70 | 71 | # @param [String] db_name 72 | # @return [Database] 73 | def database(db_name) 74 | m = @api.list_databases 75 | m.each {|name,(count, created_at, updated_at, org, permission)| 76 | if name == db_name 77 | return Database.new(self, name, nil, count, created_at, updated_at, org, permission) 78 | end 79 | } 80 | raise NotFoundError, "Database '#{db_name}' does not exist" 81 | end 82 | 83 | # @param [String] db 84 | # @param [String] table 85 | # @option params [Fixnum] :expire_days days to expire table 86 | # @option params [Boolean] :include_v (true) include v column on Hive 87 | # @option params [Boolean] :detect_schema (true) detect schema on import 88 | # @return [true] 89 | def create_log_table(db_name, table_name, params={}) 90 | @api.create_log_table(db_name, table_name, params) 91 | end 92 | 93 | # Swap table names 94 | # 95 | # @param [String] db_name 96 | # @param [String] table_name1 97 | # @param [String] table_name2 98 | # @return [true] 99 | def swap_table(db_name, table_name1, table_name2) 100 | @api.swap_table(db_name, table_name1, table_name2) 101 | end 102 | 103 | # @param [String] db_name 104 | # @param [String] table_name 105 | # @param [String] schema 106 | # @return [true] 107 | def update_schema(db_name, table_name, schema) 108 | @api.update_schema(db_name, table_name, schema.to_json) 109 | end 110 | 111 | # @param [String] db 112 | # @param [String] table 113 | # @option params [Fixnum] :expire_days days to expire table 114 | # @option params [Boolean] :include_v (true) include v column on Hive 115 | # @option params [Boolean] :detect_schema (true) detect schema on import 116 | # @return [true] 117 | def update_table(db_name, table_name, params={}) 118 | @api.update_table(db_name, table_name, params) 119 | end 120 | 121 | # @param [String] db_name 122 | # @param [String] table_name 123 | # @param [Fixnum] expire_days 124 | # @return [true] 125 | def update_expire(db_name, table_name, expire_days) 126 | @api.update_expire(db_name, table_name, expire_days) 127 | end 128 | 129 | # @param [String] db_name 130 | # @param [String] table_name 131 | # @return [Symbol] 132 | def delete_table(db_name, table_name) 133 | @api.delete_table(db_name, table_name) 134 | end 135 | 136 | # @param [String] db_name 137 | # @return [Array] Tables 138 | def tables(db_name) 139 | m = @api.list_tables(db_name) 140 | m.map {|table_name, (type, schema, count, created_at, updated_at, estimated_storage_size, last_import, last_log_timestamp, expire_days, include_v)| 141 | schema = Schema.new.from_json(schema) 142 | Table.new(self, db_name, table_name, type, schema, count, created_at, updated_at, 143 | estimated_storage_size, last_import, last_log_timestamp, expire_days, include_v) 144 | } 145 | end 146 | 147 | # @param [String] db_name 148 | # @param [String] table_name 149 | # @return [Table] 150 | def table(db_name, table_name) 151 | tables(db_name).each {|t| 152 | if t.name == table_name 153 | return t 154 | end 155 | } 156 | raise NotFoundError, "Table '#{db_name}.#{table_name}' does not exist" 157 | end 158 | 159 | # @param [String] db_name 160 | # @param [String] table_name 161 | # @param [Fixnum] count 162 | # @param [Proc] block 163 | # @return [Array, nil] 164 | def tail(db_name, table_name, count, to = nil, from = nil, &block) 165 | @api.tail(db_name, table_name, count, to, from, &block) 166 | end 167 | 168 | # @param [String] db_name 169 | # @param [String] table_name 170 | # @param [String] new_db_name 171 | # @return [true] 172 | def change_database(db_name, table_name, new_db_name) 173 | @api.change_database(db_name, table_name, new_db_name) 174 | end 175 | 176 | # @param [String] db_name 177 | # @param [String] q 178 | # @param [String] result_url 179 | # @param [Fixnum] priority 180 | # @param [Fixnum] retry_limit 181 | # @param [Hash] opts 182 | # @return [Job] 183 | def query(db_name, q, result_url=nil, priority=nil, retry_limit=nil, opts={}) 184 | # for compatibility, assume type is hive unless specifically specified 185 | type = opts[:type] || opts['type'] || :hive 186 | raise ArgumentError, "The specified query type is not supported: #{type}" unless [:hive, :pig, :impala, :presto, :trino].include?(type) 187 | job_id = @api.query(q, type, db_name, result_url, priority, retry_limit, opts) 188 | Job.new(self, job_id, type, q) 189 | end 190 | 191 | # @param [Fixnum] from 192 | # @param [Fixnum] to 193 | # @param [String] status 194 | # @param [Hash] conditions 195 | # @return [Job] 196 | def jobs(from=nil, to=nil, status=nil, conditions=nil) 197 | results = @api.list_jobs(from, to, status, conditions) 198 | results.map {|job_id, type, status, query, start_at, end_at, cpu_time, 199 | result_size, result_url, priority, retry_limit, org, db, 200 | duration, num_records| 201 | Job.new(self, job_id, type, query, status, nil, nil, start_at, end_at, cpu_time, 202 | result_size, nil, result_url, nil, priority, retry_limit, org, db, 203 | duration, num_records) 204 | } 205 | end 206 | 207 | # @param [String] job_id 208 | # @return [Job] 209 | def job(job_id) 210 | job_id = job_id.to_s 211 | type, query, status, url, debug, start_at, end_at, cpu_time, 212 | result_size, result_url, hive_result_schema, priority, retry_limit, org, db, duration, num_records = @api.show_job(job_id) 213 | Job.new(self, job_id, type, query, status, url, debug, start_at, end_at, cpu_time, 214 | result_size, nil, result_url, hive_result_schema, priority, retry_limit, org, db, duration, num_records) 215 | end 216 | 217 | # @param [String] job_id 218 | # @return [String] HTTP status code 219 | def job_status(job_id) 220 | return @api.job_status(job_id) 221 | end 222 | 223 | # @param [String] job_id 224 | # @return [Object] 225 | def job_result(job_id) 226 | @api.job_result(job_id) 227 | end 228 | 229 | # @param [String] job_id 230 | # @param [String] format 231 | # @param [IO] io 232 | # @param [Proc] block 233 | # @return [String] 234 | def job_result_format(job_id, format, io=nil, &block) 235 | @api.job_result_format(job_id, format, io, &block) 236 | end 237 | 238 | def job_result_raw(job_id, format, io=nil, &block) 239 | @api.job_result_raw(job_id, format, io, &block) 240 | end 241 | 242 | # @param [String] job_id 243 | # @param [Proc] block 244 | # @return [nil] 245 | def job_result_each(job_id, &block) 246 | @api.job_result_each(job_id, &block) 247 | end 248 | 249 | # @param [String] job_id 250 | # @param [Proc] block 251 | # @return [nil] 252 | def job_result_each_with_compr_size(job_id, &block) 253 | @api.job_result_each_with_compr_size(job_id, &block) 254 | end 255 | 256 | # @param [String] job_id 257 | # @return [String] former_status 258 | def kill(job_id) 259 | @api.kill(job_id) 260 | end 261 | 262 | # @param [String] db_name 263 | # @param [String] table_name 264 | # @param [String] storage_type 265 | # @param [Hash] opts 266 | # @return [Job] 267 | def export(db_name, table_name, storage_type, opts={}) 268 | job_id = @api.export(db_name, table_name, storage_type, opts) 269 | Job.new(self, job_id, :export, nil) 270 | end 271 | 272 | # @param [String] target_job_id 273 | # @param [Hash] opts 274 | # @return [Job] 275 | def result_export(target_job_id, opts={}) 276 | job_id = @api.result_export(target_job_id, opts) 277 | Job.new(self, job_id, :result_export, nil) 278 | end 279 | 280 | # @param [String] name 281 | # @param [String] database 282 | # @param [String] table 283 | # @param [Hash] opts 284 | # @return [nil] 285 | def create_bulk_import(name, database, table, opts={}) 286 | @api.create_bulk_import(name, database, table, opts) 287 | end 288 | 289 | # @param [String] name 290 | # @return [nil] 291 | def delete_bulk_import(name) 292 | @api.delete_bulk_import(name) 293 | end 294 | 295 | # @param [String] name 296 | # @return [nil] 297 | def freeze_bulk_import(name) 298 | @api.freeze_bulk_import(name) 299 | end 300 | 301 | # @param [String] name 302 | # @return [nil] 303 | def unfreeze_bulk_import(name) 304 | @api.unfreeze_bulk_import(name) 305 | end 306 | 307 | # @param [String] name 308 | # @param [Hash] opts options for API 309 | # @return [Job] 310 | def perform_bulk_import(name, opts={}) 311 | job_id = @api.perform_bulk_import(name, opts) 312 | Job.new(self, job_id, :bulk_import, nil) 313 | end 314 | 315 | # @param [String] name 316 | # @return [nil] 317 | def commit_bulk_import(name) 318 | @api.commit_bulk_import(name) 319 | end 320 | 321 | # @param [String] name 322 | # @param [Proc] block 323 | # @return [Hash] 324 | def bulk_import_error_records(name, &block) 325 | @api.bulk_import_error_records(name, &block) 326 | end 327 | 328 | # @param [String] name 329 | # @return [BulkImport] 330 | def bulk_import(name) 331 | data = @api.show_bulk_import(name) 332 | BulkImport.new(self, data) 333 | end 334 | 335 | # @return [Array] 336 | def bulk_imports 337 | @api.list_bulk_imports.map {|data| 338 | BulkImport.new(self, data) 339 | } 340 | end 341 | 342 | # @param [String] name 343 | # @param [String] part_name 344 | # @param [String, StringIO] stream 345 | # @param [Fixnum] size 346 | # @return [nil] 347 | def bulk_import_upload_part(name, part_name, stream, size) 348 | @api.bulk_import_upload_part(name, part_name, stream, size) 349 | end 350 | 351 | # @param [String] name 352 | # @param [String] part_name 353 | # @return [nil] 354 | def bulk_import_delete_part(name, part_name) 355 | @api.bulk_import_delete_part(name, part_name) 356 | end 357 | 358 | # @param [String] name 359 | # @return [Array] 360 | def list_bulk_import_parts(name) 361 | @api.list_bulk_import_parts(name) 362 | end 363 | 364 | # @param [String] name 365 | # @param [Hash] opts 366 | # @return [Time] 367 | def create_schedule(name, opts) 368 | raise ArgumentError, "'cron' option is required" unless opts[:cron] || opts['cron'] 369 | raise ArgumentError, "'query' option is required" unless opts[:query] || opts['query'] 370 | start = @api.create_schedule(name, opts) 371 | return start && Time.parse(start) 372 | end 373 | 374 | # @param [String] name 375 | # @return [Array] 376 | def delete_schedule(name) 377 | @api.delete_schedule(name) 378 | end 379 | 380 | # @return [Array] 381 | def schedules 382 | result = @api.list_schedules 383 | result.map {|name,cron,query,database,result_url,timezone,delay,next_time,priority,retry_limit,org_name| 384 | Schedule.new(self, name, cron, query, database, result_url, timezone, delay, next_time, priority, retry_limit, org_name) 385 | } 386 | end 387 | 388 | # @param [String] name 389 | # @param [Hash] params 390 | # @return [nil] 391 | def update_schedule(name, params) 392 | @api.update_schedule(name, params) 393 | nil 394 | end 395 | 396 | # @param [String] name 397 | # @param [Fixnum] from 398 | # @param [Fixnum] to 399 | # @return [Array] 400 | def history(name, from=nil, to=nil) 401 | result = @api.history(name, from, to) 402 | result.map {|scheduled_at,job_id,type,status,query,start_at,end_at,result_url,priority,database| 403 | job_param = [job_id, type, query, status, 404 | nil, nil, # url, debug 405 | start_at, end_at, 406 | nil, # cpu_time 407 | nil, nil, # result_size, result 408 | result_url, 409 | nil, # hive_result_schema 410 | priority, 411 | nil, # retry_limit 412 | nil, # TODO org_name 413 | database] 414 | ScheduledJob.new(self, scheduled_at, *job_param) 415 | } 416 | end 417 | 418 | # @param [String] name 419 | # @param [Fixnum] time UNIX timestamp 420 | # @param [Fixnum] num 421 | # @return [Array] 422 | def run_schedule(name, time, num) 423 | results = @api.run_schedule(name, time, num) 424 | results.map {|job_id,type,scheduled_at| 425 | ScheduledJob.new(self, scheduled_at, job_id, type, nil) 426 | } 427 | end 428 | 429 | # @param [String] db_name 430 | # @param [String] table_name 431 | # @param [String] format 432 | # @param [String, StringIO] stream 433 | # @param [Fixnum] size 434 | # @param [String] unique_id 435 | # @return [Float] 436 | def import(db_name, table_name, format, stream, size, unique_id=nil) 437 | @api.import(db_name, table_name, format, stream, size, unique_id) 438 | end 439 | 440 | # @return [Array] 441 | def results 442 | results = @api.list_result 443 | rs = results.map {|name,url,organizations| 444 | Result.new(self, name, url, organizations) 445 | } 446 | return rs 447 | end 448 | 449 | # @param [String] name 450 | # @param [String] url 451 | # @param [Hash] opts 452 | # @return [true] 453 | def create_result(name, url, opts={}) 454 | @api.create_result(name, url, opts) 455 | end 456 | 457 | # @param [String] name 458 | # @return [true] 459 | def delete_result(name) 460 | @api.delete_result(name) 461 | end 462 | 463 | # @return [Array] 464 | def users 465 | list = @api.list_users 466 | list.map {|name,org,roles,email| 467 | User.new(self, name, org, roles, email) 468 | } 469 | end 470 | 471 | # @param [String] name 472 | # @param [String] org 473 | # @param [String] email 474 | # @param [String] password 475 | # @return [true] 476 | def add_user(name, org, email, password) 477 | @api.add_user(name, org, email, password) 478 | end 479 | 480 | # @param [String] user 481 | # @return [true] 482 | def remove_user(user) 483 | @api.remove_user(user) 484 | end 485 | 486 | # @param [String] user 487 | # @param [String] email 488 | # @return [true] 489 | def change_email(user, email) 490 | @api.change_email(user, email) 491 | end 492 | 493 | # @param [String] user 494 | # @return [Array] 495 | def list_apikeys(user) 496 | @api.list_apikeys(user) 497 | end 498 | 499 | # @param [String] user 500 | # @return [true] 501 | def add_apikey(user) 502 | @api.add_apikey(user) 503 | end 504 | 505 | # @param [String] user 506 | # @param [String] apikey 507 | # @return [true] 508 | def remove_apikey(user, apikey) 509 | @api.remove_apikey(user, apikey) 510 | end 511 | 512 | # @param [String] user 513 | # @param [String] password 514 | # @return [true] 515 | def change_password(user, password) 516 | @api.change_password(user, password) 517 | end 518 | 519 | # @param [String] old_password 520 | # @param [String] password 521 | # @return [true] 522 | def change_my_password(old_password, password) 523 | @api.change_my_password(old_password, password) 524 | end 525 | 526 | # => BulkLoad::Job 527 | def bulk_load_guess(job) 528 | @api.bulk_load_guess(job) 529 | end 530 | 531 | # => BulkLoad::Job 532 | def bulk_load_preview(job) 533 | @api.bulk_load_preview(job) 534 | end 535 | 536 | # => String 537 | def bulk_load_issue(database, table, job) 538 | @api.bulk_load_issue(database, table, job) 539 | end 540 | 541 | # nil -> [BulkLoad] 542 | def bulk_load_list 543 | @api.bulk_load_list 544 | end 545 | 546 | # name: String, database: String, table: String, job: BulkLoad -> BulkLoad 547 | def bulk_load_create(name, database, table, job, opts = {}) 548 | @api.bulk_load_create(name, database, table, job, opts) 549 | end 550 | 551 | # name: String -> BulkLoad 552 | def bulk_load_show(name) 553 | @api.bulk_load_show(name) 554 | end 555 | 556 | # name: String, settings: Hash -> BulkLoad 557 | def bulk_load_update(name, settings) 558 | @api.bulk_load_update(name, settings) 559 | end 560 | 561 | # name: String -> BulkLoad 562 | def bulk_load_delete(name) 563 | @api.bulk_load_delete(name) 564 | end 565 | 566 | # name: String -> [Job] 567 | def bulk_load_history(name) 568 | @api.bulk_load_history(name) 569 | end 570 | 571 | def bulk_load_run(name, scheduled_time = nil) 572 | @api.bulk_load_run(name, scheduled_time) 573 | end 574 | 575 | end 576 | 577 | end # module TreasureData 578 | -------------------------------------------------------------------------------- /lib/td/client/api/account.rb: -------------------------------------------------------------------------------- 1 | class TreasureData::API 2 | module Account 3 | 4 | #### 5 | ## Account API 6 | ## 7 | 8 | # @return [Array] 9 | def show_account 10 | code, body, res = get("/v3/account/show") 11 | if code != "200" 12 | raise_error("Show account failed", res) 13 | end 14 | js = checked_json(body, %w[account]) 15 | a = js["account"] 16 | account_id = a['id'].to_i 17 | plan = a['plan'].to_i 18 | storage_size = a['storage_size'].to_i 19 | guaranteed_cores = a['guaranteed_cores'].to_i 20 | maximum_cores = a['maximum_cores'].to_i 21 | created_at = a['created_at'] 22 | return [account_id, plan, storage_size, guaranteed_cores, maximum_cores, created_at] 23 | end 24 | 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /lib/td/client/api/bulk_import.rb: -------------------------------------------------------------------------------- 1 | class TreasureData::API 2 | module BulkImport 3 | 4 | #### 5 | ## Bulk import API 6 | ## 7 | 8 | # @param [String] name 9 | # @param [String] db 10 | # @param [String] table 11 | # @param [Hash] opts 12 | # @return [nil] 13 | def create_bulk_import(name, db, table, opts={}) 14 | params = opts.dup 15 | code, body, res = post("/v3/bulk_import/create/#{e name}/#{e db}/#{e table}", params) 16 | if code != "200" 17 | raise_error("Create bulk import failed", res) 18 | end 19 | return nil 20 | end 21 | 22 | # @param [String] name 23 | # @param [Hash] opts 24 | # @return [nil] 25 | def delete_bulk_import(name, opts={}) 26 | params = opts.dup 27 | code, body, res = post("/v3/bulk_import/delete/#{e name}", params) 28 | if code != "200" 29 | raise_error("Delete bulk import failed", res) 30 | end 31 | return nil 32 | end 33 | 34 | # @param [String] name 35 | # @return [Hash] 36 | def show_bulk_import(name) 37 | code, body, res = get("/v3/bulk_import/show/#{name}") 38 | if code != "200" 39 | raise_error("Show bulk import failed", res) 40 | end 41 | js = checked_json(body, %w[status]) 42 | return js 43 | end 44 | 45 | # @param [Hash] opts 46 | # @return [Hash] 47 | def list_bulk_imports(opts={}) 48 | params = opts.dup 49 | code, body, res = get("/v3/bulk_import/list", params) 50 | if code != "200" 51 | raise_error("List bulk imports failed", res) 52 | end 53 | js = checked_json(body, %w[bulk_imports]) 54 | return js['bulk_imports'] 55 | end 56 | 57 | # @param [String] name 58 | # @param [Hash] opts 59 | # @return [nil] 60 | def list_bulk_import_parts(name, opts={}) 61 | params = opts.dup 62 | code, body, res = get("/v3/bulk_import/list_parts/#{e name}", params) 63 | if code != "200" 64 | raise_error("List bulk import parts failed", res) 65 | end 66 | js = checked_json(body, %w[parts]) 67 | return js['parts'] 68 | end 69 | 70 | # @param [String] name 71 | # @param [String] part_name 72 | # @param [String, StringIO] stream 73 | # @param [Fixnum] size 74 | # @param [Hash] opts 75 | # @return [nil] 76 | def bulk_import_upload_part(name, part_name, stream, size, opts={}) 77 | code, body, res = put("/v3/bulk_import/upload_part/#{e name}/#{e part_name}", stream, size) 78 | if code[0] != ?2 79 | raise_error("Upload a part failed", res) 80 | end 81 | return nil 82 | end 83 | 84 | # @param [String] name 85 | # @param [String] part_name 86 | # @param [Hash] opts 87 | # @return [nil] 88 | def bulk_import_delete_part(name, part_name, opts={}) 89 | params = opts.dup 90 | code, body, res = post("/v3/bulk_import/delete_part/#{e name}/#{e part_name}", params) 91 | if code[0] != ?2 92 | raise_error("Delete a part failed", res) 93 | end 94 | return nil 95 | end 96 | 97 | # @param [String] name 98 | # @param [Hash] opts 99 | # @return [nil] 100 | def freeze_bulk_import(name, opts={}) 101 | params = opts.dup 102 | code, body, res = post("/v3/bulk_import/freeze/#{e name}", params) 103 | if code != "200" 104 | raise_error("Freeze bulk import failed", res) 105 | end 106 | return nil 107 | end 108 | 109 | # @param [String] name 110 | # @param [Hash] opts 111 | # @return [nil] 112 | def unfreeze_bulk_import(name, opts={}) 113 | params = opts.dup 114 | code, body, res = post("/v3/bulk_import/unfreeze/#{e name}", params) 115 | if code != "200" 116 | raise_error("Unfreeze bulk import failed", res) 117 | end 118 | return nil 119 | end 120 | 121 | # @param [String] name 122 | # @param [Hash] opts 123 | # @return [String] job_id 124 | def perform_bulk_import(name, opts={}) 125 | params = opts.dup 126 | code, body, res = post("/v3/bulk_import/perform/#{e name}", params) 127 | if code != "200" 128 | raise_error("Perform bulk import failed", res) 129 | end 130 | js = checked_json(body, %w[job_id]) 131 | return js['job_id'].to_s 132 | end 133 | 134 | # @param [String] name 135 | # @param [Hash] opts 136 | # @return [nil] 137 | def commit_bulk_import(name, opts={}) 138 | params = opts.dup 139 | code, body, res = post("/v3/bulk_import/commit/#{e name}", params) 140 | if code != "200" 141 | raise_error("Commit bulk import failed", res) 142 | end 143 | return nil 144 | end 145 | 146 | # @param [String] name 147 | # @param [Hash] opts 148 | # @param [Proc] block 149 | # @return [Array] 150 | def bulk_import_error_records(name, opts={}, &block) 151 | params = opts.dup 152 | code, body, res = get("/v3/bulk_import/error_records/#{e name}", params) 153 | if code != "200" 154 | raise_error("Failed to get bulk import error records", res) 155 | end 156 | if body.nil? || body.empty? 157 | if block 158 | return nil 159 | else 160 | return [] 161 | end 162 | end 163 | require File.expand_path('../compat_gzip_reader', File.dirname(__FILE__)) 164 | u = MessagePack::Unpacker.new(Zlib::GzipReader.new(StringIO.new(body))) 165 | if block 166 | begin 167 | u.each(&block) 168 | rescue EOFError 169 | end 170 | nil 171 | else 172 | result = [] 173 | begin 174 | u.each {|row| 175 | result << row 176 | } 177 | rescue EOFError 178 | end 179 | return result 180 | end 181 | end 182 | 183 | end 184 | end 185 | -------------------------------------------------------------------------------- /lib/td/client/api/bulk_load.rb: -------------------------------------------------------------------------------- 1 | class TreasureData::API 2 | module BulkLoad 3 | 4 | #### 5 | ## BulkLoad (Server-side Bulk loader) API 6 | ## 7 | 8 | # 1. POST /v3/bulk_loads/guess - stateless non REST API to return guess result as BulkLoadSession [NEW] 9 | # 2. POST /v3/bulk_loads/preview - stateless non REST API to return preview result as BulkLoadSession [NEW] 10 | # 11 | # 3. POST /v3/job/issue/:type/:database - create a job resource to run BulkLoadSession [EXTENDED] 12 | # 4. POST /v3/job/kill/:id - kill the job [ALREADY EXISTS] 13 | # 5. GET /v3/job/show/:id - get status of the job [ALREADY EXISTS] 14 | # 6. GET /v3/job/result/:id - get result of the job [NOT NEEDED IN Q4] ... because backend feature is not yet implemented 15 | # 16 | # 7. GET /v3/bulk_loads - list BulkLoadSession resources [NEW] 17 | # 8. POST /v3/bulk_loads - create BulkLoadSession [NEW] 18 | # 9. GET /v3/bulk_loads/:name - get BulkLoadSession [NEW] 19 | # 10. PUT /v3/bulk_loads/:name - update BulkLoadSession [NEW] 20 | # 11. DELETE /v3/bulk_loads/:name - delete BulkLoadSession [NEW] 21 | # 12. GET /v3/bulk_loads/:name/jobs - list BulkLoadSession job history [NEW] 22 | # 13. POST /v3/bulk_loads/:name/jobs - run BulkLoadSession [NEW] 23 | 24 | # The 'BulkLoadSession' resource in td-api is as follows; 25 | # { 26 | # "config": { 27 | # "type": "s3", 28 | # "access_key_id": s3 access key id, 29 | # "secret_access_key": s3 secret key, 30 | # "endpoint": s3 endpoint name, 31 | # "bucket": s3 bucket name, 32 | # "path_prefix": "a/prefix/of/files", 33 | # "decoders": [] 34 | # }, 35 | # "name": account_wide_unique_name, 36 | # "cron": cron_string, 37 | # "timezone": timezone_string, 38 | # "delay": delay_seconds, 39 | # "database": database_name, 40 | # "table": table_name 41 | # } 42 | 43 | LIST = '/v3/bulk_loads' 44 | SESSION = LIST + '/%s' 45 | JOB = SESSION + '/jobs' 46 | 47 | # job: Hash -> Hash 48 | def bulk_load_guess(job) 49 | # retry_request = true 50 | path = LIST + '/guess' 51 | res = api { post(path, job.to_json) } 52 | unless res.ok? 53 | raise_error('BulkLoad configuration guess failed', res) 54 | end 55 | JSON.load(res.body) 56 | end 57 | 58 | # job: Hash -> Hash 59 | def bulk_load_preview(job) 60 | # retry_request = true 61 | path = LIST + '/preview' 62 | res = api { post(path, job.to_json) } 63 | unless res.ok? 64 | raise_error('BulkLoad job preview failed', res) 65 | end 66 | JSON.load(res.body) 67 | end 68 | 69 | # job: Hash -> String (job_id) 70 | def bulk_load_issue(database, table, job) 71 | type = 'bulkload' 72 | job = job.dup 73 | job['database'] = database 74 | job['table'] = table 75 | path = "/v3/job/issue/#{e type}/#{e database}" 76 | res = api { post(path, job.to_json) } 77 | unless res.ok? 78 | raise_error('BulkLoad job issuing failed', res) 79 | end 80 | js = checked_json(res.body) 81 | js['job_id'].to_s 82 | end 83 | 84 | # nil -> [Hash] 85 | def bulk_load_list 86 | res = api { get(LIST) } 87 | unless res.ok? 88 | raise_error("BulkLoadSession list retrieve failed", res) 89 | end 90 | JSON.load(res.body) 91 | end 92 | 93 | # name: String, database: String, table: String, job: Hash -> Hash 94 | def bulk_load_create(name, database, table, job, opts = {}) 95 | job = job.dup 96 | job['name'] = name 97 | [:cron, :timezone, :delay, :time_column].each do |prop| 98 | job[prop.to_s] = opts[prop] if opts.key?(prop) 99 | end 100 | job['database'] = database 101 | job['table'] = table 102 | res = api { post(LIST, job.to_json) } 103 | unless res.ok? 104 | raise_error("BulkLoadSession: #{name} create failed", res) 105 | end 106 | JSON.load(res.body) 107 | end 108 | 109 | # name: String -> Hash 110 | def bulk_load_show(name) 111 | path = session_path(name) 112 | res = api { get(path) } 113 | unless res.ok? 114 | raise_error("BulkLoadSession: #{name} retrieve failed", res) 115 | end 116 | JSON.load(res.body) 117 | end 118 | 119 | # name: String, settings: Hash -> Hash 120 | def bulk_load_update(name, settings) 121 | path = session_path(name) 122 | res = api { put(path, settings.to_json) } 123 | unless res.ok? 124 | raise_error("BulkLoadSession: #{name} update failed", res) 125 | end 126 | JSON.load(res.body) 127 | end 128 | 129 | # name: String -> Hash 130 | def bulk_load_delete(name) 131 | path = session_path(name) 132 | res = api { delete(path) } 133 | unless res.ok? 134 | raise_error("BulkLoadSession: #{name} delete failed", res) 135 | end 136 | JSON.load(res.body) 137 | end 138 | 139 | # name: String -> [Hash] 140 | def bulk_load_history(name) 141 | path = job_path(name) 142 | res = api { get(path) } 143 | unless res.ok? 144 | raise_error("history of BulkLoadSession: #{name} retrieve failed", res) 145 | end 146 | JSON.load(res.body) 147 | end 148 | 149 | def bulk_load_run(name, scheduled_time = nil) 150 | path = job_path(name) 151 | opts = {} 152 | opts[:scheduled_time] = scheduled_time.to_s unless scheduled_time.nil? 153 | res = api { post(path, opts.to_json) } 154 | unless res.ok? 155 | raise_error("BulkLoadSession: #{name} job create failed", res) 156 | end 157 | js = checked_json(res.body) 158 | js['job_id'].to_s 159 | end 160 | 161 | private 162 | 163 | def session_path(name) 164 | SESSION % e(name) 165 | end 166 | 167 | def job_path(name) 168 | JOB % e(name) 169 | end 170 | 171 | end 172 | end 173 | -------------------------------------------------------------------------------- /lib/td/client/api/database.rb: -------------------------------------------------------------------------------- 1 | class TreasureData::API 2 | module Database 3 | 4 | #### 5 | ## Database API 6 | ## 7 | 8 | # @return [Array] names as array 9 | def list_databases 10 | code, body, res = get("/v3/database/list") 11 | if code != "200" 12 | raise_error("List databases failed", res) 13 | end 14 | js = checked_json(body, %w[databases]) 15 | result = {} 16 | js["databases"].each {|m| 17 | name = m['name'] 18 | count = m['count'] 19 | created_at = m['created_at'] 20 | updated_at = m['updated_at'] 21 | permission = m['permission'] 22 | result[name] = [count, created_at, updated_at, nil, permission] # set nil to org for API compatibiilty 23 | } 24 | return result 25 | end 26 | 27 | # @param [String] db 28 | # @return [true] 29 | def delete_database(db) 30 | code, body, res = post("/v3/database/delete/#{e db}") 31 | if code != "200" 32 | raise_error("Delete database failed", res) 33 | end 34 | return true 35 | end 36 | 37 | # @param [String] db 38 | # @param [Hash] opts 39 | # @return [true] 40 | def create_database(db, opts={}) 41 | params = opts.dup 42 | code, body, res = post("/v3/database/create/#{e db}", params) 43 | if code != "200" 44 | raise_error("Create database failed", res) 45 | end 46 | return true 47 | end 48 | 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /lib/td/client/api/export.rb: -------------------------------------------------------------------------------- 1 | class TreasureData::API 2 | module Export 3 | 4 | #### 5 | ## Export API 6 | ## 7 | 8 | # => jobId:String 9 | # @param [String] db 10 | # @param [String] table 11 | # @param [String] storage_type 12 | # @param [Hash] opts 13 | # @return [String] job_id 14 | def export(db, table, storage_type, opts={}) 15 | params = opts.dup 16 | params['storage_type'] = storage_type 17 | code, body, res = post("/v3/export/run/#{e db}/#{e table}", params) 18 | if code != "200" 19 | raise_error("Export failed", res) 20 | end 21 | js = checked_json(body, %w[job_id]) 22 | return js['job_id'].to_s 23 | end 24 | 25 | # => jobId:String 26 | # @param [String] target_job_id 27 | # @param [Hash] opts 28 | # @return [String] job_id 29 | def result_export(target_job_id, opts={}) 30 | code, body, res = post("/v3/job/result_export/#{target_job_id}", opts) 31 | if code[0] != ?2 32 | raise_error("Result Export failed", res) 33 | end 34 | js = checked_json(body, %w[job_id]) 35 | return js['job_id'].to_s 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /lib/td/client/api/import.rb: -------------------------------------------------------------------------------- 1 | class TreasureData::API 2 | module Import 3 | 4 | #### 5 | ## Import API 6 | ## 7 | 8 | # @param [String] db 9 | # @param [String] table 10 | # @param [String] format 11 | # @param [String, StringIO] stream 12 | # @param [Fixnum] size 13 | # @param [String] unique_id 14 | # @return [Float] elapsed time 15 | def import(db, table, format, stream, size, unique_id=nil) 16 | if unique_id 17 | path = "/v3/table/import_with_id/#{e db}/#{e table}/#{unique_id}/#{format}" 18 | else 19 | path = "/v3/table/import/#{e db}/#{e table}/#{format}" 20 | end 21 | opts = {} 22 | if @host == DEFAULT_ENDPOINT 23 | opts[:host] = DEFAULT_IMPORT_ENDPOINT 24 | elsif @host == TreasureData::API::OLD_ENDPOINT # backward compatibility 25 | opts[:host] = 'api-import.treasure-data.com' 26 | opts[:ssl] = false 27 | end 28 | code, body, res = put(path, stream, size, opts) 29 | if code[0] != ?2 30 | raise_error("Import failed", res) 31 | end 32 | return true 33 | end 34 | 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/td/client/api/job.rb: -------------------------------------------------------------------------------- 1 | class TreasureData::API 2 | module Job 3 | 4 | #### 5 | ## Job API 6 | ## 7 | 8 | # @param [Fixnum] from 9 | # @param [Fixnum] to (to is inclusive) 10 | # @param [String] status 11 | # @param [Hash] conditions 12 | # @return [Array] 13 | def list_jobs(from=0, to=nil, status=nil, conditions=nil) 14 | params = {} 15 | params['from'] = from.to_s if from 16 | params['to'] = to.to_s if to 17 | params['status'] = status.to_s if status 18 | params.merge!(conditions) if conditions 19 | code, body, res = get("/v3/job/list", params) 20 | if code != "200" 21 | raise_error("List jobs failed", res) 22 | end 23 | js = checked_json(body, %w[jobs]) 24 | result = [] 25 | js['jobs'].each {|m| 26 | job_id = m['job_id'] 27 | type = (m['type'] || '?').to_sym 28 | database = m['database'] 29 | status = m['status'] 30 | query = m['query'] 31 | start_at = m['start_at'] 32 | end_at = m['end_at'] 33 | cpu_time = m['cpu_time'] 34 | result_size = m['result_size'] # compressed result size in msgpack.gz format 35 | result_url = m['result'] 36 | priority = m['priority'] 37 | retry_limit = m['retry_limit'] 38 | duration = m['duration'] 39 | num_records = m['num_records'] 40 | result << [job_id, type, status, query, start_at, end_at, cpu_time, 41 | result_size, result_url, priority, retry_limit, nil, database, 42 | duration, num_records] 43 | } 44 | return result 45 | end 46 | 47 | # @param [String] job_id 48 | # @return [Array] 49 | def show_job(job_id) 50 | # use v3/job/status instead of v3/job/show to poll finish of a job 51 | code, body, res = get("/v3/job/show/#{e job_id}") 52 | if code != "200" 53 | raise_error("Show job failed", res) 54 | end 55 | js = checked_json(body, %w[status]) 56 | # TODO debug 57 | type = (js['type'] || '?').to_sym # TODO 58 | database = js['database'] 59 | query = js['query'] 60 | status = js['status'] 61 | debug = js['debug'] 62 | url = js['url'] 63 | start_at = js['start_at'] 64 | end_at = js['end_at'] 65 | cpu_time = js['cpu_time'] 66 | result_size = js['result_size'] # compressed result size in msgpack.gz format 67 | num_records = js['num_records'] 68 | duration = js['duration'] 69 | result = js['result'] # result target URL 70 | linked_result_export_job_id = js['linked_result_export_job_id'] 71 | result_export_target_job_id = js['result_export_target_job_id'] 72 | hive_result_schema = (js['hive_result_schema'] || '') 73 | if hive_result_schema.empty? 74 | hive_result_schema = nil 75 | else 76 | begin 77 | hive_result_schema = JSON.parse(hive_result_schema) 78 | rescue JSON::ParserError => e 79 | # this is a workaround for a Known Limitation in the Pig Engine which does not set a default, auto-generated 80 | # column name for anonymous columns (such as the ones that are generated from UDF like COUNT or SUM). 81 | # The schema will contain 'nil' for the name of those columns and that breaks the JSON parser since it violates 82 | # the JSON syntax standard. 83 | if type == :pig and hive_result_schema !~ /[\{\}]/ 84 | begin 85 | # NOTE: this works because a JSON 2 dimensional array is the same as a Ruby one. 86 | # Any change in the format for the hive_result_schema output may cause a syntax error, in which case 87 | # this lame attempt at fixing the problem will fail and we will be raising the original JSON exception 88 | hive_result_schema = eval(hive_result_schema) 89 | rescue SyntaxError => ignored_e 90 | raise e 91 | end 92 | hive_result_schema.each_with_index {|col_schema, idx| 93 | if col_schema[0].nil? 94 | col_schema[0] = "_col#{idx}" 95 | end 96 | } 97 | else 98 | raise e 99 | end 100 | end 101 | end 102 | priority = js['priority'] 103 | retry_limit = js['retry_limit'] 104 | return [type, query, status, url, debug, start_at, end_at, cpu_time, 105 | result_size, result, hive_result_schema, priority, retry_limit, nil, database, duration, num_records, 106 | linked_result_export_job_id, result_export_target_job_id] 107 | end 108 | 109 | # @param [String] job_id 110 | # @return [String] HTTP status 111 | def job_status(job_id) 112 | code, body, res = get("/v3/job/status/#{e job_id}") 113 | if code != "200" 114 | raise_error("Get job status failed", res) 115 | end 116 | 117 | js = checked_json(body, %w[status]) 118 | return js['status'] 119 | end 120 | 121 | # @param [String] job_id 122 | # @return [Array] 123 | def job_result(job_id) 124 | result = [] 125 | unpacker = MessagePack::Unpacker.new 126 | job_result_download(job_id) do |chunk| 127 | unpacker.feed_each(chunk) do |row| 128 | result << row 129 | end unless chunk.empty? 130 | end 131 | return result 132 | end 133 | 134 | # block is optional and must accept 1 parameter 135 | # 136 | # @param [String] job_id 137 | # @param [String] format 138 | # @param [IO] io 139 | # @param [Proc] block 140 | # @return [nil, String] 141 | def job_result_format(job_id, format, io=nil) 142 | if io 143 | job_result_download(job_id, format) do |chunk, total| 144 | io.write chunk 145 | yield total if block_given? 146 | end 147 | nil 148 | else 149 | body = String.new 150 | job_result_download(job_id, format) do |chunk| 151 | body << chunk 152 | end 153 | body 154 | end 155 | end 156 | 157 | # block is optional and must accept 1 argument 158 | # 159 | # @param [String] job_id 160 | # @param [Proc] block 161 | # @return [nil] 162 | def job_result_each(job_id, &block) 163 | upkr = MessagePack::Unpacker.new 164 | # default to decompressing the response since format is fixed to 'msgpack' 165 | job_result_download(job_id) do |chunk| 166 | upkr.feed_each(chunk, &block) unless chunk.empty? 167 | end 168 | nil 169 | end 170 | 171 | # block is optional and must accept 1 argument 172 | # 173 | # @param [String] job_id 174 | # @param [Proc] block 175 | # @return [nil] 176 | def job_result_each_with_compr_size(job_id) 177 | upkr = MessagePack::Unpacker.new 178 | # default to decompressing the response since format is fixed to 'msgpack' 179 | job_result_download(job_id) do |chunk, total| 180 | upkr.feed_each(chunk) do |unpacked| 181 | yield unpacked, total if block_given? 182 | end unless chunk.empty? 183 | end 184 | nil 185 | end 186 | 187 | # @param [String] job_id 188 | # @param [String] format 189 | # @return [String] 190 | def job_result_raw(job_id, format, io = nil) 191 | body = io ? nil : String.new 192 | job_result_download(job_id, format, false) do |chunk, total| 193 | if io 194 | io.write(chunk) 195 | yield total if block_given? 196 | else 197 | body << chunk 198 | end 199 | end 200 | body 201 | end 202 | 203 | # @param [String] job_id 204 | # @return [String] 205 | def kill(job_id) 206 | code, body, res = post("/v3/job/kill/#{e job_id}") 207 | if code != "200" 208 | raise_error("Kill job failed", res) 209 | end 210 | js = checked_json(body, %w[]) 211 | former_status = js['former_status'] 212 | return former_status 213 | end 214 | 215 | # @param [String] q 216 | # @param [String] db 217 | # @param [String] result_url 218 | # @param [Fixnum] priority 219 | # @param [Hash] opts 220 | # @return [String] job_id 221 | def hive_query(q, db=nil, result_url=nil, priority=nil, retry_limit=nil, opts={}) 222 | query(q, :hive, db, result_url, priority, retry_limit, opts) 223 | end 224 | 225 | # @param [String] q 226 | # @param [String] db 227 | # @param [String] result_url 228 | # @param [Fixnum] priority 229 | # @param [Hash] opts 230 | # @return [String] job_id 231 | def pig_query(q, db=nil, result_url=nil, priority=nil, retry_limit=nil, opts={}) 232 | query(q, :pig, db, result_url, priority, retry_limit, opts) 233 | end 234 | 235 | # @param [String] q 236 | # @param [Symbol] type 237 | # @param [String] db 238 | # @param [String] result_url 239 | # @param [Fixnum] priority 240 | # @param [Hash] opts 241 | # @return [String] job_id 242 | def query(q, type=:hive, db=nil, result_url=nil, priority=nil, retry_limit=nil, opts={}) 243 | params = {'query' => q}.merge(opts) 244 | params['result'] = result_url if result_url 245 | params['priority'] = priority if priority 246 | params['retry_limit'] = retry_limit if retry_limit 247 | code, body, res = post("/v3/job/issue/#{type}/#{e db}", params) 248 | if code != "200" 249 | raise_error("Query failed", res) 250 | end 251 | js = checked_json(body, %w[job_id]) 252 | return js['job_id'].to_s 253 | end 254 | 255 | private 256 | 257 | def validate_content_length_with_range(response, current_total_chunk_size) 258 | if expected_size = response.header['Content-Range'][0] 259 | expected_size = expected_size[/\d+$/].to_i 260 | elsif expected_size = response.header['Content-Length'][0] 261 | expected_size = expected_size.to_i 262 | end 263 | 264 | if expected_size.nil? 265 | elsif current_total_chunk_size < expected_size 266 | # too small 267 | # NOTE: 268 | # ext/openssl raises EOFError in case where underlying connection 269 | # causes an error, but httpclient ignores it. 270 | # https://github.com/nahi/httpclient/blob/v3.2.8/lib/httpclient/session.rb#L1003 271 | raise EOFError, 'httpclient IncompleteError' 272 | elsif current_total_chunk_size > expected_size 273 | # too large 274 | raise_error("Get job result failed", response) 275 | end 276 | end 277 | 278 | def validate_response_status(res, current_total_chunk_size=0) 279 | case res.status 280 | when 200 281 | if current_total_chunk_size != 0 282 | # try to resume but the server returns 200 283 | raise_error("Get job result failed", res) 284 | end 285 | when 206 # resuming 286 | else 287 | if res.status/100 == 5 288 | raise HTTPServerException 289 | end 290 | raise_error("Get job result failed", res) 291 | end 292 | end 293 | 294 | class HTTPServerException < StandardError 295 | end 296 | 297 | def job_result_download(job_id, format='msgpack', autodecode=true) 298 | client, header = new_client 299 | client.send_timeout = @send_timeout 300 | client.receive_timeout = @read_timeout 301 | header['Accept-Encoding'] = 'deflate, gzip' 302 | 303 | url = build_endpoint("/v3/job/result/#{e job_id}", @host) 304 | params = {'format' => format} 305 | 306 | unless ENV['TD_CLIENT_DEBUG'].nil? 307 | puts "DEBUG: REST GET call:" 308 | puts "DEBUG: header: " + header.to_s 309 | puts "DEBUG: url: " + url.to_s 310 | puts "DEBUG: params: " + params.to_s 311 | end 312 | 313 | # up to 7 retries with exponential (base 2) back-off starting at 'retry_delay' 314 | retry_delay = @retry_delay 315 | cumul_retry_delay = 0 316 | current_total_chunk_size = 0 317 | infl = nil 318 | begin # LOOP of Network/Server errors 319 | first_chunk_p = true 320 | response = client.get(url, params, header) do |res, chunk| 321 | # Validate only on first chunk 322 | validate_response_status(res, current_total_chunk_size) if first_chunk_p 323 | first_chunk_p = false 324 | 325 | if infl.nil? && autodecode 326 | case res.header['Content-Encoding'][0].to_s.downcase 327 | when 'gzip' 328 | infl = Zlib::Inflate.new(Zlib::MAX_WBITS + 16) 329 | when 'deflate' 330 | infl = Zlib::Inflate.new 331 | end 332 | end 333 | current_total_chunk_size += chunk.bytesize 334 | chunk = infl.inflate(chunk) if infl 335 | yield chunk, current_total_chunk_size 336 | end 337 | 338 | # for the case response body is empty 339 | # Note that webmock returns response.body as "" instead of nil 340 | validate_response_status(response, 0) if response.body.to_s.empty? 341 | 342 | # completed? 343 | validate_content_length_with_range(response, current_total_chunk_size) 344 | rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, 345 | SystemCallError, OpenSSL::SSL::SSLError, SocketError, HTTPClient::TimeoutError, 346 | HTTPServerException => e 347 | if current_total_chunk_size > 0 348 | if etag = response.header['ETag'][0] 349 | header['If-Range'] = etag 350 | header['Range'] = "bytes=#{current_total_chunk_size}-" 351 | end 352 | end 353 | 354 | $stderr.print "#{e.class}: #{e.message}. " 355 | if cumul_retry_delay < @max_cumul_retry_delay 356 | $stderr.puts "Retrying after #{retry_delay} seconds..." 357 | sleep retry_delay 358 | cumul_retry_delay += retry_delay 359 | retry_delay *= 2 360 | retry 361 | end 362 | raise 363 | end 364 | 365 | unless ENV['TD_CLIENT_DEBUG'].nil? 366 | puts "DEBUG: REST GET response:" 367 | puts "DEBUG: header: " + response.header.to_s 368 | puts "DEBUG: status: " + response.code.to_s 369 | puts "DEBUG: body: " + response.body.to_s 370 | end 371 | 372 | nil 373 | ensure 374 | infl.close if infl 375 | end 376 | 377 | class NullInflate 378 | def inflate(chunk) 379 | chunk 380 | end 381 | 382 | def close 383 | end 384 | end 385 | 386 | def create_inflalte_or_null_inflate(response) 387 | if response.header['Content-Encoding'].empty? 388 | NullInflate.new 389 | else 390 | create_inflate(response) 391 | end 392 | end 393 | 394 | def create_inflate(response) 395 | if response.header['Content-Encoding'].include?('gzip') 396 | Zlib::Inflate.new(Zlib::MAX_WBITS + 16) 397 | else 398 | Zlib::Inflate.new 399 | end 400 | end 401 | end 402 | end 403 | -------------------------------------------------------------------------------- /lib/td/client/api/result.rb: -------------------------------------------------------------------------------- 1 | class TreasureData::API 2 | module Result 3 | 4 | #### 5 | ## Result API 6 | ## 7 | 8 | # @return [Array] 9 | def list_result 10 | code, body, res = get("/v3/result/list") 11 | if code != "200" 12 | raise_error("List result table failed", res) 13 | end 14 | js = checked_json(body, %w[results]) 15 | result = [] 16 | js['results'].map {|m| 17 | result << [m['name'], m['url'], nil] # same as database 18 | } 19 | return result 20 | end 21 | 22 | # @param [String] name 23 | # @param [String] url 24 | # @param [Hash] opts 25 | # @return [true] 26 | def create_result(name, url, opts={}) 27 | params = {'url'=>url}.merge(opts) 28 | code, body, res = post("/v3/result/create/#{e name}", params) 29 | if code != "200" 30 | raise_error("Create result table failed", res) 31 | end 32 | return true 33 | end 34 | 35 | # @param [String] name 36 | # @return [true] 37 | def delete_result(name) 38 | code, body, res = post("/v3/result/delete/#{e name}") 39 | if code != "200" 40 | raise_error("Delete result table failed", res) 41 | end 42 | return true 43 | end 44 | 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /lib/td/client/api/schedule.rb: -------------------------------------------------------------------------------- 1 | class TreasureData::API 2 | module Schedule 3 | 4 | #### 5 | ## Schedule API 6 | ## 7 | 8 | # @param [String] name 9 | # @param [Hash] opts 10 | # @return [String] 11 | def create_schedule(name, opts) 12 | params = opts.update({:type=> opts[:type] || opts['type'] || 'hive'}) 13 | code, body, res = post("/v3/schedule/create/#{e name}", params) 14 | if code != "200" 15 | raise_error("Create schedule failed", res) 16 | end 17 | js = checked_json(body) 18 | return js['start'] 19 | end 20 | 21 | # @param [String] name 22 | # @return [Array] 23 | def delete_schedule(name) 24 | code, body, res = post("/v3/schedule/delete/#{e name}") 25 | if code != "200" 26 | raise_error("Delete schedule failed", res) 27 | end 28 | js = checked_json(body, %w[]) 29 | return js['cron'], js["query"] 30 | end 31 | 32 | # @return [Array] 33 | def list_schedules 34 | code, body, res = get("/v3/schedule/list") 35 | if code != "200" 36 | raise_error("List schedules failed", res) 37 | end 38 | js = checked_json(body, %w[schedules]) 39 | result = [] 40 | js['schedules'].each {|m| 41 | name = m['name'] 42 | cron = m['cron'] 43 | query = m['query'] 44 | database = m['database'] 45 | result_url = m['result'] 46 | timezone = m['timezone'] 47 | delay = m['delay'] 48 | next_time = m['next_time'] 49 | priority = m['priority'] 50 | retry_limit = m['retry_limit'] 51 | result << [name, cron, query, database, result_url, timezone, delay, next_time, priority, retry_limit, nil] # same as database 52 | } 53 | return result 54 | end 55 | 56 | # @param [String] name 57 | # @param [Hash] params 58 | # @return [nil] 59 | def update_schedule(name, params) 60 | code, body, res = post("/v3/schedule/update/#{e name}", params) 61 | if code != "200" 62 | raise_error("Update schedule failed", res) 63 | end 64 | return nil 65 | end 66 | 67 | # @param [String] name 68 | # @param [Fixnum] from 69 | # @param [Fixnum] to (to is exclusive) 70 | # @return [Array] 71 | def history(name, from=0, to=nil) 72 | params = {} 73 | params['from'] = from.to_s if from 74 | params['to'] = to.to_s if to 75 | code, body, res = get("/v3/schedule/history/#{e name}", params) 76 | if code != "200" 77 | raise_error("List history failed", res) 78 | end 79 | js = checked_json(body, %w[history]) 80 | result = [] 81 | js['history'].each {|m| 82 | job_id = m['job_id'] 83 | type = (m['type'] || '?').to_sym 84 | database = m['database'] 85 | status = m['status'] 86 | query = m['query'] 87 | start_at = m['start_at'] 88 | end_at = m['end_at'] 89 | scheduled_at = m['scheduled_at'] 90 | result_url = m['result'] 91 | priority = m['priority'] 92 | result << [scheduled_at, job_id, type, status, query, start_at, end_at, result_url, priority, database] 93 | } 94 | return result 95 | end 96 | 97 | # @param [String] name 98 | # @param [String] time 99 | # @param [Fixnum] num 100 | # @return [Array] 101 | def run_schedule(name, time, num) 102 | params = {} 103 | params = {'num' => num} if num 104 | code, body, res = post("/v3/schedule/run/#{e name}/#{e time}", params) 105 | if code != "200" 106 | raise_error("Run schedule failed", res) 107 | end 108 | js = checked_json(body, %w[jobs]) 109 | result = [] 110 | js['jobs'].each {|m| 111 | job_id = m['job_id'] 112 | scheduled_at = m['scheduled_at'] 113 | type = (m['type'] || '?').to_sym 114 | result << [job_id, type, scheduled_at] 115 | } 116 | return result 117 | end 118 | 119 | end 120 | end 121 | -------------------------------------------------------------------------------- /lib/td/client/api/server_status.rb: -------------------------------------------------------------------------------- 1 | class TreasureData::API 2 | module ServerStatus 3 | 4 | #### 5 | ## Server Status API 6 | ## 7 | 8 | # => status:String 9 | # @return [String] HTTP status code 10 | def server_status 11 | code, body, res = get('/v3/system/server_status') 12 | if code != "200" 13 | return "Server is down (#{code})" 14 | end 15 | js = checked_json(body, %w[status]) 16 | status = js['status'] 17 | return status 18 | end 19 | 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/td/client/api/table.rb: -------------------------------------------------------------------------------- 1 | class TreasureData::API 2 | module Table 3 | 4 | #### 5 | ## Table API 6 | ## 7 | 8 | # @param [String] db 9 | # @return [Array] 10 | def list_tables(db) 11 | code, body, res = get("/v3/table/list/#{e db}") 12 | if code != "200" 13 | raise_error("List tables failed", res) 14 | end 15 | js = checked_json(body, %w[tables]) 16 | result = {} 17 | js["tables"].map {|m| 18 | name = m['name'] 19 | type = (m['type'] || '?').to_sym 20 | count = (m['count'] || 0).to_i # TODO? 21 | created_at = m['created_at'] 22 | updated_at = m['updated_at'] 23 | last_import = m['counter_updated_at'] 24 | last_log_timestamp = m['last_log_timestamp'] 25 | estimated_storage_size = m['estimated_storage_size'].to_i 26 | schema = JSON.parse(m['schema'] || '[]') 27 | expire_days = m['expire_days'] 28 | include_v = m['include_v'] 29 | result[name] = [type, schema, count, created_at, updated_at, estimated_storage_size, last_import, last_log_timestamp, expire_days, include_v] 30 | } 31 | return result 32 | end 33 | 34 | # @param [String] db 35 | # @param [String] table 36 | # @param [Hash] params 37 | # @return [true] 38 | def create_log_table(db, table, params={}) 39 | create_table(db, table, :log, params) 40 | end 41 | 42 | # @param [String] db 43 | # @param [String] table 44 | # @param [String] type 45 | # @param [Hash] params 46 | # @return [true] 47 | def create_table(db, table, type, params={}) 48 | schema = schema.to_s 49 | code, body, res = post("/v3/table/create/#{e db}/#{e table}/#{type}", params) 50 | if code != "200" 51 | raise_error("Create #{type} table failed", res) 52 | end 53 | return true 54 | end 55 | private :create_table 56 | 57 | # @param [String] db 58 | # @param [String] table1 59 | # @param [String] table2 60 | # @return [true] 61 | def swap_table(db, table1, table2) 62 | code, body, res = post("/v3/table/swap/#{e db}/#{e table1}/#{e table2}") 63 | if code != "200" 64 | raise_error("Swap tables failed", res) 65 | end 66 | return true 67 | end 68 | 69 | # @param [String] db 70 | # @param [String] table 71 | # @param [String] schema_json 72 | # @return [true] 73 | def update_schema(db, table, schema_json) 74 | code, body, res = post("/v3/table/update-schema/#{e db}/#{e table}", {'schema'=>schema_json}) 75 | if code != "200" 76 | raise_error("Create schema table failed", res) 77 | end 78 | return true 79 | end 80 | 81 | # @param [String] db 82 | # @param [String] table 83 | # @param [Fixnum] expire_days 84 | # @return [true] 85 | def update_expire(db, table, expire_days) 86 | update_table(db, table, {:expire_days=>expire_days}) 87 | end 88 | 89 | # @param [String] db 90 | # @param [String] table 91 | # @option params [Fixnum] :expire_days days to expire table 92 | # @option params [Boolean] :include_v (true) include v column on Hive 93 | # @option params [Boolean] :detect_schema (true) detect schema on import 94 | # @return [true] 95 | def update_table(db, table, params={}) 96 | code, body, res = post("/v3/table/update/#{e db}/#{e table}", params) 97 | if code != "200" 98 | raise_error("Update table failed", res) 99 | end 100 | return true 101 | end 102 | 103 | # @param [String] db 104 | # @param [String] table 105 | # @return [Symbol] 106 | def delete_table(db, table) 107 | code, body, res = post("/v3/table/delete/#{e db}/#{e table}") 108 | if code != "200" 109 | raise_error("Delete table failed", res) 110 | end 111 | js = checked_json(body, %w[]) 112 | type = (js['type'] || '?').to_sym 113 | return type 114 | end 115 | 116 | # @param [String] db 117 | # @param [String] table 118 | # @param [Fixnum] count 119 | # @param [Proc] block 120 | # @return [Array, nil] 121 | def tail(db, table, count, to = nil, from = nil, &block) 122 | unless to.nil? and from.nil? 123 | warn('parameter "to" and "from" no longer work') 124 | end 125 | params = {'format' => 'msgpack'} 126 | params['count'] = count.to_s if count 127 | code, body, res = get("/v3/table/tail/#{e db}/#{e table}", params) 128 | if code != "200" 129 | raise_error("Tail table failed", res) 130 | end 131 | if block 132 | MessagePack::Unpacker.new.feed_each(body, &block) 133 | nil 134 | else 135 | result = [] 136 | MessagePack::Unpacker.new.feed_each(body) {|row| 137 | result << row 138 | } 139 | return result 140 | end 141 | end 142 | 143 | def change_database(db, table, dest_db) 144 | params = { 'dest_database_name' => dest_db } 145 | code, body, res = post("/v3/table/change_database/#{e db}/#{e table}", params) 146 | if code != "200" 147 | raise_error("Change database failed", res) 148 | end 149 | return true 150 | end 151 | 152 | end 153 | end 154 | -------------------------------------------------------------------------------- /lib/td/client/api/user.rb: -------------------------------------------------------------------------------- 1 | class TreasureData::API 2 | module User 3 | 4 | #### 5 | ## User API 6 | ## 7 | 8 | # @param [String] user 9 | # @param [String] password 10 | # @return [String] API key 11 | def authenticate(user, password) 12 | code, body, res = post("/v3/user/authenticate", {'user'=>user, 'password'=>password}) 13 | if code != "200" 14 | if code == "400" 15 | raise_error("Authentication failed", res, AuthError) 16 | else 17 | raise_error("Authentication failed", res) 18 | end 19 | end 20 | js = checked_json(body, %w[apikey]) 21 | apikey = js['apikey'] 22 | return apikey 23 | end 24 | 25 | # @return [Array] 26 | def list_users 27 | code, body, res = get("/v3/user/list") 28 | if code != "200" 29 | raise_error("List users failed", res) 30 | end 31 | js = checked_json(body, %w[users]) 32 | result = js["users"].map {|roleinfo| 33 | name = roleinfo['name'] 34 | email = roleinfo['email'] 35 | [name, nil, nil, email] # set nil to org and role for API compatibility 36 | } 37 | return result 38 | end 39 | 40 | # @param [String] name 41 | # @param [String] org 42 | # @param [String] email 43 | # @param [String] password 44 | # @return [true] 45 | def add_user(name, org, email, password) 46 | params = {'organization'=>org, :email=>email, :password=>password} 47 | code, body, res = post("/v3/user/add/#{e name}", params) 48 | if code != "200" 49 | raise_error("Adding user failed", res) 50 | end 51 | return true 52 | end 53 | 54 | # @param [String] user 55 | # @return [true] 56 | def remove_user(user) 57 | code, body, res = post("/v3/user/remove/#{e user}") 58 | if code != "200" 59 | raise_error("Removing user failed", res) 60 | end 61 | return true 62 | end 63 | 64 | # @param [String] user 65 | # @param [String] email 66 | # @return [true] 67 | def change_email(user, email) 68 | params = {'email' => email} 69 | code, body, res = post("/v3/user/email/change/#{e user}", params) 70 | if code != "200" 71 | raise_error("Changing email failed", res) 72 | end 73 | return true 74 | end 75 | 76 | # @param [String] user 77 | # @return [Array] API keys as array 78 | def list_apikeys(user) 79 | code, body, res = get("/v3/user/apikey/list/#{e user}") 80 | if code != "200" 81 | raise_error("List API keys failed", res) 82 | end 83 | js = checked_json(body, %w[apikeys]) 84 | return js['apikeys'] 85 | end 86 | 87 | # @param [String] user 88 | # @return [true] 89 | def add_apikey(user) 90 | code, body, res = post("/v3/user/apikey/add/#{e user}") 91 | if code != "200" 92 | raise_error("Adding API key failed", res) 93 | end 94 | return true 95 | end 96 | 97 | # @param [String] user 98 | # @param [String] apikey 99 | # @return [true] 100 | def remove_apikey(user, apikey) 101 | params = {'apikey' => apikey} 102 | code, body, res = post("/v3/user/apikey/remove/#{e user}", params) 103 | if code != "200" 104 | raise_error("Removing API key failed", res) 105 | end 106 | return true 107 | end 108 | 109 | # @param [String] user 110 | # @param [String] password 111 | # @return [true] 112 | def change_password(user, password) 113 | params = {'password' => password} 114 | code, body, res = post("/v3/user/password/change/#{e user}", params) 115 | if code != "200" 116 | raise_error("Changing password failed", res) 117 | end 118 | return true 119 | end 120 | 121 | # @param [String] old_password 122 | # @param [String] password 123 | # @return [true] 124 | def change_my_password(old_password, password) 125 | params = {'old_password' => old_password, 'password' => password} 126 | code, body, res = post("/v3/user/password/change", params) 127 | if code != "200" 128 | raise_error("Changing password failed", res) 129 | end 130 | return true 131 | end 132 | 133 | end 134 | end 135 | -------------------------------------------------------------------------------- /lib/td/client/api_error.rb: -------------------------------------------------------------------------------- 1 | module TreasureData 2 | 3 | class ParameterValidationError < StandardError 4 | end 5 | 6 | # Generic API error 7 | class APIError < StandardError 8 | attr_reader :api_backtrace 9 | 10 | def initialize(error_message = nil, api_backtrace = nil) 11 | super(error_message) 12 | @api_backtrace = api_backtrace == '' ? nil : api_backtrace 13 | end 14 | end 15 | 16 | # 4xx Client Errors 17 | class ClientError < APIError 18 | end 19 | 20 | # 400 Bad Request 21 | class BadRequestError < ClientError 22 | end 23 | 24 | # 401 Unauthorized 25 | class AuthError < ClientError 26 | end 27 | 28 | # 403 Forbidden, used for database permissions 29 | class ForbiddenError < ClientError 30 | end 31 | 32 | # 404 Not Found 33 | class NotFoundError < ClientError 34 | end 35 | 36 | # 405 Method Not Allowed 37 | class MethodNotAllowedError < ClientError 38 | end 39 | 40 | # 409 Conflict 41 | class AlreadyExistsError < ClientError 42 | attr_reader :conflicts_with 43 | def initialize(error_message = nil, api_backtrace = nil, conflicts_with=nil) 44 | super(error_message, api_backtrace) 45 | @conflicts_with = conflicts_with 46 | end 47 | end 48 | 49 | # 415 Unsupported Media Type 50 | class UnsupportedMediaTypeError < ClientError 51 | end 52 | 53 | # 422 Unprocessable Entity 54 | class UnprocessableEntityError < ClientError 55 | end 56 | 57 | # 429 Too Many Requests 58 | class TooManyRequestsError < ClientError 59 | end 60 | 61 | end 62 | -------------------------------------------------------------------------------- /lib/td/client/compat_gzip_reader.rb: -------------------------------------------------------------------------------- 1 | 2 | methods = Zlib::GzipReader.public_instance_methods 3 | if !methods.include?(:readpartial) && !methods.include?('readpartial') 4 | class Zlib::GzipReader 5 | # @param [Fixnum] size 6 | # @param [IO] out 7 | # @return [String] 8 | def readpartial(size, out=nil) 9 | o = read(size) 10 | if o 11 | if out 12 | out.replace(o) 13 | return out 14 | else 15 | return o 16 | end 17 | end 18 | raise EOFError, "end of file reached" 19 | end 20 | end 21 | end 22 | 23 | -------------------------------------------------------------------------------- /lib/td/client/version.rb: -------------------------------------------------------------------------------- 1 | module TreasureData 2 | class Client 3 | VERSION = '3.0.0' 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /lib/td/core_ext/openssl/ssl/sslcontext/set_params.rb: -------------------------------------------------------------------------------- 1 | require 'openssl' 2 | module OpenSSL 3 | module SSL 4 | class SSLContext 5 | 6 | # For disabling SSLv3 connection in favor of POODLE Attack protection 7 | # 8 | # Allow 'options' customize through Thread local storage since 9 | # Net::HTTP does not support 'options' configuration. 10 | # 11 | alias original_set_params set_params 12 | def set_params(params={}) 13 | original_set_params(params) 14 | self.options |= OP_NO_SSLv3 if Thread.current[:SET_SSL_OP_NO_SSLv3] 15 | end 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | 3 | # XXX skip coverage setting if run appveyor. Because, fail to push coveralls in appveyor. 4 | unless ENV['APPVEYOR'] 5 | begin 6 | if defined?(:RUBY_ENGINE) && RUBY_ENGINE == 'ruby' 7 | # SimpleCov officially supports MRI 1.9+ only for now 8 | # https://github.com/colszowka/simplecov#ruby-version-compatibility 9 | 10 | require 'simplecov' 11 | require 'coveralls' 12 | 13 | SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new([ 14 | SimpleCov::Formatter::HTMLFormatter, 15 | Coveralls::SimpleCov::Formatter 16 | ]) 17 | SimpleCov.start("test_frameworks") 18 | end 19 | rescue NameError 20 | # skip measuring coverage at Ruby 1.8 21 | end 22 | end 23 | 24 | require 'rspec' 25 | require 'webmock/rspec' 26 | WebMock.disable_net_connect!(:allow_localhost => true) 27 | 28 | include WebMock::API 29 | 30 | $LOAD_PATH << File.dirname(__FILE__)+"../lib" 31 | require 'td-client' 32 | require 'msgpack' 33 | require 'json' 34 | 35 | include TreasureData 36 | 37 | RSpec.configure do |config| 38 | # This allows you to limit a spec run to individual examples or groups 39 | # you care about by tagging them with `:focus` metadata. When nothing 40 | # is tagged with `:focus`, all examples get run. RSpec also provides 41 | # aliases for `it`, `describe`, and `context` that include `:focus` 42 | # metadata: `fit`, `fdescribe` and `fcontext`, respectively. 43 | config.filter_run_when_matching :focus 44 | end 45 | 46 | shared_context 'common helper' do 47 | let :account_id do 48 | 1 49 | end 50 | 51 | let :headers do 52 | {'Accept' => '*/*', 'Accept-Encoding' => /gzip/, 'Date' => /.*/, 'User-Agent' => /Ruby/} 53 | end 54 | 55 | def stub_api_request(method, path, opts = nil) 56 | scheme = 'https' 57 | with_opts = {:headers => headers} 58 | if opts 59 | scheme = 'http' if opts[:ssl] == false 60 | with_opts[:query] = opts[:query] if opts[:query] 61 | end 62 | stub_request(method, "#{scheme}://api.treasuredata.com#{path}").with(with_opts) 63 | end 64 | 65 | def e(s) 66 | s.to_s.gsub(/[^*\-0-9A-Z_a-z]/){|x|'%%%02X' % x.ord} 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /spec/td/client/account_api_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/client/spec_resources' 3 | require 'json' 4 | 5 | describe 'Account API' do 6 | include_context 'spec symbols' 7 | include_context 'common helper' 8 | 9 | let :api do 10 | API.new(nil) 11 | end 12 | 13 | describe 'show_account' do 14 | it 'returns account properties' do 15 | stub_api_request(:get, "/v3/account/show"). 16 | to_return(:body => {'account' => {'id' => 1, 'plan' => 0, 'storage_size' => 2, 'guaranteed_cores' => 3, 'maximum_cores' => 4, 'created_at' => '2014-12-14T17:24:00+0900'}}.to_json) 17 | expect(api.show_account).to eq([1, 0, 2, 3, 4, "2014-12-14T17:24:00+0900"]) 18 | end 19 | end 20 | 21 | end 22 | -------------------------------------------------------------------------------- /spec/td/client/api_error_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe APIError do 4 | let (:message){ 'message' } 5 | let (:api_backtrace){ double('api_backtrace') } 6 | describe 'new' do 7 | context '' do 8 | it do 9 | exc = APIError.new(message, api_backtrace) 10 | expect(exc).to be_an(APIError) 11 | expect(exc.message).to eq message 12 | expect(exc.api_backtrace).to eq api_backtrace 13 | end 14 | end 15 | context 'api_backtrace is ""' do 16 | let (:api_backtrace){ '' } 17 | it do 18 | exc = APIError.new(message, api_backtrace) 19 | expect(exc).to be_an(APIError) 20 | expect(exc.message).to eq message 21 | expect(exc.api_backtrace).to be_nil 22 | end 23 | end 24 | context 'api_backtrace is nil' do 25 | let (:api_backtrace){ nil } 26 | it do 27 | exc = APIError.new(message, api_backtrace) 28 | expect(exc).to be_an(APIError) 29 | expect(exc.message).to eq message 30 | expect(exc.api_backtrace).to be_nil 31 | end 32 | end 33 | end 34 | end 35 | 36 | describe AlreadyExistsError do 37 | let (:message){ 'message' } 38 | let (:api_backtrace){ double('api_backtrace') } 39 | let (:conflicts_with){ '12345' } 40 | describe 'new' do 41 | context '' do 42 | it do 43 | exc = AlreadyExistsError.new(message, api_backtrace) 44 | expect(exc).to be_an(AlreadyExistsError) 45 | expect(exc.message).to eq message 46 | expect(exc.api_backtrace).to eq api_backtrace 47 | end 48 | end 49 | context 'api_backtrace is ""' do 50 | let (:api_backtrace){ '' } 51 | it do 52 | exc = AlreadyExistsError.new(message, api_backtrace) 53 | expect(exc).to be_an(AlreadyExistsError) 54 | expect(exc.message).to eq message 55 | expect(exc.api_backtrace).to be_nil 56 | end 57 | end 58 | context 'api_backtrace is nil' do 59 | let (:api_backtrace){ nil } 60 | it do 61 | exc = AlreadyExistsError.new(message, api_backtrace) 62 | expect(exc).to be_an(AlreadyExistsError) 63 | expect(exc.message).to eq message 64 | expect(exc.api_backtrace).to be_nil 65 | end 66 | end 67 | context 'conflict' do 68 | it do 69 | exc = AlreadyExistsError.new(message, api_backtrace, conflicts_with) 70 | expect(exc).to be_an(AlreadyExistsError) 71 | expect(exc.message).to eq message 72 | expect(exc.api_backtrace).to eq api_backtrace 73 | expect(exc.conflicts_with).to eq conflicts_with 74 | end 75 | end 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /spec/td/client/api_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe API do 4 | it 'initialize should raise an error with invalid endpoint' do 5 | expect { 6 | API.new(nil, :endpoint => 'smtp://api.tester.com:1000') 7 | }.to raise_error(RuntimeError, /Invalid endpoint:/) 8 | end 9 | 10 | VALID_NAMES = [ 11 | 'abc', 12 | 'abc_cd', 13 | '_abc_cd', 14 | '_abc_', 15 | 'ab0_', 16 | 'ab0', 17 | ] 18 | 19 | INVALID_NAMES = { 20 | 'a' => 'a__', 21 | 'a'*257 => 'a'*253+'__', 22 | 'abcD' => 'abcd', 23 | 'a-b*' => 'a_b_', 24 | } 25 | 26 | describe 'normalizer' do 27 | it 'normalized_msgpack should convert Bignum into String' do 28 | h = {'key' => 1111111111111111111111111111111111} 29 | unpacked = MessagePack.unpack(API.normalized_msgpack(h)) 30 | expect(unpacked['key']).to eq(h['key'].to_s) 31 | 32 | h = {'key' => -1111111111111111111111111111111111} 33 | unpacked = MessagePack.unpack(API.normalized_msgpack(h)) 34 | expect(unpacked['key']).to eq(h['key'].to_s) 35 | end 36 | 37 | it 'normalized_msgpack with out argument should convert Bignum into String' do 38 | h = {'key' => 1111111111111111111111111111111111, 'key2' => -1111111111111111111111111111111111, 'key3' => 0} 39 | out = '' 40 | API.normalized_msgpack(h, out) 41 | unpacked = MessagePack.unpack(out) 42 | expect(unpacked['key']).to eq(h['key'].to_s) 43 | expect(unpacked['key2']).to eq(h['key2'].to_s) 44 | expect(unpacked['key3']).to eq(h['key3']) # don't touch non-too-big integer 45 | end 46 | 47 | it 'normalize_database_name should return normalized data' do 48 | INVALID_NAMES.each_pair {|ng,ok| 49 | expect(API.normalize_database_name(ng)).to eq(ok) 50 | } 51 | expect { 52 | API.normalize_database_name('') 53 | }.to raise_error(RuntimeError) 54 | end 55 | 56 | it 'normalize_table_name should return normalized data' do 57 | INVALID_NAMES.each_pair {|ng,ok| 58 | expect(API.normalize_table_name(ng)).to eq(ok) 59 | } 60 | # empty 61 | expect { 62 | API.normalize_table_name('') 63 | }.to raise_error(RuntimeError) 64 | end 65 | 66 | it 'normalize_database_name should return valid data' do 67 | VALID_NAMES.each {|ok| 68 | expect(API.normalize_database_name(ok)).to eq(ok) 69 | } 70 | end 71 | end 72 | 73 | describe 'validator' do 74 | describe "'validate_database_name'" do 75 | it 'should raise a ParameterValidationError exceptions' do 76 | INVALID_NAMES.each_pair {|ng,ok| 77 | expect { 78 | API.validate_database_name(ng) 79 | }.to raise_error(ParameterValidationError) 80 | } 81 | # empty 82 | expect { 83 | API.validate_database_name('') 84 | }.to raise_error(ParameterValidationError) 85 | end 86 | 87 | it 'should return valid data' do 88 | VALID_NAMES.each {|ok| 89 | expect(API.validate_database_name(ok)).to eq(ok) 90 | } 91 | end 92 | end 93 | 94 | describe "'validate_table_name'" do 95 | it 'should raise a ParameterValidationError exception' do 96 | INVALID_NAMES.each_pair {|ng,ok| 97 | expect { 98 | API.validate_table_name(ng) 99 | }.to raise_error(ParameterValidationError) 100 | } 101 | expect { 102 | API.validate_table_name('') 103 | }.to raise_error(ParameterValidationError) 104 | end 105 | 106 | it 'should return valid data' do 107 | VALID_NAMES.each {|ok| 108 | expect(API.validate_database_name(ok)).to eq(ok) 109 | } 110 | end 111 | end 112 | 113 | describe "'validate_result_set_name'" do 114 | it 'should raise a ParameterValidationError exception' do 115 | INVALID_NAMES.each_pair {|ng,ok| 116 | expect { 117 | API.validate_result_set_name(ng) 118 | }.to raise_error(ParameterValidationError) 119 | } 120 | # empty 121 | expect { 122 | API.validate_result_set_name('') 123 | }.to raise_error(ParameterValidationError) 124 | end 125 | 126 | it 'should return valid data' do 127 | VALID_NAMES.each {|ok| 128 | expect(API.validate_result_set_name(ok)).to eq(ok) 129 | } 130 | end 131 | end 132 | 133 | describe "'validate_column_name'" do 134 | it 'should raise a ParameterValidationError exception' do 135 | [''].each { |ng| 136 | expect { 137 | API.validate_column_name(ng) 138 | }.to raise_error(ParameterValidationError) 139 | } 140 | end 141 | 142 | it 'should return valid data' do 143 | VALID_NAMES.each {|ok| 144 | expect(API.validate_column_name(ok)).to eq(ok) 145 | } 146 | ['a', 'a'*255].each {|ok| 147 | expect(API.validate_column_name(ok)).to eq(ok) 148 | } 149 | end 150 | end 151 | 152 | describe "'validate_sql_alias_name'" do 153 | it 'should raise a ParameterValidationError exception' do 154 | [''].each { |ng| 155 | expect{API.validate_sql_alias_name(ng)}.to raise_error(ParameterValidationError) 156 | } 157 | valid = ("a".."z").to_a.join<<("0".."9").to_a.join<<"_" 158 | ("\x00".."\x7F").each { |ng| 159 | next if valid.include?(ng) 160 | expect{API.validate_sql_alias_name(ng)}.to raise_error(ParameterValidationError) 161 | } 162 | end 163 | 164 | it 'should return valid data' do 165 | VALID_NAMES.each {|ok| 166 | expect(API.validate_sql_alias_name(ok)).to eq(ok) 167 | } 168 | ['a', '_a', 'a_', 'a'*512].each {|ok| 169 | expect(API.validate_sql_alias_name(ok)).to eq(ok) 170 | } 171 | end 172 | end 173 | 174 | describe "'generic validate_name'" do 175 | it 'should raise a ParameterValidationError exception' do 176 | # wrong target 177 | expect { 178 | API.validate_name("", 3, 256, '') 179 | }.to raise_error(ParameterValidationError) 180 | INVALID_NAMES.each_pair {|ng,ok| 181 | expect { 182 | API.validate_name("generic", 3, 256, ng) 183 | }.to raise_error(ParameterValidationError) 184 | } 185 | # empty 186 | expect { 187 | API.validate_name("generic", 3, 256, '') 188 | }.to raise_error(ParameterValidationError) 189 | # too short - one less than left limit 190 | expect { 191 | API.validate_name("generic", 3, 256, 'ab') 192 | }.to raise_error(ParameterValidationError) 193 | end 194 | 195 | it 'should return valid data' do 196 | VALID_NAMES.each {|ok| 197 | expect(API.validate_name("generic", 3, 256, ok)).to eq(ok) 198 | } 199 | # esplore left boundary 200 | expect(API.validate_name("generic", 2, 256, 'ab')).to eq('ab') 201 | expect(API.validate_name("generic", 1, 256, 'a')).to eq('a') 202 | # explore right boundary 203 | expect(API.validate_name("generic", 3, 256, 'a' * 256)).to eq('a' * 256) 204 | expect(API.validate_name("generic", 3, 128, 'a' * 128)).to eq('a' * 128) 205 | end 206 | end 207 | 208 | describe 'checking GET API content length with ssl' do 209 | include_context 'common helper' 210 | 211 | let(:api) { API.new(nil, endpoint: endpoint) } 212 | let :packed do 213 | s = StringIO.new(String.new) 214 | Zlib::GzipWriter.wrap(s) do |f| 215 | f << ['hello', 'world'].to_json 216 | end 217 | s.string 218 | end 219 | 220 | before do 221 | stub_api_request(:get, '/v3/job/result/12345', ssl: ssl). 222 | with(:query => {'format' => 'json'}). 223 | to_return( 224 | :headers => {'Content-Encoding' => 'gzip'}.merge(content_length), 225 | :body => packed 226 | ) 227 | end 228 | 229 | subject (:get_api_call) { 230 | api.job_result_format(12345, 'json', StringIO.new(String.new)) 231 | } 232 | 233 | context 'without ssl' do 234 | let(:endpoint) { "http://#{API::DEFAULT_ENDPOINT}" } 235 | let(:ssl) { false } 236 | let(:content_length) { {'Content-Length' => packed.size} } 237 | 238 | it 'not called #completed_body?' do 239 | expect(api).not_to receive(:completed_body?) 240 | 241 | get_api_call 242 | end 243 | end 244 | 245 | context 'with ssl' do 246 | let(:endpoint) { "https://#{API::DEFAULT_ENDPOINT}" } 247 | let(:ssl) { true } 248 | 249 | context 'without Content-Length' do 250 | let(:content_length) { {} } 251 | 252 | it 'api accuess succeded' do 253 | expect { get_api_call }.not_to raise_error 254 | end 255 | end 256 | 257 | context 'with Content-Length' do 258 | context 'match Content-Length and body.size' do 259 | let(:content_length) { {'Content-Length' => packed.size} } 260 | 261 | it 'api accuess succeded' do 262 | expect { get_api_call }.not_to raise_error 263 | end 264 | end 265 | end 266 | end 267 | end 268 | end 269 | end 270 | -------------------------------------------------------------------------------- /spec/td/client/api_ssl_connection_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/client/api' 3 | require 'logger' 4 | require 'webrick' 5 | require 'webrick/https' 6 | 7 | # Workaround for https://github.com/jruby/jruby-openssl/issues/78 8 | # With recent JRuby + jruby-openssl, X509CRL#extentions_to_text causes 9 | # StringIndexOOBException when we try to dump SSL Server Certificate. 10 | # when one of extensions has "" as value. 11 | # This hack is from httpclient https://github.com/nahi/httpclient/blob/master/lib/httpclient/util.rb#L27-L46 12 | if defined? JRUBY_VERSION 13 | require 'openssl' 14 | require 'java' 15 | module OpenSSL 16 | module X509 17 | class Certificate 18 | java_import 'java.security.cert.Certificate' 19 | java_import 'java.security.cert.CertificateFactory' 20 | java_import 'java.io.ByteArrayInputStream' 21 | def to_text 22 | cf = CertificateFactory.getInstance('X.509') 23 | cf.generateCertificate(ByteArrayInputStream.new(self.to_der.to_java_bytes)).toString 24 | end 25 | end 26 | end 27 | end 28 | end 29 | 30 | describe 'API SSL connection' do 31 | DIR = File.dirname(File.expand_path(__FILE__)) 32 | 33 | after :each do 34 | @server.shutdown if @server 35 | end 36 | 37 | it 'should fail to connect SSLv3 only server' do 38 | @server = setup_server(:SSLv3) 39 | api = API.new(nil, :endpoint => "https://localhost:#{@serverport}", :retry_post_requests => false) 40 | api.ssl_ca_file = File.join(DIR, 'testRootCA.crt') 41 | expect { 42 | begin 43 | api.delete_database('no_such_database') 44 | rescue Errno::ECONNRESET 45 | raise OpenSSL::SSL::SSLError # When openssl does not support SSLv3, httpclient server will not start. For context: https://github.com/nahi/httpclient/pull/424#issuecomment-731714786 46 | end 47 | }.to raise_error OpenSSL::SSL::SSLError 48 | end 49 | 50 | it 'should succeed to access to the server with verify false option' do 51 | @server = setup_server(:TLSv1_2) 52 | api = API.new(nil, :endpoint => "https://localhost:#{@serverport}", :retry_post_requests => false, :verify => false) 53 | expect { 54 | api.delete_database('no_such_database') 55 | }.to raise_error TreasureData::NotFoundError 56 | end 57 | 58 | it 'should succeed to access to the server with self signed certificate' do 59 | @server = setup_server(:TLSv1_2) 60 | api = API.new(nil, :endpoint => "https://localhost:#{@serverport}", :retry_post_requests => false, :verify => File.join(DIR, 'testRootCA.crt')) 61 | expect { 62 | api.delete_database('no_such_database') 63 | }.to raise_error TreasureData::NotFoundError 64 | end 65 | 66 | it 'should success to connect TLSv1_2 only server' do 67 | @server = setup_server(:TLSv1_2) 68 | api = API.new(nil, :endpoint => "https://localhost:#{@serverport}", :retry_post_requests => false) 69 | api.ssl_ca_file = File.join(DIR, 'testRootCA.crt') 70 | expect { 71 | api.delete_database('no_such_database') 72 | }.to raise_error TreasureData::NotFoundError 73 | end 74 | 75 | def setup_server(ssl_version, port = 1000 + rand(1000)) 76 | logger = Logger.new(STDERR) 77 | logger.level = Logger::Severity::FATAL # avoid logging SSLError (ERROR level) 78 | @server = WEBrick::HTTPServer.new( 79 | :BindAddress => "localhost", 80 | :Logger => logger, 81 | :Port => port, 82 | :AccessLog => [], 83 | :DocumentRoot => '.', 84 | :SSLEnable => true, 85 | :SSLCACertificateFile => File.join(DIR, 'testRootCA.crt'), 86 | :SSLCertificate => cert('testServer.crt'), 87 | :SSLPrivateKey => key('testServer.key') 88 | ) 89 | @serverport = @server.config[:Port] 90 | @server.mount( 91 | '/hello', 92 | WEBrick::HTTPServlet::ProcHandler.new(method(:do_hello).to_proc) 93 | ) 94 | @server.ssl_context.ssl_version = ssl_version 95 | @server_thread = start_server_thread(@server) 96 | return @server 97 | end 98 | 99 | def do_hello(req, res) 100 | res['content-type'] = 'text/html' 101 | res.body = "hello" 102 | end 103 | 104 | def start_server_thread(server) 105 | t = Thread.new { 106 | Thread.current.abort_on_exception = true 107 | server.start 108 | } 109 | while server.status != :Running 110 | sleep 0.1 111 | unless t.alive? 112 | t.join 113 | raise 114 | end 115 | end 116 | t 117 | end 118 | 119 | def cert(filename) 120 | OpenSSL::X509::Certificate.new(File.read(File.join(DIR, filename))) 121 | end 122 | 123 | def key(filename) 124 | OpenSSL::PKey::RSA.new(File.read(File.join(DIR, filename))) 125 | end 126 | end 127 | -------------------------------------------------------------------------------- /spec/td/client/bulk_import_spec.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'spec_helper' 4 | require 'td/client/spec_resources' 5 | require 'tempfile' 6 | 7 | describe 'BulkImport API' do 8 | include_context 'spec symbols' 9 | include_context 'common helper' 10 | 11 | let :api do 12 | API.new(nil) 13 | end 14 | 15 | let :packed do 16 | s = StringIO.new 17 | Zlib::GzipWriter.wrap(s) do |f| 18 | pk = MessagePack::Packer.new(f) 19 | pk.write([1, '2', 3.0]) 20 | pk.write([4, '5', 6.0]) 21 | pk.write([7, '8', 9.0]) 22 | pk.flush 23 | end 24 | s.string 25 | end 26 | 27 | let(:endpoint_domain) { TreasureData::API::DEFAULT_IMPORT_ENDPOINT } 28 | 29 | describe 'create_bulk_import' do 30 | it 'should create a new bulk_import' do 31 | stub_api_request(:post, "/v3/bulk_import/create/#{e(bi_name)}/#{e(db_name)}/#{e(table_name)}"). 32 | to_return(:body => {'bulk_import' => bi_name}.to_json) 33 | 34 | expect(api.create_bulk_import(bi_name, db_name, table_name)).to be_nil 35 | end 36 | 37 | it 'should return 422 error with invalid name' do 38 | name = 'D' 39 | err_msg = "Validation failed: Name is too short" # " (minimum is 3 characters)" 40 | stub_api_request(:post, "/v3/bulk_import/create/#{e(name)}/#{e(db_name)}/#{e(table_name)}"). 41 | to_return(:status => 422, :body => {'message' => err_msg}.to_json) 42 | 43 | expect { 44 | api.create_bulk_import(name, db_name, table_name) 45 | }.to raise_error(TreasureData::APIError, /#{err_msg}/) 46 | end 47 | 48 | it 'should return 404 error with non exist database name' do 49 | db = 'no_such_db' 50 | err_msg = "Couldn't find UserDatabase with name = #{db}" 51 | stub_api_request(:post, "/v3/bulk_import/create/#{e(bi_name)}/#{e(db)}/#{e(table_name)}"). 52 | to_return(:status => 404, :body => {'message' => err_msg}.to_json) 53 | 54 | expect { 55 | api.create_bulk_import(bi_name, db, table_name) 56 | }.to raise_error(TreasureData::APIError, /#{err_msg}/) 57 | end 58 | 59 | it 'should return 404 error with non exist table name' do 60 | table = 'no_such_table' 61 | err_msg = "Couldn't find UserTableReference with name = #{table}" 62 | stub_api_request(:post, "/v3/bulk_import/create/#{e(bi_name)}/#{e(db_name)}/#{e(table)}"). 63 | to_return(:status => 404, :body => {'message' => err_msg}.to_json) 64 | 65 | expect { 66 | api.create_bulk_import(bi_name, db_name, table) 67 | }.to raise_error(TreasureData::APIError, /#{err_msg}/) 68 | end 69 | end 70 | 71 | describe 'delete_bulk_import' do 72 | it 'runs' do 73 | stub_api_request(:post, '/v3/bulk_import/delete/name'). 74 | with(:body => 'foo=bar') 75 | expect(api.delete_bulk_import('name', 'foo' => 'bar')).to eq(nil) 76 | end 77 | end 78 | 79 | describe 'show_bulk_import' do 80 | it 'runs' do 81 | stub_api_request(:get, '/v3/bulk_import/show/name'). 82 | to_return(:body => {'status' => 'status', 'other' => 'other'}.to_json) 83 | expect(api.show_bulk_import('name')['status']).to eq('status') 84 | end 85 | end 86 | 87 | describe 'list_bulk_imports' do 88 | it 'runs' do 89 | stub_api_request(:get, '/v3/bulk_import/list'). 90 | with(:query => 'foo=bar'). 91 | to_return(:body => {'bulk_imports' => %w(1 2 3)}.to_json) 92 | expect(api.list_bulk_imports('foo' => 'bar')).to eq(%w(1 2 3)) 93 | end 94 | end 95 | 96 | describe 'list_bulk_import_parts' do 97 | it 'runs' do 98 | stub_api_request(:get, '/v3/bulk_import/list_parts/name'). 99 | with(:query => 'foo=bar'). 100 | to_return(:body => {'parts' => %w(1 2 3)}.to_json) 101 | expect(api.list_bulk_import_parts('name', 'foo' => 'bar')).to eq(%w(1 2 3)) 102 | end 103 | end 104 | 105 | describe 'bulk_import_upload_part' do 106 | it 'runs' do 107 | t = Tempfile.new('bulk_import_spec') 108 | File.open(t.path, 'w') do |f| 109 | f << '12345' 110 | end 111 | stub_request(:put, "https://#{TreasureData::API::DEFAULT_ENDPOINT}/v3/bulk_import/upload_part/name/part"). 112 | with(:body => '12345') 113 | File.open(t.path) do |f| 114 | expect(api.bulk_import_upload_part('name', 'part', f, 5)).to eq(nil) 115 | end 116 | end 117 | 118 | if ''.respond_to?(:encode) 119 | it 'encodes part_name in UTF-8' do 120 | t = Tempfile.new('bulk_import_spec') 121 | File.open(t.path, 'w') do |f| 122 | f << '12345' 123 | end 124 | stub_request(:put, "https://#{TreasureData::API::DEFAULT_ENDPOINT}/v3/bulk_import/upload_part/name/" + CGI.escape('日本語(Japanese)'.encode('UTF-8'))). 125 | with(:body => '12345') 126 | File.open(t.path) do |f| 127 | expect(api.bulk_import_upload_part('name', '日本語(Japanese)'.encode('Windows-31J'), f, 5)).to eq(nil) 128 | end 129 | end 130 | end 131 | end 132 | 133 | describe 'bulk_import_delete_part' do 134 | it 'runs' do 135 | stub_api_request(:post, '/v3/bulk_import/delete_part/name/part') 136 | expect(api.bulk_import_delete_part('name', 'part')).to eq(nil) 137 | end 138 | end 139 | 140 | describe 'freeze_bulk_import' do 141 | it 'runs' do 142 | stub_api_request(:post, '/v3/bulk_import/freeze/name') 143 | expect(api.freeze_bulk_import('name')).to eq(nil) 144 | end 145 | end 146 | 147 | describe 'unfreeze_bulk_import' do 148 | it 'runs' do 149 | stub_api_request(:post, '/v3/bulk_import/unfreeze/name') 150 | expect(api.unfreeze_bulk_import('name')).to eq(nil) 151 | end 152 | end 153 | 154 | describe 'perform_bulk_import' do 155 | it 'runs' do 156 | stub_api_request(:post, '/v3/bulk_import/perform/name'). 157 | to_return(:body => {'job_id' => 12345}.to_json) 158 | expect(api.perform_bulk_import('name')).to eq('12345') 159 | end 160 | end 161 | 162 | describe 'commit_bulk_import' do 163 | it 'runs' do 164 | stub_api_request(:post, '/v3/bulk_import/commit/name'). 165 | to_return(:body => {'job_id' => 12345}.to_json) 166 | expect(api.commit_bulk_import('name')).to eq(nil) 167 | end 168 | end 169 | 170 | describe 'bulk_import_error_records' do 171 | it 'returns [] on empty' do 172 | stub_api_request(:get, '/v3/bulk_import/error_records/name'). 173 | to_return(:body => '') 174 | expect(api.bulk_import_error_records('name')).to eq([]) 175 | end 176 | 177 | it 'returns nil on empty if block given' do 178 | stub_api_request(:get, '/v3/bulk_import/error_records/name'). 179 | to_return(:body => '') 180 | expect(api.bulk_import_error_records('name'){}).to eq(nil) 181 | end 182 | 183 | it 'returns unpacked result' do 184 | stub_api_request(:get, '/v3/bulk_import/error_records/name'). 185 | to_return(:body => packed) 186 | expect(api.bulk_import_error_records('name')).to eq([[1, '2', 3.0], [4, '5', 6.0], [7, '8', 9.0]]) 187 | end 188 | 189 | it 'yields unpacked result if block given' do 190 | stub_api_request(:get, '/v3/bulk_import/error_records/name'). 191 | to_return(:body => packed) 192 | result = [] 193 | api.bulk_import_error_records('name') do |row| 194 | result << row 195 | end 196 | expect(result).to eq([[1, '2', 3.0], [4, '5', 6.0], [7, '8', 9.0]]) 197 | end 198 | end 199 | end 200 | -------------------------------------------------------------------------------- /spec/td/client/bulk_load_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/client/spec_resources' 3 | require 'tempfile' 4 | 5 | describe 'BulkImport API' do 6 | include_context 'spec symbols' 7 | include_context 'common helper' 8 | 9 | let :api do 10 | Client.new(nil, {:max_cumul_retry_delay => -1}) 11 | end 12 | 13 | let :retry_api do 14 | API.new(nil, {:retry_delay => 1, :max_cumul_retry_delay => 1}) 15 | end 16 | 17 | let :original_config do 18 | { 19 | :config => { 20 | :type => "s3_file", 21 | :access_key_id => "key", 22 | :secret_access_key => "secret", 23 | :endpoint => "s3.amazonaws.com", 24 | :bucket => "td-bulk-loader-test-tokyo", 25 | :path_prefix => "in/nahi/sample" 26 | } 27 | } 28 | end 29 | 30 | let :guessed_config do 31 | { 32 | "config" => { 33 | "type" => "s3_file", 34 | "access_key_id" => "key", 35 | "secret_access_key" => "secret", 36 | "endpoint" => "s3.amazonaws.com", 37 | "bucket" => "td-bulk-loader-test-tokyo", 38 | "path_prefix" => "in/nahi/sample", 39 | "parser" => { 40 | "charset" => "UTF-8", 41 | "newline" => "LF", 42 | "type" => "csv", 43 | "delimiter" => ",", 44 | "header_line" => false, 45 | "columns" => [ 46 | {"name" => "time", "type" => "long"}, 47 | {"name" => "c1", "type" => "long"}, 48 | {"name" => "c2", "type" => "string"}, 49 | {"name" => "c3", "type" => "string"}, 50 | ] 51 | }, 52 | "decoders" => [ 53 | {"type" => "gzip"} 54 | ] 55 | } 56 | } 57 | end 58 | 59 | let :preview_result do 60 | { 61 | "schema" => [ 62 | {"index" => 0, "name" => "c0", "type" => "string"}, 63 | {"index" => 1, "name" => "c1", "type" => "long"}, 64 | {"index" => 2, "name" => "c2", "type" => "string"}, 65 | {"index" => 3, "name" => "c3", "type" => "string"} 66 | ], 67 | "records" => [ 68 | ["19920116", 32864, "06612", "00195"], 69 | ["19910729", 14824, "07706", "00058"], 70 | ["19950708", 27559, "03244", "00034"], 71 | ["19931010", 11270, "03459", "00159"], 72 | ["19981117", 20461, "01409", "00128"], 73 | ["19981117", 20461, "00203", "00128"], 74 | ["19930108", 44402, "01489", "00001"], 75 | ["19960104", 16528, "04848", "00184"], 76 | ["19960104", 16528, "01766", "00184"], 77 | ["19881022", 26114, "06960", "00175"] 78 | ] 79 | } 80 | end 81 | 82 | let :bulk_load_session do 83 | guessed_config.dup.merge( 84 | { 85 | "name" => "nahi_test_1", 86 | "cron" => "@daily", 87 | "timezone" => "Asia/Tokyo", 88 | "delay" => 3600 89 | } 90 | ) 91 | end 92 | 93 | let :bulk_load_job do 94 | guessed_config.dup.merge( 95 | { 96 | "job_id" => 123456, 97 | "account_id" => 1, 98 | "status" => "success", 99 | "records" => 10, 100 | "schema" => [["c0", "string", ""], ["c1", "long", ""], ["c2", "string", ""], ["c3", "string", ""]], 101 | "database" => {"id" => 189263, "name" => "nahidb"}, 102 | "table" => {"id" => 176281, "name" => "bulkload_import_test"}, 103 | "created_at" => 1426738133, 104 | "updated_at" => 1426738145, 105 | "start_at" => 1426738134, 106 | "end_at" => 1426738144 107 | } 108 | ) 109 | end 110 | 111 | describe 'guess' do 112 | it 'returns guessed json' do 113 | stub_api_request(:post, '/v3/bulk_loads/guess'). 114 | with(:body => original_config.to_json). 115 | to_return(:body => guessed_config.to_json) 116 | expect(api.bulk_load_guess( 117 | original_config 118 | )).to eq(guessed_config) 119 | end 120 | 121 | it 'raises an error' do 122 | stub_api_request(:post, '/v3/bulk_loads/guess'). 123 | with(:body => original_config.to_json). 124 | to_return(:status => 500, :body => guessed_config.to_json) 125 | expect { 126 | api.bulk_load_guess( 127 | original_config 128 | ) 129 | }.to raise_error(TreasureData::APIError) 130 | end 131 | 132 | it 'perform redo on 500 error' do 133 | stub_api_request(:post, '/v3/bulk_loads/guess'). 134 | with(:body => original_config.to_json). 135 | to_return(:status => 500, :body => guessed_config.to_json) 136 | begin 137 | expect(retry_api.bulk_load_guess( 138 | original_config 139 | )).to != nil 140 | rescue TreasureData::APIError => e 141 | expect(e.message).to match(/^500: BulkLoad configuration guess failed/) 142 | end 143 | end 144 | 145 | it 'perform retries on connection failure' do 146 | api = retry_api 147 | allow(api.instance_eval { @api }).to receive(:post).and_raise(SocketError.new('>>')) 148 | begin 149 | retry_api.bulk_load_guess( 150 | original_config 151 | ) 152 | rescue SocketError => e 153 | expect(e.message).to eq('>> (Retried 1 times in 1 seconds)') 154 | end 155 | end 156 | end 157 | 158 | describe 'guess with old format' do 159 | it 'returns guessed json' do 160 | stub_api_request(:post, '/v3/bulk_loads/guess'). 161 | with(:body => original_config.to_json). 162 | to_return(:body => guessed_config.to_json) 163 | expect(api.bulk_load_guess( 164 | original_config 165 | )).to eq(guessed_config) 166 | end 167 | 168 | it 'raises an error' do 169 | stub_api_request(:post, '/v3/bulk_loads/guess'). 170 | with(:body => original_config.to_json). 171 | to_return(:status => 500, :body => guessed_config.to_json) 172 | expect { 173 | api.bulk_load_guess( 174 | original_config 175 | ) 176 | }.to raise_error(TreasureData::APIError) 177 | end 178 | 179 | it 'perform redo on 500 error' do 180 | stub_api_request(:post, '/v3/bulk_loads/guess'). 181 | with(:body => original_config.to_json). 182 | to_return(:status => 500, :body => guessed_config.to_json) 183 | begin 184 | expect(retry_api.bulk_load_guess( 185 | original_config 186 | )).to != nil 187 | rescue TreasureData::APIError => e 188 | expect(e.message).to match(/^500: BulkLoad configuration guess failed/) 189 | end 190 | end 191 | 192 | it 'perform retries on connection failure' do 193 | api = retry_api 194 | allow(api.instance_eval { @api }).to receive(:post).and_raise(SocketError.new('>>')) 195 | begin 196 | retry_api.bulk_load_guess( 197 | original_config 198 | ) 199 | rescue SocketError => e 200 | expect(e.message).to eq('>> (Retried 1 times in 1 seconds)') 201 | end 202 | end 203 | end 204 | 205 | describe 'preview' do 206 | it 'returns preview json' do 207 | stub_api_request(:post, '/v3/bulk_loads/preview'). 208 | with(:body => guessed_config.to_json). 209 | to_return(:body => preview_result.to_json) 210 | expect(api.bulk_load_preview( 211 | guessed_config 212 | )).to eq(preview_result) 213 | end 214 | 215 | it 'raises an error' do 216 | stub_api_request(:post, '/v3/bulk_loads/preview'). 217 | with(:body => guessed_config.to_json). 218 | to_return(:status => 500, :body => preview_result.to_json) 219 | expect { 220 | api.bulk_load_preview( 221 | guessed_config 222 | ) 223 | }.to raise_error(TreasureData::APIError) 224 | end 225 | end 226 | 227 | describe 'issue' do 228 | it 'returns job id' do 229 | expected_request = guessed_config.dup 230 | expected_request['database'] = 'database' 231 | expected_request['table'] = 'table' 232 | stub_api_request(:post, '/v3/job/issue/bulkload/database'). 233 | with(:body => expected_request.to_json). 234 | to_return(:body => {'job_id' => 12345}.to_json) 235 | expect(api.bulk_load_issue( 236 | 'database', 237 | 'table', 238 | guessed_config 239 | )).to eq('12345') 240 | end 241 | end 242 | 243 | describe 'list' do 244 | it 'returns BulkLoadSession' do 245 | stub_api_request(:get, '/v3/bulk_loads'). 246 | to_return(:body => [bulk_load_session, bulk_load_session].to_json) 247 | result = api.bulk_load_list 248 | expect(result.size).to eq(2) 249 | expect(result.first).to eq(bulk_load_session) 250 | end 251 | 252 | it 'returns empty' do 253 | stub_api_request(:get, '/v3/bulk_loads'). 254 | to_return(:body => [].to_json) 255 | expect(api.bulk_load_list.size).to eq(0) 256 | end 257 | end 258 | 259 | describe 'create' do 260 | it 'returns registered bulk_load_session' do 261 | expected_request = guessed_config.dup 262 | expected_request['name'] = 'nahi_test_1' 263 | expected_request['cron'] = '@daily' 264 | expected_request['timezone'] = 'Asia/Tokyo' 265 | expected_request['delay'] = 3600 266 | expected_request['database'] = 'database' 267 | expected_request['table'] = 'table' 268 | stub_api_request(:post, '/v3/bulk_loads'). 269 | with(:body => expected_request.to_json). 270 | to_return(:body => bulk_load_session.to_json) 271 | expect(api.bulk_load_create( 272 | 'nahi_test_1', 273 | 'database', 274 | 'table', 275 | guessed_config, 276 | { 277 | cron: '@daily', 278 | timezone: 'Asia/Tokyo', 279 | delay: 3600 280 | } 281 | )).to eq(bulk_load_session) 282 | end 283 | 284 | it 'accepts empty option' do 285 | expected_request = guessed_config.dup 286 | expected_request['name'] = 'nahi_test_1' 287 | expected_request['database'] = 'database' 288 | expected_request['table'] = 'table' 289 | stub_api_request(:post, '/v3/bulk_loads'). 290 | with(:body => expected_request.to_json). 291 | to_return(:body => bulk_load_session.to_json) 292 | expect(api.bulk_load_create( 293 | 'nahi_test_1', 294 | 'database', 295 | 'table', 296 | guessed_config 297 | )).to eq(bulk_load_session) 298 | end 299 | 300 | it 'accepts time_column option' do 301 | expected_request = guessed_config.dup 302 | expected_request['name'] = 'nahi_test_1' 303 | expected_request['time_column'] = 'c0' 304 | expected_request['database'] = 'database' 305 | expected_request['table'] = 'table' 306 | stub_api_request(:post, '/v3/bulk_loads'). 307 | with(:body => expected_request.to_json). 308 | to_return(:body => bulk_load_session.to_json) 309 | expect(api.bulk_load_create( 310 | 'nahi_test_1', 311 | 'database', 312 | 'table', 313 | guessed_config, 314 | { 315 | time_column: 'c0' 316 | } 317 | )).to eq(bulk_load_session) 318 | end 319 | end 320 | 321 | describe 'show' do 322 | it 'returns bulk_load_session' do 323 | stub_api_request(:get, '/v3/bulk_loads/nahi_test_1'). 324 | to_return(:body => bulk_load_session.to_json) 325 | expect(api.bulk_load_show('nahi_test_1')).to eq(bulk_load_session) 326 | end 327 | end 328 | 329 | describe 'update' do 330 | it 'returns updated bulk_load_session' do 331 | stub_api_request(:put, '/v3/bulk_loads/nahi_test_1'). 332 | with(:body => bulk_load_session.to_json). 333 | to_return(:body => bulk_load_session.to_json) 334 | expect(api.bulk_load_update( 335 | 'nahi_test_1', 336 | bulk_load_session 337 | )).to eq(bulk_load_session) 338 | end 339 | 340 | it 'returns updated bulk_load_session' do 341 | updated_bulk_load_session = bulk_load_session.merge({'timezone' => 'America/Los Angeles'}) 342 | stub_api_request(:put, '/v3/bulk_loads/nahi_test_1'). 343 | with(:body => updated_bulk_load_session.to_json). 344 | to_return(:body => updated_bulk_load_session.to_json) 345 | expect(api.bulk_load_update( 346 | 'nahi_test_1', 347 | updated_bulk_load_session 348 | )).to eq updated_bulk_load_session 349 | end 350 | 351 | it 'can remove the cron schedule ' do 352 | updated_bulk_load_session = bulk_load_session.merge({'cron' => ''}) 353 | # NOTE: currently the API just ignores an empty 'cron' specification update 354 | # I am assuming that once fixed, the API will return a nil for cron if unscheduled. 355 | expected_bulk_load_session = (bulk_load_session['cron'] = nil) 356 | stub_api_request(:put, '/v3/bulk_loads/nahi_test_1'). 357 | with(:body => updated_bulk_load_session.to_json). 358 | to_return(:body => expected_bulk_load_session.to_json) 359 | expect(api.bulk_load_update( 360 | 'nahi_test_1', 361 | updated_bulk_load_session 362 | )).to eq expected_bulk_load_session 363 | end 364 | end 365 | 366 | describe 'delete' do 367 | it 'returns updated bulk_load_session' do 368 | stub_api_request(:delete, '/v3/bulk_loads/nahi_test_1'). 369 | to_return(:body => bulk_load_session.to_json) 370 | expect(api.bulk_load_delete('nahi_test_1')).to eq(bulk_load_session) 371 | end 372 | end 373 | 374 | describe 'history' do 375 | it 'returns list of jobs' do 376 | stub_api_request(:get, '/v3/bulk_loads/nahi_test_1/jobs'). 377 | to_return(:body => [bulk_load_job, bulk_load_job].to_json) 378 | result = api.bulk_load_history('nahi_test_1') 379 | expect(result.size).to eq(2) 380 | expect(result.first).to eq(bulk_load_job) 381 | end 382 | end 383 | 384 | describe 'run' do 385 | it 'returns job_id' do 386 | stub_api_request(:post, '/v3/bulk_loads/nahi_test_1/jobs'). 387 | with(:body => '{}'). 388 | to_return(:body => {'job_id' => 12345}.to_json) 389 | expect(api.bulk_load_run('nahi_test_1')).to eq('12345') 390 | end 391 | 392 | it 'accepts scheduled_time' do 393 | now = Time.now.to_i 394 | stub_api_request(:post, '/v3/bulk_loads/nahi_test_1/jobs'). 395 | with(:body => {scheduled_time: now.to_s}.to_json). 396 | to_return(:body => {'job_id' => 12345}.to_json) 397 | expect(api.bulk_load_run('nahi_test_1', now)).to eq('12345') 398 | end 399 | end 400 | 401 | end 402 | -------------------------------------------------------------------------------- /spec/td/client/db_api_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/client/spec_resources' 3 | 4 | describe 'Database API' do 5 | include_context 'spec symbols' 6 | include_context 'common helper' 7 | 8 | let :api do 9 | API.new(nil) 10 | end 11 | 12 | let :client do 13 | Client.new(apikey) 14 | end 15 | 16 | describe "'create_database' API" do 17 | it 'should create a new database' do 18 | stub_api_request(:post, "/v3/database/create/#{e(db_name)}"). 19 | to_return(:body => {'database' => db_name}.to_json) 20 | 21 | expect(api.create_database(db_name)).to be true 22 | end 23 | 24 | it 'should return 400 error with invalid name' do 25 | invalid_name = 'a' 26 | err_msg = "Name must be 3 to 256 characters, got #{invalid_name.length} characters. name = '#{invalid_name}'" 27 | stub_api_request(:post, "/v3/database/create/#{e(invalid_name)}"). 28 | to_return(:status => 400, :body => {'message' => err_msg}.to_json) 29 | 30 | expect { 31 | api.create_database(invalid_name) 32 | }.to raise_error(TreasureData::APIError, /#{err_msg}/) 33 | end 34 | 35 | it 'should return 409 error with duplicated name' do 36 | err_msg = "Database #{db_name} already exists" 37 | stub_api_request(:post, "/v3/database/create/#{e(db_name)}"). 38 | to_return(:status => 409, :body => {'message' => err_msg}.to_json) 39 | 40 | expect { 41 | api.create_database(db_name) 42 | }.to raise_error(TreasureData::AlreadyExistsError, /#{err_msg}/) 43 | end 44 | end 45 | 46 | describe "'list_databases' API" do 47 | it 'should list the databases with count, created_at, updated_at, organization, and permission' do 48 | databases = [ 49 | ["db_1", 111, "2013-01-21 01:51:41 UTC", "2014-01-21 01:51:41 UTC", nil, "administrator"], 50 | ["db_2", 222, "2013-02-22 02:52:42 UTC", "2014-02-22 02:52:42 UTC", nil, "full_access"], 51 | ["db_3", 333, "2013-03-23 03:53:43 UTC", "2014-03-23 03:53:43 UTC", nil, "import_only"], 52 | ["db_4", 444, "2013-04-24 04:54:44 UTC", "2014-04-24 04:54:44 UTC", nil, "query_only"] 53 | ] 54 | stub_api_request(:get, "/v3/database/list"). 55 | to_return(:body => {'databases' => [ 56 | {'name' => databases[0][0], 'count' => databases[0][1], 'created_at' => databases[0][2], 'updated_at' => databases[0][3], 'organization' => databases[0][4], 'permission' => databases[0][5]}, 57 | {'name' => databases[1][0], 'count' => databases[1][1], 'created_at' => databases[1][2], 'updated_at' => databases[1][3], 'organization' => databases[1][4], 'permission' => databases[1][5]}, 58 | {'name' => databases[2][0], 'count' => databases[2][1], 'created_at' => databases[2][2], 'updated_at' => databases[2][3], 'organization' => databases[2][4], 'permission' => databases[2][5]}, 59 | {'name' => databases[3][0], 'count' => databases[3][1], 'created_at' => databases[3][2], 'updated_at' => databases[3][3], 'organization' => databases[3][4], 'permission' => databases[3][5]} 60 | ]}.to_json) 61 | 62 | db_list = api.list_databases 63 | databases.each {|db| 64 | expect(db_list[db[0]]).to eq(db[1..-1]) 65 | } 66 | end 67 | end 68 | 69 | describe "'databases' Client API" do 70 | it 'should return an array of Databases objects containing name, count, created_at, updated_at, organization, and permission' do 71 | databases = [ 72 | ["db_1", 111, "2013-01-21 01:51:41 UTC", "2014-01-21 01:51:41 UTC", nil, "administrator"], 73 | ["db_2", 222, "2013-02-22 02:52:42 UTC", "2014-02-22 02:52:42 UTC", nil, "full_access"], 74 | ["db_3", 333, "2013-03-23 03:53:43 UTC", "2014-03-23 03:53:43 UTC", nil, "import_only"], 75 | ["db_4", 444, "2013-04-24 04:54:44 UTC", "2014-04-24 04:54:44 UTC", nil, "query_only"] 76 | ] 77 | stub_api_request(:get, "/v3/database/list"). 78 | to_return(:body => {'databases' => [ 79 | {'name' => databases[0][0], 'count' => databases[0][1], 'created_at' => databases[0][2], 'updated_at' => databases[0][3], 'organization' => databases[0][4], 'permission' => databases[0][5]}, 80 | {'name' => databases[1][0], 'count' => databases[1][1], 'created_at' => databases[1][2], 'updated_at' => databases[1][3], 'organization' => databases[1][4], 'permission' => databases[1][5]}, 81 | {'name' => databases[2][0], 'count' => databases[2][1], 'created_at' => databases[2][2], 'updated_at' => databases[2][3], 'organization' => databases[2][4], 'permission' => databases[2][5]}, 82 | {'name' => databases[3][0], 'count' => databases[3][1], 'created_at' => databases[3][2], 'updated_at' => databases[3][3], 'organization' => databases[3][4], 'permission' => databases[3][5]} 83 | ]}.to_json) 84 | 85 | db_list = client.databases.sort_by { |e| e.name } 86 | databases.length.times {|i| 87 | expect(db_list[i].name).to eq(databases[i][0]) 88 | expect(db_list[i].count).to eq(databases[i][1]) 89 | expect(db_list[i].created_at).to eq(Time.parse(databases[i][2])) 90 | expect(db_list[i].updated_at).to eq(Time.parse(databases[i][3])) 91 | expect(db_list[i].org_name).to eq(databases[i][4]) 92 | expect(db_list[i].permission).to eq(databases[i][5].to_sym) 93 | } 94 | end 95 | end 96 | 97 | describe "'database' Client API" do 98 | it "should return the Databases object corresponding to the name and containing count, created_at, updated_at, organization, and permission" do 99 | databases = [ 100 | ["db_1", 111, "2013-01-21 01:51:41 UTC", "2014-01-21 01:51:41 UTC", nil, "administrator"], 101 | ["db_2", 222, "2013-02-22 02:52:42 UTC", "2014-02-22 02:52:42 UTC", nil, "full_access"], 102 | ["db_3", 333, "2013-03-23 03:53:43 UTC", "2014-03-23 03:53:43 UTC", nil, "import_only"], 103 | ["db_4", 444, "2013-04-24 04:54:44 UTC", "2014-04-24 04:54:44 UTC", nil, "query_only"] 104 | ] 105 | stub_api_request(:get, "/v3/database/list"). 106 | to_return(:body => {'databases' => [ 107 | {'name' => databases[0][0], 'count' => databases[0][1], 'created_at' => databases[0][2], 'updated_at' => databases[0][3], 'organization' => databases[0][4], 'permission' => databases[0][5]}, 108 | {'name' => databases[1][0], 'count' => databases[1][1], 'created_at' => databases[1][2], 'updated_at' => databases[1][3], 'organization' => databases[1][4], 'permission' => databases[1][5]}, 109 | {'name' => databases[2][0], 'count' => databases[2][1], 'created_at' => databases[2][2], 'updated_at' => databases[2][3], 'organization' => databases[2][4], 'permission' => databases[2][5]}, 110 | {'name' => databases[3][0], 'count' => databases[3][1], 'created_at' => databases[3][2], 'updated_at' => databases[3][3], 'organization' => databases[3][4], 'permission' => databases[3][5]} 111 | ]}.to_json) 112 | 113 | i = 1 114 | db = client.database(databases[i][0]) 115 | expect(db.name).to eq(databases[i][0]) 116 | expect(db.count).to eq(databases[i][1]) 117 | expect(db.created_at).to eq(Time.parse(databases[i][2])) 118 | expect(db.updated_at).to eq(Time.parse(databases[i][3])) 119 | expect(db.org_name).to eq(databases[i][4]) 120 | expect(db.permission).to eq(databases[i][5].to_sym) 121 | end 122 | end 123 | end 124 | -------------------------------------------------------------------------------- /spec/td/client/export_api_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/client/spec_resources' 3 | 4 | describe 'Export API' do 5 | include_context 'spec symbols' 6 | include_context 'common helper' 7 | 8 | let :api do 9 | API.new(nil) 10 | end 11 | 12 | describe 'export' do 13 | let :storage_type do 14 | 's3' 15 | end 16 | 17 | it 'should export successfully' do 18 | # TODO: Use correnty values 19 | params = {'file_format' => 'json.gz', 'bucket' => 'bin', 'access_key_id' => 'id', 'secret_access_key' => 'secret'} 20 | stub_api_request(:post, "/v3/export/run/#{e(db_name)}/#{e(table_name)}").with(:body => params.merge('storage_type' => storage_type)). 21 | to_return(:body => {'database' => db_name, 'job_id' => '1', 'debug' => {}}.to_json) 22 | 23 | expect(api.export(db_name, table_name, storage_type, params)).to eq('1') 24 | end 25 | 26 | it 'should return 400 error with invalid storage type' do 27 | invalid_type = 'gridfs' 28 | params = {'storage_type' => invalid_type} 29 | err_msg = "Only s3 output type is supported: #{invalid_type}" 30 | stub_api_request(:post, "/v3/export/run/#{e(db_name)}/#{e(table_name)}").with(:body => params). 31 | to_return(:status => 400, :body => {'message' => err_msg}.to_json) 32 | 33 | expect { 34 | api.export(db_name, table_name, invalid_type) 35 | }.to raise_error(TreasureData::APIError, /#{err_msg}/) 36 | end 37 | 38 | # TODO: Add other parameters spec 39 | end 40 | 41 | describe 'result_export' do 42 | it 'should export result successfully' do 43 | params = {'result' => 'mysql://user:pass@host.com/database/table'} 44 | stub_api_request(:post, "/v3/job/result_export/100").with(:body => params). 45 | to_return(:body => {'job_id' => '101'}.to_json) 46 | 47 | expect(api.result_export(100, params)).to eq('101') 48 | end 49 | end 50 | end 51 | 52 | -------------------------------------------------------------------------------- /spec/td/client/import_api_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/client/spec_resources' 3 | require 'json' 4 | require 'tempfile' 5 | 6 | describe 'Import API' do 7 | include_context 'spec symbols' 8 | include_context 'common helper' 9 | 10 | let :api do 11 | API.new(nil, :endpoint => endpoint) 12 | end 13 | 14 | let :api_old do 15 | API.new(nil, :endpoint => endpoint_old) 16 | end 17 | 18 | let :api_default do 19 | API.new(nil) 20 | end 21 | 22 | let :api_default_http do 23 | API.new(nil, :ssl => false) 24 | end 25 | 26 | let :api_unknown_host do 27 | API.new(nil, :endpoint => endpoint_unknown) 28 | end 29 | 30 | let :api_unknown_host_http do 31 | API.new(nil, :endpoint => endpoint_unknown, :ssl => false) 32 | end 33 | 34 | let(:endpoint) { 'api.treasuredata.com' } 35 | let(:endpoint_old) { TreasureData::API::OLD_ENDPOINT } 36 | let(:endpoint_unknown) { "example.com" } 37 | let(:endpoint_import) { "api-import.treasuredata.com" } 38 | let(:endpoint_import_old) { "api-import.treasure-data.com" } 39 | let(:endpoint_import_unknown) { endpoint_unknown } 40 | 41 | describe 'import' do 42 | it 'runs with unique_id' do 43 | t = Tempfile.new('import_api_spec') 44 | File.open(t.path, 'w') do |f| 45 | f << '12345' 46 | end 47 | stub_request(:put, "https://#{endpoint_import}/v3/table/import_with_id/db/table/unique_id/format"). 48 | with(:body => '12345'). 49 | to_return(:status => 200) 50 | File.open(t.path) do |f| 51 | expect(api.import('db', 'table', 'format', f, 5, 'unique_id')).to eq(true) 52 | end 53 | end 54 | 55 | it 'runs without unique_id' do 56 | t = Tempfile.new('import_api_spec') 57 | File.open(t.path, 'w') do |f| 58 | f << '12345' 59 | end 60 | stub_request(:put, "https://#{endpoint_import}/v3/table/import/db/table/format"). 61 | with(:body => '12345'). 62 | to_return(:status => 200) 63 | File.open(t.path) do |f| 64 | expect(api.import('db', 'table', 'format', f, 5)).to eq(true) 65 | end 66 | end 67 | 68 | it 'runs for old endpoint (force "http" instead of "https" for compatibility)' do 69 | t = Tempfile.new('import_api_spec') 70 | File.open(t.path, 'w') do |f| 71 | f << '12345' 72 | end 73 | stub_request(:put, "http://#{endpoint_import_old}/v3/table/import/db/table/format"). 74 | with(:body => '12345'). 75 | to_return(:status => 200) 76 | File.open(t.path) do |f| 77 | expect(api_old.import('db', 'table', 'format', f, 5)).to eq(true) 78 | end 79 | end 80 | 81 | it 'runs for no endpoint specified (default behavior)' do 82 | t = Tempfile.new('import_api_spec') 83 | File.open(t.path, 'w') do |f| 84 | f << '12345' 85 | end 86 | stub_request(:put, "https://#{endpoint_import}/v3/table/import/db/table/format"). 87 | with(:body => '12345'). 88 | to_return(:status => 200) 89 | File.open(t.path) do |f| 90 | expect(api_default.import('db', 'table', 'format', f, 5)).to eq true 91 | end 92 | end 93 | 94 | it 'runs for no endpoint specified with ssl: false' do 95 | t = Tempfile.new('import_api_spec') 96 | File.open(t.path, 'w') do |f| 97 | f << '12345' 98 | end 99 | stub_request(:put, "http://#{endpoint_import}/v3/table/import/db/table/format"). 100 | with(:body => '12345'). 101 | to_return(:status => 200) 102 | File.open(t.path) do |f| 103 | expect(api_default_http.import('db', 'table', 'format', f, 5)).to eq true 104 | end 105 | end 106 | 107 | it 'runs for unknown endpoint specified' do 108 | t = Tempfile.new('import_api_spec') 109 | File.open(t.path, 'w') do |f| 110 | f << '12345' 111 | end 112 | stub_request(:put, "https://#{endpoint_unknown}/v3/table/import/db/table/format"). 113 | with(:body => '12345'). 114 | to_return(:status => 200) 115 | File.open(t.path) do |f| 116 | expect(api_unknown_host.import('db', 'table', 'format', f, 5)).to eq true 117 | end 118 | end 119 | 120 | it 'runs for unknown endpoint with ssl=false specified' do 121 | t = Tempfile.new('import_api_spec') 122 | File.open(t.path, 'w') do |f| 123 | f << '12345' 124 | end 125 | stub_request(:put, "http://#{endpoint_unknown}/v3/table/import/db/table/format"). 126 | with(:body => '12345'). 127 | to_return(:status => 200) 128 | File.open(t.path) do |f| 129 | expect(api_unknown_host_http.import('db', 'table', 'format', f, 5)).to eq true 130 | end 131 | end 132 | 133 | it 'raises APIError' do 134 | t = Tempfile.new('import_api_spec') 135 | File.open(t.path, 'w') do |f| 136 | f << '12345' 137 | end 138 | stub_request(:put, "https://#{endpoint_import}/v3/table/import/db/table/format"). 139 | with(:body => '12345'). 140 | to_return(:status => 500) 141 | File.open(t.path) do |f| 142 | expect { 143 | api.import('db', 'table', 'format', f, 5) 144 | }.to raise_error(TreasureData::APIError) 145 | end 146 | end 147 | end 148 | end 149 | -------------------------------------------------------------------------------- /spec/td/client/model_job_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/client/spec_resources' 3 | 4 | describe 'Job Model' do 5 | include_context 'spec symbols' 6 | include_context 'common helper' 7 | include_context 'job resources' 8 | 9 | before do 10 | stub_api_request(:post, "/v3/user/authenticate"). 11 | to_return(:body => {'apikey' => 'apikey'}.to_json) 12 | end 13 | 14 | describe '#client' do 15 | subject do 16 | Job.new(client, *arguments).client 17 | end 18 | 19 | let :client do 20 | Client.authenticate('user', 'password') 21 | end 22 | 23 | let :arguments do 24 | job_attributes = raw_jobs.first 25 | [ 26 | 'job_id', 'type', 'query', 'status', 'url', 'debug', 27 | 'start_at', 'end_at', 'cpu_time', 'result_size', 'result', 'result_url', 28 | 'hive_result_schema', 'priority', 'retry_limit', 'org_name', 'db_name', 29 | 'duration', 'num_records' 30 | ].map {|name| job_attributes[name]} 31 | end 32 | 33 | it 'returns Job object having client' do 34 | expect(subject).to eq client 35 | end 36 | end 37 | 38 | describe '#auto_update_status' do 39 | let(:client) { Client.authenticate('user', 'password') } 40 | let(:job_id) { 12345678 } 41 | let(:job) { Job.new(client, job_id, nil, nil) } 42 | let(:format) { 'json' } 43 | let(:io) { StringIO.new } 44 | before { allow(job).to receive(:finished?) { false } } 45 | 46 | it 'can set' do 47 | expect(job.auto_update_status?).to eq true 48 | job.auto_update_status = false 49 | expect(job.auto_update_status?).to eq false 50 | job.auto_update_status = true 51 | expect(job.auto_update_status?).to eq true 52 | end 53 | 54 | it 'calls API if auto_update_status=true' do 55 | job.auto_update_status = true 56 | result_job = { 57 | 'job_id' => job_id, 58 | 'status' => 'queued', 59 | 'created_at' => Time.now, 60 | } 61 | stub_request(:get, "https://api.treasuredata.com/v3/job/show/#{job_id}"). 62 | to_return(:body => result_job.to_json) 63 | expect(job.query).to be_nil 64 | expect(job.status).to eq "queued" 65 | expect(job.url).to be_nil 66 | expect(job.debug).to be_nil 67 | expect(job.start_at).to be_nil 68 | expect(job.end_at).to be_nil 69 | expect(job.cpu_time).to be_nil 70 | expect(job.hive_result_schema).to be_nil 71 | expect(job.result_size).to be_nil 72 | end 73 | 74 | it "doesn't call API if auto_update_status=false" do 75 | job.auto_update_status = false 76 | expect(job.query).to be_nil 77 | expect(job.status).to be_nil 78 | expect(job.url).to be_nil 79 | expect(job.debug).to be_nil 80 | expect(job.start_at).to be_nil 81 | expect(job.end_at).to be_nil 82 | expect(job.cpu_time).to be_nil 83 | expect(job.hive_result_schema).to be_nil 84 | expect(job.result_size).to be_nil 85 | end 86 | end 87 | 88 | describe '#result_raw' do 89 | let(:client) { Client.authenticate('user', 'password') } 90 | let(:job_id) { 12345678 } 91 | let(:job) { Job.new(client, job_id, nil, nil) } 92 | let(:format) { 'json' } 93 | let(:io) { StringIO.new } 94 | 95 | context 'not finished?' do 96 | before { allow(job).to receive(:finished?) { false } } 97 | 98 | it 'do not call #job_result_raw' do 99 | expect(client).not_to receive(:job_result_raw) 100 | 101 | expect(job.result_raw(format, io)).to_not be 102 | end 103 | end 104 | 105 | context 'finished?' do 106 | before { allow(job).to receive(:finished?) { true } } 107 | 108 | it 'call #job_result_raw' do 109 | expect(client).to receive(:job_result_raw).with(job_id, format, io) 110 | 111 | job.result_raw(format, io) 112 | end 113 | end 114 | end 115 | 116 | describe '#wait' do 117 | let(:client) { Client.authenticate('user', 'password') } 118 | let(:job_id) { 12345678 } 119 | let(:job) { Job.new(client, job_id, nil, nil) } 120 | 121 | def change_job_status(status) 122 | allow(client).to receive(:job_status).with(job_id).and_return(status) 123 | end 124 | 125 | before do 126 | change_job_status(Job::STATUS_QUEUED) 127 | end 128 | 129 | context 'without timeout' do 130 | it 'waits the job to be finished' do 131 | begin 132 | thread = Thread.start { job.wait } 133 | expect(thread).to be_alive 134 | change_job_status(Job::STATUS_SUCCESS) 135 | thread.join(1) 136 | expect(thread).to be_stop 137 | ensure 138 | thread.kill # just in case 139 | end 140 | end 141 | 142 | it 'calls a given block in every wait_interval second' do 143 | # Let's try disable stubbing #sleep for now 144 | # now = 1_400_000_000 145 | # allow(self).to receive(:sleep){|arg| now += arg } 146 | # allow(Process).to receive(:clock_gettime){ now } 147 | expect { |b| 148 | begin 149 | thread = Thread.start { 150 | job.wait(nil, 0.1, &b) 151 | } 152 | sleep 0.3 153 | change_job_status(Job::STATUS_SUCCESS) 154 | thread.join(1) 155 | expect(thread).to be_stop 156 | ensure 157 | thread.kill # just in case 158 | end 159 | }.to yield_control.at_least(2).at_most(3).times 160 | end 161 | end 162 | 163 | context 'with timeout' do 164 | context 'the job running time is too long' do 165 | it 'raise Timeout::Error' do 166 | expect { 167 | job.wait(0.1) 168 | }.to raise_error(Timeout::Error) 169 | end 170 | end 171 | 172 | it 'calls a given block in every wait_interval second, and timeout' do 173 | expect { |b| 174 | begin 175 | thread = Thread.start { 176 | job.wait(0.3, 0.1, &b) 177 | } 178 | expect{ thread.value }.to raise_error(Timeout::Error) 179 | expect(thread).to be_stop 180 | ensure 181 | thread.kill # just in case 182 | end 183 | }.to yield_control.at_least(2).times 184 | end 185 | end 186 | end 187 | end 188 | -------------------------------------------------------------------------------- /spec/td/client/model_schedule_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/client/spec_resources' 3 | 4 | describe 'Schedule Model' do 5 | describe '#run' do 6 | let(:api_key) { '1234567890abcd' } 7 | let(:api) { double(:api) } 8 | let(:client) { Client.new(api_key) } 9 | let(:name) { 'schedule' } 10 | let(:schedule) { 11 | Schedule.new(client, name, '0 0 * * * *', 'select 1') 12 | } 13 | let(:time) { "2013-01-01 00:00:00" } 14 | let(:num) { 1 } 15 | 16 | before do 17 | allow(API).to receive(:new).with(api_key, {}).and_return(api) 18 | end 19 | 20 | it 'success call api' do 21 | expect(api).to receive(:run_schedule).with(name, time, num).and_return([]) 22 | 23 | schedule.run(time, num) 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /spec/td/client/model_schema_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/client/spec_resources' 3 | 4 | describe 'TreasureData::Schema::Field' do 5 | describe '.new' do 6 | context 'name="time"' do 7 | it 'raises ParameterValidationError' do 8 | expect{ Schema::Field.new('time', 'int') }.to raise_error(ParameterValidationError) 9 | end 10 | end 11 | context 'name with UTF-8' do 12 | it 'works' do 13 | name = "\u3042\u3044\u3046" 14 | f = Schema::Field.new(name, 'int') 15 | expect(f.name).to eq name 16 | expect(f.type).to eq 'int' 17 | expect(f.sql_alias).to be_nil 18 | end 19 | end 20 | context 'with sql_alias' do 21 | it 'raises' do 22 | f = Schema::Field.new('t:t', 'int', 'alice') 23 | expect(f.name).to eq 't:t' 24 | expect(f.type).to eq 'int' 25 | expect(f.sql_alias).to eq 'alice' 26 | end 27 | end 28 | context 'with sql_alias which equals to its name' do 29 | it 'works' do 30 | name = "abc" 31 | f = Schema::Field.new(name, 'int', name) 32 | expect(f.name).to eq name 33 | expect(f.type).to eq 'int' 34 | expect(f.sql_alias).to eq name 35 | end 36 | end 37 | context 'with invalid sql_alias' do 38 | it 'raises' do 39 | expect{ Schema::Field.new('t:t', 'int', 't:t') }.to raise_error(ParameterValidationError) 40 | end 41 | end 42 | end 43 | end 44 | 45 | describe 'TreasureData::Schema' do 46 | describe '.parse' do 47 | let(:columns){ ["foo:int", "BAR\u3070\u30FC:string@bar", "baz:baz!:array@baz"] } 48 | it do 49 | sc = Schema.parse(columns) 50 | expect(sc.fields.size).to eq 3 51 | expect(sc.fields[0].name).to eq 'foo' 52 | expect(sc.fields[0].type).to eq 'int' 53 | expect(sc.fields[0].sql_alias).to be_nil 54 | expect(sc.fields[1].name).to eq "BAR\u3070\u30FC" 55 | expect(sc.fields[1].type).to eq 'string' 56 | expect(sc.fields[1].sql_alias).to eq 'bar' 57 | expect(sc.fields[2].name).to eq 'baz:baz!' 58 | expect(sc.fields[2].type).to eq 'array' 59 | expect(sc.fields[2].sql_alias).to eq 'baz' 60 | end 61 | end 62 | 63 | describe '.new' do 64 | it 'works with single field' do 65 | f = Schema::Field.new('a', 'int') 66 | sc = Schema.new([f]) 67 | expect(sc.fields[0]).to eq f 68 | end 69 | it 'works with multiple fields' do 70 | f0 = Schema::Field.new('a', 'int') 71 | f1 = Schema::Field.new('b', 'int', 'b') 72 | sc = Schema.new([f0, f1]) 73 | expect(sc.fields[0]).to eq f0 74 | expect(sc.fields[1]).to eq f1 75 | end 76 | it 'raises' do 77 | f0 = Schema::Field.new('a', 'int') 78 | f1 = Schema::Field.new('b', 'int', 'a') 79 | expect{ Schema.new([f0, f1]) }.to raise_error(ArgumentError) 80 | end 81 | end 82 | 83 | describe '#fields' do 84 | it do 85 | f = Schema::Field.new('a', 'int') 86 | sc = Schema.new([f]) 87 | expect(sc.fields[0]).to eq f 88 | end 89 | end 90 | 91 | describe '#add_field' do 92 | it do 93 | f = Schema::Field.new('a', 'int') 94 | sc = Schema.new([f]) 95 | sc.add_field('b', 'double', 'bb') 96 | expect(sc.fields[1].name).to eq 'b' 97 | end 98 | it do 99 | f = Schema::Field.new('a', 'int') 100 | sc = Schema.new([f]) 101 | sc.add_field('b', 'double', 'b') 102 | expect(sc.fields[1].name).to eq 'b' 103 | end 104 | it 'raises ParameterValidationError if name is duplicated' do 105 | f = Schema::Field.new('a', 'int') 106 | sc = Schema.new([f]) 107 | expect{ sc.add_field('a', 'double') }.to raise_error(ParameterValidationError) 108 | end 109 | it 'raises ParameterValidationError if sql_alias is duplicated' do 110 | f = Schema::Field.new('a', 'int') 111 | sc = Schema.new([f]) 112 | expect{ sc.add_field('abc', 'double', 'a') }.to raise_error(ParameterValidationError) 113 | end 114 | end 115 | 116 | describe '#merge' do 117 | it do 118 | sc1 = Schema.parse(['foo:int', 'bar:float']) 119 | sc2 = Schema.parse(['bar:double', 'baz:string']) 120 | sc3 = sc1.merge(sc2) 121 | expect(sc3.fields.size).to eq 3 122 | expect(sc3.fields[0].name).to eq 'foo' 123 | expect(sc3.fields[0].type).to eq 'int' 124 | expect(sc3.fields[1].name).to eq 'bar' 125 | expect(sc3.fields[1].type).to eq 'double' 126 | expect(sc3.fields[2].name).to eq 'baz' 127 | expect(sc3.fields[2].type).to eq 'string' 128 | end 129 | it do 130 | sc1 = Schema.parse(['foo:int', 'bar:float']) 131 | sc2 = Schema.parse(['bar:double@foo']) 132 | expect{ sc1.merge(sc2) }.to raise_error(ArgumentError) 133 | end 134 | end 135 | 136 | describe '#to_json' do 137 | it do 138 | sc = Schema.parse(['foo:int', 'bar:float@baz']) 139 | expect(sc.to_json).to eq '[["foo","int"],["bar","float","baz"]]' 140 | end 141 | end 142 | 143 | describe '#from_json' do 144 | it do 145 | sc = Schema.new 146 | sc.from_json [["foo","int"],["bar","float","baz"]] 147 | expect(sc.fields.size).to eq 2 148 | expect(sc.fields[0].name).to eq 'foo' 149 | expect(sc.fields[0].type).to eq 'int' 150 | expect(sc.fields[0].sql_alias).to be_nil 151 | expect(sc.fields[1].name).to eq 'bar' 152 | expect(sc.fields[1].type).to eq 'float' 153 | expect(sc.fields[1].sql_alias).to eq 'baz' 154 | end 155 | end 156 | end 157 | -------------------------------------------------------------------------------- /spec/td/client/result_api_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/client/spec_resources' 3 | 4 | describe 'Result API' do 5 | include_context 'spec symbols' 6 | include_context 'common helper' 7 | 8 | let :api do 9 | API.new(nil) 10 | end 11 | 12 | describe 'create_result' do 13 | it 'should create a new result' do 14 | params = {'url' => result_url} 15 | stub_api_request(:post, "/v3/result/create/#{e(result_name)}").with(:body => params).to_return(:body => {'result' => result_name}.to_json) 16 | 17 | expect(api.create_result(result_name, result_url)).to be true 18 | end 19 | 20 | it 'should return 422 error with invalid name' do 21 | name = '1' 22 | params = {'url' => result_url} 23 | err_msg = "Validation failed: Name is too short" # " (minimum is 3 characters)" 24 | stub_api_request(:post, "/v3/result/create/#{e(name)}").with(:body => params). 25 | to_return(:status => 422, :body => {'message' => err_msg}.to_json) 26 | 27 | expect { 28 | api.create_result(name, result_url) 29 | }.to raise_error(TreasureData::APIError, /#{err_msg}/) 30 | end 31 | 32 | it 'should return 422 error without url' do 33 | params = {'url' => 'false'} # I want to use nil, but nil doesn't work on WebMock... 34 | err_msg = "'url' parameter is required" 35 | stub_api_request(:post, "/v3/result/create/#{e(result_name)}").with(:body => params). 36 | to_return(:status => 422, :body => {'message' => err_msg}.to_json) 37 | 38 | expect { 39 | api.create_result(result_name, false) 40 | }.to raise_error(TreasureData::APIError, /#{err_msg}/) 41 | end 42 | 43 | it 'should return 409 error with duplicated name' do 44 | params = {'url' => result_url} 45 | err_msg = "Result must be unique" 46 | stub_api_request(:post, "/v3/result/create/#{e(result_name)}").with(:body => params). 47 | to_return(:status => 409, :body => {'message' => err_msg}.to_json) 48 | 49 | expect { 50 | api.create_result(result_name, result_url) 51 | }.to raise_error(TreasureData::APIError, /#{err_msg}/) 52 | end 53 | end 54 | 55 | describe 'list_result' do 56 | it 'should return name and url' do 57 | stub_api_request(:get, '/v3/result/list'). 58 | to_return(:body => {'results' => [{'name' => 'name', 'url' => 'url'}]}.to_json) 59 | expect(api.list_result).to eq([['name', 'url', nil]]) 60 | end 61 | end 62 | 63 | describe 'delete_result' do 64 | it 'should delete the result' do 65 | stub_api_request(:post, "/v3/result/delete/#{e(result_name)}") 66 | expect(api.delete_result(result_name)).to eq(true) 67 | end 68 | 69 | it 'should raise error' do 70 | stub_api_request(:post, "/v3/result/delete/#{e(result_name)}"). 71 | to_return(:status => 404) 72 | expect { 73 | api.delete_result(result_name) 74 | }.to raise_error(TreasureData::APIError) 75 | end 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /spec/td/client/sched_api_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/client/spec_resources' 3 | 4 | describe 'Schedule API' do 5 | include_context 'spec symbols' 6 | include_context 'common helper' 7 | 8 | let :api do 9 | API.new(nil) 10 | end 11 | 12 | describe 'create_schedule' do 13 | let :opts do 14 | {'cron' => cron, 'query' => query, 'database' => db_name} 15 | end 16 | 17 | it 'should create a new schedule' do 18 | start = Time.now 19 | stub_api_request(:post, "/v3/schedule/create/#{e(sched_name)}"). 20 | with(:body => opts.merge('type' => 'hive')). 21 | to_return(:body => {'name' => sched_name, 'start' => start.to_s}.to_json) 22 | 23 | expect(api.create_schedule(sched_name, opts.merge('type' => 'hive'))).to eq(start.to_s) 24 | end 25 | 26 | it 'should create a dummy schedule' do 27 | stub_api_request(:post, "/v3/schedule/create/#{e(sched_name)}"). 28 | with(:body => opts.merge('type' => 'hive')). 29 | to_return(:body => {'name' => sched_name, 'start' => nil}.to_json) 30 | 31 | expect(api.create_schedule(sched_name, opts.merge('type' => 'hive'))).to be_nil 32 | end 33 | 34 | it 'should return 422 error with invalid name' do 35 | name = '1' 36 | err_msg = "Validation failed: Name is too short" # " (minimum is 3 characters)" 37 | stub_api_request(:post, "/v3/schedule/create/#{e(name)}"). 38 | with(:body => opts.merge('type' => 'hive')). 39 | to_return(:status => 422, :body => {'message' => err_msg}.to_json) 40 | 41 | expect { 42 | api.create_schedule(name, opts.merge('type' => 'hive')) 43 | }.to raise_error(TreasureData::APIError, /#{err_msg}/) 44 | end 45 | end 46 | 47 | describe 'delete_schedule' do 48 | it 'should delete the schedule' do 49 | stub_api_request(:post, "/v3/schedule/delete/#{e(sched_name)}"). 50 | to_return(:body => {'cron' => 'cron', 'query' => 'query'}.to_json) 51 | expect(api.delete_schedule(sched_name)).to eq(['cron', 'query']) 52 | end 53 | end 54 | 55 | describe 'update_schedule' do 56 | let :pig_query do 57 | "OUT = FOREACH (GROUP plt364 ALL) GENERATE COUNT(plt364);\n" * 200 58 | end 59 | let :opts do 60 | {'cron' => cron, 'query' => pig_query, 'database' => db_name} 61 | end 62 | 63 | it 'should not return 414 even if the query text is very long' do 64 | stub_api_request(:post, "/v3/schedule/update/#{e(sched_name)}"). 65 | with(:body => opts.merge('type' => 'pig')). 66 | to_return(:body => {'name' => sched_name, 'query' => pig_query}.to_json) 67 | 68 | expect { 69 | api.update_schedule(sched_name, opts.merge('type' => 'pig')) 70 | }.not_to raise_error 71 | end 72 | 73 | it 'should update the schedule with the new query' do 74 | stub_api_request(:post, "/v3/schedule/update/#{e(sched_name)}"). 75 | with(:body => opts.merge('type' => 'pig')). 76 | to_return(:body => {'name' => sched_name, 'query' => pig_query}.to_json) 77 | 78 | stub_api_request(:get, "/v3/schedule/list"). 79 | to_return(:body => {'schedules' => [{'name' => sched_name, 'query' => pig_query}]}.to_json) 80 | 81 | expect(api.list_schedules.first[2]).to eq(pig_query) 82 | end 83 | end 84 | 85 | describe 'history' do 86 | let :history do 87 | ['history', 'job_id', 'type', 'database', 'status', 'query', 'start_at', 'end_at', 'result', 'priority'].inject({}) { |r, e| 88 | r[e] = e 89 | r 90 | } 91 | end 92 | 93 | it 'should return history records' do 94 | stub_api_request(:get, "/v3/schedule/history/#{e(sched_name)}"). 95 | with(:query => {'from' => 0, 'to' => 100}). 96 | to_return(:body => {'history' => [history]}.to_json) 97 | expect(api.history(sched_name, 0, 100)).to eq([[nil, 'job_id', :type, 'status', 'query', 'start_at', 'end_at', 'result', 'priority', 'database']]) 98 | end 99 | end 100 | 101 | describe 'run_schedule' do 102 | it 'should return history records' do 103 | stub_api_request(:post, "/v3/schedule/run/#{e(sched_name)}/123456789"). 104 | with(:body => {'num' => '5'}). 105 | to_return(:body => {'jobs' => [{'job_id' => 'job_id', 'scheduled_at' => 'scheduled_at', 'type' => 'type'}]}.to_json) 106 | expect(api.run_schedule(sched_name, 123456789, 5)).to eq([['job_id', :type, 'scheduled_at']]) 107 | end 108 | end 109 | end 110 | -------------------------------------------------------------------------------- /spec/td/client/server_status_api_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/client/spec_resources' 3 | 4 | describe 'ServerStatus API' do 5 | include_context 'spec symbols' 6 | include_context 'common helper' 7 | 8 | let :api do 9 | API.new(nil, {:max_cumul_retry_delay => -1}) 10 | end 11 | 12 | describe 'server_status' do 13 | it 'returns status' do 14 | stub_api_request(:get, '/v3/system/server_status'). 15 | to_return(:body => {'status' => 'OK'}.to_json) 16 | expect(api.server_status).to eq('OK') 17 | end 18 | 19 | it 'returns error description' do 20 | stub_api_request(:get, '/v3/system/server_status'). 21 | to_return(:status => 500) 22 | expect(api.server_status).to eq('Server is down (500)') 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /spec/td/client/spec_resources.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/client/model' 3 | 4 | shared_context 'spec symbols' do 5 | let :apikey do 6 | '1/0123456789ABCDEFG' 7 | end 8 | 9 | let :db_name do 10 | 'db_test' 11 | end 12 | 13 | let :table_name do 14 | 'table_test' 15 | end 16 | 17 | let :sched_name do 18 | 'sched test' 19 | end 20 | 21 | let :result_name do 22 | 'test' 23 | end 24 | 25 | let :bi_name do 26 | 'bi_test' 27 | end 28 | 29 | let :cron do 30 | '* * * * *' 31 | end 32 | 33 | let :query do 34 | 'select 1' 35 | end 36 | let :result_url do 37 | 'td://@/test/table' 38 | end 39 | end 40 | 41 | shared_context 'database resources' do 42 | include_context 'common helper' 43 | 44 | let :db_names do 45 | [ 46 | 'cloud', 'yuffie', 'vincent', 'cid' 47 | ] 48 | end 49 | end 50 | 51 | shared_context 'job resources' do 52 | include_context 'database resources' 53 | 54 | MAX_JOB ||= 20 55 | 56 | let :job_types do 57 | [ 58 | ['HiveJob', 'hive'], 59 | ['ExportJob', 'export'], 60 | ['BulkImportJob', 'bulk_import'] 61 | ] 62 | end 63 | 64 | let :raw_jobs do 65 | created_at = Time.at(1356966000) 66 | types = job_types 67 | dbs = db_names 68 | (0...MAX_JOB).map { |i| 69 | job_type = types[i % types.size] 70 | status = i.odd? ? 'success' : 'error' 71 | { 72 | "job_id" => i, 73 | "url" => "https://console.treasure-data.com/jobs/#{i.to_s}?target=query", 74 | "database" => dbs[i % dbs.size].to_s, 75 | "status" => status, 76 | "type" => job_type[0].to_sym, 77 | "query" => "select #{i}", 78 | "priority" => i % 3, 79 | "result" => nil, 80 | "created_at" => created_at.to_s, 81 | "updated_at" => (created_at + (i * 10)).to_s, 82 | "start_at" => (created_at + (i * 10 * 60)).to_s, 83 | "end_at" => (created_at + (i * 10 * 3600)).to_s, 84 | "cpu_time" => i * 100 + i, 85 | "result_size" => i * 1000, 86 | 'retry_limit' => 10, 87 | "duration" => i, 88 | "num_records" => i * 1000, 89 | 'organization' => nil, 90 | 'hive_result_schema' => nil, 91 | 'debug' => { 92 | 'stderr' => "job #{i} #{status}", 93 | 'cmdout' => "job #{i} command", 94 | } 95 | } 96 | } 97 | end 98 | end 99 | -------------------------------------------------------------------------------- /spec/td/client/table_api_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/client/spec_resources' 3 | 4 | describe 'Table API' do 5 | include_context 'spec symbols' 6 | include_context 'common helper' 7 | 8 | let :api do 9 | API.new(nil) 10 | end 11 | 12 | let :client do 13 | Client.new(apikey) 14 | end 15 | 16 | describe "'create_log_table' API" do 17 | it 'should return 404 error if the database does not exist' do 18 | err_msg = "Create log table failed: Couldn't find UserDatabase with name = #{db_name}" 19 | stub_api_request(:post, "/v3/table/create/#{e db_name}/#{e(table_name)}/log"). 20 | to_return(:status => 404, :body => {'message' => err_msg}.to_json) 21 | 22 | expect { 23 | api.create_log_table(db_name, table_name) 24 | }.to raise_error(TreasureData::NotFoundError, /#{err_msg}/) 25 | end 26 | 27 | it 'should create a new table if the database exists' do 28 | stub_api_request(:post, "/v3/table/create/#{e db_name}/#{e(table_name)}/log"). 29 | to_return(:body => {'database' => db_name, 'table' => table_name, 'type' => 'log'}.to_json) 30 | expect(api.create_log_table(db_name, table_name)).to be true 31 | end 32 | 33 | it 'should create a new table with params' do 34 | stub_api_request(:post, "/v3/table/create/#{e db_name}/#{e(table_name)}/log"). 35 | with(:body => {'include_v' => 'false'}). 36 | to_return(:body => {'database' => db_name, 'table' => table_name, 'type' => 'log', 'include_v' => 'false'}.to_json) 37 | expect(api.create_log_table(db_name, table_name, include_v: false)).to be true 38 | end 39 | 40 | it 'should return 400 error with invalid name' do 41 | invalid_name = 'a' 42 | err_msg = "Name must be 3 to 256 characters, got #{invalid_name.length} characters. name = '#{invalid_name}'" 43 | stub_api_request(:post, "/v3/table/create/#{e db_name}/#{e invalid_name}/log"). 44 | to_return(:status => 400, :body => {'message' => err_msg}.to_json) 45 | 46 | expect { 47 | api.create_log_table(db_name, invalid_name) 48 | }.to raise_error(TreasureData::APIError, /#{err_msg}/) 49 | end 50 | 51 | it 'should return 409 error with duplicated name' do 52 | err_msg = "Table #{table_name} already exists" 53 | stub_api_request(:post, "/v3/table/create/#{e db_name}/#{e table_name}/log"). 54 | to_return(:status => 409, :body => {'message' => err_msg}.to_json) 55 | 56 | expect { 57 | api.create_log_table(db_name, table_name) 58 | }.to raise_error(TreasureData::AlreadyExistsError, /#{err_msg}/) 59 | end 60 | end 61 | 62 | describe "'create_log_table' client API" do 63 | it 'should return 404 error if the database does not exist' do 64 | err_msg = "Create log table failed: Couldn't find UserDatabase with name = #{db_name}" 65 | stub_api_request(:post, "/v3/table/create/#{e db_name}/#{e(table_name)}/log"). 66 | to_return(:status => 404, :body => {'message' => err_msg}.to_json) 67 | 68 | expect { 69 | client.create_log_table(db_name, table_name) 70 | }.to raise_error(TreasureData::NotFoundError, /#{err_msg}/) 71 | end 72 | 73 | it 'should create a new table if the database exists' do 74 | stub_api_request(:post, "/v3/table/create/#{e db_name}/#{e(table_name)}/log"). 75 | to_return(:body => {'database' => db_name, 'table' => table_name, 'type' => 'log'}.to_json) 76 | expect(client.create_log_table(db_name, table_name)).to be true 77 | end 78 | 79 | it 'should create a new table with params' do 80 | stub_api_request(:post, "/v3/table/create/#{e db_name}/#{e(table_name)}/log"). 81 | with(:body => {'include_v' => 'false'}). 82 | to_return(:body => {'database' => db_name, 'table' => table_name, 'type' => 'log', 'include_v' => 'false'}.to_json) 83 | expect(client.create_log_table(db_name, table_name, include_v: false)).to be true 84 | end 85 | 86 | it 'should return 400 error with invalid name' do 87 | invalid_name = 'a' 88 | err_msg = "Name must be 3 to 256 characters, got #{invalid_name.length} characters. name = '#{invalid_name}'" 89 | stub_api_request(:post, "/v3/table/create/#{e db_name}/#{e invalid_name}/log"). 90 | to_return(:status => 400, :body => {'message' => err_msg}.to_json) 91 | 92 | expect { 93 | client.create_log_table(db_name, invalid_name) 94 | }.to raise_error(TreasureData::APIError, /#{err_msg}/) 95 | end 96 | 97 | it 'should return 409 error with duplicated name' do 98 | err_msg = "Table #{table_name} already exists" 99 | stub_api_request(:post, "/v3/table/create/#{e db_name}/#{e table_name}/log"). 100 | to_return(:status => 409, :body => {'message' => err_msg}.to_json) 101 | 102 | expect { 103 | client.create_log_table(db_name, table_name) 104 | }.to raise_error(TreasureData::AlreadyExistsError, /#{err_msg}/) 105 | end 106 | end 107 | 108 | describe "'list_tables' API" do 109 | it 'should list the tables in a Hash whose values include type, count, created_at, updated_at, schema, ...' do 110 | tables = [ 111 | ["table_1", "item", "[[\"time\",\"long\"],[\"value\",\"string\"]]", 111, "2013-01-21 01:51:41 UTC", "2014-01-21 01:51:41 UTC"], 112 | ["table_2", "log", "[[\"time\",\"long\"],[\"value\",\"long\"]]", 222, "2013-02-22 02:52:42 UTC", "2014-02-22 02:52:42 UTC"], 113 | ["table_3", "item", "[[\"time\",\"long\"],[\"value\",\"string\"]]", 333, "2013-03-23 03:53:43 UTC", "2014-03-23 03:53:43 UTC"], 114 | ["table_4", "log", "[[\"time\",\"long\"],[\"value\",\"long\"]]", 444, "2013-04-24 04:54:44 UTC", "2014-04-24 04:54:44 UTC"] 115 | ] 116 | stub_api_request(:get, "/v3/table/list/#{e db_name}"). 117 | to_return(:body => {'tables' => [ 118 | {'name' => tables[0][0], 'type' => tables[0][1], 'schema' => tables[0][2], 'count' => tables[0][3], 'created_at' => tables[0][4], 'updated_at' => tables[0][5]}, 119 | {'name' => tables[1][0], 'type' => tables[1][1], 'schema' => tables[1][2], 'count' => tables[1][3], 'created_at' => tables[1][4], 'updated_at' => tables[1][5]}, 120 | {'name' => tables[2][0], 'type' => tables[2][1], 'schema' => tables[2][2], 'count' => tables[2][3], 'created_at' => tables[2][4], 'updated_at' => tables[2][5]}, 121 | {'name' => tables[3][0], 'type' => tables[3][1], 'schema' => tables[3][2], 'count' => tables[3][3], 'created_at' => tables[3][4], 'updated_at' => tables[3][5]} 122 | ]}.to_json) 123 | 124 | table_list = api.list_tables(db_name) 125 | tables.each {|table| 126 | expect(table_list[table[0]][0]).to eq(table[1].to_sym) 127 | expect(table_list[table[0]][1]).to eq(JSON.parse(table[2])) 128 | expect(table_list[table[0]][2]).to eq(table[3]) 129 | expect(table_list[table[0]][3]).to eq(table[4]) 130 | expect(table_list[table[0]][4]).to eq(table[5]) 131 | } 132 | end 133 | end 134 | 135 | describe "'tables' Client API" do 136 | it 'should return an array of Table objects' do 137 | tables = [ 138 | ["table_1", "item", "[[\"value\",\"string\"]]", 111, "2013-01-21 01:51:41 UTC", "2014-01-21 01:51:41 UTC"], 139 | ["table_2", "log", "[[\"value\",\"long\"]]", 222, "2013-02-22 02:52:42 UTC", "2014-02-22 02:52:42 UTC"], 140 | ["table_3", "item", "[[\"value\",\"string\"]]", 333, "2013-03-23 03:53:43 UTC", "2014-03-23 03:53:43 UTC"], 141 | ["table_4", "log", "[[\"value\",\"long\"]]", 444, "2013-04-24 04:54:44 UTC", "2014-04-24 04:54:44 UTC"] 142 | ] 143 | stub_api_request(:get, "/v3/table/list/#{e db_name}"). 144 | to_return(:body => {'tables' => [ 145 | {'name' => tables[0][0], 'type' => tables[0][1], 'schema' => tables[0][2], 'count' => tables[0][3], 'created_at' => tables[0][4], 'updated_at' => tables[0][5]}, 146 | {'name' => tables[1][0], 'type' => tables[1][1], 'schema' => tables[1][2], 'count' => tables[1][3], 'created_at' => tables[1][4], 'updated_at' => tables[1][5]}, 147 | {'name' => tables[2][0], 'type' => tables[2][1], 'schema' => tables[2][2], 'count' => tables[2][3], 'created_at' => tables[2][4], 'updated_at' => tables[2][5]}, 148 | {'name' => tables[3][0], 'type' => tables[3][1], 'schema' => tables[3][2], 'count' => tables[3][3], 'created_at' => tables[3][4], 'updated_at' => tables[3][5]} 149 | ]}.to_json) 150 | 151 | table_list = client.tables(db_name).sort_by { |e| e.name } 152 | 153 | db_count = 0 154 | tables.each {|table| 155 | db_count += table[3] 156 | } 157 | 158 | # REST API call to fetch the database permission 159 | stub_api_request(:get, "/v3/database/list"). 160 | to_return(:body => {'databases' => [ 161 | {'name' => db_name, 'count' => db_count, 'created_at' => tables[0][4], 'updated_at' => tables[0][5], 'permission' => 'full_access'} 162 | ]}.to_json) 163 | 164 | tables.length.times {|i| 165 | expect(table_list[i].db_name).to eq(db_name) 166 | expect(table_list[i].name).to eq(tables[i][0]) 167 | expect(table_list[i].type).to eq(tables[i][1].to_sym) 168 | expect(table_list[i].schema.to_json).to eq(eval(tables[i][2]).to_json) 169 | expect(table_list[i].count).to eq(tables[i][3]) 170 | expect(table_list[i].created_at).to eq(Time.parse(tables[i][4])) 171 | expect(table_list[i].updated_at).to eq(Time.parse(tables[i][5])) 172 | 173 | # REST API call to fetch the database permission 174 | stub_api_request(:get, "/v3/database/list"). 175 | to_return(:body => {'databases' => [ 176 | {'name' => db_name, 'count' => db_count, 'created_at' => tables[0][4], 'updated_at' => tables[0][5], 'permission' => 'full_access'} 177 | ]}.to_json) 178 | expect(table_list[i].permission).to eq(:full_access) 179 | 180 | # set up a trap to check this call never happens 181 | # - if it did, the next assertion on the count would fail 182 | stub_api_request(:get, "/v3/database/list"). 183 | to_return(:body => {'databases' => [ 184 | {'name' => db_name, 'count' => db_count + 100, 'created_at' => tables[0][4], 'updated_at' => tables[0][5], 'permission' => 'full_access'} 185 | ]}.to_json) 186 | expect(table_list[i].database.count).to eq(db_count) 187 | } 188 | end 189 | end 190 | 191 | describe "'table' Client API" do 192 | it 'should return the Table object corresponding to the name' do 193 | tables = [ 194 | ["table_1", "item", "[[\"value\",\"string\"]]", 111, "2013-01-21 01:51:41 UTC", "2014-01-21 01:51:41 UTC"], 195 | ["table_2", "log", "[[\"value\",\"long\"]]", 222, "2013-02-22 02:52:42 UTC", "2014-02-22 02:52:42 UTC"], 196 | ["table_3", "item", "[[\"value\",\"string\"]]", 333, "2013-03-23 03:53:43 UTC", "2014-03-23 03:53:43 UTC"], 197 | ["table_4", "log", "[[\"value\",\"long\"]]", 444, "2013-04-24 04:54:44 UTC", "2014-04-24 04:54:44 UTC"] 198 | ] 199 | stub_api_request(:get, "/v3/table/list/#{e db_name}"). 200 | to_return(:body => {'tables' => [ 201 | {'name' => tables[0][0], 'type' => tables[0][1], 'schema' => tables[0][2], 'count' => tables[0][3], 'created_at' => tables[0][4], 'updated_at' => tables[0][5]}, 202 | {'name' => tables[1][0], 'type' => tables[1][1], 'schema' => tables[1][2], 'count' => tables[1][3], 'created_at' => tables[1][4], 'updated_at' => tables[1][5]}, 203 | {'name' => tables[2][0], 'type' => tables[2][1], 'schema' => tables[2][2], 'count' => tables[2][3], 'created_at' => tables[2][4], 'updated_at' => tables[2][5]}, 204 | {'name' => tables[3][0], 'type' => tables[3][1], 'schema' => tables[3][2], 'count' => tables[3][3], 'created_at' => tables[3][4], 'updated_at' => tables[3][5]} 205 | ]}.to_json) 206 | 207 | i = 1 208 | table = client.table(db_name, tables[i][0]) 209 | 210 | expect(table.name).to eq(tables[i][0]) 211 | expect(table.type).to eq(tables[i][1].to_sym) 212 | expect(table.schema.to_json).to eq(eval(tables[i][2]).to_json) 213 | expect(table.count).to eq(tables[i][3]) 214 | expect(table.created_at).to eq(Time.parse(tables[i][4])) 215 | expect(table.updated_at).to eq(Time.parse(tables[i][5])) 216 | end 217 | end 218 | 219 | describe 'swap_table' do 220 | it 'should swap tables' do 221 | stub_api_request(:post, '/v3/table/swap/db/table1/table2') 222 | expect(api.swap_table('db', 'table1', 'table2')).to eq(true) 223 | end 224 | end 225 | 226 | describe 'update_expire' do 227 | it 'should update expiry days' do 228 | stub_api_request(:post, '/v3/table/update/db/table'). 229 | with(:body => {'expire_days' => '5'}). 230 | to_return(:body => {'type' => 'type'}.to_json) 231 | expect(api.update_expire('db', 'table', 5)).to eq(true) 232 | end 233 | end 234 | 235 | describe 'handle include_v' do 236 | it 'should set/unset include_v flag' do 237 | stub_api_request(:get, '/v3/table/list/db'). 238 | to_return(:body => {'tables' => [ 239 | {'name' => 'table', 'type' => 'log', 'include_v' => true}, 240 | ]}.to_json) 241 | 242 | table = client.table('db', 'table') 243 | expect(table.include_v).to eq true 244 | 245 | stub_api_request(:get, '/v3/table/list/db'). 246 | to_return(:body => {'tables' => [ 247 | {'name' => 'table', 'type' => 'log', 'include_v' => false}, 248 | ]}.to_json) 249 | 250 | stub_api_request(:post, '/v3/table/update/db/table'). 251 | with(:body => {'include_v' => "false"}). 252 | to_return(:body => {"database"=>"db","table"=>"table","type"=>"log"}.to_json) 253 | api.update_table('db', 'table', include_v: "false") 254 | 255 | table = client.table('db', 'table') 256 | expect(table.include_v).to eq false 257 | end 258 | end 259 | 260 | describe 'tail' do 261 | let :packed do 262 | s = StringIO.new 263 | pk = MessagePack::Packer.new(s) 264 | pk.write([1, 2, 3]) 265 | pk.write([4, 5, 6]) 266 | pk.flush 267 | s.string 268 | end 269 | 270 | it 'yields row if block given' do 271 | stub_api_request(:get, '/v3/table/tail/db/table'). 272 | with(:query => {'format' => 'msgpack', 'count' => '10'}). 273 | to_return(:body => packed) 274 | result = [] 275 | api.tail('db', 'table', 10) do |row| 276 | result << row 277 | end 278 | expect(result).to eq([[1, 2, 3], [4, 5, 6]]) 279 | end 280 | 281 | it 'returns rows' do 282 | stub_api_request(:get, '/v3/table/tail/db/table'). 283 | with(:query => {'format' => 'msgpack', 'count' => '10'}). 284 | to_return(:body => packed) 285 | expect(api.tail('db', 'table', 10)).to eq([[1, 2, 3], [4, 5, 6]]) 286 | end 287 | 288 | it 'shows deprecated warning for from and to' do 289 | stub_api_request(:get, '/v3/table/tail/db/table'). 290 | with(:query => {'format' => 'msgpack', 'count' => '10'}). 291 | to_return(:body => packed) 292 | r, w = IO.pipe 293 | begin 294 | backup = $stderr.dup 295 | $stderr.reopen(w) 296 | api.tail('db', 'table', 10, 100, 0) 297 | ensure 298 | $stderr.reopen(backup) 299 | w.close 300 | end 301 | expect(r.read).to eq(%Q(parameter "to" and "from" no longer work\n)) 302 | end 303 | end 304 | 305 | describe 'change_database' do 306 | it 'should change the database belonging to' do 307 | stub_api_request(:post, "/v3/table/change_database/src_db/table"). 308 | with(:body => {'dest_database_name' => 'dst_db'}). 309 | to_return(:body => {'database' => 'dst_db', 'table' => 'table', 'type' => 'log'}.to_json) 310 | expect(api.change_database('src_db', 'table', 'dst_db')).to be true 311 | end 312 | 313 | it 'should return 403 error if dest database is inaccessible' do 314 | err_msg = 'Access denied for the destination' 315 | stub_api_request(:post, "/v3/table/change_database/src_db/table"). 316 | with(:body => {'dest_database_name' => 'inaccessible_db'}). 317 | to_return(:status => 403, :body => {'message' => err_msg}.to_json) 318 | expect { 319 | api.change_database('src_db', 'table', 'inaccessible_db') 320 | }.to raise_error(TreasureData::ForbiddenError, /#{err_msg}/) 321 | end 322 | 323 | it 'should return 403 error if there is a same name table in the dest database' do 324 | err_msg = 'Table table already exists in the destination' 325 | stub_api_request(:post, "/v3/table/change_database/src_db/table"). 326 | with(:body => {'dest_database_name' => 'dst_db'}). 327 | to_return(:status => 403, :body => {'message' => err_msg}.to_json) 328 | expect { 329 | api.change_database('src_db', 'table', 'dst_db') 330 | }.to raise_error(TreasureData::ForbiddenError, /#{err_msg}/) 331 | end 332 | 333 | it 'should return 404 error if the dest database does not exist' do 334 | err_msg = 'Destination database is not found' 335 | stub_api_request(:post, "/v3/table/change_database/src_db/table"). 336 | with(:body => {'dest_database_name' => 'notexist_db'}). 337 | to_return(:status => 404, :body => {'message' => err_msg}.to_json) 338 | expect { 339 | api.change_database('src_db', 'table', 'notexist_db') 340 | }.to raise_error(TreasureData::NotFoundError, /#{err_msg}/) 341 | end 342 | end 343 | end 344 | -------------------------------------------------------------------------------- /spec/td/client/user_api_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/client/spec_resources' 3 | require 'json' 4 | 5 | describe 'User API' do 6 | include_context 'spec symbols' 7 | include_context 'common helper' 8 | 9 | let :api do 10 | API.new(nil) 11 | end 12 | 13 | describe 'authenticate' do 14 | it 'returns apikey' do 15 | stub_api_request(:post, "/v3/user/authenticate"). 16 | to_return(:body => {'apikey' => 'apikey'}.to_json) 17 | expect(api.authenticate('user', 'password')).to eq('apikey') 18 | end 19 | 20 | it 'raises AuthError for authentication failure' do 21 | stub_api_request(:post, "/v3/user/authenticate"). 22 | to_return(:status => 400, :body => {'apikey' => 'apikey'}.to_json) 23 | expect { 24 | api.authenticate('user', 'password') 25 | }.to raise_error(TreasureData::AuthError) 26 | end 27 | 28 | it 'raises APIError for other error' do 29 | stub_api_request(:post, "/v3/user/authenticate"). 30 | to_return(:status => 500, :body => {'apikey' => 'apikey'}.to_json) 31 | expect { 32 | api.authenticate('user', 'password') 33 | }.to raise_error(TreasureData::APIError) 34 | end 35 | end 36 | 37 | describe 'list_users' do 38 | it 'returns users' do 39 | stub_api_request(:get, "/v3/user/list"). 40 | to_return(:body => {'users' => [{'name' => 'name1', 'email' => 'email1'}, {'name' => 'name2', 'email' => 'email2'}]}.to_json) 41 | expect(api.list_users).to eq([ 42 | ['name1', nil, nil, 'email1'], 43 | ['name2', nil, nil, 'email2'], 44 | ]) 45 | end 46 | end 47 | 48 | describe 'add_user' do 49 | it 'runs' do 50 | stub_api_request(:post, "/v3/user/add/name").to_return(:body => {}.to_json) 51 | expect(api.add_user('name', "org", 'name+suffix@example.com', 'password')).to eq(true) 52 | end 53 | 54 | # TODO 55 | it 'does not escape sp but it must be a bug' do 56 | stub_api_request(:post, "/v3/user/add/!%20%20%20%20@%23$%25%5E&*()_%2B%7C~%2Ecom").to_return(:body => {}.to_json) 57 | expect(api.add_user('! @#$%^&*()_+|~.com', "org", 'name+suffix@example.com', 'password')).to eq(true) 58 | end 59 | end 60 | 61 | describe 'remove_user' do 62 | it 'runs' do 63 | stub_api_request(:post, "/v3/user/remove/name").to_return(:body => {}.to_json) 64 | expect(api.remove_user('name')).to eq(true) 65 | end 66 | end 67 | 68 | describe 'change_email' do 69 | it 'runs' do 70 | stub_api_request(:post, "/v3/user/email/change/name"). 71 | with(:body => {'email' => 'new@email.com'}). 72 | to_return(:body => {}.to_json) 73 | expect(api.change_email('name', 'new@email.com')).to eq(true) 74 | end 75 | end 76 | 77 | describe 'list_apikeys' do 78 | it 'runs' do 79 | stub_api_request(:get, "/v3/user/apikey/list/name"). 80 | to_return(:body => {'apikeys' => ['key1', 'key2']}.to_json) 81 | expect(api.list_apikeys('name')).to eq(['key1', 'key2']) 82 | end 83 | end 84 | 85 | describe 'add_apikey' do 86 | it 'does not return the generated apikey because you can list apikey afterwards' do 87 | stub_api_request(:post, "/v3/user/apikey/add/name"). 88 | to_return(:body => {'apikey' => 'apikey'}.to_json) 89 | expect(api.add_apikey('name')).to eq(true) 90 | end 91 | end 92 | 93 | describe 'remove_apikey' do 94 | it 'runs' do 95 | stub_api_request(:post, "/v3/user/apikey/remove/name"). 96 | to_return(:body => {}.to_json) 97 | expect(api.remove_apikey('name', 'apikey')).to eq(true) 98 | end 99 | end 100 | 101 | describe 'change password' do 102 | it 'runs' do 103 | stub_api_request(:post, "/v3/user/password/change/name"). 104 | with(:body => {'password' => 'password'}). 105 | to_return(:body => {}.to_json) 106 | expect(api.change_password('name', 'password')).to eq(true) 107 | end 108 | end 109 | 110 | describe 'change my password' do 111 | it 'runs' do 112 | stub_api_request(:post, "/v3/user/password/change"). 113 | with(:body => {'old_password' => 'old_password', 'password' => 'password'}). 114 | to_return(:body => {}.to_json) 115 | expect(api.change_my_password('old_password', 'password')).to eq(true) 116 | end 117 | end 118 | end 119 | -------------------------------------------------------------------------------- /spec/td/client_sched_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/client/spec_resources' 3 | 4 | describe 'Schedule Command' do 5 | include_context 'spec symbols' 6 | include_context 'common helper' 7 | 8 | let :api do 9 | API.new(nil) 10 | end 11 | 12 | let :client do 13 | client = TreasureData::Client.new('dummy') 14 | client.instance_variable_set('@api', api) 15 | client 16 | end 17 | 18 | describe 'create' do 19 | let :opts do 20 | {:database => db_name, :cron => '', :type => 'hive', :query => 'select 1;'} 21 | end 22 | 23 | before do 24 | stub_api_request(:post, "/v3/schedule/create/#{e(sched_name)}"). 25 | with(:body => opts). 26 | to_return(:body => {'name' => sched_name, 'start' => start}.to_json) 27 | end 28 | context 'start is now' do 29 | let (:start){ Time.now.round } 30 | it 'returns Time object' do 31 | expect(client.create_schedule(sched_name, opts)).to eq(start) 32 | end 33 | end 34 | 35 | context 'start is nil' do 36 | let (:start){ nil } 37 | it do 38 | expect(client.create_schedule(sched_name, opts)).to eq(start) 39 | end 40 | end 41 | end 42 | 43 | describe 'history' do 44 | let :opts do 45 | {'database' => db_name} 46 | end 47 | 48 | let :history do 49 | ['history', 'scheduled_at', 'job_id', 'type', 'database', 'status', 'query', 'start_at', 'end_at', 'result', 'priority'].inject({}) { |r, e| 50 | r[e] = e 51 | r 52 | } 53 | end 54 | 55 | it 'returns scheduled_job' do 56 | h = history; h['scheduled_at'] = '2015-02-17 14:16:00 +0900' 57 | stub_api_request(:get, "/v3/schedule/history/#{e(sched_name)}?from=0&to=19"). 58 | to_return(:body => {'count' => 1, 'history' => [h]}.to_json) 59 | 60 | client.history(sched_name, 0, 19).each do |scheduled_job| 61 | expect(scheduled_job.scheduled_at.xmlschema).to eq(Time.parse('2015-02-17T14:16:00+09:00').xmlschema) #avoid depending on CI's Locale 62 | expect(scheduled_job.job_id).to eq('job_id') 63 | expect(scheduled_job.status).to eq('status') 64 | expect(scheduled_job.priority).to eq('priority') 65 | expect(scheduled_job.result_url).to eq('result') 66 | end 67 | end 68 | 69 | it 'works when scheduled_at == ""' do 70 | h = history; h['scheduled_at'] = '' 71 | stub_api_request(:get, "/v3/schedule/history/#{e(sched_name)}?from=0&to=19"). 72 | to_return(:body => {'count' => 1, 'history' => [h]}.to_json) 73 | 74 | client.history(sched_name, 0, 19).each do |scheduled_job| 75 | expect(scheduled_job.scheduled_at).to eq(nil) 76 | end 77 | end 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /spec/td/client_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'td/client/spec_resources' 3 | 4 | describe 'Command' do 5 | include_context 'spec symbols' 6 | include_context 'common helper' 7 | include_context 'job resources' 8 | 9 | before do 10 | stub_api_request(:post, "/v3/user/authenticate"). 11 | to_return(:body => {'apikey' => 'apikey'}.to_json) 12 | end 13 | 14 | let :client do 15 | Client.authenticate('user', 'password') 16 | end 17 | 18 | describe '#job' do 19 | before do 20 | stub_api_request(:get, "/v3/job/list").to_return(:body => {'jobs' => raw_jobs}.to_json) 21 | end 22 | 23 | it 'return jobs created with API result' do 24 | jobs = client.jobs 25 | 26 | expect(jobs).to be_kind_of Array 27 | jobs.each.with_index do |job, i| 28 | expect(job.job_id).to eq raw_jobs[i]['job_id'] 29 | expect(job.type).to eq raw_jobs[i]['type'] 30 | expect(job.status).to eq raw_jobs[i]['status'] 31 | expect(job.query).to eq raw_jobs[i]['query'] 32 | expect(job.start_at).to eq Time.parse(raw_jobs[i]['start_at']) 33 | expect(job.end_at).to eq Time.parse(raw_jobs[i]['end_at']) 34 | expect(job.cpu_time).to eq raw_jobs[i]['cpu_time'] 35 | expect(job.result_size).to eq raw_jobs[i]['result_size'] 36 | expect(job.result_url).to eq raw_jobs[i]['result_url'] 37 | expect(job.priority).to eq raw_jobs[i]['priority'] 38 | expect(job.retry_limit).to eq raw_jobs[i]['retry_limit'] 39 | expect(job.org_name).to eq raw_jobs[i]['organization'] 40 | expect(job.db_name).to eq raw_jobs[i]['database'] 41 | expect(job.duration).to eq raw_jobs[i]['duration'] 42 | expect(job.num_records).to eq raw_jobs[i]['num_records'] 43 | end 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /td-client.gemspec: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | $:.push File.expand_path('../lib', __FILE__) 3 | require 'td/client/version' 4 | 5 | Gem::Specification.new do |gem| 6 | gem.name = "td-client" 7 | gem.summary = "Treasure Data API library for Ruby" 8 | gem.description = "Treasure Data API library for Ruby" 9 | gem.authors = ["Treasure Data, Inc."] 10 | gem.email = "support@treasure-data.com" 11 | gem.homepage = "http://treasuredata.com/" 12 | gem.version = TreasureData::Client::VERSION 13 | gem.has_rdoc = false 14 | gem.test_files = Dir["spec/**/*_spec.rb"] 15 | gem.files = Dir["lib/**/*", "ext/**/*", "data/**/*", "spec/**/*.rb"] 16 | gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } 17 | gem.require_paths = ["lib"] 18 | gem.required_ruby_version = '>= 2.1' if RUBY_ENGINE != 'jruby' 19 | gem.license = "Apache-2.0" 20 | 21 | gem.add_dependency "msgpack", ">= 0.5.6", "< 2" 22 | gem.add_dependency "httpclient", ">= 2.7" 23 | gem.add_development_dependency "rspec", "~> 3.0" 24 | gem.add_development_dependency 'coveralls_reborn' 25 | gem.add_development_dependency "webmock", "~> 3.25.1" 26 | gem.add_development_dependency "mutex_m" 27 | gem.add_development_dependency 'simplecov', '>= 0.21.2' 28 | gem.add_development_dependency 'rake' 29 | gem.add_development_dependency 'yard' 30 | gem.add_development_dependency 'webrick' 31 | if defined?(JRUBY_VERSION) && JRUBY_VERSION.start_with?('1.7.') 32 | gem.add_development_dependency 'tins', '< 1.7' 33 | gem.add_development_dependency 'public_suffix', '< 1.5' 34 | gem.add_development_dependency 'term-ansicolor', '< 1.4' 35 | end 36 | end 37 | --------------------------------------------------------------------------------