├── Rakefile ├── Gemfile ├── bin └── apache-loggen ├── lib ├── apache-loggen │ ├── version.rb │ └── base.rb ├── apache-loggen-boot.rb └── apache-loggen.rb ├── .gitignore ├── apache-loggen.gemspec ├── README.md └── LICENSE.txt /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | -------------------------------------------------------------------------------- /bin/apache-loggen: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'apache-loggen' 4 | -------------------------------------------------------------------------------- /lib/apache-loggen/version.rb: -------------------------------------------------------------------------------- 1 | module LogGenerator 2 | VERSION = "0.0.5" 3 | end 4 | -------------------------------------------------------------------------------- /lib/apache-loggen-boot.rb: -------------------------------------------------------------------------------- 1 | #coding: utf-8 2 | 3 | require 'apache-loggen' 4 | LogGenerator.generate(nil, LogGenerator::Apache.new) 5 | 6 | -------------------------------------------------------------------------------- /lib/apache-loggen.rb: -------------------------------------------------------------------------------- 1 | #coding: utf-8 2 | 3 | require 'apache-loggen/base' 4 | LogGenerator.generate(nil, LogGenerator::Apache.new) 5 | 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | .bundle 4 | .config 5 | .yardoc 6 | Gemfile.lock 7 | InstalledFiles 8 | _yardoc 9 | coverage 10 | doc/ 11 | lib/bundler/man 12 | pkg 13 | rdoc 14 | spec/reports 15 | test/tmp 16 | test/version_tmp 17 | tmp 18 | -------------------------------------------------------------------------------- /apache-loggen.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'apache-loggen/version' 5 | 6 | Gem::Specification.new do |gem| 7 | gem.name = "apache-loggen" 8 | gem.version = LogGenerator::VERSION 9 | gem.authors = ["tamtam180"] 10 | gem.email = ["kirscheless@gmail.com"] 11 | gem.description = %q{dummy apache-log generator} 12 | gem.summary = %q{dummy apache-log generator} 13 | gem.homepage = "https://github.com/tamtam180/apache_log_gen" 14 | gem.license = 'Apache 2.0' 15 | 16 | gem.files = `git ls-files`.split($/) - %w[.gitignore] 17 | gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) } 18 | gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) 19 | gem.require_paths = ["lib"] 20 | 21 | gem.add_dependency "json" 22 | end 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # はじめに 2 | 3 | Apacheログに対して何かの処理を行いたい場合に使うためのテストツールです。 4 | こんな場合に有用なんではないかなと。 5 | 6 | * productionでもアクセスが足りない 7 | * 手元でproductionレベルの流速を試したい 8 | * apacheのログファイルを持ってくるのが面倒 9 | などなど.. 10 | 11 | 12 | 一応、流速がなだらかになるようになっています。 13 | 100msよりも割り込み精度が低いOSの場合はその限りではありません。 14 | 15 | ## 性能 16 | 私のへっぽこ開発PCで毎秒12,000レコードほど生成します。 17 | 18 | Fedora16(CPU:2Core, Mem:3GB) on VirtualBox on Windows7(Corei7 M640 2.8GB, Mem:8GB) 19 | Ruby 1.9.3p194 20 | 21 | # インストール 22 | 23 | gem install apache-loggen 24 | 25 | apache-loggenというコマンドがgems/binに作られます。 26 | 27 | # 使い方 28 | 29 | ``` 30 | apache-loggen [options] [file] 31 | 32 | --limit=COUNT 最大何件出力するか。デフォルトは0で無制限。 33 | --rate=RATE 毎秒何レコード生成するか。デフォルトは0秒で流量制限無し。 34 | --rotate=SECOND ファイルローテーションをする間隔。デフォルトは0(行わない)。 35 | ファイル名を指定した場合は無効。 36 | --progress STDERRに生成速度の表示をする 37 | --json Json形式で出力 38 | 39 | file を指定した場合はそのファイルへ出力する 40 | file を省略した場合はSTDOUTへ出力する 41 | ``` 42 | 43 | # 例 44 | 45 | ## STDOUTにレコード出力 46 | apache-loggen 47 | 48 | ## JSONで出力 49 | apache-loggen --json 50 | 51 | ## 毎秒100レコードの速度でファイル「abc.log」に出力 52 | apache-loggen --rate=100 abc.log 53 | 54 | ## 10秒ごとにファイルのローテーションを行う 55 | apache-loggen --rotate=10 abc.og 56 | 57 | ## 生成状況を表示する 58 | apache-loggen --rotate=10 --progress abc.log 59 | ---- 60 | file rotate. rename to ./abc.2012-10-17_113723.log 61 | file rotate. rename to ./abc.2012-10-17_113733.log 62 | 220[rec] 9.90[rec/s] 63 | 64 | ## 5000件出力で打ちきる 65 | apache-loggen --limit=5000 66 | 67 | # ローテーションルール 68 | abc.log -> abc.[yyyy-MM-dd_HHmmss].log 69 | 70 | 71 | # 他のログを生成したい場合 72 | 再利用できるようにある程度クラス化してあるので、Apache以外のログも対応可能です。 73 | 74 | 以下のコードでログ生成を開始します。 75 | 76 | require 'apache-loggen/base' 77 | LogGenerator.generate(conf=nil, gen_obj=nil, &block) 78 | ---- 79 | apache-loggen/base をrequireします。 80 | conf にnilを渡すとARGVをパースします。 81 | conf にHashを渡すと、デフォルトのオプションから、渡したものだけ上書きします。 82 | 83 | 実際にログを生成する部分は、LogGenerator::Baseを継承し、generate(context, config)というメソッドを定義する必要があります。 84 | デフォルトではLogGenerator::Apacheというクラスが存在します。 85 | Apacheログをベースに何かいじる場合は、これを利用すると良いと思います。 86 | 87 | もっと気軽に生成したい場合は、以下のようにブロックを渡すことでGeneratorの代わりとなります。 88 | 89 | LogGenerator.generate() do | context, config, record | 90 | # ログを1つ分生成する。 91 | Time.now.to_s + "\n" 92 | end 93 | 94 | 上記のrecordは、第2引数のgen_objも指定した場合にその結果を受け取り、さらにblockで加工する場合に使います。 95 | 96 | ## Apacheのログの出力形式を変更したい 97 | 98 | ```ruby 99 | require 'apache-loggen/base' 100 | class MyGen < LogGenerator::Apache 101 | def format(record, config) 102 | # 今回はJSONを無視する 103 | return %[[#{Time.now.strftime('%d/%b/%Y:%H:%M:%S %z')}] #{record["path"]}\n] 104 | end 105 | end 106 | LogGenerator.generate(nil, MyGen.new) 107 | ``` 108 | 109 | ## Apacheのログに新しく情報を追加したい 110 | ```ruby 111 | require 'apache-loggen/base' 112 | class MyGen < LogGenerator::Apache 113 | # オリジナル実装はhashをJSONか1行の文字列にしているが 114 | # 今回はそれに情報を追加する 115 | def format(record, config) 116 | record["process_time"] = grand(1000000) + 1000 117 | if config[:json] then 118 | return record.to_json + "\n" 119 | else 120 | return %[#{record['host']} - #{record['user']} [#{Time.now.strftime('%d/%b/%Y:%H:%M:%S %z')}] "#{record['method']} #{record['path']} HTTP/1.1" #{record['code']} #{record['size']} "#{record['referer']}" "#{record['agent']}" #{record['process_time']}\n] 121 | end 122 | end 123 | end 124 | LogGenerator.generate(nil, MyGen.new) 125 | ``` 126 | 127 | ## 完全に独自のログ形式を出力したい 128 | 129 | ```ruby 130 | require 'apache-loggen/base' 131 | class MyGen < LogGenerator::Base 132 | def generate(context, config) 133 | return "#{Time.now.to_s} #{context.inspect}\n" 134 | end 135 | end 136 | LogGenerator.generate(nil, MyGen.new) 137 | ``` 138 | 139 | もしくは、 140 | 141 | ```ruby 142 | require 'apache-loggen/base' 143 | LogGenerator.generate do | context, config, record | 144 | "#{Time.now.to_s} #{context.inspect}\n" 145 | end 146 | ``` 147 | 148 | 149 | # 履歴 150 | - 0.0.5 Limitで指定した値よりも1つレコードが多く出力されるのを修正。 151 | - 0.0.4 Ruby-1.8.7でも動くようにした。 152 | - 0.0.3 Rate=1くらいの低速度の場合、Flushが走らないので明示的にFlushするようにした。 153 | - 0.0.2 RubyGemsに登録。コマンドを用意した。クラスの再利用ができるようにした。 154 | - 0.0.1 はじめてのリリース 155 | 156 | # ライセンス 157 | Apache License, Version 2.0 158 | 159 | # 謝辞 160 | 161 | TreasureDataのスクリプトをパクりました。 162 | 163 | オリジナルとの差異は、ログの日付を現在の日付で出力する点です。 164 | 165 | * https://github.com/treasure-data/td 166 | * https://github.com/treasure-data/td/blob/master/data/sample_apache_gen.rb 167 | 168 | 169 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /lib/apache-loggen/base.rb: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | require 'rubygems' 4 | require 'time' 5 | require 'fileutils' 6 | require 'optparse' 7 | require 'json' 8 | 9 | module LogGenerator 10 | 11 | class Base 12 | def generate(context, config) 13 | # must override 14 | exit 1 15 | end 16 | end 17 | 18 | class Apache < Base 19 | 20 | RECORDS = 5000 21 | HOSTS = RECORDS/4 22 | PAGES = RECORDS/4 23 | 24 | AGENT_LIST_STRING = <<-END 25 | Mozilla/5.0 (iPhone; CPU iPhone OS 5_0_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A405 Safari/7534.48.3 26 | Mozilla/5.0 (iPad; CPU OS 5_0_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A405 Safari/7534.48.3 27 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0) 28 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0) 29 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0) 30 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 31 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 32 | Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) 33 | Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) 34 | Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 35 | Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 36 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7 37 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7 38 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11 39 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11 40 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11 41 | Mozilla/5.0 (Windows NT 6.0; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 42 | Mozilla/5.0 (Windows NT 6.0; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 43 | Mozilla/5.0 (Windows NT 6.0; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 44 | Mozilla/5.0 (Windows NT 6.0; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 45 | Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11 46 | Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11 47 | Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.46 Safari/535.11 48 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1 49 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1 50 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1 51 | Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; YTB730; GTB7.2; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C; .NET4.0E; Media Center PC 6.0) 52 | Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; YTB730; GTB7.2; EasyBits GO v1.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C) 53 | Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; GTB7.2; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C) 54 | Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; YTB730; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C) 55 | Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; WOW64; Trident/4.0; GTB6; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30618; .NET4.0C) 56 | Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; YTB720; GTB7.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729) 57 | Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; BTRS122159; GTB7.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; BRI/2) 58 | END 59 | AGENT_LIST = AGENT_LIST_STRING.split("\n").map{|a|a.lstrip} 60 | 61 | PAGE_CATEGORIES = %w[ 62 | books 63 | books 64 | books 65 | electronics 66 | electronics 67 | electronics 68 | electronics 69 | electronics 70 | electronics 71 | software 72 | software 73 | software 74 | software 75 | games 76 | games 77 | games 78 | office 79 | office 80 | cameras 81 | computers 82 | finance 83 | giftcards 84 | garden 85 | health 86 | music 87 | sports 88 | toys 89 | networking 90 | jewelry 91 | ] 92 | 93 | # インナークラスから使うためにmodule化 94 | # 使う場合はinclude RANDする 95 | module RAND 96 | if RUBY_VERSION >= '1.9.1' 97 | RANDOM = Random.new 98 | def grand(n) 99 | RANDOM.rand(n) 100 | end 101 | else 102 | def grand(n) 103 | rand(n) 104 | end 105 | end 106 | end 107 | include RAND 108 | 109 | def initialize() 110 | 111 | @pages = [] 112 | PAGES.times do 113 | @pages << Page.new 114 | end 115 | 116 | @hosts = [] 117 | HOSTS.times do 118 | @hosts << Host.new 119 | end 120 | 121 | end 122 | 123 | def generate(context, config) 124 | 125 | page = @pages[grand(@pages.size)] 126 | host = @hosts[grand(@hosts.size)] 127 | record = { 128 | 'host' => host.ip, 129 | 'user' => '-', 130 | 'method' => page.method, 131 | 'path' => page.path, 132 | 'code' => grand(10000) == 0 ? 500 : page.code, 133 | 'referer' => (grand(2) == 0 ? @pages[grand(@pages.size)].path : page.referer) || '-', 134 | 'size' => page.size, 135 | 'agent' => host.agent, 136 | } 137 | 138 | return format(record, config) 139 | 140 | end 141 | 142 | def format(record, config) 143 | if config[:json] then 144 | return record.to_json + "\n" 145 | else 146 | return %[#{record['host']} - #{record['user']} [#{Time.now.strftime('%d/%b/%Y:%H:%M:%S %z')}] "#{record['method']} #{record['path']} HTTP/1.1" #{record['code']} #{record['size']} "#{record['referer']}" "#{record['agent']}"\n] 147 | end 148 | end 149 | 150 | class Host 151 | include RAND 152 | def initialize 153 | @ip = "#{(grand(210)+20)/4*4}.#{(grand(210)+20)/3*3}.#{grand(210)+20}.#{grand(210)+20}" 154 | @agents = [] 155 | end 156 | 157 | attr_reader :ip 158 | 159 | def agent 160 | if @agents.size == 4 161 | @agents[grand(4)] 162 | else 163 | agent = AGENT_LIST[grand(AGENT_LIST.size)] 164 | @agents << agent 165 | agent 166 | end 167 | end 168 | end 169 | 170 | class Page 171 | include RAND 172 | def initialize 173 | cate = PAGE_CATEGORIES[grand(PAGE_CATEGORIES.size)] 174 | item = grand(RECORDS) 175 | 176 | if grand(2) == 0 177 | w = [cate, PAGE_CATEGORIES[grand(PAGE_CATEGORIES.size)]] 178 | else 179 | w = [cate] 180 | end 181 | q = w.map {|k| k[0..0].upcase + k[1..-1] }.join('+') 182 | search_path = "/search/?c=#{q}" 183 | google_ref = "http://www.google.com/search?ie=UTF-8&q=google&sclient=psy-ab&q=#{q}&oq=#{q}&aq=f&aqi=g-vL1&aql=&pbx=1&bav=on.2,or.r_gc.r_pw.r_qf.,cf.osb&biw=#{grand(5000)}&bih=#{grand(600)}" 184 | 185 | case grand(12) 186 | when 0,1,2,3,4,5 187 | @path = "/category/#{cate}" 188 | @referers = [nil, nil, nil, nil, nil, nil, nil, google_ref] 189 | @method = 'GET' 190 | @code = 200 191 | 192 | when 6 193 | @path = "/category/#{cate}?from=#{grand(3)*10}" 194 | @referers = [search_path, "/category/#{cate}"] 195 | @method = 'GET' 196 | @code = 200 197 | 198 | when 7,8,9,10 199 | @path = "/item/#{cate}/#{item}" 200 | @referers = [search_path, search_path, google_ref, "/category/#{cate}"] 201 | @method = 'GET' 202 | if grand(100) == 0 203 | @code = 404 204 | else 205 | @code = 200 206 | end 207 | 208 | when 11 209 | @path = search_path 210 | @referers = [nil] 211 | @method = 'POST' 212 | @code = 200 213 | end 214 | 215 | @size = grand(100) + 40 216 | end 217 | 218 | attr_reader :path, :size, :method, :code 219 | 220 | def referer 221 | if grand(2) == 0 222 | @referers[grand(@referers.size)] 223 | end 224 | end 225 | 226 | end 227 | 228 | end 229 | 230 | 231 | # Windowsだと割り込みが55msや10msだったりとするので100msごとに処理するように。 232 | # 汚いソースになっちゃった・・。 233 | # MultimediaTimer使えばいいんだけど、めんどくさ。 234 | class Executors 235 | FIXED_RATE = 100 236 | def self.exec(config) 237 | 238 | rate_per_sec = config[:rate] 239 | display = config[:progress] 240 | 241 | limited = rate_per_sec > 0 242 | if limited then 243 | mspr = 1000.0 / rate_per_sec # ms per rec. 244 | rate = rate_per_sec.to_f / (1000 / FIXED_RATE) # rec per 100ms 245 | end 246 | start_time = Time.now 247 | 248 | time = last_display = Time.now 249 | count = 0 250 | total_count = 0 251 | while true do 252 | 253 | break unless yield({ 254 | :start_time => start_time, 255 | :total_count => total_count, 256 | :elapsed_time => (Time.now - start_time).round, 257 | }) 258 | 259 | total_count += 1 260 | count += 1 261 | 262 | if limited && count >= rate then 263 | spent = ((Time.now - time) * 1000).round 264 | sleep_ms = mspr - spent 265 | sleep(sleep_ms / 1000.0) if sleep_ms > 0 266 | time = Time.now 267 | count = 0 268 | end 269 | 270 | if display then 271 | if Time.now - last_display >= 1.0 then 272 | $stderr.printf("\r%d[rec] %.2f[rec/s]", total_count, total_count / (Time.now - start_time + 0.001)) 273 | last_display = Time.now 274 | end 275 | end 276 | 277 | end 278 | end 279 | end 280 | 281 | class MyWriter 282 | def initialize(filename) 283 | @filename = filename 284 | @io = nil 285 | rotate() 286 | end 287 | def rotate() 288 | if @filename == nil then 289 | @io = $stdout 290 | return nil 291 | else 292 | dir = File.dirname(@filename) 293 | name = File.basename(@filename, '.*') + '.' + Time.now.strftime('%Y-%m-%d_%H%M%S') + File.extname(@filename) 294 | FileUtils.mkdir_p(dir) unless File.exists?(dir) 295 | if @io != nil then 296 | File.rename(@filename, name) 297 | @io.close 298 | end 299 | @io = open(@filename, "a") 300 | return File.join(dir, name) 301 | end 302 | end 303 | def write(str) 304 | return @io.write(str) 305 | end 306 | def flush() 307 | @io.flush() 308 | end 309 | def close() 310 | if @filename != nil && @io != nil && !@io.closed? then 311 | @io.close 312 | end 313 | end 314 | end 315 | 316 | class Generator 317 | DEFAULT_CONFIG = { 318 | :limit => 0, 319 | :rate => 0, 320 | :rotate => 0, 321 | :progress => false, 322 | :json => false, 323 | :filename => nil, 324 | } 325 | def self.execute(conf={}, gen_obj=nil, &block) 326 | 327 | config = DEFAULT_CONFIG.merge(conf) 328 | writer = MyWriter.new(config[:filename]) 329 | gen_kick = gen_obj && gen_obj.is_a?(Base) 330 | 331 | # 実行 332 | last_rotate = Time.now.to_i 333 | Executors.exec(config) do | context | 334 | 335 | if config[:rotate] > 0 && (last_rotate + config[:rotate]) <= Time.now.to_i then 336 | rotated_file = writer.rotate() 337 | if config[:progress] then 338 | $stderr.write "\rfile rotate. rename to #{rotated_file}\n" 339 | end 340 | last_rotate = Time.now.to_i 341 | end 342 | 343 | # レコード生成 344 | record = gen_obj.generate(context, config) if gen_kick 345 | record = block.call(context, config, record) if block 346 | 347 | # 出力 348 | writer.write(record) 349 | writer.flush() 350 | 351 | not (config[:limit] > 0 && config[:limit] <= context[:total_count]+1) 352 | 353 | end 354 | writer.close 355 | 356 | end 357 | end 358 | 359 | def parse_config() 360 | config = {} 361 | op = OptionParser.new 362 | op.on('--limit=COUNT', '最大何件出力するか。デフォルトは0で無制限。'){|v| config[:limit] = v.to_i } 363 | op.on('--rate=RATE', '毎秒何レコード生成するか。デフォルトは0で流量制限無し。'){|v| config[:rate] = v.to_i } 364 | op.on('--rotate=SECOND', 'ローテーションする間隔。デフォルトは0。'){|v| config[:rotate] = v.to_i } 365 | op.on('--progress', 'レートの表示をする。'){|v| config[:progress] = true } 366 | op.on('--json', 'json形式の出力'){|v| config[:json] = true } 367 | op.parse!(ARGV) 368 | # ファイルかSTDOUTか 369 | config[:filename] = ARGV[0] if not ARGV.empty? 370 | return config 371 | end 372 | def generate(conf=nil, gen_obj=nil, &block) 373 | conf = parse_config() if conf == nil 374 | Generator.execute(conf, gen_obj, &block) 375 | end 376 | module_function :parse_config 377 | module_function :generate 378 | 379 | end 380 | 381 | --------------------------------------------------------------------------------