├── .github
    ├── dependabot.yml
    └── workflows
    │   └── linux.yml
├── .gitignore
├── AUTHORS
├── ChangeLog
├── Gemfile
├── README.rdoc
├── Rakefile
├── VERSION
├── bin
    └── mongo-tail
├── fluent-plugin-mongo.gemspec
├── lib
    └── fluent
    │   └── plugin
    │       ├── in_mongo_tail.rb
    │       ├── logger_support.rb
    │       ├── mongo_auth.rb
    │       ├── out_mongo.rb
    │       └── out_mongo_replset.rb
└── test
    ├── helper.rb
    └── plugin
        ├── test_in_mongo_tail.rb
        ├── test_out_mongo.rb
        └── test_out_mongo_replset.rb


/.github/dependabot.yml:
--------------------------------------------------------------------------------
1 | version: 2
2 | updates:
3 |   - package-ecosystem: 'github-actions'
4 |     directory: '/'
5 |     schedule:
6 |       interval: 'weekly'
7 | 


--------------------------------------------------------------------------------
/.github/workflows/linux.yml:
--------------------------------------------------------------------------------
 1 | name: Test
 2 | on:
 3 |   push:
 4 |     branches: [master]
 5 |   pull_request:
 6 |     branches: [master]
 7 | jobs:
 8 |   build:
 9 |     runs-on: ${{ matrix.os }}
10 |     continue-on-error: ${{ matrix.experimental }}
11 |     strategy:
12 |       fail-fast: false
13 |       matrix:
14 |         ruby: [ '3.2', '3.1', '3.0', '2.7' ]
15 |         mongodb-version: ['5.0', '4.4', '4.2']
16 |         os:
17 |           - ubuntu-latest
18 |         experimental: [false]
19 |         include:
20 |           - ruby: head
21 |             os: ubuntu-latest
22 |             experimental: true
23 |             mongodb-version: '4.0'
24 |           - ruby: head
25 |             os: ubuntu-latest
26 |             experimental: true
27 |             mongodb-version: '4.2'
28 |           - ruby: head
29 |             os: ubuntu-latest
30 |             experimental: true
31 |             mongodb-version: '4.4'
32 | 
33 |     name: Ruby ${{ matrix.ruby }} and MongoDB ${{ matrix.mongodb-version }} on ${{ matrix.os }}
34 |     steps:
35 |     - uses: actions/checkout@v3
36 |     - uses: ruby/setup-ruby@v1
37 |       with:
38 |         ruby-version: ${{ matrix.ruby }}
39 |     - name: Start MongoDB
40 |       uses: supercharge/mongodb-github-action@1.8.0
41 |       with:
42 |         mongodb-version: ${{ matrix.mongodb-version }}
43 |     - name: unit testing
44 |       env:
45 |         CI: true
46 |       run: |
47 |         gem install bundler rake
48 |         bundle install --jobs 4 --retry 3
49 |         bundle exec rake test
50 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /fluent/
2 | /pkg/
3 | /coverage/
4 | /vendor/
5 | Gemfile.lock
6 | /test/plugin/data/
7 | /test/tools/data/


--------------------------------------------------------------------------------
/AUTHORS:
--------------------------------------------------------------------------------
1 | Masahiro Nakagawa <repeatedly _at_ gmail.com>
2 | Yuichi Tateno <hotchpotch _at_ gmail.com>
3 | 


--------------------------------------------------------------------------------
/ChangeLog:
--------------------------------------------------------------------------------
  1 | Release 1.6.0 - 2022/07/15
  2 | 
  3 | 	* Update mongo gem dependency
  4 | 	* out_mongo: Add object_id_keys parameter to convert string to Mongo ObjectId
  5 | 
  6 | Release 1.5.0 - 2020/11/26
  7 | 
  8 | 	* out_mongo: Support nested fields in date_keys
  9 | 
 10 | Release 1.4.1 - 2020/08/21
 11 | 
 12 | 	* out_mongo: Add expire_after parameter
 13 | 
 14 | Release 1.4.0 - 2020/04/28
 15 | 
 16 | 	* out_mongo: Add date_keys parameter to support MongoDB Date object in record fields.
 17 | 
 18 | Release 1.3.0 - 2019/04/24
 19 | 
 20 | 	* out_mongo: Support auth_mech parameter to allow other authentication
 21 | 	* in_mongo_tail: Fix replicat set issue
 22 | 
 23 | Release 1.2.2 - 2019/04/01
 24 | 
 25 | 	* out_mongo: Don't handle database placholders when specifying connection_string parameter
 26 | 
 27 | Release 1.2.1 - 2018/12/18
 28 | 
 29 | 	* out_mongo_replset: Fix internal signature mismatch
 30 | 
 31 | Release 1.2.0 - 2018/11/29
 32 | 
 33 | 	* out_mongo: Support placeholder in database parameter
 34 | 
 35 | Release 1.1.2 - 2018/07/19
 36 | 
 37 | 	* Update mongo gem dependency
 38 | 
 39 | 
 40 | Release 1.1.1 - 2018/05/17
 41 | 
 42 | 	* out_mongo/out_mongo_replset: Use built-in Output#format to fix EventTime serialization
 43 | 
 44 | 
 45 | Release 1.1.0 - 2018/01/18
 46 | 
 47 | 	* in_mongo: Add batch_size parameter
 48 | 	* out_mongo: Handle collection options correctly
 49 | 
 50 | 
 51 | Release 1.0.0 - 2017/11/26
 52 | 
 53 | 	* Use new Plugin API
 54 | 	* Update fluentd version to v0.14 or later
 55 | 
 56 | 
 57 | Release 0.7.16 - 2016/10/05
 58 | 
 59 | 	* out_mongo: Log warn / deprecated message for invalid record handling
 60 | 
 61 | 
 62 | Release 0.7.15 - 2016/08/16
 63 | 
 64 | 	* in_mongo_tail: Add object_id_keys parameter to convert ObjectId object into string
 65 | 
 66 | 
 67 | Release 0.7.14 - 2016/08/06
 68 | 
 69 | 	* Fix unexpected value generation with replace_xxx parameters
 70 | 
 71 | 
 72 | Release 0.7.13 - 2016/06/03
 73 | 
 74 | 	* Add mongodb_smaller_bson_limit parameter to disable MongoDB v1.7 or earlier versions by default.
 75 | 
 76 | 
 77 | Release 0.7.12 - 2016/02/09
 78 | 
 79 | 	* Support saving last_id to mongod instead of local file
 80 | 
 81 | 
 82 | Release 0.7.11 - 2015/11/26
 83 | 
 84 | 	* Add secret option to related parameters
 85 | 	* Support label in input plugin
 86 | 	* Add socket_pool_size parameter
 87 | 
 88 | 
 89 | Release 0.7.10 - 2015/05/28
 90 | 
 91 | 	* Add SSL authentication options
 92 | 
 93 | 
 94 | Release 0.7.9 - 2015/04/01
 95 | 
 96 | 	* Force to use mongo gem v1
 97 | 
 98 | 
 99 | Release 0.7.8 - 2015/03/23
100 | 
101 | 	* Add url option to support URL configuration for MongoDB connection
102 | 
103 | 
104 | Release 0.7.7 - 2015/03/19
105 | 
106 | 	* Fix in_mongo_tail shutdown mechanizm for handling stop sequence correctly.
107 | 
108 | 
109 | Release 0.7.6 - 2015/02/12
110 | 
111 | 	* Relax fluentd dependency to support v0.12 or later
112 | 
113 | 
114 | Release 0.7.5 - 2015/01/05
115 | 
116 | 	* Add journaled option to support journaled write
117 | 
118 | 
119 | Release 0.7.4 - 2014/11/10
120 | 
121 | 	* Sanitize keys of each hash of an array
122 | 	* Add config_set_default to mongo_replset
123 | 
124 | 
125 | Release 0.7.3 - 2014/03/09
126 | 
127 | 	* Add replace_dot_in_key_with and replace_dollar_in_key_with parameters to sanitize invalid key
128 | 	* Add ssl parameter to enable SSL connection
129 | 	* Relax gem version dependency
130 | 
131 | 
132 | Release 0.7.2 - 2014/02/05
133 | 
134 | 	* Support log_level option
135 | 
136 | 
137 | Release 0.7.1 - 2013/07/31
138 | 
139 | 	* Fix incomprehensible code indent
140 | 	* Remove mongo_backup output
141 | 	* Fix getting version from mongod for broken mongod support
142 | 
143 | 
144 | Release 0.7.0 - 2013/03/20
145 | 
146 | 	* Upgrade mongo gem least version to 1.8
147 | 	* Upgrade fluentd gem least version to 0.10.9
148 | 	* Use new classes with mongo gem version 1.8
149 | 	* Replace safe with write_concern option in output plugins
150 | 	* Change buffer_chunk_limit to 8MB when mongod version is 1.8 or later.
151 | 
152 | 
153 | Release 0.6.13 - 2013/01/15
154 | 
155 | 	* Add exclude_broken_fields config to output plugins
156 | 
157 | 
158 | Release 0.6.12 - 2012/12/28
159 | 
160 | 	* Fix mongo 2.2 capped? problem in mongo_tail
161 | 	* Add wait_time config to mongo_tail
162 | 
163 | 
164 | Release 0.6.11 - 2012/12/04
165 | 
166 | 	* Use buildInfo instead of serverStatus to check version
167 | 	  https://github.com/fluent/fluent-plugin-mongo/pull/20
168 | 
169 | 
170 | Release 0.6.10 - 2012/10/17
171 | 
172 | 	* mongo-tail always flush STDOUT for pipeline usage.
173 | 	  https://github.com/fluent/fluent-plugin-mongo/issues/16
174 | 	* Fix capped collection checking for 2.1.x or later versions.
175 | 
176 | 
177 | Release 0.6.9 - 2012/10/12
178 | 
179 | 	* Fix invalid use of '~>' in gemspec.
180 | 
181 | 
182 | Release 0.6.8 - 2012/10/12
183 | 
184 | 	* Lock fluentd gem version with 0.10.x.
185 | 	* Lock mongo gem version with 1.6.x.
186 | 
187 | 
188 | Release 0.6.7 - 2012/03/31
189 | 
190 | 	* Fix invaild record handling with BSON::Binary
191 | 	  https://github.com/fluent/fluent-plugin-mongo/issues/12
192 | 	* Change disable_collection_check strategy
193 | 	  https://github.com/fluent/fluent-plugin-mongo/commit/d840c948f45302ecd73af67c0b0022e3e905f955
194 | 
195 | 
196 | Release 0.6.6 - 2012/03/01
197 | 
198 | 	* Update mongo gem 1.5.2 -> 1.6.0 or later
199 | 	* Move buffer_chunk_limit checking from configure to start
200 | 
201 | 
202 | Release 0.6.5 - 2012/02/27
203 | 
204 | 	* Fix "mongo_replset unexpectedly requires 'host' in configuration"
205 | 	  https://github.com/fluent/fluent-plugin-mongo/issues/9
206 | 
207 | 
208 | Release 0.6.4 - 2012/02/16
209 | 
210 | 	* Add 'disable_collection_check' parameter
211 | 	* Fix mongod_version bug
212 | 
213 | 
214 | Release 0.6.3 - 2012/02/08
215 | 
216 | 	* Add authentication support to input / output plugins
217 | 	  https://github.com/fluent/fluent-plugin-mongo/pull/8
218 | 
219 | 
220 | Release 0.6.2 - 2012/01/23
221 | 
222 | 	* Add :safe to Connection#new options
223 | 	  https://github.com/fluent/fluent-plugin-mongo/issues/7
224 | 	* out_mongo_tag_collection renamed to out_mongo_tag_mapped
225 | 
226 | 
227 | Release 0.6.1 - 2012/01/17
228 | 
229 | 	* Add mongo_replset explanation to README
230 | 
231 | 
232 | Release 0.6.0 - 2012/01/16
233 | 
234 | 	* Add mongo_replset for Replica Set
235 | 	* out_mongo_tag_collection merged into out_mongo.
236 | 	  Please use tag_mapped mode.
237 | 	* Support invalid documets handling
238 | 
239 | 
240 | Release 0.5.3 - 2011/12/20
241 | 
242 | 	* Fix "use format time argument when record to mongodb"
243 | 	  https://github.com/fluent/fluent-plugin-mongo/pull/6
244 | 
245 | 
246 | Release 0.5.2 - 2011/11/29
247 | 
248 | 	* Fix mongod_version
249 | 	* Fix "configure of ouput plugins raises an exception when mongod is down"
250 | 	  https://github.com/fluent/fluent-plugin-mongo/issues/4
251 | 
252 | 
253 | Release 0.5.1 - 2011/11/26
254 | 
255 | 	* Fix typo
256 | 
257 | 
258 | Release 0.5.0 - 2011/11/26
259 | 
260 | 	* Jeweler to Bundler
261 | 	* Add in_mongo_tail
262 | 	* Add out_mongo_tag_collection
263 | 	* Add default 'collection' configuration to mongo_tag_collection
264 | 	* Update the version of dependency modules
265 | 	* Fix "MongoDB and Ruby-Driver have a size limit of insert operation."
266 | 	  https://github.com/fluent/fluent-plugin-mongo/issues/3
267 | 
268 | 
269 | Release 0.4.0 - 2011/10/16
270 | 
271 | 	* Support fluentd 0.10.1
272 | 	* Add out_mongo.rb test
273 | 
274 | 
275 | Release 0.3.1 - 2011/10/05
276 | 
277 | 	* Add mongo-tail tool
278 | 	  https://github.com/fluent/fluent-plugin-mongo/issues/1
279 | 
280 | 
281 | Release 0.3.0 - 2011/10/03
282 | 
283 | 	* Add mongo_backup
284 | 
285 | 
286 | Release 0.2.1 - 2011/10/02
287 | 
288 | 	* Fix mongo dependency
289 | 	* Fix typo
290 | 	* Fix configuration parsing
291 | 	* Replace MongoBuffer with MongoOutput's <store>
292 | 	* Fix duplicated insert to backup
293 | 
294 | 
295 | Release 0.2.0 - 2011/09/28
296 | 
297 | 	* MongoOutput becomes BufferedOutput
298 | 
299 | 
300 | Release 0.1.0 - 2011/09/28
301 | 
302 | 	* Add out_mongo
303 | 


--------------------------------------------------------------------------------
/Gemfile:
--------------------------------------------------------------------------------
1 | source 'https://rubygems.org'
2 | 
3 | gemspec
4 | 
5 | gem "simplecov", :require => false
6 | 


--------------------------------------------------------------------------------
/README.rdoc:
--------------------------------------------------------------------------------
  1 | = MongoDB plugin for {Fluentd}[http://github.com/fluent/fluentd]
  2 | 
  3 | fluent-plugin-mongo provides input and output plugins for {Fluentd}[http://fluentd.org] ({GitHub}[http://github.com/fluent/fluentd])
  4 | 
  5 | = Requirements
  6 | 
  7 |     |fluent-plugin-mongo|   fluentd  |  ruby  |
  8 |     |-------------------|------------|--------|
  9 |     |     >= 1.0.0      | >= 0.14.12 | >= 2.1 |
 10 |     |     <  1.0.0      | >= 0.12.0  | >= 1.9 |
 11 | 
 12 | = Installation
 13 | 
 14 | == Gems
 15 | 
 16 | The gem is hosted at {Rubygems.org}[http://rubygems.org]. You can install the gem as follows:
 17 | 
 18 |     $ fluent-gem install fluent-plugin-mongo
 19 | 
 20 | = Plugins
 21 | 
 22 | == Output plugin
 23 | 
 24 | === mongo
 25 | 
 26 | Store Fluentd event to MongoDB database.
 27 | 
 28 | ==== Configuration
 29 | 
 30 | Use _mongo_ type in match.
 31 | 
 32 |     <match mongo.**>
 33 |       @type mongo
 34 | 
 35 |       # You can choose two approaches, connection_string or each parameter
 36 |       # 1. connection_string for MongoDB URI
 37 |       connection_string mongodb://fluenter:10000/fluent
 38 | 
 39 |       # 2. specify each parameter
 40 |       database fluent
 41 |       host fluenter
 42 |       port 10000
 43 | 
 44 |       # collection name to insert
 45 |       collection test
 46 | 
 47 |       # Set 'user' and 'password' for authentication.
 48 |       # These options are not used when use connection_string parameter.
 49 |       user handa
 50 |       password shinobu
 51 | 
 52 |       # Set 'capped' if you want to use capped collection
 53 |       capped
 54 |       capped_size 100m
 55 | 
 56 |       # Specify date fields in record to use MongoDB's Date object (Optional) default: nil
 57 |       # Supported data types are String/Integer/Float/Fuentd EventTime.
 58 |       # For Integer type, milliseconds epoch and seconds epoch are supported.
 59 |       # eg: updated_at: "2020-02-01T08:22:23.780Z" or updated_at: 1580546457010
 60 |       date_keys updated_at
 61 | 
 62 |       # Specify id fields in record to use MongoDB's BSON ObjectID (Optional) default: nil
 63 |       # eg: my_id: "507f1f77bcf86cd799439011"
 64 |       object_id_keys my_id
 65 | 
 66 |       # Other buffer configurations here
 67 |     </match>
 68 | 
 69 | For _connection_string_ parameter, see https://docs.mongodb.com/manual/reference/connection-string/ article for more detail.
 70 | 
 71 | ===== built-in placeholders
 72 | 
 73 | fluent-plugin-mongo support built-in placeholders.
 74 | _database_ and _collection_ parameters can handle them.
 75 | 
 76 | Here is an example to use built-in placeholders:
 77 | 
 78 |     <match mongo.**>
 79 |       @type mongo
 80 | 
 81 |       database ${tag[0]}
 82 | 
 83 |       # collection name to insert
 84 |       collection ${tag[1]}-%Y%m%d
 85 | 
 86 |       # Other buffer configurations here
 87 |       <buffer tag, time>
 88 |         @type memory
 89 |         timekey 3600
 90 |       </buffer>
 91 |     </match>
 92 | 
 93 | In more detail, please refer to the officilal document for built-in placeholders: https://docs.fluentd.org/v1.0/articles/buffer-section#placeholders
 94 | 
 95 | === mongo(tag mapped mode)
 96 | 
 97 | Tag mapped to MongoDB collection automatically.
 98 | 
 99 | ==== Configuration
100 | 
101 | Use _tag_mapped_ parameter in match of _mongo_ type.
102 | 
103 | If tag name is "foo.bar", auto create collection "foo.bar" and insert data.
104 | 
105 |     <match forward.*>
106 |       @type mongo
107 |       database fluent
108 | 
109 |       # Set 'tag_mapped' if you want to use tag mapped mode.
110 |       tag_mapped
111 | 
112 |       # If tag is "forward.foo.bar", then prefix "forward." is removed.
113 |       # Collection name to insert is "foo.bar".
114 |       remove_tag_prefix forward.
115 | 
116 |       # This configuration is used if tag not found. Default is 'untagged'.
117 |       collection misc
118 | 
119 |       # Other configurations here
120 |     </match>
121 | 
122 | === mongo_replset
123 | 
124 | Replica Set version of mongo.
125 | 
126 | ==== Configuration
127 | 
128 | ===== v0.8 or later
129 | 
130 |     <match mongo.**>
131 |       @type mongo_replset
132 |       database fluent
133 |       collection logs
134 | 
135 |       nodes localhost:27017,localhost:27018
136 | 
137 |       # The replica set name
138 |       replica_set myapp
139 | 
140 |       # num_retries is threshold at failover, default is 60.
141 |       # If retry count reached this threshold, mongo plugin raises an exception.
142 |       num_retries 30
143 | 
144 |       # following optional parameters passed to mongo-ruby-driver.
145 |       # See mongo-ruby-driver docs for more detail: https://docs.mongodb.com/ruby-driver/master/tutorials/ruby-driver-create-client/
146 |       # Specifies the read preference mode
147 |       #read secondary
148 |     </match>
149 | 
150 | ===== v0.7 or ealier
151 | 
152 | Use _mongo_replset_ type in match.
153 | 
154 |     <match mongo.**>
155 |       @type mongo_replset
156 |       database fluent
157 |       collection logs
158 | 
159 |       # each node separated by ','
160 |       nodes localhost:27017,localhost:27018,localhost:27019
161 | 
162 |       # following optional parameters passed to mongo-ruby-driver.
163 |       #name replset_name
164 |       #read secondary
165 |       #refresh_mode sync
166 |       #refresh_interval 60
167 |       #num_retries 60
168 |     </match>
169 | 
170 | == Input plugin
171 | 
172 | === mongo_tail
173 | 
174 | Tail capped collection to input data.
175 | 
176 | ==== Configuration
177 | 
178 | Use _mongo_tail_ type in source.
179 | 
180 |     <source>
181 |       @type mongo_tail
182 |       database fluent
183 |       collection capped_log
184 | 
185 |       tag app.mongo_log
186 | 
187 |       # waiting time when there is no next document. default is 1s.
188 |       wait_time 5
189 | 
190 |       # Convert 'time'(BSON's time) to fluent time(Unix time).
191 |       time_key time
192 | 
193 |       # Convert ObjectId to string
194 |       object_id_keys ["id_key"]
195 |     </source>
196 | 
197 | You can also use _url_ to specify the database to connect.
198 | 
199 |     <source>
200 |       @type mongo_tail
201 |       url mongodb://user:password@192.168.0.13:10249,192.168.0.14:10249/database
202 |       collection capped_log
203 |       ...
204 |     </source>
205 | 
206 | This allows the plugin to read data from a replica set.
207 | 
208 | You can save last ObjectId to tail over server's shutdown to file.
209 | 
210 |     <source>
211 |       ...
212 | 
213 |       id_store_file /Users/repeatedly/devel/fluent-plugin-mongo/last_id
214 |     </source>
215 | 
216 | Or Mongo collection can be used to keep last ObjectID.
217 | 
218 |     <source>
219 |       ...
220 | 
221 |       id_store_collection last_id
222 |     </source>
223 | 
224 | Make sure the collection is capped. The plugin inserts records but does not remove at all.
225 | 
226 | = NOTE
227 | 
228 | == replace_dot_in_key_with and replace_dollar_in_key_with
229 | 
230 | BSON records which include '.' or start with '$' are invalid and they will be stored as broken data to MongoDB. If you want to sanitize keys, you can use _replace_dot_in_key_with_ and _replace_dollar_in_key_with_.
231 | 
232 |     <match forward.*>
233 |       ...
234 |       # replace '.' in keys with '__dot__'
235 |       replace_dot_in_key_with __dot__
236 | 
237 |       # replace '$' in keys with '__dollar__'
238 |       # Note: This replaces '$' only on first character
239 |       replace_dollar_in_key_with __dollar__
240 |       ...
241 |     </match>
242 | 
243 | == Broken data as a BSON
244 | 
245 | NOTE: This feature will be removed since v0.8
246 | 
247 | Fluentd event sometimes has an invalid record as a BSON.
248 | In such case, Mongo plugin marshals an invalid record using Marshal.dump
249 | and re-inserts its to same collection as a binary.
250 | 
251 | If passed following invalid record:
252 | 
253 |     {"key1": "invalid value", "key2": "valid value", "time": ISODate("2012-01-15T21:09:53Z") }
254 | 
255 | then Mongo plugin converts this record to following format:
256 | 
257 |     {"__broken_data": BinData(0, Marshal.dump result of {"key1": "invalid value", "key2": "valid value"}), "time": ISODate("2012-01-15T21:09:53Z") }
258 | 
259 | Mongo-Ruby-Driver cannot detect an invalid attribute,
260 | so Mongo plugin marshals all attributes excluding Fluentd keys("tag_key" and "time_key").
261 | 
262 | You can deserialize broken data using Mongo and Marshal.load. Sample code is below:
263 | 
264 |     # _collection_ is an instance of Mongo::Collection
265 |     collection.find({'__broken_data' => {'$exists' => true}}).each do |doc|
266 |       p Marshal.load(doc['__broken_data'].to_s) #=> {"key1": "invalid value", "key2": "valid value"}
267 |     end
268 | 
269 | === ignore_invalid_record
270 | 
271 | If you want to ignore an invalid record, set _true_ to _ignore_invalid_record_ parameter in match.
272 | 
273 |     <match forward.*>
274 |       ...
275 | 
276 |       # ignore invalid documents at write operation
277 |       ignore_invalid_record true
278 | 
279 |       ...
280 |     </match>
281 | 
282 | === exclude_broken_fields
283 | 
284 | If you want to exclude some fields from broken data marshaling, use _exclude_broken_fields_ to specfiy the keys.
285 | 
286 |     <match forward.*>
287 |       ...
288 | 
289 |       # key2 is excluded from __broken_data.
290 |       # e.g. {"__broken_data": BinData(0, Marshal.dump result of {"key1": "invalid value"}), "key2": "valid value", "time": ISODate("2012-01-15T21:09:53Z")
291 |       exclude_broken_fields key2
292 | 
293 |       ...
294 |     </match>
295 | 
296 | Specified value is a comma separated keys(e.g. key1,key2,key3).
297 | This parameter is useful for excluding shard keys in shard environment.
298 | 
299 | == Buffer size limitation
300 | 
301 | Mongo plugin has the limitation of buffer size.
302 | Because MongoDB and mongo-ruby-driver checks the total object size at each insertion.
303 | If total object size gets over the size limitation, then
304 | MongoDB returns error or mongo-ruby-driver raises an exception.
305 | 
306 | So, Mongo plugin resets _buffer_chunk_limit_ if configurated value is larger than above limitation:
307 | - Before v1.8, max of _buffer_chunk_limit_ is 2MB
308 | - After  v1.8, max of _buffer_chunk_limit_ is 8MB
309 | 
310 | = Tool
311 | 
312 | You can tail mongo capped collection.
313 | 
314 |     $ mongo-tail -f
315 | 
316 | = Test
317 | 
318 | Run following command:
319 | 
320 |     $ bundle exec rake test
321 | 
322 | You can use 'mongod' environment variable for specified mongod:
323 | 
324 |     $ mongod=/path/to/mongod bundle exec rake test
325 | 
326 | Note that source code in test/tools are from mongo-ruby-driver.
327 | 
328 | = Copyright
329 | 
330 | Copyright:: Copyright (c) 2011- Masahiro Nakagawa
331 | License::   Apache License, Version 2.0
332 | 


--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
 1 | require 'bundler'
 2 | Bundler::GemHelper.install_tasks
 3 | 
 4 | require 'rake/testtask'
 5 | 
 6 | Rake::TestTask.new(:test) do |test|
 7 |   test.libs << 'test'
 8 |   test.test_files = FileList['test/plugin/*.rb']
 9 |   test.verbose = true
10 | end
11 | 
12 | task :coverage do |t|
13 |   ENV['SIMPLE_COV'] = '1'
14 |   Rake::Task["test"].invoke
15 | end
16 | 
17 | task :default => [:build]
18 | 


--------------------------------------------------------------------------------
/VERSION:
--------------------------------------------------------------------------------
1 | 1.6.0
2 | 


--------------------------------------------------------------------------------
/bin/mongo-tail:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env ruby
 2 | 
 3 | # tail like CLI for mongo capped collection
 4 | 
 5 | require 'optparse'
 6 | require 'json'
 7 | 
 8 | require 'mongo'
 9 | 
10 | TailConfig = {
11 |   d: 'fluent',
12 |   c: 'out_mongo_backup',
13 |   h: 'localhost',
14 |   p: 27017,
15 |   n: 10
16 | }
17 | 
18 | OptionParser.new { |opt|
19 |   opt.on('-d VAL', 'The name of database') { |v| TailConfig[:d] = v }
20 |   opt.on('-c VAL', 'The name of collection') { |v| TailConfig[:c] = v }
21 |   opt.on('-h VAL', 'The host of mongodb server') { |v| TailConfig[:h] = v }
22 |   opt.on('-p VAL', 'The port of mongodb server') { |v| TailConfig[:p] = Integer(v) }
23 |   opt.on('-n VAL', 'The number of documents') { |v| TailConfig[:n] = Integer(v) }
24 |   opt.on('-f', 'This option causes tail to not stop when end of collection is reached, but rather to wait for additional data to be appended to the collection') { |v|
25 |     TailConfig[:f] = true
26 |   }
27 | 
28 |   opt.parse!(ARGV)
29 | }
30 | 
31 | def get_client_options(conf)
32 |   client_options = {}
33 |   client_options[:database] = conf[:d]
34 |   client_options
35 | end
36 | 
37 | def get_collection_options
38 |   collection_options = {}
39 |   collection_options[:capped] = true
40 |   collection_options
41 | end
42 | 
43 | def get_capped_collection(conf)
44 |   client_options = get_client_options(conf)
45 |   collection_options = get_collection_options
46 |   client = Mongo::Client.new(["#{conf[:h]}:#{conf[:p]}"], client_options)
47 |   collection = client["#{conf[:c]}", collection_options]
48 |   if collection.capped?
49 |     collection
50 |   else
51 |     puts "#{conf[:c]} is not capped. mongo-tail can not tail normal collection."
52 |   end
53 | end
54 | 
55 | def create_skip_number(collection, conf)
56 |   skip = collection.count - conf[:n]
57 |   skip
58 | end
59 | 
60 | def tail_n(collection, conf)
61 |   collection.find().skip(create_skip_number(collection, conf)).each {|doc|
62 |     puts doc.to_json
63 |   }
64 | end
65 | 
66 | def tail_forever(collection, conf)
67 |   loop {
68 |     option['_id'] = {'$gt' => BSON::ObjectId(@last_id)} if @last_id
69 | 
70 |     documents = collection.find(option)
71 |     if documents.count >= 1
72 |       documents.each {|doc|
73 |         STDOUT.puts doc.to_json
74 |         STDOUT.flush
75 |         if id = doc.delete('_id')
76 |           @last_id = id.to_s
77 |         end
78 |       }
79 |     else
80 |       sleep 1
81 |     end
82 |   }
83 | end
84 | 
85 | exit unless collection = get_capped_collection(TailConfig)
86 | 
87 | if TailConfig[:f]
88 |   tail_forever(collection, TailConfig)
89 | else
90 |   tail_n(collection, TailConfig)
91 | end
92 | 


--------------------------------------------------------------------------------
/fluent-plugin-mongo.gemspec:
--------------------------------------------------------------------------------
 1 | # encoding: utf-8
 2 | $:.push File.expand_path('../lib', __FILE__)
 3 | 
 4 | Gem::Specification.new do |gem|
 5 |   gem.name        = "fluent-plugin-mongo"
 6 |   gem.description = "MongoDB plugin for Fluentd"
 7 |   gem.homepage    = "https://github.com/fluent/fluent-plugin-mongo"
 8 |   gem.summary     = gem.description
 9 |   gem.licenses    = ["Apache-2.0"]
10 |   gem.version     = File.read("VERSION").strip
11 |   gem.authors     = ["Masahiro Nakagawa"]
12 |   gem.email       = "repeatedly@gmail.com"
13 |   #gem.platform    = Gem::Platform::RUBY
14 |   gem.files       = `git ls-files`.split("\n")
15 |   gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
16 |   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17 |   gem.require_paths = ['lib']
18 | 
19 |   gem.add_dependency "fluentd", [">= 0.14.22", "< 2"]
20 |   gem.add_runtime_dependency "mongo", ">= 2.15.0", "< 2.19.0"
21 |   gem.add_development_dependency "rake", ">= 0.9.2"
22 |   gem.add_development_dependency "simplecov", ">= 0.5.4"
23 |   gem.add_development_dependency "rr", ">= 1.0.0"
24 |   gem.add_development_dependency "test-unit", ">= 3.0.0"
25 |   gem.add_development_dependency "timecop", "~> 0.9.4"
26 |   gem.add_development_dependency "webrick", ">= 1.7.0"
27 | end
28 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/in_mongo_tail.rb:
--------------------------------------------------------------------------------
  1 | # coding: utf-8
  2 | require 'mongo'
  3 | require 'bson'
  4 | require 'fluent/plugin/input'
  5 | require 'fluent/plugin/mongo_auth'
  6 | require 'fluent/plugin/logger_support'
  7 | 
  8 | module Fluent::Plugin
  9 |   class MongoTailInput < Input
 10 |     Fluent::Plugin.register_input('mongo_tail', self)
 11 | 
 12 |     helpers :timer
 13 | 
 14 |     include Fluent::MongoAuthParams
 15 |     include Fluent::MongoAuth
 16 |     include Fluent::LoggerSupport
 17 | 
 18 |     desc "MongoDB database"
 19 |     config_param :database, :string, default: nil
 20 |     desc "MongoDB collection"
 21 |     config_param :collection, :string
 22 |     desc "MongoDB host"
 23 |     config_param :host, :string, default: 'localhost'
 24 |     desc "MongoDB port"
 25 |     config_param :port, :integer, default: 27017
 26 |     desc "Tailing interval"
 27 |     config_param :wait_time, :integer, default: 1
 28 |     desc "MongoDB node URL"
 29 |     config_param :url, :string, default: nil
 30 | 
 31 |     desc "Input tag"
 32 |     config_param :tag, :string, default: nil
 33 |     desc "Treat key as tag"
 34 |     config_param :tag_key, :string, default: nil
 35 |     desc "Treat key as time"
 36 |     config_param :time_key, :string, default: nil
 37 |     desc "Time format"
 38 |     config_param :time_format, :string, default: nil
 39 |     config_param :object_id_keys, :array, default: nil
 40 | 
 41 |     desc "To store last ObjectID"
 42 |     config_param :id_store_file, :string, default: nil
 43 | 
 44 |     desc "SSL connection"
 45 |     config_param :ssl, :bool, default: false
 46 | 
 47 |     desc "Batch size for each find"
 48 |     config_param :batch_size, :integer, default: nil
 49 | 
 50 |     def initialize
 51 |       super
 52 | 
 53 |       @client_options = {}
 54 |       @connection_options = {}
 55 |     end
 56 | 
 57 |     def configure(conf)
 58 |       super
 59 | 
 60 |       if !@tag and !@tag_key
 61 |         raise Fluent::ConfigError, "'tag' or 'tag_key' option is required on mongo_tail input"
 62 |       end
 63 | 
 64 |       if @database && @url
 65 |         raise Fluent::ConfigError, "Both 'database' and 'url' can not be set"
 66 |       end
 67 | 
 68 |       if !@database && !@url
 69 |         raise Fluent::ConfigError, "One of 'database' or 'url' must be specified"
 70 |       end
 71 | 
 72 |       @last_id = @id_store_file ? get_last_id : nil
 73 |       @connection_options[:ssl] = @ssl
 74 | 
 75 |       if @batch_size && @batch_size <= 0
 76 |         raise Fluent::ConfigError, "Batch size must be positive."
 77 |       end
 78 | 
 79 |       configure_logger(@mongo_log_level)
 80 |     end
 81 | 
 82 |     def start
 83 |       super
 84 | 
 85 |       @file = get_id_store_file if @id_store_file
 86 |       @collection = get_collection
 87 |       # Resume tailing from last inserted id.
 88 |       # Because tailable option is obsoleted since mongo driver 2.0.
 89 |       @last_id = get_last_inserted_id if !@id_store_file and get_last_inserted_id
 90 |       timer_execute(:in_mongo_tail_watcher, @wait_time, &method(:run))
 91 |     end
 92 | 
 93 |     def shutdown
 94 |       if @id_store_file
 95 |         save_last_id
 96 |         @file.close
 97 |       end
 98 | 
 99 |       @client.close
100 | 
101 |       super
102 |     end
103 | 
104 |     def run
105 |       option = {}
106 |       begin
107 |         option['_id'] = {'$gt' => BSON::ObjectId(@last_id)} if @last_id
108 |         documents = @collection.find(option)
109 |         documents = documents.limit(@batch_size) if @batch_size
110 |         if documents.count >= 1
111 |           process_documents(documents)
112 |         end
113 |       rescue
114 |         # ignore Exceptions
115 |       end
116 |     end
117 | 
118 |     private
119 | 
120 |     def client
121 |       @client_options[:database] = @database if @database
122 |       @client_options[:user] = @user if @user
123 |       @client_options[:password] = @password if @password
124 |       Mongo::Client.new(node_string, @client_options)
125 |     end
126 | 
127 |     def get_collection
128 |       @client = client
129 |       @client = authenticate(@client)
130 |       @client["#{@collection}"]
131 |     end
132 | 
133 |     def node_string
134 |       case
135 |       when @database
136 |         ["#{@host}:#{@port}"]
137 |       when @url
138 |         @url
139 |       end
140 |     end
141 | 
142 |     def process_documents(documents)
143 |       es = Fluent::MultiEventStream.new
144 |       documents.each {|doc|
145 |         time = if @time_key
146 |                  t = doc.delete(@time_key)
147 |                  t.nil? ? Fluent::Engine.now : t.to_i
148 |                else
149 |                  Fluent::Engine.now
150 |                end
151 |         @tag = if @tag_key
152 |                 t = doc.delete(@tag_key)
153 |                 t.nil? ? 'mongo.missing_tag' : t
154 |               else
155 |                 @tag
156 |               end
157 |         if @object_id_keys
158 |           @object_id_keys.each {|id_key|
159 |             doc[id_key] = doc[id_key].to_s
160 |           }
161 |         end
162 | 
163 |         if id = doc.delete('_id')
164 |           @last_id = id.to_s
165 |           doc['_id_str'] = @last_id
166 |           save_last_id if @id_store_file
167 |         end
168 |         es.add(time, doc)
169 |       }
170 |       router.emit_stream(@tag, es)
171 |     end
172 | 
173 |     def get_last_inserted_id
174 |       last_inserted_id = nil
175 |       documents = @collection.find()
176 |       if documents.count >= 1
177 |         documents.each {|doc|
178 |           if id = doc.delete('_id')
179 |             last_inserted_id = id
180 |           end
181 |         }
182 |       end
183 |       last_inserted_id
184 |     end
185 | 
186 |     def get_id_store_file
187 |       file = File.open(@id_store_file, 'w')
188 |       file.sync
189 |       file
190 |     end
191 | 
192 |     def get_last_id
193 |       if File.exist?(@id_store_file)
194 |         BSON::ObjectId(File.read(@id_store_file)).to_s rescue nil
195 |       else
196 |         nil
197 |       end
198 |     end
199 | 
200 |     def save_last_id
201 |       @file.pos = 0
202 |       @file.write(@last_id)
203 |     end
204 |   end
205 | end
206 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/logger_support.rb:
--------------------------------------------------------------------------------
 1 | module Fluent
 2 |   module LoggerSupport
 3 |     def self.included(klass)
 4 |       klass.instance_eval {
 5 |         desc "MongoDB log level"
 6 |         config_param :mongo_log_level, :string, default: 'info'
 7 |       }
 8 |     end
 9 | 
10 |     def configure_logger(mongo_log_level)
11 |       Mongo::Logger.level = case @mongo_log_level.downcase
12 |                             when 'fatal'
13 |                               Logger::FATAL
14 |                             when 'error'
15 |                               Logger::ERROR
16 |                             when 'warn'
17 |                               Logger::WARN
18 |                             when 'info'
19 |                               Logger::INFO
20 |                             when 'debug'
21 |                               Logger::DEBUG
22 |                             end
23 |     end
24 |   end
25 | end
26 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/mongo_auth.rb:
--------------------------------------------------------------------------------
 1 | module Fluent
 2 |   module MongoAuthParams
 3 |     def self.included(klass)
 4 |       klass.instance_eval {
 5 |         desc "MongoDB user"
 6 |         config_param :user, :string, default: nil
 7 |         desc "MongoDB password"
 8 |         config_param :password, :string, default: nil, secret: true
 9 |         desc "MongoDB authentication database"
10 |         config_param :auth_source, :string, default: nil
11 |         desc "MongoDB authentication mechanism"
12 |         config_param :auth_mech, :string, default: nil
13 |       }
14 |     end
15 |   end
16 | 
17 |   module MongoAuth
18 |     def authenticate(client)
19 |       begin
20 |         if [@user, @password, @auth_source].all?
21 |           client = client.with(user: @user, password: @password, auth_source: @auth_source)
22 |         elsif [@user, @password].all?
23 |           client = client.with(user: @user, password: @password)
24 |         elsif [@user, @auth_source, @auth_mech].all?
25 |           client = client.with(user: @user, auth_source: @auth_source, auth_mech: @auth_mech.to_sym)
26 |         end
27 |       rescue Mongo::Auth::Unauthorized => e
28 |         log.fatal e
29 |         exit!
30 |       end
31 |       client
32 |     end
33 |   end
34 | end
35 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/out_mongo.rb:
--------------------------------------------------------------------------------
  1 | require 'mongo'
  2 | require 'msgpack'
  3 | require 'fluent/plugin/output'
  4 | require 'fluent/plugin/mongo_auth'
  5 | require 'fluent/plugin/logger_support'
  6 | 
  7 | module Fluent::Plugin
  8 |   class MongoOutput < Output
  9 |     Fluent::Plugin.register_output('mongo', self)
 10 | 
 11 |     helpers :event_emitter, :inject, :compat_parameters, :record_accessor
 12 | 
 13 |     include Fluent::MongoAuthParams
 14 |     include Fluent::MongoAuth
 15 |     include Fluent::LoggerSupport
 16 | 
 17 |     DEFAULT_BUFFER_TYPE = "memory"
 18 | 
 19 |     config_set_default :include_tag_key, false
 20 |     config_set_default :include_time_key, true
 21 | 
 22 |     desc "MongoDB connection string"
 23 |     config_param :connection_string, :default => nil
 24 |     desc "MongoDB database"
 25 |     config_param :database, :string, :default => nil
 26 |     desc "MongoDB collection"
 27 |     config_param :collection, :string, default: 'untagged'
 28 |     desc "MongoDB host"
 29 |     config_param :host, :string, default: 'localhost'
 30 |     desc "MongoDB port"
 31 |     config_param :port, :integer, default: 27017
 32 |     desc "MongoDB write_concern"
 33 |     config_param :write_concern, :integer, default: nil
 34 |     desc "MongoDB journaled"
 35 |     config_param :journaled, :bool, default: false
 36 |     desc "Replace dot with specified string"
 37 |     config_param :replace_dot_in_key_with, :string, default: nil
 38 |     desc "Replace dollar with specified string"
 39 |     config_param :replace_dollar_in_key_with, :string, default: nil
 40 | 
 41 |     # Additional date field to be used to Date object
 42 |     desc "Specify keys to use MongoDB's Date. Supported value types are Integer/Float/EventTime/String"
 43 |     config_param :date_keys, :array, default: nil
 44 |     desc "Specify if the fields in date_keys are of type Integer or Float"
 45 |     config_param :parse_string_number_date, :bool, default: false
 46 |     desc "Specify keys to use MongoDB's ObjectId"
 47 |     config_param :object_id_keys, :array, default: nil
 48 | 
 49 |     # tag mapping mode
 50 |     desc "Use tag_mapped mode"
 51 |     config_param :tag_mapped, :bool, default: false,
 52 |                  deprecated: "use '${tag}' placeholder in collection parameter."
 53 |     desc "Remove tag prefix"
 54 |     config_param :remove_tag_prefix, :string, default: nil,
 55 |                  deprecated: "use @label instead for event routing."
 56 |     # expire indexes
 57 |     desc "Specify expire after seconds"
 58 |     config_param :expire_after, :time, default: 0
 59 | 
 60 |     # SSL connection
 61 |     config_param :ssl, :bool, default: false
 62 |     config_param :ssl_cert, :string, default: nil
 63 |     config_param :ssl_key, :string, default: nil
 64 |     config_param :ssl_key_pass_phrase, :string, default: nil, secret: true
 65 |     config_param :ssl_verify, :bool, default: false
 66 |     config_param :ssl_ca_cert, :string, default: nil
 67 | 
 68 | 
 69 |     config_section :buffer do
 70 |       config_set_default :@type, DEFAULT_BUFFER_TYPE
 71 |       config_set_default :chunk_keys, ['tag']
 72 |     end
 73 | 
 74 |     attr_reader :client_options, :collection_options
 75 | 
 76 |     def initialize
 77 |       super
 78 | 
 79 |       @nodes = nil
 80 |       @client_options = {}
 81 |       @collection_options = {capped: false}
 82 |       @date_accessors = {}
 83 |       @object_id_accessors = {}
 84 |     end
 85 | 
 86 |     # Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
 87 |     LIMIT_BEFORE_v1_8 = 2 * 1024 * 1024  # 2MB = 4MB  / 2
 88 |     LIMIT_AFTER_v1_8 =  8 * 1024 * 1024  # 8MB = 16MB / 2
 89 | 
 90 |     def configure(conf)
 91 |       if conf.has_key?('buffer_chunk_limit')
 92 |         configured_chunk_limit_size = Fluent::Config.size_value(conf['buffer_chunk_limit'])
 93 |         estimated_limit_size = LIMIT_AFTER_v1_8
 94 |         estimated_limit_size_conf = '8m'
 95 |         if conf.has_key?('mongodb_smaller_bson_limit') && Fluent::Config.bool_value(conf['mongodb_smaller_bson_limit'])
 96 |           estimated_limit_size = LIMIT_BEFORE_v1_8
 97 |           estimated_limit_size_conf = '2m'
 98 |         end
 99 |         if configured_chunk_limit_size > estimated_limit_size
100 |           log.warn ":buffer_chunk_limit(#{conf['buffer_chunk_limit']}) is large. Reset :buffer_chunk_limit with #{estimated_limit_size_conf}"
101 |           conf['buffer_chunk_limit'] = estimated_limit_size_conf
102 |         end
103 |       else
104 |         if conf.has_key?('mongodb_smaller_bson_limit') && Fluent::Config.bool_value(conf['mongodb_smaller_bson_limit'])
105 |           conf['buffer_chunk_limit'] = '2m'
106 |         else
107 |           conf['buffer_chunk_limit'] = '8m'
108 |         end
109 |       end
110 |       # 'config_set_default :include_time_key, true' is ignored in compat_parameters_convert so need manual setting
111 |       if conf.elements('inject').empty?
112 |         if conf.has_key?('include_time_key')
113 |           if Fluent::Config.bool_value(conf['include_time_key']) && !conf.has_key?('time_key')
114 |             conf['time_key'] = 'time'
115 |           end
116 |         else
117 |           conf['time_key'] = 'time'
118 |         end
119 |       end
120 | 
121 |       compat_parameters_convert(conf, :inject)
122 | 
123 |       super
124 | 
125 |       if @auth_mech && !Mongo::Auth::SOURCES.has_key?(@auth_mech.to_sym)
126 |         raise Fluent::ConfigError, Mongo::Auth::InvalidMechanism.new(@auth_mech.to_sym)
127 |       end
128 | 
129 |       if @connection_string.nil? && @database.nil?
130 |         raise Fluent::ConfigError,  "connection_string or database parameter is required"
131 |       end
132 | 
133 |       if conf.has_key?('tag_mapped')
134 |         log.warn "'tag_mapped' feature is replaced with built-in config placeholder. Please consider to use 'collection ${tag}'."
135 |         @collection = '${tag}'
136 |       end
137 |       raise Fluent::ConfigError, "normal mode requires collection parameter" if !@tag_mapped and !conf.has_key?('collection')
138 | 
139 |       if conf.has_key?('capped')
140 |         raise Fluent::ConfigError, "'capped_size' parameter is required on <store> of Mongo output" unless conf.has_key?('capped_size')
141 |         @collection_options[:capped] = true
142 |         @collection_options[:size] = Fluent::Config.size_value(conf['capped_size'])
143 |         @collection_options[:max] = Fluent::Config.size_value(conf['capped_max']) if conf.has_key?('capped_max')
144 |       end
145 | 
146 |       if remove_tag_prefix = conf['remove_tag_prefix']
147 |         @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
148 |       end
149 | 
150 |       @client_options[:write] = {j: @journaled}
151 |       @client_options[:write].merge!({w: @write_concern}) unless @write_concern.nil?
152 |       @client_options[:ssl] = @ssl
153 | 
154 |       if @ssl
155 |         @client_options[:ssl_cert] = @ssl_cert
156 |         @client_options[:ssl_key] = @ssl_key
157 |         @client_options[:ssl_key_pass_phrase] = @ssl_key_pass_phrase
158 |         @client_options[:ssl_verify] = @ssl_verify
159 |         @client_options[:ssl_ca_cert] = @ssl_ca_cert
160 |       end
161 |       @nodes = ["#{@host}:#{@port}"] if @nodes.nil?
162 | 
163 |       configure_logger(@mongo_log_level)
164 | 
165 |       log.debug "Setup mongo configuration: mode = #{@tag_mapped ? 'tag mapped' : 'normal'}"
166 | 
167 |       if @date_keys
168 |         @date_keys.each { |field_name|
169 |           @date_accessors[field_name.to_s] = record_accessor_create(field_name)
170 |         }
171 |         log.debug "Setup record accessor for every date key"
172 |       end
173 |       if @object_id_keys
174 |         @object_id_keys.each { |field_name|
175 |           @object_id_accessors[field_name.to_s] = record_accessor_create(field_name)
176 |         }
177 |         log.debug "Setup record accessor for every object_id key"
178 |       end
179 |     end
180 | 
181 |     def start
182 |       @client = client
183 |       @client = authenticate(@client)
184 |       @collections = {}
185 |       super
186 |     end
187 | 
188 |     def shutdown
189 |       @client.close
190 |       super
191 |     end
192 | 
193 |     def formatted_to_msgpack_binary
194 |       true
195 |     end
196 | 
197 |     def multi_workers_ready?
198 |       true
199 |     end
200 | 
201 |     def write(chunk)
202 |       collection_name = extract_placeholders(@collection, chunk)
203 |       # In connection_string case, we shouldn't handle extract_placeholers for @database.
204 |       database_name = extract_placeholders(@database, chunk) unless @connection_string
205 |       operate(database_name, format_collection_name(collection_name), collect_records(chunk))
206 |     end
207 | 
208 |     private
209 | 
210 |     def client(database = @database)
211 |       if @connection_string
212 |         Mongo::Client.new(@connection_string)
213 |       else
214 |         @client_options[:database] = database
215 |         @client_options[:user] = @user if @user
216 |         @client_options[:password] = @password if @password
217 |         Mongo::Client.new(@nodes, @client_options)
218 |       end
219 |     end
220 | 
221 |     def collect_records(chunk)
222 |       records = []
223 |       time_key = @inject_config.time_key if @inject_config
224 |       date_keys = @date_keys
225 |       object_id_keys = @object_id_keys
226 | 
227 |       tag = chunk.metadata.tag
228 |       chunk.msgpack_each {|time, record|
229 |         record = inject_values_to_record(tag, time, record)
230 |         # MongoDB uses BSON's Date for time.
231 |         record[time_key] = Time.at(time || record[time_key]) if time_key
232 | 
233 |         if date_keys
234 |           @date_accessors.each_pair { |date_key, date_key_accessor|
235 |             begin
236 |               date_value = date_key_accessor.call(record)
237 |               case date_value
238 |               when Fluent::EventTime
239 |                 value_to_set = date_value.to_time
240 |               when Integer
241 |                 value_to_set = if date_value > 9999999999
242 |                                  # epoch with milliseconds: e.g. javascript
243 |                                  Time.at(date_value / 1000.0)
244 |                                else
245 |                                  # epoch with seconds: e.g. ruby
246 |                                  Time.at(date_value)
247 |                                end
248 |               when Float
249 |                 value_to_set = Time.at(date_value)
250 |               else
251 |                 if @parse_string_number_date
252 |                   if date_value.to_i.to_s == date_value
253 |                     date_value = date_value.to_i
254 |                     value_to_set = if date_value > 9999999999
255 |                                      # epoch with milliseconds: e.g. javascript
256 |                                      date_value / 1000.0
257 |                                    else
258 |                                      # epoch with seconds: e.g. ruby
259 |                                      date_value
260 |                                    end
261 |                   elsif date_value.to_f.to_s == date_value
262 |                     date_value = date_value.to_f
263 |                   end
264 |                   value_to_set = date_value.is_a?(String) ? Time.parse(date_value) : Time.at(date_value)
265 |                 else
266 |                   value_to_set = Time.parse(date_value)
267 |                 end
268 |               end
269 | 
270 |               date_key_accessor.set(record, value_to_set)
271 |             rescue ArgumentError
272 |               log.warn "Failed to parse '#{date_key}' field. Expected date types are Integer/Float/String/EventTime: #{date_value}"
273 |               date_key_accessor.set(record, nil)
274 |             end
275 |           }
276 |         end
277 |         if object_id_keys
278 |           @object_id_accessors.each_pair { |object_id_key, object_id_key_accessor|
279 |             begin
280 |               object_id_value = object_id_key_accessor.call(record)
281 |               value_to_set = BSON::ObjectId(object_id_value)
282 |               object_id_key_accessor.set(record, value_to_set)
283 |             rescue BSON::ObjectId::Invalid
284 |               log.warn "Failed to parse '#{object_id_key}' field. Expected object_id types are String: #{object_id_value}"
285 |               object_id_key_accessor.set(record, nil)
286 |             end
287 |           }
288 |         end
289 |         records << record
290 |       }
291 |       records
292 |     end
293 | 
294 |     FORMAT_COLLECTION_NAME_RE = /(^\.+)|(\.+$)/
295 | 
296 |     def format_collection_name(collection_name)
297 |       formatted = collection_name
298 |       formatted = formatted.gsub(@remove_tag_prefix, '') if @remove_tag_prefix
299 |       formatted = formatted.gsub(FORMAT_COLLECTION_NAME_RE, '')
300 |       formatted = @collection if formatted.size == 0 # set default for nil tag
301 |       formatted
302 |     end
303 | 
304 |     def list_collections_enabled?
305 |       @client.cluster.next_primary(false).features.list_collections_enabled?
306 |     end
307 | 
308 |     def collection_exists?(name)
309 |       if list_collections_enabled?
310 |         r = @client.database.command(
311 |           { :listCollections => 1, :filter => { :name => name } }
312 |         ).first
313 |         r[:ok] && r[:cursor][:firstBatch].size == 1
314 |       else
315 |         @client.database.collection_names.include?(name)
316 |       end
317 |     end
318 | 
319 |     def get_collection(database, name,  options)
320 |       @client = client(database) if database && @database != database
321 |       return @client[name] if @collections[name]
322 | 
323 |       unless collection_exists?(name)
324 |         log.trace "Create collection #{name} with options #{options}"
325 |         @client[name, options].create
326 |         if @expire_after > 0 && @inject_config
327 |           log.trace "Create expiring index with key: \"#{@inject_config.time_key}\" and seconds: \"#{@expire_after}\""
328 |           @client[name].indexes.create_one(
329 |             {"#{@inject_config.time_key}": 1},
330 |             expire_after: @expire_after
331 |           )
332 |         end
333 |       end
334 |       @collections[name] = true
335 |       @client[name]
336 |     end
337 | 
338 |     def forget_collection(name)
339 |       @collections.delete(name)
340 |     end
341 | 
342 |     def operate(database, collection, records)
343 |       begin
344 |         if @replace_dot_in_key_with
345 |           records.map! do |r|
346 |             replace_key_of_hash(r, ".", @replace_dot_in_key_with)
347 |           end
348 |         end
349 |         if @replace_dollar_in_key_with
350 |           records.map! do |r|
351 |             replace_key_of_hash(r, /^\$/, @replace_dollar_in_key_with)
352 |           end
353 |         end
354 | 
355 |         get_collection(database, collection, @collection_options).insert_many(records)
356 |       rescue Mongo::Error::BulkWriteError => e
357 |         log.warn "#{records.size - e.result["n_inserted"]} documents are not inserted. Maybe these documents are invalid as a BSON."
358 |         forget_collection(collection)
359 |       rescue ArgumentError => e
360 |         log.warn e
361 |       end
362 |       records
363 |     end
364 | 
365 |     def replace_key_of_hash(hash_or_array, pattern, replacement)
366 |       case hash_or_array
367 |       when Array
368 |         hash_or_array.map do |elm|
369 |           replace_key_of_hash(elm, pattern, replacement)
370 |         end
371 |       when Hash
372 |         result = Hash.new
373 |         hash_or_array.each_pair do |k, v|
374 |           k = k.gsub(pattern, replacement)
375 | 
376 |           if v.is_a?(Hash) || v.is_a?(Array)
377 |             result[k] = replace_key_of_hash(v, pattern, replacement)
378 |           else
379 |             result[k] = v
380 |           end
381 |         end
382 |         result
383 |       else
384 |         hash_or_array
385 |       end
386 |     end
387 |   end
388 | end
389 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/out_mongo_replset.rb:
--------------------------------------------------------------------------------
 1 | require 'fluent/plugin/out_mongo'
 2 | 
 3 | module Fluent::Plugin
 4 |   class MongoOutputReplset < MongoOutput
 5 |     Fluent::Plugin.register_output('mongo_replset', self)
 6 | 
 7 |     config_set_default :include_tag_key, false
 8 |     config_set_default :include_time_key, true
 9 | 
10 |     desc "Nodes of replica set"
11 |     config_param :nodes, :array, value_type: :string, :default => nil
12 |     desc "Replica set name"
13 |     config_param :replica_set, :string
14 |     desc "Read from specified role"
15 |     config_param :read, :string, :default => nil
16 |     desc "Retry number"
17 |     config_param :num_retries, :integer, :default => 60
18 | 
19 |     def configure(conf)
20 |       super
21 | 
22 |       if replica_set = conf['replica_set']
23 |         @client_options[:replica_set] = replica_set
24 |       end
25 |       if read = conf['read']
26 |         @client_options[:read] = read.to_sym
27 |       end
28 | 
29 |       log.debug "Setup replica set configuration: #{conf['replica_set']}"
30 |     end
31 | 
32 |     def write(chunk)
33 |       super
34 |     end
35 | 
36 |     private
37 | 
38 |     def operate(database, client, records)
39 |       rescue_connection_failure do
40 |         super(database, client, records)
41 |       end
42 |     end
43 | 
44 |     def rescue_connection_failure
45 |       retries = 0
46 |       begin
47 |         yield
48 |       rescue Mongo::Error::OperationFailure => e
49 |         retries += 1
50 |         raise e if retries > @num_retries
51 | 
52 |         log.warn "Failed to operate to Replica Set. Try to retry: retry count = #{retries}"
53 | 
54 |         sleep 0.5
55 |         retry
56 |       end
57 |     end
58 |   end
59 | end
60 | 


--------------------------------------------------------------------------------
/test/helper.rb:
--------------------------------------------------------------------------------
1 | require 'test/unit'
2 | require 'fluent/test'
3 | require 'mongo'
4 | require 'fluent/plugin/out_mongo'
5 | require 'fluent/plugin/out_mongo_replset'
6 | require 'fluent/plugin/in_mongo_tail'
7 | 


--------------------------------------------------------------------------------
/test/plugin/test_in_mongo_tail.rb:
--------------------------------------------------------------------------------
  1 | require "helper"
  2 | require "fluent/test/driver/input"
  3 | require "fluent/test/helpers"
  4 | require "timecop"
  5 | 
  6 | class MongoTailInputTest < Test::Unit::TestCase
  7 |   def setup
  8 |     Fluent::Test.setup
  9 |     setup_mongod
 10 |   end
 11 | 
 12 |   def teardown
 13 |     teardown_mongod
 14 |   end
 15 | 
 16 |   def collection_name
 17 |     'test'
 18 |   end
 19 | 
 20 |   def database_name
 21 |     'fluent_test'
 22 |   end
 23 | 
 24 |   def port
 25 |     27017
 26 |   end
 27 | 
 28 |   def default_config
 29 |     %[
 30 |       type mongo_tail
 31 |       database test
 32 |       collection log
 33 |       tag_key tag
 34 |       time_key time
 35 |       id_store_file /tmp/fluent_mongo_last_id
 36 |     ]
 37 |   end
 38 | 
 39 |   def setup_mongod
 40 |     options = {}
 41 |     options[:database] = database_name
 42 |     @client = ::Mongo::Client.new(["localhost:#{port}"], options)
 43 |   end
 44 | 
 45 |   def teardown_mongod
 46 |     @client[collection_name].drop
 47 |   end
 48 | 
 49 |   def create_driver(conf=default_config)
 50 |     Fluent::Test::Driver::Input.new(Fluent::Plugin::MongoTailInput).configure(conf)
 51 |   end
 52 | 
 53 |   def test_configure
 54 |     d = create_driver
 55 |     assert_equal('localhost', d.instance.host)
 56 |     assert_equal(27017, d.instance.port)
 57 |     assert_equal('test', d.instance.database)
 58 |     assert_equal('log', d.instance.collection)
 59 |     assert_equal('tag', d.instance.tag_key)
 60 |     assert_equal('time', d.instance.time_key)
 61 |     assert_equal('/tmp/fluent_mongo_last_id', d.instance.id_store_file)
 62 |   end
 63 | 
 64 |   def test_configure_with_logger_conf
 65 |     d = create_driver(default_config + %[
 66 |       mongo_log_level error
 67 |     ])
 68 | 
 69 |     expected = "error"
 70 |     assert_equal(expected, d.instance.mongo_log_level)
 71 |   end
 72 | 
 73 |   class TailInputTest < self
 74 |     include Fluent::Test::Helpers
 75 | 
 76 |     def setup_mongod
 77 |       options = {}
 78 |       options[:database] = database_name
 79 |       @client = ::Mongo::Client.new(["localhost:#{port}"], options)
 80 |       @time = Time.now
 81 |       Timecop.freeze(@time)
 82 |     end
 83 | 
 84 |     def teardown_mongod
 85 |       @client[collection_name].drop
 86 |       Timecop.return
 87 |     end
 88 | 
 89 |     def test_emit
 90 |       d = create_driver(%[
 91 |         @type mongo_tail
 92 |         database #{database_name}
 93 |         collection #{collection_name}
 94 |         tag input.mongo
 95 |         time_key time
 96 |         batch_size 10000
 97 |       ])
 98 |       d.run(expect_records: 1, timeout: 5) do
 99 |         @client[collection_name].insert_one({message: "test"})
100 |       end
101 |       events = d.events
102 |       assert_equal "input.mongo", events[0][0]
103 |       assert events[0][1].is_a?(Fluent::EventTime)
104 |       assert_equal "test", events[0][2]["message"]
105 |     end
106 | 
107 |     def test_emit_with_tag_time_keys
108 |       d = create_driver(%[
109 |         @type mongo_tail
110 |         database #{database_name}
111 |         collection #{collection_name}
112 |         tag input.mongo
113 |         tag_key tag
114 |         time_key time
115 |       ])
116 |       d.run(expect_records: 1, timeout: 5) do
117 |         @client[collection_name].insert_one({message: "test", tag: "user.defined", time: Time.at(Fluent::Engine.now)})
118 |       end
119 |       events = d.events
120 |       assert_equal "user.defined", events[0][0]
121 |       assert_equal event_time(@time.to_s), events[0][1]
122 |       assert_equal "test", events[0][2]["message"]
123 |     end
124 | 
125 |     def test_emit_after_last_id
126 |       d = create_driver(%[
127 |         @type mongo_tail
128 |         database #{database_name}
129 |         collection #{collection_name}
130 |         tag input.mongo.last_id
131 |         time_key time
132 |       ])
133 |       @client[collection_name].insert_one({message: "can't obtain"})
134 | 
135 |       d.run(expect_records: 1, timeout: 5) do
136 |         @client[collection_name].insert_one({message: "can obtain"})
137 |       end
138 |       events = d.events
139 |       assert_equal 1, events.size
140 |       assert_equal "input.mongo.last_id", events[0][0]
141 |       assert events[0][1].is_a?(Fluent::EventTime)
142 |       assert_equal "can obtain", events[0][2]["message"]
143 |     end
144 |   end
145 | 
146 |   class MongoAuthenticateTest < self
147 |     require 'fluent/plugin/mongo_auth'
148 |     include ::Fluent::MongoAuth
149 | 
150 |     def setup_mongod
151 |       options = {}
152 |       options[:database] = database_name
153 |       @client = ::Mongo::Client.new(["localhost:#{port}"], options)
154 |       @client.database.users.create('fluent', password: 'password',
155 |                                     roles: [Mongo::Auth::Roles::READ_WRITE])
156 |     end
157 | 
158 |     def teardown_mongod
159 |       @client[collection_name].drop
160 |       @client.database.users.remove('fluent')
161 |     end
162 | 
163 |     def test_authenticate
164 |       d = create_driver(default_config + %[
165 |         user fluent
166 |         password password
167 |       ])
168 | 
169 |       assert authenticate(@client)
170 |     end
171 |   end
172 | end
173 | 


--------------------------------------------------------------------------------
/test/plugin/test_out_mongo.rb:
--------------------------------------------------------------------------------
  1 | # coding: utf-8
  2 | require "helper"
  3 | require "fluent/test/driver/output"
  4 | require "fluent/test/helpers"
  5 | 
  6 | class MongoOutputTest < ::Test::Unit::TestCase
  7 |   include Fluent::Test::Helpers
  8 | 
  9 |   def setup
 10 |     Fluent::Test.setup
 11 |     setup_mongod
 12 |   end
 13 | 
 14 |   def teardown
 15 |     teardown_mongod
 16 |   end
 17 | 
 18 |   def collection_name
 19 |     'test'
 20 |   end
 21 | 
 22 |   def database_name
 23 |     'fluent_test'
 24 |   end
 25 | 
 26 |   def port
 27 |     27017
 28 |   end
 29 | 
 30 |   def default_config
 31 |     %[
 32 |       type mongo
 33 |       database #{database_name}
 34 |       collection #{collection_name}
 35 |       include_time_key true
 36 |     ]
 37 |   end
 38 | 
 39 |   def setup_mongod(database = database_name)
 40 |     options = {}
 41 |     options[:database] = database
 42 |     @client = ::Mongo::Client.new(["localhost:#{port}"], options)
 43 |   end
 44 | 
 45 |   def teardown_mongod(collection = collection_name)
 46 |     @client[collection].drop
 47 |   end
 48 | 
 49 |   def create_driver(conf=default_config)
 50 |     Fluent::Test::Driver::Output.new(Fluent::Plugin::MongoOutput).configure(conf)
 51 |   end
 52 | 
 53 |   def test_configure
 54 |     d = create_driver(%[
 55 |       @type mongo
 56 |       database fluent_test
 57 |       collection test_collection
 58 | 
 59 |       capped
 60 |       capped_size 100
 61 |     ])
 62 | 
 63 |     assert_equal('fluent_test', d.instance.database)
 64 |     assert_equal('test_collection', d.instance.collection)
 65 |     assert_equal('localhost', d.instance.host)
 66 |     assert_equal(port, d.instance.port)
 67 |     assert_equal({capped: true, size: 100}, d.instance.collection_options)
 68 |     assert_equal({ssl: false, write: {j: false}}, d.instance.client_options)
 69 |     assert_nil d.instance.connection_string
 70 |   end
 71 | 
 72 |   def test_configure_with_connection_string
 73 |     d = create_driver(%[
 74 |       @type mongo
 75 |       connection_string mongodb://localhost/fluent_test
 76 |       collection test_collection
 77 |       capped
 78 |       capped_size 100
 79 |     ])
 80 |     assert_equal('mongodb://localhost/fluent_test', d.instance.connection_string)
 81 |     assert_nil d.instance.database
 82 |   end
 83 | 
 84 |   def test_configure_without_connection_string_or_database
 85 |     assert_raise Fluent::ConfigError do
 86 |       d = create_driver(%[
 87 |         @type mongo
 88 |         collection test_collection
 89 |         capped
 90 |         capped_size 100
 91 |       ])
 92 |     end
 93 |   end
 94 | 
 95 |   def test_configure_auth_mechanism
 96 |     Mongo::Auth::SOURCES.each do |key, value|
 97 |       conf = default_config + %[
 98 |         auth_mech #{key}
 99 |       ]
100 |       d = create_driver(conf)
101 |       assert_equal(key.to_s, d.instance.auth_mech)
102 |     end
103 |     assert_raise Fluent::ConfigError do
104 |       conf = default_config + %[
105 |         auth_mech invalid
106 |       ]
107 |       d = create_driver(conf)
108 |     end
109 |   end
110 | 
111 |   def test_configure_with_ssl
112 |     conf = default_config + %[
113 |       ssl true
114 |     ]
115 |     d = create_driver(conf)
116 |     expected = {
117 |       write: {
118 |         j: false,
119 |       },
120 |       ssl: true,
121 |       ssl_cert: nil,
122 |       ssl_key: nil,
123 |       ssl_key_pass_phrase: nil,
124 |       ssl_verify: false,
125 |       ssl_ca_cert: nil,
126 |     }
127 |     assert_equal(expected, d.instance.client_options)
128 |   end
129 | 
130 |   def test_configure_with_tag_mapped
131 |     conf = default_config + %[
132 |       tag_mapped true
133 |       remove_tag_prefix raw.
134 |     ]
135 |     d = create_driver(conf)
136 |     assert_true(d.instance.tag_mapped)
137 |     assert_equal(/^raw\./, d.instance.remove_tag_prefix)
138 |     assert_equal('${tag}', d.instance.collection)
139 |   end
140 | 
141 |   def test_configure_with_write_concern
142 |     d = create_driver(default_config + %[
143 |       write_concern 2
144 |     ])
145 | 
146 |     expected = {
147 |       ssl: false,
148 |       write: {
149 |         j: false,
150 |         w: 2,
151 |       },
152 |     }
153 |     assert_equal(expected, d.instance.client_options)
154 |   end
155 | 
156 |   def test_configure_with_journaled
157 |     d = create_driver(default_config + %[
158 |       journaled true
159 |     ])
160 | 
161 |     expected = {
162 |       ssl: false,
163 |       write: {
164 |         j: true,
165 |       },
166 |     }
167 |     assert_equal(expected, d.instance.client_options)
168 |   end
169 | 
170 |   def test_configure_with_logger_conf
171 |     d = create_driver(default_config + %[
172 |       mongo_log_level fatal
173 |     ])
174 | 
175 |     expected = "fatal"
176 |     assert_equal(expected, d.instance.mongo_log_level)
177 |   end
178 | 
179 |   def get_documents(collection = collection_name)
180 |     @client[collection].find.to_a.map {|e| e.delete('_id'); e}
181 |   end
182 | 
183 |   def get_indexes(collection = collection_name)
184 |     @client[collection].indexes
185 |   end
186 | 
187 |   def emit_documents(d)
188 |     time = event_time("2011-01-02 13:14:15 UTC")
189 |     d.feed(time, {'a' => 1})
190 |     d.feed(time, {'a' => 2})
191 |     time
192 |   end
193 | 
194 |   def test_write
195 |     d = create_driver
196 |     d.run(default_tag: 'test') do
197 |       emit_documents(d)
198 |     end
199 |     actual_documents = get_documents
200 |     time = event_time("2011-01-02 13:14:15 UTC")
201 |     expected = [{'a' => 1, d.instance.inject_config.time_key => Time.at(time).localtime},
202 |                 {'a' => 2, d.instance.inject_config.time_key => Time.at(time).localtime}]
203 |     assert_equal(expected, actual_documents)
204 |   end
205 | 
206 |   def test_write_with_connection_string
207 |     d = create_driver(%[
208 |       @type mongo
209 |       connection_string mongodb://localhost:#{port}/#{database_name}
210 |       collection #{collection_name}
211 |       capped
212 |       capped_size 100
213 |     ])
214 |     assert_equal("mongodb://localhost:#{port}/#{database_name}", d.instance.connection_string)
215 |     assert_nil d.instance.database
216 | 
217 |     d.run(default_tag: 'test') do
218 |       emit_documents(d)
219 |     end
220 |     actual_documents = get_documents
221 |     time = event_time("2011-01-02 13:14:15 UTC")
222 |     expected = [{'a' => 1, d.instance.inject_config.time_key => Time.at(time).localtime},
223 |                 {'a' => 2, d.instance.inject_config.time_key => Time.at(time).localtime}]
224 |     assert_equal(expected, actual_documents)
225 |   end
226 | 
227 |   def test_write_with_expire_index
228 |     d = create_driver(%[
229 |       @type mongo
230 |       connection_string mongodb://localhost:#{port}/#{database_name}
231 |       collection #{collection_name}
232 |       capped
233 |       capped_size 100
234 |       expire_after 120
235 |     ])
236 |     assert_equal("mongodb://localhost:#{port}/#{database_name}", d.instance.connection_string)
237 |     assert_nil d.instance.database
238 |         d.run(default_tag: 'test') do
239 |       emit_documents(d)
240 |     end
241 |     actual_documents = get_documents
242 |     time = event_time("2011-01-02 13:14:15 UTC")
243 |     expected = [{'a' => 1, d.instance.inject_config.time_key => Time.at(time).localtime},
244 |                 {'a' => 2, d.instance.inject_config.time_key => Time.at(time).localtime}]
245 |     assert_equal(expected, actual_documents)
246 | 
247 |     indexes = get_indexes()
248 |     expire_after_hash = indexes.map {|e| e.select{|k, v| k == "expireAfterSeconds"} }.reject{|e| e.empty?}.first
249 |     assert_equal({"expireAfterSeconds"=>120.0}, expire_after_hash)
250 |   end
251 | 
252 |   class WriteWithCollectionPlaceholder < self
253 |     def setup
254 |       @tag = 'custom'
255 |       setup_mongod
256 |     end
257 | 
258 |     def teardown
259 |       teardown_mongod(@tag)
260 |     end
261 | 
262 |     def test_write_with_collection_placeholder
263 |       d = create_driver(%[
264 |         @type mongo
265 |         database #{database_name}
266 |         collection ${tag}
267 |         include_time_key true
268 |       ])
269 |       d.run(default_tag: @tag) do
270 |         emit_documents(d)
271 |       end
272 |       actual_documents = get_documents(@tag)
273 |       time = event_time("2011-01-02 13:14:15 UTC")
274 |       expected = [{'a' => 1, d.instance.inject_config.time_key => Time.at(time).localtime},
275 |                   {'a' => 2, d.instance.inject_config.time_key => Time.at(time).localtime}]
276 |       assert_equal(expected, actual_documents)
277 |     end
278 |   end
279 | 
280 |   class WriteWithDatabasePlaceholder < self
281 |     def setup
282 |       @tag = 'custom'
283 |       setup_mongod(@tag)
284 |     end
285 | 
286 |     def teardown
287 |       teardown_mongod
288 |     end
289 | 
290 |     def test_write_with_database_placeholder
291 |       d = create_driver(%[
292 |         @type mongo
293 |         database ${tag}
294 |         collection #{collection_name}
295 |         include_time_key true
296 |       ])
297 |       d.run(default_tag: @tag) do
298 |         emit_documents(d)
299 |       end
300 | 
301 |       actual_documents = get_documents
302 |       time = event_time("2011-01-02 13:14:15 UTC")
303 |       expected = [{'a' => 1, d.instance.inject_config.time_key => Time.at(time).localtime},
304 |                   {'a' => 2, d.instance.inject_config.time_key => Time.at(time).localtime}]
305 |       assert_equal(expected, actual_documents)
306 |     end
307 |   end
308 | 
309 |   def test_write_at_enable_tag
310 |     d = create_driver(default_config + %[
311 |       include_tag_key true
312 |       include_time_key false
313 |     ])
314 |     d.run(default_tag: 'test') do
315 |       emit_documents(d)
316 |     end
317 |     actual_documents = get_documents
318 |     expected = [{'a' => 1, d.instance.inject_config.tag_key => 'test'},
319 |                 {'a' => 2, d.instance.inject_config.tag_key => 'test'}]
320 |     assert_equal(expected, actual_documents)
321 |   end
322 | 
323 |   def emit_invalid_documents(d)
324 |     time = event_time("2011-01-02 13:14:15 UTC")
325 |     d.feed(time, {'a' => 3, 'b' => "c", '$last' => '石動'})
326 |     d.feed(time, {'a' => 4, 'b' => "d", 'first' => '菖蒲'.encode('EUC-JP').force_encoding('UTF-8')})
327 |     time
328 |   end
329 | 
330 |   def test_write_with_invalid_recoreds_with_keys_containing_dot_and_dollar
331 |     d = create_driver(default_config + %[
332 |       replace_dot_in_key_with _dot_
333 |       replace_dollar_in_key_with _dollar_
334 |     ])
335 | 
336 |     time = event_time("2011-01-02 13:14:15 UTC")
337 |     d.run(default_tag: 'test') do
338 |       d.feed(time, {
339 |         "foo.bar1" => {
340 |            "$foo$bar" => "baz"
341 |         },
342 |         "foo.bar2" => [
343 |           {
344 |             "$foo$bar" => "baz"
345 |           }
346 |         ],
347 |       })
348 |     end
349 | 
350 |     documents = get_documents
351 |     expected = {"foo_dot_bar1" => {
352 |                   "_dollar_foo$bar"=>"baz"
353 |                 },
354 |                 "foo_dot_bar2" => [
355 |                   {
356 |                     "_dollar_foo$bar"=>"baz"
357 |                   },
358 |                 ], "time" => Time.at(time).localtime
359 |                }
360 |     assert_equal(1, documents.size)
361 |     assert_equal(expected, documents[0])
362 |   end
363 | 
364 |   class WithAuthenticateTest < self
365 |     def setup_mongod
366 |       options = {}
367 |       options[:database] = database_name
368 |       @client = ::Mongo::Client.new(["localhost:#{port}"], options)
369 |       @client.database.users.create('fluent', password: 'password',
370 |                                     roles: [Mongo::Auth::Roles::READ_WRITE])
371 |     end
372 | 
373 |     def teardown_mongod
374 |       @client[collection_name].drop
375 |       @client.database.users.remove('fluent')
376 |     end
377 | 
378 |     def test_write_with_authenticate
379 |       d = create_driver(default_config + %[
380 |         user fluent
381 |         password password
382 |       ])
383 |       d.run(default_tag: 'test') do
384 |         emit_documents(d)
385 |       end
386 |       actual_documents = get_documents
387 |       time = event_time("2011-01-02 13:14:15 UTC")
388 |       expected = [{'a' => 1, d.instance.inject_config.time_key => Time.at(time).localtime},
389 |                   {'a' => 2, d.instance.inject_config.time_key => Time.at(time).localtime}]
390 |       assert_equal(expected, actual_documents)
391 |     end
392 |   end
393 | 
394 |   class MongoAuthenticateTest < self
395 |     require 'fluent/plugin/mongo_auth'
396 |     include ::Fluent::MongoAuth
397 | 
398 |     def setup_mongod
399 |       options = {}
400 |       options[:database] = database_name
401 |       @client = ::Mongo::Client.new(["localhost:#{port}"], options)
402 |       @client.database.users.create('fluent', password: 'password',
403 |                                     roles: [Mongo::Auth::Roles::READ_WRITE])
404 |     end
405 | 
406 |     def teardown_mongod
407 |       @client[collection_name].drop
408 |       @client.database.users.remove('fluent')
409 |     end
410 | 
411 |     def test_authenticate
412 |       d = create_driver(default_config + %[
413 |         user fluent
414 |         password password
415 |       ])
416 | 
417 |       assert authenticate(@client)
418 |     end
419 |   end
420 | 
421 |   sub_test_case 'date_keys' do
422 |     setup do
423 |       @updated_at_str = "2020-02-01T08:22:23.780Z"
424 |       @updated_at_t = Time.parse(@updated_at_str)
425 |     end
426 | 
427 |     def emit_date_documents(d)
428 |       time = event_time("2011-01-02 13:14:15 UTC")
429 |       d.feed(time, {'a' => 1, updated_at: @updated_at_str})
430 |       d.feed(time, {'a' => 2, updated_at: @updated_at_t.to_f})
431 |       d.feed(time, {'a' => 3, updated_at: @updated_at_t.to_i})
432 |       time
433 |     end
434 | 
435 |     def emit_invalid_date_documents(d)
436 |       time = event_time("2011-01-02 13:14:15 UTC")
437 |       d.feed(time, {'a' => 1, updated_at: "Invalid Date String"})
438 |       time
439 |     end
440 | 
441 |     def emit_nested_date_documents(d)
442 |       time = event_time("2011-01-02 13:14:15 UTC")
443 |       d.feed(time, {'a' => 1, updated_at: { 'time': @updated_at_str}})
444 |       d.feed(time, {'a' => 2, updated_at: { 'time': @updated_at_t.to_f}})
445 |       d.feed(time, {'a' => 3, updated_at: { 'time': @updated_at_t.to_i}})
446 |       time
447 |     end
448 | 
449 |     def emit_nested_invalid_date_documents(d)
450 |       time = event_time("2011-01-02 13:14:15 UTC")
451 |       d.feed(time, {'a' => 1, 'updated_at': { 'time': "Invalid Date String"}})
452 |       time
453 |     end
454 | 
455 |     def test_write_with_date_keys
456 |       d = create_driver(default_config + %[
457 |         date_keys updated_at
458 |         time_key created_at
459 |       ])
460 | 
461 |       d.run(default_tag: 'test') do
462 |         emit_date_documents(d)
463 |       end
464 | 
465 |       actual_documents = get_documents
466 |       date_key = d.instance.date_keys.first
467 |       actual_documents.each_with_index { |doc, i|
468 |         assert_equal(i + 1, doc['a'])
469 |         assert doc[date_key].is_a?(Time)
470 |       }
471 |     end
472 | 
473 |     def test_write_with_parsed_date_key_invalid_string
474 |       d = create_driver(default_config + %[
475 |             date_keys updated_at
476 |             time_key created_at
477 |           ])
478 | 
479 |       d.run(default_tag: 'test') do
480 |         emit_invalid_date_documents(d)
481 |       end
482 |       actual_documents = get_documents
483 |       assert_nil actual_documents.first['updated_at']
484 |     end
485 | 
486 |     def test_write_with_date_nested_keys
487 |       d = create_driver(default_config + %[
488 |         replace_dot_in_key_with _
489 |         replace_dollar_in_key_with _
490 |         date_keys $.updated_at.time
491 |         time_key created_at
492 |       ])
493 | 
494 |       d.run(default_tag: 'test') do
495 |         emit_nested_date_documents(d)
496 |       end
497 | 
498 |       actual_documents = get_documents
499 |       actual_documents.each_with_index { |doc, i|
500 |         assert_equal(i + 1, doc['a'])
501 |         assert doc['updated_at']['time'].is_a?(Time)
502 |       }
503 |     end
504 | 
505 |     def test_write_with_parsed_date_nested_key_invalid_string
506 |       d = create_driver(default_config + %[
507 |             replace_dot_in_key_with _
508 |             replace_dollar_in_key_with _
509 |             date_keys $.updated_at.time
510 |             time_key created_at
511 |           ])
512 | 
513 |       d.run(default_tag: 'test') do
514 |         emit_nested_invalid_date_documents(d)
515 |       end
516 |       actual_documents = get_documents
517 |       assert_nil actual_documents.first['updated_at']['time']
518 |     end
519 |   end
520 | 
521 |   sub_test_case 'object_id_keys' do
522 |     setup do
523 |       @my_id_str = "507f1f77bcf86cd799439011"
524 |     end
525 | 
526 |     def emit_date_documents(d)
527 |       time = event_time("2011-01-02 13:14:15 UTC")
528 |       d.feed(time, {'a' => 1, my_id: @my_id_str})
529 |       time
530 |     end
531 | 
532 |     def emit_invalid_date_documents(d)
533 |       time = event_time("2011-01-02 13:14:15 UTC")
534 |       d.feed(time, {'a' => 1, my_id: "Invalid ObjectId String"})
535 |       time
536 |     end
537 | 
538 |     def emit_nested_date_documents(d)
539 |       time = event_time("2011-01-02 13:14:15 UTC")
540 |       d.feed(time, {'a' => 1, my_id: { 'id': @my_id_str}})
541 |       time
542 |     end
543 | 
544 |     def emit_nested_invalid_date_documents(d)
545 |       time = event_time("2011-01-02 13:14:15 UTC")
546 |       d.feed(time, {'a' => 1, 'my_id': { 'id': "Invalid ObjectId String"}})
547 |       time
548 |     end
549 | 
550 |     def test_write_with_object_id_keys
551 |       d = create_driver(default_config + %[
552 |         object_id_keys my_id
553 |       ])
554 | 
555 |       d.run(default_tag: 'test') do
556 |         emit_date_documents(d)
557 |       end
558 | 
559 |       actual_documents = get_documents
560 |       object_id_key = d.instance.object_id_keys.first
561 |       actual_documents.each_with_index { |doc, i|
562 |         assert_equal(i + 1, doc['a'])
563 |         assert doc[object_id_key].is_a?(BSON::ObjectId)
564 |       }
565 |     end
566 | 
567 |     def test_write_with_parsed_object_id_key_invalid_string
568 |       d = create_driver(default_config + %[
569 |             object_id_keys my_id
570 |           ])
571 | 
572 |       d.run(default_tag: 'test') do
573 |         emit_invalid_date_documents(d)
574 |       end
575 |       actual_documents = get_documents
576 |       assert_nil actual_documents.first['my_id']
577 |     end
578 | 
579 |     def test_write_with_date_nested_keys
580 |       d = create_driver(default_config + %[
581 |         replace_dot_in_key_with _
582 |         replace_dollar_in_key_with _
583 |         object_id_keys $.my_id.id
584 |       ])
585 | 
586 |       d.run(default_tag: 'test') do
587 |         emit_nested_date_documents(d)
588 |       end
589 | 
590 |       actual_documents = get_documents
591 |       actual_documents.each_with_index { |doc, i|
592 |         assert_equal(i + 1, doc['a'])
593 |         assert doc['my_id']['id'].is_a?(BSON::ObjectId)
594 |       }
595 |     end
596 | 
597 |     def test_write_with_parsed_date_nested_key_invalid_string
598 |       d = create_driver(default_config + %[
599 |             replace_dot_in_key_with _
600 |             replace_dollar_in_key_with _
601 |             object_id_keys $.my_id.id
602 |           ])
603 | 
604 |       d.run(default_tag: 'test') do
605 |         emit_nested_invalid_date_documents(d)
606 |       end
607 |       actual_documents = get_documents
608 |       assert_nil actual_documents.first['my_id']['id']
609 |     end
610 |   end
611 | end
612 | 


--------------------------------------------------------------------------------
/test/plugin/test_out_mongo_replset.rb:
--------------------------------------------------------------------------------
  1 | require "helper"
  2 | require "fluent/test/driver/output"
  3 | require "fluent/test/helpers"
  4 | require 'fluent/mixin' # for TimeFormatter
  5 | 
  6 | class MongoReplsetOutputTest < ::Test::Unit::TestCase
  7 |   include Fluent::Test::Helpers
  8 | 
  9 |   def setup
 10 |     Fluent::Test.setup
 11 |   end
 12 | 
 13 |   def teardown
 14 |   end
 15 | 
 16 |   def collection_name
 17 |     'test'
 18 |   end
 19 | 
 20 |   def database_name
 21 |     'fluent_test'
 22 |   end
 23 | 
 24 |   def nodes
 25 |     ["localhost:#{port}"]
 26 |   end
 27 | 
 28 |   def port
 29 |     27018
 30 |   end
 31 | 
 32 |   def default_config
 33 |     %[
 34 |       @type mongo_replset
 35 |       nodes localhost:27018
 36 |       database #{database_name}
 37 |       collection #{collection_name}
 38 |       include_time_key true
 39 |       replica_set rs0
 40 |     ]
 41 |   end
 42 | 
 43 |   def create_driver(conf=default_config)
 44 |     Fluent::Test::Driver::Output.new(Fluent::Plugin::MongoOutputReplset).configure(conf)
 45 |   end
 46 | 
 47 |   def test_configure
 48 |     d = create_driver(%[
 49 |       @type mongo_replset
 50 |       port 27018
 51 |       database fluent_test
 52 |       collection test_collection
 53 |       replica_set rs0
 54 |     ])
 55 | 
 56 |     assert_equal('fluent_test', d.instance.database)
 57 |     assert_equal('test_collection', d.instance.collection)
 58 |     assert_equal('localhost', d.instance.host)
 59 |     assert_equal(27018, d.instance.port)
 60 |     assert_equal({replica_set: 'rs0', :ssl=>false, :write=>{:j=>false}},
 61 |                  d.instance.client_options)
 62 |   end
 63 | 
 64 |   def test_configure_with_nodes
 65 |     d = create_driver(%[
 66 |       @type mongo_replset
 67 |       nodes localhost:27018,localhost:27019
 68 |       database fluent_test
 69 |       collection test_collection
 70 |       replica_set rs0
 71 |     ])
 72 | 
 73 |     assert_equal('fluent_test', d.instance.database)
 74 |     assert_equal('test_collection', d.instance.collection)
 75 |     assert_equal(['localhost:27018', 'localhost:27019'], d.instance.nodes)
 76 |     assert_equal({replica_set: 'rs0', :ssl=>false, :write=>{:j=>false}},
 77 |                  d.instance.client_options)
 78 |   end
 79 | 
 80 |   def test_configure_with_logger_conf
 81 |     d = create_driver(default_config + %[
 82 |       mongo_log_level fatal
 83 |     ])
 84 | 
 85 |     expected = "fatal"
 86 |     assert_equal(expected, d.instance.mongo_log_level)
 87 |   end
 88 | 
 89 |   class ReplisetWriteTest < self
 90 |     def setup
 91 |       omit("Replica set setup is too hard in CI.") if ENV['CI']
 92 | 
 93 |       setup_mongod
 94 |     end
 95 | 
 96 |     def teardown
 97 |       omit("Replica set setup is too hard in CI.") if ENV['CI']
 98 | 
 99 |       teardown_mongod
100 |     end
101 | 
102 |     def setup_mongod
103 |       options = {}
104 |       options[:database] = database_name
105 |       @client = ::Mongo::Client.new(nodes, options)
106 |     end
107 | 
108 |     def teardown_mongod
109 |       @client[collection_name].drop
110 |     end
111 | 
112 |     def get_documents
113 |       @client[collection_name].find.to_a.map {|e| e.delete('_id'); e}
114 |     end
115 | 
116 |     def emit_documents(d)
117 |       time = event_time("2011-01-02 13:14:15 UTC")
118 |       d.feed(time, {'a' => 1})
119 |       d.feed(time, {'a' => 2})
120 |       time
121 |     end
122 | 
123 |     def test_write
124 |       d = create_driver
125 |       d.run(default_tag: 'test') do
126 |         emit_documents(d)
127 |       end
128 |       actual_documents = get_documents
129 |       time = event_time("2011-01-02 13:14:15 UTC")
130 |       expected = [{'a' => 1, d.instance.inject_config.time_key => Time.at(time).localtime},
131 |                   {'a' => 2, d.instance.inject_config.time_key => Time.at(time).localtime}]
132 |       assert_equal(expected, actual_documents)
133 |     end
134 |   end
135 | end
136 | 


--------------------------------------------------------------------------------