├── .gitignore
├── LICENSE
├── README.md
├── buf_disk.go
├── buf_disk_test.go
├── buffer.go
├── config.go
├── config_test.go
├── glob.go
├── gofluent.conf
├── in_forward.go
├── in_tail.go
├── in_tail_test.go
├── input.go
├── logger.go
├── main.go
├── out_forward.go
├── out_httpsqs.go
├── out_mongodb.go
├── out_mongodb_test.go
├── out_stdout.go
├── output.go
├── pipeline_runner.go
├── plugin_interface.go
├── plugin_runner.go
├── router.go
├── router_test.go
└── wercker.yml
/.gitignore:
--------------------------------------------------------------------------------
1 | # Compiled Object files, Static and Dynamic libs (Shared Objects)
2 | *.o
3 | *.a
4 | *.so
5 |
6 | # Folders
7 | _obj
8 | _test
9 | src
10 |
11 | # Architecture specific extensions/prefixes
12 | *.[568vq]
13 | [568vq].out
14 |
15 | *.cgo1.go
16 | *.cgo2.c
17 | _cgo_defun.c
18 | _cgo_gotypes.go
19 | _cgo_export.*
20 |
21 | _testmain.go
22 |
23 | fluent-go
24 | *.exe
25 | *.test
26 | *.prof
27 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "{}"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright {yyyy} {name of copyright owner}
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
203 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | gofluent
2 | ========
3 | [](https://app.wercker.com/project/bykey/1908429c90a6d7bd437a63210fb1a97f)
4 |
5 | This program is something acting like fluentd rewritten in Go.
6 |
7 | Table of Contents
8 | =================
9 |
10 | * [Introduction](#introduction)
11 | * [Architecture](#architecture)
12 | * [Implementation](#implementation)
13 | * [Overview](#overview)
14 | * [Data flow](#data-flow)
15 | * [Plugins](#plugins)
16 | * [Tail Input Plugin](#tail-input-plugin)
17 | * [Httpsqs Output Plugin](#httpsqs-output-plugin)
18 | * [Stdout Output Plugin](#stdout-output-plugin)
19 | * [Mongodb Output Plugin](#mongodb-output-plugin)
20 |
21 | Introduction
22 | ============
23 |
24 | Fluentd is originally written in CRuby, which has too many dependecies.
25 |
26 | I hope fluentd to be simpler and cleaner, as its main feature is simplicity and rubostness.
27 |
28 | Architecture
29 | ============
30 |
31 | ```
32 | +---------+ +---------+ +---------+ +---------+
33 | | server1 | | server2 | | server3 | | serverN |
34 | |---------| |---------| |---------| |---------|
35 | | | | | | | | |
36 | |---------| |---------| |---------| |---------|
37 | |gofluent | |gofluent | |gofluent | |gofluent |
38 | +---------+ +---------+ +---------+ +---------+
39 | | | | |
40 | -----------------------------------------------
41 | |
42 | | HTTP POST
43 | V
44 | +-----------------+
45 | | |
46 | | Httpmq |
47 | | |
48 | +-----------------+
49 | |
50 | | HTTP GET
51 | V
52 | +-----------------+ +-----------------+
53 | | | | |
54 | | Preprocessor | --------------> | Storage |
55 | | | | |
56 | +-----------------+ +-----------------+
57 | ```
58 |
59 | Implementation
60 | ==============
61 | Overview
62 | --------
63 |
64 | ```
65 | Input -> Router -> Output
66 | ```
67 | Data flow
68 | ---------
69 |
70 | ```
71 | -------<--------
72 | | |
73 | V | generate pool
74 | InputRunner.inputRecycleChan | recycling
75 | | | | ^
76 | | ------->------- \
77 | | ^ ^ \
78 | InputRunner.inChan | | \
79 | | | | \
80 | | | | \
81 | consume | | | \
82 | V | | \
83 | Input(Router.inChan) ----> Router ----> (Router.outChan)Output.inChan
84 |
85 | ```
86 |
87 | Plugins
88 | =======
89 |
90 | Tail Input Plugin
91 | -----------------
92 | The in_tail input plugin allows gofluent to read events from the tail of text files. Its behavior is similar to the tail -F command.
93 |
94 | Example Configuration
95 |
96 | in_tail is included in gofluent’s core. No additional installation process is required.
97 | ```
98 |
99 | type tail
100 | path /var/log/httpd-access.log
101 | pos_file /var/log/httpd-access.log.pos
102 | tag apache.access
103 | format /^(?P[^ ]*) [^ ]* (?P[^ ]*) \[(?P
105 | ```
106 | *type (required)*
107 | The value must be tail.
108 |
109 | *tag (required)*
110 | The tag of the event.
111 |
112 | *path (required)*
113 | The paths to read.
114 |
115 | *format (required)*
116 | The format of the log. It is the name of a template or regexp surrounded by ‘/’.
117 | The regexp must have at least one named capture (?P\PATTERN).
118 |
119 | The following templates are supported:
120 | - regexp
121 | - json
122 | One JSON map, per line. This is the most straight forward format :).
123 | ```
124 | format json
125 | ```
126 | *pos_file (highly recommended)*
127 | This parameter is highly recommended. gofluent will record the position it last read into this file.
128 | ```
129 | pos_file /var/log/access.log.pos
130 | ```
131 |
132 | *sync_interval*
133 | The sync interval of pos file, default is 2s.
134 |
135 | Httpsqs Output Plugin
136 | ---------------------
137 | The out_httpsqs output plugin allows gofluent to send data to httpsqs mq.
138 |
139 | Example Configuration
140 |
141 | out_httpsqs is included in gofluent’s core. No additional installation process is required.
142 | ```
143 |
144 | type httpsqs
145 | host localhost
146 | port 1218
147 | flush_interval 10
148 |
149 | ```
150 | *type (required)*
151 | The value must be httpsqs.
152 |
153 | *host (required)*
154 | The output target host ip.
155 |
156 | *port (required)*
157 | The output target host port.
158 |
159 | *auth (highly recommended)*
160 | The auth password for httpsqs.
161 |
162 | *flush_interval*
163 | The flush interval for sending data to httpsqs.
164 |
165 | *gzip*
166 | The gzip switch, default is on.
167 |
168 | Forward Output Plugin
169 | ---------------------
170 | The out_forward output plugin allows gofluent to forward events to another gofluent.
171 |
172 | Example Configuration
173 |
174 | out_forward is included in gofluent’s core. No additional installation process is required.
175 | ```
176 |
177 | type forward
178 | host localhost
179 | port 1218
180 | flush_interval 10
181 |
182 | ```
183 | *type (required)*
184 | The value must be forward.
185 |
186 | *host (required)*
187 | The output target host ip.
188 |
189 | *port (required)*
190 | The output target host port.
191 |
192 | *flush_interval*
193 | The flush interval for sending data to httpsqs.
194 |
195 | *connect_timeout*
196 | The connect timeout value.
197 |
198 | *sync_interval*
199 | The sync interval of metadata file, default is 2s.
200 |
201 | *buffer_path(required)*
202 | The disk buffer path for output plugin, default is /tmp/test.
203 |
204 | *buffer_queue_limit*
205 | The queue limit of disk buffer, default is 64M.
206 |
207 | *buffer_chunk_limit*
208 | The chunk limit of disk buffer to forward, default is 8M.
209 |
210 | Stdout Output Plugin
211 | --------------------
212 | The out_stdout output plugin allows gofluent to print events to stdout.
213 |
214 | Example Configuration
215 |
216 | out_stdout is included in gofluent’s core. No additional installation process is required.
217 | ```
218 |
219 | type stdout
220 |
221 | ```
222 | *type (required)*
223 | The value must be stdout.
224 |
225 | Mongodb Output Plugin
226 | ---------------------
227 | The out_mongodb output plugin allows gofluent to send message to mongodb.
228 |
229 | Example Configuration
230 |
231 | out_mongodb is included in gofluent's core. No additional installation process is required.
232 | ```
233 |
234 | type mongodb
235 | host localhost
236 | port 27017
237 | capped on
238 | capped_size 1024
239 | database test
240 | collection test
241 | user test
242 | password test
243 |
244 | ```
245 | *type(required)*
246 | The value must be mongodb.
247 |
248 | *host (required)*
249 | The output target host ip.
250 |
251 | *port (required)*
252 | The output target host port.
253 |
254 | *capped (highly recommended)*
255 | To create a capped collection.
256 |
257 | *capped_size*
258 | Specify the actual size(MB) of the capped collection.
259 |
260 | *database(required)*
261 | The name of the database to be created or used.
262 |
263 | *collection(required)*
264 | The name of the collection to be created or used.
265 |
266 | *user(highly recommended)*
267 | The user name to login the database.
268 |
269 | *password(highly recommended)*
270 | The password to login the database.
271 |
--------------------------------------------------------------------------------
/buf_disk.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bufio"
5 | "bytes"
6 | "encoding/binary"
7 | "errors"
8 | "fmt"
9 | "io"
10 | "os"
11 | "path"
12 | "sync"
13 | "sync/atomic"
14 | "time"
15 | )
16 |
17 | // diskQueue implements the BackendQueue interface
18 | // providing a filesystem backed FIFO queue
19 | type diskQueue struct {
20 | // 64bit atomic vars need to be first for proper alignment on 32bit platforms
21 |
22 | // run-time state (also persisted to disk)
23 | readPos int64
24 | writePos int64
25 | readFileNum int64
26 | writeFileNum int64
27 | depth int64
28 |
29 | sync.RWMutex
30 |
31 | // instantiation time metadata
32 | name string
33 | dataPath string
34 | maxBytesPerFile int64 // currently this cannot change once created
35 | syncEvery int64 // number of writes per fsync
36 | syncTimeout time.Duration // duration of time per fsync
37 | exitFlag int32
38 | needSync bool
39 |
40 | // keeps track of the position where we have read
41 | // (but not yet sent over readChan)
42 | nextReadPos int64
43 | nextReadFileNum int64
44 |
45 | readFile *os.File
46 | writeFile *os.File
47 | reader *bufio.Reader
48 | writeBuf bytes.Buffer
49 |
50 | // exposed via ReadChan()
51 | readChan chan []byte
52 |
53 | // internal channels
54 | writeChan chan []byte
55 | writeResponseChan chan error
56 | emptyChan chan int
57 | emptyResponseChan chan error
58 | exitChan chan int
59 | exitSyncChan chan int
60 |
61 | logger logger
62 | }
63 |
64 | // newDiskQueue instantiates a new instance of diskQueue, retrieving metadata
65 | // from the filesystem and starting the read ahead goroutine
66 | func newDiskQueue(name string, dataPath string, maxBytesPerFile int64,
67 | syncEvery int64, syncTimeout time.Duration,
68 | logger logger) BackendQueue {
69 | d := diskQueue{
70 | name: name,
71 | dataPath: dataPath,
72 | maxBytesPerFile: maxBytesPerFile,
73 | readChan: make(chan []byte),
74 | writeChan: make(chan []byte),
75 | writeResponseChan: make(chan error),
76 | emptyChan: make(chan int),
77 | emptyResponseChan: make(chan error),
78 | exitChan: make(chan int),
79 | exitSyncChan: make(chan int),
80 | syncEvery: syncEvery,
81 | syncTimeout: syncTimeout,
82 | logger: logger,
83 | }
84 |
85 | // no need to lock here, nothing else could possibly be touching this instance
86 | err := d.retrieveMetaData()
87 | if err != nil && !os.IsNotExist(err) {
88 | d.logf("ERROR: diskqueue(%s) failed to retrieveMetaData - %s", d.name, err)
89 | }
90 |
91 | go d.ioLoop()
92 |
93 | return &d
94 | }
95 |
96 | func (d *diskQueue) logf(f string, args ...interface{}) {
97 | if d.logger == nil {
98 | return
99 | }
100 | d.logger.Output(2, fmt.Sprintf(f, args...))
101 | }
102 |
103 | // Depth returns the depth of the queue
104 | func (d *diskQueue) Depth() int64 {
105 | return atomic.LoadInt64(&d.depth)
106 | }
107 |
108 | // ReadChan returns the []byte channel for reading data
109 | func (d *diskQueue) ReadChan() chan []byte {
110 | return d.readChan
111 | }
112 |
113 | // Put writes a []byte to the queue
114 | func (d *diskQueue) Put(data []byte) error {
115 | d.RLock()
116 | defer d.RUnlock()
117 |
118 | if d.exitFlag == 1 {
119 | return errors.New("exiting")
120 | }
121 |
122 | d.writeChan <- data
123 | return <-d.writeResponseChan
124 | }
125 |
126 | // Close cleans up the queue and persists metadata
127 | func (d *diskQueue) Close() error {
128 | err := d.exit(false)
129 | if err != nil {
130 | return err
131 | }
132 | return d.sync()
133 | }
134 |
135 | func (d *diskQueue) Delete() error {
136 | return d.exit(true)
137 | }
138 |
139 | func (d *diskQueue) exit(deleted bool) error {
140 | d.Lock()
141 | defer d.Unlock()
142 |
143 | d.exitFlag = 1
144 |
145 | if deleted {
146 | d.logf("DISKQUEUE(%s): deleting", d.name)
147 | } else {
148 | d.logf("DISKQUEUE(%s): closing", d.name)
149 | }
150 |
151 | close(d.exitChan)
152 | // ensure that ioLoop has exited
153 | <-d.exitSyncChan
154 |
155 | if d.readFile != nil {
156 | d.readFile.Close()
157 | d.readFile = nil
158 | }
159 |
160 | if d.writeFile != nil {
161 | d.writeFile.Close()
162 | d.writeFile = nil
163 | }
164 |
165 | return nil
166 | }
167 |
168 | // Empty destructively clears out any pending data in the queue
169 | // by fast forwarding read positions and removing intermediate files
170 | func (d *diskQueue) Empty() error {
171 | d.RLock()
172 | defer d.RUnlock()
173 |
174 | if d.exitFlag == 1 {
175 | return errors.New("exiting")
176 | }
177 |
178 | d.logf("DISKQUEUE(%s): emptying", d.name)
179 |
180 | d.emptyChan <- 1
181 | return <-d.emptyResponseChan
182 | }
183 |
184 | func (d *diskQueue) deleteAllFiles() error {
185 | err := d.skipToNextRWFile()
186 |
187 | innerErr := os.Remove(d.metaDataFileName())
188 | if innerErr != nil && !os.IsNotExist(innerErr) {
189 | d.logf("ERROR: diskqueue(%s) failed to remove metadata file - %s", d.name, innerErr)
190 | return innerErr
191 | }
192 |
193 | return err
194 | }
195 |
196 | func (d *diskQueue) skipToNextRWFile() error {
197 | var err error
198 |
199 | if d.readFile != nil {
200 | d.readFile.Close()
201 | d.readFile = nil
202 | }
203 |
204 | if d.writeFile != nil {
205 | d.writeFile.Close()
206 | d.writeFile = nil
207 | }
208 |
209 | for i := d.readFileNum; i <= d.writeFileNum; i++ {
210 | fn := d.fileName(i)
211 | innerErr := os.Remove(fn)
212 | if innerErr != nil && !os.IsNotExist(innerErr) {
213 | d.logf("ERROR: diskqueue(%s) failed to remove data file - %s", d.name, innerErr)
214 | err = innerErr
215 | }
216 | }
217 |
218 | d.writeFileNum++
219 | d.writePos = 0
220 | d.readFileNum = d.writeFileNum
221 | d.readPos = 0
222 | d.nextReadFileNum = d.writeFileNum
223 | d.nextReadPos = 0
224 | atomic.StoreInt64(&d.depth, 0)
225 |
226 | return err
227 | }
228 |
229 | // readOne performs a low level filesystem read for a single []byte
230 | // while advancing read positions and rolling files, if necessary
231 | func (d *diskQueue) readOne() ([]byte, error) {
232 | var err error
233 | var msgSize int32
234 |
235 | if d.readFile == nil {
236 | curFileName := d.fileName(d.readFileNum)
237 | d.readFile, err = os.OpenFile(curFileName, os.O_RDONLY, 0600)
238 | if err != nil {
239 | return nil, err
240 | }
241 |
242 | d.logf("DISKQUEUE(%s): readOne() opened %s", d.name, curFileName)
243 |
244 | if d.readPos > 0 {
245 | _, err = d.readFile.Seek(d.readPos, 0)
246 | if err != nil {
247 | d.readFile.Close()
248 | d.readFile = nil
249 | return nil, err
250 | }
251 | }
252 |
253 | d.reader = bufio.NewReader(d.readFile)
254 | }
255 |
256 | err = binary.Read(d.reader, binary.BigEndian, &msgSize)
257 | if err != nil {
258 | d.readFile.Close()
259 | d.readFile = nil
260 | return nil, err
261 | }
262 |
263 | readBuf := make([]byte, msgSize)
264 | _, err = io.ReadFull(d.reader, readBuf)
265 | if err != nil {
266 | d.readFile.Close()
267 | d.readFile = nil
268 | return nil, err
269 | }
270 |
271 | totalBytes := int64(4 + msgSize)
272 |
273 | // we only advance next* because we have not yet sent this to consumers
274 | // (where readFileNum, readPos will actually be advanced)
275 | d.nextReadPos = d.readPos + totalBytes
276 | d.nextReadFileNum = d.readFileNum
277 |
278 | // TODO: each data file should embed the maxBytesPerFile
279 | // as the first 8 bytes (at creation time) ensuring that
280 | // the value can change without affecting runtime
281 | if d.nextReadPos > d.maxBytesPerFile {
282 | if d.readFile != nil {
283 | d.readFile.Close()
284 | d.readFile = nil
285 | }
286 |
287 | d.nextReadFileNum++
288 | d.nextReadPos = 0
289 | }
290 |
291 | return readBuf, nil
292 | }
293 |
294 | // writeOne performs a low level filesystem write for a single []byte
295 | // while advancing write positions and rolling files, if necessary
296 | func (d *diskQueue) writeOne(data []byte) error {
297 | var err error
298 |
299 | if d.writeFile == nil {
300 | curFileName := d.fileName(d.writeFileNum)
301 | d.writeFile, err = os.OpenFile(curFileName, os.O_RDWR|os.O_CREATE, 0600)
302 | if err != nil {
303 | return err
304 | }
305 |
306 | d.logf("DISKQUEUE(%s): writeOne() opened %s", d.name, curFileName)
307 |
308 | if d.writePos > 0 {
309 | _, err = d.writeFile.Seek(d.writePos, 0)
310 | if err != nil {
311 | d.writeFile.Close()
312 | d.writeFile = nil
313 | return err
314 | }
315 | }
316 | }
317 |
318 | dataLen := len(data)
319 |
320 | d.writeBuf.Reset()
321 | err = binary.Write(&d.writeBuf, binary.BigEndian, int32(dataLen))
322 | if err != nil {
323 | return err
324 | }
325 |
326 | _, err = d.writeBuf.Write(data)
327 | if err != nil {
328 | return err
329 | }
330 |
331 | // only write to the file once
332 | _, err = d.writeFile.Write(d.writeBuf.Bytes())
333 | if err != nil {
334 | d.writeFile.Close()
335 | d.writeFile = nil
336 | return err
337 | }
338 |
339 | totalBytes := int64(4 + dataLen)
340 | d.writePos += totalBytes
341 | atomic.AddInt64(&d.depth, 1)
342 |
343 | if d.writePos > d.maxBytesPerFile {
344 | d.writeFileNum++
345 | d.writePos = 0
346 |
347 | // sync every time we start writing to a new file
348 | err = d.sync()
349 | if err != nil {
350 | d.logf("ERROR: diskqueue(%s) failed to sync - %s", d.name, err)
351 | }
352 |
353 | if d.writeFile != nil {
354 | d.writeFile.Close()
355 | d.writeFile = nil
356 | }
357 | }
358 |
359 | return err
360 | }
361 |
362 | // sync fsyncs the current writeFile and persists metadata
363 | func (d *diskQueue) sync() error {
364 | if d.writeFile != nil {
365 | err := d.writeFile.Sync()
366 | if err != nil {
367 | d.writeFile.Close()
368 | d.writeFile = nil
369 | return err
370 | }
371 | }
372 |
373 | err := d.persistMetaData()
374 | if err != nil {
375 | return err
376 | }
377 |
378 | d.needSync = false
379 | return nil
380 | }
381 |
382 | // retrieveMetaData initializes state from the filesystem
383 | func (d *diskQueue) retrieveMetaData() error {
384 | var f *os.File
385 | var err error
386 |
387 | fileName := d.metaDataFileName()
388 | f, err = os.OpenFile(fileName, os.O_RDONLY, 0600)
389 | if err != nil {
390 | return err
391 | }
392 | defer f.Close()
393 |
394 | var depth int64
395 | _, err = fmt.Fscanf(f, "%d\n%d,%d\n%d,%d\n",
396 | &depth,
397 | &d.readFileNum, &d.readPos,
398 | &d.writeFileNum, &d.writePos)
399 | if err != nil {
400 | return err
401 | }
402 | atomic.StoreInt64(&d.depth, depth)
403 | d.nextReadFileNum = d.readFileNum
404 | d.nextReadPos = d.readPos
405 |
406 | return nil
407 | }
408 |
409 | // persistMetaData atomically writes state to the filesystem
410 | func (d *diskQueue) persistMetaData() error {
411 | var f *os.File
412 | var err error
413 |
414 | fileName := d.metaDataFileName()
415 | tmpFileName := fileName + ".tmp"
416 |
417 | // write to tmp file
418 | f, err = os.OpenFile(tmpFileName, os.O_RDWR|os.O_CREATE, 0600)
419 | if err != nil {
420 | return err
421 | }
422 |
423 | _, err = fmt.Fprintf(f, "%d\n%d,%d\n%d,%d\n",
424 | atomic.LoadInt64(&d.depth),
425 | d.readFileNum, d.readPos,
426 | d.writeFileNum, d.writePos)
427 | if err != nil {
428 | f.Close()
429 | return err
430 | }
431 | f.Sync()
432 | f.Close()
433 |
434 | // atomically rename
435 | return os.Rename(tmpFileName, fileName)
436 | }
437 |
438 | func (d *diskQueue) metaDataFileName() string {
439 | return fmt.Sprintf(path.Join(d.dataPath, "%s.diskqueue.meta.dat"), d.name)
440 | }
441 |
442 | func (d *diskQueue) fileName(fileNum int64) string {
443 | return fmt.Sprintf(path.Join(d.dataPath, "%s.diskqueue.%06d.dat"), d.name, fileNum)
444 | }
445 |
446 | func (d *diskQueue) checkTailCorruption(depth int64) {
447 | if d.readFileNum < d.writeFileNum || d.readPos < d.writePos {
448 | return
449 | }
450 |
451 | // we've reached the end of the diskqueue
452 | // if depth isn't 0 something went wrong
453 | if depth != 0 {
454 | if depth < 0 {
455 | d.logf(
456 | "ERROR: diskqueue(%s) negative depth at tail (%d), metadata corruption, resetting 0...",
457 | d.name, depth)
458 | } else if depth > 0 {
459 | d.logf(
460 | "ERROR: diskqueue(%s) positive depth at tail (%d), data loss, resetting 0...",
461 | d.name, depth)
462 | }
463 | // force set depth 0
464 | atomic.StoreInt64(&d.depth, 0)
465 | d.needSync = true
466 | }
467 |
468 | if d.readFileNum != d.writeFileNum || d.readPos != d.writePos {
469 | if d.readFileNum > d.writeFileNum {
470 | d.logf(
471 | "ERROR: diskqueue(%s) readFileNum > writeFileNum (%d > %d), corruption, skipping to next writeFileNum and resetting 0...",
472 | d.name, d.readFileNum, d.writeFileNum)
473 | }
474 |
475 | if d.readPos > d.writePos {
476 | d.logf(
477 | "ERROR: diskqueue(%s) readPos > writePos (%d > %d), corruption, skipping to next writeFileNum and resetting 0...",
478 | d.name, d.readPos, d.writePos)
479 | }
480 |
481 | d.skipToNextRWFile()
482 | d.needSync = true
483 | }
484 | }
485 |
486 | func (d *diskQueue) moveForward() {
487 | oldReadFileNum := d.readFileNum
488 | d.readFileNum = d.nextReadFileNum
489 | d.readPos = d.nextReadPos
490 | depth := atomic.AddInt64(&d.depth, -1)
491 |
492 | // see if we need to clean up the old file
493 | if oldReadFileNum != d.nextReadFileNum {
494 | // sync every time we start reading from a new file
495 | d.needSync = true
496 |
497 | fn := d.fileName(oldReadFileNum)
498 | err := os.Remove(fn)
499 | if err != nil {
500 | d.logf("ERROR: failed to Remove(%s) - %s", fn, err)
501 | }
502 | }
503 |
504 | d.checkTailCorruption(depth)
505 | }
506 |
507 | func (d *diskQueue) handleReadError() {
508 | // jump to the next read file and rename the current (bad) file
509 | if d.readFileNum == d.writeFileNum {
510 | // if you can't properly read from the current write file it's safe to
511 | // assume that something is fucked and we should skip the current file too
512 | if d.writeFile != nil {
513 | d.writeFile.Close()
514 | d.writeFile = nil
515 | }
516 | d.writeFileNum++
517 | d.writePos = 0
518 | }
519 |
520 | badFn := d.fileName(d.readFileNum)
521 | badRenameFn := badFn + ".bad"
522 |
523 | d.logf(
524 | "NOTICE: diskqueue(%s) jump to next file and saving bad file as %s",
525 | d.name, badRenameFn)
526 |
527 | err := os.Rename(badFn, badRenameFn)
528 | if err != nil {
529 | d.logf(
530 | "ERROR: diskqueue(%s) failed to rename bad diskqueue file %s to %s",
531 | d.name, badFn, badRenameFn)
532 | }
533 |
534 | d.readFileNum++
535 | d.readPos = 0
536 | d.nextReadFileNum = d.readFileNum
537 | d.nextReadPos = 0
538 |
539 | // significant state change, schedule a sync on the next iteration
540 | d.needSync = true
541 | }
542 |
543 | // ioLoop provides the backend for exposing a go channel (via ReadChan())
544 | // in support of multiple concurrent queue consumers
545 | //
546 | // it works by looping and branching based on whether or not the queue has data
547 | // to read and blocking until data is either read or written over the appropriate
548 | // go channels
549 | //
550 | // conveniently this also means that we're asynchronously reading from the filesystem
551 | func (d *diskQueue) ioLoop() {
552 | var dataRead []byte
553 | var err error
554 | var count int64
555 | var r chan []byte
556 |
557 | syncTicker := time.NewTicker(d.syncTimeout)
558 |
559 | for {
560 | // dont sync all the time :)
561 | if count == d.syncEvery {
562 | count = 0
563 | d.needSync = true
564 | }
565 |
566 | if d.needSync {
567 | err = d.sync()
568 | if err != nil {
569 | d.logf("ERROR: diskqueue(%s) failed to sync - %s", d.name, err)
570 | }
571 | }
572 |
573 | if (d.readFileNum < d.writeFileNum) || (d.readPos < d.writePos) {
574 | if d.nextReadPos == d.readPos {
575 | dataRead, err = d.readOne()
576 | if err != nil {
577 | d.logf("ERROR: reading from diskqueue(%s) at %d of %s - %s",
578 | d.name, d.readPos, d.fileName(d.readFileNum), err)
579 | d.handleReadError()
580 | continue
581 | }
582 | }
583 | r = d.readChan
584 | } else {
585 | r = nil
586 | }
587 |
588 | select {
589 | // the Go channel spec dictates that nil channel operations (read or write)
590 | // in a select are skipped, we set r to d.readChan only when there is data to read
591 | case r <- dataRead:
592 | // moveForward sets needSync flag if a file is removed
593 | count++
594 | d.moveForward()
595 | case <-d.emptyChan:
596 | d.emptyResponseChan <- d.deleteAllFiles()
597 | count = 0
598 | case dataWrite := <-d.writeChan:
599 | count++
600 | d.writeResponseChan <- d.writeOne(dataWrite)
601 | case <-syncTicker.C:
602 | if count > 0 {
603 | count = 0
604 | d.needSync = true
605 | }
606 | case <-d.exitChan:
607 | goto exit
608 | }
609 | }
610 |
611 | exit:
612 | d.logf("DISKQUEUE(%s): closing ... ioLoop", d.name)
613 | syncTicker.Stop()
614 | d.exitSyncChan <- 1
615 | }
616 |
--------------------------------------------------------------------------------
/buf_disk_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bufio"
5 | . "github.com/smartystreets/goconvey/convey"
6 | "log"
7 | "os"
8 | "path"
9 | "strconv"
10 | "sync"
11 | "sync/atomic"
12 | "testing"
13 | "time"
14 | )
15 |
16 | func TestDiskQueue(t *testing.T) {
17 |
18 | Convey("TestDiskQueue", t, func() {
19 | l := log.New(os.Stderr, "", log.LstdFlags)
20 | dqName := "test_disk_queue" + strconv.Itoa(int(time.Now().Unix()))
21 | dq := newDiskQueue(dqName, os.TempDir(), 1024, 2500, 2*time.Second, l)
22 | So(dq, ShouldNotEqual, nil)
23 | So(dq.Depth(), ShouldEqual, int64(0))
24 |
25 | msg := []byte("test")
26 | err := dq.Put(msg)
27 | So(err, ShouldEqual, nil)
28 | So(dq.Depth(), ShouldEqual, int64(1))
29 |
30 | msgOut := <-dq.ReadChan()
31 | So(string(msgOut), ShouldEqual, string(msg))
32 | dq.Delete()
33 | })
34 |
35 | Convey("TestDiskQueueRoll", t, func() {
36 | l := log.New(os.Stderr, "", log.LstdFlags)
37 | dqName := "test_disk_queue_roll" + strconv.Itoa(int(time.Now().Unix()))
38 | dq := newDiskQueue(dqName, os.TempDir(), 100, 2500, 2*time.Second, l)
39 | So(dq, ShouldNotEqual, nil)
40 | So(dq.Depth(), ShouldEqual, int64(0))
41 |
42 | msg := []byte("aaaaaaaaaa")
43 | for i := 0; i < 10; i++ {
44 | err := dq.Put(msg)
45 | So(err, ShouldEqual, nil)
46 | So(dq.Depth(), ShouldEqual, int64(i+1))
47 | }
48 |
49 | So(dq.(*diskQueue).writeFileNum, ShouldEqual, int64(1))
50 | So(dq.(*diskQueue).writePos, ShouldEqual, int64(28))
51 | dq.Delete()
52 | })
53 | }
54 |
55 | func TestDiskQueueEmpty(t *testing.T) {
56 | Convey("TestDiskQueueEmpty", t, func() {
57 | l := log.New(os.Stderr, "", log.LstdFlags)
58 | dqName := "test_disk_queue_empty" + strconv.Itoa(int(time.Now().Unix()))
59 | dq := newDiskQueue(dqName, os.TempDir(), 100, 2500, 2*time.Second, l)
60 | So(dq, ShouldNotEqual, nil)
61 | So(dq.Depth(), ShouldEqual, int64(0))
62 |
63 | msg := []byte("aaaaaaaaaa")
64 |
65 | for i := 0; i < 100; i++ {
66 | err := dq.Put(msg)
67 | So(err, ShouldEqual, nil)
68 | So(dq.Depth(), ShouldEqual, int64(i+1))
69 | }
70 |
71 | for i := 0; i < 3; i++ {
72 | <-dq.ReadChan()
73 | }
74 |
75 | for {
76 | if dq.Depth() == 97 {
77 | break
78 | }
79 | time.Sleep(50 * time.Millisecond)
80 | }
81 | So(dq.Depth(), ShouldEqual, int64(97))
82 |
83 | numFiles := dq.(*diskQueue).writeFileNum
84 | dq.Empty()
85 |
86 | f, err := os.OpenFile(dq.(*diskQueue).metaDataFileName(), os.O_RDONLY, 0600)
87 | So(f, ShouldEqual, (*os.File)(nil))
88 | So(os.IsNotExist(err), ShouldEqual, true)
89 | for i := int64(0); i <= numFiles; i++ {
90 | f, err := os.OpenFile(dq.(*diskQueue).fileName(i), os.O_RDONLY, 0600)
91 | So(f, ShouldEqual, (*os.File)(nil))
92 | So(os.IsNotExist(err), ShouldEqual, true)
93 | }
94 | So(dq.Depth(), ShouldEqual, int64(0))
95 | So(dq.(*diskQueue).readFileNum, ShouldEqual, dq.(*diskQueue).writeFileNum)
96 | So(dq.(*diskQueue).readPos, ShouldEqual, dq.(*diskQueue).writePos)
97 | So(dq.(*diskQueue).nextReadPos, ShouldEqual, dq.(*diskQueue).readPos)
98 | So(dq.(*diskQueue).nextReadFileNum, ShouldEqual, dq.(*diskQueue).readFileNum)
99 | for i := 0; i < 100; i++ {
100 | err := dq.Put(msg)
101 | So(err, ShouldEqual, nil)
102 | So(dq.Depth(), ShouldEqual, int64(i+1))
103 | }
104 |
105 | for i := 0; i < 100; i++ {
106 | <-dq.ReadChan()
107 | }
108 |
109 | for {
110 | if dq.Depth() == 0 {
111 | break
112 | }
113 | time.Sleep(50 * time.Millisecond)
114 | }
115 |
116 | So(dq.Depth(), ShouldEqual, int64(0))
117 | So(dq.(*diskQueue).readFileNum, ShouldEqual, dq.(*diskQueue).writeFileNum)
118 | So(dq.(*diskQueue).readPos, ShouldEqual, dq.(*diskQueue).writePos)
119 | So(dq.(*diskQueue).nextReadPos, ShouldEqual, dq.(*diskQueue).readPos)
120 | dq.Delete()
121 | })
122 | }
123 |
124 | func TestDiskQueueCorruption(t *testing.T) {
125 | Convey("TestDiskQueueCorruption", t, func() {
126 | l := log.New(os.Stderr, "", log.LstdFlags)
127 | dqName := "test_disk_queue_corruption" + strconv.Itoa(int(time.Now().Unix()))
128 | dq := newDiskQueue(dqName, os.TempDir(), 1000, 5, 2*time.Second, l)
129 |
130 | msg := make([]byte, 123)
131 | for i := 0; i < 25; i++ {
132 | dq.Put(msg)
133 | }
134 |
135 | So(dq.Depth(), ShouldEqual, int64(25))
136 |
137 | // corrupt the 2nd file
138 | dqFn := dq.(*diskQueue).fileName(1)
139 | os.Truncate(dqFn, 500)
140 |
141 | for i := 0; i < 19; i++ {
142 | So(string(<-dq.ReadChan()), ShouldEqual, string(msg))
143 | }
144 |
145 | // corrupt the 4th (current) file
146 | dqFn = dq.(*diskQueue).fileName(3)
147 | os.Truncate(dqFn, 100)
148 |
149 | dq.Put(msg)
150 |
151 | So(string(<-dq.ReadChan()), ShouldEqual, string(msg))
152 | dq.Delete()
153 | })
154 | }
155 |
156 | func TestDiskQueueTorture(t *testing.T) {
157 | Convey("TestDiskQueueTorture", t, func() {
158 | var wg sync.WaitGroup
159 |
160 | l := log.New(os.Stderr, "", log.LstdFlags)
161 | dqName := "test_disk_queue_torture" + strconv.Itoa(int(time.Now().Unix()))
162 | dq := newDiskQueue(dqName, os.TempDir(), 262144, 2500, 2*time.Second, l)
163 | So(dq, ShouldNotEqual, nil)
164 | So(dq.Depth(), ShouldEqual, int64(0))
165 |
166 | msg := []byte("aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff")
167 |
168 | numWriters := 4
169 | numReaders := 4
170 | readExitChan := make(chan int)
171 | writeExitChan := make(chan int)
172 |
173 | var depth int64
174 | for i := 0; i < numWriters; i++ {
175 | wg.Add(1)
176 | go func() {
177 | defer wg.Done()
178 | for {
179 | time.Sleep(100000 * time.Nanosecond)
180 | select {
181 | case <-writeExitChan:
182 | return
183 | default:
184 | err := dq.Put(msg)
185 | if err == nil {
186 | atomic.AddInt64(&depth, 1)
187 | }
188 | }
189 | }
190 | }()
191 | }
192 |
193 | time.Sleep(1 * time.Second)
194 |
195 | dq.Close()
196 |
197 | t.Logf("closing writeExitChan")
198 | close(writeExitChan)
199 | wg.Wait()
200 |
201 | t.Logf("restarting diskqueue")
202 |
203 | dq = newDiskQueue(dqName, os.TempDir(), 262144, 2500, 2*time.Second, l)
204 | So(dq, ShouldNotEqual, nil)
205 | So(dq.Depth(), ShouldEqual, depth)
206 |
207 | var read int64
208 | for i := 0; i < numReaders; i++ {
209 | wg.Add(1)
210 | go func() {
211 | defer wg.Done()
212 | for {
213 | time.Sleep(100000 * time.Nanosecond)
214 | select {
215 | case <-dq.ReadChan():
216 | atomic.AddInt64(&read, 1)
217 | case <-readExitChan:
218 | return
219 | }
220 | }
221 | }()
222 | }
223 |
224 | t.Logf("waiting for depth 0")
225 | for {
226 | if dq.Depth() == 0 {
227 | break
228 | }
229 | time.Sleep(50 * time.Millisecond)
230 | }
231 |
232 | t.Logf("closing readExitChan")
233 | close(readExitChan)
234 | wg.Wait()
235 |
236 | So(read, ShouldEqual, depth)
237 |
238 | dq.Delete()
239 | })
240 | }
241 |
242 | func BenchmarkDiskQueuePut(b *testing.B) {
243 | b.StopTimer()
244 | l := log.New(os.Stderr, "", log.LstdFlags)
245 | dqName := "bench_disk_queue_put" + strconv.Itoa(b.N) + strconv.Itoa(int(time.Now().Unix()))
246 | dq := newDiskQueue(dqName, os.TempDir(), 1024768*100, 2500, 2*time.Second, l)
247 | size := 1024
248 | b.SetBytes(int64(size))
249 | data := make([]byte, size)
250 | b.StartTimer()
251 |
252 | for i := 0; i < b.N; i++ {
253 | dq.Put(data)
254 | }
255 | dq.Delete()
256 | }
257 |
258 | func BenchmarkDiskWrite(b *testing.B) {
259 | b.StopTimer()
260 | fileName := "bench_disk_queue_put" + strconv.Itoa(b.N) + strconv.Itoa(int(time.Now().Unix()))
261 | f, _ := os.OpenFile(path.Join(os.TempDir(), fileName), os.O_RDWR|os.O_CREATE, 0600)
262 | size := 256
263 | b.SetBytes(int64(size))
264 | data := make([]byte, size)
265 | b.StartTimer()
266 |
267 | for i := 0; i < b.N; i++ {
268 | f.Write(data)
269 | }
270 | f.Sync()
271 | os.Remove(f.Name())
272 | }
273 |
274 | func BenchmarkDiskWriteBuffered(b *testing.B) {
275 | b.StopTimer()
276 | fileName := "bench_disk_queue_put" + strconv.Itoa(b.N) + strconv.Itoa(int(time.Now().Unix()))
277 | f, _ := os.OpenFile(path.Join(os.TempDir(), fileName), os.O_RDWR|os.O_CREATE, 0600)
278 | size := 256
279 | b.SetBytes(int64(size))
280 | data := make([]byte, size)
281 | w := bufio.NewWriterSize(f, 1024*4)
282 | b.StartTimer()
283 |
284 | for i := 0; i < b.N; i++ {
285 | w.Write(data)
286 | if i%1024 == 0 {
287 | w.Flush()
288 | }
289 | }
290 | w.Flush()
291 | f.Sync()
292 | os.Remove(f.Name())
293 | }
294 |
295 | // this benchmark should be run via:
296 | // $ go test -test.bench 'DiskQueueGet' -test.benchtime 0.1
297 | // (so that it does not perform too many iterations)
298 | func BenchmarkDiskQueueGet(b *testing.B) {
299 | b.StopTimer()
300 | l := log.New(os.Stderr, "", log.LstdFlags)
301 | dqName := "bench_disk_queue_get" + strconv.Itoa(b.N) + strconv.Itoa(int(time.Now().Unix()))
302 | dq := newDiskQueue(dqName, os.TempDir(), 1024768, 2500, 2*time.Second, l)
303 | for i := 0; i < b.N; i++ {
304 | dq.Put([]byte("aaaaaaaaaaaaaaaaaaaaaaaaaaa"))
305 | }
306 | b.StartTimer()
307 | size := 256
308 | b.SetBytes(int64(size))
309 |
310 | for i := 0; i < b.N; i++ {
311 | <-dq.ReadChan()
312 | }
313 |
314 | dq.Delete()
315 | }
316 |
--------------------------------------------------------------------------------
/buffer.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import ()
4 |
5 | // BackendQueue represents the behavior for the secondary message
6 | // storage system
7 | type BackendQueue interface {
8 | Put([]byte) error
9 | ReadChan() chan []byte // this is expected to be an *unbuffered* channel
10 | Close() error
11 | Delete() error
12 | Depth() int64
13 | Empty() error
14 | }
15 |
--------------------------------------------------------------------------------
/config.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bufio"
5 | "errors"
6 | "fmt"
7 | "io"
8 | "net/http"
9 | "net/url"
10 | "os"
11 | "path"
12 | "regexp"
13 | "strings"
14 | )
15 |
16 | type Opener interface {
17 | FileSystem() http.FileSystem
18 | BasePath() string
19 | NewOpener(path string) Opener
20 | }
21 |
22 | type Config struct {
23 | Root *ConfigElement
24 | }
25 |
26 | type ConfigElement struct {
27 | Name string
28 | Args string
29 | Attrs map[string]string
30 | Elems []*ConfigElement
31 | }
32 |
33 | type LineReader interface {
34 | Next() (string, error)
35 | Close() error
36 | Filename() string
37 | LineNumber() int
38 | }
39 |
40 | type parserContext struct {
41 | tag string
42 | tagArgs string
43 | elems []*ConfigElement
44 | attrs map[string]string
45 | opener Opener
46 | }
47 |
48 | type DefaultLineReader struct {
49 | filename string
50 | lineNumber int
51 | inner *bufio.Reader
52 | closer io.Closer
53 | }
54 |
55 | type DefaultOpener http.Dir
56 |
57 | var (
58 | stripCommentRegexp = regexp.MustCompile("\\s*(?:#.*)?$")
59 | startTagRegexp = regexp.MustCompile("^<([a-zA-Z0-9_]+)\\s*(.+?)?>$")
60 | attrRegExp = regexp.MustCompile("^([a-zA-Z0-9_]+)\\s+(.*)$")
61 | )
62 |
63 | func (reader *DefaultLineReader) Next() (string, error) {
64 | reader.lineNumber += 1
65 | line, isPrefix, err := reader.inner.ReadLine()
66 | if isPrefix {
67 | return "", errors.New(fmt.Sprintf("Line too long in %s at line %d", reader.filename, reader.lineNumber))
68 | }
69 | if err != nil {
70 | return "", err
71 | }
72 | return string(line), err
73 | }
74 |
75 | func (reader *DefaultLineReader) Close() error {
76 | if reader.closer != nil {
77 | return reader.closer.Close()
78 | }
79 | return nil
80 | }
81 |
82 | func (reader *DefaultLineReader) Filename() string {
83 | return reader.filename
84 | }
85 |
86 | func (reader *DefaultLineReader) LineNumber() int {
87 | return reader.lineNumber
88 | }
89 |
90 | func NewDefaultLineReader(filename string, reader io.Reader) *DefaultLineReader {
91 | closer, ok := reader.(io.Closer)
92 | if !ok {
93 | closer = nil
94 | }
95 | return &DefaultLineReader{
96 | filename: filename,
97 | lineNumber: 0,
98 | inner: bufio.NewReader(reader),
99 | closer: closer,
100 | }
101 | }
102 |
103 | func (opener DefaultOpener) FileSystem() http.FileSystem {
104 | return http.Dir(opener)
105 | }
106 |
107 | func (opener DefaultOpener) BasePath() string {
108 | return string(opener)
109 | }
110 |
111 | func (opener DefaultOpener) NewOpener(path_ string) Opener {
112 | if !path.IsAbs(path_) {
113 | path_ = path.Join(string(opener), path_)
114 | }
115 | return DefaultOpener(path_)
116 | }
117 |
118 | func NewLineReader(opener Opener, filename string) (LineReader, error) {
119 | //file, err := opener.FileSystem().Open(filename)
120 | file, err := os.Open(filename)
121 | if err != nil {
122 | return nil, err
123 | }
124 | return NewDefaultLineReader(filename, file), nil
125 | }
126 |
127 | func (opener DefaultOpener) Open(filename string) (http.File, error) {
128 | return http.Dir(opener).Open(filename)
129 | }
130 |
131 | func makeParserContext(tag string, tagArgs string, opener Opener) *parserContext {
132 | return &parserContext{
133 | tag: tag,
134 | tagArgs: tagArgs,
135 | elems: make([]*ConfigElement, 0),
136 | attrs: make(map[string]string),
137 | opener: opener,
138 | }
139 | }
140 |
141 | func makeConfigElementFromContext(context *parserContext) *ConfigElement {
142 | return &ConfigElement{
143 | Name: context.tag,
144 | Args: context.tagArgs,
145 | Attrs: context.attrs,
146 | Elems: context.elems,
147 | }
148 | }
149 |
150 | func handleInclude(reader LineReader, context *parserContext, attrValue string) error {
151 | url_, err := url.Parse(attrValue)
152 | if err != nil {
153 | return err
154 | }
155 | if url_.Scheme == "file" || url_.Path == attrValue {
156 | var abspathPattern string
157 | if path.IsAbs(url_.Path) {
158 | abspathPattern = url_.Path
159 | } else {
160 | abspathPattern = path.Join(context.opener.BasePath(), url_.Path)
161 | }
162 | files, err := Glob(context.opener.FileSystem(), abspathPattern)
163 | if err != nil {
164 | return err
165 | }
166 | for _, file := range files {
167 | newReader, err := NewLineReader(context.opener, file)
168 | if err != nil {
169 | return err
170 | }
171 | defer newReader.Close()
172 | parseConfig(newReader, &parserContext{
173 | tag: context.tag,
174 | tagArgs: context.tagArgs,
175 | elems: context.elems,
176 | attrs: context.attrs,
177 | opener: context.opener.NewOpener(path.Dir(file)),
178 | })
179 | }
180 | return nil
181 | } else {
182 | return errors.New("Not implemented!")
183 | }
184 | }
185 |
186 | func handleSpecialAttrs(reader LineReader, context *parserContext, attrName string, attrValue string) (bool, error) {
187 | if attrName == "include" {
188 | return false, handleInclude(reader, context, attrValue)
189 | }
190 | return false, nil
191 | }
192 |
193 | func parseConfig(reader LineReader, context *parserContext) error {
194 | tagEnd := ""
195 | for {
196 | line, err := reader.Next()
197 | if err != nil {
198 | if err == io.EOF {
199 | break
200 | } else {
201 | return err
202 | }
203 | }
204 |
205 | line = strings.TrimLeft(line, " \t\r\n")
206 | line = stripCommentRegexp.ReplaceAllLiteralString(line, "")
207 | if len(line) == 0 {
208 | continue
209 | } else if submatch := startTagRegexp.FindStringSubmatch(line); submatch != nil {
210 | subcontext := makeParserContext(
211 | submatch[1],
212 | submatch[2],
213 | nil,
214 | )
215 | err = parseConfig(reader, subcontext)
216 | if err != nil {
217 | return err
218 | }
219 | context.elems = append(context.elems, makeConfigElementFromContext(subcontext))
220 | } else if line == tagEnd {
221 | break
222 | } else if submatch := attrRegExp.FindStringSubmatch(line); submatch != nil {
223 | handled, err := handleSpecialAttrs(reader, context, submatch[1], submatch[2])
224 | if err != nil {
225 | return err
226 | }
227 | if !handled {
228 | context.attrs[submatch[1]] = submatch[2]
229 | }
230 | } else {
231 | return errors.New(fmt.Sprintf("Parse error in %s at line %s", reader.Filename(), reader.LineNumber()))
232 | }
233 | }
234 | return nil
235 | }
236 |
237 | func ParseConfig(opener Opener, filename string) (*Config, error) {
238 | context := makeParserContext("(root)", "", opener)
239 | reader, err := NewLineReader(nil, filename)
240 | if err != nil {
241 | return nil, err
242 | }
243 | defer reader.Close()
244 | err = parseConfig(reader, context)
245 | if err != nil {
246 | return nil, err
247 | }
248 | return &Config{Root: makeConfigElementFromContext(context)}, nil
249 | }
250 |
--------------------------------------------------------------------------------
/config_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | . "github.com/smartystreets/goconvey/convey"
5 | "os"
6 | "testing"
7 | )
8 |
9 | func Test_ReadConf(t *testing.T) {
10 | content := `
11 | type tail
12 | path /var/log/apache2.log
13 | format /^(?[^ ]*)$/
14 | pos_file /var/log/apache2.pos
15 | tag test
16 |
17 |
18 |
19 | type httpsqs
20 | host localhost
21 | auth 123456
22 | flush_interval 10
23 | `
24 |
25 | testFile := "/tmp/test.file"
26 | os.Remove(testFile)
27 |
28 | Convey("Write config to a file and read from it", t, func() {
29 | Convey("Write config to a file", func() {
30 | f, err := os.OpenFile(testFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
31 | So(err, ShouldEqual, nil)
32 | _, err = f.WriteString(content)
33 | So(err, ShouldEqual, nil)
34 | f.Close()
35 | })
36 |
37 | Convey("Read from it", func() {
38 | config, err := ParseConfig(nil, testFile)
39 | So(err, ShouldEqual, nil)
40 | So(config, ShouldNotEqual, nil)
41 | for _, v := range config.Root.Elems {
42 | if v.Name == "source" {
43 | So(v.Attrs["type"], ShouldEqual, "tail")
44 | So(v.Attrs["path"], ShouldEqual, "/var/log/apache2.log")
45 | So(v.Attrs["format"], ShouldEqual, "/^(?[^ ]*)$/")
46 | So(v.Attrs["pos_file"], ShouldEqual, "/var/log/apache2.pos")
47 | So(v.Attrs["tag"], ShouldEqual, "test")
48 | } else if v.Name == "match" {
49 | So(v.Args, ShouldEqual, "test.**")
50 | So(v.Attrs["type"], ShouldEqual, "httpsqs")
51 | So(v.Attrs["host"], ShouldEqual, "localhost")
52 | So(v.Attrs["auth"], ShouldEqual, "123456")
53 | So(v.Attrs["flush_interval"], ShouldEqual, "10")
54 | }
55 | }
56 | })
57 | })
58 | os.Remove(testFile)
59 | }
60 |
--------------------------------------------------------------------------------
/glob.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "net/http"
5 | "path"
6 | "regexp"
7 | "strings"
8 | )
9 |
10 | type globMatcherContext struct {
11 | fs http.FileSystem
12 | path string
13 | restOfComponents []string
14 | resultCollector func(string) error
15 | }
16 |
17 | func doMatch(context globMatcherContext) error {
18 | if len(context.restOfComponents) == 0 {
19 | return context.resultCollector(context.path)
20 | }
21 |
22 | f, err := context.fs.Open(context.path)
23 | if err != nil {
24 | return err
25 | }
26 | defer f.Close()
27 |
28 | info, err := f.Stat()
29 | if err != nil {
30 | return err
31 | }
32 | if !info.IsDir() {
33 | return nil
34 | }
35 |
36 | entries, err := f.Readdir(-1)
37 | if err != nil {
38 | return err
39 | }
40 |
41 | for _, entry := range entries {
42 | name := entry.Name()
43 | matched, err := path.Match(context.restOfComponents[0], name)
44 | if err != nil {
45 | return err
46 | }
47 | if matched {
48 | err := doMatch(globMatcherContext{
49 | fs: context.fs,
50 | path: path.Join(context.path, name),
51 | restOfComponents: context.restOfComponents[1:],
52 | resultCollector: context.resultCollector,
53 | })
54 | if err != nil {
55 | return err
56 | }
57 | }
58 | }
59 | return nil
60 | }
61 |
62 | func Glob(fs http.FileSystem, pattern string) ([]string, error) {
63 | retval := make([]string, 0)
64 | components := strings.Split(pattern, "/")
65 | var path_ string
66 | if len(components) > 0 && components[0] == "" {
67 | path_ = "/"
68 | components = components[1:]
69 | } else {
70 | path_ = "."
71 | }
72 | return retval, doMatch(globMatcherContext{
73 | fs: fs,
74 | path: path_,
75 | restOfComponents: components,
76 | resultCollector: func(match string) error {
77 | retval = append(retval, match)
78 | return nil
79 | },
80 | })
81 | }
82 |
83 | type PatternError struct {
84 | message string
85 | }
86 |
87 | func (self *PatternError) Error() string {
88 | return self.message
89 | }
90 |
91 | func buildRegexpFromGlobPatternInner(pattern string, startPos int) (string, int, error) {
92 | state := 0
93 | chunk := ""
94 |
95 | patternLength := len(pattern)
96 | var i int
97 | for i = startPos; i < patternLength; i += 1 {
98 | if state == 0 {
99 | c := pattern[i]
100 | if c == '*' {
101 | state = 2
102 | } else if c == '}' || c == ',' {
103 | break
104 | } else if c == '{' {
105 | chunk += "(?:"
106 | first := true
107 | for {
108 | i += 1
109 | subchunk, lastPos, err := buildRegexpFromGlobPatternInner(pattern, i)
110 | if err != nil {
111 | return "", 0, err
112 | }
113 | if lastPos == patternLength {
114 | return "", 0, &PatternError{"unexpected end of pattern (in expectation of '}' or ',')"}
115 | }
116 | if !first {
117 | chunk += "|"
118 | }
119 | i = lastPos
120 | chunk += "(?:" + subchunk + ")"
121 | first = false
122 | if pattern[lastPos] == '}' {
123 | break
124 | } else if pattern[lastPos] != ',' {
125 | return "", 0, &PatternError{"never get here"}
126 | }
127 | }
128 | chunk += ")"
129 | } else {
130 | chunk += regexp.QuoteMeta(string(c))
131 | }
132 | } else if state == 1 {
133 | // escape
134 | c := pattern[i]
135 | chunk += regexp.QuoteMeta(string(c))
136 | state = 0
137 | } else if state == 2 {
138 | c := pattern[i]
139 | if c == '*' {
140 | state = 3
141 | } else {
142 | chunk += "[^.]*" + regexp.QuoteMeta(string(c))
143 | state = 0
144 | }
145 | } else if state == 3 {
146 | // recursive any
147 | c := pattern[i]
148 | if c == '*' {
149 | return "", 0, &PatternError{"unexpected *"}
150 | } else if c == '.' {
151 | chunk += "(?:.*\\.|^)"
152 | } else {
153 | chunk += ".*" + regexp.QuoteMeta(string(c))
154 | }
155 | state = 0
156 | }
157 | }
158 | if state == 2 {
159 | chunk += "[^.]*"
160 | } else if state == 3 {
161 | chunk += ".*"
162 | }
163 | return chunk, i, nil
164 | }
165 |
166 | func BuildRegexpFromGlobPattern(pattern string) (string, error) {
167 | chunk, pos, err := buildRegexpFromGlobPatternInner(pattern, 0)
168 | if err != nil {
169 | return "", err
170 | }
171 | if pos != len(pattern) {
172 | return "", &PatternError{"unexpected '" + string(pattern[pos]) + "'"}
173 | }
174 | return "^" + chunk + "$", nil
175 | }
176 |
--------------------------------------------------------------------------------
/gofluent.conf:
--------------------------------------------------------------------------------
1 |
2 | type tail
3 | path /usr/local/ysec_agent/log/exec_log.json
4 | format json
5 | pos_file /tmp/exec_log.pos
6 | tag ysec_agent.exec_log
7 |
8 |
9 | #
10 | # type mongodb
11 | # host localhost
12 | # capped on
13 | # capped_size 1024
14 | # database test
15 | # collection test
16 | # user test
17 | # password test
18 | #
19 |
20 |
21 | type forward
22 | host 172.19.108.101
23 | port 8888
24 | flush_interval 10
25 | buffer_queue_limit 64
26 | buffer_chunk_limit 8
27 |
28 |
--------------------------------------------------------------------------------
/in_forward.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "log"
5 | "net"
6 | "sync"
7 | )
8 |
9 | type InputForward struct {
10 | Host string
11 | Port string
12 | }
13 |
14 | func (this *InputForward) Init(cf map[string]string) error {
15 | value := cf["bind"]
16 | if len(value) > 0 {
17 | this.Host = value
18 | } else {
19 | log.Panicln("No bind info configured.")
20 | }
21 |
22 | value = cf["port"]
23 | if len(value) > 0 {
24 | this.Port = value
25 | } else {
26 | log.Panicln("No port info configured.")
27 | }
28 |
29 | return nil
30 | }
31 |
32 | func (this *InputForward) Run(runner iRunner) error {
33 | var listener net.Listener
34 | var conn net.Conn
35 | var err error
36 | var wg sync.WaitGroup
37 |
38 | for {
39 | if conn, err = listener.Accept(); err != nil {
40 | if err.(net.Error).Temporary() {
41 | log.Println("TCP accept failed:", err)
42 | continue
43 | } else {
44 | break
45 | }
46 |
47 | wg.Add(1)
48 |
49 | go this.handleConn(conn, wg)
50 | }
51 | }
52 |
53 | wg.Wait()
54 | return nil
55 | }
56 |
57 | func (this *InputForward) handleConn(conn net.Conn, wg sync.WaitGroup) {
58 | defer wg.Done()
59 |
60 | }
61 |
62 | func init() {
63 | RegisterInput("forward", func() interface{} {
64 | return new(InputForward)
65 | })
66 | }
67 |
--------------------------------------------------------------------------------
/in_tail.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/ActiveState/tail"
5 | "github.com/ugorji/go/codec"
6 | "io/ioutil"
7 | "log"
8 | "os"
9 | "reflect"
10 | "regexp"
11 | "strconv"
12 | "strings"
13 | "time"
14 | )
15 |
16 | type inputTail struct {
17 | path string
18 | format string
19 | tag string
20 | pos_file string
21 |
22 | offset int64
23 | sync_interval int
24 | codec *codec.JsonHandle
25 | time_key string
26 | }
27 |
28 | func (self *inputTail) Init(f map[string]string) error {
29 |
30 | self.sync_interval = 2
31 |
32 | value := f["path"]
33 | if len(value) > 0 {
34 | self.path = value
35 | }
36 |
37 | value = f["format"]
38 | if len(value) > 0 {
39 | self.format = value
40 | if value == "json" {
41 | _codec := codec.JsonHandle{}
42 | _codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
43 | self.codec = &_codec
44 |
45 | value = f["time_key"]
46 | if len(value) > 0 {
47 | self.time_key = value
48 | } else {
49 | self.time_key = "time"
50 | }
51 | }
52 | }
53 |
54 | value = f["tag"]
55 | if len(value) > 0 {
56 | self.tag = value
57 | }
58 |
59 | value = f["pos_file"]
60 | if len(value) > 0 {
61 | self.pos_file = value
62 |
63 | str, err := ioutil.ReadFile(self.pos_file)
64 | if err != nil {
65 | log.Println("ioutil.ReadFile:", err)
66 | }
67 |
68 | f, err := os.Open(self.path)
69 | if err != nil {
70 | log.Println("os.Open:", err)
71 | }
72 |
73 | info, err := f.Stat()
74 | if err != nil {
75 | log.Println("f.Stat:", err)
76 | self.offset = 0
77 | } else {
78 | offset, _ := strconv.Atoi(string(str))
79 | if int64(offset) > info.Size() {
80 | self.offset = info.Size()
81 | } else {
82 | self.offset = int64(offset)
83 | }
84 | }
85 | }
86 |
87 | value = f["sync_interval"]
88 | if len(value) > 0 {
89 | sync_interval, err := strconv.Atoi(value)
90 | if err != nil {
91 | return err
92 | }
93 | self.sync_interval = sync_interval
94 | }
95 |
96 | return nil
97 | }
98 |
99 | func (self *inputTail) Run(runner InputRunner) error {
100 | defer func() {
101 | if err := recover(); err != nil {
102 | logs.Fatalln("recover panic at err:", err)
103 | }
104 | }()
105 |
106 | var seek int
107 | if self.offset > 0 {
108 | seek = os.SEEK_SET
109 | } else {
110 | seek = os.SEEK_END
111 | }
112 |
113 | t, err := tail.TailFile(self.path, tail.Config{
114 | Poll: true,
115 | ReOpen: true,
116 | Follow: true,
117 | MustExist: false,
118 | Location: &tail.SeekInfo{int64(self.offset), seek}})
119 | if err != nil {
120 | return err
121 | }
122 |
123 | f, err := os.OpenFile(self.pos_file, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
124 | if err != nil {
125 | log.Fatalln("os.OpenFile", err)
126 | }
127 | defer f.Close()
128 |
129 | var re regexp.Regexp
130 | if string(self.format[0]) == string("/") || string(self.format[len(self.format)-1]) == string("/") {
131 | format := strings.Trim(self.format, "/")
132 | trueformat := regexp.MustCompile("\\(\\?<").ReplaceAllString(format, "(?P<")
133 | if trueformat != format {
134 | log.Printf("pos_file:%s, format:%s", self.path, trueformat)
135 | }
136 | re = *regexp.MustCompile(trueformat)
137 | self.format = "regexp"
138 | } else if self.format == "json" {
139 |
140 | }
141 |
142 | tick := time.NewTicker(time.Second * time.Duration(self.sync_interval))
143 | count := 0
144 |
145 | for {
146 | select {
147 | case <-tick.C:
148 | {
149 | if count > 0 {
150 | offset, err := t.Tell()
151 | if err != nil {
152 | log.Println("Tell return error: ", err)
153 | continue
154 | }
155 |
156 | str := strconv.Itoa(int(offset))
157 |
158 | _, err = f.WriteAt([]byte(str), 0)
159 | if err != nil {
160 | log.Println("f.WriteAt", err)
161 | return err
162 | }
163 |
164 | count = 0
165 | }
166 | }
167 | case line := <-t.Lines:
168 | {
169 | pack := <-runner.InChan()
170 |
171 | pack.MsgBytes = []byte(line.Text)
172 | pack.Msg.Tag = self.tag
173 | pack.Msg.Timestamp = line.Time.Unix()
174 |
175 | if self.format == "regexp" {
176 | text := re.FindSubmatch([]byte(line.Text))
177 | if text == nil {
178 | pack.Recycle()
179 | continue
180 | }
181 |
182 | for i, name := range re.SubexpNames() {
183 | if len(name) > 0 {
184 | pack.Msg.Data[name] = string(text[i])
185 | }
186 | }
187 | } else if self.format == "json" {
188 | dec := codec.NewDecoderBytes([]byte(line.Text), self.codec)
189 | err := dec.Decode(&pack.Msg.Data)
190 | if err != nil {
191 | log.Println("json.Unmarshal", err)
192 | pack.Recycle()
193 | continue
194 | } else {
195 | t, ok := pack.Msg.Data[self.time_key]
196 | if ok {
197 | if time, xx := t.(uint64); xx {
198 | pack.Msg.Timestamp = int64(time)
199 | delete(pack.Msg.Data, self.time_key)
200 | } else if time64, oo := t.(int64); oo {
201 | pack.Msg.Timestamp = time64
202 | delete(pack.Msg.Data, self.time_key)
203 | } else {
204 | log.Println("time is not int64, ", t, " typeof:", reflect.TypeOf(t))
205 | pack.Recycle()
206 | continue
207 | }
208 | }
209 | }
210 | }
211 |
212 | count++
213 | runner.RouterChan() <- pack
214 | }
215 | }
216 | }
217 |
218 | err = t.Wait()
219 | if err != nil {
220 | return err
221 | }
222 |
223 | return err
224 | }
225 |
226 | func init() {
227 | RegisterInput("tail", func() interface{} {
228 | return new(inputTail)
229 | })
230 | }
231 |
--------------------------------------------------------------------------------
/in_tail_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | . "github.com/smartystreets/goconvey/convey"
5 | "io/ioutil"
6 | "log"
7 | "os"
8 | "testing"
9 | "time"
10 | )
11 |
12 | func TestTailJson(t *testing.T) {
13 | log.SetOutput(ioutil.Discard)
14 | cf := make(map[string]string)
15 | cf["path"] = "/tmp/test.file"
16 | cf["format"] = "json"
17 | cf["tag"] = "test"
18 | cf["pos_file"] = "/tmp/test.pos"
19 | tail := new(inputTail)
20 |
21 | Convey("Init tail plugin", t, func() {
22 | err := tail.Init(cf)
23 | Convey("tail.Init err should be nil", func() {
24 | So(err, ShouldEqual, nil)
25 | })
26 | })
27 |
28 | rChan := make(chan *PipelinePack)
29 | InputRecycleChan := make(chan *PipelinePack, 1)
30 | iPack := NewPipelinePack(InputRecycleChan)
31 | InputRecycleChan <- iPack
32 | iRunner := NewInputRunner(InputRecycleChan, rChan)
33 | os.Remove(cf["pos_file"])
34 | os.Remove(cf["path"])
35 |
36 | f, _ := os.OpenFile(cf["path"], os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
37 |
38 | go tail.Run(iRunner)
39 | time.Sleep(1 * time.Second)
40 | f.Write([]byte("{\"data\":\"test\",\"hello\":\"world\"}\n"))
41 | f.Close()
42 |
43 | pack := <-iRunner.RouterChan()
44 |
45 | Convey("Create tail file and write something", t, func() {
46 | Convey("pipelinepack should get the json map", func() {
47 | So(pack.Msg.Data["data"], ShouldEqual, "test")
48 | So(pack.Msg.Data["hello"], ShouldEqual, "world")
49 | })
50 | })
51 |
52 | os.Remove(cf["pos_file"])
53 | os.Remove(cf["path"])
54 | }
55 |
56 | func TestTailRegex(t *testing.T) {
57 | log.SetOutput(ioutil.Discard)
58 | cf := make(map[string]string)
59 | cf["path"] = "/tmp/test.file"
60 | cf["format"] = "/^(?P.*), (?P.*)$/"
61 | cf["tag"] = "test"
62 | cf["pos_file"] = "/tmp/test.pos"
63 | tail := new(inputTail)
64 |
65 | Convey("Init tail plugin", t, func() {
66 | err := tail.Init(cf)
67 | Convey("tail.Init err should be nil", func() {
68 | So(err, ShouldEqual, nil)
69 | })
70 | })
71 |
72 | rChan := make(chan *PipelinePack)
73 | InputRecycleChan := make(chan *PipelinePack, 1)
74 | iPack := NewPipelinePack(InputRecycleChan)
75 | InputRecycleChan <- iPack
76 | iRunner := NewInputRunner(InputRecycleChan, rChan)
77 | os.Remove(cf["pos_file"])
78 | os.Remove(cf["path"])
79 |
80 | f, _ := os.OpenFile(cf["path"], os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
81 |
82 | go tail.Run(iRunner)
83 | time.Sleep(1 * time.Second)
84 | f.Write([]byte("test, world\n"))
85 | f.Close()
86 |
87 | pack := <-iRunner.RouterChan()
88 |
89 | Convey("Create tail file and write something", t, func() {
90 | Convey("pipelinepack should get the regex map", func() {
91 | So(pack.Msg.Tag, ShouldEqual, "test")
92 | So(pack.Msg.Data["data"], ShouldEqual, "test")
93 | So(pack.Msg.Data["hello"], ShouldEqual, "world")
94 | })
95 | })
96 |
97 | os.Remove(cf["pos_file"])
98 | os.Remove(cf["path"])
99 | }
100 |
101 | func TestSyncIntervalBug(t *testing.T) {
102 | cf := make(map[string]string)
103 | cf["path"] = "/tmp/test.file"
104 | cf["format"] = "/^(?P.*), (?P.*)$/"
105 | cf["tag"] = "test"
106 | cf["sync_interval"] = "4"
107 | cf["pos_file"] = "./test.pos"
108 | tail := new(inputTail)
109 | err := tail.Init(cf)
110 | if err != nil {
111 | t.Fail()
112 | }
113 |
114 | rChan := make(chan *PipelinePack)
115 | InputRecycleChan := make(chan *PipelinePack, 1)
116 | iPack := NewPipelinePack(InputRecycleChan)
117 | InputRecycleChan <- iPack
118 | iRunner := NewInputRunner(InputRecycleChan, rChan)
119 |
120 | f, _ := os.OpenFile(cf["path"], os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
121 |
122 | go tail.Run(iRunner)
123 | time.Sleep(1 * time.Second)
124 | f.Write([]byte("test, world\n"))
125 | f.Close()
126 |
127 | os.Remove(cf["pos_file"])
128 | os.Remove(cf["path"])
129 |
130 | }
131 |
132 | func TestNamedRegexMatch(t *testing.T) {
133 | cf := make(map[string]string)
134 | cf["path"] = "/tmp/test.file"
135 | cf["format"] = "/(^(?P.*)), (?P.*)$/"
136 | cf["tag"] = "test"
137 | cf["sync_interval"] = "4"
138 | cf["pos_file"] = "./test.pos"
139 | tail := new(inputTail)
140 | err := tail.Init(cf)
141 | if err != nil {
142 | t.Fail()
143 | }
144 |
145 | rChan := make(chan *PipelinePack)
146 | InputRecycleChan := make(chan *PipelinePack, 1)
147 | iPack := NewPipelinePack(InputRecycleChan)
148 | InputRecycleChan <- iPack
149 | iRunner := NewInputRunner(InputRecycleChan, rChan)
150 |
151 | f, _ := os.OpenFile(cf["path"], os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
152 |
153 | go tail.Run(iRunner)
154 | time.Sleep(1 * time.Second)
155 | f.Write([]byte("test, world\n"))
156 | f.Close()
157 |
158 | pack := <-iRunner.RouterChan()
159 |
160 | if len(pack.Msg.Data) == 3 {
161 | t.Fatal(pack.Msg.Data)
162 | }
163 |
164 | os.Remove(cf["pos_file"])
165 | os.Remove(cf["path"])
166 |
167 | }
168 |
169 | func TestPcreStyleRegex(t *testing.T) {
170 | cf := make(map[string]string)
171 | cf["path"] = "/tmp/test.file"
172 | cf["format"] = "/(^(?.*)), (?.*)$/"
173 | cf["tag"] = "test"
174 | cf["sync_interval"] = "4"
175 | cf["pos_file"] = "./test.pos"
176 | tail := new(inputTail)
177 | err := tail.Init(cf)
178 | if err != nil {
179 | t.Fail()
180 | }
181 |
182 | rChan := make(chan *PipelinePack)
183 | InputRecycleChan := make(chan *PipelinePack, 1)
184 | iPack := NewPipelinePack(InputRecycleChan)
185 | InputRecycleChan <- iPack
186 | iRunner := NewInputRunner(InputRecycleChan, rChan)
187 |
188 | f, _ := os.OpenFile(cf["path"], os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
189 |
190 | go tail.Run(iRunner)
191 | time.Sleep(1 * time.Second)
192 | f.Write([]byte("test, world\n"))
193 | f.Close()
194 |
195 | pack := <-iRunner.RouterChan()
196 |
197 | if len(pack.Msg.Data) == 3 {
198 | t.Fatal(pack.Msg.Data)
199 | }
200 |
201 | os.Remove(cf["pos_file"])
202 | os.Remove(cf["path"])
203 | }
204 |
--------------------------------------------------------------------------------
/input.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "log"
5 | )
6 |
7 | var input_plugins = make(map[string]func() interface{})
8 |
9 | func RegisterInput(name string, input func() interface{}) {
10 | if input == nil {
11 | log.Fatalln("input: Register input is nil")
12 | }
13 |
14 | if _, ok := input_plugins[name]; ok {
15 | log.Fatalln("input: Register called twice for input " + name)
16 | }
17 |
18 | input_plugins[name] = input
19 | }
20 |
--------------------------------------------------------------------------------
/logger.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | type logger interface {
4 | Output(maxdepth int, s string) error
5 | }
6 |
--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "flag"
5 | "io"
6 | "log"
7 | "net/http"
8 | _ "net/http/pprof"
9 | "os"
10 | )
11 |
12 | type GlobalConfig struct {
13 | PoolSize int
14 | }
15 |
16 | var logs *log.Logger
17 |
18 | func DefaultGC() *GlobalConfig {
19 | gc := new(GlobalConfig)
20 | gc.PoolSize = 1000
21 | return gc
22 | }
23 |
24 | func main() {
25 | c := flag.String("c", "gofluent.conf", "config filepath")
26 | p := flag.String("p", "", "write cpu profile to file")
27 | v := flag.String("v", "error.log", "log file path")
28 | flag.Parse()
29 |
30 | f, err := os.OpenFile(*v, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
31 | if err != nil {
32 | log.Fatalln("os.Open failed, err:", err)
33 | }
34 | defer f.Close()
35 |
36 | w := io.MultiWriter(f, os.Stdout)
37 | logs = log.New(w, "", log.Ldate|log.Ltime|log.Lshortfile)
38 | log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
39 | log.SetOutput(w)
40 |
41 | if *p != "" {
42 | go func() {
43 | log.Println(http.ListenAndServe("0.0.0.0:"+*p, nil))
44 | }()
45 | }
46 |
47 | gc := DefaultGC()
48 | config := NewPipeLineConfig(gc)
49 | config.LoadConfig(*c)
50 |
51 | Run(config)
52 | }
53 |
--------------------------------------------------------------------------------
/out_forward.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bytes"
5 | "github.com/ugorji/go/codec"
6 | "log"
7 | "net"
8 | "path/filepath"
9 | "reflect"
10 | "strconv"
11 | "time"
12 | )
13 |
14 | type OutputForward struct {
15 | host string
16 | port int
17 |
18 | connect_timeout int
19 | flush_interval int
20 | sync_interval int
21 | buffer_queue_limit int64
22 | buffer_chunk_limit int64
23 |
24 | buffer_path string
25 |
26 | codec *codec.MsgpackHandle
27 | enc *codec.Encoder
28 | conn net.Conn
29 | msg_buffer bytes.Buffer
30 | buffer bytes.Buffer
31 | backend BackendQueue
32 | }
33 |
34 | func (self *OutputForward) Init(config map[string]string) error {
35 | _codec := codec.MsgpackHandle{}
36 | _codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
37 | _codec.RawToString = false
38 | _codec.StructToArray = true
39 |
40 | self.host = "localhost"
41 | self.port = 8888
42 | self.flush_interval = 10
43 | self.sync_interval = 2
44 | self.buffer_path = "/tmp/test"
45 | self.buffer_queue_limit = 64 * 1024 * 1024
46 | self.buffer_chunk_limit = 8 * 1024 * 1024
47 | self.connect_timeout = 10
48 | self.codec = &_codec
49 |
50 | value := config["host"]
51 | if len(value) > 0 {
52 | self.host = value
53 | }
54 |
55 | value = config["port"]
56 | if len(value) > 0 {
57 | self.port, _ = strconv.Atoi(value)
58 | }
59 |
60 | value = config["connect_timeout"]
61 | if len(value) > 0 {
62 | self.connect_timeout, _ = strconv.Atoi(value)
63 | }
64 |
65 | value = config["flush_interval"]
66 | if len(value) > 0 {
67 | self.flush_interval, _ = strconv.Atoi(value)
68 | }
69 |
70 | value = config["sync_interval"]
71 | if len(value) > 0 {
72 | sync_interval, err := strconv.Atoi(value)
73 | if err != nil {
74 | return err
75 | }
76 | self.sync_interval = sync_interval
77 | }
78 |
79 | value = config["buffer_path"]
80 | if len(value) > 0 {
81 | self.buffer_path = value
82 | }
83 |
84 | value = config["buffer_queue_limit"]
85 | if len(value) > 0 {
86 | buffer_queue_limit, err := strconv.Atoi(value)
87 | if err != nil {
88 | return err
89 | }
90 | self.buffer_queue_limit = int64(buffer_queue_limit) * 1024 * 1024
91 | }
92 |
93 | value = config["buffer_chunk_limit"]
94 | if len(value) > 0 {
95 | buffer_chunk_limit, err := strconv.Atoi(value)
96 | if err != nil {
97 | return err
98 | }
99 | self.buffer_chunk_limit = int64(buffer_chunk_limit) * 1024 * 1024
100 | }
101 | return nil
102 | }
103 |
104 | func (self *OutputForward) Run(runner OutputRunner) error {
105 | sync_interval := time.Duration(self.sync_interval)
106 | base := filepath.Base(self.buffer_path)
107 | dir := filepath.Dir(self.buffer_path)
108 | self.backend = newDiskQueue(base, dir, self.buffer_queue_limit, 2500, sync_interval*time.Second, logs)
109 |
110 | tick := time.NewTicker(time.Second * time.Duration(self.flush_interval))
111 |
112 | for {
113 | select {
114 | case <-tick.C:
115 | {
116 | if self.backend.Depth() > 0 {
117 | log.Printf("flush %d left", self.backend.Depth())
118 | self.flush()
119 | }
120 | }
121 | case pack := <-runner.InChan():
122 | {
123 | self.encodeRecordSet(pack.Msg)
124 | pack.Recycle()
125 | }
126 | }
127 | }
128 |
129 | }
130 |
131 | func (self *OutputForward) flush() error {
132 | if self.conn == nil {
133 | conn, err := net.DialTimeout("tcp", self.host+":"+strconv.Itoa(self.port), time.Second*time.Duration(self.connect_timeout))
134 | if err != nil {
135 | log.Println("net.DialTimeout failed, err", err)
136 | return err
137 | } else {
138 | self.conn = conn
139 | }
140 | }
141 |
142 | defer self.conn.Close()
143 |
144 | count := 0
145 | depth := self.backend.Depth()
146 |
147 | if self.buffer.Len() == 0 {
148 | for i := int64(0); i < depth; i++ {
149 | self.buffer.Write(<-self.backend.ReadChan())
150 | count++
151 | if int64(self.buffer.Len()) > self.buffer_chunk_limit {
152 | break
153 | }
154 | }
155 | }
156 |
157 | log.Println("buffer sent:", self.buffer.Len(), "count:", count)
158 | n, err := self.buffer.WriteTo(self.conn)
159 | if err != nil {
160 | log.Printf("Write failed. size: %d, buf size: %d, error: %#v", n, self.buffer.Len(), err.Error())
161 | self.conn = nil
162 | return err
163 | }
164 | if n > 0 {
165 | log.Printf("Forwarded: %d bytes (left: %d bytes)\n", n, self.buffer.Len())
166 | }
167 |
168 | self.conn = nil
169 |
170 | return nil
171 |
172 | }
173 |
174 | func (self *OutputForward) encodeRecordSet(msg Message) error {
175 | v := []interface{}{msg.Tag, msg.Timestamp, msg.Data}
176 | if self.enc == nil {
177 | self.enc = codec.NewEncoder(&self.msg_buffer, self.codec)
178 | }
179 | err := self.enc.Encode(v)
180 | if err != nil {
181 | return err
182 | }
183 | self.backend.Put(self.msg_buffer.Bytes())
184 | self.msg_buffer.Reset()
185 | return err
186 | }
187 |
188 | func init() {
189 | RegisterOutput("forward", func() interface{} {
190 | return new(OutputForward)
191 | })
192 | }
193 |
--------------------------------------------------------------------------------
/out_httpsqs.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bytes"
5 | "compress/gzip"
6 | "encoding/json"
7 | "fmt"
8 | "io/ioutil"
9 | "log"
10 | "net/http"
11 | "strconv"
12 | "time"
13 | )
14 |
15 | type outputHttpsqs struct {
16 | host string
17 | port int
18 |
19 | auth string
20 | flush_interval int
21 | debug bool
22 | gzip bool
23 | buffer map[string][]byte
24 | client *http.Client
25 | count int
26 | }
27 |
28 | func (self *outputHttpsqs) Init(f map[string]string) error {
29 | self.host = "localhost"
30 | self.port = 1218
31 | self.flush_interval = 10
32 | self.gzip = true
33 | self.client = &http.Client{}
34 | self.buffer = make(map[string][]byte, 0)
35 |
36 | var value string
37 |
38 | value = f["host"]
39 | if len(value) > 0 {
40 | self.host = value
41 | }
42 |
43 | value = f["port"]
44 | if len(value) > 0 {
45 | self.port, _ = strconv.Atoi(value)
46 | }
47 |
48 | value = f["auth"]
49 | if len(value) > 0 {
50 | self.auth = value
51 | }
52 |
53 | value = f["flush_interval"]
54 | if len(value) > 0 {
55 | self.flush_interval, _ = strconv.Atoi(value)
56 | }
57 |
58 | value = f["gzip"]
59 | if len(value) > 0 {
60 | if value == "off" {
61 | self.gzip = false
62 | }
63 | }
64 |
65 | return nil
66 | }
67 |
68 | func (self *outputHttpsqs) Run(runner OutputRunner) error {
69 |
70 | tick := time.NewTicker(time.Second * time.Duration(self.flush_interval))
71 |
72 | for {
73 | select {
74 | case <-tick.C:
75 | {
76 | if len(self.buffer) > 0 {
77 | self.flush()
78 | }
79 | }
80 | case pack := <-runner.InChan():
81 | {
82 | b, err := json.Marshal(pack.Msg.Data)
83 |
84 | if err != nil {
85 | log.Println("json.Marshal:", err)
86 | pack.Recycle()
87 | continue
88 | }
89 |
90 | if len(self.buffer) == 0 {
91 | self.buffer[pack.Msg.Tag] = append(self.buffer[pack.Msg.Tag], byte('['))
92 | } else if len(self.buffer) > 0 {
93 | self.buffer[pack.Msg.Tag] = append(self.buffer[pack.Msg.Tag], byte(','))
94 | }
95 |
96 | self.count++
97 | self.buffer[pack.Msg.Tag] = append(self.buffer[pack.Msg.Tag], b...)
98 | pack.Recycle()
99 | }
100 | }
101 | }
102 | }
103 |
104 | func (self *outputHttpsqs) flush() {
105 | for k, v := range self.buffer {
106 | url := fmt.Sprintf("http://%s:%d/?name=%s&opt=put&auth=%s", self.host, self.port, k, self.auth)
107 |
108 | v = append(v, byte(']'))
109 | var buf bytes.Buffer
110 | var req *http.Request
111 |
112 | if self.gzip == true {
113 | gzw := gzip.NewWriter(&buf)
114 | gzw.Write([]byte(v))
115 | gzw.Close()
116 | req, _ = http.NewRequest("POST", url, bytes.NewReader(buf.Bytes()))
117 | } else {
118 | req, _ = http.NewRequest("POST", url, bytes.NewReader([]byte(v)))
119 | }
120 |
121 | req.Header.Add("Content-Encoding", "gzip")
122 | req.Header.Add("Content-Type", "application/json")
123 |
124 | log.Println("url:", url, "count:", self.count, "length:", len(v), "gziped:", buf.Len())
125 |
126 | resp, err := self.client.Do(req)
127 | if err != nil {
128 | log.Println("post failed:", err)
129 | continue
130 | }
131 |
132 | v, _ := ioutil.ReadAll(resp.Body)
133 | log.Println("StatusCode:", resp.StatusCode, string(v), "Pos:", resp.Header.Get("Pos"))
134 |
135 | resp.Body.Close()
136 | self.buffer[k] = self.buffer[k][0:0]
137 | delete(self.buffer, k)
138 | self.count = 0
139 | }
140 | }
141 |
142 | func init() {
143 | RegisterOutput("httpsqs", func() interface{} {
144 | return new(outputHttpsqs)
145 | })
146 | }
147 |
--------------------------------------------------------------------------------
/out_mongodb.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | mgo "gopkg.in/mgo.v2"
5 | "log"
6 | "strconv"
7 | )
8 |
9 | type outputMongo struct {
10 | host string
11 | port string
12 | database string
13 | collection string
14 | user string
15 | password string
16 | capped bool
17 | capped_size int
18 | failed_count int
19 | }
20 |
21 | func (this *outputMongo) Init(cf map[string]string) error {
22 | this.host = "localhost"
23 | this.port = "27017"
24 | this.capped = false
25 | this.failed_count = 0
26 |
27 | value := cf["host"]
28 | if len(value) > 0 {
29 | this.host = value
30 | }
31 |
32 | value = cf["port"]
33 | if len(value) > 0 {
34 | this.port = value
35 | }
36 |
37 | value = cf["database"]
38 | if len(value) > 0 {
39 | this.database = value
40 | }
41 |
42 | value = cf["collection"]
43 | if len(value) > 0 {
44 | this.collection = value
45 | }
46 |
47 | value = cf["user"]
48 | if len(value) > 0 {
49 | this.user = value
50 | }
51 |
52 | value = cf["password"]
53 | if len(value) > 0 {
54 | this.password = value
55 | }
56 |
57 | value = cf["capped"]
58 | if len(value) > 0 {
59 | if value == "on" {
60 | this.capped = true
61 | }
62 | }
63 |
64 | value = cf["capped_size"]
65 | if len(value) > 0 {
66 | this.capped_size, _ = strconv.Atoi(value)
67 | }
68 |
69 | return nil
70 | }
71 |
72 | func (this *outputMongo) Run(runner OutputRunner) error {
73 |
74 | //[mongodb://][user:pass@]host1[:port1][,host2[:port2],...][/database][?options]
75 | url := "mongodb://"
76 | if len(this.user) != 0 && len(this.password) != 0 {
77 | url += this.user + ":" + this.password + "@"
78 | }
79 | url += this.host + ":" + this.port + "/" + this.database
80 |
81 | session, err := mgo.Dial(url)
82 | if err != nil {
83 | log.Println("mgo.Dial failed, err:", err)
84 | return err
85 | }
86 |
87 | info := &mgo.CollectionInfo{
88 | Capped: this.capped,
89 | MaxBytes: this.capped_size * 1024 * 1024,
90 | }
91 |
92 | coll := session.DB(this.database).C(this.collection)
93 | err = coll.Create(info)
94 | if err != nil && err.Error() != "collection already exists" {
95 | return err
96 | }
97 |
98 | for {
99 | select {
100 | case pack := <-runner.InChan():
101 | {
102 | session.Refresh()
103 | coll := session.DB(this.database).C(this.collection)
104 |
105 | err = coll.Insert(pack.Msg.Data)
106 | if err != nil {
107 | this.failed_count++
108 | log.Println("insert failed, count=", this.failed_count, "err:", err)
109 | pack.Recycle()
110 | continue
111 | }
112 |
113 | pack.Recycle()
114 | }
115 | }
116 | }
117 | }
118 |
119 | func init() {
120 | RegisterOutput("mongodb", func() interface{} {
121 | return new(outputMongo)
122 | })
123 | }
124 |
--------------------------------------------------------------------------------
/out_mongodb_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | . "github.com/smartystreets/goconvey/convey"
5 | mgo "gopkg.in/mgo.v2"
6 | "testing"
7 | "time"
8 | )
9 |
10 | func TestCreateAndInsert(t *testing.T) {
11 | cf := map[string]string{
12 | "tag": "test",
13 | "host": "localhost",
14 | "port": "27017",
15 | "database": "test",
16 | "collection": "test",
17 | "capped": "on",
18 | "capped_size": "1024",
19 | }
20 | mongo := new(outputMongo)
21 | mongo.Init(cf)
22 | pack := new(PipelinePack)
23 | pack.Msg.Data = map[string]interface{}{
24 | "data": "test",
25 | "hello": "world",
26 | }
27 | inChan := make(chan *PipelinePack, 1)
28 | oRunner := NewOutputRunner(inChan)
29 | inChan <- pack
30 |
31 | go mongo.Run(oRunner)
32 | time.Sleep(1 * time.Second)
33 |
34 | Convey("Test create and insert ops", t, func() {
35 |
36 | //[mongodb://][user:pass@]host1[:port1][,host2[:port2],...][/database][?options]
37 | url := "mongodb://" +
38 | cf["host"] + ":" + cf["port"] + "/" + cf["database"]
39 |
40 | session, err := mgo.Dial(url)
41 | if err != nil {
42 | So(err.Error(), ShouldEqual, "no reachable servers")
43 | return
44 | }
45 | So(session, ShouldNotEqual, nil)
46 | defer session.Close()
47 | coll := session.DB(cf["database"]).C(cf["collection"])
48 | So(coll, ShouldNotEqual, nil)
49 |
50 | result := make(map[string]string)
51 | err1 := coll.Find(nil).One(&result)
52 | So(err1, ShouldEqual, nil)
53 | So(result["data"], ShouldEqual, "test")
54 | So(result["hello"], ShouldEqual, "world")
55 | coll.DropCollection()
56 | })
57 | }
58 |
--------------------------------------------------------------------------------
/out_stdout.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "log"
5 | )
6 |
7 | type OutputStdout struct {
8 | }
9 |
10 | func (self *OutputStdout) Init(f map[string]string) error {
11 | return nil
12 | }
13 |
14 | func (self *OutputStdout) Run(runner OutputRunner) error {
15 |
16 | for {
17 | pack := <-runner.InChan()
18 | log.Println("stdout", pack.Msg)
19 | pack.Recycle()
20 | }
21 |
22 | return nil
23 | }
24 |
25 | func init() {
26 | RegisterOutput("stdout", func() interface{} {
27 | return new(OutputStdout)
28 | })
29 | }
30 |
--------------------------------------------------------------------------------
/output.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "log"
5 | )
6 |
7 | var output_plugins = make(map[string]func() interface{})
8 |
9 | func RegisterOutput(name string, out func() interface{}) {
10 | if out == nil {
11 | log.Fatalln("output: Register output is nil")
12 | }
13 |
14 | if _, ok := output_plugins[name]; ok {
15 | log.Fatalln("output: Register called twice for output " + name)
16 | }
17 |
18 | output_plugins[name] = out
19 | }
20 |
--------------------------------------------------------------------------------
/pipeline_runner.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "log"
5 | "sync/atomic"
6 | )
7 |
8 | type Message struct {
9 | Tag string
10 | Timestamp int64
11 | Data map[string]interface{}
12 | }
13 |
14 | type PipelinePack struct {
15 | MsgBytes []byte
16 | Msg Message
17 | RecycleChan chan *PipelinePack
18 | RefCount int32
19 | }
20 |
21 | func NewPipelinePack(recycleChan chan *PipelinePack) (pack *PipelinePack) {
22 | msgBytes := make([]byte, 100)
23 | data := make(map[string]interface{})
24 | msg := Message{Data: data}
25 | return &PipelinePack{
26 | MsgBytes: msgBytes,
27 | Msg: msg,
28 | RecycleChan: recycleChan,
29 | RefCount: 1,
30 | }
31 | }
32 |
33 | func (this *PipelinePack) Zero() {
34 | this.MsgBytes = this.MsgBytes[:cap(this.MsgBytes)]
35 | this.Msg.Data = make(map[string]interface{})
36 | this.RefCount = 1
37 | }
38 |
39 | func (this *PipelinePack) Recycle() {
40 | cnt := atomic.AddInt32(&this.RefCount, -1)
41 | if cnt == 0 {
42 | this.Zero()
43 | this.RecycleChan <- this
44 | }
45 | }
46 |
47 | type PipelineConfig struct {
48 | Gc *GlobalConfig
49 | InputRunners []interface{}
50 | OutputRunners []interface{}
51 | router Router
52 | }
53 |
54 | func NewPipeLineConfig(gc *GlobalConfig) *PipelineConfig {
55 | config := new(PipelineConfig)
56 | config.router.Init()
57 | config.Gc = gc
58 |
59 | return config
60 | }
61 |
62 | func (this *PipelineConfig) LoadConfig(path string) error {
63 | configure, _ := ParseConfig(nil, path)
64 | for _, v := range configure.Root.Elems {
65 | if v.Name == "source" {
66 | this.InputRunners = append(this.InputRunners, v.Attrs)
67 | } else if v.Name == "match" {
68 | v.Attrs["tag"] = v.Args
69 | this.OutputRunners = append(this.OutputRunners, v.Attrs)
70 | }
71 | }
72 |
73 | return nil
74 | }
75 |
76 | func Run(config *PipelineConfig) {
77 | log.Println("Starting gofluent...")
78 |
79 | rChan := make(chan *PipelinePack, config.Gc.PoolSize)
80 | config.router.AddInChan(rChan)
81 |
82 | for _, input_config := range config.InputRunners {
83 | cf := input_config.(map[string]string)
84 |
85 | InputRecycleChan := make(chan *PipelinePack, config.Gc.PoolSize)
86 | for i := 0; i < config.Gc.PoolSize; i++ {
87 | iPack := NewPipelinePack(InputRecycleChan)
88 | InputRecycleChan <- iPack
89 | }
90 | iRunner := NewInputRunner(InputRecycleChan, rChan)
91 |
92 | go iRunner.Start(cf)
93 | }
94 |
95 | for _, output_config := range config.OutputRunners {
96 | cf := output_config.(map[string]string)
97 |
98 | inChan := make(chan *PipelinePack, config.Gc.PoolSize)
99 | oRunner := NewOutputRunner(inChan)
100 | config.router.AddOutChan(cf["tag"], oRunner.InChan())
101 |
102 | go oRunner.Start(cf)
103 | }
104 |
105 | config.router.Loop()
106 | }
107 |
--------------------------------------------------------------------------------
/plugin_interface.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | type Input interface {
4 | Init(config map[string]string) error
5 | Run(in InputRunner) error
6 | }
7 |
8 | type Output interface {
9 | Init(config map[string]string) error
10 | Run(out OutputRunner) error
11 | }
12 |
--------------------------------------------------------------------------------
/plugin_runner.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "log"
5 | )
6 |
7 | type InputRunner interface {
8 | InChan() chan *PipelinePack
9 | RouterChan() chan *PipelinePack
10 | Start(cf map[string]string)
11 | }
12 |
13 | type iRunner struct {
14 | inChan chan *PipelinePack
15 | routerChan chan *PipelinePack
16 | }
17 |
18 | func NewInputRunner(in, router chan *PipelinePack) InputRunner {
19 | return &iRunner{
20 | inChan: in,
21 | routerChan: router,
22 | }
23 | }
24 |
25 | func (this *iRunner) InChan() chan *PipelinePack {
26 | return this.inChan
27 | }
28 |
29 | func (this *iRunner) RouterChan() chan *PipelinePack {
30 | return this.routerChan
31 | }
32 |
33 | func (this *iRunner) Start(cf map[string]string) {
34 | intput_type, ok := cf["type"]
35 | if !ok {
36 | log.Fatalln("no type configured")
37 | }
38 |
39 | input, ok := input_plugins[intput_type]
40 | if !ok {
41 | log.Fatalln("unkown type ", intput_type)
42 | }
43 |
44 | in := input()
45 |
46 | err := in.(Input).Init(cf)
47 | if err != nil {
48 | log.Fatalln("in.(Input).Init", err)
49 | }
50 |
51 | err = in.(Input).Run(this)
52 | if err != nil {
53 | log.Fatalln("in.(Input).Run", err)
54 | }
55 | }
56 |
57 | type OutputRunner interface {
58 | InChan() chan *PipelinePack
59 | Start(cf map[string]string)
60 | }
61 |
62 | type oRunner struct {
63 | inChan chan *PipelinePack
64 | }
65 |
66 | func NewOutputRunner(in chan *PipelinePack) OutputRunner {
67 | return &oRunner{
68 | inChan: in,
69 | }
70 | }
71 |
72 | func (this *oRunner) InChan() chan *PipelinePack {
73 | return this.inChan
74 | }
75 |
76 | func (this *oRunner) Start(cf map[string]string) {
77 | output_type, ok := cf["type"]
78 | if !ok {
79 | log.Fatalln("no type configured")
80 | }
81 |
82 | output_plugin, ok := output_plugins[output_type]
83 | if !ok {
84 | log.Fatalln("unkown type ", output_type)
85 | }
86 |
87 | out := output_plugin()
88 |
89 | err := out.(Output).Init(cf)
90 | if err != nil {
91 | log.Fatalln("out.(Output).Init", err)
92 | }
93 |
94 | err = out.(Output).Run(this)
95 | if err != nil {
96 | log.Fatalln("out.(Output).Run", err)
97 | }
98 | }
99 |
--------------------------------------------------------------------------------
/router.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "log"
5 | "regexp"
6 | "sync/atomic"
7 | )
8 |
9 | type Router struct {
10 | inChan chan *PipelinePack
11 | outChan map[*regexp.Regexp]chan *PipelinePack
12 | }
13 |
14 | func (self *Router) Init() {
15 | self.outChan = make(map[*regexp.Regexp]chan *PipelinePack)
16 | }
17 |
18 | func (self *Router) AddOutChan(matchtag string, outChan chan *PipelinePack) error {
19 | chunk, err := BuildRegexpFromGlobPattern(matchtag)
20 | if err != nil {
21 | return err
22 | }
23 |
24 | re, err := regexp.Compile(chunk)
25 | if err != nil {
26 | return err
27 | }
28 |
29 | self.outChan[re] = outChan
30 | return nil
31 | }
32 |
33 | func (self *Router) AddInChan(inChan chan *PipelinePack) {
34 | self.inChan = inChan
35 | }
36 |
37 | func (self *Router) Loop() {
38 | for pack := range self.inChan {
39 |
40 | for re, outChan := range self.outChan {
41 | flag := re.MatchString(pack.Msg.Tag)
42 | if flag == true {
43 | atomic.AddInt32(&pack.RefCount, 1)
44 | select {
45 | case outChan <- pack:
46 | default:
47 | {
48 | log.Println("outChan fulled, tag=", pack.Msg.Tag)
49 | <-outChan
50 | outChan <- pack
51 | }
52 | }
53 | }
54 | }
55 |
56 | pack.Recycle()
57 | }
58 | }
59 |
--------------------------------------------------------------------------------
/router_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | . "github.com/smartystreets/goconvey/convey"
5 | "testing"
6 | )
7 |
8 | func TestBuildRegexpFromGlobPattern_0(t *testing.T) {
9 | a, err := BuildRegexpFromGlobPattern("a.b.c")
10 | if err != nil {
11 | t.Fail()
12 | }
13 | if a != "^a\\.b\\.c$" {
14 | t.Fail()
15 | }
16 | }
17 |
18 | func TestBuildRegexpFromGlobPattern_1(t *testing.T) {
19 | a, err := BuildRegexpFromGlobPattern("a.*.c")
20 | if err != nil {
21 | t.Fail()
22 | }
23 | if a != "^a\\.[^.]*\\.c$" {
24 | t.Fail()
25 | }
26 | }
27 |
28 | func TestBuildRegexpFromGlobPattern_2(t *testing.T) {
29 | a, err := BuildRegexpFromGlobPattern("**c")
30 | if err != nil {
31 | t.Fail()
32 | }
33 | if a != "^.*c$" {
34 | t.Fail()
35 | }
36 | }
37 |
38 | func TestBuildRegexpFromGlobPattern_3(t *testing.T) {
39 | a, err := BuildRegexpFromGlobPattern("**.c")
40 | if err != nil {
41 | t.Fail()
42 | }
43 | if a != "^(?:.*\\.|^)c$" {
44 | t.Fail()
45 | }
46 | }
47 |
48 | func TestBuildRegexpFromGlobPattern_4(t *testing.T) {
49 | a, err := BuildRegexpFromGlobPattern("a.{b,c}.d")
50 |
51 | if err != nil {
52 | t.Fail()
53 | }
54 | if a != "^a\\.(?:(?:b)|(?:c))\\.d$" {
55 | t.Fail()
56 | }
57 | }
58 |
59 | func TestRouter(t *testing.T) {
60 | in := make(chan *PipelinePack)
61 | out := make(chan *PipelinePack, 1)
62 | router := new(Router)
63 |
64 | router.Init()
65 | router.AddInChan(in)
66 | router.AddOutChan("test.**", out)
67 | go router.Loop()
68 | one := NewPipelinePack(in)
69 | one.Msg.Tag = "test.one"
70 | in <- one
71 |
72 | Convey("Confirm the outputs from outchan", t, func() {
73 | // Get the one pipelinepack and confirm it
74 | So(len(out), ShouldEqual, 1)
75 | res := <-out
76 | So(res.Msg.Tag, ShouldEqual, "test.one")
77 |
78 | // We should get the three pipelinepack,
79 | // instead of blocking the main loop of router
80 | two := NewPipelinePack(in)
81 | two.Msg.Tag = "test.two"
82 | in <- two
83 |
84 | three := NewPipelinePack(in)
85 | three.Msg.Tag = "test.three"
86 | in <- three
87 |
88 | So(len(out), ShouldEqual, 1)
89 | res = <-out
90 | So(res.Msg.Tag, ShouldEqual, "test.three")
91 |
92 | })
93 | }
94 |
--------------------------------------------------------------------------------
/wercker.yml:
--------------------------------------------------------------------------------
1 | box: wercker/golang
2 | # Build definition
3 | build:
4 | # The steps that will be executed on build
5 | steps:
6 | # Sets the go workspace and places you package
7 | # at the right place in the workspace tree
8 | - setup-go-workspace
9 |
10 | # Gets the dependencies
11 | - script:
12 | name: go get
13 | code: |
14 | cd $WERCKER_SOURCE_DIR
15 | go version
16 | go get -t ./...
17 |
18 | # Build the project
19 | - script:
20 | name: go build
21 | code: |
22 | go build ./...
23 |
24 | # Test the project
25 | - script:
26 | name: go test
27 | code: |
28 | go test ./...
29 |
--------------------------------------------------------------------------------