├── .gitignore ├── LICENSE ├── LicenseVerifier.class ├── README.md ├── crack_xpack.sh ├── elasticsearch-developer-cheatsheet.md ├── elasticsearch-devops-cheatsheet.md ├── filebeat ├── filebeat.yaml └── metricbeat.yml ├── flume ├── dir2kafka.conf ├── kafka2es.conf ├── kafka2hdfs.conf └── kafka2hive.conf ├── img ├── deploy2.png ├── deploy3.png └── deploy4.png ├── install_apm_server.sh ├── install_elasticsearch.sh ├── install_filebeat.sh ├── install_kibana.sh ├── install_logstash.sh ├── install_rsyslog.sh ├── license.json ├── logrotate.d ├── mysql ├── nginx ├── rabbitmq └── redis ├── logstash ├── conf.d │ ├── filter-ceph.conf │ ├── filter-firewall.conf │ ├── filter-haproxy.conf │ ├── filter-ip2location.conf │ ├── filter-java.conf │ ├── filter-jetty.conf │ ├── filter-metrics.conf │ ├── filter-mongodb.conf │ ├── filter-multiline.conf │ ├── filter-mysql-slow.conf │ ├── filter-nginx-access.conf │ ├── filter-nginx-error.conf │ ├── filter-percona-audit-log.conf │ ├── filter-php-slow.conf │ ├── filter-redis.conf │ ├── filter-referer.conf │ ├── filter-solr.conf │ ├── filter-tomcat.conf │ ├── filter-trafficserver.conf │ ├── filter-useragent.conf │ ├── input-beats.conf │ ├── input-http_poller.conf │ ├── input-kafka.conf │ ├── input-netflow.conf │ ├── input-retry.conf │ ├── input-syslog.conf │ ├── input-varnishlog.conf │ ├── output-es.conf │ ├── output-hdfs.conf │ ├── output-kafka.conf │ └── output-opentsdb.conf ├── conf │ ├── ip2location.datx │ ├── referers.yml │ └── useragent.yaml └── intro.txt ├── rsyslog.conf └── rsyslog.d ├── fwd.conf ├── grok.conf ├── kafka.conf ├── log.template.conf ├── log4j.conf ├── log4php.conf ├── log4python.conf ├── messsages.conf ├── mysql-error.conf ├── mysql-slow.conf ├── nginx-access.conf ├── nginx-error.conf ├── php-fpm-error.conf ├── php-fpm-slow.conf └── python-error.conf /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 xiaoma tech 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /LicenseVerifier.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiaomatech/log/47a6d4e914909b0b74876aa1921f54bd619fb4d3/LicenseVerifier.class -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 日志配置 2 | 3 | 4 | 5 | ``` 6 | https://www.elastic.co/guide/en/logstash/current/deploying-and-scaling.html 7 | ``` 8 | -------------------------------------------------------------------------------- /crack_xpack.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | yum install -y java-1.8.0-openjdk-devel 4 | 5 | version=6.2.3 6 | 7 | mkdir test && cd test 8 | jar -xvf /usr/share/elasticsearch/plugins/x-pack/x-pack-core/x-pack-core-*.jar 9 | 10 | echo -ne ''' 11 | package org.elasticsearch.license; 12 | 13 | public class LicenseVerifier 14 | { 15 | public static boolean verifyLicense(final License license, final byte[] encryptedPublicKeyData) { 16 | return true; 17 | } 18 | 19 | public static boolean verifyLicense(final License license) { 20 | return true; 21 | } 22 | } 23 | '''>LicenseVerifier.java 24 | 25 | javac -cp "/usr/share/elasticsearch/lib/elasticsearch-*.jar:/usr/share/elasticsearch/lib/lucene-core-*.jar:/usr/share/elasticsearch/plugins/x-pack/x-pack-core/x-pack-core-*.jar" LicenseVerifier.java 26 | 27 | cp ./LicenseVerifier.class org/elasticsearch/license/ 28 | 29 | jar -cvf x-pack-core-$version.jar ./* 30 | 31 | cp x-pack-core-$version.jar /usr/share/elasticsearch/plugins/x-pack/x-pack-core/x-pack-core-$version.jar 32 | 33 | -------------------------------------------------------------------------------- /elasticsearch-developer-cheatsheet.md: -------------------------------------------------------------------------------- 1 | Data Manipulation 2 | ================= 3 | 4 | Put/Get/Delete index 5 | -------------------- 6 | 7 | curl -XPUT localhost:9200/index-name -d '{"settings": { "number_of_shards": 1}}' 8 | curl -XGET localhost:9200/index-name?pretty 9 | curl -XDELETE localhost:9200/index-name 10 | 11 | Put/Get/Delete template 12 | ----------------------- 13 | 14 | curl -XPUT localhost:9200/_template/template-name -d '{ 15 | "template": "logs*", 16 | "mappings": { 17 | "foo-type": { 18 | "properties": { 19 | "foo-field": { 20 | "type": "text" 21 | } 22 | } 23 | } 24 | }, 25 | "settings": { 26 | "number_of_shards": 1 27 | } 28 | }' 29 | curl -XGET localhost:9200/_template/template-name?pretty 30 | curl -XDELETE localhost:9200/_template/template-name 31 | 32 | Bulk API 33 | -------- 34 | 35 | echo '{"index": { "_index": "logs01", "_type": "logs"}} 36 | {"title": "this is an error"} 37 | {"index": { "_index": "logs02", "_type": "logs"}} 38 | {"title": "this is a warning"} 39 | {"delete": { "_index": "logs03", "_type": "logs", "_id": "abc123"}} 40 | ' > /tmp/bulk 41 | curl localhost:9200/_bulk?pretty –data-binary @/tmp/bulk 42 | 43 | Ingest API (put/get/delete/simulate pipeline) 44 | --------------------------------------------- 45 | 46 | curl -XPUT localhost:9200/_ingest/pipeline/apache -d '{ 47 | "description": "grok apache logs", 48 | "processors": [ 49 | { 50 | "grok": { 51 | "field": "message", 52 | "patterns": ["%{COMBINEDAPACHELOG}%{GREEDYDATA:additional_fields}"] 53 | } 54 | } 55 | ] 56 | }' 57 | curl -XGET localhost:9200/_ingest/pipeline/apache?pretty 58 | curl -XDELETE localhost:9200/_ingest/pipeline/apache 59 | curl -XPOSTlocalhost:9200/_ingest/pipeline/_simulate -d '{ 60 | "pipeline": { 61 | "description": "grok apache logs", 62 | "processors": [ 63 | { 64 | "grok": { 65 | "field": "message", 66 | "patterns": [ 67 | "%{COMBINEDAPACHELOG}%{GREEDYDATA:additional_fields}" 68 | ] 69 | } 70 | } 71 | ] 72 | }, 73 | "docs": [ 74 | { 75 | "_source": { 76 | "message": "example.com – – [22/Apr/2016:18:52:51 +1200] \"GET /images/photos/455.jpg HTTP/1.1\" 200 986 \"-\" \"Mozilla/5.0\" \"-\"" 77 | } 78 | } 79 | ] 80 | }' 81 | 82 | Mapping parameters 83 | ================== 84 | 85 | Field types 86 | ----------- 87 | 88 | curl -XPUT localhost:9200/index-name -d '{ 89 | "mappings": { 90 | "foo-type": { 91 | "properties": { 92 | "foo": { 93 | "type": "text" 94 | } 95 | } 96 | } 97 | } 98 | }' 99 | 100 | By default, string fields are mapped as both: 101 | 102 | - **text** – full-text search 103 | - **keyword** – exact search, sorting and aggregations 104 | 105 | Numeric: **byte, short, integer, long, float, scaled_float, half_float** 106 | Others: **boolean, ip, geo_point, geo_shape** 107 | 108 | Analysis 109 | -------- 110 | 111 | Analyzer components: 112 | 113 | - [character filters] 114 | - tokenizer 115 | - [token filters] 116 | 117 | 118 | 119 | curl -XPUT localhost:9200/index-name -d '{ 120 | "settings": { 121 | "analysis": { 122 | "char_filter": { 123 | "my_mapping_char_filter": { 124 | "type": "mapping", 125 | "mappings": ["& => and"] 126 | } 127 | }, 128 | "analyzer": { 129 | "my_custom_analyzer": { 130 | "char_filter": ["my_mapping_char_filter"], 131 | "tokenizer": "whitespace", 132 | "filter": ["lowercase"] 133 | } 134 | } 135 | } 136 | }, 137 | "mappings": { 138 | "foo-type": { 139 | "properties": { 140 | "foo": { 141 | "type": "text", 142 | "analyzer": "my_custom_analyzer" 143 | } 144 | } 145 | } 146 | } 147 | 148 | 149 | Analyze API: 150 | ------------ 151 | 152 | curl -XPOST localhost:9200/index-name/_analyze -d '{ 153 | "text": ["Fish & Chips"], 154 | "analyzer": "my_custom_analyzer" 155 | }' 156 | # reply 157 | { 158 | "tokens": [ 159 | { 160 | "token": "fish", 161 | "start_offset": 0, 162 | "end_offset": 4, 163 | "type": "word", 164 | "position": 0 165 | }, 166 | { 167 | "token": "and", 168 | … 169 | }, 170 | { 171 | "token": "chips", 172 | … 173 | 174 | Important default analyzers: 175 | 176 | 177 | - **standard** – tokenizes European languages OK, lowercases 178 | - language (e.g. **english, dutch**) – selects the appropriate tokenizer (often **standard**), lowercases, removes stopwords and stems 179 | 180 | Important character filters: 181 | 182 | - **html_strip** – removes HTML elements and decodes HTML character 183 | entities 184 | - **pattern_replace** – replaces regular expression matches 185 | 186 | Important tokenizers: 187 | 188 | - **standard** – the same used in the Standard Analyzer 189 | - **letter** – tokens are only groups of letters 190 | - **whitespace** – treats whitespaces as separators 191 | - **pattern** – regular expression as separator 192 | - **keyword** – treats the whole string as a token 193 | 194 | Important token filters: 195 | 196 | - **lowercase** or **uppercase** – folds cases 197 | - **asciifolding** – folds non-ASCII characters to ASCII equivalents for 198 | european languages 199 | - **stemmer** – reduces words to their roots (with configurable 200 | aggressiveness) 201 | - **synonym** – adds synonym tokens to the index 202 | - **ngram** – creates tokens out of groups of consecutive letters 203 | - **edge ngram** – ngrams for prefixes 204 | - **reverse** – flips character order (combine with edge ngram for suffix 205 | matching) 206 | - **shingle** – word ngrams 207 | 208 | Mapping options 209 | --------------- 210 | 211 | curl -XPUT localhost:9200/index-name -d '{ 212 | "mappings": { 213 | "foo-type": { 214 | "properties": { 215 | "foo": { 216 | "type": "text", 217 | "index_options": "docs", 218 | "norms": false, 219 | "fields": { 220 | "keyword": { 221 | "type": "keyword", 222 | "doc_values": true, 223 | "index": false 224 | } 225 | } 226 | } 227 | } 228 | } 229 | } 230 | }' 231 | 232 | - **doc_values** (true/false) – for sorting and aggregations on a field 233 | - **index** (true/false) – for searching on a field 234 | - **index_options** – whether to index only the term (**docs**), or also its 235 | frequency (**freqs**) and where it occurs (**positions** and **offsets**) 236 | - **norms** (true/false) – for normalizing scores relative to field length 237 | - **ignore_above** – don’t index terms bigger than N characters 238 | 239 | Queries 240 | ======= 241 | 242 | Full-text search 243 | ---------------- 244 | 245 | Lucene query syntax: **query_string** 246 | 247 | curl localhost:9200/index-name/_search -d '{ 248 | "query": { 249 | "query_string": { 250 | "query": "+fish +chips" 251 | } 252 | } 253 | }' 254 | 255 | Options: 256 | 257 | - **field:value** to look in **field**, or search in all fields (default) or in 258 | a specified **default_field** 259 | - +requiredTerm -excludedTerm. Or you can say **requiredTerm1** AND **requiredTerm2** 260 | - **(firstName AND lastName) OR alias** 261 | - **Ealsticsearch~1** (fuzziness of one character to tolerate typos) 262 | - **"Sematext consulting Elasticsearch"~2** (slop of two words) 263 | - **E?asticse*** 264 | - **date:[2017-01-01 TO 2018-01-01] OR rating:[3 TO *]** 265 | - **boostThisTermByTen^10** 266 | - escape special characters (?*~^:+-), use a backslash (\) 267 | 268 | Text-box like search: **match** 269 | 270 | 271 | "query": { 272 | "match": { 273 | "foo": { 274 | "query": "bar baz", 275 | "operator": "OR" 276 | } 277 | } 278 | } 279 | 280 | Options: 281 | 282 | - fuzziness allows typos to be tolerated 283 | - cutoff_frequency high-frequency terms are searched only on results of 284 | the low-frequency terms For match on multiple fields: multi_match 285 | 286 | "multi_match": { 287 | "query": "fish chips", 288 | "fields": ["foo", "bar"] 289 | } 290 | 291 | Can set **type** to: 292 | 293 | - **best_fields** (default): takes the highest scoring field into account, 294 | optionally taking a fraction of the others (as defined by 295 | **tie_breaker**) 296 | - **most_fields**: sums up scores of all fields (equivalent to best_fields 297 | with tie_breaker=1) 298 | - **cross_fields**: treats multiple fields as one 299 | - **phrase**: like best_fields, but matches phrases with a configurable 300 | slop 301 | - **phrase_prefix**: like phrase, but considers the prefix of the last term 302 | 303 | Filtering 304 | --------- 305 | 306 | Exact values: **term** and **terms** 307 | 308 | "term": { 309 | "foo": "fish" 310 | } 311 | 312 | **range** 313 | 314 | "range": { 315 | "retweets": { 316 | "gte": 10, 317 | "lte": 20 318 | } 319 | } 320 | 321 | Wrappers 322 | -------- 323 | 324 | Combining other queries: **bool** 325 | 326 | "bool": { 327 | "must": { 328 | "match": { 329 | "foo": "fish" 330 | } 331 | }, 332 | "filter": { 333 | "range": { 334 | "retweets": { 335 | "gte": 10 336 | } 337 | } 338 | } 339 | } 340 | 341 | Clauses: 342 | 343 | - **must**: queries required both to produce a hit and for scoring 344 | - **should**: queries that, if matched, contribute to the score 345 | - **filter**: required queries, not influencing score (cacheable) 346 | - **must_not**: cacheable queries that are required not to match 347 | 348 | Alters score to [a subset of] results: **function_score** 349 | 350 | 351 | "function_score": { 352 | "query": { 353 | "match": { 354 | "foo": "fish" 355 | } 356 | }, 357 | "functions": [ 358 | { 359 | "filter": { 360 | "range": { 361 | "retweets": { 362 | "gte": 10 363 | } 364 | } 365 | }, 366 | "weight": 5 367 | } 368 | ] 369 | } 370 | 371 | Functions: 372 | 373 | - weight/random_score: multiply the score by a static or a random 374 | number 375 | - field_value_factor: multiply the score by a factor (e.g. square root) 376 | of the value of a field 377 | - linear/exp/gauss decay: reduce the score based on how far the value 378 | of a field is from a specified origin script: use a script to 379 | generate a weight 380 | 381 | Aggregations 382 | ============ 383 | 384 | curl localhost:9200/index-name/_search -d '{ 385 | "size": 0, 386 | "aggs": { 387 | "most_foos": { 388 | "terms": { 389 | "field": "foo.keyword" 390 | } 391 | } 392 | } 393 | }' 394 | 395 | Term occurrences 396 | ---------------- 397 | 398 | - **terms**: by default, most occurrences of a term. Can order by other 399 | criteria (including other aggregations) 400 | - **significant_terms**: terms occurring more often in the query results 401 | compared to overall. More expensive, may want to use the **sampler** 402 | aggregation 403 | 404 | Ranges 405 | ------ 406 | 407 | - **range**: buckets of documents from defined numeric ranges 408 | - **date_range/ip_range**: same as range, but for dates and IPs 409 | - **histogram/date_histogram**: ranges are fixed from an interval 410 | 411 | Statistics 412 | ---------- 413 | 414 | 415 | 416 | "aggs": { 417 | "avg_retweets": { 418 | "avg": { 419 | "field": "retweets" 420 | } 421 | } 422 | } 423 | 424 | - **value_count/min/max/avg/sum** of values from a field 425 | - **percentiles** from a numeric field are approximate 426 | - **cardinality** of terms is also approximate 427 | 428 | Grouping by nesting aggregations 429 | -------------------------------- 430 | 431 | The following gets the top results, ordered by **_score**, grouped by the value of **bar** (one hit per value). 432 | 433 | 434 | "query": { 435 | "match": { 436 | "foo": "fish" 437 | } 438 | }, 439 | "size": 0, 440 | "aggs": { 441 | "most_foo": { 442 | "terms": { 443 | "field": "bar.keyword", 444 | "order": { 445 | "max_score": "desc" 446 | } 447 | }, 448 | "aggs": { 449 | "max_score": { 450 | "max": { 451 | "script": { 452 | "inline": "_score" 453 | } 454 | } 455 | }, 456 | "top_hit": { 457 | "top_hits": { 458 | "size": 1 459 | } 460 | } 461 | } 462 | } 463 | } 464 | 465 | Document relationships 466 | ====================== 467 | 468 | Objects 469 | ------- 470 | 471 | Good for one-to-one relations or when you’re searching a single field: 472 | 473 | curl -XPOST localhost:9200/blog/posts/ -d '{ 474 | "title": "Fish & Chips", 475 | "author": { 476 | "first_name": "John", 477 | "last_name": "Smith" 478 | } 479 | }' 480 | 481 | Nested 482 | ------ 483 | 484 | When you need boundaries between objects (e.g. **first_name:jane AND last_name:smith**). Mapping needs to specify that the parent field is **nested**: 485 | 486 | 487 | "mappings": { 488 | "posts": { 489 | "properties": { 490 | "authors": { 491 | "type": "nested" 492 | } 493 | } 494 | } 495 | } 496 | 497 | Documents look like regular objects (even though they’re separate Lucene documents): 498 | 499 | 500 | "authors": [ 501 | { 502 | "first_name": "John", 503 | "last_name": "Smith" 504 | }, 505 | { 506 | "first_name": "Jane", 507 | "last_name": "Adams" 508 | } 509 | ] 510 | 511 | Queries (and aggregations) need to be aware of this and do the join: 512 | 513 | 514 | "query": { 515 | "nested": { 516 | "path": "authors", 517 | "query": { 518 | "match": { 519 | "authors.first_name": "Jane" 520 | } 521 | } 522 | } 523 | } 524 | 525 | Parent-child 526 | ------------ 527 | 528 | When updates are frequent and you want to avoid reindexing the whole ensemble (as you would with nested documents). These are completely separate documents, going in different types: 529 | 530 | 531 | "mappings": { 532 | "authors": { 533 | "_parent": { 534 | "type": "posts" 535 | } 536 | } 537 | } 538 | 539 | Children point to parents via the **_parent field**: 540 | 541 | curl -XPOST localhost:9200/blog/posts/1 -d '{ 542 | "title": "Fish & Chips" 543 | }' 544 | curl -XPOST localhost:9200/blog/authors?parent=1 -d '{ 545 | "first_name": "John", 546 | "last_name": "Smith" 547 | }' 548 | curl -XPOSTlocalhost:9200/blog/authors?parent=1 -d '{ 549 | "first_name": "Jane", 550 | "last_name": "Adams" 551 | }' 552 | Like with nested documents, the query has to specify that a join needs to be done: 553 | "query": { 554 | "has_child": { 555 | "type": "authors", 556 | "query": { 557 | "match": { 558 | "first_name": "Jane" 559 | } 560 | } 561 | } 562 | } 563 | -------------------------------------------------------------------------------- /elasticsearch-devops-cheatsheet.md: -------------------------------------------------------------------------------- 1 | DevOps cheatsheet 2 | ================= 3 | 4 | Allocation 5 | ========== 6 | 7 | Allocation awareness 8 | -------------------- 9 | 10 | Avoids putting two copies of the same shard on nodes with the same attribute (e.g. rack, availability zone). For example: 11 | 12 | node.attr.availability_zone: us-east1 # in elasticsearch.yml 13 | 14 | Awareness is enabled at the cluster level: 15 | 16 | curl -XPUT localhost:9200/_cluster/settings?pretty -d '{ 17 | "persistent" : { 18 | "cluster.routing.allocation.awareness.attributes" : "availability_zone" 19 | } 20 | }' 21 | 22 | Allocation filtering 23 | -------------------- 24 | 25 | Shards of an index can prefer/avoid nodes with certain attributes. Good for having hot/cold tiers: 26 | node.attr.temperature: hot # in elasticsearch.yml 27 | 28 | At index creation, you can assign shards to the hot nodes: 29 | 30 | curl -XPUT localhost:9200/logs01 -d '{ 31 | "settings": { 32 | "index.routing.allocation.include.tag": "hot" 33 | } 34 | }' 35 | 36 | Later on, you can change this value to **cold** move the shards to nodes having **temperature** set to **cold**. 37 | 38 | Delayed allocation 39 | ------------------ 40 | 41 | Avoids the domino effect of relocation when a node is restarted or temporarily unavailable: 42 | 43 | curl -XPUT localhost:9200/$INDEX/_settings -d '{ 44 | "settings": { 45 | "index.unassigned.node_left.delayed_timeout": "5m" 46 | } 47 | }' 48 | 49 | Caches 50 | ====== 51 | 52 | Query cache 53 | ----------- 54 | 55 | Defaults to 10% of heap: 56 | 57 | indices.queries.cache.size: 7% # in elasticsearch.yml 58 | 59 | By default, queries running in the **filter** context will be cached if they run repeatedly, and only on larger segments. You can override this and cache everything in **elasticsearch.yml**: 60 | 61 | index.queries.cache.everything: true 62 | 63 | Request cache 64 | ------------- 65 | 66 | Caches results of aggregations on indices that haven’t changed. Defaults to 1% of heap: 67 | 68 | indices.requests.cache.size: 2% 69 | 70 | Indexing buffer 71 | --------------- 72 | 73 | A node-level buffer for indexing, before a flush will commit to disk. Defaults to 10% of heap: 74 | 75 | indices.memory.index_buffer_size: 5% 76 | 77 | Page recycler 78 | ------------- 79 | 80 | Big arrays used by aggregations are put here so they can be reused. Defaults to 10% of heap: 81 | 82 | cache.recycler.page.limit.heap: 5% 83 | 84 | Field data 85 | ---------- 86 | 87 | The only way to do sorting/aggregations on text fields. Avoid it if possible. If not, limit it through per-request circuit breakers: 88 | 89 | indices.breaker.fielddata.limit: 10% 90 | 91 | And by limiting the overall size: 92 | 93 | indices.fielddata.cache.size: 20% 94 | 95 | Merges 96 | ====== 97 | 98 | Force merge 99 | ----------- 100 | 101 | Might be worth merging indices that don’t change into a handful of big segments: 102 | 103 | curl -XPOST localhost:9200/$INDEX/_forcemerge?max_num_segments=5 104 | 105 | Segments need to be merged in order to change the compression level, so you can do that before force merging: 106 | 107 | curl -XPUT localhost:9200/$INDEX/_settings -d '{ 108 | "index.codec": "best_compression" 109 | }' 110 | 111 | Thread pools 112 | ------------ 113 | 114 | **size** = number of parallel requests, and **queue_size** = number of waiting requests: 115 | 116 | threadpool.search.size: 8 117 | threadpool.search.queue_size: 5000 118 | threadpool.bulk.size: 12 119 | threadpool.bulk.queue_size: 500 120 | 121 | Merge policy 122 | ------------ 123 | 124 | There are multiple knobs here. Most importantly: 125 | Segments per tier. Defaults to 10. Higher values allow for more segments, giving better indexing throughput at the expense of search latency, disk space, memory and open file handles 126 | Max merge at once. Defaults to 10. Lower values lower the impact of merging, but will make the process slower (which can potentially throttle indexing) 127 | Max merged segment. Defaults to 5GB. Lower values result in less merges of large segments, but require more merges of small segments, trading spikes for overall load. 128 | 129 | curl -XPUT localhost:9200/$INDEX/_settings -d '{ 130 | "index.merge.policy": { 131 | "segments_per_tier": 50, 132 | "max_merge_at_once": 50, 133 | "max_merged_segment": "1gb" 134 | } 135 | }' 136 | 137 | Shrink index 138 | ------------ 139 | 140 | Shrink an index into a new one with less shards (factor of the current number of shards): 141 | 142 | curl -XPOST localhost:9200/logs01/_shrink/logs01_shrinked -d '{ 143 | "settings": { 144 | "index.number_of_shards": 1 145 | } 146 | }' 147 | 148 | Troubleshooting: get info 149 | ========================= 150 | 151 | Cat health 152 | ---------- 153 | 154 | curl localhost:9200/_cat/health?v 155 | 156 | **v is for “verbose”**, shows column headers. Gives number of nodes, shards (started, initializing, relocating) and cluster color: 157 | 158 | - All primaries and replicas are up 159 | - All primaries are up, but not all replicas 160 | - Not all primaries are up 161 | 162 | Cat nodes 163 | --------- 164 | 165 | curl localhost:9200/_cat/nodes?v 166 | 167 | Shows figures like load and heap usage of nodes. You can select columns via the help parameter to get other metrics. 168 | 169 | _cat/allocation 170 | --------------- 171 | 172 | How many shards are on each node and how much disk space they take (vs free space). 173 | 174 | _cat/indices 175 | ------------ 176 | 177 | How big is each index; how many shards and replicas it has. 178 | 179 | _cat/shards 180 | =========== 181 | 182 | How big is each shard and on which node it is. Shows whether a shard is STARTED, UNASSIGNED, INITIALIZING or RELOCATING. You can easily **grep** though those values when you have many shards 183 | 184 | _cat/segments 185 | ------------- 186 | 187 | How big each segment in each shard is (including memory usage). You can filter by index, for example: 188 | 189 | curl localhost:9200/_cat/segments/$INDEX?v 190 | 191 | If you look at the files, you’ll see different extensions. Most importantly (in terms of memory and storage): 192 | 193 | - .cfs, .cfe: These are **compound segments** 194 | - .fdt: **Stored fields (like _source)** 195 | - .tim: **Term dictionary**, used when searching in indexed fields 196 | - .doc: **Frequency** of each term in each document (for scoring) 197 | - .pos: **Positional** information (for phrase searches) 198 | - .pay: **Payloads**, most notably character offsets (for the 199 | Postings-based highlighters) 200 | - .nvd, .nvm: Field lengths (a.k.a. **norms** - also used for storing) 201 | - .dvd, .dvm: **Doc values** (used for sorting and aggregations) 202 | - .tv?: **Term vectors** (used for the term-vector-based highlighters) 203 | - .dii, .dim: **Point values** (for geo fields as well as numerics) 204 | 205 | More information can be found here (for Elasticsearch 5.x, which uses Lucene 6.x, you may need to change the version): 206 | [https://lucene.apache.org/core/6_0_0/core/org/apache/lucene/codecs/lucene60/package-summary.html](https://lucene.apache.org/core/6_0_0/core/org/apache/lucene/codecs/lucene60/package-summary.html) 207 | 208 | _cat/pending_tasks 209 | ------------------ 210 | 211 | In-progress operations in your cluster. You’d typically catch long-running ones (e.g. snapshot, force merge) or the ones that get queued up when the cluster is in trouble and the master gets overloaded (e.g. lots of mapping/cluster state updates). 212 | _cat/thread_pool 213 | How many threads are active (working on) searches, bulk indexing and so on. You can also see how many are enqueued (queue) compared to the queue.size and how many were rejected (usually because the queue was full). 214 | 215 | _cat/fielddata 216 | -------------- 217 | 218 | How much heap field data (the in memory equivalent of doc values) takes. Per field, per node. 219 | 220 | Nodes stats 221 | ----------- 222 | 223 | curl localhost:9200/_nodes/stats?pretty 224 | 225 | Gives back statistics of all nodes in the cluster. You can filter nodes, too, like **_nodes/_local/stats** just for the current node. Relevant metrics include: 226 | 227 | - How much time was spent in queries, fetches, indexing, merging, etc 228 | - How much memory current segments take, broken by type (e.g. term 229 | dictionary, doc values) which is a good indicator of the live set 230 | - Current and maximum amount of heap usage per pool. Good indicator of 231 | 232 | Nodes hot threads 233 | ----------------- 234 | 235 | curl localhost:9200/_nodes/hot_threads 236 | 237 | Tells you what’s keeping Elasticsearch busy. Add **type=wait** or **type=block** to see what’s keeping it from being busy. You can also filter nodes like **_nodes/_local/hot_threads** 238 | 239 | OS stats 240 | -------- 241 | 242 | **top, iotop, dstat, iostat** help figure out what the bottleneck is. Usually: 243 | 244 | - Aggregations are CPU-intensive and memory-intensive. The last part 245 | may translate into high GC (check the logs for longer GC events) 246 | - Full-text search (without aggregations) is IO latency sensitive 247 | - Indexing (especially merging) is CPU intensive and IO throughput 248 | intensive 249 | - Snapshots, replication and replication are network and disk intensive 250 | 251 | Cluster allocation explain 252 | -------------------------- 253 | 254 | Shows all the decisions that make a particular shard not to be allocated on different nodes: 255 | 256 | curl localhost:9200/_cluster/allocation/explain?pretty -d'{ 257 | "index": "INDEX_NAME", 258 | "shard": 0, 259 | "primary": true 260 | }' 261 | 262 | Also accepts the node name as a **node** value in the body to show the explanation only for it. 263 | 264 | Indices shard stores 265 | -------------------- 266 | 267 | curl localhost:9200/$INDEX/_shard_stores?pretty 268 | 269 | Returns the last exception that occurred while opening shards of this index. 270 | 271 | Troubleshooting actions 272 | ======================= 273 | 274 | Total shards per node 275 | --------------------- 276 | 277 | How many shards an index can have on each node (good for force-balancing the cluster): 278 | 279 | curl -XPUT localhost:9200/$INDEX/_settings -d '{ 280 | "index.routing.allocation.total_shards_per_node": 2 281 | }' 282 | 283 | Disk allocation thresholds 284 | -------------------------- 285 | 286 | Prevents nodes from running out of disk. 287 | 288 | Low watermark: when to stop allocating new shards. 289 | High watermark: when to relocate existing shards. 290 | 291 | curl -XPUT localhost:9200/_cluster/settings -d '{ 292 | "persistent" : { 293 | "cluster.routing.allocation.disk.watermark.low" : "70%", 294 | "cluster.routing.allocation.disk.watermark.high" : "85%" 295 | } 296 | }' 297 | 298 | Shard reroute (allocate, move and cancel) 299 | ----------------------------------------- 300 | 301 | Allows you to try and allocate a shard manually, or cancel a replication/relocation, or to move a shard: 302 | 303 | curl -XPOST localhost:9200/_cluster/reroute -d '{ 304 | "commands" : [ { 305 | "move" : 306 | { 307 | "index" : "INDEX_NAME", "shard" : SHARD_NUMBER, 308 | "from_node" : "SOURCE_NODE", "to_node" : "DESTINATION_NODE" 309 | } 310 | } 311 | ] 312 | }' 313 | 314 | Concurrent replications, relocations and bandwidth 315 | -------------------------------------------------- 316 | 317 | How many shards can be replicated from each node: 318 | 319 | curl -XPUT localhost:9200/_cluster/settings?pretty -d '{ 320 | "persistent" : { 321 | "cluster.routing.allocation.node_concurrent_recoveries": 2 322 | } 323 | }' 324 | 325 | How many shards can move around, cluster-wide: 326 | 327 | curl -XPUT localhost:9200/_cluster/settings?pretty -d '{ 328 | "persistent" : { 329 | "cluster.routing.allocation.cluster_concurrent_rebalance": 2 330 | } 331 | }' 332 | 333 | How much bandwidth can recovery/rebalancing take: 334 | 335 | curl -XPUT localhost:9200/_cluster/settings?pretty -d '{ 336 | "persistent" : { 337 | "indices.recovery.max_bytes_per_sec": "20mb" 338 | } 339 | }' 340 | 341 | Transaction log settings 342 | ------------------------ 343 | 344 | Trade durability for performance (less IOPS): 345 | 346 | curl -XPUT localhost:9200/$INDEX/_settings -d '{ 347 | "index.translog": { 348 | "index.translog.durability": "async" 349 | } 350 | }' 351 | 352 | GC tuning 353 | --------- 354 | 355 | If survivor space is mostly full, you can increase it by lowering -XX:SurvivorRatio in jvm.options (default is 8 on most platforms). 356 | 357 | If the whole young generation (survivor + eden) is mostly full, you can increase it via -XX:NewSize. 358 | 359 | On large heaps (>30GB, usually you’d want to stay under 30GB to get compressed pointers, but 60-90GB may be needed on some big boxes), using G1 instead of CMS should help. To do that, replace: 360 | 361 | -XX:+UseConcMarkSweepGC 362 | -XX:CMSInitiatingOccupancyFraction=75 363 | -XX:+UseCMSInitiatingOccupancyOnly 364 | 365 | With: 366 | 367 | -XX:+UseG1GC 368 | 369 | Clear caches 370 | ------------ 371 | 372 | Quick way to free some heap: 373 | 374 | curl -XPOST localhost:9200/_cache/clear 375 | -------------------------------------------------------------------------------- /filebeat/filebeat.yaml: -------------------------------------------------------------------------------- 1 | filebeat.prospectors: 2 | - type: log 3 | paths: 4 | - /data/logs/web.log 5 | fields: 6 | service: spring-service 7 | log_type: log4j 8 | multiline.pattern: '^[[:digit:]]{4}-[[:digit:]]{2}-[[:digit:]]{2}' 9 | multiline.negate: true 10 | multiline.match: after 11 | exclude_files: [".gz$"] 12 | 13 | - type: log 14 | fields: 15 | service: python-service 16 | log_type: python-logging 17 | paths: ['/data/logs/python/*.log'] 18 | multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}' 19 | multiline.negate: true 20 | multiline.match: after 21 | exclude_files: [".gz$"] 22 | 23 | - type: log 24 | fields: 25 | service: spring-web 26 | log_type: tomcat 27 | paths: ['/data/logs/tomcat/*.log'] 28 | multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}:[0-9]{3}' 29 | multiline.negate: true 30 | multiline.match: after 31 | exclude_files: [".gz$"] 32 | 33 | - type: log 34 | paths: ['/data/logs/*-warg.log'] 35 | fields: 36 | service: spring-web-access 37 | log_type: log4j 38 | multiline.pattern: '^\[' 39 | multiline.negate: true 40 | multiline.match: after 41 | exclude_files: [".gz$"] 42 | 43 | - type: log 44 | paths: ['/data/logs/error*.log'] 45 | fields: 46 | service: spring-web-error 47 | log_type: log4j 48 | multiline.pattern: "^\s" 49 | multiline.negate: true 50 | multiline.match: before 51 | exclude_files: [".gz$"] 52 | 53 | filebeat.modules: 54 | - module: system 55 | syslog: 56 | enabled: true 57 | auth: 58 | enabled: true 59 | 60 | - module: auditd 61 | log: 62 | enabled: true 63 | 64 | - module: nginx 65 | access: 66 | enabled: true 67 | var.paths: ["/data/logs/nginx/access.log*"] 68 | error: 69 | enabled: true 70 | var.paths: ["/data/logs/nginx/error.log*"] 71 | 72 | - module: mysql 73 | error: 74 | enabled: true 75 | var.paths: 76 | - /data/logs/mysql/error.log* 77 | - /data/logs/mysqld.log* 78 | slowlog: 79 | enabled: true 80 | var.paths: 81 | - /data/logs/mysql/mysql-slow.log* 82 | - /var/lib/mysql/{{.builtin.hostname}}-slow.logs 83 | - module: redis 84 | log: 85 | enabled: true 86 | var.paths: 87 | - /data/logs/redis/redis-server.log* 88 | log: 89 | enabled: true 90 | var.paths: 91 | - /data/logs/redis/redis-server.log* 92 | slowlog: 93 | var.hosts: 94 | - "localhost:6379" 95 | var.password: "" 96 | 97 | queue.mem: 98 | events: 8192 99 | flush.min_events: 512 100 | flush.timeout: 5s 101 | 102 | output.logstash: 103 | hosts: ["logstash1.meizu.mz:5044","logstash2.meizu.mz:5044"] 104 | bulk_max_size: 1024 105 | loadbalance: true 106 | worker: 16 107 | 108 | #output.elasticsearch: 109 | # hosts: ["http://es1.meizu.mz:9200","http://es2.meizu.mz:9200"] 110 | # loadbalance: true 111 | # worker: 16 112 | 113 | #output.kafka: 114 | # # initial brokers for reading cluster metadata 115 | # hosts: ["kafka1:9092", "kafka2:9092", "kafka3:9092"] 116 | # # message topic selection + partitioning 117 | # topic: '%{[fields.log_topic]}' 118 | # partition.round_robin: 119 | # reachable_only: false 120 | # required_acks: 1 121 | # compression: snappy 122 | # max_message_bytes: 1000000 123 | 124 | setup.kibana: 125 | host: "kibana.meizu.mz:5601" 126 | enabled: true 127 | 128 | logging.level: warning 129 | logging.to_files: true 130 | logging.to_syslog: false 131 | logging.files: 132 | path: /var/log/filebeat 133 | name: filebeat.log 134 | keepfiles: 7 135 | permissions: 0644 136 | 137 | 138 | setup.template.name: "filebeat" 139 | setup.template.fields: "fields.yml" 140 | setup.template.overwrite: false 141 | setup.template.settings: 142 | index.number_of_shards: 32 143 | index.number_of_replicas: 2 144 | 145 | xpack.monitoring: 146 | enabled: true 147 | elasticsearch: 148 | hosts: ["http://example.com:9200", "http://example2.com:9200"] 149 | username: elastic 150 | password: test 151 | 152 | #filebeat.autodiscover: 153 | # providers: 154 | # - type: docker 155 | # templates: 156 | # - condition: 157 | # equals: 158 | # docker.container.image: redis 159 | # config: 160 | # - type: docker 161 | # containers.ids: 162 | # - "${data.docker.container.id}" 163 | # exclude_lines: ["^\\s+[\\-`('.|_]"] -------------------------------------------------------------------------------- /filebeat/metricbeat.yml: -------------------------------------------------------------------------------- 1 | metricbeat.modules: 2 | - module: system 3 | period: 10s 4 | metricsets: 5 | - cpu 6 | - load 7 | - memory 8 | - network 9 | - process 10 | - process_summary 11 | #- core 12 | #- diskio 13 | #- socket 14 | processes: ['.*'] 15 | process.include_top_n: 16 | by_cpu: 5 # include top 5 processes by CPU 17 | by_memory: 5 # include top 5 processes by memory 18 | 19 | - module: system 20 | period: 1m 21 | metricsets: 22 | - filesystem 23 | - fsstat 24 | processors: 25 | - drop_event.when.regexp: 26 | system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' 27 | 28 | - module: system 29 | period: 15m 30 | metricsets: 31 | - uptime 32 | 33 | 34 | - module: apache 35 | metricsets: ["status"] 36 | enabled: true 37 | period: 1s 38 | hosts: ["http://127.0.0.1"] 39 | 40 | - module: ceph 41 | metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk", "osd_tree"] 42 | period: 10s 43 | hosts: ["localhost:5000"] 44 | 45 | - module: docker 46 | metricsets: ["container", "cpu", "diskio", "healthcheck", "info", "memory", "network"] 47 | hosts: ["unix:///var/run/docker.sock"] 48 | period: 10s 49 | # To connect to Docker over TLS you must specify a client and CA certificate. 50 | #ssl: 51 | #certificate_authority: "/etc/pki/root/ca.pem" 52 | #certificate: "/etc/pki/client/cert.pem" 53 | #key: "/etc/pki/client/cert.key" 54 | 55 | - module: dropwizard 56 | metricsets: ["collector"] 57 | period: 10s 58 | hosts: ["localhost:8080"] 59 | metrics_path: /metrics/metrics 60 | namespace: example 61 | 62 | - module: elasticsearch 63 | metricsets: ["node", "node_stats"] 64 | period: 10s 65 | hosts: ["localhost:9200"] 66 | 67 | - module: etcd 68 | metricsets: ["leader", "self", "store"] 69 | period: 10s 70 | hosts: ["localhost:2379"] 71 | 72 | - module: golang 73 | metricsets: ["expvar","heap"] 74 | period: 10s 75 | hosts: ["localhost:6060"] 76 | heap.path: "/debug/vars" 77 | expvar: 78 | namespace: "example" 79 | path: "/debug/vars" 80 | 81 | - module: graphite 82 | metricsets: ["server"] 83 | enabled: true 84 | # protocol: "udp" 85 | # templates: 86 | # - filter: "test.*.bash.*" # This would match metrics like test.localhost.bash.stats 87 | # namespace: "test" 88 | # template: ".host.shell.metric*" # test.localhost.bash.stats would become metric=stats and tags host=localhost,shell=bash 89 | # delimiter: "_" 90 | 91 | - module: haproxy 92 | metricsets: ["info", "stat"] 93 | period: 10s 94 | hosts: ["tcp://127.0.0.1:14567"] 95 | 96 | 97 | - module: http 98 | metricsets: ["json"] 99 | period: 10s 100 | hosts: ["localhost:80"] 101 | namespace: "json_namespace" 102 | path: "/" 103 | #body: "" 104 | #method: "GET" 105 | #request.enabled: false 106 | #response.enabled: false 107 | 108 | - module: http 109 | metricsets: ["server"] 110 | host: "localhost" 111 | port: "8080" 112 | enabled: false 113 | # paths: 114 | # - path: "/foo" 115 | # namespace: "foo" 116 | # fields: # added to the the response in root. overwrites existing fields 117 | # key: "value" 118 | 119 | - module: kibana 120 | metricsets: ["status"] 121 | period: 10s 122 | hosts: ["localhost:5601"] 123 | 124 | 125 | # Node metrics, from kubelet: 126 | - module: kubernetes 127 | metricsets: 128 | - node 129 | - system 130 | - pod 131 | - container 132 | - volume 133 | period: 10s 134 | hosts: ["localhost:10255"] 135 | 136 | # State metrics from kube-state-metrics service: 137 | - module: kubernetes 138 | enabled: false 139 | metricsets: 140 | - state_node 141 | - state_deployment 142 | - state_replicaset 143 | - state_pod 144 | - state_container 145 | period: 10s 146 | hosts: ["kube-state-metrics:8080"] 147 | 148 | # Kubernetes events 149 | - module: kubernetes 150 | enabled: false 151 | metricsets: 152 | - event 153 | 154 | - module: logstash 155 | metricsets: ["node", "node_stats"] 156 | enabled: false 157 | period: 10s 158 | hosts: ["localhost"] 159 | 160 | - module: memcached 161 | metricsets: ["stats"] 162 | period: 10s 163 | hosts: ["localhost:11211"] 164 | 165 | - module: mongodb 166 | metricsets: ["dbstats", "status"] 167 | period: 10s 168 | 169 | # The hosts must be passed as MongoDB URLs in the format: 170 | # [mongodb://][user:pass@]host[:port]. 171 | # The username and password can also be set using the respective configuration 172 | # options. The credentials in the URL take precedence over the username and 173 | # password configuration options. 174 | hosts: ["localhost:27017"] 175 | 176 | # Username to use when connecting to MongoDB. Empty by default. 177 | #username: user 178 | 179 | # Password to use when connecting to MongoDB. Empty by default. 180 | #password: pass 181 | 182 | - module: mysql 183 | metricsets: ["status"] 184 | period: 10s 185 | 186 | # Host DSN should be defined as "user:pass@tcp(127.0.0.1:3306)/" 187 | # The username and password can either be set in the DSN or using the username 188 | # and password config options. Those specified in the DSN take precedence. 189 | hosts: ["root:secret@tcp(127.0.0.1:3306)/"] 190 | 191 | # Username of hosts. Empty by default. 192 | #username: root 193 | 194 | # Password of hosts. Empty by default. 195 | #password: secret 196 | 197 | # By setting raw to true, all raw fields from the status metricset will be added to the event. 198 | #raw: false 199 | 200 | - module: nginx 201 | metricsets: ["stubstatus"] 202 | period: 10s 203 | 204 | # Nginx hosts 205 | hosts: ["http://127.0.0.1"] 206 | 207 | # Path to server status. Default server-status 208 | #server_status_path: "server-status" 209 | 210 | - module: php_fpm 211 | metricsets: ["pool"] 212 | period: 10s 213 | status_path: "/status" 214 | hosts: ["localhost:8080"] 215 | 216 | 217 | - module: redis 218 | metricsets: ["info", "keyspace"] 219 | period: 10s 220 | 221 | # Redis hosts 222 | hosts: ["127.0.0.1:6379"] 223 | 224 | # Timeout after which time a metricset should return an error 225 | # Timeout is by default defined as period, as a fetch of a metricset 226 | # should never take longer then period, as otherwise calls can pile up. 227 | #timeout: 1s 228 | 229 | # Optional fields to be added to each event 230 | #fields: 231 | # datacenter: west 232 | 233 | # Network type to be used for redis connection. Default: tcp 234 | #network: tcp 235 | 236 | # Max number of concurrent connections. Default: 10 237 | #maxconn: 10 238 | 239 | # Filters can be used to reduce the number of fields sent. 240 | #processors: 241 | # - include_fields: 242 | # fields: ["beat", "metricset", "redis.info.stats"] 243 | 244 | # Redis AUTH password. Empty by default. 245 | #password: foobared 246 | 247 | 248 | - module: rabbitmq 249 | metricsets: ["node", "queue"] 250 | period: 10s 251 | hosts: ["localhost:15672"] 252 | 253 | username: guest 254 | password: guest 255 | 256 | - module: prometheus 257 | metricsets: ["stats"] 258 | period: 10s 259 | hosts: ["localhost:9090"] 260 | metrics_path: /metrics 261 | #namespace: example 262 | 263 | 264 | queue.mem: 265 | events: 8192 266 | flush.min_events: 512 267 | flush.timeout: 5s 268 | 269 | output.elasticsearch: 270 | hosts: ["http://es1.meizu.mz:9200","http://es2.meizu.mz:9200"] 271 | loadbalance: true 272 | worker: 16 273 | 274 | #output.kafka: 275 | # # initial brokers for reading cluster metadata 276 | # hosts: ["kafka1:9092", "kafka2:9092", "kafka3:9092"] 277 | # # message topic selection + partitioning 278 | # topic: '%{[fields.log_topic]}' 279 | # partition.round_robin: 280 | # reachable_only: false 281 | # required_acks: 1 282 | # compression: snappy 283 | # max_message_bytes: 1000000 284 | 285 | setup.kibana: 286 | host: "kibana.meizu.mz:5601" 287 | enabled: true 288 | 289 | setup.template.name: "metricbeat" 290 | setup.template.fields: "fields.yml" 291 | setup.template.overwrite: false 292 | setup.template.settings: 293 | index.number_of_shards: 32 294 | index.number_of_replicas: 2 295 | 296 | logging.level: warning 297 | logging.to_files: true 298 | logging.to_syslog: false 299 | logging.files: 300 | path: /var/log/metricbeat 301 | name: metricbeat.log 302 | keepfiles: 7 303 | permissions: 0644 304 | 305 | 306 | xpack.monitoring: 307 | enabled: true 308 | elasticsearch: 309 | hosts: ["http://example.com:9200", "http://example2.com:9200"] 310 | username: elastic 311 | password: test 312 | -------------------------------------------------------------------------------- /flume/dir2kafka.conf: -------------------------------------------------------------------------------- 1 | ingest.sources = from_dir_src 2 | ingest.channels = to_kafka_channel 3 | ingest.sinks = to_kafka_sink 4 | 5 | ingest.sources.from_dir_src.type = spooldir 6 | ingest.sources.from_dir_src.channels = to_kafka_channel 7 | ingest.sources.from_dir_src.spoolDir = /data/logs/nginx 8 | ingest.sources.from_dir_src.fileHeader = true 9 | ingest.sources.from_dir_src.recursiveDirectorySearch = true 10 | ingest.sources.from_dir_src.basenameHeader = true 11 | ingest.sources.from_dir_src.basenameHeaderKey = basename 12 | ingest.sources.from_dir_src.parentDirectoryHeader = true 13 | ingest.sources.from_dir_src.parentDirectoryHeaderKey = parentDirectory 14 | ingest.sources.from_dir_src.relativeParentDirectoryHeader = true 15 | ingest.sources.from_dir_src.relativeParentDirectoryHeaderKey = relativeParentDirectory 16 | #ingest.sources.from_dir_src.deserializer = org.apache.flume.sink.solr.morphline.BlobDeserializer$Builder 17 | 18 | #ingest.sources = from_dir_src 19 | #ingest.sources.from_dir_src.type = TAILDIR 20 | #ingest.sources.from_dir_src.positionFile = /var/log/flume/taildir_position.json 21 | #ingest.sources.from_dir_src.filegroups = f1 f2 22 | #ingest.sources.from_dir_src.filegroups.f1 = /data/logs/nginx/*_access.log 23 | #ingest.sources.from_dir_src.headers.f1.headerKey1 = nginx 24 | #ingest.sources.from_dir_src.headers.f1.headerKey2 = access 25 | #ingest.sources.from_dir_src.filegroups.f2 = /var/log/nginx/*_error.log 26 | #ingest.sources.from_dir_src.headers.f2.headerKey1 = nginx 27 | #ingest.sources.from_dir_src.headers.f2.headerKey2 = error 28 | #ingest.sources.from_dir_src.fileHeader = true 29 | 30 | 31 | #ingest.channels.to_kafka_channel.type = file 32 | #ingest.channels.to_kafka_channel.checkpointDir = /tmp/flume/to_avro_sink/checkpoint 33 | #ingest.channels.to_kafka_channel.dataDirs = /tmp/flume/to_avro_sink/data 34 | #ingest.channels.to_kafka_channel.transactionCapacity = 10000 35 | #ingest.channels.to_kafka_channel.capacity = 1000000 36 | 37 | #ingest.channels = c1 c2 c3 38 | #ingest.sources.from_kafka_src.selector.type = multiplexing 39 | #ingest.sources.from_kafka_src.selector.header = xxxName 40 | #ingest.sources.from_kafka_src.selector.mapping.value1 = c1 41 | #ingest.sources.from_kafka_src.selector.mapping.value2 = c1 c2 42 | #ingest.sources.from_kafka_src.selector.mapping.value3 = c3 43 | 44 | #ingest.channels = c1 c2 c3 45 | #ingest.sources.from_kafka_src.selector.type = replicating 46 | #ingest.sources.from_kafka_src.channels = c1 c2 c3 47 | #ingest.sources.from_kafka_src.selector.optional = c3 48 | 49 | ingest.channels.to_kafka_channel.type = memory 50 | ingest.channels.to_kafka_channel.capacity = 100000 51 | ingest.channels.to_kafka_channel.transactionCapacity = 100000 52 | 53 | 54 | #ingest.sinkgroups = g1 55 | #ingest.sinkgroups.g1.sinks = k1 k2 k3 56 | #ingest.sinkgroups.g1.processor.type = load_balance 57 | #ingest.sinkgroups.g1.processor.backoff = true 58 | #ingest.sinkgroups.g1.processor.selector = random 59 | ##ingest.sinkgroups.g1.processor.selector = round_robin 60 | 61 | #ingest.sinkgroups = g1 62 | #ingest.sinkgroups.g1.sinks = k1 k2 k3 63 | #ingest.sinkgroups.g1.processor.type = failover 64 | #ingest.sinkgroups.g1.processor.priority.k1 = 5 65 | #ingest.sinkgroups.g1.processor.priority.k2 = 10 66 | #ingest.sinkgroups.g1.processor.priority.k3 = 15 67 | #ingest.sinkgroups.g1.processor.maxpenalty = 10000 68 | 69 | 70 | ingest.sinks.to_kafka_sink.channel = to_kafka_channel 71 | ingest.sinks.to_kafka_sink.type = org.apache.flume.sink.kafka.KafkaSink 72 | ingest.sinks.to_kafka_sink.kafka.topic = mytopic 73 | #ingest.sinks.to_kafka_sink.kafka.topic = %{basename} 74 | ingest.sinks.to_kafka_sink.kafka.bootstrap.servers = kafkingest:6667,kafka2:6667,kafka3:6667 75 | ingest.sinks.to_kafka_sink.kafka.flumeBatchSize = 2000 76 | ingest.sinks.to_kafka_sink.kafka.producer.acks = 1 77 | ingest.sinks.to_kafka_sink.kafka.producer.linger.ms = 1 78 | ingest.sinks.to_kafka_sink.kafka.producer.compression.type = snappy 79 | ingest.sinks.to_kafka_sink.kafka.kafka.producer.type = async 80 | ingest.sinks.to_kafka_sink.kafka.kafka.encoding = UTF-8 81 | ingest.sinks.to_kafka_sink.kafka.kafka.partitioner.class = com.hisun.flume.sink.kafka.partition.RandomPartioner 82 | #ingest.sinks.to_kafka_sink.kafka.producer.security.protocol = SASL_PLAINTEXT 83 | #ingest.sinks.to_kafka_sink.kafka.producer.sasl.mechanism = GSSAPI 84 | #ingest.sinks.to_kafka_sink.kafka.producer.sasl.kerberos.service.name = kafka -------------------------------------------------------------------------------- /flume/kafka2es.conf: -------------------------------------------------------------------------------- 1 | ingest.sources = from_kafka_src 2 | ingest.channels = to_es_channel 3 | ingest.sinks = to_es_sink 4 | 5 | ingest.sources.from_kafka_src.type = org.apache.flume.source.kafka.KafkaSource 6 | ingest.sources.from_kafka_src.kafka.bootstrap.servers = kafka1:6667,kafka2:6667,kafka3:6667 7 | ingest.sources.from_kafka_src.kafka.topics.regex = ^topic[0-9]$ 8 | #ingest.sources.from_kafka_src.kafka.topics = mytopic 9 | ingest.sources.from_kafka_src.kafka.batchSize = 10000 10 | ingest.sources.from_kafka_src.channels = to_es_channel 11 | ingest.sources.from_kafka_src.kafka.consumer.group.id = flume-consumer 12 | ingest.sources.from_kafka_src.kafka.consumer.timeout.ms = 6000 13 | ingest.sources.from_kafka_src.kafka.zookeeper.session.timeout.ms=6000 14 | ingest.sources.from_kafka_src.kafka.rebalance.max.retries=5 15 | ingest.sources.from_kafka_src.kafka.rebalance.backoff.ms=1500 16 | #ingest.sources.from_kafka_src.kafka.consumer.security.protocol = SASL_PLAINTEXT 17 | #ingest.sources.from_kafka_src.kafka.consumer.sasl.mechanism = GSSAPI 18 | #ingest.sources.from_kafka_src.kafka.consumer.sasl.kerberos.service.name = kafka 19 | 20 | 21 | #ingest.channels = c1 c2 c3 22 | #ingest.sources.from_kafka_src.selector.type = multiplexing 23 | #ingest.sources.from_kafka_src.selector.header = xxxName 24 | #ingest.sources.from_kafka_src.selector.mapping.value1 = c1 25 | #ingest.sources.from_kafka_src.selector.mapping.value2 = c1 c2 26 | #ingest.sources.from_kafka_src.selector.mapping.value3 = c3 27 | 28 | #ingest.channels = c1 c2 c3 29 | #ingest.sources.from_kafka_src.selector.type = replicating 30 | #ingest.sources.from_kafka_src.channels = c1 c2 c3 31 | #ingest.sources.from_kafka_src.selector.optional = c3 32 | 33 | #ingest.channels.to_es_channel.type = file 34 | #ingest.channels.to_es_channel.checkpointDir = /tmp/flume/to_es_sink/checkpoint 35 | #ingest.channels.to_es_channel.dataDirs = /tmp/flume/to_es_sink/data 36 | #ingest.channels.to_es_channel.transactionCapacity = 10000 37 | #ingest.channels.to_es_channel.capacity = 1000000 38 | 39 | ingest.channels.to_es_channel.type = memory 40 | ingest.channels.to_es_channel.capacity = 100000 41 | ingest.channels.to_es_channel.transactionCapacity = 100000 42 | 43 | 44 | #ingest.sinkgroups = g1 45 | #ingest.sinkgroups.g1.sinks = k1 k2 k3 46 | #ingest.sinkgroups.g1.processor.type = load_balance 47 | #ingest.sinkgroups.g1.processor.backoff = true 48 | #ingest.sinkgroups.g1.processor.selector = random 49 | ##ingest.sinkgroups.g1.processor.selector = round_robin 50 | 51 | #ingest.sinkgroups = g1 52 | #ingest.sinkgroups.g1.sinks = k1 k2 k3 53 | #ingest.sinkgroups.g1.processor.type = failover 54 | #ingest.sinkgroups.g1.processor.priority.k1 = 5 55 | #ingest.sinkgroups.g1.processor.priority.k2 = 10 56 | #ingest.sinkgroups.g1.processor.priority.k3 = 15 57 | #ingest.sinkgroups.g1.processor.maxpenalty = 10000 58 | 59 | ingest.sinks.to_es_sink.type = elasticsearch 60 | ingest.sinks.to_es_sink.hostNames = es1:9200,es2:9300 61 | ingest.sinks.to_es_sink.indexName = foo_index 62 | ingest.sinks.to_es_sink.indexType = bar_type 63 | ingest.sinks.to_es_sink.clusterName = foobar_cluster 64 | ingest.sinks.to_es_sink.batchSize = 500 65 | ingest.sinks.to_es_sink.ttl = 5d 66 | ingest.sinks.to_es_sink.serializer = org.apache.flume.sink.elasticsearch.ElasticSearchDynamicSerializer 67 | ingest.sinks.to_es_sink.channel = to_es_channel -------------------------------------------------------------------------------- /flume/kafka2hdfs.conf: -------------------------------------------------------------------------------- 1 | ingest.sources = from_kafka_src 2 | ingest.channels = to_avro_channel 3 | ingest.sinks = to_avro_sink 4 | 5 | ingest.sources.from_kafka_src.type = org.apache.flume.source.kafka.KafkaSource 6 | ingest.sources.from_kafka_src.kafka.bootstrap.servers = kafka1:6667,kafka2:6667,kafka3:6667 7 | ingest.sources.from_kafka_src.kafka.topics.regex = ^topic[0-9]$ 8 | #ingest.sources.from_kafka_src.kafka.topics = mytopic 9 | ingest.sources.from_kafka_src.kafka.batchSize = 10000 10 | ingest.sources.from_kafka_src.channels = to_avro_channel 11 | ingest.sources.from_kafka_src.kafka.consumer.group.id = flume-consumer 12 | ingest.sources.from_kafka_src.kafka.consumer.timeout.ms = 6000 13 | ingest.sources.from_kafka_src.kafka.zookeeper.session.timeout.ms=6000 14 | ingest.sources.from_kafka_src.kafka.rebalance.max.retries=5 15 | ingest.sources.from_kafka_src.kafka.rebalance.backoff.ms=1500 16 | #ingest.sources.from_kafka_src.kafka.consumer.security.protocol = SASL_PLAINTEXT 17 | #ingest.sources.from_kafka_src.kafka.consumer.sasl.mechanism = GSSAPI 18 | #ingest.sources.from_kafka_src.kafka.consumer.sasl.kerberos.service.name = kafka 19 | 20 | #ingest.channels = c1 c2 c3 21 | #ingest.sources.from_kafka_src.selector.type = multiplexing 22 | #ingest.sources.from_kafka_src.selector.header = xxxName 23 | #ingest.sources.from_kafka_src.selector.mapping.value1 = c1 24 | #ingest.sources.from_kafka_src.selector.mapping.value2 = c1 c2 25 | #ingest.sources.from_kafka_src.selector.mapping.value3 = c3 26 | 27 | #ingest.channels = c1 c2 c3 28 | #ingest.sources.from_kafka_src.selector.type = replicating 29 | #ingest.sources.from_kafka_src.channels = c1 c2 c3 30 | #ingest.sources.from_kafka_src.selector.optional = c3 31 | 32 | #ingest.channels.to_avro_channel.type = file 33 | #ingest.channels.to_avro_channel.checkpointDir = /tmp/flume/to_avro_sink/checkpoint 34 | #ingest.channels.to_avro_channel.dataDirs = /tmp/flume/to_avro_sink/data 35 | #ingest.channels.to_avro_channel.transactionCapacity = 10000 36 | #ingest.channels.to_avro_channel.capacity = 1000000 37 | 38 | ingest.channels.to_avro_channel.type = memory 39 | ingest.channels.to_avro_channel.capacity = 100000 40 | ingest.channels.to_avro_channel.transactionCapacity = 100000 41 | 42 | 43 | #ingest.sinkgroups = g1 44 | #ingest.sinkgroups.g1.sinks = k1 k2 k3 45 | #ingest.sinkgroups.g1.processor.type = load_balance 46 | #ingest.sinkgroups.g1.processor.backoff = true 47 | #ingest.sinkgroups.g1.processor.selector = random 48 | ##ingest.sinkgroups.g1.processor.selector = round_robin 49 | 50 | #ingest.sinkgroups = g1 51 | #ingest.sinkgroups.g1.sinks = k1 k2 k3 52 | #ingest.sinkgroups.g1.processor.type = failover 53 | #ingest.sinkgroups.g1.processor.priority.k1 = 5 54 | #ingest.sinkgroups.g1.processor.priority.k2 = 10 55 | #ingest.sinkgroups.g1.processor.priority.k3 = 15 56 | #ingest.sinkgroups.g1.processor.maxpenalty = 10000 57 | 58 | ingest.sinks.to_avro_sink.type = hdfs 59 | ingest.sinks.to_avro_sink.hdfs.path = /user/flume/task/avro/%Y/%n/%e/%k 60 | ingest.sinks.to_avro_sink.hdfs.filePrefix = flume_task_avro 61 | ingest.sinks.to_avro_sink.hdfs.fileSuffix = .avro 62 | ingest.sinks.to_avro_sink.hdfs.rollCount = 100 63 | ingest.sinks.to_avro_sink.hdfs.rollInterval = 0 64 | ingest.sinks.to_avro_sink.hdfs.rollSize = 0 65 | ingest.sinks.to_avro_sink.hdfs.useLocalTimeStamp = true 66 | ingest.sinks.to_avro_sink.hdfs.timeZone = UTC 67 | ingest.sinks.to_avro_sink.hdfs.writeFormat = Text 68 | ingest.sinks.to_avro_sink.hdfs.fileType = DataStream 69 | ingest.sinks.to_avro_sink.serializer = avro_event 70 | ingest.sinks.to_avro_sink.channel = to_avro_channel -------------------------------------------------------------------------------- /flume/kafka2hive.conf: -------------------------------------------------------------------------------- 1 | ingest.sources = from_kafka_src 2 | ingest.channels = to_hive_channel 3 | ingest.sinks = to_hive_sink 4 | 5 | ingest.sources.from_kafka_src.type = org.apache.flume.source.kafka.KafkaSource 6 | ingest.sources.from_kafka_src.kafka.bootstrap.servers = kafka1:6667,kafka2:6667,kafka3:6667 7 | ingest.sources.from_kafka_src.kafka.topics.regex = ^topic[0-9]$ 8 | #ingest.sources.from_kafka_src.kafka.topics = mytopic 9 | ingest.sources.from_kafka_src.channels = to_avro_channel 10 | ingest.sources.from_kafka_src.kafka.consumer.group.id = flume-consumer 11 | #ingest.sources.from_kafka_src.kafka.consumer.security.protocol = SASL_PLAINTEXT 12 | #ingest.sources.from_kafka_src.kafka.consumer.sasl.mechanism = GSSAPI 13 | #ingest.sources.from_kafka_src.kafka.consumer.sasl.kerberos.service.name = kafka 14 | 15 | #ingest.sources.from_avro_src.type = avro 16 | #ingest.sources.from_avro_src.channels = to_avro_channel 17 | #ingest.sources.from_avro_src.bind = 0.0.0.0 18 | #ingest.sources.from_avro_src.port = 4141 19 | 20 | 21 | #ingest.channels = c1 c2 c3 22 | #ingest.sources.from_kafka_src.selector.type = multiplexing 23 | #ingest.sources.from_kafka_src.selector.header = xxxName 24 | #ingest.sources.from_kafka_src.selector.mapping.value1 = c1 25 | #ingest.sources.from_kafka_src.selector.mapping.value2 = c1 c2 26 | #ingest.sources.from_kafka_src.selector.mapping.value3 = c3 27 | 28 | #ingest.channels = c1 c2 c3 29 | #ingest.sources.from_kafka_src.selector.type = replicating 30 | #ingest.sources.from_kafka_src.channels = c1 c2 c3 31 | #ingest.sources.from_kafka_src.selector.optional = c3 32 | 33 | #ingest.channels.to_avro_channel.type = file 34 | #ingest.channels.to_avro_channel.checkpointDir = /tmp/flume/to_avro_sink/checkpoint 35 | #ingest.channels.to_avro_channel.dataDirs = /tmp/flume/to_avro_sink/data 36 | #ingest.channels.to_avro_channel.transactionCapacity = 10000 37 | #ingest.channels.to_avro_channel.capacity = 1000000 38 | 39 | ingest.channels.to_avro_channel.type = memory 40 | ingest.channels.to_avro_channel.capacity = 100000 41 | ingest.channels.to_avro_channel.transactionCapacity = 100000 42 | 43 | 44 | #ingest.sinkgroups = g1 45 | #ingest.sinkgroups.g1.sinks = k1 k2 k3 46 | #ingest.sinkgroups.g1.processor.type = load_balance 47 | #ingest.sinkgroups.g1.processor.backoff = true 48 | #ingest.sinkgroups.g1.processor.selector = random 49 | ##ingest.sinkgroups.g1.processor.selector = round_robin 50 | 51 | #ingest.sinkgroups = g1 52 | #ingest.sinkgroups.g1.sinks = k1 k2 k3 53 | #ingest.sinkgroups.g1.processor.type = failover 54 | #ingest.sinkgroups.g1.processor.priority.k1 = 5 55 | #ingest.sinkgroups.g1.processor.priority.k2 = 10 56 | #ingest.sinkgroups.g1.processor.priority.k3 = 15 57 | #ingest.sinkgroups.g1.processor.maxpenalty = 10000 58 | 59 | ingest.sinks.to_hive_sink.type = hive 60 | ingest.sinks.to_hive_sink.hive.metastore = thrift://hive.meizu.mz:9083 61 | ingest.sinks.to_hive_sink.hive.database = default 62 | ingest.sinks.to_hive_sink.hive.table = test 63 | ingest.sinks.to_hive_sink.hive.partition = asia,%{country},%y-%m-%d-%H-%M 64 | ingest.sinks.to_hive_sink.batchSize = 10000 65 | ingest.sinks.to_hive_sink.timeZone = UTC 66 | ingest.sinks.to_hive_sink.useLocalTimeStamp = false 67 | ingest.sinks.to_hive_sink.round = true 68 | ingest.sinks.to_hive_sink.roundUnit = second 69 | ingest.sinks.to_hive_sink.roundValue=1 70 | ingest.sinks.to_hive_sink.serializer = DELIMITED 71 | ingest.sinks.to_hive_sink.serializer.delimiter = , 72 | ingest.sinks.to_hive_sink.serializer.serdeSeparator = '\t' 73 | ingest.sinks.to_hive_sink.serializer.fieldnames = userid,notify,targetid,timestamp 74 | ingest.sinks.to_hive_sink.channel = to_hive_channel -------------------------------------------------------------------------------- /img/deploy2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiaomatech/log/47a6d4e914909b0b74876aa1921f54bd619fb4d3/img/deploy2.png -------------------------------------------------------------------------------- /img/deploy3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiaomatech/log/47a6d4e914909b0b74876aa1921f54bd619fb4d3/img/deploy3.png -------------------------------------------------------------------------------- /img/deploy4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiaomatech/log/47a6d4e914909b0b74876aa1921f54bd619fb4d3/img/deploy4.png -------------------------------------------------------------------------------- /install_apm_server.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | version=6.3.2 4 | 5 | rpm -ivh https://artifacts.elastic.co/downloads/apm-server/apm-server-$version-x86_64.rpm 6 | 7 | echo -ne ''' 8 | apm-server: 9 | host: "localhost:8200" 10 | frontend: 11 | enabled: true 12 | 13 | setup.template.enabled: true 14 | setup.template.settings: 15 | index: 16 | number_of_shards: 16 17 | codec: best_compression 18 | 19 | setup.kibana: 20 | host: "localhost:5601" 21 | 22 | output.elasticsearch: 23 | hosts: ["ElasticsearchAddress:9200"] 24 | worker: 8 25 | bulk_max_size: 5120 26 | username: "elastic" 27 | password: "elastic" 28 | indices: 29 | - index: "apm-%{[beat.version]}-sourcemap" 30 | when.contains: 31 | processor.event: "sourcemap" 32 | 33 | - index: "apm-%{[beat.version]}-error-%{+yyyy.MM.dd}" 34 | when.contains: 35 | processor.event: "error" 36 | 37 | - index: "apm-%{[beat.version]}-transaction-%{+yyyy.MM.dd}" 38 | when.contains: 39 | processor.event: "transaction" 40 | 41 | - index: "apm-%{[beat.version]}-span-%{+yyyy.MM.dd}" 42 | when.contains: 43 | processor.event: "span" 44 | 45 | path.data: /data/apm-server 46 | path.logs: /data/logs/apm-server 47 | 48 | setup.dashboards.enabled: true 49 | 50 | http.enabled: true 51 | '''>/etc/apm-server/apm-server.yml 52 | 53 | systemctl enable apm-server 54 | systemctl start apm-server 55 | -------------------------------------------------------------------------------- /install_elasticsearch.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | version=6.2.3 4 | 5 | rpm -ivh https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$version.rpm 6 | /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch https://artifacts.elastic.co/downloads/elasticsearch-plugins/ingest-geoip/ingest-geoip-$version.zip 7 | /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch https://artifacts.elastic.co/downloads/elasticsearch-plugins/ingest-user-agent/ingest-user-agent-$version.zip 8 | /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-$version.zip 9 | /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch analysis-smartcn 10 | /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch https://github.com/medcl/elasticsearch-analysis-pinyin/releases/download/v$version/elasticsearch-analysis-pinyin-$version.zip 11 | 12 | #https://github.com/o19s/elasticsearch-learning-to-rank 13 | 14 | mkdir -p /etc/sysctl.d 15 | echo "vm.max_map_count=262144" > /etc/sysctl.d/11-es.conf 16 | echo -ne '* soft nproc 8192 \nelasticsearch - nofile 65536 \nelasticsearch soft memlock unlimited \nelasticsearch hard memlock unlimited' > /etc/security/limits.d/es.conf 17 | 18 | echo -ne ''' 19 | ES_HOME=/usr/share/elasticsearch/ 20 | 21 | ES_HEAP_SIZE=128m 22 | MAX_OPEN_FILES=65535 23 | MAX_MAP_COUNT=262144 24 | LOG_DIR=/var/log/elasticsearch 25 | WORK_DIR=/tmp/elasticsearch/ 26 | CONF_DIR=/etc/elasticsearch/ 27 | ES_PATH_CONF=/etc/elasticsearch/ 28 | RESTART_ON_UPGRADE=true 29 | PID_DIR=/var/run/elasticsearch 30 | JAVA_HOME=/usr/java/default 31 | ES_STARTUP_SLEEP_TIME=5 32 | ES_JAVA_OPTS="-Des.index.max_number_of_shards=128 -verbose:gc -Xloggc:/var/log/elasticsearch/elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:ErrorFile=/var/log/elasticsearch/elasticsearch_err.log -XX:ParallelGCThreads=8" 33 | MAX_LOCKED_MEMORY=unlimited 34 | MAX_MAP_COUNT=262144 35 | '''> /etc/sysconfig/elasticsearch 36 | 37 | memory=`free -g | awk 'NR==2{printf $2}'`g 38 | 39 | 40 | echo -ne ''' 41 | -Xms'''$memory''' 42 | -Xmx'''$memory''' 43 | -XX:+UseConcMarkSweepGC 44 | -XX:CMSInitiatingOccupancyFraction=75 45 | -XX:+UseCMSInitiatingOccupancyOnly 46 | -XX:+AlwaysPreTouch 47 | -server 48 | -Xss1m 49 | -Djava.awt.headless=true 50 | -Dfile.encoding=UTF-8 51 | -Djna.nosys=true 52 | -Djdk.io.permissionsUseCanonicalPath=true 53 | -XX:-OmitStackTraceInFastThrow 54 | -Dio.netty.allocator.type=unpooled 55 | -Dio.netty.noUnsafe=true 56 | -Dio.netty.noKeySetOptimization=true 57 | -Dio.netty.recycler.maxCapacityPerThread=0 58 | -Dlog4j.shutdownHookEnabled=false 59 | -Dlog4j2.disable.jmx=true 60 | -XX:+HeapDumpOnOutOfMemoryError 61 | -XX:+PrintGCDetails 62 | -XX:+PrintGCTimeStamps 63 | -XX:+PrintGCDateStamps 64 | -XX:+PrintClassHistogram 65 | -XX:+PrintTenuringDistribution 66 | -XX:+PrintGCApplicationStoppedTime 67 | -XX:+UseGCLogFileRotation 68 | -XX:NumberOfGCLogFiles=32 69 | -XX:GCLogFileSize=128M 70 | '''> /etc/elasticsearch/jvm.options 71 | 72 | SERVER_IP=`/sbin/ifconfig | grep 'inet'| grep -v '127.0.0.1' |head -n1 |tr -s ' '|cut -d ' ' -f3 | cut -d: -f2` 73 | hostname=`hostname -f` 74 | 75 | echo -ne ''' 76 | cluster.name: logcenter 77 | node.name: '''$hostname''' 78 | network.host: '''$SERVER_IP''' 79 | discovery.zen.ping.unicast.hosts: ["10.19.0.97","10.19.0.98","10.19.0.99","10.19.0.100"] 80 | discovery.zen.minimum_master_nodes: 2 81 | path: 82 | data: 83 | - /data01/es 84 | - /data02/es 85 | - /data03/es 86 | - /data04/es 87 | - /data05/es 88 | - /data06/es 89 | - /data07/es 90 | - /data08/es 91 | - /data09/es 92 | - /data10/es 93 | - /data11/es 94 | - /data12/es 95 | logs: /var/log/elasticsearch 96 | 97 | bootstrap.system_call_filter: false 98 | bootstrap.memory_lock: true 99 | http.port: 9200 100 | 101 | action.destructive_requires_name: true 102 | 103 | indices.requests.cache.size: 2% 104 | indices.breaker.fielddata.limit: 10% 105 | indices.fielddata.cache.size: 20% 106 | 107 | thread_pool.search.queue_size: 5000 108 | thread_pool.bulk.queue_size: 6000 109 | 110 | xpack.security.enabled: false 111 | 112 | gateway.recover_after_time: 5m 113 | 114 | cluster.routing.allocation.same_shard.host: true 115 | 116 | indices.recovery.max_bytes_per_sec: 200mb 117 | indices.memory.index_buffer_size: 20% 118 | 119 | node.attr.role: hot 120 | 121 | #gateway.recover_after_data_nodes: 7 122 | #gateway.recover_after_master_nodes: 2 123 | ''' > /etc/elasticsearch/elasticsearch.yml 124 | 125 | 126 | #master node: 127 | #node.master: true 128 | #node.data: false 129 | #node.ingest: false 130 | #node.ml: false 131 | #xpack.ml.enabled: true 132 | 133 | #data node: 134 | #node.master: false 135 | #node.data: true 136 | #node.ingest: false 137 | #node.ml: false 138 | 139 | #ingest node: 140 | #node.master: false 141 | #node.data: false 142 | #node.ingest: true 143 | #search.remote.connect: false 144 | #node.ml: false 145 | 146 | #coordinating node 147 | #node.master: false 148 | #node.data: false 149 | #node.ingest: false 150 | #search.remote.connect: false 151 | #node.ml: false 152 | 153 | #machine learning node 154 | #node.master: false 155 | #node.data: false 156 | #node.ingest: false 157 | #search.remote.connect: false 158 | #node.ml: true 159 | #xpack.ml.enabled: true 160 | 161 | sudo swapoff -a 162 | 163 | systemctl enable elasticsearch 164 | systemctl start elasticsearch 165 | 166 | /usr/share/elasticsearch/bin/x-pack/setup-passwords auto -u 'http://'$SERVER_IP':9200' 167 | 168 | curl -XPUT -u elastic:test -H 'Content-Type: application/json' 'http://'$SERVER_IP':9200/_xpack/license?acknowledge=true' -d @license.json 169 | 170 | 171 | curl -XPOST 'http://'$SERVER_IP':9200/_xpack/security/user/remote_monitor?pretty' -H 'Content-Type: application/json' -d' 172 | { 173 | "password" : "test", 174 | "roles" : [ "remote_monitoring_agent"], 175 | "full_name" : "Internal Agent For Remote Monitoring" 176 | } 177 | ' 178 | 179 | 180 | curl -XPUT 'http://'$SERVER_IP':9200/_template/index_template' -H 'Content-Type: application/json' -d '{ 181 | "index_patterns" : ["*"], 182 | "order": 0, 183 | "template": "*", 184 | "settings" : { 185 | "index.number_of_replicas" : 1, 186 | "index.number_of_shards" : 64, 187 | "index.routing_partition_size" : 4, 188 | "index.merge.policy": { 189 | "segments_per_tier": 50, 190 | "max_merge_at_once": 50, 191 | "max_merged_segment": "1gb" 192 | }, 193 | "persistent" : { 194 | "cluster.routing.allocation.disk.watermark.high" : "98%", 195 | "indices.recovery.max_bytes_per_sec": "200mb", 196 | "indices.store.throttle.max_bytes_per_sec" : "100mb" 197 | }, 198 | "index.translog.flush_threshold_size":"1gb", 199 | "index.translog.sync_interval":"30s", 200 | "index.translog.durability": "async", 201 | "index.refresh_interval": "30s", 202 | "index.merge.scheduler.max_thread_count": 1, 203 | "indices.memory.index_buffer_size":"20%", 204 | "index.queries.cache.everything": true, 205 | "index.indexing.slowlog.level" : "info", 206 | "index.indexing.slowlog.source" : "1000", 207 | "index.indexing.slowlog.threshold.index.debug" : "2s", 208 | "index.indexing.slowlog.threshold.index.info" : "5s", 209 | "index.indexing.slowlog.threshold.index.trace" : "500ms", 210 | "index.indexing.slowlog.threshold.index.warn" : "10s", 211 | "index.search.slowlog.level" : "info", 212 | "index.search.slowlog.threshold.fetch.debug" : "500ms", 213 | "index.search.slowlog.threshold.fetch.info" : "800ms", 214 | "index.search.slowlog.threshold.fetch.trace" : "200ms", 215 | "index.search.slowlog.threshold.fetch.warn" : "1s", 216 | "index.search.slowlog.threshold.query.debug" : "2s", 217 | "index.search.slowlog.threshold.query.info" : "5s", 218 | "index.search.slowlog.threshold.query.trace" : "500ms", 219 | "index.search.slowlog.threshold.query.warn" : "10s" 220 | }, 221 | "mappings": { 222 | "_default_": { 223 | "dynamic_templates": [ 224 | { 225 | "strings_as_keywords": { 226 | "mapping": { 227 | "index": "not_analyzed" 228 | }, 229 | "match_mapping_type": "string" 230 | } 231 | } 232 | ] 233 | } 234 | }, 235 | "aliases": {} 236 | }' 237 | -------------------------------------------------------------------------------- /install_filebeat.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | version=6.2.3 4 | 5 | rpm -ivh https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-$version-x86_64.rpm 6 | rpm -ivh https://artifacts.elastic.co/downloads/beats/metricbeat/metricbeat-$version-x86_64.rpm 7 | 8 | #filebeat模块的配置在/etc/filebeat/modules.d/ 9 | #https://www.elastic.co/guide/en/beats/filebeat/current/exported-fields.html 10 | #https://www.elastic.co/guide/en/beats/filebeat/current/defining-processors.html 11 | #参考 /etc/filebeat/filebeat.reference.yml 12 | cp filebeat/filebeat.yaml /etc/filebeat/filebeat.yml 13 | 14 | #metricbeat模块的配置在 /etc/metricbeat/modules.d 15 | #https://www.elastic.co/guide/en/beats/metricbeat/current/exported-fields.html 16 | #https://www.elastic.co/guide/en/beats/metricbeat/current/configuring-howto-metricbeat.html 17 | #参考 /etc/metricbeat/metricbeat.reference.yml 18 | cp filebeat/metricbeat.yml /etc/metricbeat/metricbeat.yml 19 | 20 | filebeat setup --dashboards 21 | filebeat setup --machine-learning 22 | filebeat setup --template 23 | 24 | metricbeat setup --dashboards 25 | metricbeat setup --machine-learning 26 | metricbeat setup --template 27 | 28 | #匹配规则在 ll /usr/share/filebeat/module/*/*/ingest/* 29 | #可根据特有日志格式修改对应的json 30 | 31 | systemctl enable filebeat 32 | systemctl start filebeat 33 | 34 | systemctl enable metricbeat 35 | systemctl start metricbeat -------------------------------------------------------------------------------- /install_kibana.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | version=6.2.3 4 | 5 | yum install -y fontconfig freetype 6 | 7 | rpm -ivh https://artifacts.elastic.co/downloads/kibana/kibana-$version-x86_64.rpm 8 | /usr/share/kibana/bin/kibana-plugin install https://github.com/elasticfence/kaae/releases/download/snapshot/kaae-latest.tar.gz 9 | /usr/share/kibana/bin/kibana-plugin install https://github.com/sivasamyk/logtrail/releases/download/0.1.6/logtrail-5.x-0.1.6.zip 10 | /usr/share/kibana/bin/kibana-plugin install https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-$version.zip 11 | 12 | 13 | echo -ne ''' 14 | xpack.security.enabled: false 15 | elasticsearch.username: kibana 16 | elasticsearch.password: test 17 | '''>/etc/kibana/kibana.yaml 18 | 19 | systemctl enable kibana 20 | systemctl start kibana -------------------------------------------------------------------------------- /install_logstash.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | version=6.2.3 4 | 5 | rpm -ivh https://artifacts.elastic.co/downloads/logstash/logstash-$version.rpm 6 | /usr/share/logstash/bin/logstash-plugin install logstash-output-opentsdb 7 | /usr/share/logstash/bin/logstash-plugin install --no-verify https://raw.githubusercontent.com/xiaomatech/logstash-filter-ipip/master/logstash-filter-ipip-2.0.0.gem 8 | /usr/share/logstash/bin/logstash-plugin install --no-verify https://raw.githubusercontent.com/xiaomatech/logstash-filter-referer/master/referer-parser-0.3.0.gem 9 | /usr/share/logstash/bin/logstash-plugin install --no-verify https://raw.githubusercontent.com/xiaomatech/logstash-filter-referer/master/logstash-filter-referer-1.0.0.gem 10 | /usr/share/logstash/bin/logstash-plugin install --no-verify https://raw.githubusercontent.com/xiaomatech/logstash-filter-redis/master/logstash-filter-redis-1.0.0.gem 11 | 12 | /usr/share/logstash/bin/logstash-plugin install https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-$version.zip 13 | 14 | sudo yum install -y GeoIP-data 15 | mkdir -p /data/logs/logstash 16 | 17 | mkdir -p /etc/logstash/patterns 18 | /bin/cp -rf /usr/share/logstash/vendor/bundle/jruby/*/gems/logstash-patterns-core-*/patterns/* /etc/logstash/patterns/ 19 | 20 | echo -ne "LOGSTASH_HOME=/usr/share/logstash\nJRUBY_HOME=\$LOGSTASH_HOME/vendor/jruby">/etc/profile.d/logstash 21 | source /etc/profile.d/logstash 22 | 23 | /usr/share/logstash/vendor/jruby/bin/jruby -S /usr/share/logstash/vendor/jruby/bin/gem sources --add https://gems.ruby-china.org/ --remove https://rubygems.org/ 24 | 25 | nproc=$[`nproc`*2 -1] 26 | cp -r ./logstash/* /etc/logstash/ 27 | 28 | echo -ne ''' 29 | JAVACMD=/usr/bin/java 30 | LS_HOME=/usr/share/logstash 31 | LS_SETTINGS_DIR=/etc/logstash 32 | LS_OPTS="--path.settings ${LS_SETTINGS_DIR}" 33 | LS_JAVA_OPTS="" 34 | LS_USER=logstash 35 | LS_GROUP=logstash 36 | LS_PIDFILE=/var/run/logstash.pid 37 | LS_GC_LOG_FILE=/var/log/logstash/gc.log 38 | LS_OPEN_FILES=65536 39 | LS_NICE=19 40 | SERVICE_NAME="logstash" 41 | SERVICE_DESCRIPTION="logstash" 42 | LS_HEAP_SIZE=8g 43 | ''' > /etc/logstash/startup.options 44 | 45 | echo -ne ''' 46 | path.data: /var/lib/logstash 47 | dead_letter_queue.enable: true 48 | path.dead_letter_queue: /var/lib/logstash/dead_letter_queue 49 | path.config: /etc/logstash/conf.d/*.conf 50 | path.logs: /var/log/logstash 51 | pipeline.workers: '''$nproc''' 52 | pipeline.output.workers: '''$nproc''' 53 | pipeline.batch.size: 2000 54 | pipeline.batch.delay: 3 55 | queue.page_capacity: 256mb 56 | queue.max_bytes: 8gb 57 | queue.drain: true 58 | queue.type: memory 59 | slowlog.threshold.warn: 2s 60 | slowlog.threshold.info: 1s 61 | slowlog.threshold.debug: 500ms, 62 | slowlog.threshold.trace: 100ms 63 | config.reload.automatic: true 64 | 65 | xpack.security.enabled: false 66 | xpack.monitoring.elasticsearch.url: http://elasticsearch:9200 67 | xpack.management.enabled: true 68 | xpack.management.elasticsearch.url: http://elasticsearch:9200 69 | xpack.management.logstash.poll_interval: 5s 70 | xpack.management.pipeline.id: ["nginx_log", "java_log", "mysql_log"] 71 | 72 | '''> /etc/logstash/logstash.yml 73 | 74 | systemctl enable logstash 75 | systemctl start logstash 76 | -------------------------------------------------------------------------------- /install_rsyslog.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | yum install -y rsyslog-kafka rsyslog-mmjsonparse rsyslog-mmnormalize rsyslog-mmfields rsyslog-mmanon 4 | 5 | mkdir -p /data/rsyslog 6 | /bin/cp ./rsyslog.conf /etc/ 7 | /bin/cp -rf ./rsyslog.d /etc/ 8 | 9 | systemctl start rsyslog 10 | -------------------------------------------------------------------------------- /license.json: -------------------------------------------------------------------------------- 1 | { 2 | "license": { 3 | "uid": "b9e226d1-ae19-4d05-a238-f7a9e9177c27", 4 | "type": "platinum", 5 | "issue_date_in_millis": 1521676800000, 6 | "expiry_date_in_millis": 4102416000000, 7 | "max_nodes": 100000, 8 | "issued_to": "xiaomatech", 9 | "issuer": "Web Form", 10 | "signature": "AAAAAwAAAA3nnDuu5y7pFV6VEgwbAAABmC9ZN0hjZDBGYnVyRXpCOW5Bb3FjZDAxOWpSbTVoMVZwUzRxVk1PSmkxaktJRVl5MUYvUWh3bHZVUTllbXNPbzBUemtnbWpBbmlWRmRZb25KNFlBR2x0TXc2K2p1Y1VtMG1UQU9TRGZVSGRwaEJGUjE3bXd3LzRqZ05iLzRteWFNekdxRGpIYlFwYkJiNUs0U1hTVlJKNVlXekMrSlVUdFIvV0FNeWdOYnlESDc3MWhlY3hSQmdKSjJ2ZTcvYlBFOHhPQlV3ZHdDQ0tHcG5uOElCaDJ4K1hob29xSG85N0kvTWV3THhlQk9NL01VMFRjNDZpZEVXeUtUMXIyMlIveFpJUkk2WUdveEZaME9XWitGUi9WNTZVQW1FMG1DenhZU0ZmeXlZakVEMjZFT2NvOWxpZGlqVmlHNC8rWVVUYzMwRGVySHpIdURzKzFiRDl4TmM1TUp2VTBOUlJZUlAyV0ZVL2kvVk10L0NsbXNFYVZwT3NSU082dFNNa2prQ0ZsclZ4NTltbU1CVE5lR09Bck93V2J1Y3c9PQAAAQCeNSwESKwTxifEoHKocUVeJGLgeV+w9R1ISSVMprGzUFCNmgnMEbWh9mKJjhdIrV5CnbHlIU0exYwfMDdg2gyO+IKqDEycgAFnRIyPwbSjqjAhYN/V1IdELwdiUlLKjIy+FQ5c8qmOHbKHltiIb8VpggnCjhm74+JVvLGvHcaaM5Chg10tLjd51DUt/QXfGYRhnTfBZWw7m6ZlE0VreXrAb1al3H4zr5WVK5B82rrqqEj+zhzZqdF1F4cmMqt1Z5/hbzQGSOUc/g5r8KfVxJIeZamE8fqrN3IypZtjOWlkYfd6DHBgdm0EJPlela9jR1XfP9hmk1a9NlZ4teuc27gb", 11 | "start_date_in_millis": 1521676800000 12 | } 13 | } -------------------------------------------------------------------------------- /logrotate.d/mysql: -------------------------------------------------------------------------------- 1 | /data/logs/mysql/mysqld.log { 2 | # create 600 mysql mysql 3 | notifempty 4 | daily 5 | rotate 5 6 | missingok 7 | compress 8 | postrotate 9 | # just if mysqld is really running 10 | if test -x /usr/bin/mysqladmin && \ 11 | /usr/bin/mysqladmin ping &>/dev/null 12 | then 13 | /usr/bin/mysqladmin flush-logs 14 | fi 15 | endscript 16 | } -------------------------------------------------------------------------------- /logrotate.d/nginx: -------------------------------------------------------------------------------- 1 | /data/logs/nginx/*.log { 2 | daily 3 | missingok 4 | rotate 52 5 | compress 6 | delaycompress 7 | notifempty 8 | sharedscripts 9 | postrotate 10 | [ -f /var/run/nginx.pid ] && kill -USR1 `cat /var/run/nginx.pid` 11 | endscript 12 | } -------------------------------------------------------------------------------- /logrotate.d/rabbitmq: -------------------------------------------------------------------------------- 1 | /data/logs/rabbitmq/*.log { 2 | weekly 3 | missingok 4 | rotate 20 5 | compress 6 | delaycompress 7 | notifempty 8 | sharedscripts 9 | postrotate 10 | /usr/sbin/rabbitmqctl -q rotate_logs 11 | endscript 12 | } 13 | -------------------------------------------------------------------------------- /logrotate.d/redis: -------------------------------------------------------------------------------- 1 | /data/logs/redis/*.log { 2 | weekly 3 | rotate 10 4 | copytruncate 5 | delaycompress 6 | compress 7 | notifempty 8 | missingok 9 | } 10 | -------------------------------------------------------------------------------- /logstash/conf.d/filter-ceph.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | if [log_type] == "ceph" { 3 | grok { 4 | match => [ "message", "%{TIMESTAMP_ISO8601} mon%{BASE10NUM:mon_number} %{IPORHOST:host}:%{POSINT:port}/0 %{BASE10NUM}%{GREEDYDATA}client%{BASE10NUM:clientnum}%{GREEDYDATA}: %{QUOTEDSTRING:command}%{GREEDYDATA}root=%{WORD:device}%{GREEDYDATA}rack=%{HOSTNAME:rack}%{GREEDYDATA}host=%{HOSTNAME}%{GREEDYDATA}%{NUMBER:id},%{GREEDYDATA}%{NUMBER:weight}"] 5 | overwrite => [ "message" ] 6 | } 7 | } 8 | } -------------------------------------------------------------------------------- /logstash/conf.d/filter-firewall.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | if [log_type] == "fortinet" { 3 | mutate { 4 | gsub => [ "message", "= ", "=NA " ] 5 | } 6 | 7 | grok { 8 | match => ["message", "type=%{DATA:event_type}\s+"] 9 | tag_on_failure => [] 10 | } 11 | grok { 12 | match => ["message", "<%{DATA}>%{GREEDYDATA:kv}"] 13 | tag_on_failure => [] 14 | } 15 | kv { 16 | source => "kv" 17 | exclude_keys => [ "type" ] 18 | } 19 | mutate { 20 | gsub => [ "log", "= ", "=NA " ] 21 | } 22 | kv { 23 | source => "log" 24 | target => "SubLog" 25 | } 26 | grok { 27 | match => ["message", "custom: DOM-ALL, dns_query=%{DATA:dns_query};"] 28 | tag_on_failure => [ "" ] 29 | } 30 | mutate { 31 | rename => { "action" => "action" } 32 | rename => { "addr" => "addr_ip" } 33 | rename => { "age" => "age" } 34 | rename => { "assigned" => "assigned_ip" } 35 | rename => { "assignip" => "assign_ip" } 36 | rename => { "ap" => "access_point" } 37 | rename => { "app" => "application" } 38 | rename => { "appcat" => "application_category" } 39 | rename => { "applist" => "application_list" } 40 | rename => { "apprisk" => "application_risk" } 41 | rename => { "approfile" => "accessPoint_profile" } 42 | rename => { "apscan" => "access_point_scan" } 43 | rename => { "apstatus" => "acces_point_status" } 44 | rename => { "aptype" => "access_point_type" } 45 | rename => { "authproto" => "authentication_protocol" } 46 | rename => { "bandwidth" => "bandwidth" } 47 | rename => { "banned_src" => "banned_source" } 48 | rename => { "cat" => "category" } 49 | rename => { "catdesc" => "category_description" } 50 | rename => { "cfgattr" => "configuration_attribute" } 51 | rename => { "cfgobj" => "configuration_object" } 52 | rename => { "cfgpath" => "configuration_path" } 53 | rename => { "cfgtid" => "configuration_transaction_id" } 54 | rename => { "channel" => "channel" } 55 | rename => { "community" => "community" } 56 | rename => { "cookies" => "cookies" } 57 | rename => { "craction" => "cr_action" } 58 | rename => { "crlevel" => "cr_level" } 59 | rename => { "crscore" => "cr_score" } 60 | rename => { "datarange" => "data_range" } 61 | rename => { "desc" => "description" } 62 | rename => { "detectionmethod" => "detection_method" } 63 | rename => { "devid" => "device_id" } 64 | rename => { "devname" => "device_name" } 65 | rename => { "devtype" => "device_type" } 66 | rename => { "dhcp_msg" => "dhcp_message" } 67 | rename => { "disklograte" => "disk_lograte" } 68 | rename => { "dstcountry" => "destination_country" } 69 | rename => { "dstintf" => "destination_interface" } 70 | rename => { "dstip" => "destination_ip" } 71 | rename => { "dstport" => "destination_port" } 72 | rename => { "duration" => "elapsed_time" } 73 | rename => { "error_num" => "error_number" } 74 | rename => { "espauth" => "esp_authentication" } 75 | rename => { "esptransform" => "esp_transform" } 76 | rename => { "eventid" => "event_id" } 77 | rename => { "eventtype" => "event_type" } 78 | rename => { "fazlograte" => "faz_lograte" } 79 | rename => { "filename" => "file_name" } 80 | rename => { "filesize" => "file_size" } 81 | rename => { "filetype" => "file_type" } 82 | rename => { "hostname" => "hostname" } 83 | rename => { "ip" => "source_ip" } 84 | rename => { "localip" => "source_ip" } 85 | rename => { "locip" => "local_ip" } 86 | rename => { "locport" => "source_port" } 87 | rename => { "logid" => "log_id" } 88 | rename => { "logver" => "log_version" } 89 | rename => { "manuf" => "manufacturer" } 90 | rename => { "mem" => "memory" } 91 | rename => { "meshmode" => "mesh_mode" } 92 | rename => { "msg" => "message" } 93 | rename => { "nextstat" => "next_stat" } 94 | rename => { "onwire" => "on_wire" } 95 | rename => { "osname" => "os_name" } 96 | rename => { "osversion" => "unauthenticated_user" } 97 | rename => { "outintf" => "outbound_interface" } 98 | rename => { "peer_notif" => "peer_notification" } 99 | rename => { "phase2_name" => "phase2_name" } 100 | rename => { "policyid" => "policy_id" } 101 | rename => { "policytype" => "policy_type" } 102 | rename => { "port" => "port" } 103 | rename => { "probeproto" => "probe_protocol" } 104 | rename => { "proto" => "protocol_number" } 105 | rename => { "radioband" => "radio_band" } 106 | rename => { "radioidclosest" => "radio_id_closest" } 107 | rename => { "radioiddetected" => "radio_id_detected" } 108 | rename => { "rcvd" => "bytes_received" } 109 | rename => { "rcvdbyte" => "bytes_received" } 110 | rename => { "rcvdpkt" => "packets_received" } 111 | rename => { "remip" => "destination_ip" } 112 | rename => { "remport" => "remote_port" } 113 | rename => { "reqtype" => "request_type" } 114 | rename => { "scantime" => "scan_time" } 115 | rename => { "securitymode" => "security_mode" } 116 | rename => { "sent" => "bytes_sent" } 117 | rename => { "sentbyte" => "bytes_sent" } 118 | rename => { "sentpkt" => "packets_sent" } 119 | rename => { "session_id" => "session_id" } 120 | rename => { "setuprate" => "setup_rate" } 121 | rename => { "sn" => "serial" } 122 | rename => { "snclosest" => "serial_closest_access_point" } 123 | rename => { "sndetected" => "serial_access_point_that_detected_rogue_ap" } 124 | rename => { "snmeshparent" => "serial_mesh_parent" } 125 | rename => { "srccountry" => "source_country" } 126 | rename => { "srcip" => "source_ip" } 127 | rename => { "srcmac" => "source_mac" } 128 | rename => { "srcname" => "source_name" } 129 | rename => { "srcintf" => "source_interface" } 130 | rename => { "srcport" => "source_port" } 131 | rename => { "stacount" => "station_count" } 132 | rename => { "stamac" => "static_mac" } 133 | rename => { "srccountry" => "source_country" } 134 | rename => { "srcip" => "source_ip" } 135 | rename => { "srcmac" => "source_mac" } 136 | rename => { "srcname" => "source_name" } 137 | rename => { "sn" => "serial" } 138 | rename => { "srcintf" => "source_interface" } 139 | rename => { "srcport" => "source_port" } 140 | rename => { "total" => "total_bytes" } 141 | rename => { "totalsession" => "total_sessions" } 142 | rename => { "trandisp" => "nat_translation_type" } 143 | rename => { "tranip" => "nat_destination_ip" } 144 | rename => { "tranport" => "nat_destination_port" } 145 | rename => { "transip" => "nat_source_ip" } 146 | rename => { "transport" => "nat_source_port" } 147 | rename => { "tunnelid" => "tunnel_id" } 148 | rename => { "tunnelip" => "tunnel_ip" } 149 | rename => { "tunneltype" => "tunnel_type" } 150 | rename => { "unauthuser" => "unauthenticated_user_source" } 151 | rename => { "unauthusersource" => "os_version" } 152 | rename => { "vendorurl" => "vendor_url" } 153 | rename => { "vpntunnel" => "vpn_tunnel" } 154 | rename => { "vulncat" => "vulnerability_category" } 155 | rename => { "vulncmt" => "vulnerability_count" } 156 | rename => { "vulnid" => "vulnerability_id" } 157 | rename => { "vulnname" => "vulnerability_name" } 158 | rename => { "vulnref" => "vulnerability_reference" } 159 | rename => { "vulnscore" => "vulnerability_score" } 160 | rename => { "xauthgroup" => "x_authentication_group" } 161 | rename => { "xauthuser" => "x_authentication_user" } 162 | rename => { "[SubLog][appid]" => "sub_application_id" } 163 | rename => { "[SubLog][devid]" => "sub_device_id" } 164 | rename => { "[SubLog][dstip]" => "sub_destination_ip" } 165 | rename => { "[SubLog][srcip]" => "sub_source_ip" } 166 | rename => { "[SubLog][dstport]" => "sub_destination_port" } 167 | rename => { "[SubLog][eventtype]" => "sub_event_type" } 168 | rename => { "[SubLog][proto]" => "sub_protocol_number" } 169 | rename => { "[SubLog][date]" => "sub_date" } 170 | rename => { "[SubLog][time]" => "sub_time" } 171 | rename => { "[SubLog][srcport]" => "sub_source_port" } 172 | rename => { "[SubLog][subtype]" => "sub_subtype" } 173 | rename => { "[SubLog][devname]" => "sub_device_name" } 174 | rename => { "[SubLog][itime]" => "sub_itime" } 175 | rename => { "[SubLog][level]" => "sub_level" } 176 | rename => { "[SubLog][logid]" => "sub_log_id" } 177 | rename => { "[SubLog][logver]" => "sub_log_version" } 178 | rename => { "[SubLog][type]" => "sub_event_type" } 179 | rename => { "[SubLog][vd]" => "sub_vd" } 180 | rename => { "[SubLog][action]" => "sub_action" } 181 | rename => { "[SubLog][logdesc]" => "sub_destination_ip" } 182 | rename => { "[SubLog][policyid]" => "sub_olicy_id" } 183 | rename => { "[SubLog][reason]" => "sub_reason" } 184 | rename => { "[SubLog][service]" => "sub_service" } 185 | rename => { "[SubLog][sessionid]" => "sub_session_id" } 186 | rename => { "[SubLog][src]" => "sub_source_ip" } 187 | rename => { "[SubLog][status]" => "sub_status" } 188 | rename => { "[SubLog][ui]" => "sub_ui" } 189 | rename => { "[SubLog][urlfilteridx]" => "sub_url_filter_idx" } 190 | strip => [ "bytes_sent", "bytes_received" ] 191 | convert => [ "bytes_sent", "integer" ] 192 | convert => [ "bytes_received", "integer" ] 193 | convert => [ "cr_score", "integer" ] 194 | convert => [ "cr_action", "integer" ] 195 | convert => [ "elapsed_time", "integer" ] 196 | convert => [ "destination_port", "integer" ] 197 | convert => [ "source_port", "integer" ] 198 | convert => [ "local_port", "integer" ] 199 | convert => [ "remote_port", "integer" ] 200 | convert => [ "packets_sent", "integer" ] 201 | convert => [ "packets_received", "integer" ] 202 | convert => [ "port", "integer" ] 203 | convert => [ "ProtocolNumber", "integer" ] 204 | convert => [ "XAuthUser", "string" ] 205 | remove_field => [ "kv", "log" ] 206 | } 207 | if [tunnel_ip] == "N/A" { 208 | mutate { 209 | remove_field => [ "tunnel_ip" ] 210 | } 211 | } 212 | if [nat_destination_ip] { 213 | mutate { 214 | add_field => { "ips" => [ "%{nat_destination_ip}" ] } 215 | add_field => { "destination_ips" => [ "%{nat_destination_ip}" ] } 216 | } 217 | } 218 | if [sub_destination_ip] { 219 | mutate { 220 | add_field => { "ips" => [ "%{sub_destination_ip}" ] } 221 | add_field => { "destination_ips" => [ "%{sub_destination_ip}" ] } 222 | } 223 | } 224 | if [nat_source_ip] { 225 | mutate { 226 | add_field => { "ips" => [ "%{nat_source_ip}" ] } 227 | add_field => { "source_ips" => [ "%{nat_source_ip}" ] } 228 | } 229 | } 230 | if [sub_source_ip] { 231 | mutate { 232 | add_field => { "ips" => [ "%{sub_source_ip}" ] } 233 | add_field => { "source_ips" => [ "%{sub_source_ip}" ] } 234 | } 235 | } 236 | if [addr_ip] { 237 | mutate { 238 | add_field => { "ips" => [ "%{addr_ip}" ] } 239 | } 240 | } 241 | if [assign_ip] { 242 | mutate { 243 | add_field => { "ips" => [ "%{assign_ip}" ] } 244 | } 245 | } 246 | if [assigned_ip] { 247 | mutate { 248 | add_field => { "ips" => [ "%{assigned_ip}" ] } 249 | } 250 | } 251 | grok { 252 | match => ["message", "type=%{DATA:event_type}\s+"] 253 | } 254 | if [date] and [time] { 255 | mutate { 256 | add_field => { "receive_time" => "%{date} %{time}" } 257 | remove_field => [ "date", "time" ] 258 | } 259 | date { 260 | timezone => "America/Chicago" 261 | match => [ "receive_time", "YYYY-MM-dd HH:mm:ss" ] 262 | target => "receive_time" 263 | } 264 | mutate { 265 | rename => { "receive_time" => "@timestamp" } 266 | } 267 | } else { 268 | mutate { 269 | add_tag => [ "missing_date" ] 270 | } 271 | } 272 | mutate { 273 | #add_tag => [ "conf_file_6200"] 274 | } 275 | } 276 | } -------------------------------------------------------------------------------- /logstash/conf.d/filter-haproxy.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | if [log_type] == "haproxy" { 3 | grok { 4 | match => ["message", "%{HAPROXYHTTP}"] 5 | named_captures_only => true 6 | } 7 | geoip { 8 | cache_size => 8192000 9 | source => "client_ip" 10 | add_tag => ["geoip"] 11 | database => "/usr/share/GeoIP/GeoIPCity.dat" 12 | fields => ['city_name','continent_code','country_code2','country_code3','country_name','dma_code','ip','latitude','longitude','postal_code','region_name'] 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /logstash/conf.d/filter-ip2location.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | if "_jsonparsefailure" in [tags] { 3 | mutate { 4 | gsub => ["message", "[\\\\]", ""] 5 | } 6 | mutate { remove_tag => [ "tags", "_jsonparsefailure" ] } 7 | json { source => "message" } 8 | } 9 | 10 | if [log_type] == "nginx_access" { 11 | geoip { 12 | cache_size => 8192000 13 | source => 'remote_addr' 14 | add_tag => ["geoip"] 15 | database => "/usr/share/GeoIP/GeoIPCity.dat" 16 | fields => ['city_name','continent_code','country_code2','country_code3','country_name','dma_code','ip','latitude','longitude','postal_code','region_name'] 17 | } 18 | 19 | ipip { 20 | source => "remote_addr" 21 | add_tag => ["ipip"] 22 | fields => ['country', 'province', 'city', 'district', 'isp', 'lat', 'lag', 'timezone_name', 'timezone', 'zip', 'phonecode', 'countrycode', 'region'] 23 | } 24 | } 25 | } -------------------------------------------------------------------------------- /logstash/conf.d/filter-java.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | 3 | if [log_type] == "log4j" { 4 | grok { 5 | match => ["message" => "%{20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}:timestamp}%{SPACE}%{LOGLEVEL:level}%{SPACE}\[%{THREADNAME:thread}\]%{SPACE}\[%{JAVACLASS:class}\]%{SPACE}%{GREEDYDATA:logmessage}"] 6 | } 7 | date { 8 | match => [ "timestamp", "yyyy-MM-dd HH:mm:ss,SSS" ] 9 | } 10 | 11 | if [type] == "java" and "multiline" in [tags] { 12 | grok { 13 | match => ["message", "%{JAVASTACKTRACEPART}"] 14 | add_tag => "java_exception" 15 | } 16 | } 17 | 18 | } 19 | } -------------------------------------------------------------------------------- /logstash/conf.d/filter-jetty.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | 3 | if [log_type] == "jetty_access" { 4 | grok { 5 | match => ["message" => "%{IP:clientip} \- \- \[%{MONTHDAY}\/%{MONTH}\/%{YEAR}\:%{TIME} %{ISO8601_TIMEZONE}] "%{WORD:request} \/ HTTP\/%{NUMBER:http_version}" %{NUMBER:response_code} %{NUMBER:response_time}"] 6 | add_tag => "jetty_access" 7 | } 8 | 9 | geoip { 10 | source => "clientip" 11 | add_tag => ["geoip"] 12 | database => "/usr/share/GeoIP/GeoIPCity.dat" 13 | fields => ["country_name", "country_code2", "region_name", "city_name", "real_region_name", "latitude", "longitude"] 14 | remove_field => ["[geoip][longitude]", "[geoip][latitude]"] 15 | } 16 | } 17 | 18 | } -------------------------------------------------------------------------------- /logstash/conf.d/filter-metrics.conf: -------------------------------------------------------------------------------- 1 | #如果最近一分钟 504 请求的个数超过 100 个就报警 2 | filter { 3 | metrics { 4 | meter => "error_%{status}" 5 | add_tag => "metric" 6 | ignore_older_than => 10 7 | } 8 | if "metric" in [tags] { 9 | ruby { 10 | code => "event.cancel if (event.get('[error_504][rate_1m]') * 60 > 100)" 11 | } 12 | } 13 | } 14 | 15 | output { 16 | if "metric" in [tags] { 17 | exec { 18 | command => "echo \"Out of threshold: %{[error_504][rate_1m]}\"" 19 | } 20 | } 21 | } 22 | 23 | 24 | #box and shisker plot 25 | filter { 26 | metrics { 27 | timer => {"rt" => "%{request_time}"} 28 | percentiles => [25, 75] 29 | add_tag => "percentile" 30 | } 31 | if "percentile" in [tags] { 32 | ruby { 33 | code => "l=event.get('[rt][p75]')-event.get('[rt][p25]');event.set('[rt][low]', event.get('[rt][p25]')-l);event.set('[rt][high]',event.get('[rt][p75]')+l)" 34 | } 35 | } 36 | } 37 | output { 38 | if "percentile" in [tags] and ([rt][last] > [rt][high] or [rt][last] < [rt][low]) { 39 | exec { 40 | command => "echo \"Anomaly: %{[rt][last]}\"" 41 | } 42 | } 43 | } -------------------------------------------------------------------------------- /logstash/conf.d/filter-mongodb.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | if [log_type] == "mongodb" { 3 | grok { 4 | match => ["message","%{MONGO3_LOG}"] 5 | } 6 | 7 | if [message] =~ "nreturned" { 8 | grok { 9 | match => [ "message", "%{MONGO_SLOWQUERY}"] 10 | add_tag => [ "query" ] 11 | } 12 | } 13 | 14 | date { 15 | match => [ "timestamp", "MMM d HH:mm:ss", "MMM d HH:mm:ss.SSS" ,"MMM dd HH:mm:ss.SSS", "MMM DD HH:mm:ss" ] 16 | remove_field => [ "timestamp" ] 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /logstash/conf.d/filter-multiline.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | if [log_type] == "php_slow" { 3 | codec => multiline { 4 | pattern => '^$' 5 | what => 'next' 6 | negate => true 7 | } 8 | } 9 | 10 | if [log_type] == "log4j" { 11 | codec => multiline { 12 | pattern => "(^.+Exception: .+)|(^\s+at .+)|(^\s+... \d+ more)|(^\s*Caused by:.+)|(^\s+...\s[\d]+[\s\w]+$)" 13 | what => "previous" 14 | } 15 | } 16 | 17 | if [log_type] == "mysql_slow" { 18 | codec => multiline { 19 | pattern => "^# User@Host:" 20 | negate => true 21 | what => "previous" 22 | } 23 | } 24 | 25 | if [log_type] == "other" { 26 | #以时间戳开始 27 | codec => multiline { 28 | pattern => "^%{TIMESTAMP_ISO8601} " 29 | negate => true 30 | what => previous 31 | } 32 | 33 | #\字符在一行末尾表示该行继续 34 | codec => multiline { 35 | pattern => "\\$" 36 | what => "next" 37 | } 38 | 39 | #以空白开始的行与上一行合并 40 | codec => multiline { 41 | pattern => "^\s" 42 | what => "previous" 43 | } 44 | } 45 | } -------------------------------------------------------------------------------- /logstash/conf.d/filter-mysql-slow.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | if [log_type] == "mysql_slow" { 3 | # drop sleep events 4 | grok { 5 | match => { "message" => "SELECT SLEEP" } 6 | add_tag => [ "sleep_drop" ] 7 | tag_on_failure => [] # prevent default _grokparsefailure tag on real records 8 | } 9 | if "sleep_drop" in [tags] { 10 | drop {} 11 | } 12 | 13 | grok { 14 | match => { "message" => "(?m)^#\s+User@Host:\s+%{USER:user}\[[^\]]+\]\s+@\s+(?:(?\S*) )?\[(?:%{IPV4:clientip})?\]\s+Id:\s+%{NUMBER:row_id:int}\n#\s+Query_time:\s+%{NUMBER:query_time:float}\s+Lock_time:\s+%{NUMBER:lock_time:float}\s+Rows_sent:\s+%{NUMBER:rows_sent:int}\s+Rows_examined:\s+%{NUMBER:rows_examined:int}\n\s*(?:use %{DATA:database};\s*\n)?SET\s+timestamp=%{NUMBER:timestamp};\n\s*(?(?\w+)\b.*;)\s*(?:\n#\s+Time)?.*$" } 15 | } 16 | date { 17 | match => [ "timestamp", "UNIX", "YYYY-MM-dd HH:mm:ss"] 18 | remove_field => [ "timestamp" ] 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /logstash/conf.d/filter-nginx-access.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | if [log_type] == "nginx_access" { 3 | 4 | json { 5 | source => "message" 6 | } 7 | 8 | mutate { 9 | convert => ["size", "integer", "responsetime", "float", "upstream_response_time", "float"] 10 | } 11 | 12 | if [http_cookie] { 13 | kv { 14 | prefix => "cookie_" 15 | source => "http_cookie" 16 | field_split => "; " 17 | } 18 | } 19 | if [request] { 20 | ruby { 21 | init => "@kname = ['method','uri','verb']" 22 | code => " 23 | new_event = LogStash::Event.new(Hash[@kname.zip(event.get('request').split(' '))]) 24 | new_event.remove('@timestamp') 25 | event.append(new_event)"" 26 | " 27 | } 28 | if [uri] { 29 | ruby { 30 | init => "@kname = ['url_path','url_args']" 31 | code => " 32 | new_event = LogStash::Event.new(Hash[@kname.zip(event.get('uri').split('?'))]) 33 | new_event.remove('@timestamp') 34 | event.append(new_event)"" 35 | " 36 | } 37 | kv { 38 | prefix => "url_" 39 | source => "url_args" 40 | field_split => "& " 41 | remove_field => [ "url_args","uri","request" ] 42 | } 43 | } 44 | } 45 | mutate { 46 | convert => [ 47 | "status" : "integer", 48 | "size" , "integer", 49 | "request_length", "integer", 50 | "upstream_response_time", "float", 51 | "request_time", "float" 52 | ] 53 | } 54 | date { 55 | match => [ "time_local", "dd/MMM/yyyy:hh:mm:ss Z" ] 56 | locale => "en" 57 | } 58 | 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /logstash/conf.d/filter-nginx-error.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | if [log_type] == "nginx_error" { 3 | grok { 4 | match => ["message", "^%{%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME}:timestamp} \[%{LOGLEVEL}\] %{POSINT:pid}#%{NUMBER}: \*%{NUMBER} %{GREEDYDATA:msg}"] 5 | } 6 | 7 | date { 8 | match => ['timestamp', 'YYYY/MMM/dd HH:mm:ss'] 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /logstash/conf.d/filter-percona-audit-log.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | if [log_type] == "mysql_audit_log" { 3 | mutate { 4 | gsub => ["message", "\\\\n", " "] 5 | gsub => ["message", "\t", " "] 6 | replace => ["message", "%{message}"] 7 | } 8 | 9 | json { 10 | source => "message" 11 | } 12 | 13 | mutate { 14 | remove_field => ["message"] 15 | gsub => ["[audit_record][timestamp]", " UTC", "Z"] 16 | replace => ["[audit_record][timestamp]", "%{[audit_record][timestamp]}"] 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /logstash/conf.d/filter-php-slow.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | if [log_type] == "php_slow" { 3 | grok { 4 | patterns_dir => '/etc/logstash/patterns' 5 | match => [ 6 | "message", "\[%{%{MONTHDAY}-%{MONTH}-%{YEAR}\s+%{TIME}:timestamp}\] \[%{GREEDYDATA:php_fpm_pool}\] pid {POSINT}", 7 | "message", "\[%{GREEDYDATA}\] %{GREEDYDATA:func_name} %{UNIXPATH:script_path}" 8 | ] 9 | } 10 | date { match => ['timestamp', 'dd-MMM-YYYY HH:mm:ss'] } 11 | } 12 | } -------------------------------------------------------------------------------- /logstash/conf.d/filter-redis.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | redis { 3 | host => "127.0.0.1" 4 | port => 6379 5 | db => 0 6 | # password => "" 7 | action => "GET" 8 | key => "%{host}" 9 | field => "%{host}" 10 | name => "ext" 11 | default => "" 12 | } 13 | } -------------------------------------------------------------------------------- /logstash/conf.d/filter-referer.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | if [log_type] == "nginx_access" { 3 | if [http_referer] != "-" { 4 | referer { 5 | target => "referer" 6 | source => "http_referer" 7 | referers_file => "/etc/logstash/conf/referers.yaml" 8 | } 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /logstash/conf.d/filter-solr.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | if [log_type] == "solr" and [message] !~ /(.+)/ { 3 | drop { } 4 | } 5 | if [type] == "solr_log" and "multiline" in [tags] { 6 | grok { 7 | match => [ "message", "%{JAVASTACKTRACEPART}" ] 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /logstash/conf.d/filter-tomcat.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | 3 | if [log_type] == "tomcat" and [message] !~ /(.+)/ { 4 | drop { } 5 | } 6 | if [log_type] == "tomcat" and "multiline" in [tags] { 7 | grok { 8 | match => [ "message", "%{JAVASTACKTRACEPART}" ] 9 | } 10 | } 11 | 12 | if [log_type] == "tomcat" { 13 | grok { 14 | match => [ "message", "%{TOMCATLOG}", "message", "%{CATALINALOG}" ] 15 | } 16 | 17 | geoip { 18 | cache_size => 8192000 19 | source => "clientip" 20 | add_tag => ["geoip"] 21 | database => "/usr/share/GeoIP/GeoIPCity.dat" 22 | fields => ['city_name','continent_code','country_code2','country_code3','country_name','dma_code','ip','latitude','longitude','postal_code','region_name'] 23 | remove_field => ["[geoip][longitude]", "[geoip][latitude]"] 24 | } 25 | mutate { 26 | convert => [ "[geoip][coordinates]", "float"] 27 | } 28 | date { 29 | match => [ "timestamp", "yyyy-MM-dd HH:mm:ss,SSS Z", "MMM dd, yyyy HH:mm:ss a" ] 30 | } 31 | } 32 | 33 | } -------------------------------------------------------------------------------- /logstash/conf.d/filter-trafficserver.conf: -------------------------------------------------------------------------------- 1 | #Squid Compatible 2 | filter { 3 | if [log_type] == "trafficserver_log" { 4 | grok { 5 | match => {"message" => "%{NUMBER:timestamp} %{NUMBER:timetoserve} %{IPORHOST:clientip} %{WORD:cachecode}/%{NUMBER:response} %{NUMBER:bytes} %{WORD:verb} %{NOTSPACE:request} %{USER:auth} %{NOTSPACE:route} %{DATA:contenttype}"} 6 | } 7 | date { 8 | match => ["timestamp", "UNIX"] 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /logstash/conf.d/filter-useragent.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | if [log_type] == "nginx_access" { 3 | if [http_user_agent] != "-" { 4 | useragent { 5 | target => "useragent" 6 | source => "http_user_agent" 7 | regexes => "/etc/logstash/conf/useragent.yaml" 8 | } 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /logstash/conf.d/input-beats.conf: -------------------------------------------------------------------------------- 1 | input { 2 | beats { 3 | port => 5044 4 | } 5 | } -------------------------------------------------------------------------------- /logstash/conf.d/input-http_poller.conf: -------------------------------------------------------------------------------- 1 | input { 2 | http_poller { 3 | urls => { 4 | test1 => "http://localhost:9200" 5 | test2 => { 6 | # Supports all options supported by ruby's Manticore HTTP client 7 | method => get 8 | user => "AzureDiamond" 9 | password => "hunter2" 10 | url => "http://localhost:9200/_cluster/health" 11 | headers => { 12 | Accept => "application/json" 13 | } 14 | } 15 | } 16 | request_timeout => 60 17 | # Supports "cron", "every", "at" and "in" schedules by rufus scheduler 18 | schedule => { cron => "* * * * * UTC"} 19 | codec => "json" 20 | # A hash of request metadata info (timing, response headers, etc.) will be sent here 21 | metadata_target => "http_poller_metadata" 22 | } 23 | } -------------------------------------------------------------------------------- /logstash/conf.d/input-kafka.conf: -------------------------------------------------------------------------------- 1 | #https://www.elastic.co/guide/en/logstash/current/plugins-inputs-kafka.html 2 | input { 3 | kafka { 4 | #codec => avro { 5 | # schema_uri => "/tmp/schema.avsc" # "http://example.com/schema.avsc" 6 | #} 7 | 8 | bootstrap_servers => "kafka1:6667,kafka2:6667,kafka3:6667" 9 | group_id => 'logcenter' 10 | topics => ["nginx_access"] 11 | #topics_pattern => "nginx_access*" 12 | #exclude_internal_topics => "" 13 | codec => json 14 | consumer_threads => 16 15 | decorate_events => true 16 | consumer_restart_on_error => true 17 | tags => ["logcenter-kafka","nginx_access"] 18 | decorate_events => true 19 | #jaas_path => "" 20 | #security_protocol => "SASL_PLAINTEXT" 21 | } 22 | } -------------------------------------------------------------------------------- /logstash/conf.d/input-netflow.conf: -------------------------------------------------------------------------------- 1 | input { 2 | udp { 3 | port => 2055 4 | codec => netflow 5 | receive_buffer_bytes => 16777216 6 | workers => 16 7 | } 8 | } -------------------------------------------------------------------------------- /logstash/conf.d/input-retry.conf: -------------------------------------------------------------------------------- 1 | input { 2 | dead_letter_queue { 3 | path => "/var/lib/logstash/dead_letter_queue" 4 | commit_offsets => true 5 | pipeline_id => "main" 6 | } 7 | } -------------------------------------------------------------------------------- /logstash/conf.d/input-syslog.conf: -------------------------------------------------------------------------------- 1 | input { 2 | tcp { 3 | port => "5140" 4 | } 5 | 6 | tcp { 7 | port => "5141" 8 | } 9 | 10 | tcp { 11 | port => "5142" 12 | } 13 | 14 | tcp { 15 | port => "5143" 16 | } 17 | } 18 | 19 | filter { 20 | grok { 21 | match => ["message", "%{SYSLOGLINE}" ] 22 | } 23 | syslog_pri { } 24 | } 25 | -------------------------------------------------------------------------------- /logstash/conf.d/input-varnishlog.conf: -------------------------------------------------------------------------------- 1 | input { 2 | if [type] == "varnish_log" { 3 | varnishlog { 4 | threads => 8 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /logstash/conf.d/output-es.conf: -------------------------------------------------------------------------------- 1 | output { 2 | elasticsearch { 3 | cluster = "elasticsearch-cluster" 4 | hosts => ['es1:9200','es2:9200'] 5 | index => "logcenter-%{+YYYY.MM.dd}" 6 | pool_max => 6000 7 | pool_max_per_route => 600 8 | } 9 | } -------------------------------------------------------------------------------- /logstash/conf.d/output-hdfs.conf: -------------------------------------------------------------------------------- 1 | output { 2 | webhdfs { 3 | host => "hdfs_host" 4 | port => 50070 , 5 | standby_host => "hdfs_standby_host" , 6 | standby_port => 50070, 7 | user => "logcenter" 8 | path => "/usr/logcenter/dt=%{+Y}-%{+M}-%{+d}/logstash-%{+H}.log" 9 | flush_size => 500 10 | compress => "snappy" 11 | idle_flush_time => 10 12 | retry_interval => 0.5 13 | workers => 32 14 | #use_kerberos_auth = true 15 | #kerberos_keytab = '/etc/security/keytabs/hdfs.headless.keytab' 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /logstash/conf.d/output-kafka.conf: -------------------------------------------------------------------------------- 1 | output { 2 | kafka { 3 | codec => json 4 | bootstrap_servers => "kafka1:6667,kafka2:6667,kafka3:6667" 5 | topic_id => "logcenter" 6 | batch_size => 16384 7 | compression_type => "snappy" 8 | 9 | #jaas_path => "" 10 | #security_protocol => "SASL_PLAINTEXT" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /logstash/conf.d/output-opentsdb.conf: -------------------------------------------------------------------------------- 1 | # opentsdb输入信息格式为：put metric timestamp value tagname=tagvalue tag2=value2 2 | # 在logstash-output-opentsdb插件metrics配置中默认已经输入timestamp 3 | # 因此metrics需要配置的第一个参数为metricName，第二个参数为 value 之后依次为tagname,tagValue 4 | output { 5 | opentsdb { 6 | host => '***.***.***.***' 7 | port => 4242 8 | metrics => [ 9 | "%{metricName}", 10 | "%{data}", 11 | "host", 12 | "%{metricHost}", 13 | "port", 14 | "%{port}" 15 | ] 16 | } 17 | } -------------------------------------------------------------------------------- /logstash/conf/ip2location.datx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiaomatech/log/47a6d4e914909b0b74876aa1921f54bd619fb4d3/logstash/conf/ip2location.datx -------------------------------------------------------------------------------- /logstash/conf/referers.yml: -------------------------------------------------------------------------------- 1 | # ####################################################################################################### 2 | # 3 | # ALL SUPPORTED REFERERS 4 | # 5 | # Broken down into: 6 | # 7 | # 1. Medium-unknown providers 8 | # 2. Email providers 9 | # 3. Social providers 10 | # 4. Search providers 11 | # 5. Paid media 12 | 13 | 14 | # ####################################################################################################### 15 | # 16 | # MEDIUM-UNKNOWN PROVIDERS 17 | # 18 | # We know the source, but not the medium. 19 | # This section is useful for reducing false positives in the other sections 20 | 21 | unknown: 22 | 23 | Google: 24 | domains: 25 | - support.google.com 26 | - developers.google.com 27 | - maps.google.com 28 | - accounts.google.com 29 | - drive.google.com 30 | - sites.google.com 31 | - groups.google.com 32 | - groups.google.co.uk 33 | 34 | Yahoo!: 35 | domains: 36 | - finance.yahoo.com 37 | - news.yahoo.com 38 | - eurosport.yahoo.com 39 | - sports.yahoo.com 40 | - astrology.yahoo.com 41 | - travel.yahoo.com 42 | - answers.yahoo.com 43 | - screen.yahoo.com 44 | - weather.yahoo.com 45 | - messenger.yahoo.com 46 | - games.yahoo.com 47 | - shopping.yahoo.net 48 | - movies.yahoo.com 49 | - cars.yahoo.com 50 | - lifestyle.yahoo.com 51 | - omg.yahoo.com 52 | - match.yahoo.net 53 | 54 | 55 | 56 | # ####################################################################################################### 57 | # 58 | # EMAIL PROVIDERS 59 | 60 | email: 61 | 62 | 126 Mail: 63 | domains: 64 | - mail.126.com 65 | 66 | 163 Mail: 67 | domains: 68 | - mail.163.com 69 | 70 | 2degrees: 71 | domains: 72 | - webmail.2degreesbroadband.co.nz 73 | 74 | Adam Internet: 75 | domains: 76 | - webmail.adam.com.au 77 | 78 | AOL Mail: 79 | domains: 80 | - mail.aol.com 81 | 82 | Bigpond: 83 | domains: 84 | - webmail.bigpond.com 85 | - webmail2.bigpond.com 86 | - email.telstra.com 87 | - basic.messaging.bigpond.com 88 | 89 | Commander: 90 | domains: 91 | - webmail.commander.net.au 92 | 93 | Daum Mail: 94 | domains: 95 | - mail2.daum.net 96 | - mail.daum.net 97 | 98 | Dodo: 99 | domains: 100 | - webmail.dodo.com.au 101 | 102 | Freenet: 103 | domains: 104 | - webmail.freenet.de 105 | 106 | Gmail: 107 | domains: 108 | - mail.google.com 109 | - inbox.google.com 110 | 111 | iiNet: 112 | domains: 113 | - webmail.iinet.net.au 114 | - mail.iinet.net.au 115 | 116 | Inbox.com: 117 | domains: 118 | - inbox.com 119 | 120 | iPrimus: 121 | domains: 122 | - webmail.iprimus.com.au 123 | 124 | Mynet Mail: 125 | domains: 126 | - mail.mynet.com 127 | 128 | Naver Mail: 129 | domains: 130 | - mail.naver.com 131 | 132 | Netspace: 133 | domains: 134 | - webmail.netspace.net.au 135 | 136 | Optus Zoo: 137 | domains: 138 | - webmail.optuszoo.com.au 139 | - webmail.optusnet.com.au 140 | 141 | Orange Webmail: 142 | domains: 143 | - orange.fr/webmail 144 | 145 | Outlook.com: 146 | domains: 147 | - mail.live.com 148 | - outlook.live.com 149 | 150 | QQ Mail: 151 | domains: 152 | - mail.qq.com 153 | 154 | Seznam Mail: 155 | domains: 156 | - email.seznam.cz 157 | 158 | Virgin: 159 | domains: 160 | - webmail.virginbroadband.com.au 161 | 162 | Vodafone: 163 | domains: 164 | - webmail.vodafone.co.nz 165 | 166 | Westnet: 167 | domains: 168 | - webmail.westnet.com.au 169 | 170 | Yahoo! Mail: 171 | domains: 172 | - mail.yahoo.net 173 | - mail.yahoo.com 174 | - mail.yahoo.co.uk 175 | - mail.yahoo.co.jp 176 | 177 | Zoho: 178 | domains: 179 | - mail.zoho.com 180 | 181 | # ####################################################################################################### 182 | # 183 | # SOCIAL PROVIDERS 184 | 185 | social: 186 | 187 | Facebook: 188 | domains: 189 | - facebook.com 190 | - fb.me 191 | - m.facebook.com 192 | - l.facebook.com 193 | - lm.facebook.com 194 | 195 | Qzone: 196 | domains: 197 | - qzone.qq.com 198 | 199 | Habbo: 200 | domains: 201 | - habbo.com 202 | 203 | Twitter: 204 | domains: 205 | - twitter.com 206 | - t.co 207 | 208 | Instagram: 209 | domains: 210 | - instagram.com 211 | 212 | Youtube: 213 | domains: 214 | - youtube.com 215 | - youtu.be 216 | 217 | Vimeo: 218 | domains: 219 | - vimeo.com 220 | 221 | Renren: 222 | domains: 223 | - renren.com 224 | 225 | Windows Live Spaces: 226 | domains: 227 | - login.live.com 228 | 229 | LinkedIn: 230 | domains: 231 | - linkedin.com 232 | - lnkd.in 233 | 234 | Bebo: 235 | domains: 236 | - bebo.com 237 | 238 | Vkontakte: 239 | domains: 240 | - vk.com 241 | - vkontakte.ru 242 | 243 | Tagged: 244 | domains: 245 | - login.tagged.com 246 | 247 | Orkut: 248 | domains: 249 | - orkut.com 250 | 251 | Myspace: 252 | domains: 253 | - myspace.com 254 | 255 | Friendster: 256 | domains: 257 | - friendster.com 258 | 259 | Badoo: 260 | domains: 261 | - badoo.com 262 | 263 | hi5: 264 | domains: 265 | - hi5.com 266 | 267 | Netlog: 268 | domains: 269 | - netlog.com 270 | 271 | Flixster: 272 | domains: 273 | - flixster.com 274 | 275 | MyLife: 276 | domains: 277 | - mylife.ru 278 | 279 | Paper.li: 280 | domains: 281 | - paper.li 282 | 283 | Classmates: 284 | domains: 285 | - classmates.com 286 | 287 | GitHub: 288 | domains: 289 | - github.com 290 | 291 | Google+: 292 | domains: 293 | - url.google.com 294 | - plus.google.com 295 | 296 | Douban: 297 | domains: 298 | - douban.com 299 | 300 | Odnoklassniki: 301 | domains: 302 | - odnoklassniki.ru 303 | 304 | Viadeo: 305 | domains: 306 | - viadeo.com 307 | 308 | Flickr: 309 | domains: 310 | - flickr.com 311 | 312 | WeeWorld: 313 | domains: 314 | - weeworld.com 315 | 316 | Last.fm: 317 | domains: 318 | - lastfm.ru 319 | 320 | MyHeritage: 321 | domains: 322 | - myheritage.com 323 | 324 | Xanga: 325 | domains: 326 | - xanga.com 327 | 328 | Mixi: 329 | domains: 330 | - mixi.jp 331 | 332 | Cyworld: 333 | domains: 334 | - global.cyworld.com 335 | 336 | Gaia Online: 337 | domains: 338 | - gaiaonline.com 339 | 340 | Skyrock: 341 | domains: 342 | - skyrock.com 343 | 344 | BlackPlanet: 345 | domains: 346 | - blackplanet.com 347 | 348 | myYearbook: 349 | domains: 350 | - myyearbook.com 351 | 352 | Fotolog: 353 | domains: 354 | - fotolog.com 355 | 356 | Friends Reunited: 357 | domains: 358 | - friendsreunited.com 359 | 360 | LiveJournal: 361 | domains: 362 | - livejournal.ru 363 | 364 | StudiVZ: 365 | domains: 366 | - studivz.net 367 | 368 | StackOverflow: 369 | domains: 370 | - stackoverflow.com 371 | 372 | Sonico.com: 373 | domains: 374 | - sonico.com 375 | 376 | Pinterest: 377 | domains: 378 | - pinterest.com 379 | 380 | Plaxo: 381 | domains: 382 | - plaxo.com 383 | 384 | Geni: 385 | domains: 386 | - geni.com 387 | 388 | Tuenti: 389 | domains: 390 | - tuenti.com 391 | 392 | XING: 393 | domains: 394 | - xing.com 395 | 396 | Taringa!: 397 | domains: 398 | - taringa.net 399 | 400 | Tumblr: 401 | domains: 402 | - tumblr.com 403 | - t.umblr.com 404 | 405 | Nasza-klasa.pl: 406 | domains: 407 | - nk.pl 408 | 409 | StumbleUpon: 410 | domains: 411 | - stumbleupon.com 412 | 413 | SourceForge: 414 | domains: 415 | - sourceforge.net 416 | 417 | Hyves: 418 | domains: 419 | - hyves.nl 420 | 421 | WAYN: 422 | domains: 423 | - wayn.com 424 | 425 | Buzznet: 426 | domains: 427 | - buzznet.com 428 | 429 | Multiply: 430 | domains: 431 | - multiply.com 432 | 433 | Foursquare: 434 | domains: 435 | - foursquare.com 436 | 437 | vKruguDruzei.ru: 438 | domains: 439 | - vkrugudruzei.ru 440 | 441 | Mail.ru: 442 | domains: 443 | - my.mail.ru 444 | 445 | MoiKrug.ru: 446 | domains: 447 | - moikrug.ru 448 | 449 | Reddit: 450 | domains: 451 | - reddit.com 452 | 453 | Hacker News: 454 | domains: 455 | - news.ycombinator.com 456 | 457 | Identi.ca: 458 | domains: 459 | - identi.ca 460 | 461 | Weibo: 462 | domains: 463 | - weibo.com 464 | - t.cn 465 | 466 | Delicious: 467 | domains: 468 | - delicious.com 469 | 470 | Pocket: 471 | domains: 472 | - getpocket.com 473 | 474 | ITU Sozluk: 475 | domains: 476 | - itusozluk.com 477 | 478 | Instela: 479 | domains: 480 | - instela.com 481 | 482 | Eksi Sozluk: 483 | domains: 484 | - Sozluk.com 485 | - sourtimes.org 486 | 487 | Uludag Sozluk: 488 | domains: 489 | - uludagsozluk.com 490 | - ulusozluk.com 491 | 492 | Inci Sozluk: 493 | domains: 494 | - inci.sozlukspot.com 495 | - incisozluk.com 496 | - incisozluk.cc 497 | 498 | Hocam.com: 499 | domains: 500 | - hocam.com 501 | 502 | Donanimhaber: 503 | domains: 504 | - donanimhaber.com 505 | 506 | Disqus: 507 | domains: 508 | - redirect.disqus.com 509 | - disq.us 510 | - disqus.com 511 | 512 | Quora: 513 | domains: 514 | - quora.com 515 | 516 | Whirlpool: 517 | domains: 518 | - forums.whirlpool.net.au 519 | 520 | # ####################################################################################################### 521 | # 522 | # SEARCH PROVIDERS 523 | 524 | search: 525 | 526 | 1.cz: 527 | parameters: 528 | - q 529 | domains: 530 | - 1.cz 531 | 532 | # 123people TODO 533 | 534 | 1&1: 535 | parameters: 536 | - q 537 | domains: 538 | - search.1and1.com 539 | 540 | 1und1: 541 | parameters: 542 | - su 543 | domains: 544 | - search.1und1.de 545 | 546 | 360.cn: 547 | parameters: 548 | - q 549 | domains: 550 | - so.360.cn 551 | - www.so.com 552 | 553 | Abacho: 554 | parameters: 555 | - q 556 | domains: 557 | - www.abacho.de 558 | - www.abacho.com 559 | - www.abacho.co.uk 560 | - www.se.abacho.com 561 | - www.tr.abacho.com 562 | - www.abacho.at 563 | - www.abacho.fr 564 | - www.abacho.es 565 | - www.abacho.ch 566 | - www.abacho.it 567 | 568 | ABCsøk: 569 | parameters: 570 | - q 571 | domains: 572 | - abcsolk.no 573 | - verden.abcsok.no 574 | 575 | Acoon: 576 | parameters: 577 | - begriff 578 | domains: 579 | - www.acoon.de 580 | 581 | Alexa: 582 | parameters: 583 | - q 584 | domains: 585 | - alexa.com 586 | - search.toolbars.alexa.com 587 | 588 | Alice Adsl: 589 | parameters: 590 | - q 591 | domains: 592 | - rechercher.aliceadsl.fr 593 | 594 | AllTheWeb: 595 | parameters: 596 | - q 597 | domains: 598 | - www.alltheweb.com 599 | 600 | all.by: 601 | parameters: 602 | - query 603 | domains: 604 | - all.by 605 | 606 | Altavista: 607 | parameters: 608 | - q 609 | domains: 610 | - www.altavista.com 611 | - search.altavista.com 612 | - listings.altavista.com 613 | - altavista.de 614 | - altavista.fr 615 | - be-nl.altavista.com 616 | - be-fr.altavista.com 617 | 618 | Amazon: 619 | parameters: 620 | - keywords 621 | domains: 622 | - amazon.com 623 | - www.amazon.com 624 | 625 | AOL: 626 | parameters: 627 | - q 628 | - query 629 | domains: 630 | - search.aol.com 631 | - search.aol.it 632 | - aolsearch.aol.com 633 | - aolsearch.com 634 | - www.aolrecherche.aol.fr 635 | - www.aolrecherches.aol.fr 636 | - www.aolimages.aol.fr 637 | - aim.search.aol.com 638 | - www.recherche.aol.fr 639 | - recherche.aol.fr 640 | - find.web.aol.com 641 | - recherche.aol.ca 642 | - aolsearch.aol.co.uk 643 | - search.aol.co.uk 644 | - aolrecherche.aol.fr 645 | - sucheaol.aol.de 646 | - suche.aol.de 647 | - suche.aolsvc.de 648 | - aolbusqueda.aol.com.mx 649 | - alicesuche.aol.de 650 | - alicesuchet.aol.de 651 | - suchet2.aol.de 652 | - search.hp.my.aol.com.au 653 | - search.hp.my.aol.de 654 | - search.hp.my.aol.it 655 | - search-intl.netscape.com 656 | 657 | Apollo Latvia: 658 | parameters: 659 | - q 660 | domains: 661 | - apollo.lv/portal/search/ 662 | 663 | APOLL07: 664 | parameters: 665 | - query 666 | domains: 667 | - apollo7.de 668 | 669 | Apontador: 670 | parameters: 671 | - q 672 | domains: 673 | - apontador.com.br 674 | - www.apontador.com.br 675 | 676 | Aport: 677 | parameters: 678 | - r 679 | domains: 680 | - sm.aport.ru 681 | 682 | arama: 683 | parameters: 684 | - q 685 | domains: 686 | - arama.com 687 | 688 | Arcor: 689 | parameters: 690 | - Keywords 691 | domains: 692 | - www.arcor.de 693 | 694 | Arianna: 695 | parameters: 696 | - query 697 | domains: 698 | - arianna.libero.it 699 | - www.arianna.com 700 | 701 | Ask: 702 | parameters: 703 | - q 704 | domains: 705 | - ask.com 706 | - www.ask.com 707 | - web.ask.com 708 | - int.ask.com 709 | - mws.ask.com 710 | - uk.ask.com 711 | - images.ask.com 712 | - ask.reference.com 713 | - www.askkids.com 714 | - iwon.ask.com 715 | - www.ask.co.uk 716 | - www.qbyrd.com 717 | - search-results.com 718 | - uk.search-results.com 719 | - www.search-results.com 720 | - int.search-results.com 721 | 722 | Ask Toolbar: 723 | parameters: 724 | - searchfor 725 | domains: 726 | - search.tb.ask.com 727 | 728 | Atlas: 729 | parameters: 730 | - q 731 | domains: 732 | - searchatlas.centrum.cz 733 | 734 | Austronaut: 735 | parameters: 736 | - q 737 | domains: 738 | - www2.austronaut.at 739 | - www1.astronaut.at 740 | 741 | Babylon: 742 | parameters: 743 | - q 744 | domains: 745 | - search.babylon.com 746 | - searchassist.babylon.com 747 | 748 | Baidu: 749 | parameters: 750 | - wd 751 | - word 752 | - kw 753 | - k 754 | domains: 755 | - www.baidu.com 756 | - www1.baidu.com 757 | - zhidao.baidu.com 758 | - tieba.baidu.com 759 | - news.baidu.com 760 | - web.gougou.com 761 | - m.baidu.com 762 | 763 | Biglobe: 764 | parameters: 765 | - q 766 | domains: 767 | - cgi.search.biglobe.ne.jp 768 | 769 | Bing: 770 | parameters: 771 | - q 772 | - Q 773 | domains: 774 | - bing.com 775 | - www.bing.com 776 | - msnbc.msn.com 777 | - dizionario.it.msn.com 778 | - cc.bingj.com 779 | - m.bing.com 780 | 781 | Bing Images: 782 | parameters: 783 | - q 784 | - Q 785 | domains: 786 | - bing.com/images/search 787 | - www.bing.com/images/search 788 | 789 | blekko: 790 | parameters: 791 | - q 792 | domains: 793 | - blekko.com 794 | 795 | Blogdigger: 796 | parameters: 797 | - q 798 | domains: 799 | - www.blogdigger.com 800 | 801 | Blogpulse: 802 | parameters: 803 | - query 804 | domains: 805 | - www.blogpulse.com 806 | 807 | Bluewin: 808 | parameters: 809 | - searchTerm 810 | domains: 811 | - search.bluewin.ch 812 | 813 | British Telecommunications: 814 | parameters: 815 | - p 816 | domains: 817 | - search.bt.com 818 | 819 | canoe.ca: 820 | parameters: 821 | - q 822 | domains: 823 | - web.canoe.ca 824 | 825 | Centrum: 826 | parameters: 827 | - q 828 | domains: 829 | - serach.centrum.cz 830 | - morfeo.centrum.cz 831 | 832 | Certified-Toolbar: 833 | parameters: 834 | - q 835 | domains: 836 | - search.certified-toolbar.com 837 | 838 | Charter: 839 | parameters: 840 | - q 841 | domains: 842 | - www.charter.net 843 | 844 | Clix: 845 | parameters: 846 | - question 847 | domains: 848 | - pesquisa.clix.pt 849 | 850 | Conduit: 851 | parameters: 852 | - q 853 | domains: 854 | - search.conduit.com 855 | 856 | Comcast: 857 | parameters: 858 | - q 859 | domains: 860 | - serach.comcast.net 861 | 862 | Crawler: 863 | parameters: 864 | - q 865 | domains: 866 | - www.crawler.com 867 | 868 | Compuserve: 869 | parameters: 870 | - query 871 | domains: 872 | - websearch.cs.com 873 | 874 | Cuil: 875 | parameters: 876 | - q 877 | domains: 878 | - www.cuil.com 879 | 880 | Daemon search: 881 | parameters: 882 | - q 883 | domains: 884 | - daemon-search.com 885 | - my.daemon-search.com 886 | 887 | Dalesearch: 888 | parameters: 889 | - q 890 | domains: 891 | - www.dalesearch.com 892 | 893 | DasOertliche: 894 | parameters: 895 | - kw 896 | domains: 897 | - www.dasoertliche.de 898 | 899 | DasTelefonbuch: 900 | parameters: 901 | - kw 902 | domains: 903 | - www1.dastelefonbuch.de 904 | 905 | Daum: 906 | parameters: 907 | - q 908 | domains: 909 | - search.daum.net 910 | 911 | Delfi latvia: 912 | parameters: 913 | - q 914 | domains: 915 | - smart.delfi.lv 916 | 917 | Delfi: 918 | parameters: 919 | - q 920 | domains: 921 | - otsing.delfi.ee 922 | 923 | Digg: 924 | parameters: 925 | - s 926 | domains: 927 | - digg.com 928 | 929 | dmoz: 930 | parameters: 931 | - q 932 | domains: 933 | - dmoz.org 934 | - editors.dmoz.org 935 | 936 | Dodo: 937 | parameters: 938 | - q 939 | domains: 940 | - google.dodo.com.au 941 | 942 | DuckDuckGo: 943 | parameters: 944 | - q 945 | domains: 946 | - duckduckgo.com 947 | 948 | earthlink: 949 | parameters: 950 | - q 951 | domains: 952 | - search.earthlink.net 953 | 954 | Ecosia: 955 | parameters: 956 | - q 957 | domains: 958 | - ecosia.org 959 | 960 | Eniro: 961 | parameters: 962 | - q 963 | - search_word 964 | domains: 965 | - www.eniro.se 966 | 967 | Eurip: 968 | parameters: 969 | - q 970 | domains: 971 | - www.eurip.com 972 | 973 | Euroseek: 974 | parameters: 975 | - string 976 | domains: 977 | - www.euroseek.com 978 | 979 | Everyclick: 980 | parameters: 981 | - keyword 982 | domains: 983 | - www.everyclick.com 984 | 985 | Excite: 986 | parameters: 987 | - q 988 | - search 989 | domains: 990 | - search.excite.it 991 | - search.excite.fr 992 | - search.excite.de 993 | - search.excite.co.uk 994 | - serach.excite.es 995 | - search.excite.nl 996 | - msxml.excite.com 997 | - www.excite.co.jp 998 | 999 | Exalead: 1000 | parameters: 1001 | - q 1002 | domains: 1003 | - www.exalead.fr 1004 | - www.exalead.com 1005 | 1006 | eo: 1007 | parameters: 1008 | - x_query 1009 | domains: 1010 | - eo.st 1011 | 1012 | Fast Browser Search: 1013 | parameters: 1014 | - q 1015 | domains: 1016 | - www.fastbrowsersearch.com 1017 | 1018 | Francite: 1019 | parameters: 1020 | - name 1021 | domains: 1022 | - recherche.francite.com 1023 | 1024 | Finderoo: 1025 | parameters: 1026 | - q 1027 | domains: 1028 | - www.finderoo.com 1029 | 1030 | Findwide: 1031 | parameters: 1032 | - k 1033 | domains: 1034 | - search.findwide.com 1035 | 1036 | Fireball: 1037 | parameters: 1038 | - q 1039 | domains: 1040 | - www.fireball.de 1041 | 1042 | Firstfind: 1043 | parameters: 1044 | - qry 1045 | domains: 1046 | - www.firstsfind.com 1047 | 1048 | Fixsuche: 1049 | parameters: 1050 | - q 1051 | domains: 1052 | - www.fixsuche.de 1053 | 1054 | Flix: 1055 | parameters: 1056 | - keyword 1057 | domains: 1058 | - www.flix.de 1059 | 1060 | Forestle: 1061 | parameters: 1062 | - q 1063 | domains: 1064 | - forestle.org 1065 | - www.forestle.org 1066 | - forestle.mobi 1067 | 1068 | Free: 1069 | parameters: 1070 | - q 1071 | domains: 1072 | - search.free.fr 1073 | - search1-2.free.fr 1074 | - search1-1.free.fr 1075 | 1076 | Freecause: 1077 | parameters: 1078 | - p 1079 | domains: 1080 | - search.freecause.com 1081 | 1082 | Freenet: 1083 | parameters: 1084 | - query 1085 | - Keywords 1086 | domains: 1087 | - suche.freenet.de 1088 | 1089 | Freshweather: 1090 | parameters: 1091 | - q 1092 | domains: 1093 | - www.fresh-weather.com 1094 | 1095 | FriendFeed: 1096 | parameters: 1097 | - q 1098 | domains: 1099 | - friendfeed.com 1100 | 1101 | GAIS: 1102 | parameters: 1103 | - q 1104 | domains: 1105 | - gais.cs.ccu.edu.tw 1106 | 1107 | Geona: 1108 | parameters: 1109 | - q 1110 | domains: 1111 | - geona.net 1112 | 1113 | Genieo: 1114 | parameters: 1115 | - q 1116 | domains: 1117 | - search.genieo.com 1118 | 1119 | Gigablast: 1120 | parameters: 1121 | - q 1122 | domains: 1123 | - www.gigablast.com 1124 | - dir.gigablast.com 1125 | 1126 | Globososo: 1127 | parameters: 1128 | - q 1129 | domains: 1130 | - searches.globososo.com 1131 | - search.globososo.com 1132 | 1133 | GMX: 1134 | parameters: 1135 | - su 1136 | domains: 1137 | - suche.gmx.net 1138 | 1139 | Gnadenmeer: 1140 | parameters: 1141 | - keyword 1142 | domains: 1143 | - www.gnadenmeer.de 1144 | 1145 | Gomeo: 1146 | parameters: 1147 | - Keywords 1148 | domains: 1149 | - www.gomeo.com 1150 | 1151 | goo: 1152 | parameters: 1153 | - MT 1154 | domains: 1155 | - search.goo.ne.jp 1156 | - ocnsearch.goo.ne.jp 1157 | 1158 | Google: 1159 | parameters: 1160 | - q 1161 | - query # For www.cnn.com (powered by Google) 1162 | - Keywords # For gooofullsearch.com (powered by Google) 1163 | domains: 1164 | - www.google.com 1165 | - www.google.ac 1166 | - www.google.ad 1167 | - www.google.com.af 1168 | - www.google.com.ag 1169 | - www.google.com.ai 1170 | - www.google.am 1171 | - www.google.it.ao 1172 | - www.google.com.ar 1173 | - www.google.as 1174 | - www.google.at 1175 | - www.google.com.au 1176 | - www.google.az 1177 | - www.google.ba 1178 | - www.google.com.bd 1179 | - www.google.be 1180 | - www.google.bf 1181 | - www.google.bg 1182 | - www.google.com.bh 1183 | - www.google.bi 1184 | - www.google.bj 1185 | - www.google.com.bn 1186 | - www.google.com.bo 1187 | - www.google.com.br 1188 | - www.google.bs 1189 | - www.google.co.bw 1190 | - www.google.com.by 1191 | - www.google.by 1192 | - www.google.com.bz 1193 | - www.google.ca 1194 | - www.google.com.kh 1195 | - www.google.cc 1196 | - www.google.cd 1197 | - www.google.cf 1198 | - www.google.cat 1199 | - www.google.cg 1200 | - www.google.ch 1201 | - www.google.ci 1202 | - www.google.co.ck 1203 | - www.google.cl 1204 | - www.google.cm 1205 | - www.google.cn 1206 | - www.google.com.co 1207 | - www.google.co.cr 1208 | - www.google.com.cu 1209 | - www.google.cv 1210 | - www.google.com.cy 1211 | - www.google.cz 1212 | - www.google.de 1213 | - www.google.dj 1214 | - www.google.dk 1215 | - www.google.dm 1216 | - www.google.com.do 1217 | - www.google.dz 1218 | - www.google.com.ec 1219 | - www.google.ee 1220 | - www.google.com.eg 1221 | - www.google.es 1222 | - www.google.com.et 1223 | - www.google.fi 1224 | - www.google.com.fj 1225 | - www.google.fm 1226 | - www.google.fr 1227 | - www.google.ga 1228 | - www.google.gd 1229 | - www.google.ge 1230 | - www.google.gf 1231 | - www.google.gg 1232 | - www.google.com.gh 1233 | - www.google.com.gi 1234 | - www.google.gl 1235 | - www.google.gm 1236 | - www.google.gp 1237 | - www.google.gr 1238 | - www.google.com.gt 1239 | - www.google.gy 1240 | - www.google.com.hk 1241 | - www.google.hn 1242 | - www.google.hr 1243 | - www.google.ht 1244 | - www.google.hu 1245 | - www.google.co.id 1246 | - www.google.iq 1247 | - www.google.ie 1248 | - www.google.co.il 1249 | - www.google.im 1250 | - www.google.co.in 1251 | - www.google.io 1252 | - www.google.is 1253 | - www.google.it 1254 | - www.google.je 1255 | - www.google.com.jm 1256 | - www.google.jo 1257 | - www.google.co.jp 1258 | - www.google.co.ke 1259 | - www.google.ki 1260 | - www.google.kg 1261 | - www.google.co.kr 1262 | - www.google.com.kw 1263 | - www.google.kz 1264 | - www.google.la 1265 | - www.google.com.lb 1266 | - www.google.com.lc 1267 | - www.google.li 1268 | - www.google.lk 1269 | - www.google.co.ls 1270 | - www.google.lt 1271 | - www.google.lu 1272 | - www.google.lv 1273 | - www.google.com.ly 1274 | - www.google.co.ma 1275 | - www.google.md 1276 | - www.google.me 1277 | - www.google.mg 1278 | - www.google.mk 1279 | - www.google.ml 1280 | - www.google.mn 1281 | - www.google.ms 1282 | - www.google.com.mt 1283 | - www.google.mu 1284 | - www.google.mv 1285 | - www.google.mw 1286 | - www.google.com.mx 1287 | - www.google.com.my 1288 | - www.google.co.mz 1289 | - www.google.com.na 1290 | - www.google.ne 1291 | - www.google.com.nf 1292 | - www.google.com.ng 1293 | - www.google.com.ni 1294 | - www.google.nl 1295 | - www.google.no 1296 | - www.google.com.np 1297 | - www.google.nr 1298 | - www.google.nu 1299 | - www.google.co.nz 1300 | - www.google.com.om 1301 | - www.google.com.pa 1302 | - www.google.com.pe 1303 | - www.google.com.ph 1304 | - www.google.com.pk 1305 | - www.google.pl 1306 | - www.google.pn 1307 | - www.google.com.pr 1308 | - www.google.ps 1309 | - www.google.pt 1310 | - www.google.com.py 1311 | - www.google.com.qa 1312 | - www.google.ro 1313 | - www.google.rs 1314 | - www.google.ru 1315 | - www.google.rw 1316 | - www.google.com.sa 1317 | - www.google.com.sb 1318 | - www.google.sc 1319 | - www.google.se 1320 | - www.google.com.sg 1321 | - www.google.sh 1322 | - www.google.si 1323 | - www.google.sk 1324 | - www.google.com.sl 1325 | - www.google.sn 1326 | - www.google.sm 1327 | - www.google.so 1328 | - www.google.st 1329 | - www.google.com.sv 1330 | - www.google.td 1331 | - www.google.tg 1332 | - www.google.co.th 1333 | - www.google.com.tj 1334 | - www.google.tk 1335 | - www.google.tl 1336 | - www.google.tm 1337 | - www.google.to 1338 | - www.google.com.tn 1339 | - www.google.tn 1340 | - www.google.com.tr 1341 | - www.google.tt 1342 | - www.google.com.tw 1343 | - www.google.co.tz 1344 | - www.google.com.ua 1345 | - www.google.co.ug 1346 | - www.google.ae 1347 | - www.google.co.uk 1348 | - www.google.us 1349 | - www.google.com.uy 1350 | - www.google.co.uz 1351 | - www.google.com.vc 1352 | - www.google.co.ve 1353 | - www.google.vg 1354 | - www.google.co.vi 1355 | - www.google.com.vn 1356 | - www.google.vu 1357 | - www.google.ws 1358 | - www.google.co.za 1359 | - www.google.co.zm 1360 | - www.google.co.zw 1361 | - google.com 1362 | - google.ac 1363 | - google.ad 1364 | - google.com.af 1365 | - google.com.ag 1366 | - google.com.ai 1367 | - google.am 1368 | - google.it.ao 1369 | - google.com.ar 1370 | - google.as 1371 | - google.at 1372 | - google.com.au 1373 | - google.az 1374 | - google.ba 1375 | - google.com.bd 1376 | - google.be 1377 | - google.bf 1378 | - google.bg 1379 | - google.com.bh 1380 | - google.bi 1381 | - google.bj 1382 | - google.com.bn 1383 | - google.com.bo 1384 | - google.com.br 1385 | - google.bs 1386 | - google.co.bw 1387 | - google.com.by 1388 | - google.by 1389 | - google.com.bz 1390 | - google.ca 1391 | - google.com.kh 1392 | - google.cc 1393 | - google.cd 1394 | - google.cf 1395 | - google.cat 1396 | - google.cg 1397 | - google.ch 1398 | - google.ci 1399 | - google.co.ck 1400 | - google.cl 1401 | - google.cm 1402 | - google.cn 1403 | - google.com.co 1404 | - google.co.cr 1405 | - google.com.cu 1406 | - google.cv 1407 | - google.com.cy 1408 | - google.cz 1409 | - google.de 1410 | - google.dj 1411 | - google.dk 1412 | - google.dm 1413 | - google.com.do 1414 | - google.dz 1415 | - google.com.ec 1416 | - google.ee 1417 | - google.com.eg 1418 | - google.es 1419 | - google.com.et 1420 | - google.fi 1421 | - google.com.fj 1422 | - google.fm 1423 | - google.fr 1424 | - google.ga 1425 | - google.gd 1426 | - google.ge 1427 | - google.gf 1428 | - google.gg 1429 | - google.com.gh 1430 | - google.com.gi 1431 | - google.gl 1432 | - google.gm 1433 | - google.gp 1434 | - google.gr 1435 | - google.com.gt 1436 | - google.gy 1437 | - google.com.hk 1438 | - google.hn 1439 | - google.hr 1440 | - google.ht 1441 | - google.hu 1442 | - google.co.id 1443 | - google.iq 1444 | - google.ie 1445 | - google.co.il 1446 | - google.im 1447 | - google.co.in 1448 | - google.io 1449 | - google.is 1450 | - google.it 1451 | - google.je 1452 | - google.com.jm 1453 | - google.jo 1454 | - google.co.jp 1455 | - google.co.ke 1456 | - google.ki 1457 | - google.kg 1458 | - google.co.kr 1459 | - google.com.kw 1460 | - google.kz 1461 | - google.la 1462 | - google.com.lb 1463 | - google.com.lc 1464 | - google.li 1465 | - google.lk 1466 | - google.co.ls 1467 | - google.lt 1468 | - google.lu 1469 | - google.lv 1470 | - google.com.ly 1471 | - google.co.ma 1472 | - google.md 1473 | - google.me 1474 | - google.mg 1475 | - google.mk 1476 | - google.ml 1477 | - google.mn 1478 | - google.ms 1479 | - google.com.mt 1480 | - google.mu 1481 | - google.mv 1482 | - google.mw 1483 | - google.com.mx 1484 | - google.com.my 1485 | - google.co.mz 1486 | - google.com.na 1487 | - google.ne 1488 | - google.com.nf 1489 | - google.com.ng 1490 | - google.com.ni 1491 | - google.nl 1492 | - google.no 1493 | - google.com.np 1494 | - google.nr 1495 | - google.nu 1496 | - google.co.nz 1497 | - google.com.om 1498 | - google.com.pa 1499 | - google.com.pe 1500 | - google.com.ph 1501 | - google.com.pk 1502 | - google.pl 1503 | - google.pn 1504 | - google.com.pr 1505 | - google.ps 1506 | - google.pt 1507 | - google.com.py 1508 | - google.com.qa 1509 | - google.ro 1510 | - google.rs 1511 | - google.ru 1512 | - google.rw 1513 | - google.com.sa 1514 | - google.com.sb 1515 | - google.sc 1516 | - google.se 1517 | - google.com.sg 1518 | - google.sh 1519 | - google.si 1520 | - google.sk 1521 | - google.com.sl 1522 | - google.sn 1523 | - google.sm 1524 | - google.so 1525 | - google.st 1526 | - google.com.sv 1527 | - google.td 1528 | - google.tg 1529 | - google.co.th 1530 | - google.com.tj 1531 | - google.tk 1532 | - google.tl 1533 | - google.tm 1534 | - google.to 1535 | - google.com.tn 1536 | - google.com.tr 1537 | - google.tt 1538 | - google.com.tw 1539 | - google.co.tz 1540 | - google.com.ua 1541 | - google.co.ug 1542 | - google.ae 1543 | - google.co.uk 1544 | - google.us 1545 | - google.com.uy 1546 | - google.co.uz 1547 | - google.com.vc 1548 | - google.co.ve 1549 | - google.vg 1550 | - google.co.vi 1551 | - google.com.vn 1552 | - google.vu 1553 | - google.ws 1554 | - google.co.za 1555 | - google.co.zm 1556 | - google.co.zw 1557 | - google.tn 1558 | # powered by Google 1559 | - search.avg.com 1560 | - isearch.avg.com 1561 | - www.cnn.com 1562 | - darkoogle.com 1563 | - search.darkoogle.com 1564 | - search.foxtab.com 1565 | - www.gooofullsearch.com 1566 | - search.hiyo.com 1567 | - search.incredimail.com 1568 | - search1.incredimail.com 1569 | - search2.incredimail.com 1570 | - search3.incredimail.com 1571 | - search4.incredimail.com 1572 | - search.incredibar.com 1573 | - search.sweetim.com 1574 | - www.fastweb.it 1575 | - search.juno.com 1576 | - find.tdc.dk 1577 | - searchresults.verizon.com 1578 | - search.walla.co.il 1579 | - search.alot.com 1580 | # Google Earch 1581 | - www.googleearth.de 1582 | - www.googleearth.fr 1583 | # Google Cache 1584 | - webcache.googleusercontent.com 1585 | # Google SSL 1586 | - encrypted.google.com 1587 | # Syndicated search 1588 | - googlesyndicatedsearch.com 1589 | 1590 | Google Blogsearch: 1591 | parameters: 1592 | - q 1593 | domains: 1594 | - blogsearch.google.ac 1595 | - blogsearch.google.ad 1596 | - blogsearch.google.ae 1597 | - blogsearch.google.am 1598 | - blogsearch.google.as 1599 | - blogsearch.google.at 1600 | - blogsearch.google.az 1601 | - blogsearch.google.ba 1602 | - blogsearch.google.be 1603 | - blogsearch.google.bf 1604 | - blogsearch.google.bg 1605 | - blogsearch.google.bi 1606 | - blogsearch.google.bj 1607 | - blogsearch.google.bs 1608 | - blogsearch.google.by 1609 | - blogsearch.google.ca 1610 | - blogsearch.google.cat 1611 | - blogsearch.google.cc 1612 | - blogsearch.google.cd 1613 | - blogsearch.google.cf 1614 | - blogsearch.google.cg 1615 | - blogsearch.google.ch 1616 | - blogsearch.google.ci 1617 | - blogsearch.google.cl 1618 | - blogsearch.google.cm 1619 | - blogsearch.google.cn 1620 | - blogsearch.google.co.bw 1621 | - blogsearch.google.co.ck 1622 | - blogsearch.google.co.cr 1623 | - blogsearch.google.co.id 1624 | - blogsearch.google.co.il 1625 | - blogsearch.google.co.in 1626 | - blogsearch.google.co.jp 1627 | - blogsearch.google.co.ke 1628 | - blogsearch.google.co.kr 1629 | - blogsearch.google.co.ls 1630 | - blogsearch.google.co.ma 1631 | - blogsearch.google.co.mz 1632 | - blogsearch.google.co.nz 1633 | - blogsearch.google.co.th 1634 | - blogsearch.google.co.tz 1635 | - blogsearch.google.co.ug 1636 | - blogsearch.google.co.uk 1637 | - blogsearch.google.co.uz 1638 | - blogsearch.google.co.ve 1639 | - blogsearch.google.co.vi 1640 | - blogsearch.google.co.za 1641 | - blogsearch.google.co.zm 1642 | - blogsearch.google.co.zw 1643 | - blogsearch.google.com 1644 | - blogsearch.google.com.af 1645 | - blogsearch.google.com.ag 1646 | - blogsearch.google.com.ai 1647 | - blogsearch.google.com.ar 1648 | - blogsearch.google.com.au 1649 | - blogsearch.google.com.bd 1650 | - blogsearch.google.com.bh 1651 | - blogsearch.google.com.bn 1652 | - blogsearch.google.com.bo 1653 | - blogsearch.google.com.br 1654 | - blogsearch.google.com.by 1655 | - blogsearch.google.com.bz 1656 | - blogsearch.google.com.co 1657 | - blogsearch.google.com.cu 1658 | - blogsearch.google.com.cy 1659 | - blogsearch.google.com.do 1660 | - blogsearch.google.com.ec 1661 | - blogsearch.google.com.eg 1662 | - blogsearch.google.com.et 1663 | - blogsearch.google.com.fj 1664 | - blogsearch.google.com.gh 1665 | - blogsearch.google.com.gi 1666 | - blogsearch.google.com.gt 1667 | - blogsearch.google.com.hk 1668 | - blogsearch.google.com.jm 1669 | - blogsearch.google.com.kh 1670 | - blogsearch.google.com.kw 1671 | - blogsearch.google.com.lb 1672 | - blogsearch.google.com.lc 1673 | - blogsearch.google.com.ly 1674 | - blogsearch.google.com.mt 1675 | - blogsearch.google.com.mx 1676 | - blogsearch.google.com.my 1677 | - blogsearch.google.com.na 1678 | - blogsearch.google.com.nf 1679 | - blogsearch.google.com.ng 1680 | - blogsearch.google.com.ni 1681 | - blogsearch.google.com.np 1682 | - blogsearch.google.com.om 1683 | - blogsearch.google.com.pa 1684 | - blogsearch.google.com.pe 1685 | - blogsearch.google.com.ph 1686 | - blogsearch.google.com.pk 1687 | - blogsearch.google.com.pr 1688 | - blogsearch.google.com.py 1689 | - blogsearch.google.com.qa 1690 | - blogsearch.google.com.sa 1691 | - blogsearch.google.com.sb 1692 | - blogsearch.google.com.sg 1693 | - blogsearch.google.com.sl 1694 | - blogsearch.google.com.sv 1695 | - blogsearch.google.com.tj 1696 | - blogsearch.google.com.tn 1697 | - blogsearch.google.com.tr 1698 | - blogsearch.google.com.tw 1699 | - blogsearch.google.com.ua 1700 | - blogsearch.google.com.uy 1701 | - blogsearch.google.com.vc 1702 | - blogsearch.google.com.vn 1703 | - blogsearch.google.cv 1704 | - blogsearch.google.cz 1705 | - blogsearch.google.de 1706 | - blogsearch.google.dj 1707 | - blogsearch.google.dk 1708 | - blogsearch.google.dm 1709 | - blogsearch.google.dz 1710 | - blogsearch.google.ee 1711 | - blogsearch.google.es 1712 | - blogsearch.google.fi 1713 | - blogsearch.google.fm 1714 | - blogsearch.google.fr 1715 | - blogsearch.google.ga 1716 | - blogsearch.google.gd 1717 | - blogsearch.google.ge 1718 | - blogsearch.google.gf 1719 | - blogsearch.google.gg 1720 | - blogsearch.google.gl 1721 | - blogsearch.google.gm 1722 | - blogsearch.google.gp 1723 | - blogsearch.google.gr 1724 | - blogsearch.google.gy 1725 | - blogsearch.google.hn 1726 | - blogsearch.google.hr 1727 | - blogsearch.google.ht 1728 | - blogsearch.google.hu 1729 | - blogsearch.google.ie 1730 | - blogsearch.google.im 1731 | - blogsearch.google.io 1732 | - blogsearch.google.iq 1733 | - blogsearch.google.is 1734 | - blogsearch.google.it 1735 | - blogsearch.google.it.ao 1736 | - blogsearch.google.je 1737 | - blogsearch.google.jo 1738 | - blogsearch.google.kg 1739 | - blogsearch.google.ki 1740 | - blogsearch.google.kz 1741 | - blogsearch.google.la 1742 | - blogsearch.google.li 1743 | - blogsearch.google.lk 1744 | - blogsearch.google.lt 1745 | - blogsearch.google.lu 1746 | - blogsearch.google.lv 1747 | - blogsearch.google.md 1748 | - blogsearch.google.me 1749 | - blogsearch.google.mg 1750 | - blogsearch.google.mk 1751 | - blogsearch.google.ml 1752 | - blogsearch.google.mn 1753 | - blogsearch.google.ms 1754 | - blogsearch.google.mu 1755 | - blogsearch.google.mv 1756 | - blogsearch.google.mw 1757 | - blogsearch.google.ne 1758 | - blogsearch.google.nl 1759 | - blogsearch.google.no 1760 | - blogsearch.google.nr 1761 | - blogsearch.google.nu 1762 | - blogsearch.google.pl 1763 | - blogsearch.google.pn 1764 | - blogsearch.google.ps 1765 | - blogsearch.google.pt 1766 | - blogsearch.google.ro 1767 | - blogsearch.google.rs 1768 | - blogsearch.google.ru 1769 | - blogsearch.google.rw 1770 | - blogsearch.google.sc 1771 | - blogsearch.google.se 1772 | - blogsearch.google.sh 1773 | - blogsearch.google.si 1774 | - blogsearch.google.sk 1775 | - blogsearch.google.sm 1776 | - blogsearch.google.sn 1777 | - blogsearch.google.so 1778 | - blogsearch.google.st 1779 | - blogsearch.google.td 1780 | - blogsearch.google.tg 1781 | - blogsearch.google.tk 1782 | - blogsearch.google.tl 1783 | - blogsearch.google.tm 1784 | - blogsearch.google.to 1785 | - blogsearch.google.tt 1786 | - blogsearch.google.us 1787 | - blogsearch.google.vg 1788 | - blogsearch.google.vu 1789 | - blogsearch.google.ws 1790 | 1791 | Google Images: 1792 | parameters: 1793 | - q 1794 | domains: 1795 | - google.ac/imgres 1796 | - google.ad/imgres 1797 | - google.ae/imgres 1798 | - google.am/imgres 1799 | - google.as/imgres 1800 | - google.at/imgres 1801 | - google.az/imgres 1802 | - google.ba/imgres 1803 | - google.be/imgres 1804 | - google.bf/imgres 1805 | - google.bg/imgres 1806 | - google.bi/imgres 1807 | - google.bj/imgres 1808 | - google.bs/imgres 1809 | - google.by/imgres 1810 | - google.ca/imgres 1811 | - google.cat/imgres 1812 | - google.cc/imgres 1813 | - google.cd/imgres 1814 | - google.cf/imgres 1815 | - google.cg/imgres 1816 | - google.ch/imgres 1817 | - google.ci/imgres 1818 | - google.cl/imgres 1819 | - google.cm/imgres 1820 | - google.cn/imgres 1821 | - google.co.bw/imgres 1822 | - google.co.ck/imgres 1823 | - google.co.cr/imgres 1824 | - google.co.id/imgres 1825 | - google.co.il/imgres 1826 | - google.co.in/imgres 1827 | - google.co.jp/imgres 1828 | - google.co.ke/imgres 1829 | - google.co.kr/imgres 1830 | - google.co.ls/imgres 1831 | - google.co.ma/imgres 1832 | - google.co.mz/imgres 1833 | - google.co.nz/imgres 1834 | - google.co.th/imgres 1835 | - google.co.tz/imgres 1836 | - google.co.ug/imgres 1837 | - google.co.uk/imgres 1838 | - google.co.uz/imgres 1839 | - google.co.ve/imgres 1840 | - google.co.vi/imgres 1841 | - google.co.za/imgres 1842 | - google.co.zm/imgres 1843 | - google.co.zw/imgres 1844 | - google.com/imgres 1845 | - google.com.af/imgres 1846 | - google.com.ag/imgres 1847 | - google.com.ai/imgres 1848 | - google.com.ar/imgres 1849 | - google.com.au/imgres 1850 | - google.com.bd/imgres 1851 | - google.com.bh/imgres 1852 | - google.com.bn/imgres 1853 | - google.com.bo/imgres 1854 | - google.com.br/imgres 1855 | - google.com.by/imgres 1856 | - google.com.bz/imgres 1857 | - google.com.co/imgres 1858 | - google.com.cu/imgres 1859 | - google.com.cy/imgres 1860 | - google.com.do/imgres 1861 | - google.com.ec/imgres 1862 | - google.com.eg/imgres 1863 | - google.com.et/imgres 1864 | - google.com.fj/imgres 1865 | - google.com.gh/imgres 1866 | - google.com.gi/imgres 1867 | - google.com.gt/imgres 1868 | - google.com.hk/imgres 1869 | - google.com.jm/imgres 1870 | - google.com.kh/imgres 1871 | - google.com.kw/imgres 1872 | - google.com.lb/imgres 1873 | - google.com.lc/imgres 1874 | - google.com.ly/imgres 1875 | - google.com.mt/imgres 1876 | - google.com.mx/imgres 1877 | - google.com.my/imgres 1878 | - google.com.na/imgres 1879 | - google.com.nf/imgres 1880 | - google.com.ng/imgres 1881 | - google.com.ni/imgres 1882 | - google.com.np/imgres 1883 | - google.com.om/imgres 1884 | - google.com.pa/imgres 1885 | - google.com.pe/imgres 1886 | - google.com.ph/imgres 1887 | - google.com.pk/imgres 1888 | - google.com.pr/imgres 1889 | - google.com.py/imgres 1890 | - google.com.qa/imgres 1891 | - google.com.sa/imgres 1892 | - google.com.sb/imgres 1893 | - google.com.sg/imgres 1894 | - google.com.sl/imgres 1895 | - google.com.sv/imgres 1896 | - google.com.tj/imgres 1897 | - google.com.tn/imgres 1898 | - google.com.tr/imgres 1899 | - google.com.tw/imgres 1900 | - google.com.ua/imgres 1901 | - google.com.uy/imgres 1902 | - google.com.vc/imgres 1903 | - google.com.vn/imgres 1904 | - google.cv/imgres 1905 | - google.cz/imgres 1906 | - google.de/imgres 1907 | - google.dj/imgres 1908 | - google.dk/imgres 1909 | - google.dm/imgres 1910 | - google.dz/imgres 1911 | - google.ee/imgres 1912 | - google.es/imgres 1913 | - google.fi/imgres 1914 | - google.fm/imgres 1915 | - google.fr/imgres 1916 | - google.ga/imgres 1917 | - google.gd/imgres 1918 | - google.ge/imgres 1919 | - google.gf/imgres 1920 | - google.gg/imgres 1921 | - google.gl/imgres 1922 | - google.gm/imgres 1923 | - google.gp/imgres 1924 | - google.gr/imgres 1925 | - google.gy/imgres 1926 | - google.hn/imgres 1927 | - google.hr/imgres 1928 | - google.ht/imgres 1929 | - google.hu/imgres 1930 | - google.ie/imgres 1931 | - google.im/imgres 1932 | - google.io/imgres 1933 | - google.iq/imgres 1934 | - google.is/imgres 1935 | - google.it/imgres 1936 | - google.it.ao/imgres 1937 | - google.je/imgres 1938 | - google.jo/imgres 1939 | - google.kg/imgres 1940 | - google.ki/imgres 1941 | - google.kz/imgres 1942 | - google.la/imgres 1943 | - google.li/imgres 1944 | - google.lk/imgres 1945 | - google.lt/imgres 1946 | - google.lu/imgres 1947 | - google.lv/imgres 1948 | - google.md/imgres 1949 | - google.me/imgres 1950 | - google.mg/imgres 1951 | - google.mk/imgres 1952 | - google.ml/imgres 1953 | - google.mn/imgres 1954 | - google.ms/imgres 1955 | - google.mu/imgres 1956 | - google.mv/imgres 1957 | - google.mw/imgres 1958 | - google.ne/imgres 1959 | - google.nl/imgres 1960 | - google.no/imgres 1961 | - google.nr/imgres 1962 | - google.nu/imgres 1963 | - google.pl/imgres 1964 | - google.pn/imgres 1965 | - google.ps/imgres 1966 | - google.pt/imgres 1967 | - google.ro/imgres 1968 | - google.rs/imgres 1969 | - google.ru/imgres 1970 | - google.rw/imgres 1971 | - google.sc/imgres 1972 | - google.se/imgres 1973 | - google.sh/imgres 1974 | - google.si/imgres 1975 | - google.sk/imgres 1976 | - google.sm/imgres 1977 | - google.sn/imgres 1978 | - google.so/imgres 1979 | - google.st/imgres 1980 | - google.td/imgres 1981 | - google.tg/imgres 1982 | - google.tk/imgres 1983 | - google.tl/imgres 1984 | - google.tm/imgres 1985 | - google.to/imgres 1986 | - google.tt/imgres 1987 | - google.us/imgres 1988 | - google.vg/imgres 1989 | - google.vu/imgres 1990 | - images.google.ws 1991 | - images.google.ac 1992 | - images.google.ad 1993 | - images.google.ae 1994 | - images.google.am 1995 | - images.google.as 1996 | - images.google.at 1997 | - images.google.az 1998 | - images.google.ba 1999 | - images.google.be 2000 | - images.google.bf 2001 | - images.google.bg 2002 | - images.google.bi 2003 | - images.google.bj 2004 | - images.google.bs 2005 | - images.google.by 2006 | - images.google.ca 2007 | - images.google.cat 2008 | - images.google.cc 2009 | - images.google.cd 2010 | - images.google.cf 2011 | - images.google.cg 2012 | - images.google.ch 2013 | - images.google.ci 2014 | - images.google.cl 2015 | - images.google.cm 2016 | - images.google.cn 2017 | - images.google.co.bw 2018 | - images.google.co.ck 2019 | - images.google.co.cr 2020 | - images.google.co.id 2021 | - images.google.co.il 2022 | - images.google.co.in 2023 | - images.google.co.jp 2024 | - images.google.co.ke 2025 | - images.google.co.kr 2026 | - images.google.co.ls 2027 | - images.google.co.ma 2028 | - images.google.co.mz 2029 | - images.google.co.nz 2030 | - images.google.co.th 2031 | - images.google.co.tz 2032 | - images.google.co.ug 2033 | - images.google.co.uk 2034 | - images.google.co.uz 2035 | - images.google.co.ve 2036 | - images.google.co.vi 2037 | - images.google.co.za 2038 | - images.google.co.zm 2039 | - images.google.co.zw 2040 | - images.google.com 2041 | - images.google.com.af 2042 | - images.google.com.ag 2043 | - images.google.com.ai 2044 | - images.google.com.ar 2045 | - images.google.com.au 2046 | - images.google.com.bd 2047 | - images.google.com.bh 2048 | - images.google.com.bn 2049 | - images.google.com.bo 2050 | - images.google.com.br 2051 | - images.google.com.by 2052 | - images.google.com.bz 2053 | - images.google.com.co 2054 | - images.google.com.cu 2055 | - images.google.com.cy 2056 | - images.google.com.do 2057 | - images.google.com.ec 2058 | - images.google.com.eg 2059 | - images.google.com.et 2060 | - images.google.com.fj 2061 | - images.google.com.gh 2062 | - images.google.com.gi 2063 | - images.google.com.gt 2064 | - images.google.com.hk 2065 | - images.google.com.jm 2066 | - images.google.com.kh 2067 | - images.google.com.kw 2068 | - images.google.com.lb 2069 | - images.google.com.lc 2070 | - images.google.com.ly 2071 | - images.google.com.mt 2072 | - images.google.com.mx 2073 | - images.google.com.my 2074 | - images.google.com.na 2075 | - images.google.com.nf 2076 | - images.google.com.ng 2077 | - images.google.com.ni 2078 | - images.google.com.np 2079 | - images.google.com.om 2080 | - images.google.com.pa 2081 | - images.google.com.pe 2082 | - images.google.com.ph 2083 | - images.google.com.pk 2084 | - images.google.com.pr 2085 | - images.google.com.py 2086 | - images.google.com.qa 2087 | - images.google.com.sa 2088 | - images.google.com.sb 2089 | - images.google.com.sg 2090 | - images.google.com.sl 2091 | - images.google.com.sv 2092 | - images.google.com.tj 2093 | - images.google.com.tn 2094 | - images.google.com.tr 2095 | - images.google.com.tw 2096 | - images.google.com.ua 2097 | - images.google.com.uy 2098 | - images.google.com.vc 2099 | - images.google.com.vn 2100 | - images.google.cv 2101 | - images.google.cz 2102 | - images.google.de 2103 | - images.google.dj 2104 | - images.google.dk 2105 | - images.google.dm 2106 | - images.google.dz 2107 | - images.google.ee 2108 | - images.google.es 2109 | - images.google.fi 2110 | - images.google.fm 2111 | - images.google.fr 2112 | - images.google.ga 2113 | - images.google.gd 2114 | - images.google.ge 2115 | - images.google.gf 2116 | - images.google.gg 2117 | - images.google.gl 2118 | - images.google.gm 2119 | - images.google.gp 2120 | - images.google.gr 2121 | - images.google.gy 2122 | - images.google.hn 2123 | - images.google.hr 2124 | - images.google.ht 2125 | - images.google.hu 2126 | - images.google.ie 2127 | - images.google.im 2128 | - images.google.io 2129 | - images.google.iq 2130 | - images.google.is 2131 | - images.google.it 2132 | - images.google.it.ao 2133 | - images.google.je 2134 | - images.google.jo 2135 | - images.google.kg 2136 | - images.google.ki 2137 | - images.google.kz 2138 | - images.google.la 2139 | - images.google.li 2140 | - images.google.lk 2141 | - images.google.lt 2142 | - images.google.lu 2143 | - images.google.lv 2144 | - images.google.md 2145 | - images.google.me 2146 | - images.google.mg 2147 | - images.google.mk 2148 | - images.google.ml 2149 | - images.google.mn 2150 | - images.google.ms 2151 | - images.google.mu 2152 | - images.google.mv 2153 | - images.google.mw 2154 | - images.google.ne 2155 | - images.google.nl 2156 | - images.google.no 2157 | - images.google.nr 2158 | - images.google.nu 2159 | - images.google.pl 2160 | - images.google.pn 2161 | - images.google.ps 2162 | - images.google.pt 2163 | - images.google.ro 2164 | - images.google.rs 2165 | - images.google.ru 2166 | - images.google.rw 2167 | - images.google.sc 2168 | - images.google.se 2169 | - images.google.sh 2170 | - images.google.si 2171 | - images.google.sk 2172 | - images.google.sm 2173 | - images.google.sn 2174 | - images.google.so 2175 | - images.google.st 2176 | - images.google.td 2177 | - images.google.tg 2178 | - images.google.tk 2179 | - images.google.tl 2180 | - images.google.tm 2181 | - images.google.to 2182 | - images.google.tt 2183 | - images.google.us 2184 | - images.google.vg 2185 | - images.google.vu 2186 | 2187 | Google News: 2188 | parameters: 2189 | - q 2190 | domains: 2191 | - news.google.ac 2192 | - news.google.ad 2193 | - news.google.ae 2194 | - news.google.am 2195 | - news.google.as 2196 | - news.google.at 2197 | - news.google.az 2198 | - news.google.ba 2199 | - news.google.be 2200 | - news.google.bf 2201 | - news.google.bg 2202 | - news.google.bi 2203 | - news.google.bj 2204 | - news.google.bs 2205 | - news.google.by 2206 | - news.google.ca 2207 | - news.google.cat 2208 | - news.google.cc 2209 | - news.google.cd 2210 | - news.google.cf 2211 | - news.google.cg 2212 | - news.google.ch 2213 | - news.google.ci 2214 | - news.google.cl 2215 | - news.google.cm 2216 | - news.google.cn 2217 | - news.google.co.bw 2218 | - news.google.co.ck 2219 | - news.google.co.cr 2220 | - news.google.co.id 2221 | - news.google.co.il 2222 | - news.google.co.in 2223 | - news.google.co.jp 2224 | - news.google.co.ke 2225 | - news.google.co.kr 2226 | - news.google.co.ls 2227 | - news.google.co.ma 2228 | - news.google.co.mz 2229 | - news.google.co.nz 2230 | - news.google.co.th 2231 | - news.google.co.tz 2232 | - news.google.co.ug 2233 | - news.google.co.uk 2234 | - news.google.co.uz 2235 | - news.google.co.ve 2236 | - news.google.co.vi 2237 | - news.google.co.za 2238 | - news.google.co.zm 2239 | - news.google.co.zw 2240 | - news.google.com 2241 | - news.google.com.af 2242 | - news.google.com.ag 2243 | - news.google.com.ai 2244 | - news.google.com.ar 2245 | - news.google.com.au 2246 | - news.google.com.bd 2247 | - news.google.com.bh 2248 | - news.google.com.bn 2249 | - news.google.com.bo 2250 | - news.google.com.br 2251 | - news.google.com.by 2252 | - news.google.com.bz 2253 | - news.google.com.co 2254 | - news.google.com.cu 2255 | - news.google.com.cy 2256 | - news.google.com.do 2257 | - news.google.com.ec 2258 | - news.google.com.eg 2259 | - news.google.com.et 2260 | - news.google.com.fj 2261 | - news.google.com.gh 2262 | - news.google.com.gi 2263 | - news.google.com.gt 2264 | - news.google.com.hk 2265 | - news.google.com.jm 2266 | - news.google.com.kh 2267 | - news.google.com.kw 2268 | - news.google.com.lb 2269 | - news.google.com.lc 2270 | - news.google.com.ly 2271 | - news.google.com.mt 2272 | - news.google.com.mx 2273 | - news.google.com.my 2274 | - news.google.com.na 2275 | - news.google.com.nf 2276 | - news.google.com.ng 2277 | - news.google.com.ni 2278 | - news.google.com.np 2279 | - news.google.com.om 2280 | - news.google.com.pa 2281 | - news.google.com.pe 2282 | - news.google.com.ph 2283 | - news.google.com.pk 2284 | - news.google.com.pr 2285 | - news.google.com.py 2286 | - news.google.com.qa 2287 | - news.google.com.sa 2288 | - news.google.com.sb 2289 | - news.google.com.sg 2290 | - news.google.com.sl 2291 | - news.google.com.sv 2292 | - news.google.com.tj 2293 | - news.google.com.tn 2294 | - news.google.com.tr 2295 | - news.google.com.tw 2296 | - news.google.com.ua 2297 | - news.google.com.uy 2298 | - news.google.com.vc 2299 | - news.google.com.vn 2300 | - news.google.cv 2301 | - news.google.cz 2302 | - news.google.de 2303 | - news.google.dj 2304 | - news.google.dk 2305 | - news.google.dm 2306 | - news.google.dz 2307 | - news.google.ee 2308 | - news.google.es 2309 | - news.google.fi 2310 | - news.google.fm 2311 | - news.google.fr 2312 | - news.google.ga 2313 | - news.google.gd 2314 | - news.google.ge 2315 | - news.google.gf 2316 | - news.google.gg 2317 | - news.google.gl 2318 | - news.google.gm 2319 | - news.google.gp 2320 | - news.google.gr 2321 | - news.google.gy 2322 | - news.google.hn 2323 | - news.google.hr 2324 | - news.google.ht 2325 | - news.google.hu 2326 | - news.google.ie 2327 | - news.google.im 2328 | - news.google.io 2329 | - news.google.iq 2330 | - news.google.is 2331 | - news.google.it 2332 | - news.google.it.ao 2333 | - news.google.je 2334 | - news.google.jo 2335 | - news.google.kg 2336 | - news.google.ki 2337 | - news.google.kz 2338 | - news.google.la 2339 | - news.google.li 2340 | - news.google.lk 2341 | - news.google.lt 2342 | - news.google.lu 2343 | - news.google.lv 2344 | - news.google.md 2345 | - news.google.me 2346 | - news.google.mg 2347 | - news.google.mk 2348 | - news.google.ml 2349 | - news.google.mn 2350 | - news.google.ms 2351 | - news.google.mu 2352 | - news.google.mv 2353 | - news.google.mw 2354 | - news.google.ne 2355 | - news.google.nl 2356 | - news.google.no 2357 | - news.google.nr 2358 | - news.google.nu 2359 | - news.google.pl 2360 | - news.google.pn 2361 | - news.google.ps 2362 | - news.google.pt 2363 | - news.google.ro 2364 | - news.google.rs 2365 | - news.google.ru 2366 | - news.google.rw 2367 | - news.google.sc 2368 | - news.google.se 2369 | - news.google.sh 2370 | - news.google.si 2371 | - news.google.sk 2372 | - news.google.sm 2373 | - news.google.sn 2374 | - news.google.so 2375 | - news.google.st 2376 | - news.google.td 2377 | - news.google.tg 2378 | - news.google.tk 2379 | - news.google.tl 2380 | - news.google.tm 2381 | - news.google.to 2382 | - news.google.tt 2383 | - news.google.us 2384 | - news.google.vg 2385 | - news.google.vu 2386 | - news.google.ws 2387 | 2388 | Google Product Search: 2389 | parameters: 2390 | - q 2391 | domains: 2392 | - google.ac/products 2393 | - google.ad/products 2394 | - google.ae/products 2395 | - google.am/products 2396 | - google.as/products 2397 | - google.at/products 2398 | - google.az/products 2399 | - google.ba/products 2400 | - google.be/products 2401 | - google.bf/products 2402 | - google.bg/products 2403 | - google.bi/products 2404 | - google.bj/products 2405 | - google.bs/products 2406 | - google.by/products 2407 | - google.ca/products 2408 | - google.cat/products 2409 | - google.cc/products 2410 | - google.cd/products 2411 | - google.cf/products 2412 | - google.cg/products 2413 | - google.ch/products 2414 | - google.ci/products 2415 | - google.cl/products 2416 | - google.cm/products 2417 | - google.cn/products 2418 | - google.co.bw/products 2419 | - google.co.ck/products 2420 | - google.co.cr/products 2421 | - google.co.id/products 2422 | - google.co.il/products 2423 | - google.co.in/products 2424 | - google.co.jp/products 2425 | - google.co.ke/products 2426 | - google.co.kr/products 2427 | - google.co.ls/products 2428 | - google.co.ma/products 2429 | - google.co.mz/products 2430 | - google.co.nz/products 2431 | - google.co.th/products 2432 | - google.co.tz/products 2433 | - google.co.ug/products 2434 | - google.co.uk/products 2435 | - google.co.uz/products 2436 | - google.co.ve/products 2437 | - google.co.vi/products 2438 | - google.co.za/products 2439 | - google.co.zm/products 2440 | - google.co.zw/products 2441 | - google.com/products 2442 | - google.com.af/products 2443 | - google.com.ag/products 2444 | - google.com.ai/products 2445 | - google.com.ar/products 2446 | - google.com.au/products 2447 | - google.com.bd/products 2448 | - google.com.bh/products 2449 | - google.com.bn/products 2450 | - google.com.bo/products 2451 | - google.com.br/products 2452 | - google.com.by/products 2453 | - google.com.bz/products 2454 | - google.com.co/products 2455 | - google.com.cu/products 2456 | - google.com.cy/products 2457 | - google.com.do/products 2458 | - google.com.ec/products 2459 | - google.com.eg/products 2460 | - google.com.et/products 2461 | - google.com.fj/products 2462 | - google.com.gh/products 2463 | - google.com.gi/products 2464 | - google.com.gt/products 2465 | - google.com.hk/products 2466 | - google.com.jm/products 2467 | - google.com.kh/products 2468 | - google.com.kw/products 2469 | - google.com.lb/products 2470 | - google.com.lc/products 2471 | - google.com.ly/products 2472 | - google.com.mt/products 2473 | - google.com.mx/products 2474 | - google.com.my/products 2475 | - google.com.na/products 2476 | - google.com.nf/products 2477 | - google.com.ng/products 2478 | - google.com.ni/products 2479 | - google.com.np/products 2480 | - google.com.om/products 2481 | - google.com.pa/products 2482 | - google.com.pe/products 2483 | - google.com.ph/products 2484 | - google.com.pk/products 2485 | - google.com.pr/products 2486 | - google.com.py/products 2487 | - google.com.qa/products 2488 | - google.com.sa/products 2489 | - google.com.sb/products 2490 | - google.com.sg/products 2491 | - google.com.sl/products 2492 | - google.com.sv/products 2493 | - google.com.tj/products 2494 | - google.com.tn/products 2495 | - google.com.tr/products 2496 | - google.com.tw/products 2497 | - google.com.ua/products 2498 | - google.com.uy/products 2499 | - google.com.vc/products 2500 | - google.com.vn/products 2501 | - google.cv/products 2502 | - google.cz/products 2503 | - google.de/products 2504 | - google.dj/products 2505 | - google.dk/products 2506 | - google.dm/products 2507 | - google.dz/products 2508 | - google.ee/products 2509 | - google.es/products 2510 | - google.fi/products 2511 | - google.fm/products 2512 | - google.fr/products 2513 | - google.ga/products 2514 | - google.gd/products 2515 | - google.ge/products 2516 | - google.gf/products 2517 | - google.gg/products 2518 | - google.gl/products 2519 | - google.gm/products 2520 | - google.gp/products 2521 | - google.gr/products 2522 | - google.gy/products 2523 | - google.hn/products 2524 | - google.hr/products 2525 | - google.ht/products 2526 | - google.hu/products 2527 | - google.ie/products 2528 | - google.im/products 2529 | - google.io/products 2530 | - google.iq/products 2531 | - google.is/products 2532 | - google.it/products 2533 | - google.it.ao/products 2534 | - google.je/products 2535 | - google.jo/products 2536 | - google.kg/products 2537 | - google.ki/products 2538 | - google.kz/products 2539 | - google.la/products 2540 | - google.li/products 2541 | - google.lk/products 2542 | - google.lt/products 2543 | - google.lu/products 2544 | - google.lv/products 2545 | - google.md/products 2546 | - google.me/products 2547 | - google.mg/products 2548 | - google.mk/products 2549 | - google.ml/products 2550 | - google.mn/products 2551 | - google.ms/products 2552 | - google.mu/products 2553 | - google.mv/products 2554 | - google.mw/products 2555 | - google.ne/products 2556 | - google.nl/products 2557 | - google.no/products 2558 | - google.nr/products 2559 | - google.nu/products 2560 | - google.pl/products 2561 | - google.pn/products 2562 | - google.ps/products 2563 | - google.pt/products 2564 | - google.ro/products 2565 | - google.rs/products 2566 | - google.ru/products 2567 | - google.rw/products 2568 | - google.sc/products 2569 | - google.se/products 2570 | - google.sh/products 2571 | - google.si/products 2572 | - google.sk/products 2573 | - google.sm/products 2574 | - google.sn/products 2575 | - google.so/products 2576 | - google.st/products 2577 | - google.td/products 2578 | - google.tg/products 2579 | - google.tk/products 2580 | - google.tl/products 2581 | - google.tm/products 2582 | - google.to/products 2583 | - google.tt/products 2584 | - google.us/products 2585 | - google.vg/products 2586 | - google.vu/products 2587 | - google.ws/products 2588 | - www.google.ac/products 2589 | - www.google.ad/products 2590 | - www.google.ae/products 2591 | - www.google.am/products 2592 | - www.google.as/products 2593 | - www.google.at/products 2594 | - www.google.az/products 2595 | - www.google.ba/products 2596 | - www.google.be/products 2597 | - www.google.bf/products 2598 | - www.google.bg/products 2599 | - www.google.bi/products 2600 | - www.google.bj/products 2601 | - www.google.bs/products 2602 | - www.google.by/products 2603 | - www.google.ca/products 2604 | - www.google.cat/products 2605 | - www.google.cc/products 2606 | - www.google.cd/products 2607 | - www.google.cf/products 2608 | - www.google.cg/products 2609 | - www.google.ch/products 2610 | - www.google.ci/products 2611 | - www.google.cl/products 2612 | - www.google.cm/products 2613 | - www.google.cn/products 2614 | - www.google.co.bw/products 2615 | - www.google.co.ck/products 2616 | - www.google.co.cr/products 2617 | - www.google.co.id/products 2618 | - www.google.co.il/products 2619 | - www.google.co.in/products 2620 | - www.google.co.jp/products 2621 | - www.google.co.ke/products 2622 | - www.google.co.kr/products 2623 | - www.google.co.ls/products 2624 | - www.google.co.ma/products 2625 | - www.google.co.mz/products 2626 | - www.google.co.nz/products 2627 | - www.google.co.th/products 2628 | - www.google.co.tz/products 2629 | - www.google.co.ug/products 2630 | - www.google.co.uk/products 2631 | - www.google.co.uz/products 2632 | - www.google.co.ve/products 2633 | - www.google.co.vi/products 2634 | - www.google.co.za/products 2635 | - www.google.co.zm/products 2636 | - www.google.co.zw/products 2637 | - www.google.com/products 2638 | - www.google.com.af/products 2639 | - www.google.com.ag/products 2640 | - www.google.com.ai/products 2641 | - www.google.com.ar/products 2642 | - www.google.com.au/products 2643 | - www.google.com.bd/products 2644 | - www.google.com.bh/products 2645 | - www.google.com.bn/products 2646 | - www.google.com.bo/products 2647 | - www.google.com.br/products 2648 | - www.google.com.by/products 2649 | - www.google.com.bz/products 2650 | - www.google.com.co/products 2651 | - www.google.com.cu/products 2652 | - www.google.com.cy/products 2653 | - www.google.com.do/products 2654 | - www.google.com.ec/products 2655 | - www.google.com.eg/products 2656 | - www.google.com.et/products 2657 | - www.google.com.fj/products 2658 | - www.google.com.gh/products 2659 | - www.google.com.gi/products 2660 | - www.google.com.gt/products 2661 | - www.google.com.hk/products 2662 | - www.google.com.jm/products 2663 | - www.google.com.kh/products 2664 | - www.google.com.kw/products 2665 | - www.google.com.lb/products 2666 | - www.google.com.lc/products 2667 | - www.google.com.ly/products 2668 | - www.google.com.mt/products 2669 | - www.google.com.mx/products 2670 | - www.google.com.my/products 2671 | - www.google.com.na/products 2672 | - www.google.com.nf/products 2673 | - www.google.com.ng/products 2674 | - www.google.com.ni/products 2675 | - www.google.com.np/products 2676 | - www.google.com.om/products 2677 | - www.google.com.pa/products 2678 | - www.google.com.pe/products 2679 | - www.google.com.ph/products 2680 | - www.google.com.pk/products 2681 | - www.google.com.pr/products 2682 | - www.google.com.py/products 2683 | - www.google.com.qa/products 2684 | - www.google.com.sa/products 2685 | - www.google.com.sb/products 2686 | - www.google.com.sg/products 2687 | - www.google.com.sl/products 2688 | - www.google.com.sv/products 2689 | - www.google.com.tj/products 2690 | - www.google.com.tn/products 2691 | - www.google.com.tr/products 2692 | - www.google.com.tw/products 2693 | - www.google.com.ua/products 2694 | - www.google.com.uy/products 2695 | - www.google.com.vc/products 2696 | - www.google.com.vn/products 2697 | - www.google.cv/products 2698 | - www.google.cz/products 2699 | - www.google.de/products 2700 | - www.google.dj/products 2701 | - www.google.dk/products 2702 | - www.google.dm/products 2703 | - www.google.dz/products 2704 | - www.google.ee/products 2705 | - www.google.es/products 2706 | - www.google.fi/products 2707 | - www.google.fm/products 2708 | - www.google.fr/products 2709 | - www.google.ga/products 2710 | - www.google.gd/products 2711 | - www.google.ge/products 2712 | - www.google.gf/products 2713 | - www.google.gg/products 2714 | - www.google.gl/products 2715 | - www.google.gm/products 2716 | - www.google.gp/products 2717 | - www.google.gr/products 2718 | - www.google.gy/products 2719 | - www.google.hn/products 2720 | - www.google.hr/products 2721 | - www.google.ht/products 2722 | - www.google.hu/products 2723 | - www.google.ie/products 2724 | - www.google.im/products 2725 | - www.google.io/products 2726 | - www.google.iq/products 2727 | - www.google.is/products 2728 | - www.google.it/products 2729 | - www.google.it.ao/products 2730 | - www.google.je/products 2731 | - www.google.jo/products 2732 | - www.google.kg/products 2733 | - www.google.ki/products 2734 | - www.google.kz/products 2735 | - www.google.la/products 2736 | - www.google.li/products 2737 | - www.google.lk/products 2738 | - www.google.lt/products 2739 | - www.google.lu/products 2740 | - www.google.lv/products 2741 | - www.google.md/products 2742 | - www.google.me/products 2743 | - www.google.mg/products 2744 | - www.google.mk/products 2745 | - www.google.ml/products 2746 | - www.google.mn/products 2747 | - www.google.ms/products 2748 | - www.google.mu/products 2749 | - www.google.mv/products 2750 | - www.google.mw/products 2751 | - www.google.ne/products 2752 | - www.google.nl/products 2753 | - www.google.no/products 2754 | - www.google.nr/products 2755 | - www.google.nu/products 2756 | - www.google.pl/products 2757 | - www.google.pn/products 2758 | - www.google.ps/products 2759 | - www.google.pt/products 2760 | - www.google.ro/products 2761 | - www.google.rs/products 2762 | - www.google.ru/products 2763 | - www.google.rw/products 2764 | - www.google.sc/products 2765 | - www.google.se/products 2766 | - www.google.sh/products 2767 | - www.google.si/products 2768 | - www.google.sk/products 2769 | - www.google.sm/products 2770 | - www.google.sn/products 2771 | - www.google.so/products 2772 | - www.google.st/products 2773 | - www.google.td/products 2774 | - www.google.tg/products 2775 | - www.google.tk/products 2776 | - www.google.tl/products 2777 | - www.google.tm/products 2778 | - www.google.to/products 2779 | - www.google.tt/products 2780 | - www.google.us/products 2781 | - www.google.vg/products 2782 | - www.google.vu/products 2783 | - www.google.ws/products 2784 | 2785 | Google Video: 2786 | parameters: 2787 | - q 2788 | domains: 2789 | - video.google.com 2790 | 2791 | Goyellow.de: 2792 | parameters: 2793 | - MDN 2794 | domains: 2795 | - www.goyellow.de 2796 | 2797 | Gule Sider: 2798 | parameters: 2799 | - q 2800 | domains: 2801 | - www.gulesider.no 2802 | 2803 | HighBeam: 2804 | parameters: 2805 | - q 2806 | domains: 2807 | - www.highbeam.com 2808 | 2809 | Hit-Parade: 2810 | parameters: 2811 | - p7 2812 | domains: 2813 | - req.-hit-parade.com 2814 | - class.hit-parade.com 2815 | - www.hit-parade.com 2816 | 2817 | Holmes: 2818 | parameters: 2819 | - q 2820 | domains: 2821 | - holmes.ge 2822 | 2823 | Hooseek.com: 2824 | parameters: 2825 | - recherche 2826 | domains: 2827 | - www.hooseek.com 2828 | 2829 | Hotbot: 2830 | parameters: 2831 | - query 2832 | domains: 2833 | - www.hotbot.com 2834 | 2835 | Icerockeet: 2836 | parameters: 2837 | - q 2838 | domains: 2839 | - blogs.icerocket.com 2840 | 2841 | ICQ: 2842 | parameters: 2843 | - q 2844 | domains: 2845 | - www.icq.com 2846 | - search.icq.com 2847 | 2848 | Ilse: 2849 | parameters: 2850 | - search_for 2851 | domains: 2852 | - www.ilse.nl 2853 | 2854 | Inbox.com: 2855 | parameters: 2856 | - q 2857 | domains: 2858 | - inbox.com/search/ 2859 | 2860 | InfoSpace: 2861 | parameters: 2862 | - q 2863 | - s 2864 | domains: 2865 | - infospace.com 2866 | - dogpile.com 2867 | - www.dogpile.com 2868 | - metacrawler.com 2869 | - webfetch.com 2870 | - webcrawler.com 2871 | - search.kiwee.com 2872 | # powered by InfoSpace 2873 | - isearch.babylon.com 2874 | - start.facemoods.com 2875 | - search.magnetic.com 2876 | - search.searchcompletion.com 2877 | - clusty.com 2878 | 2879 | Flyingbird: 2880 | parameters: 2881 | - q 2882 | domains: 2883 | - inspsearch.com 2884 | - viview.inspsearch.com 2885 | 2886 | Interia: 2887 | parameters: 2888 | - q 2889 | domains: 2890 | - www.google.interia.pl 2891 | 2892 | I-play: 2893 | parameters: 2894 | - q 2895 | domains: 2896 | - start.iplay.com 2897 | 2898 | I.ua: 2899 | parameters: 2900 | - q 2901 | domains: 2902 | - search.i.ua 2903 | 2904 | IXquick: 2905 | parameters: 2906 | - query 2907 | domains: 2908 | - ixquick.com 2909 | - www.eu.ixquick.com 2910 | - ixquick.de 2911 | - www.ixquick.de 2912 | - us.ixquick.com 2913 | - s1.us.ixquick.com 2914 | - s2.us.ixquick.com 2915 | - s3.us.ixquick.com 2916 | - s4.us.ixquick.com 2917 | - s5.us.ixquick.com 2918 | - eu.ixquick.com 2919 | - s8-eu.ixquick.com 2920 | - s1-eu.ixquick.de 2921 | 2922 | Jyxo: 2923 | parameters: 2924 | - q 2925 | domains: 2926 | - jyxo.1188.cz 2927 | 2928 | Jungle Spider: 2929 | parameters: 2930 | - q 2931 | domains: 2932 | - www.jungle-spider.de 2933 | 2934 | Jungle Key: 2935 | parameters: 2936 | - query 2937 | domains: 2938 | - junglekey.com 2939 | - junglekey.fr 2940 | 2941 | Kataweb: 2942 | parameters: 2943 | - q 2944 | domains: 2945 | - www.kataweb.it 2946 | 2947 | Kvasir: 2948 | parameters: 2949 | - q 2950 | domains: 2951 | - www.kvasir.no 2952 | 2953 | kununu: 2954 | parameters: 2955 | - q 2956 | domains: 2957 | - kununu.com 2958 | 2959 | Latne: 2960 | parameters: 2961 | - q 2962 | domains: 2963 | - www.latne.lv 2964 | 2965 | La Toile Du Quebec Via Google: 2966 | parameters: 2967 | - q 2968 | domains: 2969 | - www.toile.com 2970 | - web.toile.com 2971 | 2972 | Looksmart: 2973 | parameters: 2974 | - key 2975 | domains: 2976 | - www.looksmart.com 2977 | 2978 | Lo.st: 2979 | parameters: 2980 | - x_query 2981 | domains: 2982 | - lo.st 2983 | 2984 | Lycos: 2985 | parameters: 2986 | - query 2987 | domains: 2988 | - search.lycos.com 2989 | - www.lycos.com 2990 | - lycos.com 2991 | 2992 | maailm: 2993 | parameters: 2994 | - tekst 2995 | domains: 2996 | - www.maailm.com 2997 | 2998 | Mail.ru: 2999 | parameters: 3000 | - q 3001 | domains: 3002 | - go.mail.ru 3003 | 3004 | Mamma: 3005 | parameters: 3006 | - query 3007 | domains: 3008 | - www.mamma.com 3009 | - mamma75.mamma.com 3010 | 3011 | Marktplaats: 3012 | parameters: 3013 | - query 3014 | domains: 3015 | - www.marktplaats.nl 3016 | 3017 | Maxwebsearch: 3018 | parameters: 3019 | - query 3020 | domains: 3021 | - maxwebsearch.com 3022 | 3023 | Meta: 3024 | parameters: 3025 | - q 3026 | domains: 3027 | - meta.ua 3028 | 3029 | MetaCrawler.de: 3030 | parameters: 3031 | - qry 3032 | domains: 3033 | - s1.metacrawler.de 3034 | - s2.metacrawler.de 3035 | - s3.metacrawler.de 3036 | 3037 | Metager: 3038 | parameters: 3039 | - eingabe 3040 | domains: 3041 | - meta.rrzn.uni-hannover.de 3042 | - www.metager.de 3043 | 3044 | Metager2: 3045 | parameters: 3046 | - q 3047 | domains: 3048 | - metager2.de 3049 | 3050 | Meinestadt: 3051 | parameters: 3052 | - words 3053 | domains: 3054 | - www.meinestadt.de 3055 | 3056 | Mister Wong: 3057 | parameters: 3058 | - Keywords 3059 | domains: 3060 | - www.mister-wong.com 3061 | - www.mister-wong.de 3062 | 3063 | Monstercrawler: 3064 | parameters: 3065 | - qry 3066 | domains: 3067 | - www.monstercrawler.com 3068 | 3069 | Mozbot: 3070 | parameters: 3071 | - q 3072 | domains: 3073 | - www.mozbot.fr 3074 | - www.mozbot.co.uk 3075 | - www.mozbot.com 3076 | 3077 | El Mundo: 3078 | parameters: 3079 | - q 3080 | domains: 3081 | - ariadna.elmundo.es 3082 | 3083 | MySearch: 3084 | parameters: 3085 | - searchfor 3086 | - searchFor 3087 | domains: 3088 | - www.mysearch.com 3089 | - ms114.mysearch.com 3090 | - ms146.mysearch.com 3091 | - kf.mysearch.myway.com 3092 | - ki.mysearch.myway.com 3093 | - search.myway.com 3094 | - search.mywebsearch.com 3095 | 3096 | Najdi: 3097 | parameters: 3098 | - q 3099 | domains: 3100 | - www.najdi.si 3101 | 3102 | Nate: 3103 | parameters: 3104 | - q 3105 | domains: 3106 | - search.nate.com 3107 | 3108 | Naver: 3109 | parameters: 3110 | - query 3111 | domains: 3112 | - search.naver.com 3113 | 3114 | Naver Images: 3115 | parameters: 3116 | - query 3117 | domains: 3118 | - image.search.naver.com 3119 | - imagesearch.naver.com 3120 | 3121 | Needtofind: 3122 | parameters: 3123 | - searchfor 3124 | domains: 3125 | - ko.search.need2find.com 3126 | 3127 | Neti: 3128 | parameters: 3129 | - query 3130 | domains: 3131 | - www.neti.ee 3132 | 3133 | Nifty: 3134 | parameters: 3135 | - q 3136 | domains: 3137 | - search.nifty.com 3138 | 3139 | Nigma: 3140 | parameters: 3141 | - s 3142 | domains: 3143 | - nigma.ru 3144 | 3145 | Onet: 3146 | parameters: 3147 | - qt 3148 | domains: 3149 | - szukaj.onet.pl 3150 | 3151 | Online.no: 3152 | parameters: 3153 | - q 3154 | domains: 3155 | - online.no 3156 | 3157 | Opplysningen 1881: 3158 | parameters: 3159 | - Query 3160 | domains: 3161 | - www.1881.no 3162 | 3163 | Orange: 3164 | parameters: 3165 | - q 3166 | - kw 3167 | domains: 3168 | - busca.orange.es 3169 | - search.orange.co.uk 3170 | - lemoteur.orange.fr 3171 | 3172 | Paperball: 3173 | parameters: 3174 | - q 3175 | domains: 3176 | - www.paperball.de 3177 | 3178 | PeoplePC: 3179 | parameters: 3180 | - q 3181 | domains: 3182 | - search.peoplepc.com 3183 | 3184 | Picsearch: 3185 | parameters: 3186 | - q 3187 | domains: 3188 | - www.picsearch.com 3189 | 3190 | Plazoo: 3191 | parameters: 3192 | - q 3193 | domains: 3194 | - www.plazoo.com 3195 | 3196 | Poisk.ru: 3197 | parameters: 3198 | - q 3199 | domains: 3200 | - poisk.ru 3201 | 3202 | PriceRunner: 3203 | parameters: 3204 | - q 3205 | domains: 3206 | - www.pricerunner.co.uk 3207 | 3208 | qip: 3209 | parameters: 3210 | - query 3211 | domains: 3212 | - search.qip.ru 3213 | 3214 | Qualigo: 3215 | parameters: 3216 | - q 3217 | domains: 3218 | - www.qualigo.at 3219 | - www.qualigo.ch 3220 | - www.qualigo.de 3221 | - www.qualigo.nl 3222 | 3223 | Rakuten: 3224 | parameters: 3225 | - qt 3226 | domains: 3227 | - websearch.rakuten.co.jp 3228 | 3229 | Rambler: 3230 | parameters: 3231 | - query 3232 | - words 3233 | domains: 3234 | - nova.rambler.ru 3235 | 3236 | RPMFind: 3237 | parameters: 3238 | - query 3239 | domains: 3240 | - rpmfind.net 3241 | - fr2.rpmfind.net 3242 | 3243 | Road Runner Search: 3244 | parameters: 3245 | - q 3246 | domains: 3247 | - search.rr.com 3248 | 3249 | Sapo: 3250 | parameters: 3251 | - q 3252 | domains: 3253 | - pesquisa.sapo.pt 3254 | 3255 | # Add Scour.com 3256 | 3257 | Search This: 3258 | parameters: 3259 | - q 3260 | domains: 3261 | - www.searchthis.com 3262 | 3263 | Search.com: 3264 | parameters: 3265 | - q 3266 | domains: 3267 | - www.search.com 3268 | 3269 | Search.ch: 3270 | parameters: 3271 | - q 3272 | domains: 3273 | - www.search.ch 3274 | 3275 | Searchalot: 3276 | parameters: 3277 | - q 3278 | domains: 3279 | - searchalot.com 3280 | 3281 | SearchCanvas: 3282 | parameters: 3283 | - q 3284 | domains: 3285 | - www.searchcanvas.com 3286 | 3287 | Searchy: 3288 | parameters: 3289 | - q 3290 | domains: 3291 | - www.searchy.co.uk 3292 | 3293 | # Add setooz.com 3294 | 3295 | Seznam: 3296 | parameters: 3297 | - q 3298 | domains: 3299 | - search.seznam.cz 3300 | 3301 | Sharelook: 3302 | parameters: 3303 | - keyword 3304 | domains: 3305 | - www.sharelook.fr 3306 | 3307 | Skynet: 3308 | parameters: 3309 | - q 3310 | domains: 3311 | - www.skynet.be 3312 | 3313 | The Smart Search: 3314 | parameters: 3315 | - q 3316 | domains: 3317 | - thesmartsearch.net 3318 | - www.thesmartsearch.net 3319 | 3320 | Sogou: 3321 | parameters: 3322 | - query 3323 | - w 3324 | domains: 3325 | - www.sougou.com 3326 | - www.soso.com 3327 | 3328 | Softonic: 3329 | parameters: 3330 | - q 3331 | domains: 3332 | - search.softonic.com 3333 | 3334 | SoSoDesk: 3335 | parameters: 3336 | - q 3337 | domains: 3338 | - sosodesktop.com 3339 | - search.sosodesktop.com 3340 | 3341 | Snapdo: 3342 | parameters: 3343 | - q 3344 | domains: 3345 | - search.snapdo.com 3346 | 3347 | Startpagina: 3348 | parameters: 3349 | - q 3350 | domains: 3351 | - startgoogle.startpagina.nl 3352 | 3353 | Startsiden: 3354 | parameters: 3355 | - q 3356 | domains: 3357 | - www.startsiden.no 3358 | 3359 | suche.info: 3360 | parameters: 3361 | - q 3362 | domains: 3363 | - suche.info 3364 | 3365 | Suchmaschine.com: 3366 | parameters: 3367 | - suchstr 3368 | domains: 3369 | - www.suchmaschine.com 3370 | 3371 | Suchnase: 3372 | parameters: 3373 | - q 3374 | domains: 3375 | - www.suchnase.de 3376 | 3377 | TalkTalk: 3378 | parameters: 3379 | - query 3380 | domains: 3381 | - www.talktalk.co.uk 3382 | 3383 | Technorati: 3384 | parameters: 3385 | - q 3386 | domains: 3387 | - technorati.com 3388 | 3389 | Telstra: 3390 | parameters: 3391 | - find 3392 | domains: 3393 | - search.media.telstra.com.au 3394 | 3395 | Teoma: 3396 | parameters: 3397 | - q 3398 | domains: 3399 | - www.teoma.com 3400 | 3401 | Terra: 3402 | parameters: 3403 | - query 3404 | domains: 3405 | - buscador.terra.es 3406 | - buscador.terra.cl 3407 | - buscador.terra.com.br 3408 | 3409 | Tiscali: 3410 | parameters: 3411 | - q 3412 | - key 3413 | domains: 3414 | - search.tiscali.it 3415 | - search-dyn.tiscali.it 3416 | - hledani.tiscali.cz 3417 | 3418 | Tixuma: 3419 | parameters: 3420 | - sc 3421 | domains: 3422 | - www.tixuma.de 3423 | 3424 | T-Online: 3425 | parameters: 3426 | - q 3427 | domains: 3428 | - suche.t-online.de 3429 | - brisbane.t-online.de 3430 | - navigationshilfe.t-online.de 3431 | 3432 | Toolbarhome: 3433 | parameters: 3434 | - q 3435 | domains: 3436 | - www.toolbarhome.com 3437 | - vshare.toolbarhome.com 3438 | 3439 | Trouvez.com: 3440 | parameters: 3441 | - query 3442 | domains: 3443 | - www.trouvez.com 3444 | 3445 | TrovaRapido: 3446 | parameters: 3447 | - q 3448 | domains: 3449 | - www.trovarapido.com 3450 | 3451 | Trusted-Search: 3452 | parameters: 3453 | - w 3454 | domains: 3455 | - www.trusted--search.com 3456 | 3457 | Tut.by: 3458 | parameters: 3459 | - query 3460 | domains: 3461 | - search.tut.by 3462 | 3463 | Twingly: 3464 | parameters: 3465 | - q 3466 | domains: 3467 | - www.twingly.com 3468 | 3469 | UKR.net: 3470 | parameters: 3471 | - q 3472 | domains: 3473 | - search.ukr.net 3474 | 3475 | uol.com.br: 3476 | parameters: 3477 | - q 3478 | domains: 3479 | - busca.uol.com.br 3480 | 3481 | URL.ORGanizier: 3482 | parameters: 3483 | - q 3484 | domains: 3485 | - www.url.org 3486 | 3487 | Vinden: 3488 | parameters: 3489 | - q 3490 | domains: 3491 | - www.vinden.nl 3492 | 3493 | Vindex: 3494 | parameters: 3495 | - search_for 3496 | domains: 3497 | - www.vindex.nl 3498 | - search.vindex.nl 3499 | 3500 | Virgilio: 3501 | parameters: 3502 | - qs 3503 | domains: 3504 | - ricerca.virgilio.it 3505 | - ricercaimmagini.virgilio.it 3506 | - ricercavideo.virgilio.it 3507 | - ricercanews.virgilio.it 3508 | - mobile.virgilio.it 3509 | 3510 | Voila: 3511 | parameters: 3512 | - rdata 3513 | - kw 3514 | domains: 3515 | - search.ke.voila.fr 3516 | - www.lemoteur.fr 3517 | 3518 | Volny: 3519 | parameters: 3520 | - search 3521 | domains: 3522 | - web.volny.cz 3523 | 3524 | Walhello : 3525 | parameters: 3526 | - key 3527 | domains: 3528 | - www.walhello.info 3529 | - www.walhello.com 3530 | - www.walhello.de 3531 | - www.walhello.nl 3532 | 3533 | Web.de: 3534 | parameters: 3535 | - su 3536 | domains: 3537 | - suche.web.de 3538 | 3539 | Web.nl: 3540 | parameters: 3541 | - zoekwoord 3542 | domains: 3543 | - www.web.nl 3544 | 3545 | Weborama: 3546 | parameters: 3547 | - QUERY 3548 | domains: 3549 | - www.weborama.com 3550 | 3551 | WebSearch: 3552 | parameters: 3553 | - qkw 3554 | - q 3555 | domains: 3556 | - www.websearch.com 3557 | 3558 | Winamp: 3559 | parameters: 3560 | - q 3561 | domains: 3562 | - search.winamp.com 3563 | 3564 | Witch: 3565 | parameters: 3566 | - search 3567 | domains: 3568 | - www.witch.de 3569 | 3570 | Wirtualna Polska: 3571 | parameters: 3572 | - szukaj 3573 | domains: 3574 | - szukaj.wp.pl 3575 | 3576 | WWW: 3577 | parameters: 3578 | - query 3579 | domains: 3580 | - search.www.ee 3581 | 3582 | X-recherche: 3583 | parameters: 3584 | - MOTS 3585 | domains: 3586 | - www.x-recherche.com 3587 | 3588 | Yahoo!: 3589 | parameters: 3590 | - p 3591 | - q 3592 | domains: 3593 | - search.yahoo.com 3594 | - yahoo.com 3595 | - ar.search.yahoo.com 3596 | - ar.yahoo.com 3597 | - au.search.yahoo.com 3598 | - au.yahoo.com 3599 | - br.search.yahoo.com 3600 | - br.yahoo.com 3601 | - cade.searchde.yahoo.com 3602 | - cade.yahoo.com 3603 | - chinese.searchinese.yahoo.com 3604 | - chinese.yahoo.com 3605 | - cn.search.yahoo.com 3606 | - cn.yahoo.com 3607 | - de.search.yahoo.com 3608 | - de.yahoo.com 3609 | - dk.search.yahoo.com 3610 | - dk.yahoo.com 3611 | - es.search.yahoo.com 3612 | - es.yahoo.com 3613 | - espanol.searchpanol.yahoo.com 3614 | - espanol.yahoo.com 3615 | - fr.search.yahoo.com 3616 | - fr.yahoo.com 3617 | - ie.search.yahoo.com 3618 | - ie.yahoo.com 3619 | - it.search.yahoo.com 3620 | - it.yahoo.com 3621 | - kr.search.yahoo.com 3622 | - kr.yahoo.com 3623 | - mx.search.yahoo.com 3624 | - mx.yahoo.com 3625 | - no.search.yahoo.com 3626 | - no.yahoo.com 3627 | - nz.search.yahoo.com 3628 | - nz.yahoo.com 3629 | - one.cn.yahoo.com 3630 | - one.searchn.yahoo.com 3631 | - qc.search.yahoo.com 3632 | - qc.yahoo.com 3633 | - se.search.yahoo.com 3634 | - se.yahoo.com 3635 | - search.searcharch.yahoo.com 3636 | - uk.search.yahoo.com 3637 | - uk.yahoo.com 3638 | - www.yahoo.co.jp 3639 | - search.yahoo.co.jp 3640 | # powered by Yahoo 3641 | - www.cercato.it 3642 | - search.offerbox.com 3643 | - ys.mirostart.com 3644 | 3645 | Yahoo! Images: 3646 | parameters: 3647 | - p 3648 | - q 3649 | domains: 3650 | - image.yahoo.cn 3651 | - images.search.yahoo.com 3652 | 3653 | Yam: 3654 | parameters: 3655 | - k 3656 | domains: 3657 | - search.yam.com 3658 | 3659 | Yandex: 3660 | parameters: 3661 | - text 3662 | domains: 3663 | - yandex.ru 3664 | - yandex.ua 3665 | - yandex.com 3666 | - yandex.by 3667 | - www.yandex.ru 3668 | - www.yandex.ua 3669 | - www.yandex.com 3670 | - www.yandex.by 3671 | 3672 | Yandex Images: 3673 | parameters: 3674 | - text 3675 | domains: 3676 | - images.yandex.ru 3677 | - images.yandex.ua 3678 | - images.yandex.com 3679 | 3680 | Yasni: 3681 | parameters: 3682 | - query 3683 | domains: 3684 | - www.yasni.de 3685 | - www.yasni.com 3686 | - www.yasni.co.uk 3687 | - www.yasni.ch 3688 | - www.yasni.at 3689 | 3690 | Yatedo: 3691 | parameters: 3692 | - q 3693 | domains: 3694 | - www.yatedo.com 3695 | - www.yatedo.fr 3696 | 3697 | # Add Yellowmap: 3698 | 3699 | Yippy: 3700 | parameters: 3701 | - q 3702 | - query 3703 | domains: 3704 | - search.yippy.com 3705 | 3706 | YouGoo: 3707 | parameters: 3708 | - q 3709 | domains: 3710 | - www.yougoo.fr 3711 | 3712 | Zapmeta: 3713 | parameters: 3714 | - q 3715 | - query 3716 | domains: 3717 | - www.zapmeta.com 3718 | - www.zapmeta.nl 3719 | - www.zapmeta.de 3720 | - uk.zapmeta.com 3721 | 3722 | Zoek: 3723 | parameters: 3724 | - q 3725 | domains: 3726 | - www3.zoek.nl 3727 | 3728 | Zhongsou: 3729 | parameters: 3730 | - w 3731 | domains: 3732 | - p.zhongsou.com 3733 | 3734 | Zoeken: 3735 | parameters: 3736 | - q 3737 | domains: 3738 | - www.zoeken.nl 3739 | 3740 | Zoohoo: 3741 | parameters: 3742 | - q 3743 | domains: 3744 | - zoohoo.cz 3745 | 3746 | 3747 | 3748 | # ####################################################################################################### 3749 | # 3750 | # PAID MEDIA 3751 | 3752 | paid: 3753 | 3754 | Acuity Ads: 3755 | domains: 3756 | - acuityplatform.com 3757 | 3758 | Adform: 3759 | domains: 3760 | - adform.net 3761 | 3762 | Adition: 3763 | domains: 3764 | - adition.com 3765 | 3766 | AdNET: 3767 | domains: 3768 | - adnet.de 3769 | 3770 | AdRoll: 3771 | domains: 3772 | - adroll.com 3773 | 3774 | AdSpirit: 3775 | domains: 3776 | - adspirit.de 3777 | - rtbcity.com 3778 | - plusperformance.com 3779 | 3780 | AppNexus: 3781 | domains: 3782 | - ib.adnxs.com 3783 | - adnxs.com 3784 | - 247realmedia.com 3785 | 3786 | AudienceScience: 3787 | domains: 3788 | - wunderloop.net 3789 | 3790 | BidSwitch: 3791 | domains: 3792 | - bidswitch.net 3793 | 3794 | Casale Media: 3795 | domains: 3796 | - casalemedia.com 3797 | 3798 | Criteo: 3799 | domains: 3800 | - cas.jp.as.criteo.com 3801 | - cas.criteo.com 3802 | 3803 | Doubleclick: 3804 | domains: 3805 | - ad.doubleclick.net 3806 | - ad-apac.doubleclick.net 3807 | - s0.2mdn.net 3808 | - s1.2mdn.net 3809 | - dp.g.doubleclick.net 3810 | - pubads.g.doubleclick.net 3811 | 3812 | Eyeota: 3813 | domains: 3814 | - eyeota.net 3815 | 3816 | Flashtalking: 3817 | domains: 3818 | - flashtalking.com 3819 | - servedby.flashtalking.com 3820 | 3821 | Fluct: 3822 | domains: 3823 | - adingo.jp 3824 | 3825 | Google: 3826 | domains: 3827 | - www.googleadservices.com 3828 | - partner.googleadservices.com 3829 | - googleads.g.doubleclick.net 3830 | - tpc.googlesyndication.com 3831 | - googleadservices.com 3832 | - imasdk.googleapis.com 3833 | 3834 | LifeStreet: 3835 | domains: 3836 | - lfstmedia.com 3837 | 3838 | Jivox: 3839 | domains: 3840 | - jivox.com 3841 | 3842 | MicroAd: 3843 | domains: 3844 | - microad.jp 3845 | 3846 | Mixpo: 3847 | domains: 3848 | - mixpo.com 3849 | 3850 | Mozo: 3851 | domains: 3852 | - mozo.com.au 3853 | - a.mozo.com.au 3854 | 3855 | Neustar AdAdvisor: 3856 | domains: 3857 | - adadvisor.net 3858 | 3859 | ONE by AOL: 3860 | domains: 3861 | - nexage.com 3862 | 3863 | OpenX: 3864 | domains: 3865 | - us-ads.openx.net 3866 | - openx.net 3867 | - servedbyopenx.com 3868 | - openxenterprise.com 3869 | 3870 | Outbrain: 3871 | domains: 3872 | - paid.outbrain.com 3873 | 3874 | Plista: 3875 | domains: 3876 | - farm.plista.com 3877 | 3878 | PubMatic: 3879 | domains: 3880 | - sshowads.pubmatic.com 3881 | 3882 | Rubicon Project: 3883 | domains: 3884 | - optimized-by.rubiconproject.com 3885 | 3886 | Sizmek: 3887 | domains: 3888 | - bs.serving-sys.com 3889 | 3890 | Sociomantic Labs: 3891 | domains: 3892 | - sociomantic.com 3893 | 3894 | Sonobi: 3895 | domains: 3896 | - sonobi.com 3897 | 3898 | Sovrn: 3899 | domains: 3900 | - lijit.com 3901 | 3902 | SteelHouse: 3903 | domains: 3904 | - steelhousemedia.com 3905 | 3906 | StickyADS.tv: 3907 | domains: 3908 | - stickyadstv.com 3909 | - sfx.stickyadstv.com 3910 | 3911 | Taboola: 3912 | domains: 3913 | - trc.taboola.com 3914 | - api.taboola.com 3915 | - taboola.com 3916 | 3917 | Tribal Fusion: 3918 | domains: 3919 | - cdnx.tribalfusion.com 3920 | 3921 | White Pages: 3922 | domains: 3923 | - www.whitepages.com.au 3924 | - mobile.whitepages.com.au 3925 | 3926 | Yieldmo: 3927 | domains: 3928 | - yieldmo.com 3929 | 3930 | ZEDO: 3931 | domains: 3932 | - zedo.com 3933 | - z1.zedo.com -------------------------------------------------------------------------------- /logstash/intro.txt: -------------------------------------------------------------------------------- 1 | 参考资料 2 | https://www.gitbook.com/book/chenryn/elk-stack-guide-cn/details 3 | 4 | logstash处理事件有三个阶段：input ---> filter ---> output.input产生事件,filter 对事件进行修改,output输出到其它地方. 5 | input和output支持解码,可以对进入的或者退出管道的数据进行编码或解码而无需单独经过过滤器处理. 6 | 7 | 常用的input有： 8 | file：从文件系统中读取文件,类似于Linux下的tail -0F 9 | syslog：监听在514端口的系统日志信息,并解析成RFC3164格式 10 | redis：从redis服务器读取,同时使用redis channel和redis list 11 | beats: 通过Filebeat发送事件 12 | 13 | filter是logstash管道中间处理的设备.可以结合条件语句对符合标准的事件进行处理. 14 | grok: 解析和结构化任何文本.Grok 目前是logstash最好的方式对非结构化日志数据解析成结构化和可查询化.logstash内置了120个匹配模式,满足大部分需求. 15 | mutate: 在事件字段执行一般的转换.可以重命名、删除、替换和修改事件字段 16 | drop: 完全丢弃事件,如debug事件 17 | clone: 复制事件,可能添加或者删除字段 18 | geoip: 添加有关IP地址地理位置信息 19 | 20 | 21 | output是logstash管道的最后一个阶段.一个事件可以经过多个output.但是一旦所有输出处理完,该事件已经执行完. 22 | 常用的output有： 23 | elasticsearch: 发送事件数据到 Elasticsearch.如果要将数据保存在一个高效、便捷、易于查询的格式,elasticsearch将是不二人选 24 | file: 将事件数据写入到磁盘文件上 25 | graphite: 发送事件数据到graphite 26 | statsd: 发送事件数据到 statsd 27 | 28 | 29 | codec有json、msgpack、plain(text). 30 | json: 以json格式编码或者解码数据. 31 | multiline: 合并多行文本事件,如java异常和堆栈跟踪信息到一个单一事件. 32 | 33 | 34 | 故障容错 35 | 事件从一个管道到另一个管道使用内部的Ruby SizedQueue队列实现的.一个SizedQueue有最大的项目数.当队列达到最大值,所有的写入队列将会被阻塞. 36 | logstash设置每个队列大小为20.这意味着最多20个事件可以挂起进入下一个阶段,这可以防止数据丢失和保持logstash作为一个数据存储系统.这些内部队列不用于长期存放信息. 37 | 小队列意味着当logstash任务繁重或者管道临时有问题时,更容易堵塞.当出现问题时,要么队列不限制要么丢弃信息.队列不限制时,会无限的增长一直超出内存大小,导致崩溃,从而队列中的所有信息丢失.在多数情况下,丢弃消息也是不希望接受的. 38 | 大多数output会不断尝试受故障影响的事件.output失败或者下游服务的问题如磁盘满、权限问题、网络故障、服务中止. 39 | 如果output失败,output线程等待直到output能成功发送消息.output停止从output队列读取,这意味着事件填满了队列. 40 | 当output队列满了,过滤器是被阻塞的,因此它们不能写入新的事件到输出队列.虽然写入到output队列被阻塞了,过滤器停止从filter队列读取.最终,可能会导致filter队列(input--->filter)满. 41 | 一个满的filter队列,阻塞input写入到filter.这将导致所有input停止处理数据无论是新的事件. 42 | 在理想的情况下,这种行为类似于当tcp窗口关闭为0.没有新的数据发送,因为接收器还没有处理完当前队列的数据,直到下游(output)问题解决,消息重新流动起来. -------------------------------------------------------------------------------- /rsyslog.conf: -------------------------------------------------------------------------------- 1 | $ModLoad imklog 2 | $ModLoad omfwd 3 | #$ModLoad omfile 4 | #module(load="impstats" interval="120" severity="6" log.syslog="off" format="json" resetCounters="on") 5 | #module(load="imuxsock" SysSock.RateLimit.Interval="0") 6 | #module(load="mmnormalize") 7 | #module(load="mmfields") 8 | 9 | module(load="imfile" mode="inotify") 10 | module(load="mmsequence") 11 | 12 | #$ActionFileEnableSync on 13 | $SystemLogRateLimitInterval 0 14 | 15 | #### RULES #### 16 | *.info;mail.none;authpriv.none;cron.none /var/log/messages 17 | authpriv.* /var/log/secure 18 | cron.* ~ 19 | kern.* /var/log/kern.log 20 | daemon.* /var/log/daemon.log 21 | syslog.* /var/log/syslog 22 | 23 | $WorkDirectory /data/rsyslog 24 | #http://www.rsyslog.com/doc/master/rainerscript/queue_parameters.html 25 | main_queue( 26 | queue.filename="queue" 27 | queue.workerthreads="10" 28 | queue.discardmark="1000000000" 29 | queue.highwatermark="80000" 30 | queue.lowwatermark="1000" 31 | queue.size="100000" 32 | queue.type="linkedList" 33 | queue.workerthreadMinimumMessages="500" 34 | queue.timeoutWorkerthreadShutdown="1000" 35 | queue.timeoutEnqueue="10000" 36 | queue.dequeuebatchsize="1000" 37 | queue.maxdiskspace="10G" 38 | queue.discardseverity="8" 39 | queue.checkpointinterval="10" 40 | queue.timeoutshutdown="10" 41 | queue.timeoutactioncompletion="10" 42 | queue.timeoutworkerthreadshutdown="10" 43 | queue.workerthreadminimummessages="5000" 44 | queue.maxfilesize="500M" 45 | queue.saveonshutdown="on" 46 | ) 47 | 48 | $PreserveFQDN on 49 | 50 | $InputFileSeverity error 51 | $InputFileFacility local6 52 | $InputFilePersistStateInterval 20 53 | $InputFilePollInterval 1 54 | $InputFileReadMode 1 55 | $InputRunFileMonitor 56 | 57 | $WorkDirectory /data/rsyslog/ 58 | 59 | $IncludeConfig /etc/rsyslog.d/*.conf 60 | 61 | *.* @@logcenter.example.com;syslog 62 | 63 | -------------------------------------------------------------------------------- /rsyslog.d/fwd.conf: -------------------------------------------------------------------------------- 1 | #通过omfwd 后端logstash 可线性扩展 2 | Ruleset(name="forward_rule" ) { 3 | Action( type="mmsequence" mode="instance" from="0" to="4" var="$.seq" ) 4 | if $.seq == "0" then { 5 | action (type="omfwd" Target="logstash.example.com" Port="5140" Protocol="tcp" queue.size="150000" queue.dequeuebatchsize="2000" ) 6 | } 7 | if $.seq == "1" then { 8 | action (type="omfwd" Target="logstash.example.com" Port="5141" Protocol="tcp" queue.size="150000" queue.dequeuebatchsize="2000" ) 9 | } 10 | if $.seq == "2" then { 11 | action (type="omfwd" Target="logstash.example.com" Port="5142" Protocol="tcp" queue.size="150000" queue.dequeuebatchsize="2000" ) 12 | } 13 | if $.seq == "3" then { 14 | action (type="omfwd" Target="logstash.example.com" Port="5143" Protocol="tcp" queue.size="150000" queue.dequeuebatchsize="2000" ) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /rsyslog.d/grok.conf: -------------------------------------------------------------------------------- 1 | module(load="mmgrok") 2 | 3 | ruleset(name="grok_rule"){ 4 | action( 5 | type="mmgrok" 6 | patterndir="/data/ops/pattern" 7 | match="%{WORD:test}" 8 | source="msg" 9 | target="!msg" 10 | ) 11 | } 12 | 13 | -------------------------------------------------------------------------------- /rsyslog.d/kafka.conf: -------------------------------------------------------------------------------- 1 | module(load="omkafka") 2 | 3 | ruleset(name="kafka_rule_nginx_access"){ 4 | action( 5 | type="omkafka" 6 | broker=["kafka1:9092","kafka2:9092","kafka3:9092"] 7 | topic='nginx_access' 8 | template="syslog" 9 | confParam=["compression.codec=gzip","queue.buffering.max.messages=2000000","socket.timeout.ms=5","socket.keepalive.enable=true"] 10 | partitions.auto="on" 11 | action.resumeRetryCount="-1" 12 | ) 13 | } 14 | 15 | #input(type="imfile" Ruleset ="kafka_rule" File="/var/log/messages" Tag="messages") 16 | -------------------------------------------------------------------------------- /rsyslog.d/log.template.conf: -------------------------------------------------------------------------------- 1 | template(name="syslog" type="list") { 2 | constant(value="@timestamp") property(name="timereported" dateFormat="rfc3339") 3 | constant(value="message") property(name="msg") 4 | constant(value="tag") property(name="syslogtag") 5 | constant(value="host") property(name="fromhost") 6 | constant(value="hostname") property(name="hostname") 7 | constant(value="host_ip") property(name="fromhost-ip") 8 | constant(value="severity") property(name="syslogseverity-text") 9 | constant(value="facility") property(name="syslogfacility-text") 10 | constant(value="programname") property(name="programname") 11 | constant(value="app-name") property(name="app-name") 12 | } 13 | 14 | $template common,"{%timestamp:::date-rfc3339,jsonf:@timestamp%,%source:::jsonf:@source_host%,\"@source\":\"syslog://%fromhost-ip:::json%\",\"@message\":\"%timestamp% %app-name%:%msg:::json%\",\"@fields\":{%syslogfacility-text:::jsonf:facility%,%syslogseverity-text:::jsonf:severity%,%app-name:::jsonf:program%,%procid:::jsonf:processid%}}" 15 | 16 | $ActionFileDefaultTemplate syslog -------------------------------------------------------------------------------- /rsyslog.d/log4j.conf: -------------------------------------------------------------------------------- 1 | input(type="imfile" 2 | ruleset="forward_rule" 3 | File="/data/logs/log4j/*.log" 4 | Tag="log4j" 5 | Severity="info" 6 | reopenOnTruncate="on" 7 | addMetadata="on" 8 | freshStartTail="on" 9 | escapeLF="on" 10 | ) -------------------------------------------------------------------------------- /rsyslog.d/log4php.conf: -------------------------------------------------------------------------------- 1 | input(type="imfile" 2 | Ruleset ="forward_rule" 3 | File="/data/logs/log4php/*.log" 4 | Tag="log4php" 5 | Severity="error" 6 | reopenOnTruncate="on" 7 | addMetadata="on" 8 | freshStartTail="on" 9 | escapeLF="on" 10 | ) 11 | 12 | -------------------------------------------------------------------------------- /rsyslog.d/log4python.conf: -------------------------------------------------------------------------------- 1 | input(type="imfile" 2 | Ruleset ="forward_rule" 3 | File="/data/logs/log4python/*.log" 4 | Tag="log4python" 5 | Severity="error" 6 | reopenOnTruncate="on" 7 | addMetadata="on" 8 | freshStartTail="on" 9 | escapeLF="on" 10 | ) -------------------------------------------------------------------------------- /rsyslog.d/messsages.conf: -------------------------------------------------------------------------------- 1 | input(type="imfile" 2 | Ruleset ="forward_rule" 3 | File="/var/log/messsages" 4 | Tag="sys_messsages" 5 | Severity="info" 6 | reopenOnTruncate="on" 7 | addMetadata="on" 8 | freshStartTail="on" 9 | escapeLF="on" 10 | ) -------------------------------------------------------------------------------- /rsyslog.d/mysql-error.conf: -------------------------------------------------------------------------------- 1 | input(type="imfile" 2 | Ruleset ="forward_rule" 3 | File="/data/logs/mysql/error.log" 4 | Tag="mysql_error" 5 | Severity="error" 6 | reopenOnTruncate="on" 7 | addMetadata="on" 8 | freshStartTail="on" 9 | escapeLF="on" 10 | ) -------------------------------------------------------------------------------- /rsyslog.d/mysql-slow.conf: -------------------------------------------------------------------------------- 1 | input(type="imfile" 2 | Ruleset ="forward_rule" 3 | File="/data/logs/mysql/slow.log" 4 | Tag="mysql_slow" 5 | Severity="info" 6 | reopenOnTruncate="on" 7 | addMetadata="on" 8 | freshStartTail="on" 9 | escapeLF="on" 10 | startmsg.regex="^[0-9]") 11 | ) -------------------------------------------------------------------------------- /rsyslog.d/nginx-access.conf: -------------------------------------------------------------------------------- 1 | input(type="imfile" 2 | Ruleset ="kafka_rule_nginx_access" 3 | File="/data/logs/nginx/access.log" 4 | Tag="nginx_access" 5 | Severity="info" 6 | reopenOnTruncate="on" 7 | addMetadata="on" 8 | freshStartTail="on" 9 | escapeLF="on" 10 | ) -------------------------------------------------------------------------------- /rsyslog.d/nginx-error.conf: -------------------------------------------------------------------------------- 1 | input(type="imfile" 2 | Ruleset ="forward_rule" 3 | File="/data/logs/nginx/error.log" 4 | Tag="nginx_error" 5 | Severity="error" 6 | reopenOnTruncate="on" 7 | addMetadata="on" 8 | freshStartTail="on" 9 | escapeLF="on" 10 | ) 11 | -------------------------------------------------------------------------------- /rsyslog.d/php-fpm-error.conf: -------------------------------------------------------------------------------- 1 | input(type="imfile" 2 | Ruleset ="forward_rule" 3 | File="/data/logs/php/*error.log" 4 | Tag="php_error" 5 | Severity="error" 6 | reopenOnTruncate="on" 7 | addMetadata="on" 8 | freshStartTail="on" 9 | escapeLF="on" 10 | ) 11 | 12 | -------------------------------------------------------------------------------- /rsyslog.d/php-fpm-slow.conf: -------------------------------------------------------------------------------- 1 | input(type="imfile" 2 | Ruleset ="forward_rule" 3 | File="/data/logs/php/*slow.log" 4 | Tag="php_slow" 5 | Severity="error" 6 | reopenOnTruncate="on" 7 | addMetadata="on" 8 | freshStartTail="on" 9 | escapeLF="on" 10 | ) 11 | 12 | -------------------------------------------------------------------------------- /rsyslog.d/python-error.conf: -------------------------------------------------------------------------------- 1 | input(type="imfile" 2 | Ruleset ="forward_rule" 3 | File="/data/logs/python/error.log" 4 | Tag="python_error" 5 | Severity="error" 6 | reopenOnTruncate="on" 7 | addMetadata="on" 8 | freshStartTail="on" 9 | escapeLF="on" 10 | ) --------------------------------------------------------------------------------