├── .gitignore ├── LICENSE ├── README.md ├── aws ├── README.md ├── build.gradle └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── cloudflare │ │ │ └── elastic │ │ │ └── ElasticLambdaForwarder.java │ └── resources │ │ └── log4j2.xml │ └── test │ ├── java │ └── com │ │ └── cloudflare │ │ └── elastic │ │ └── TestElasticLambdaForwarder.java │ └── resources │ └── sample-event.json ├── build.gradle ├── conf ├── README.md ├── cloudflare-index-template.json ├── cloudflare-ingest-pipeline-daily.json ├── cloudflare-ingest-pipeline-weekly.json ├── install-artifacts.sh └── logstash-cloudflare.conf ├── dashboards ├── README.md ├── dashboards.json └── import-kibana-dashboards.png ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat └── settings.gradle /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore Gradle project-specific cache directory 2 | .gradle 3 | 4 | # Ignore Gradle build output directory 5 | build 6 | 7 | .idea/ 8 | *.iml 9 | *.ipr 10 | *.iws 11 | build-idea/ 12 | .DS_Store 13 | 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cloudflare-elastic 2 | 3 | ### Use Elasticsearch and Kibana to visualize Cloudflare logs 4 | 5 | # AWS Lambda Function for Forwarding Cloudflare Logs to Elasticsearch 6 | 7 | This lambda function will collect Cloudflare logs from an S3 bucket and forward them to an Elasticsearch cluster running on **[Elastic Cloud](https://cloud.elastic.co)**. 8 | 9 | The official documentation can be found on [Cloudflare's](https://developers.cloudflare.com/logs/analytics-integrations/elastic/) site 10 | 11 | ### Build 12 | **```./gradlew clean build```** 13 | 14 | 15 | ### Install 16 | 17 | Upload build/distributions/cloudflare-elastic-aws.zip to AWS. Due to the size of the distribution, you may need to upload the file to an S3 bucket before configuring. 18 | 19 | 20 | ### Configure 21 | 22 | ##### Configure handler method 23 | 24 | com.cloudflare.elastic.ElasticLambdaForwarder::handleRequest 25 | 26 | ##### Configure environment variables 27 | 28 | Certain environment variables must be configured so that the function can connect to the Elasticsearch cluster. 29 | 30 | | Environment Variable | Description | 31 | | --- | --- | 32 | | elastic_hostname | Fully qualified domain name of the Elasticsearch endpoint | 33 | | elastic_username | The username of the Elasticsearch user, e.g. elastic | 34 | | elastic_password | The password of the Elasticsearch user | 35 | 36 | Additionally, the following environment variables can optionally be configured. 37 | 38 | | Environment Variable | Description | Default value | 39 | | --- | --- | --- | 40 | | elastic_port | Endpoint port number | 9243 | 41 | | elastic_index | The index pattern to use | cloudflare-* | 42 | | elastic_pipeline | The ingest pipeline to use for pre-processing (**cloudflare-pipeline-(weekly, daily)**) | cloudflare-pipeline-weekly | 43 | | elastic_use_https | Whether to use SSL/TLS to connect | true | 44 | | elastic_bulk_actions | Number of log messages to send to Elasticsearch per batch; can be tuned for scale and speed | 100 | 45 | | elastic_bulk_concurrency | Number of concurrent requests to Elasticsearch; can be tuned for scale and speed | 2 | 46 | | elastic_debug | Enable verbose logging | false | 47 | | aws_access_key | Can be used to override permissions from execution role; typically not needed | | 48 | | aws_secret_key | Can be used to override permissions from execution role; typically not needed | | 49 | 50 | 51 | ### Install Additional Artifacts 52 | 53 | Change to the *conf* directory and issue the following command. 54 | ``` 55 | ./install-artifacts.sh -u elastic -p -e https://__:9243 56 | ``` 57 | 58 | This will install the ingest node processor and index templates. 59 | 60 | 61 | -------------------------------------------------------------------------------- /aws/README.md: -------------------------------------------------------------------------------- 1 | # AWS Lambda Function for Forwarding Cloudflare Logs to Elasticsearch 2 | 3 | This lambda function will collect Cloudflare logs from an S3 bucket and forward them to an Elasticsearch cluster running on **[Elastic Cloud](https://cloud.elastic.co)**. 4 | 5 | ### Build 6 | **```./gradlew clean build```** 7 | 8 | 9 | ### Install 10 | 11 | Upload build/distributions/cloudflare-elastic-aws.zip to AWS. Due to the size of the distribution, you may need to upload the file to an S3 bucket before configuring. 12 | 13 | 14 | ### Configure 15 | 16 | ##### Configure handler method 17 | 18 | com.cloudflare.elastic.ElasticLambdaForwarder::handleRequest 19 | 20 | ##### Configure environment variables 21 | 22 | Certain environment variables must be configured so that the function can connect to the Elasticsearch cluster. 23 | 24 | | Environment Variable | Description | 25 | | --- | --- | 26 | | elastic_hostname | Fully qualified domain name of the Elasticsearch endpoint | 27 | | elastic_username | The username of the Elasticsearch user, e.g. elastic | 28 | | elastic_password | The password of the Elasticsearch user | 29 | 30 | Additionally, the following environment variables can optionally be configured. 31 | 32 | | Environment Variable | Description | Default value | 33 | | --- | --- | --- | 34 | | elastic_port | Endpoint port number | 9243 | 35 | | elastic_index | The index pattern to use | cloudflare-* | 36 | | elastic_pipeline | The ingest pipeline to use for pre-processing (**cloudflare-pipeline-(weekly, daily)**) | cloudflare-pipeline-weekly | 37 | | elastic_use_https | Whether to use SSL/TLS to connect | true | 38 | | elastic_bulk_actions | Number of log messages to send to Elasticsearch per batch; can be tuned for scale and speed | 100 | 39 | | elastic_bulk_concurrency | Number of concurrent requests to Elasticsearch; can be tuned for scale and speed | 2 | 40 | | elastic_debug | Enable verbose logging | false | 41 | | aws_access_key | Can be used to override permissions from execution role; typically not needed | | 42 | | aws_secret_key | Can be used to override permissions from execution role; typically not needed | | 43 | 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /aws/build.gradle: -------------------------------------------------------------------------------- 1 | 2 | plugins { 3 | id 'java-library' 4 | } 5 | 6 | dependencies { 7 | testCompile 'junit:junit:4.12' 8 | 9 | compile ( 10 | 'com.amazonaws:aws-lambda-java-core:1.1.0', 11 | 'com.amazonaws:aws-lambda-java-events:1.1.0', 12 | 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.3.1', 13 | 'org.apache.logging.log4j:log4j-core:2.15.0', 14 | ) 15 | } 16 | 17 | task buildZip(type: Zip) { 18 | from compileJava 19 | from processResources 20 | into('lib') { 21 | from configurations.compileClasspath 22 | } 23 | } 24 | 25 | build.dependsOn buildZip 26 | 27 | -------------------------------------------------------------------------------- /aws/src/main/java/com/cloudflare/elastic/ElasticLambdaForwarder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.cloudflare.elastic; 15 | 16 | import com.amazonaws.auth.AWSCredentials; 17 | import com.amazonaws.services.lambda.runtime.LambdaLogger; 18 | import com.amazonaws.services.lambda.runtime.RequestHandler; 19 | import com.amazonaws.services.lambda.runtime.Context; 20 | import com.amazonaws.services.lambda.runtime.events.S3Event; 21 | import com.amazonaws.services.s3.AmazonS3; 22 | import com.amazonaws.services.s3.AmazonS3Client; 23 | import com.amazonaws.services.s3.AmazonS3ClientBuilder; 24 | import com.amazonaws.services.s3.event.S3EventNotification; 25 | import com.amazonaws.services.s3.model.GetObjectRequest; 26 | import com.amazonaws.services.s3.model.S3Object; 27 | import com.amazonaws.services.s3.model.S3ObjectInputStream; 28 | 29 | import org.apache.http.HttpHost; 30 | import org.apache.http.auth.AuthScope; 31 | import org.apache.http.auth.UsernamePasswordCredentials; 32 | import org.apache.http.client.CredentialsProvider; 33 | import org.apache.http.impl.client.BasicCredentialsProvider; 34 | import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; 35 | import org.elasticsearch.action.ActionListener; 36 | import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; 37 | import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; 38 | import org.elasticsearch.action.bulk.BackoffPolicy; 39 | import org.elasticsearch.action.bulk.BulkProcessor; 40 | import org.elasticsearch.action.bulk.BulkRequest; 41 | import org.elasticsearch.action.bulk.BulkResponse; 42 | import org.elasticsearch.action.index.IndexRequest; 43 | import org.elasticsearch.client.RequestOptions; 44 | import org.elasticsearch.client.RestClient; 45 | import org.elasticsearch.client.RestClientBuilder; 46 | import org.elasticsearch.client.RestHighLevelClient; 47 | import org.elasticsearch.cluster.health.ClusterHealthStatus; 48 | import org.elasticsearch.common.unit.ByteSizeUnit; 49 | import org.elasticsearch.common.unit.ByteSizeValue; 50 | import org.elasticsearch.common.unit.TimeValue; 51 | import org.elasticsearch.common.xcontent.XContentType; 52 | 53 | import java.io.BufferedReader; 54 | import java.io.IOException; 55 | import java.io.InputStreamReader; 56 | import java.io.PrintWriter; 57 | import java.io.StringWriter; 58 | import java.nio.charset.StandardCharsets; 59 | import java.util.concurrent.TimeUnit; 60 | import java.util.function.BiConsumer; 61 | import java.util.zip.GZIPInputStream; 62 | 63 | public class ElasticLambdaForwarder implements RequestHandler 64 | { 65 | private static final String ENV_ELASTIC_HOST = "elastic_hostname"; 66 | private static final String ENV_ELASTIC_PORT = "elastic_port"; 67 | private static final String ENV_ELASTIC_INDEX = "elastic_index"; 68 | private static final String ENV_ELASTIC_USERNAME = "elastic_username"; 69 | private static final String ENV_ELASTIC_PASSWORD = "elastic_password"; 70 | private static final String ENV_ELASTIC_PIPELINE = "elastic_pipeline"; 71 | private static final String ENV_ELASTIC_HTTPS = "elastic_use_https"; 72 | private static final String ENV_ELASTIC_BULK_ACTIONS = "elastic_bulk_actions"; 73 | private static final String ENV_ELASTIC_CONCURRENCY = "elastic_bulk_concurrency"; 74 | private static final String ENV_ELASTIC_DEBUG = "elastic_debug"; 75 | private static final String ENV_AWS_ACCESS_KEY = "aws_access_key"; 76 | private static final String ENV_AWS_SECRET_KEY = "aws_secret_key"; 77 | 78 | private LambdaLogger logger = null; 79 | 80 | private static int BULK_ACTIONS = 100; 81 | private static int BULK_CONCURRENCY = 2; 82 | private static int ELASTIC_PORT = 9243; 83 | private static String ELASTIC_INDEX = "cloudflare"; 84 | private static String ELASTIC_PIPELINE = "cloudflare-pipeline-weekly"; 85 | private static String ELASTIC_HOSTNAME = null; 86 | private static String ELASTIC_USERNAME = null; 87 | private static String ELASTIC_PASSWORD = null; 88 | private static boolean ELASTIC_HTTPS = true; 89 | private static boolean ELASTIC_DEBUG = false; 90 | private static boolean USE_AWS_CREDENTIALS = false; 91 | 92 | 93 | @Override 94 | public Void handleRequest(S3Event event, Context context) 95 | { 96 | logger = context.getLogger(); 97 | 98 | try { 99 | initialize(); 100 | 101 | logger.log(String.format( 102 | "Parameters: hostname [%s:%d] index [%s], username [%s], pipeline [%s], ssl/tls [%b]", 103 | ELASTIC_HOSTNAME, ELASTIC_PORT, ELASTIC_INDEX, ELASTIC_USERNAME, ELASTIC_PIPELINE, ELASTIC_HTTPS)); 104 | 105 | RestHighLevelClient es = client(ELASTIC_HOSTNAME, ELASTIC_PORT, ELASTIC_USERNAME, ELASTIC_PASSWORD, ELASTIC_HTTPS); 106 | 107 | AmazonS3 s3Client = getS3Client(); 108 | BulkProcessor processor = processor(es, BULK_ACTIONS, BULK_CONCURRENCY); 109 | 110 | try { 111 | process(es, s3Client, event, processor, ELASTIC_INDEX, ELASTIC_PIPELINE); 112 | } 113 | finally { 114 | logger.log("Finished processing; flushing any remaining logs..."); 115 | processor.flush(); 116 | processor.awaitClose(60L, TimeUnit.SECONDS); 117 | logger.log("Elasticsearch processor shut down"); 118 | } 119 | } 120 | catch (Exception e) { 121 | logger.log(String.format("%s\n%s", e.getMessage(), trace(e))); 122 | } 123 | 124 | return null; 125 | } 126 | 127 | private void process(RestHighLevelClient es, AmazonS3 s3Client, S3Event event, BulkProcessor processor, String index, String pipeline) 128 | { 129 | if (ELASTIC_DEBUG) { 130 | logger.log("S3 event record: " + event.toJson()); 131 | } 132 | 133 | for (S3EventNotification.S3EventNotificationRecord record : event.getRecords()) 134 | { 135 | final String key = record.getS3().getObject().getKey(); 136 | final String bucket = record.getS3().getBucket().getName(); 137 | 138 | logger.log("Processing: " + bucket + ":" + key); 139 | S3Object object = s3Client.getObject(new GetObjectRequest(bucket, key)); 140 | 141 | S3ObjectInputStream s3stream = object.getObjectContent(); 142 | 143 | try (GZIPInputStream gzip = new GZIPInputStream(s3stream); 144 | BufferedReader br = new BufferedReader(new InputStreamReader(gzip, StandardCharsets.UTF_8))) 145 | { 146 | for (String line; (line = br.readLine()) != null;) { 147 | processor.add(new IndexRequest(index).setPipeline(pipeline).source(line, XContentType.JSON)); 148 | } 149 | } 150 | catch (IOException e) { 151 | e.printStackTrace(); 152 | } 153 | } 154 | } 155 | 156 | private BulkProcessor processor(RestHighLevelClient client, int bulkActions, int bulkConcurrency) 157 | { 158 | BulkProcessor.Listener listener = new BulkProcessor.Listener() { 159 | @Override 160 | public void beforeBulk(long executionId, BulkRequest request) { 161 | logger.log(String.format("Flushing [%s] logs to elasticsearch", request.numberOfActions())); 162 | } 163 | 164 | @Override 165 | public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { 166 | if (response.hasFailures()) { 167 | logger.log(response.buildFailureMessage()); 168 | } 169 | } 170 | 171 | @Override 172 | public void afterBulk(long executionId, BulkRequest request, Throwable failure) { 173 | logger.log(String.format("%s\n%s", failure.getMessage(), trace(failure))); 174 | } 175 | }; 176 | 177 | BiConsumer> bulkConsumer = 178 | (request, bulkListener) -> 179 | client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener); 180 | 181 | BulkProcessor.Builder builder = BulkProcessor.builder(bulkConsumer, listener); 182 | 183 | builder.setBulkActions(bulkActions); 184 | builder.setBulkSize(new ByteSizeValue(10L, ByteSizeUnit.MB)); 185 | builder.setConcurrentRequests(bulkConcurrency); 186 | builder.setFlushInterval(TimeValue.timeValueSeconds(2L)); 187 | builder.setBackoffPolicy( 188 | BackoffPolicy.exponentialBackoff(TimeValue.timeValueSeconds(1), 10)); 189 | 190 | return builder.build(); 191 | } 192 | 193 | private RestHighLevelClient client(String endpoint, int port, String username, String password, boolean https) throws IOException 194 | { 195 | CredentialsProvider provider = new BasicCredentialsProvider(); 196 | provider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); 197 | 198 | RestHighLevelClient client = new RestHighLevelClient( 199 | RestClient.builder( 200 | new HttpHost(endpoint, port, https ? "https" : "http")). 201 | setHttpClientConfigCallback( 202 | new RestClientBuilder.HttpClientConfigCallback() { 203 | @Override 204 | public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder builder) 205 | { 206 | return builder.setDefaultCredentialsProvider(provider); 207 | }})); 208 | 209 | // Verify connection 210 | ClusterHealthResponse health = client.cluster().health(new ClusterHealthRequest(), RequestOptions.DEFAULT); 211 | ClusterHealthStatus status = health.getStatus(); 212 | logger.log("Connected to cluster: [" + health.getClusterName() + "] status: [" + status + "]"); 213 | 214 | if (status == ClusterHealthStatus.RED) { 215 | throw new RuntimeException("Elasticsearch cluster is in RED state; aborting"); 216 | } 217 | 218 | return client; 219 | } 220 | 221 | private AmazonS3 getS3Client() 222 | { 223 | if (USE_AWS_CREDENTIALS) { 224 | return new AmazonS3Client(new AWSCredentials() { 225 | @Override 226 | public String getAWSAccessKeyId() { 227 | return System.getenv(ENV_AWS_ACCESS_KEY); 228 | } 229 | 230 | @Override 231 | public String getAWSSecretKey() { 232 | return System.getenv(ENV_AWS_SECRET_KEY); 233 | } 234 | }); 235 | } 236 | else { 237 | return AmazonS3ClientBuilder.defaultClient(); 238 | } 239 | } 240 | 241 | private void initialize() 242 | { 243 | String bulk = System.getenv(ENV_ELASTIC_BULK_ACTIONS); 244 | if (bulk != null) { 245 | BULK_ACTIONS = Integer.parseInt(bulk); 246 | } 247 | 248 | String concurrency = System.getenv(ENV_ELASTIC_CONCURRENCY); 249 | if (concurrency != null) { 250 | BULK_CONCURRENCY = Integer.parseInt(concurrency); 251 | } 252 | 253 | String port = System.getenv(ENV_ELASTIC_PORT); 254 | if (port != null) { 255 | ELASTIC_PORT = Integer.parseInt(port); 256 | } 257 | 258 | String index = System.getenv(ENV_ELASTIC_INDEX); 259 | if (index != null && !index.isEmpty()) { 260 | ELASTIC_INDEX = index; 261 | } 262 | 263 | String pipeline = System.getenv(ENV_ELASTIC_PIPELINE); 264 | if (pipeline != null && !pipeline.isEmpty()) { 265 | ELASTIC_PIPELINE = pipeline; 266 | } 267 | 268 | String https = System.getenv(ENV_ELASTIC_HTTPS); 269 | if (https != null) { 270 | ELASTIC_HTTPS = Boolean.parseBoolean(https); 271 | } 272 | 273 | String debug = System.getenv(ENV_ELASTIC_DEBUG); 274 | if (debug != null) { 275 | ELASTIC_DEBUG = Boolean.parseBoolean(debug); 276 | } 277 | 278 | ELASTIC_HOSTNAME = System.getenv(ENV_ELASTIC_HOST); 279 | if (ELASTIC_HOSTNAME == null || ELASTIC_HOSTNAME.isEmpty()) { 280 | throw new RuntimeException("Elastic hostname not set; please set '" + ENV_ELASTIC_HOST + "'"); 281 | } 282 | 283 | ELASTIC_USERNAME = System.getenv(ENV_ELASTIC_USERNAME); 284 | if (ELASTIC_USERNAME == null || ELASTIC_USERNAME.isEmpty()) { 285 | throw new RuntimeException("Elastic username not set; please set '" + ENV_ELASTIC_USERNAME + "'"); 286 | } 287 | 288 | ELASTIC_PASSWORD = System.getenv(ENV_ELASTIC_PASSWORD); 289 | if (ELASTIC_PASSWORD == null || ELASTIC_PASSWORD.isEmpty()) { 290 | throw new RuntimeException("Elastic password not set; please set '" + ENV_ELASTIC_PASSWORD + "'"); 291 | } 292 | 293 | if (System.getenv(ENV_AWS_ACCESS_KEY) != null && System.getenv(ENV_AWS_SECRET_KEY) != null) { 294 | USE_AWS_CREDENTIALS = true; 295 | } 296 | } 297 | 298 | private String trace(Throwable t) 299 | { 300 | StringWriter sw = new StringWriter(); 301 | t.printStackTrace(new PrintWriter(sw)); 302 | return t.toString(); 303 | } 304 | } 305 | -------------------------------------------------------------------------------- /aws/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /aws/src/test/java/com/cloudflare/elastic/TestElasticLambdaForwarder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.cloudflare.elastic; 15 | 16 | import com.amazonaws.services.lambda.runtime.ClientContext; 17 | import com.amazonaws.services.lambda.runtime.CognitoIdentity; 18 | import com.amazonaws.services.lambda.runtime.Context; 19 | import com.amazonaws.services.lambda.runtime.LambdaLogger; 20 | import com.amazonaws.services.s3.event.S3EventNotification; 21 | import org.junit.Test; 22 | 23 | import com.amazonaws.services.lambda.runtime.events.S3Event; 24 | 25 | import java.io.BufferedReader; 26 | import java.io.IOException; 27 | import java.io.InputStreamReader; 28 | import java.util.stream.Collectors; 29 | 30 | public class TestElasticLambdaForwarder 31 | { 32 | @Test 33 | public void TestLambdaFunction() 34 | { 35 | S3Event event = new S3Event(S3Event().getRecords()); 36 | new ElasticLambdaForwarder().handleRequest(event, context()); 37 | } 38 | 39 | private S3EventNotification S3Event() 40 | { 41 | String json = ""; 42 | try (BufferedReader buffer = new BufferedReader(new InputStreamReader(this.getClass().getResourceAsStream("/sample-event.json")))) { 43 | json = buffer.lines().collect(Collectors.joining()); 44 | } 45 | catch (IOException e) { 46 | e.printStackTrace(); 47 | } 48 | 49 | S3EventNotification event = S3Event.parseJson(json); 50 | return event; 51 | } 52 | 53 | private Context context() 54 | { 55 | return new Context() { 56 | @Override 57 | public String getAwsRequestId() 58 | { 59 | return null; 60 | } 61 | 62 | @Override 63 | public String getLogGroupName() 64 | { 65 | return null; 66 | } 67 | 68 | @Override 69 | public String getLogStreamName() 70 | { 71 | return null; 72 | } 73 | 74 | @Override 75 | public String getFunctionName() 76 | { 77 | return null; 78 | } 79 | 80 | @Override 81 | public String getFunctionVersion() 82 | { 83 | return null; 84 | } 85 | 86 | @Override 87 | public String getInvokedFunctionArn() 88 | { 89 | return null; 90 | } 91 | 92 | @Override 93 | public CognitoIdentity getIdentity() 94 | { 95 | return null; 96 | } 97 | 98 | @Override 99 | public ClientContext getClientContext() 100 | { 101 | return null; 102 | } 103 | 104 | @Override 105 | public int getRemainingTimeInMillis() 106 | { 107 | return 0; 108 | } 109 | 110 | @Override 111 | public int getMemoryLimitInMB() 112 | { 113 | return 0; 114 | } 115 | 116 | @Override 117 | public LambdaLogger getLogger() 118 | { 119 | return new LambdaLogger() { 120 | @Override 121 | public void log(String string) 122 | { 123 | System.out.println(string); 124 | } 125 | }; 126 | } 127 | }; 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /aws/src/test/resources/sample-event.json: -------------------------------------------------------------------------------- 1 | { 2 | "Records": [ 3 | { 4 | "awsRegion": "us-east-1", 5 | "eventName": "ObjectCreated:Put", 6 | "eventSource": "aws:s3", 7 | "eventTime": "2019-08-20T23:50:25.237Z", 8 | "eventVersion": "2.1", 9 | "requestParameters": { 10 | "sourceIPAddress": "96.74.98.114" 11 | }, 12 | "responseElements": { 13 | "x-amz-id-2": "cm4JWvCxJ38sKPSwg1P1KTkokJ5bqARc8ZgSytlk4Kczzzs0PHCXlw4ElglR8CcO+O8rZ5y6aXI=", 14 | "x-amz-request-id": "B8A56BDD74C5419D" 15 | }, 16 | "s3": { 17 | "configurationId": "a01cbfec-2239-4b54-bd87-0f3a418353ec", 18 | "bucket": { 19 | "name": "andrew-cloudflare-logs", 20 | "ownerIdentity": { 21 | "principalId": "A165S1FYTEWVZY" 22 | }, 23 | "arn": "arn:aws:s3:::andrew-cloudflare-logs" 24 | }, 25 | "object": { 26 | "key": "sample-log-small.json.gz", 27 | "size": 790, 28 | "eTag": "2ccf37aee4e038437fbb458181da7111", 29 | "versionId": "", 30 | "sequencer": "005D5C87412C3129C9", 31 | "urlDecodedKey": "sample-log-small.json.gz" 32 | }, 33 | "s3SchemaVersion": "1.0" 34 | }, 35 | "userIdentity": { 36 | "principalId": "A165S1FYTEWVZY" 37 | }, 38 | "glacierEventData": null 39 | } 40 | ] 41 | } -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | 2 | subprojects { 3 | repositories { 4 | mavenCentral() 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /conf/README.md: -------------------------------------------------------------------------------- 1 | 2 | ##### Configure Elasticsearch 3 | 4 | Elasticsearch needs to be configured with an ingest pipeline and an index template. You can use the provided script to easily install the required artifacts. 5 | 6 | **```./install-artifacts.sh -e https://: -u -p ```** 7 | -------------------------------------------------------------------------------- /conf/cloudflare-index-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "index_patterns": [ 3 | "cloudflare-*" 4 | ], 5 | "mappings": { 6 | "properties": { 7 | "observer.vendor": { 8 | "type": "keyword" 9 | }, 10 | "observer.type": { 11 | "type": "keyword" 12 | }, 13 | "ecs.version": { 14 | "type": "keyword" 15 | }, 16 | "event.dataset": { 17 | "type": "keyword" 18 | }, 19 | "cloudflare.waf.flags": { 20 | "type": "keyword" 21 | }, 22 | "cloudflare.firewall.matches.actions" : { 23 | "type": "keyword" 24 | }, 25 | "cloudflare.firewall.matches.rule_ids" : { 26 | "type": "keyword" 27 | }, 28 | "cloudflare.firewall.matches.sources" : { 29 | "type": "keyword" 30 | }, 31 | "cloudflare.edge.colo.code" : { 32 | "type": "keyword" 33 | }, 34 | "cloudflare.client.ssl.protocol": { 35 | "type": "keyword" 36 | }, 37 | "cloudflare.client.request.protocol" : { 38 | "type": "keyword" 39 | }, 40 | "cloudflare.waf.rule.id": { 41 | "type": "keyword" 42 | }, 43 | "cloudflare.edge.colo.id": { 44 | "type": "integer" 45 | }, 46 | "source.geo": { 47 | "properties": { 48 | "ip": { 49 | "type": "ip" 50 | }, 51 | "postal_code": { 52 | "type": "keyword" 53 | }, 54 | "location": { 55 | "type": "geo_point" 56 | }, 57 | "dma_code": { 58 | "type": "long" 59 | }, 60 | "country_code3": { 61 | "type": "keyword" 62 | }, 63 | "latitude": { 64 | "type": "float" 65 | }, 66 | "longitude": { 67 | "type": "float" 68 | }, 69 | "region_name": { 70 | "type": "keyword" 71 | }, 72 | "city_name": { 73 | "type": "keyword" 74 | }, 75 | "timezone": { 76 | "type": "keyword" 77 | }, 78 | "country_code2": { 79 | "type": "keyword" 80 | }, 81 | "continent_code": { 82 | "type": "keyword" 83 | }, 84 | "country_name": { 85 | "type": "keyword" 86 | }, 87 | "region_code": { 88 | "type": "keyword" 89 | }, 90 | "continent_name": { 91 | "type": "keyword" 92 | }, 93 | "region_iso_code": { 94 | "type": "keyword" 95 | } 96 | } 97 | }, 98 | "cloudflare.origin.response.time": { 99 | "type": "long" 100 | }, 101 | "cloudflare.origin.ip": { 102 | "type": "ip" 103 | }, 104 | "@version": { 105 | "type": "keyword" 106 | }, 107 | "cloudflare.cache.status": { 108 | "type": "keyword" 109 | }, 110 | "source.geo.country_iso_code": { 111 | "type": "keyword" 112 | }, 113 | "cloudflare.edge.server.ip": { 114 | "type": "ip" 115 | }, 116 | "observer.ip": { 117 | "type": "ip" 118 | }, 119 | "cloudflare.waf.matched_var": { 120 | "type": "keyword" 121 | }, 122 | "cloudflare.waf.action": { 123 | "type": "keyword" 124 | }, 125 | "cloudflare.edge.pathing.status": { 126 | "type": "keyword" 127 | }, 128 | "source.as.number": { 129 | "type": "long" 130 | }, 131 | "cloudflare.edge.rate.limit.id": { 132 | "type": "long" 133 | }, 134 | "user_agent.original": { 135 | "type": "keyword" 136 | }, 137 | "user_agent.device.name": { 138 | "type": "keyword" 139 | }, 140 | "user_agent.name": { 141 | "type": "keyword" 142 | }, 143 | "user_agent.os.full": { 144 | "type": "keyword" 145 | }, 146 | "user_agent.os.kernel": { 147 | "type": "keyword" 148 | }, 149 | "user_agent.os.platform": { 150 | "type": "keyword" 151 | }, 152 | "user_agent.os.name": { 153 | "type": "keyword" 154 | }, 155 | "user_agent.os.version": { 156 | "type": "keyword" 157 | }, 158 | "user_agent.version": { 159 | "type": "keyword" 160 | }, 161 | "user_agent.major": { 162 | "type": "keyword" 163 | }, 164 | "user_agent.minor": { 165 | "type": "keyword" 166 | }, 167 | "user_agent.patch": { 168 | "type": "keyword" 169 | }, 170 | "user_agent.build": { 171 | "type": "keyword" 172 | }, 173 | "user_agent.os_name": { 174 | "type": "keyword" 175 | }, 176 | "user_agent.os_major": { 177 | "type": "keyword" 178 | }, 179 | "user_agent.os_minor": { 180 | "type": "keyword" 181 | }, 182 | "cloudflare.edge.response.compression_ratio": { 183 | "type": "float" 184 | }, 185 | "cloudflare.parent.ray_id": { 186 | "type": "keyword" 187 | }, 188 | "http.request.referrer": { 189 | "type": "keyword" 190 | }, 191 | "cloudflare.cache.tiered.fill": { 192 | "type": "boolean" 193 | }, 194 | "cloudflare.ray_id": { 195 | "type": "keyword" 196 | }, 197 | "cloudflare.client.ip.class": { 198 | "type": "keyword" 199 | }, 200 | "http.version": { 201 | "type": "keyword" 202 | }, 203 | "cloudflare.edge.request.host": { 204 | "type": "keyword" 205 | }, 206 | "cloudflare.cache.response.bytes": { 207 | "type": "long" 208 | }, 209 | "http.response.bytes": { 210 | "type": "long" 211 | }, 212 | "cloudflare.device.type": { 213 | "type": "keyword" 214 | }, 215 | "cloudflare.edge.rate.limit.action": { 216 | "type": "keyword" 217 | }, 218 | "cloudflare.edge.end.timestamp": { 219 | "type": "date" 220 | }, 221 | "event.end": { 222 | "type": "date" 223 | }, 224 | "cloudflare.origin.response.http.expires": { 225 | "type": "date", 226 | "format": "E, d MMM uuuu HH:mm:ss 'UTC'" 227 | }, 228 | "cloudflare.edge.response.status": { 229 | "type": "long" 230 | }, 231 | "http.request.bytes": { 232 | "type": "long" 233 | }, 234 | "cloudflare.cache.response.status": { 235 | "type": "long" 236 | }, 237 | "http.response.status_code": { 238 | "type": "long" 239 | }, 240 | "url.domain": { 241 | "type": "keyword" 242 | }, 243 | "cloudflare.origin.ssl.protocol": { 244 | "type": "keyword" 245 | }, 246 | "url.full": { 247 | "type": "keyword" 248 | }, 249 | "cloudflare.origin.response.bytes": { 250 | "type": "long" 251 | }, 252 | "cloudflare.worker.cpu_time": { 253 | "type": "long" 254 | }, 255 | "cloudflare.origin.response.http.last_modified": { 256 | "type": "date", 257 | "ignore_malformed": true 258 | }, 259 | "cloudflare.edge.start.timestamp": { 260 | "type": "date" 261 | }, 262 | "event.start": { 263 | "type": "date" 264 | }, 265 | "cloudflare.edge.pathing.src": { 266 | "type": "keyword" 267 | }, 268 | "cloudflare.waf.profile": { 269 | "type": "keyword" 270 | }, 271 | "cloudflare.edge.pathing.op": { 272 | "type": "keyword" 273 | }, 274 | "cloudflare.security_level": { 275 | "type": "keyword" 276 | }, 277 | "cloudflare.waf.rule.message": { 278 | "type": "keyword" 279 | }, 280 | "cloudflare.http.response.status_code": { 281 | "type": "long" 282 | }, 283 | "@timestamp": { 284 | "type": "date" 285 | }, 286 | "server.ip": { 287 | "type": "ip" 288 | }, 289 | "destination.ip": { 290 | "type": "ip" 291 | }, 292 | "http.request.method": { 293 | "type": "keyword" 294 | }, 295 | "cloudflare.zone_id": { 296 | "type": "integer" 297 | }, 298 | "source.port": { 299 | "type": "long" 300 | }, 301 | "source.address" : { 302 | "type" : "keyword" 303 | }, 304 | "source.ip" : { 305 | "type" : "ip" 306 | }, 307 | "client.port": { 308 | "type": "long" 309 | }, 310 | "client.ip" : { 311 | "type" : "ip" 312 | }, 313 | "client.address": { 314 | "type" : "keyword" 315 | }, 316 | "cloudflare.worker.subrequest": { 317 | "type": "boolean" 318 | }, 319 | "cloudflare.worker.subrequest_count": { 320 | "type": "long" 321 | }, 322 | "cloudflare.worker.status": { 323 | "type": "keyword" 324 | }, 325 | "cloudflare.edge.response.content_type": { 326 | "type": "keyword" 327 | }, 328 | "cloudflare.client.ssl.cipher": { 329 | "type": "keyword" 330 | }, 331 | "cloudflare.edge.response.bytes": { 332 | "type": "long" 333 | }, 334 | "url.path": { 335 | "type": "keyword" 336 | } 337 | } 338 | }, 339 | "settings": { 340 | "index": { 341 | "number_of_shards": "1", 342 | "number_of_replicas": "1", 343 | "mapping.ignore_malformed": true 344 | } 345 | } 346 | } 347 | -------------------------------------------------------------------------------- /conf/cloudflare-ingest-pipeline-daily.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Cloudflare Log Pipeline (Daily Indices)", 3 | "processors": [ 4 | { 5 | "date": { 6 | "field": "EdgeStartTimestamp", 7 | "formats": [ 8 | "uuuu-MM-dd'T'HH:mm:ssX", 9 | "uuuu-MM-dd'T'HH:mm:ss.SSSX", 10 | "yyyy-MM-dd'T'HH:mm:ssZ", 11 | "yyyy-MM-dd'T'HH:mm:ss.SSSZ" 12 | ], 13 | "timezone": "UTC", 14 | "target_field": "@timestamp" 15 | } 16 | }, 17 | { 18 | "date": { 19 | "field": "EdgeStartTimestamp", 20 | "formats": [ 21 | "uuuu-MM-dd'T'HH:mm:ssX", 22 | "uuuu-MM-dd'T'HH:mm:ss.SSSX", 23 | "yyyy-MM-dd'T'HH:mm:ssZ", 24 | "yyyy-MM-dd'T'HH:mm:ss.SSSZ" 25 | ], 26 | "timezone": "UTC", 27 | "target_field": "EdgeStartTimestamp" 28 | } 29 | }, 30 | { 31 | "date_index_name": { 32 | "field": "EdgeStartTimestamp", 33 | "index_name_prefix": "cloudflare-", 34 | "date_rounding": "d", 35 | "timezone": "UTC", 36 | "date_formats": [ 37 | "uuuu-MM-dd'T'HH:mm:ssX", 38 | "uuuu-MM-dd'T'HH:mm:ss.SSSX", 39 | "yyyy-MM-dd'T'HH:mm:ssZ", 40 | "yyyy-MM-dd'T'HH:mm:ss.SSSZ" 41 | ] 42 | } 43 | }, 44 | { 45 | "geoip": { 46 | "field": "ClientIP", 47 | "target_field": "source.geo", 48 | "properties": [ 49 | "ip", 50 | "country_name", 51 | "continent_name", 52 | "region_iso_code", 53 | "region_name", 54 | "city_name", 55 | "timezone", 56 | "location" 57 | ] 58 | } 59 | }, 60 | { 61 | "dissect": { 62 | "field": "ClientRequestProtocol", 63 | "pattern": "HTTP/%{http.version}" 64 | } 65 | }, 66 | { 67 | "rename": { 68 | "field": "ClientRequestProtocol", 69 | "target_field": "cloudflare.client.request.protocol", 70 | "ignore_missing": true 71 | } 72 | }, 73 | { 74 | "set": { 75 | "field": "ecs.version", 76 | "value": "1.1.0" 77 | } 78 | }, 79 | { 80 | "set": { 81 | "field": "observer.vendor", 82 | "value": "Cloudflare" 83 | } 84 | }, 85 | { 86 | "set": { 87 | "field": "observer.type", 88 | "value": "proxy" 89 | } 90 | }, 91 | { 92 | "set": { 93 | "field": "event.dataset", 94 | "value": "cloudflare.log" 95 | } 96 | }, 97 | { 98 | "rename": { 99 | "field": "CacheCacheStatus", 100 | "target_field": "cloudflare.cache.status", 101 | "ignore_missing": true 102 | } 103 | }, 104 | { 105 | "set": { 106 | "field": "http.response.bytes", 107 | "value": "{{CacheResponseBytes}}" 108 | } 109 | }, 110 | { 111 | "rename": { 112 | "field": "CacheResponseBytes", 113 | "target_field": "cloudflare.cache.response.bytes", 114 | "ignore_missing": true 115 | } 116 | }, 117 | { 118 | "set": { 119 | "field": "http.response.status_code", 120 | "value": "{{CacheResponseStatus}}" 121 | } 122 | }, 123 | { 124 | "rename": { 125 | "field": "CacheResponseStatus", 126 | "target_field": "cloudflare.cache.response.status", 127 | "ignore_missing": true 128 | } 129 | }, 130 | { 131 | "rename": { 132 | "field": "CacheTieredFill", 133 | "target_field": "cloudflare.cache.tiered.fill", 134 | "ignore_missing": true 135 | } 136 | }, 137 | { 138 | "rename": { 139 | "field": "ClientASN", 140 | "target_field": "source.as.number", 141 | "ignore_missing": true 142 | } 143 | }, 144 | { 145 | "rename": { 146 | "field": "ClientCountry", 147 | "target_field": "source.geo.country_iso_code", 148 | "ignore_missing": true 149 | } 150 | }, 151 | { 152 | "rename": { 153 | "field": "ClientDeviceType", 154 | "target_field": "cloudflare.device.type", 155 | "ignore_missing": true 156 | } 157 | }, 158 | { 159 | "set": { 160 | "field": "source.address", 161 | "value": "{{ClientIP}}" 162 | } 163 | }, 164 | { 165 | "set": { 166 | "field": "client.ip", 167 | "value": "{{ClientIP}}" 168 | } 169 | }, 170 | { 171 | "set": { 172 | "field": "client.address", 173 | "value": "{{ClientIP}}" 174 | } 175 | }, 176 | { 177 | "rename": { 178 | "field": "ClientIP", 179 | "target_field": "source.ip", 180 | "ignore_missing": true 181 | } 182 | }, 183 | { 184 | "rename": { 185 | "field": "ClientIPClass", 186 | "target_field": "cloudflare.client.ip.class", 187 | "ignore_missing": true 188 | } 189 | }, 190 | { 191 | "rename": { 192 | "field": "ClientRequestBytes", 193 | "target_field": "http.request.bytes", 194 | "ignore_missing": true 195 | } 196 | }, 197 | { 198 | "rename": { 199 | "field": "ClientRequestHost", 200 | "target_field": "url.domain", 201 | "ignore_missing": true 202 | } 203 | }, 204 | { 205 | "rename": { 206 | "field": "ClientRequestMethod", 207 | "target_field": "http.request.method", 208 | "ignore_missing": true 209 | } 210 | }, 211 | { 212 | "rename": { 213 | "field": "ClientRequestPath", 214 | "target_field": "url.path", 215 | "ignore_missing": true 216 | } 217 | }, 218 | { 219 | "rename": { 220 | "field": "ClientRequestReferer", 221 | "target_field": "http.request.referrer", 222 | "ignore_missing": true 223 | } 224 | }, 225 | { 226 | "rename": { 227 | "field": "ClientRequestURI", 228 | "target_field": "url.full", 229 | "ignore_missing": true 230 | } 231 | }, 232 | { 233 | "user_agent" : { 234 | "field" : "ClientRequestUserAgent", 235 | "target_field": "user_agent", 236 | "ecs": true, 237 | "ignore_missing": true 238 | } 239 | }, 240 | { 241 | "set": { 242 | "field": "user_agent.original", 243 | "value": "{{ClientRequestUserAgent}}" 244 | } 245 | }, 246 | { 247 | "remove" : { 248 | "field": "ClientRequestUserAgent", 249 | "ignore_missing": true 250 | } 251 | }, 252 | { 253 | "rename": { 254 | "field": "ClientSSLCipher", 255 | "target_field": "cloudflare.client.ssl.cipher", 256 | "ignore_missing": true 257 | } 258 | }, 259 | { 260 | "rename": { 261 | "field": "ClientSSLProtocol", 262 | "target_field": "cloudflare.client.ssl.protocol", 263 | "ignore_missing": true 264 | } 265 | }, 266 | { 267 | "set": { 268 | "field": "client.port", 269 | "value": "{{ClientSrcPort}}" 270 | } 271 | }, 272 | { 273 | "rename": { 274 | "field": "ClientSrcPort", 275 | "target_field": "source.port", 276 | "ignore_missing": true 277 | } 278 | }, 279 | { 280 | "rename": { 281 | "field": "EdgeColoCode", 282 | "target_field": "cloudflare.edge.colo.code", 283 | "ignore_missing": true 284 | } 285 | }, 286 | { 287 | "rename": { 288 | "field": "EdgeColoID", 289 | "target_field": "cloudflare.edge.colo.id", 290 | "ignore_missing": true 291 | } 292 | }, 293 | { 294 | "set": { 295 | "field": "event.end", 296 | "value": "{{EdgeEndTimestamp}}" 297 | } 298 | }, 299 | { 300 | "rename": { 301 | "field": "EdgeEndTimestamp", 302 | "target_field": "cloudflare.edge.end.timestamp", 303 | "ignore_missing": true 304 | } 305 | }, 306 | { 307 | "rename": { 308 | "field": "EdgePathingOp", 309 | "target_field": "cloudflare.edge.pathing.op", 310 | "ignore_missing": true 311 | } 312 | }, 313 | { 314 | "rename": { 315 | "field": "EdgePathingSrc", 316 | "target_field": "cloudflare.edge.pathing.src", 317 | "ignore_missing": true 318 | } 319 | }, 320 | { 321 | "rename": { 322 | "field": "EdgePathingStatus", 323 | "target_field": "cloudflare.edge.pathing.status", 324 | "ignore_missing": true 325 | } 326 | }, 327 | { 328 | "rename": { 329 | "field": "EdgeRateLimitAction", 330 | "target_field": "cloudflare.edge.rate.limit.action", 331 | "ignore_missing": true 332 | } 333 | }, 334 | { 335 | "rename": { 336 | "field": "EdgeRateLimitID", 337 | "target_field": "cloudflare.edge.rate.limit.id", 338 | "ignore_missing": true 339 | } 340 | }, 341 | { 342 | "rename": { 343 | "field": "EdgeRequestHost", 344 | "target_field": "cloudflare.edge.request.host", 345 | "ignore_missing": true 346 | } 347 | }, 348 | { 349 | "rename": { 350 | "field": "EdgeResponseBytes", 351 | "target_field": "cloudflare.edge.response.bytes", 352 | "ignore_missing": true 353 | } 354 | }, 355 | { 356 | "rename": { 357 | "field": "EdgeResponseCompressionRatio", 358 | "target_field": "cloudflare.edge.response.compression_ratio", 359 | "ignore_missing": true 360 | } 361 | }, 362 | { 363 | "rename": { 364 | "field": "EdgeResponseContentType", 365 | "target_field": "cloudflare.edge.response.content_type", 366 | "ignore_missing": true 367 | } 368 | }, 369 | { 370 | "rename": { 371 | "field": "EdgeResponseStatus", 372 | "target_field": "cloudflare.edge.response.status", 373 | "ignore_missing": true 374 | } 375 | }, 376 | { 377 | "set": { 378 | "field": "observer.ip", 379 | "value": "{{EdgeServerIP}}" 380 | } 381 | }, 382 | { 383 | "rename": { 384 | "field": "EdgeServerIP", 385 | "target_field": "cloudflare.edge.server.ip", 386 | "ignore_missing": true 387 | } 388 | }, 389 | { 390 | "set": { 391 | "field": "event.start", 392 | "value": "{{EdgeStartTimestamp}}" 393 | } 394 | }, 395 | { 396 | "rename": { 397 | "field": "EdgeStartTimestamp", 398 | "target_field": "cloudflare.edge.start.timestamp", 399 | "ignore_missing": true 400 | } 401 | }, 402 | { 403 | "rename": { 404 | "field": "FirewallMatchesActions", 405 | "target_field": "cloudflare.firewall.matches.actions", 406 | "ignore_missing": true 407 | } 408 | }, 409 | { 410 | "rename": { 411 | "field": "FirewallMatchesSources", 412 | "target_field": "cloudflare.firewall.matches.sources", 413 | "ignore_missing": true 414 | } 415 | }, 416 | { 417 | "rename": { 418 | "field": "FirewallMatchesRuleIDs", 419 | "target_field": "cloudflare.firewall.matches.rule_ids", 420 | "ignore_missing": true 421 | } 422 | }, 423 | { 424 | "set": { 425 | "field": "server.ip", 426 | "value": "{{OriginIP}}" 427 | } 428 | }, 429 | { 430 | "set": { 431 | "field": "cloudflare.origin.ip", 432 | "value": "{{OriginIP}}" 433 | } 434 | }, 435 | { 436 | "rename": { 437 | "field": "OriginIP", 438 | "target_field": "destination.ip", 439 | "ignore_missing": true 440 | } 441 | }, 442 | { 443 | "rename": { 444 | "field": "OriginResponseBytes", 445 | "target_field": "cloudflare.origin.response.bytes", 446 | "ignore_missing": true 447 | } 448 | }, 449 | { 450 | "remove" : { 451 | "if" : "ctx.OriginResponseHTTPExpires == ''", 452 | "field" : "OriginResponseHTTPExpires" 453 | } 454 | }, 455 | { 456 | "rename": { 457 | "field": "OriginResponseHTTPExpires", 458 | "target_field": "cloudflare.origin.response.http.expires", 459 | "ignore_missing": true 460 | } 461 | }, 462 | { 463 | "remove" : { 464 | "if" : "ctx.OriginResponseHTTPLastModified == ''", 465 | "field" : "OriginResponseHTTPLastModified" 466 | } 467 | }, 468 | { 469 | "rename": { 470 | "field": "OriginResponseHTTPLastModified", 471 | "target_field": "cloudflare.origin.response.http.last_modified", 472 | "ignore_missing": true 473 | } 474 | }, 475 | { 476 | "rename": { 477 | "field": "OriginResponseStatus", 478 | "target_field": "cloudflare.http.response.status_code", 479 | "ignore_missing": true 480 | } 481 | }, 482 | { 483 | "rename": { 484 | "field": "OriginResponseTime", 485 | "target_field": "cloudflare.origin.response.time", 486 | "ignore_missing": true 487 | } 488 | }, 489 | { 490 | "rename": { 491 | "field": "OriginSSLProtocol", 492 | "target_field": "cloudflare.origin.ssl.protocol", 493 | "ignore_missing": true 494 | } 495 | }, 496 | { 497 | "rename": { 498 | "field": "ParentRayID", 499 | "target_field": "cloudflare.parent.ray_id", 500 | "ignore_missing": true 501 | } 502 | }, 503 | { 504 | "rename": { 505 | "field": "RayID", 506 | "target_field": "cloudflare.ray_id", 507 | "ignore_missing": true 508 | } 509 | }, 510 | { 511 | "rename": { 512 | "field": "SecurityLevel", 513 | "target_field": "cloudflare.security_level", 514 | "ignore_missing": true 515 | } 516 | }, 517 | { 518 | "rename": { 519 | "field": "WAFAction", 520 | "target_field": "cloudflare.waf.action", 521 | "ignore_missing": true 522 | } 523 | }, 524 | { 525 | "rename": { 526 | "field": "WAFFlags", 527 | "target_field": "cloudflare.waf.flags", 528 | "ignore_missing": true 529 | } 530 | }, 531 | { 532 | "rename": { 533 | "field": "WAFMatchedVar", 534 | "target_field": "cloudflare.waf.matched_var", 535 | "ignore_missing": true 536 | } 537 | }, 538 | { 539 | "rename": { 540 | "field": "WAFProfile", 541 | "target_field": "cloudflare.waf.profile", 542 | "ignore_missing": true 543 | } 544 | }, 545 | { 546 | "rename": { 547 | "field": "WAFRuleID", 548 | "target_field": "cloudflare.waf.rule.id", 549 | "ignore_missing": true 550 | } 551 | }, 552 | { 553 | "rename": { 554 | "field": "WAFRuleMessage", 555 | "target_field": "cloudflare.waf.rule.message", 556 | "ignore_missing": true 557 | } 558 | }, 559 | { 560 | "rename": { 561 | "field": "WorkerCPUTime", 562 | "target_field": "cloudflare.worker.cpu_time", 563 | "ignore_missing": true 564 | } 565 | }, 566 | { 567 | "rename": { 568 | "field": "WorkerStatus", 569 | "target_field": "cloudflare.worker.status", 570 | "ignore_missing": true 571 | } 572 | }, 573 | { 574 | "rename": { 575 | "field": "WorkerSubrequest", 576 | "target_field": "cloudflare.worker.subrequest", 577 | "ignore_missing": true 578 | } 579 | }, 580 | { 581 | "rename": { 582 | "field": "WorkerSubrequestCount", 583 | "target_field": "cloudflare.worker.subrequest_count", 584 | "ignore_missing": true 585 | } 586 | }, 587 | { 588 | "rename": { 589 | "field": "ZoneID", 590 | "target_field": "cloudflare.zone_id", 591 | "ignore_missing": true 592 | } 593 | } 594 | ] 595 | } 596 | 597 | -------------------------------------------------------------------------------- /conf/cloudflare-ingest-pipeline-weekly.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Cloudflare Log Pipeline (Weekly Indices)", 3 | "processors": [ 4 | { 5 | "date": { 6 | "field": "EdgeStartTimestamp", 7 | "formats": [ 8 | "uuuu-MM-dd'T'HH:mm:ssX", 9 | "uuuu-MM-dd'T'HH:mm:ss.SSSX", 10 | "yyyy-MM-dd'T'HH:mm:ssZ", 11 | "yyyy-MM-dd'T'HH:mm:ss.SSSZ" 12 | ], 13 | "timezone": "UTC", 14 | "target_field": "@timestamp" 15 | } 16 | }, 17 | { 18 | "date": { 19 | "field": "EdgeStartTimestamp", 20 | "formats": [ 21 | "uuuu-MM-dd'T'HH:mm:ssX", 22 | "uuuu-MM-dd'T'HH:mm:ss.SSSX", 23 | "yyyy-MM-dd'T'HH:mm:ssZ", 24 | "yyyy-MM-dd'T'HH:mm:ss.SSSZ" 25 | ], 26 | "timezone": "UTC", 27 | "target_field": "EdgeStartTimestamp" 28 | } 29 | }, 30 | { 31 | "date_index_name": { 32 | "field": "EdgeStartTimestamp", 33 | "index_name_prefix": "cloudflare-", 34 | "date_rounding": "w", 35 | "timezone": "UTC", 36 | "date_formats": [ 37 | "uuuu-MM-dd'T'HH:mm:ssX", 38 | "uuuu-MM-dd'T'HH:mm:ss.SSSX", 39 | "yyyy-MM-dd'T'HH:mm:ssZ", 40 | "yyyy-MM-dd'T'HH:mm:ss.SSSZ" 41 | ] 42 | } 43 | }, 44 | { 45 | "geoip": { 46 | "field": "ClientIP", 47 | "target_field": "source.geo", 48 | "properties": [ 49 | "ip", 50 | "country_name", 51 | "continent_name", 52 | "region_iso_code", 53 | "region_name", 54 | "city_name", 55 | "timezone", 56 | "location" 57 | ] 58 | } 59 | }, 60 | { 61 | "dissect": { 62 | "field": "ClientRequestProtocol", 63 | "pattern": "HTTP/%{http.version}" 64 | } 65 | }, 66 | { 67 | "rename": { 68 | "field": "ClientRequestProtocol", 69 | "target_field": "cloudflare.client.request.protocol", 70 | "ignore_missing": true 71 | } 72 | }, 73 | { 74 | "set": { 75 | "field": "ecs.version", 76 | "value": "1.1.0" 77 | } 78 | }, 79 | { 80 | "set": { 81 | "field": "observer.vendor", 82 | "value": "Cloudflare" 83 | } 84 | }, 85 | { 86 | "set": { 87 | "field": "observer.type", 88 | "value": "proxy" 89 | } 90 | }, 91 | { 92 | "set": { 93 | "field": "event.dataset", 94 | "value": "cloudflare.log" 95 | } 96 | }, 97 | { 98 | "rename": { 99 | "field": "CacheCacheStatus", 100 | "target_field": "cloudflare.cache.status", 101 | "ignore_missing": true 102 | } 103 | }, 104 | { 105 | "set": { 106 | "field": "http.response.bytes", 107 | "value": "{{CacheResponseBytes}}" 108 | } 109 | }, 110 | { 111 | "rename": { 112 | "field": "CacheResponseBytes", 113 | "target_field": "cloudflare.cache.response.bytes", 114 | "ignore_missing": true 115 | } 116 | }, 117 | { 118 | "set": { 119 | "field": "http.response.status_code", 120 | "value": "{{CacheResponseStatus}}" 121 | } 122 | }, 123 | { 124 | "rename": { 125 | "field": "CacheResponseStatus", 126 | "target_field": "cloudflare.cache.response.status", 127 | "ignore_missing": true 128 | } 129 | }, 130 | { 131 | "rename": { 132 | "field": "CacheTieredFill", 133 | "target_field": "cloudflare.cache.tiered.fill", 134 | "ignore_missing": true 135 | } 136 | }, 137 | { 138 | "rename": { 139 | "field": "ClientASN", 140 | "target_field": "source.as.number", 141 | "ignore_missing": true 142 | } 143 | }, 144 | { 145 | "rename": { 146 | "field": "ClientCountry", 147 | "target_field": "source.geo.country_iso_code", 148 | "ignore_missing": true 149 | } 150 | }, 151 | { 152 | "rename": { 153 | "field": "ClientDeviceType", 154 | "target_field": "cloudflare.device.type", 155 | "ignore_missing": true 156 | } 157 | }, 158 | { 159 | "set": { 160 | "field": "source.address", 161 | "value": "{{ClientIP}}" 162 | } 163 | }, 164 | { 165 | "set": { 166 | "field": "client.ip", 167 | "value": "{{ClientIP}}" 168 | } 169 | }, 170 | { 171 | "set": { 172 | "field": "client.address", 173 | "value": "{{ClientIP}}" 174 | } 175 | }, 176 | { 177 | "rename": { 178 | "field": "ClientIP", 179 | "target_field": "source.ip", 180 | "ignore_missing": true 181 | } 182 | }, 183 | { 184 | "rename": { 185 | "field": "ClientIPClass", 186 | "target_field": "cloudflare.client.ip.class", 187 | "ignore_missing": true 188 | } 189 | }, 190 | { 191 | "rename": { 192 | "field": "ClientRequestBytes", 193 | "target_field": "http.request.bytes", 194 | "ignore_missing": true 195 | } 196 | }, 197 | { 198 | "rename": { 199 | "field": "ClientRequestHost", 200 | "target_field": "url.domain", 201 | "ignore_missing": true 202 | } 203 | }, 204 | { 205 | "rename": { 206 | "field": "ClientRequestMethod", 207 | "target_field": "http.request.method", 208 | "ignore_missing": true 209 | } 210 | }, 211 | { 212 | "rename": { 213 | "field": "ClientRequestPath", 214 | "target_field": "url.path", 215 | "ignore_missing": true 216 | } 217 | }, 218 | { 219 | "rename": { 220 | "field": "ClientRequestReferer", 221 | "target_field": "http.request.referrer", 222 | "ignore_missing": true 223 | } 224 | }, 225 | { 226 | "rename": { 227 | "field": "ClientRequestURI", 228 | "target_field": "url.full", 229 | "ignore_missing": true 230 | } 231 | }, 232 | { 233 | "user_agent" : { 234 | "field" : "ClientRequestUserAgent", 235 | "target_field": "user_agent", 236 | "ecs": true, 237 | "ignore_missing": true 238 | } 239 | }, 240 | { 241 | "set": { 242 | "field": "user_agent.original", 243 | "value": "{{ClientRequestUserAgent}}" 244 | } 245 | }, 246 | { 247 | "remove" : { 248 | "field": "ClientRequestUserAgent", 249 | "ignore_missing": true 250 | } 251 | }, 252 | { 253 | "rename": { 254 | "field": "ClientSSLCipher", 255 | "target_field": "cloudflare.client.ssl.cipher", 256 | "ignore_missing": true 257 | } 258 | }, 259 | { 260 | "rename": { 261 | "field": "ClientSSLProtocol", 262 | "target_field": "cloudflare.client.ssl.protocol", 263 | "ignore_missing": true 264 | } 265 | }, 266 | { 267 | "set": { 268 | "field": "client.port", 269 | "value": "{{ClientSrcPort}}" 270 | } 271 | }, 272 | { 273 | "rename": { 274 | "field": "ClientSrcPort", 275 | "target_field": "source.port", 276 | "ignore_missing": true 277 | } 278 | }, 279 | { 280 | "rename": { 281 | "field": "EdgeColoCode", 282 | "target_field": "cloudflare.edge.colo.code", 283 | "ignore_missing": true 284 | } 285 | }, 286 | { 287 | "rename": { 288 | "field": "EdgeColoID", 289 | "target_field": "cloudflare.edge.colo.id", 290 | "ignore_missing": true 291 | } 292 | }, 293 | { 294 | "set": { 295 | "field": "event.end", 296 | "value": "{{EdgeEndTimestamp}}" 297 | } 298 | }, 299 | { 300 | "rename": { 301 | "field": "EdgeEndTimestamp", 302 | "target_field": "cloudflare.edge.end.timestamp", 303 | "ignore_missing": true 304 | } 305 | }, 306 | { 307 | "rename": { 308 | "field": "EdgePathingOp", 309 | "target_field": "cloudflare.edge.pathing.op", 310 | "ignore_missing": true 311 | } 312 | }, 313 | { 314 | "rename": { 315 | "field": "EdgePathingSrc", 316 | "target_field": "cloudflare.edge.pathing.src", 317 | "ignore_missing": true 318 | } 319 | }, 320 | { 321 | "rename": { 322 | "field": "EdgePathingStatus", 323 | "target_field": "cloudflare.edge.pathing.status", 324 | "ignore_missing": true 325 | } 326 | }, 327 | { 328 | "rename": { 329 | "field": "EdgeRateLimitAction", 330 | "target_field": "cloudflare.edge.rate.limit.action", 331 | "ignore_missing": true 332 | } 333 | }, 334 | { 335 | "rename": { 336 | "field": "EdgeRateLimitID", 337 | "target_field": "cloudflare.edge.rate.limit.id", 338 | "ignore_missing": true 339 | } 340 | }, 341 | { 342 | "rename": { 343 | "field": "EdgeRequestHost", 344 | "target_field": "cloudflare.edge.request.host", 345 | "ignore_missing": true 346 | } 347 | }, 348 | { 349 | "rename": { 350 | "field": "EdgeResponseBytes", 351 | "target_field": "cloudflare.edge.response.bytes", 352 | "ignore_missing": true 353 | } 354 | }, 355 | { 356 | "rename": { 357 | "field": "EdgeResponseCompressionRatio", 358 | "target_field": "cloudflare.edge.response.compression_ratio", 359 | "ignore_missing": true 360 | } 361 | }, 362 | { 363 | "rename": { 364 | "field": "EdgeResponseContentType", 365 | "target_field": "cloudflare.edge.response.content_type", 366 | "ignore_missing": true 367 | } 368 | }, 369 | { 370 | "rename": { 371 | "field": "EdgeResponseStatus", 372 | "target_field": "cloudflare.edge.response.status", 373 | "ignore_missing": true 374 | } 375 | }, 376 | { 377 | "set": { 378 | "field": "observer.ip", 379 | "value": "{{EdgeServerIP}}" 380 | } 381 | }, 382 | { 383 | "rename": { 384 | "field": "EdgeServerIP", 385 | "target_field": "cloudflare.edge.server.ip", 386 | "ignore_missing": true 387 | } 388 | }, 389 | { 390 | "set": { 391 | "field": "event.start", 392 | "value": "{{EdgeStartTimestamp}}" 393 | } 394 | }, 395 | { 396 | "rename": { 397 | "field": "EdgeStartTimestamp", 398 | "target_field": "cloudflare.edge.start.timestamp", 399 | "ignore_missing": true 400 | } 401 | }, 402 | { 403 | "rename": { 404 | "field": "FirewallMatchesActions", 405 | "target_field": "cloudflare.firewall.matches.actions", 406 | "ignore_missing": true 407 | } 408 | }, 409 | { 410 | "rename": { 411 | "field": "FirewallMatchesSources", 412 | "target_field": "cloudflare.firewall.matches.sources", 413 | "ignore_missing": true 414 | } 415 | }, 416 | { 417 | "rename": { 418 | "field": "FirewallMatchesRuleIDs", 419 | "target_field": "cloudflare.firewall.matches.rule_ids", 420 | "ignore_missing": true 421 | } 422 | }, 423 | { 424 | "set": { 425 | "field": "server.ip", 426 | "value": "{{OriginIP}}" 427 | } 428 | }, 429 | { 430 | "set": { 431 | "field": "cloudflare.origin.ip", 432 | "value": "{{OriginIP}}" 433 | } 434 | }, 435 | { 436 | "rename": { 437 | "field": "OriginIP", 438 | "target_field": "destination.ip", 439 | "ignore_missing": true 440 | } 441 | }, 442 | { 443 | "rename": { 444 | "field": "OriginResponseBytes", 445 | "target_field": "cloudflare.origin.response.bytes", 446 | "ignore_missing": true 447 | } 448 | }, 449 | { 450 | "remove" : { 451 | "if" : "ctx.OriginResponseHTTPExpires == ''", 452 | "field" : "OriginResponseHTTPExpires" 453 | } 454 | }, 455 | { 456 | "rename": { 457 | "field": "OriginResponseHTTPExpires", 458 | "target_field": "cloudflare.origin.response.http.expires", 459 | "ignore_missing": true 460 | } 461 | }, 462 | { 463 | "remove" : { 464 | "if" : "ctx.OriginResponseHTTPLastModified == ''", 465 | "field" : "OriginResponseHTTPLastModified" 466 | } 467 | }, 468 | { 469 | "rename": { 470 | "field": "OriginResponseHTTPLastModified", 471 | "target_field": "cloudflare.origin.response.http.last_modified", 472 | "ignore_missing": true 473 | } 474 | }, 475 | { 476 | "rename": { 477 | "field": "OriginResponseStatus", 478 | "target_field": "cloudflare.http.response.status_code", 479 | "ignore_missing": true 480 | } 481 | }, 482 | { 483 | "rename": { 484 | "field": "OriginResponseTime", 485 | "target_field": "cloudflare.origin.response.time", 486 | "ignore_missing": true 487 | } 488 | }, 489 | { 490 | "rename": { 491 | "field": "OriginSSLProtocol", 492 | "target_field": "cloudflare.origin.ssl.protocol", 493 | "ignore_missing": true 494 | } 495 | }, 496 | { 497 | "rename": { 498 | "field": "ParentRayID", 499 | "target_field": "cloudflare.parent.ray_id", 500 | "ignore_missing": true 501 | } 502 | }, 503 | { 504 | "rename": { 505 | "field": "RayID", 506 | "target_field": "cloudflare.ray_id", 507 | "ignore_missing": true 508 | } 509 | }, 510 | { 511 | "rename": { 512 | "field": "SecurityLevel", 513 | "target_field": "cloudflare.security_level", 514 | "ignore_missing": true 515 | } 516 | }, 517 | { 518 | "rename": { 519 | "field": "WAFAction", 520 | "target_field": "cloudflare.waf.action", 521 | "ignore_missing": true 522 | } 523 | }, 524 | { 525 | "rename": { 526 | "field": "WAFFlags", 527 | "target_field": "cloudflare.waf.flags", 528 | "ignore_missing": true 529 | } 530 | }, 531 | { 532 | "rename": { 533 | "field": "WAFMatchedVar", 534 | "target_field": "cloudflare.waf.matched_var", 535 | "ignore_missing": true 536 | } 537 | }, 538 | { 539 | "rename": { 540 | "field": "WAFProfile", 541 | "target_field": "cloudflare.waf.profile", 542 | "ignore_missing": true 543 | } 544 | }, 545 | { 546 | "rename": { 547 | "field": "WAFRuleID", 548 | "target_field": "cloudflare.waf.rule.id", 549 | "ignore_missing": true 550 | } 551 | }, 552 | { 553 | "rename": { 554 | "field": "WAFRuleMessage", 555 | "target_field": "cloudflare.waf.rule.message", 556 | "ignore_missing": true 557 | } 558 | }, 559 | { 560 | "rename": { 561 | "field": "WorkerCPUTime", 562 | "target_field": "cloudflare.worker.cpu_time", 563 | "ignore_missing": true 564 | } 565 | }, 566 | { 567 | "rename": { 568 | "field": "WorkerStatus", 569 | "target_field": "cloudflare.worker.status", 570 | "ignore_missing": true 571 | } 572 | }, 573 | { 574 | "rename": { 575 | "field": "WorkerSubrequest", 576 | "target_field": "cloudflare.worker.subrequest", 577 | "ignore_missing": true 578 | } 579 | }, 580 | { 581 | "rename": { 582 | "field": "WorkerSubrequestCount", 583 | "target_field": "cloudflare.worker.subrequest_count", 584 | "ignore_missing": true 585 | } 586 | }, 587 | { 588 | "rename": { 589 | "field": "ZoneID", 590 | "target_field": "cloudflare.zone_id", 591 | "ignore_missing": true 592 | } 593 | } 594 | ] 595 | } 596 | 597 | -------------------------------------------------------------------------------- /conf/install-artifacts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function usage { 4 | echo "" 5 | echo "usage: install-artifacts.sh -u -p -e " 6 | echo " -u elasticsearch username" 7 | echo " -p elasticsearch password" 8 | echo " -e elasticsearch endpoint, e.g. https://localhost:9200" 9 | echo "" 10 | } 11 | 12 | username="" 13 | password="" 14 | endpoint="" 15 | auth="" 16 | 17 | while getopts ":u:p:e:h" opt; do 18 | case $opt in 19 | u) 20 | username=$OPTARG 21 | ;; 22 | p) 23 | password=$OPTARG 24 | ;; 25 | e) 26 | endpoint=$OPTARG 27 | ;; 28 | h) 29 | usage 30 | exit 1 31 | ;; 32 | \?) 33 | usage 34 | exit 1 35 | ;; 36 | esac 37 | done 38 | 39 | 40 | if [[ "$username" && "$password" ]]; 41 | then 42 | auth="-u $username:$password" 43 | fi 44 | 45 | if [ -z "$endpoint" ]; 46 | then 47 | usage 48 | exit 1 49 | fi 50 | 51 | 52 | echo "" 53 | echo "Installing ingest pipeline to $endpoint for daily indices" 54 | curl -X PUT $auth "$endpoint/_ingest/pipeline/cloudflare-pipeline-daily" -H 'Content-Type: application/json' -d @cloudflare-ingest-pipeline-daily.json 55 | echo "" 56 | echo "Installing ingest pipeline to $endpoint for weekly indices" 57 | curl -X PUT $auth "$endpoint/_ingest/pipeline/cloudflare-pipeline-weekly" -H 'Content-Type: application/json' -d @cloudflare-ingest-pipeline-weekly.json 58 | echo "" 59 | echo "Installing index template to $endpoint" 60 | curl -X PUT $auth "$endpoint/_template/cloudflare" -H 'Content-Type: application/json' -d @cloudflare-index-template.json 61 | echo "" 62 | -------------------------------------------------------------------------------- /conf/logstash-cloudflare.conf: -------------------------------------------------------------------------------- 1 | input { 2 | stdin {} 3 | } 4 | 5 | filter { 6 | json { 7 | source => "message" 8 | } 9 | geoip { 10 | source => "ClientIP" 11 | target => "geoip" 12 | } 13 | mutate { 14 | rename => ["geoip", "ClientIP" ] 15 | } 16 | 17 | date { 18 | match => [ "EdgeStartTimestamp", "yyyy-MM-dd'T'HH:mm:ssZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZ" ] 19 | timezone => "UTC" 20 | target => "@timestamp" 21 | } 22 | date { 23 | match => [ "EdgeStartTimestamp", "yyyy-MM-dd'T'HH:mm:ssZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZ" ] 24 | timezone => "UTC" 25 | target => "EdgeStartTimestamp" 26 | } 27 | useragent { 28 | source => "ClientRequestUserAgent" 29 | target => "UserAgent" 30 | } 31 | } 32 | 33 | output { 34 | stdout { 35 | codec => rubydebug 36 | } 37 | elasticsearch { 38 | hosts => [" < YOUR ELASTICSEARCH HOSTS GO HERE > "] 39 | index => "cloudflare-%{+YYYY.MM.dd}" 40 | user => "elastic" 41 | password => "< YOUR ELASTICSEARCH PASSWORD GOES HERE >" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /dashboards/README.md: -------------------------------------------------------------------------------- 1 | 2 | ##### Kibana Dashboards 3 | 4 | Pre-built dashboards for viewing Cloudflare logs and metrics. You must import these into Kibana using the **Import Saved Objects** interface (Management/Kibana/Saved Objects). 5 | 6 | 7 | ![Import Dashboards](./import-kibana-dashboards.png?raw=true "Title") 8 | -------------------------------------------------------------------------------- /dashboards/import-kibana-dashboards.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudflarearchive/cloudflare-elastic/6fe7b2ac9d943da80ac1ca99b1513fba0e62f9fd/dashboards/import-kibana-dashboards.png -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudflarearchive/cloudflare-elastic/6fe7b2ac9d943da80ac1ca99b1513fba0e62f9fd/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Mon Dec 02 17:06:08 PST 2019 2 | distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-all.zip 3 | distributionBase=GRADLE_USER_HOME 4 | distributionPath=wrapper/dists 5 | zipStorePath=wrapper/dists 6 | zipStoreBase=GRADLE_USER_HOME 7 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS='"-Xmx64m"' 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn () { 37 | echo "$*" 38 | } 39 | 40 | die () { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save () { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS="-Xmx64m" 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | 2 | rootProject.name = 'cloudflare-elastic' 3 | 4 | def prefix(prefix) { 5 | rootProject.name = prefix 6 | rootProject.children.each { it.name = "${prefix}-${it.name}" } 7 | } 8 | 9 | include 'aws' 10 | 11 | prefix('cloudflare-elastic') 12 | 13 | --------------------------------------------------------------------------------