├── docs ├── partner_editable │ ├── regions.adoc │ ├── specialized_knowledge.adoc │ ├── licenses.adoc │ ├── service_limits.adoc │ ├── overview_target_and_usage.adoc │ ├── _settings.adoc │ ├── product_description.adoc │ ├── faq_troubleshooting.adoc │ ├── pre-reqs.adoc │ ├── deployment_options.adoc │ ├── architecture.adoc │ ├── deploy_steps.adoc │ └── additional_info.adoc └── images │ ├── cfn_outputs.png │ ├── cluster-master-sfrf-met.png │ ├── indexer-clustering-menu.png │ ├── search-head-distributed-search-menu.png │ ├── splunk-enterprise-architecture-on-aws.png │ └── search-head-distributed-search-success.png ├── .gitmodules ├── NOTICE.txt ├── ci ├── taskcat.yml └── defaults.json ├── README.md ├── LICENSE.txt ├── templates ├── splunk-enterprise-master.template └── splunk-enterprise.template └── scripts └── user_data.sh /docs/partner_editable/regions.adoc: -------------------------------------------------------------------------------- 1 | - All AWS Regions 2 | -------------------------------------------------------------------------------- /docs/images/cfn_outputs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/quickstart-splunk-enterprise/HEAD/docs/images/cfn_outputs.png -------------------------------------------------------------------------------- /docs/images/cluster-master-sfrf-met.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/quickstart-splunk-enterprise/HEAD/docs/images/cluster-master-sfrf-met.png -------------------------------------------------------------------------------- /docs/images/indexer-clustering-menu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/quickstart-splunk-enterprise/HEAD/docs/images/indexer-clustering-menu.png -------------------------------------------------------------------------------- /docs/images/search-head-distributed-search-menu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/quickstart-splunk-enterprise/HEAD/docs/images/search-head-distributed-search-menu.png -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "submodules/quickstart-aws-vpc"] 2 | path = submodules/quickstart-aws-vpc 3 | url = ../../aws-quickstart/quickstart-aws-vpc.git 4 | branch = master 5 | -------------------------------------------------------------------------------- /docs/images/splunk-enterprise-architecture-on-aws.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/quickstart-splunk-enterprise/HEAD/docs/images/splunk-enterprise-architecture-on-aws.png -------------------------------------------------------------------------------- /docs/images/search-head-distributed-search-success.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/quickstart-splunk-enterprise/HEAD/docs/images/search-head-distributed-search-success.png -------------------------------------------------------------------------------- /docs/partner_editable/specialized_knowledge.adoc: -------------------------------------------------------------------------------- 1 | // Describe or link to specific knowledge requirements; for example: “familiarity with basic concepts in the areas of networking, database operations, and data encryption” or “familiarity with .” 2 | 3 | This Quick Start assumes familiarity with basic concepts of networking and Linux system administration, as well as basic knowledge of {partner-product-name} 4 | -------------------------------------------------------------------------------- /docs/partner_editable/licenses.adoc: -------------------------------------------------------------------------------- 1 | This Quick Start requires a subscription to the Amazon Machine Image (AMI) for {partner-product-name}, which is available from AWS Marketplace. For subscription instructions, see step 2 under "Subscribe to the {partner-product-name} AMI" in the link:#_deployment_steps[deployment steps]. In addition, to fully utilize the Quick Start environment, a {partner-product-name} license is required. If you do not have one, please contact sales@splunk.com 2 | -------------------------------------------------------------------------------- /docs/partner_editable/service_limits.adoc: -------------------------------------------------------------------------------- 1 | // Replace the in each row to specify the number of resources used in this deployment. Remove the rows for resources that aren’t used. 2 | 3 | |=== 4 | |Resource |This deployment uses 5 | 6 | // Space needed to maintain table headers 7 | |VPCs |1 8 | |AWS Identity and Access Management (IAM) security groups |2 or more 9 | |IAM roles |2 or more 10 | |Auto Scaling groups |1 11 | |Classic Load Balancers |2 12 | |EC2 Instances |5 or more 13 | |EBS Volumes|5 or more 14 | |S3 Buckets |1 15 | |=== 16 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2016-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 4 | 5 | http://aws.amazon.com/apache2.0/ 6 | 7 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 8 | -------------------------------------------------------------------------------- /docs/partner_editable/overview_target_and_usage.adoc: -------------------------------------------------------------------------------- 1 | This Quick Start provides architectural guidance and step-by-step instructions for a high availability deployment of {partner-product-name} on AWS. The guide addresses common scalability, high-availability, and security considerations for your deployment. 2 | 3 | This guide is intended for a variety of audiences, including IT infrastructure architects, administrators, and DevOps professionals who are planning to implement or extend their Splunk Enterprise deployments on the AWS Cloud. It also provides direct links for viewing and launching AWS CloudFormation templates that assist with automating the deployment 4 | -------------------------------------------------------------------------------- /ci/taskcat.yml: -------------------------------------------------------------------------------- 1 | global: 2 | marketplace-ami: true 3 | owner: quickstart@amazon.com 4 | qsname: quickstart-splunk-enterprise 5 | regions: 6 | - ap-northeast-1 7 | - ap-northeast-2 8 | - ap-south-1 9 | - ap-southeast-1 10 | - ap-southeast-2 11 | - ca-central-1 12 | - eu-central-1 13 | - eu-west-1 14 | - eu-west-2 15 | - sa-east-1 16 | - us-east-1 17 | - us-east-2 18 | - us-west-1 19 | - us-west-2 20 | reporting: true 21 | tests: 22 | splunk-enterprise: 23 | parameter_input: defaults.json 24 | template_file: splunk-enterprise-master.template 25 | regions: 26 | - us-west-1 27 | - us-east-2 28 | -------------------------------------------------------------------------------- /docs/partner_editable/_settings.adoc: -------------------------------------------------------------------------------- 1 | :quickstart-project-name: quickstart-splunk-enterprise 2 | :partner-product-name: Splunk Enterprise 3 | :partner-company-name: Splunk Inc. 4 | :doc-month: March 5 | :doc-year: 2021 6 | :partner-contributors: Bill Bartlett, {partner-company-name} 7 | :quickstart-contributors: Shivansh Singh, Amazon Web Services 8 | :deployment_time: 25 minutes 9 | :default_deployment_region: us-east-1 10 | // Uncomment these two attributes if you are leveraging 11 | // - an AWS Marketplace listing. 12 | // Additional content will be auto-generated based on these attributes. 13 | :marketplace_subscription: 14 | :marketplace_listing_url: https://aws.amazon.com/marketplace/pp/Splunk-Splunk-Enterprise/B00PUXWXNE 15 | -------------------------------------------------------------------------------- /docs/partner_editable/product_description.adoc: -------------------------------------------------------------------------------- 1 | This Quick Start deploys {partner-product-name} on the AWS Cloud 2 | 3 | {partner-product-name} is the platform for turning data into doing. By monitoring and analyzing everything from customer clickstreams and transactions to security events and network activity, {partner-product-name} is a scalable and reliable data platform for investigating, monitoring, analyzing and acting on your data. With a full range of powerful search, analysis, alert, and visualization capabilities along with prepackaged content for many typical use cases, users can quickly discover and share insights. 4 | 5 | For more details about the features and functionality of {partner-product-name}, see the https://docs.splunk.com/Documentation/Splunk[Splunk Enterprise documentation^]. 6 | 7 | 8 | -------------------------------------------------------------------------------- /docs/partner_editable/faq_troubleshooting.adoc: -------------------------------------------------------------------------------- 1 | // Add any tips or answers to anticipated questions. This could include the following troubleshooting information. If you don’t have any other Q&A to add, change “FAQ” to “Troubleshooting.” 2 | 3 | == FAQ 4 | *Q.* Why is my search factor and/or replication factor not being met? 5 | 6 | *A.* The most common reason for this is the cluster replication or search factor is set higher than the minimum indexers in a site. For example, if you create a 5 node cluster across two sites, you will have 3 indexers in site1, and 2 indexers in site2. In this example, if you also configured RF and/or SF = 3, Splunk will not be able to meet the required replication or search factor. (RF or SF = 3, but you only have 2 indexers in site2) For further reading, this topic is outlined in https://docs.splunk.com/Documentation/Splunk/8.1.2/Indexer/Bucketreplicationissues#Multisite_cluster_does_not_meet_its_replication_or_search_factors[Splunk documentation^]. 7 | -------------------------------------------------------------------------------- /docs/partner_editable/pre-reqs.adoc: -------------------------------------------------------------------------------- 1 | // If no preperation is required, remove all content from here. 2 | There are two important steps to verify prior to launching this Quick start: 3 | 4 | * Ensure that the SmartStore bucket that is defined in the "SmartStoreBucketName" parameter *does not exist*. This Quick Start will attempt to create that bucket with an appropriate bucket policy. If that bucket already exists, the Quick Start will fail. 5 | * Upload a valid Splunk license to an S3 bucket owned by the user launching the Quick Start. Take note of the bucket name as well as the path to the license file. An example may look something like this: 6 | ** Splunk license file is named "splunk.license" and uploaded to an S3 bucket called "my-s3-bucket" under the 'directory' called "license" 7 | *** Parameter "SplunkLicenseBucket" should be configured to: my-s3-bucket 8 | *** Parameter "SplunkLicensePath" should be configured to: license/splunk.license (note the lack of a leading "/" on the license path) 9 | 10 | -------------------------------------------------------------------------------- /ci/defaults.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "ParameterKey": "AvailabilityZones", 4 | "ParameterValue": "$[taskcat_genaz_2]" 5 | }, 6 | { 7 | "ParameterKey": "WebClientLocation", 8 | "ParameterValue": "72.21.196.66/32" 9 | }, 10 | { 11 | "ParameterKey": "HECClientLocation", 12 | "ParameterValue": "10.0.0.0/16" 13 | }, 14 | { 15 | "ParameterKey": "KeyName", 16 | "ParameterValue": "$[taskcat_getkeypair]" 17 | }, 18 | { 19 | "ParameterKey": "SSHClientLocation", 20 | "ParameterValue": "10.0.0.0/16" 21 | }, 22 | { 23 | "ParameterKey": "SplunkAdminPassword", 24 | "ParameterValue": "$[taskcat_genpass_10]" 25 | }, 26 | { 27 | "ParameterKey": "SplunkClusterSecret", 28 | "ParameterValue": "$[taskcat_genpass_10]" 29 | }, 30 | { 31 | "ParameterKey": "SplunkIndexerDiscoverySecret", 32 | "ParameterValue": "$[taskcat_genpass_10]" 33 | }, 34 | { 35 | "ParameterKey": "QSS3BucketName", 36 | "ParameterValue": "$[taskcat_autobucket]" 37 | } 38 | ] 39 | -------------------------------------------------------------------------------- /docs/partner_editable/deployment_options.adoc: -------------------------------------------------------------------------------- 1 | // There are generally two deployment options. If additional are required, add them here 2 | 3 | This Quick Start provides two deployment options: 4 | 5 | * *Deploy {partner-product-name} into a new VPC (end-to-end deployment)*. This option builds a new AWS environment consisting of the VPC, subnets, security groups, load balancers, and other infrastructure components. It then deploys {partner-product-name} into this new VPC. 6 | 7 | * *Deploy {partner-product-name} into an existing VPC*. This option provisions {partner-product-name} in your existing AWS infrastructure. 8 | 9 | The Quick Start provides separate templates for these options. It also lets you configure Classless Inter-Domain Routing (CIDR) blocks, instance types, and {partner-product-name} settings, as discussed later in this guide. 10 | 11 | For further information about the dimensions of a Splunk Enterprise deployment, see the {partner-product-name} https://docs.splunk.com/Documentation/Splunk/latest/Capacity/DimensionsofaSplunkEnterprisedeployment[capacity planning manual^]. Finally, for additional information specific to an AWS deployment, see {partner-product-name} https://www.splunk.com/pdfs/technical-briefs/deploying-splunk-enterprise-on-amazon-web-services-technical-brief.pdf[AWS tech brief^] for instance sizing considerations. 12 | -------------------------------------------------------------------------------- /docs/partner_editable/architecture.adoc: -------------------------------------------------------------------------------- 1 | Deploying this Quick Start for a new VPC with default parameters builds the following {partner-product-name} environment in the AWS Cloud. 2 | [#architecture1] 3 | .Quick Start architecture for {partner-product-name} on AWS 4 | [link=images/splunk-enterprise-architecture-on-aws.png] 5 | image::../images/splunk-enterprise-architecture-on-aws.png[Architecture,width=648,height=439] 6 | 7 | As shown in figure 1, the Quick Start sets up the following: 8 | 9 | * A VPC configured across two or three Availability Zones, depending on your selection. The Quick Start provisions one public subnet in each Availability Zone. 10 | * Two Elastic Load Balancing (ELB) load balancers: one to load-balance HTTP web traffic to the search head instances, and the other to load-balance HTTP event traffic destined for the Splunk HTTP Event Collector (HEC) across all indexer instances. 11 | * An IAM user with fine-grained permissions for access to AWS services necessary for the initial deployment process. 12 | * Appropriate security groups for each instance or function to restrict access to only necessary protocols and ports. 13 | * Amazon Simple Storage Service (Amazon S3) bucket for Splunk SmartStore usage. 14 | * In the public subnets, EC2 instances for {partner-product-name}, including the following: 15 | ** {partner-product-name} indexer cluster with the number of indexers you specify (3-10), distributed across the number of Availability Zones you specify (2 or 3). The Splunk receiver (splunktcp) and Splunk HEC are enabled across all indexers. 16 | ** Splunk search head(s), either stand-alone or in a 3-node cluster, based on your input during deployment. In the latter case, the search heads are distributed across the number of Availability Zones you specify. 17 | ** Splunk license server and indexer cluster master, co-located. 18 | ** Splunk search head deployer, where applicable. 19 | 20 | 21 | If you decide to deploy Splunk Enterprise into your existing VPC, please see link:#_deployment_options[deployment options] later in this guide. The Quick Start assumes that the infrastructure components already exist, and deploys Splunk Enterprise into the environment you specify. 22 | 23 | 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Splunk Enterprise on AWS - Quick Start 2 | 3 | Source code associated with [Splunk Enterprise AWS Quick Start](https://fwd.aws/r7QNJ) 4 | 5 | ## Usage 6 | 7 | Use these templates to deploy a highly available Splunk Enterprise environment across multiple AZs (2 or 3) in a given AWS region. 8 | AZ-aware indexer clustering is enabled for horizontal scaling and to guarantee data is replicated in every AZ. 9 | AZ-aware Search head clustering (3 nodes by default) can also be enabled for horizontal scaling and to guarantee data is available for search in every AZ. 10 | 11 | View the accompanying [deployment guide](https://fwd.aws/bGBmy) for everything you need to get started. Refer to 'Deployment Steps' section for a step-by-step walkthrough on how to use these templates in AWS console. 12 | 13 | ### Prerequisites 14 | 15 | Before getting started with the template configuration, you will need to make your Splunk Enterprise license privately accessible for CloudFormation template deployment via S3 download. The following steps will guide you through that process. *(Note: This step is required. A non-trial Splunk Enterprise license is required to allow our template to configure the Splunk deployment. If you don't already have a Splunk Enterprise license, you can obtain one by contacting sales@splunk.com.)* 16 | 17 | 1. From the AWS Console, select "S3" under the "Storage" heading, or by simply typing "S3" into the search bar. 18 | 2. You can either select an existing private bucket to upload to, or create a new one. If you select an existing bucket, make sure its access policy does not grant public access. By default, all the S3 resources are private, so only the AWS account that created the resources can access them. For this exercise, I'm outlining how to create a new bucket. 19 | 3. Click "create bucket" 20 | 4. Name your bucket, and select your region. In this example, I will use "bbartlett-splunk-config". Your bucket name must be unique, and you should select the same region where you plan on deploying Splunk.

![new bucket example](https://s3-us-west-2.amazonaws.com/splk-bbartlett/splunk_newbucket.png)

21 | 5. Once you've created your bucket, select your new bucket from the list of buckets. 22 | 6. Click "Upload" on the upper left of the page 23 | 7. Click "Add Files" 24 | 8. Select your license file. 25 | 9. Click "Start Upload" on the lower right of the page. 26 | 10. Once the license has finished uploading, you'll need the bucket name and the filename to use with the CloudFormation template. 27 | 28 | ## License 29 | 30 | This project is licensed under Apache License 2.0 - see [LICENSE.txt](./LICENSE.txt) file for details 31 | 32 | ## Help 33 | 34 | If you have any problems or general questions, please file an issue in the parent repository: 35 | https://github.com/aws-quickstart/quickstart-splunk-enterprise/issues 36 | 37 | 38 | -------------------------------------------------------------------------------- /docs/partner_editable/deploy_steps.adoc: -------------------------------------------------------------------------------- 1 | // We need to work around Step numbers here if we are going to potentially exclude the AMI subscription 2 | === Sign in to your AWS account 3 | 4 | . Sign in to your AWS account at https://aws.amazon.com with an AWS Identity and Access Management (IAM) user role that has the necessary permissions. For details, see link:#_planning_the_deployment[Planning the deployment] earlier in this guide. 5 | . Make sure that your AWS account is configured correctly, as discussed in the link:#_technical_requirements[Technical requirements] section. 6 | 7 | // Optional based on Marketplace listing. Not to be edited 8 | ifdef::marketplace_subscription[] 9 | === Subscribe to the {partner-product-name} AMI 10 | 11 | This Quick Start requires a subscription to the Amazon Machine Image (AMI) for {partner-product-name} in AWS Marketplace. 12 | 13 | . Sign in to your AWS account. 14 | . {marketplace_listing_url}[Open the page for the {partner-product-name} AMI in AWS Marketplace], and then choose *Continue to Subscribe*. 15 | . Review the terms and conditions for software usage, and then choose *Accept Terms*. + 16 | A confirmation page loads, and an email confirmation is sent to the account owner. For detailed subscription instructions, see the https://aws.amazon.com/marketplace/help/200799470[AWS Marketplace documentation^]. 17 | 18 | . When the subscription process is complete, close AWS Marketplace without further action. *Do not* provision the software from AWS Marketplace, as the Quick Start deploys the AMI for you. 19 | endif::marketplace_subscription[] 20 | // \Not to be edited 21 | 22 | === Launch the Quick Start 23 | 24 | NOTE: You are responsible for the cost of the AWS services used while running this Quick Start reference deployment. There is no additional cost for using this Quick Start. For full details, see the pricing pages for each AWS service used by this Quick Start. Prices are subject to change. 25 | 26 | . Sign in to your AWS account, and choose one of the following options to launch the AWS CloudFormation template. For help with choosing an option, see link:#_deployment_options[deployment options] earlier in this guide. 27 | 28 | [cols=",] 29 | |=== 30 | |https://fwd.aws/MNRVe[Deploy {partner-product-name} into a new VPC on AWS^] 31 | |https://fwd.aws/DD3gQ[Deploy {partner-product-name} into an existing VPC on AWS^] 32 | |=== 33 | 34 | WARNING: If you’re deploying {partner-product-name} into an existing VPC, make sure that your VPC has at least two subnets in different Availability Zones for the indexers and search head(s), and that the subnets aren’t shared. If you choose to deploy into three AZ, then your VPC will require at least three available AZ with three separate subnets. This Quick Start doesn’t support https://docs.aws.amazon.com/vpc/latest/userguide/vpc-sharing.html[shared subnets^]. 35 | 36 | Each deployment takes about {deployment_time} to complete. 37 | 38 | [start=2] 39 | . Check the AWS Region that’s displayed in the upper-right corner of the navigation bar, and change it if necessary. This Region is where {partner-product-name} will be deployed. The template is launched in the {default_deployment_region} Region by default. 40 | 41 | [start=3] 42 | . On the *Create stack* page, keep the default setting for the template URL, and then choose *Next*. 43 | . On the *Specify stack details* page, change the stack name if needed. Review the parameters for the template. Provide values for the parameters that require input. For all other parameters, review the default settings and customize them as necessary. 44 | 45 | // In the following tables, parameters are listed by category and described separately for the two deployment options: 46 | 47 | // * Parameters for deploying {partner-product-name} into a new VPC 48 | 49 | // * Parameters for deploying {partner-product-name} into an existing VPC 50 | -------------------------------------------------------------------------------- /docs/partner_editable/additional_info.adoc: -------------------------------------------------------------------------------- 1 | // Add steps as necessary for accessing the software, post-configuration, and testing. Don’t include full usage instructions for your software, but add links to your product documentation for that information. 2 | //Should any sections not be applicable, remove them 3 | 4 | == Post deployment steps 5 | // If steps are required to test the deployment, add them here. If not, remove the heading 6 | 7 | After the Quick Start has successfully completed, you can log into your {partner-product-name} deployment from a web browser and verify configuration. 8 | 9 | ==== Verify Distributed Search 10 | . Begin by logging into {partner-product-name} search head to verify all of the indexers are available for search. To log into the {partner-product-name} search head, navigate your browser to the URL shown in the CloudFormation Outputs labeled "SearchHeadURL" with the credentials of "admin" and the password configured with the "SplunkAdminPassword" parameter when launching the Quick Start. 11 | . Navigate to Settings -> 'Distributed search' menu item as shown below 12 | 13 | image:../images/search-head-distributed-search-menu.png[distributed_search_menu,width=850,height=294,link="../docs/images/search-head-distributed-search-menu.png"] 14 | [start=3] 15 | . Click on "Search peers" 16 | 17 | . A screen similar to the screenshot below indicates that distributed search in good standing. (This example was created with a 4 node indexer cluster.) 18 | 19 | image:../images/search-head-distributed-search-success.png[distributed_search_success,width=850,height=202,link="../docs/images/search-head-distributed-search-success.png"] 20 | 21 | ==== Verify Indexer Replication Status 22 | . Begin by logging into {partner-product-name} cluster master to verify all of the indexers are successfully replicating buckets across the cluster. To log into the {partner-product-name} cluster master, navigate your browser to the URL shown in the CloudFormation Outputs labeled "ClusterMasterURL" with the credentials of "admin" and the password configured with the "SplunkAdminPassword" parameter when launching the Quick Start. 23 | . Navigate to Settings -> 'Indexer clustering' menu item as shown below 24 | 25 | image:../images/indexer-clustering-menu.png[indexer_clustering_menu,width=850,height=294,link="../docs/images/indexer-clustering-menu.png"] 26 | [start=3] 27 | . A screen similar to the screenshot below indicates that both the search factor and replication factor are in good standing. (This example was created with a 4 node indexer cluster across 2 AZ) 28 | .. _Please note that it will likely take a few minutes after the Quick Start has successfully launched before the buckets are replicated and this status window shows both search factor and replication factor being met. If there are errors for replication and/or search factor after 10 minutes, please see the link:#_faq[FAQ section] below._ 29 | 30 | image:../images/cluster-master-sfrf-met.png[indexer_clustering_success,width=850,height=202,link="../docs/images/cluster-master-sfrf-met.png"] 31 | 32 | == Security 33 | // Provide post-deployment best practices for using the technology on AWS, including considerations such as migrating data, backups, ensuring high performance, high availability, etc. Link to software documentation for detailed information. 34 | 35 | The {partner-product-name} Quick Start exposes three user-configurable security group access parameters: 'WebClientLocation', 'HECClientLocation', and 'SSHClientLocation'. Be sure that the 'SSHClientLocation' parameter is accessible only on tightly controlled authorized network ranges as this allows direct access to the instances. The parameter 'WebClientLocation' allows connections to the {partner-product-name} web interfaces, while 'HECClientLocation' controls access to the load balancer in front of the {partner-product-name} HTTP Event Collector listener. 36 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /templates/splunk-enterprise-master.template: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Description: Splunk deployment with indexer clustering, search head (optional search head clustering), and cluster master/license server. 3 | Parameters: 4 | AvailabilityZones: 5 | Description: List of Availability Zones to use for the subnets in the VPC (logical order preserved). This must match the Number of Availability Zones parameter value. 6 | Type: List 7 | NumberOfAZs: 8 | AllowedValues: 9 | - '2' 10 | - '3' 11 | Default: '2' 12 | Description: Number of Availability Zones to use in the VPC. This must match your selections in the list of Availability Zones parameter. 13 | Type: String 14 | WebClientLocation: 15 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 16 | ConstraintDescription: Must be a valid IP range in x.x.x.x/x notation. Use 0.0.0.0/0 for no restrictions. 17 | Description: 'The IP address range that is allowed to connect to the Splunk web interface. Note: a value of 0.0.0.0/0 will allow access from ANY ip address.' 18 | MaxLength: '19' 19 | MinLength: '9' 20 | Type: String 21 | HECClientLocation: 22 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 23 | ConstraintDescription: Must be a valid IP range in x.x.x.x/x notation. Use 0.0.0.0/0 for no restrictions. 24 | Description: 'The IP address range that is allowed to send data to Splunk HTTP Event Collector. Note: a value of 0.0.0.0/0 will allow access from ANY ip address.' 25 | MaxLength: '19' 26 | MinLength: '9' 27 | Type: String 28 | IndexerInstanceType: 29 | AllowedValues: 30 | - m5.4xlarge 31 | - m5.8xlarge 32 | - c5.4xlarge 33 | - c5.9xlarge 34 | - c5.18xlarge 35 | - i3.4xlarge 36 | - i3.8xlarge 37 | - i3en.3xlarge 38 | - i3en.6xlarge 39 | - i3en.12xlarge 40 | Description: EC2 instance type for Splunk Indexers 41 | ConstraintDescription: must be a valid EC2 instance type. 42 | Default: i3.4xlarge 43 | Type: String 44 | SearchHeadInstanceType: 45 | AllowedValues: 46 | - r5.4xlarge 47 | - r5.8xlarge 48 | - r5.16xlarge 49 | - c5.4xlarge 50 | - c5.9xlarge 51 | - m5.2xlarge 52 | - m5.4xlarge 53 | - m5.8xlarge 54 | - m5.12xlarge 55 | Description: EC2 instance type for Splunk Search Heads 56 | ConstraintDescription: must be a valid EC2 instance type. 57 | Default: c5.4xlarge 58 | Type: String 59 | KeyName: 60 | ConstraintDescription: Must be the name of an existing EC2 KeyPair. 61 | Description: Name of an existing EC2 KeyPair to enable SSH access to the instance 62 | Type: AWS::EC2::KeyPair::KeyName 63 | PublicSubnet1CIDR: 64 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 65 | ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x. 66 | Default: 10.0.1.0/24 67 | Description: The address space that will be assigned to the first Splunk server subnet. (x.x.x.x/x notation) 68 | Type: String 69 | PublicSubnet2CIDR: 70 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 71 | ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x 72 | Default: 10.0.2.0/24 73 | Description: The address space that will be assigned to the second Splunk server subnet. (x.x.x.x/x notation) 74 | Type: String 75 | PublicSubnet3CIDR: 76 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 77 | ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x. 78 | Default: 10.0.3.0/24 79 | Description: The address space that will be assigned to the second Splunk server subnet. (x.x.x.x/x notation) 80 | Type: String 81 | QSS3BucketName: 82 | Description: S3 bucket name for the Quick Start assets. 83 | Default: '' 84 | Type: String 85 | QSS3KeyPrefix: 86 | Default: quickstart-splunk-enterprise/ 87 | Description: S3 key prefix for the Quick Start assets. 88 | Type: String 89 | SHCEnabled: 90 | AllowedValues: 91 | - 'yes' 92 | - 'no' 93 | Default: 'no' 94 | Description: Do you want to build a Splunk search head cluster? yes or no 95 | Type: String 96 | SSHClientLocation: 97 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 98 | ConstraintDescription: Must be a valid IP range in x.x.x.x/x notation. Use 0.0.0.0/0 for no restrictions. 99 | Description: 'The IP address range that is allowed to SSH to the EC2 instances. Note: a value of 0.0.0.0/0 will allow access from ANY ip address' 100 | MaxLength: '19' 101 | MinLength: '9' 102 | Type: String 103 | SplunkAdminPassword: 104 | AllowedPattern: (?=^.{6,255}$)((?=.*\d)(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[^A-Za-z0-9])(?=.*[a-z])|(?=.*[^A-Za-z0-9])(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[A-Z])(?=.*[^A-Za-z0-9]))^.* 105 | ConstraintDescription: Must be at least 8 characters containing letters, numbers and symbols. 106 | Description: Admin password for Splunk. Must be at least 8 characters containing letters, numbers and symbols 107 | MaxLength: '32' 108 | MinLength: '6' 109 | NoEcho: 'true' 110 | Type: String 111 | SplunkIndexerCount: 112 | ConstraintDescription: must be a valid number, 4-10 113 | Default: '4' 114 | Description: How many Splunk indexers to launch. [4-10] 115 | MaxValue: '10' 116 | MinValue: '4' 117 | Type: Number 118 | SplunkIndexerDiskSize: 119 | ConstraintDescription: must be a valid number, 100-16000 120 | Default: '334' 121 | Description: The size of the attached EBS volume to the Splunk indexers. (in GB) 122 | MaxValue: '16000' 123 | MinValue: '100' 124 | Type: Number 125 | SplunkSearchHeadDiskSize: 126 | ConstraintDescription: must be a valid number, 100-16000 127 | Default: '334' 128 | Description: The size of the attached EBS volume to the Splunk search head(s). (in GB) 129 | MaxValue: '16000' 130 | MinValue: '100' 131 | Type: Number 132 | SplunkLicenseBucket: 133 | AllowedPattern: (?=^.{3,63}$)(?!xn--)([a-z0-9](?:[a-z0-9-]*)[a-z0-9])$ 134 | ConstraintDescription: 'Required for QuickStart to function and must be a valid s3 bucket' 135 | Description: 'Name of private S3 bucket with licenses to be accessed via authenticated requests' 136 | MinLength: '3' 137 | MaxLength: '63' 138 | Type: String 139 | SplunkLicensePath: 140 | ConstraintDescription: 'Required for QuickStart to function and must point to a valid Splunk license' 141 | AllowedPattern: ([0-9]|[A-Z]|[a-z]|[\/\._-])+ 142 | Description: 'Path to license file in S3 Bucket, without leading /. (ex: license/splunk.license)' 143 | MinLength: '2' 144 | MaxLength: '128' 145 | Type: String 146 | SplunkReplicationFactor: 147 | ConstraintDescription: must be a valid number, 2-4 148 | Default: '2' 149 | Description: How many copies of data should be stored in the Splunk Indexer Cluster 150 | MaxValue: '4' 151 | MinValue: '2' 152 | Type: Number 153 | SplunkSearchFactor: 154 | ConstraintDescription: must be a valid number, 2-4 155 | Default: '2' 156 | Description: How many copies of data should be searchable in the Splunk indexer clusters 157 | MaxValue: '4' 158 | MinValue: '2' 159 | Type: Number 160 | SplunkClusterSecret: 161 | AllowedPattern: (?=^.{6,255}$)((?=.*\d)(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[^A-Za-z0-9])(?=.*[a-z])|(?=.*[^A-Za-z0-9])(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[A-Z])(?=.*[^A-Za-z0-9]))^.* 162 | ConstraintDescription: Must be at least 8 characters containing letters, numbers and symbols. 163 | Description: Shared cluster secret for Search Head and Indexer clusters. Must be at least 8 characters containing letters, numbers and symbols. 164 | MaxLength: '32' 165 | MinLength: '6' 166 | NoEcho: 'true' 167 | Type: String 168 | SplunkIndexerDiscoverySecret: 169 | AllowedPattern: (?=^.{6,255}$)((?=.*\d)(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[^A-Za-z0-9])(?=.*[a-z])|(?=.*[^A-Za-z0-9])(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[A-Z])(?=.*[^A-Za-z0-9]))^.* 170 | ConstraintDescription: Must be at least 8 characters containing letters, numbers and symbols. 171 | Description: Security key used for communication between your forwarders and the cluster master. This value should also be used by forwarders in order to retrieve list of available peer nodes from cluster master. Must be at least 8 characters containing letters, numbers and symbols. 172 | MaxLength: '32' 173 | MinLength: '8' 174 | NoEcho: 'true' 175 | Type: String 176 | VPCCIDR: 177 | AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2}) 178 | ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x. 179 | Default: 10.0.0.0/16 180 | Description: The address space that will be assigned to the entire VPC where Splunk will reside. (Recommend at least a /16) 181 | MaxLength: '19' 182 | MinLength: '9' 183 | Type: String 184 | SmartStoreBucketName: 185 | Default: '' 186 | Description: Name of bucket that will be created for SmartStore storage 187 | Type: String 188 | Metadata: 189 | QuickStartDocumentation: 190 | EntrypointName: "Splunk QuickStart (New VPC)" 191 | AWS::CloudFormation::Interface: 192 | ParameterGroups: 193 | - Label: 194 | default: AWS Instance and Network Settings 195 | Parameters: 196 | - IndexerInstanceType 197 | - SearchHeadInstanceType 198 | - KeyName 199 | - WebClientLocation 200 | - HECClientLocation 201 | - SSHClientLocation 202 | - AvailabilityZones 203 | - NumberOfAZs 204 | - VPCCIDR 205 | - PublicSubnet1CIDR 206 | - PublicSubnet2CIDR 207 | - PublicSubnet3CIDR 208 | - Label: 209 | default: Splunk Settings 210 | Parameters: 211 | - SplunkAdminPassword 212 | - SplunkClusterSecret 213 | - SplunkIndexerDiscoverySecret 214 | - SplunkLicenseBucket 215 | - SplunkLicensePath 216 | - SplunkIndexerCount 217 | - SplunkIndexerDiskSize 218 | - SplunkSearchHeadDiskSize 219 | - SplunkReplicationFactor 220 | - SplunkSearchFactor 221 | - SmartStoreBucketName 222 | - SHCEnabled 223 | - Label: 224 | default: AWS Quick Start Configuration 225 | Parameters: 226 | - QSS3BucketName 227 | - QSS3KeyPrefix 228 | ParameterLabels: 229 | AvailabilityZones: 230 | default: Availability Zones 231 | SplunkSearchHeadDiskSize: 232 | default: Size (in GB) of Splunk search head disk 233 | NumberOfAZs: 234 | default: Number of Availability Zones 235 | WebClientLocation: 236 | default: Permitted CIDR for Splunk web interface 237 | HECClientLocation: 238 | default: Permitted CIDR for Splunk HTTP event collector input 239 | IndexerInstanceType: 240 | default: EC2 instance type for Splunk indexer 241 | SearchHeadInstanceType: 242 | default: EC2 instance type for Splunk search head 243 | KeyName: 244 | default: Key Name 245 | PublicSubnet1CIDR: 246 | default: Public Subnet 1 CIDR 247 | PublicSubnet2CIDR: 248 | default: Public Subnet 2 CIDR 249 | PublicSubnet3CIDR: 250 | default: Public Subnet 3 CIDR 251 | QSS3BucketName: 252 | default: QuickStart S3 Bucket Name 253 | QSS3KeyPrefix: 254 | default: QuickStart S3 Key Prefix 255 | SHCEnabled: 256 | default: Enable Search Head Cluster? 257 | SSHClientLocation: 258 | default: Permitted CIDR for ssh 259 | SplunkAdminPassword: 260 | default: Splunk Admin Password 261 | SplunkIndexerCount: 262 | default: No. of Splunk Indexers 263 | SplunkIndexerDiskSize: 264 | default: Indexer Disk Size 265 | SplunkLicenseBucket: 266 | default: Splunk License Bucket 267 | SplunkLicensePath: 268 | default: Splunk License S3 Bucket Path 269 | SplunkReplicationFactor: 270 | default: Index Cluster Replication Factor 271 | SplunkSearchFactor: 272 | default: Index Cluster Search Factor 273 | SmartStoreBucketName: 274 | default: Name of bucket that will be created for SmartStore storage 275 | SplunkClusterSecret: 276 | default: Shared Security Key for Cluster Nodes 277 | SplunkIndexerDiscoverySecret: 278 | default: Shared Security Key for Forwarders using Indexer Discovery 279 | VPCCIDR: 280 | default: VPC CIDR 281 | Conditions: 282 | Create3AZ: !Equals 283 | - !Ref 'NumberOfAZs' 284 | - '3' 285 | Resources: 286 | VPCStack: 287 | Type: AWS::CloudFormation::Stack 288 | Properties: 289 | TemplateURL: !Sub 'https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}submodules/quickstart-aws-vpc/templates/aws-vpc.template.yaml' 290 | Parameters: 291 | AvailabilityZones: !Join 292 | - ',' 293 | - !Ref 'AvailabilityZones' 294 | CreatePrivateSubnets: 'false' 295 | CreatePublicSubnets: 'true' 296 | CreateNATGateways: 'false' 297 | NumberOfAZs: !Ref 'NumberOfAZs' 298 | PublicSubnet1CIDR: !Ref 'PublicSubnet1CIDR' 299 | PublicSubnet2CIDR: !Ref 'PublicSubnet2CIDR' 300 | PublicSubnet3CIDR: !Ref 'PublicSubnet3CIDR' 301 | VPCCIDR: !Ref 'VPCCIDR' 302 | TimeoutInMinutes: 15 303 | SplunkStack: 304 | Type: AWS::CloudFormation::Stack 305 | Properties: 306 | TemplateURL: !Sub 'https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}templates/splunk-enterprise.template' 307 | Parameters: 308 | VPCID: !GetAtt 'VPCStack.Outputs.VPCID' 309 | VPCCIDR: !GetAtt 'VPCStack.Outputs.VPCCIDR' 310 | PublicSubnet1ID: !GetAtt 'VPCStack.Outputs.PublicSubnet1ID' 311 | PublicSubnet2ID: !GetAtt 'VPCStack.Outputs.PublicSubnet2ID' 312 | PublicSubnet3ID: !If 313 | - Create3AZ 314 | - !GetAtt 'VPCStack.Outputs.PublicSubnet3ID' 315 | - !GetAtt 'VPCStack.Outputs.PublicSubnet2ID' 316 | NumberOfAZs: !Ref 'NumberOfAZs' 317 | IndexerInstanceType: !Ref 'IndexerInstanceType' 318 | SearchHeadInstanceType: !Ref 'SearchHeadInstanceType' 319 | SplunkAdminPassword: !Ref 'SplunkAdminPassword' 320 | SplunkClusterSecret: !Ref 'SplunkClusterSecret' 321 | SplunkIndexerDiscoverySecret: !Ref 'SplunkIndexerDiscoverySecret' 322 | SplunkLicenseBucket: !Ref 'SplunkLicenseBucket' 323 | SplunkLicensePath: !Ref 'SplunkLicensePath' 324 | KeyName: !Ref 'KeyName' 325 | SSHClientLocation: !Ref 'SSHClientLocation' 326 | HECClientLocation: !Ref 'HECClientLocation' 327 | WebClientLocation: !Ref 'WebClientLocation' 328 | SplunkIndexerCount: !Ref 'SplunkIndexerCount' 329 | SHCEnabled: !Ref 'SHCEnabled' 330 | SplunkIndexerDiskSize: !Ref 'SplunkIndexerDiskSize' 331 | SmartStoreBucketName: !Ref 'SmartStoreBucketName' 332 | SplunkReplicationFactor: !Ref 'SplunkReplicationFactor' 333 | TimeoutInMinutes: 45 334 | Outputs: 335 | SearchHeadURL: 336 | Description: Splunk Enterprise - Search Head URL 337 | Value: !GetAtt 'SplunkStack.Outputs.SearchHeadURL' 338 | ClusterMasterURL: 339 | Description: Splunk Enterprise - Cluster Master URL 340 | Value: !GetAtt 'SplunkStack.Outputs.ClusterMasterURL' 341 | ClusterMasterManagementURL: 342 | Description: Splunk Enterprise - Cluster Master Management URL (required for Indexer Discovery) 343 | Value: !GetAtt 'SplunkStack.Outputs.ClusterMasterManagementURL' 344 | DeployerURL: 345 | Description: Splunk Enterprise - Search Head Cluster Deployer URL 346 | Value: !GetAtt 'SplunkStack.Outputs.DeployerURL' 347 | HttpEventCollectorURL: 348 | Description: HTTP Event Collector URL 349 | Value: !GetAtt 'SplunkStack.Outputs.HttpEventCollectorURL' 350 | HttpEventCollectorToken: 351 | Description: HTTP Event Collector Token 352 | Value: !GetAtt 'SplunkStack.Outputs.HttpEventCollectorToken' 353 | -------------------------------------------------------------------------------- /scripts/user_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | 3 | #### start universal functions 4 | function base 5 | { 6 | 7 | # variables 8 | export LOCALIP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4) 9 | export INSTANCEID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) 10 | export SPLUNK_USER=splunk 11 | export SPLUNK_BIN=/opt/splunk/bin/splunk 12 | export SPLUNK_HOME=/opt/splunk 13 | 14 | # make cloud-init output log readable by root only to protect sensitive parameter values 15 | chmod 600 /var/log/cloud-init-output.log 16 | 17 | #- The newer version of the Splunk AMI does not come with Splunk pre-installed. Instead 18 | #- Splunk is installed via ansible as part of cloud-init. The following code (starting at line 30) is 19 | #- needed to ensure these install scripts are ran prior to the remainder of the Cloudformation 20 | #- user scripts. Without doing this first, the Splunk installer is ran after CloudFormation's 21 | #- cloud-init scripts, leaving no Splunk install to configure. 22 | 23 | #- remove the cloud-init scripts from running 24 | rm -f /etc/cloud/cloud.cfg.d/20_install_splunk.cfg 25 | rm -f /var/lib/cloud/instance/scripts/runcmd 26 | 27 | # run the ansible code 28 | (cd /opt/splunk-ansible && time sudo -u ec2-user -E -S bash -c "SPLUNK_BUILD_URL=/tmp/splunk.tgz SPLUNK_ENABLE_SERVICE=true SPLUNK_PASSWORD=SPLUNK-$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id) ansible-playbook -i inventory/environ.py site.yml") 29 | 30 | #- as of 8.2.0, aws-cfn-bootstrap is no longer pre-installed on the AMI. 31 | #- install aws-cfn-bootstrap package 32 | yum -y install aws-cfn-bootstrap 33 | 34 | 35 | # setup auth with user-selected admin password 36 | mv $SPLUNK_HOME/etc/passwd $SPLUNK_HOME/etc/passwd.bak 37 | cat >> $SPLUNK_HOME/etc/system/local/user-seed.conf << end 38 | [user_info] 39 | USERNAME = admin 40 | PASSWORD = $ADMIN_PASSWORD 41 | end 42 | 43 | sed -i '/guid/d' $SPLUNK_HOME/etc/instance.cfg 44 | touch $SPLUNK_HOME/etc/.ui_login 45 | 46 | # restart Splunk for admin password update 47 | $SPLUNK_BIN restart 48 | } 49 | 50 | function restart_signal 51 | { 52 | 53 | # restart splunk 54 | $SPLUNK_BIN restart 55 | 56 | # communicate back to CloudFormation the status of the instance creation 57 | /opt/aws/bin/cfn-signal -e $? --stack $STACK_NAME --resource $RESOURCE --region $AWS_REGION 58 | 59 | # disable splunk user login 60 | usermod --expiredate 1 splunk 61 | } 62 | 63 | #### end universal config 64 | 65 | ##### 66 | #### start role-specific functions 67 | ##### 68 | 69 | ### 70 | # setup nvme drives for i3 indexers 71 | function nvme_setup 72 | { 73 | # first, determine the instance type. 74 | ec2_type=$(curl -s http://169.254.169.254/latest/meta-data/instance-type) 75 | 76 | # this script is intended to run on i3* instance types. 77 | if [[ "$ec2_type" != *"i3"* ]] 78 | then 79 | return 0 80 | fi 81 | 82 | # find the attached nvme drives. lsblk could work here, but utilizing the nvme-list utility due to 83 | # json formatting and simpler parsing. install the nvme-cli and jq packages to accomplish this. 84 | yum -y install nvme-cli jq >/dev/null 85 | 86 | # save the nvme drive information to a temp file for parsing 87 | nvme list --output-format=json > /tmp/nvme_drive.json 88 | 89 | # declare the nvme device array 90 | declare -a nvme_devices 91 | unset nvme_devices 92 | 93 | for nvme_device in $(jq '.Devices[] | .DevicePath' /tmp/nvme_drive.json) 94 | do 95 | # test to ensure that the storage device is instance storage. in testing, I have 96 | # seen EBS volues show as NVME. this logic will ensure attached EBS devices are not 97 | # added to the nvme raid0 98 | nvme_model_type=$(jq -r '.Devices[] | select(.DevicePath=='$nvme_device') | .ModelNumber' /tmp/nvme_drive.json) 99 | if [[ $nvme_model_type = *"NVMe Instance Storage"* ]] 100 | then 101 | # unfortunate 'hack' here to remove the quotes from the device name. without them, the jq lookup 102 | # will fail in the previous step. however, they need to be removed for the md raid creation later. 103 | # additionally, since there needs to be a space between device names for the md create, convert 104 | # quotes to spaces, and remove leading space. this leaves "$nvme_device " (note trailing space) 105 | # stored in the array. this will allow for simply using the contents of the array as an argument for 106 | # building the raid0 device 107 | nvme_device=$(echo $nvme_device|sed 's/"/ /g'| sed 's/^ //g') 108 | 109 | # save device list in nvme_devices array 110 | nvme_devices+=("$nvme_device") 111 | else 112 | # if the nvme model type is not instance storage, continue to the next iteration of the loop 113 | continue 114 | fi 115 | done 116 | 117 | # name of the raid device to create 118 | raid_device="/dev/md0" 119 | 120 | # mount point of the raid device 121 | raid_mount="/opt/splunk" 122 | 123 | # make directory for mount point 124 | mkdir -p $raid_mount 125 | 126 | # create the raid device 127 | mdadm --create $raid_device --level=raid0 --raid-devices=${#nvme_devices[@]} ${nvme_devices[@]} 128 | 129 | # create filesystem on raid device 130 | if [ ${#nvme_devices[@]} -eq 1 ] 131 | then 132 | discardOption="" 133 | else 134 | discardOption="-E nodiscard" 135 | fi 136 | 137 | mkfs.ext4 -m 2 -F -F ${discardOption} $raid_device 138 | 139 | # add entry to fstab for mounting on reboot 140 | echo "$raid_device $raid_mount auto defaults,nofail,noatime 0 2" >>/etc/fstab 141 | 142 | # mount device 143 | mount $raid_device 144 | 145 | } 146 | 147 | ### 148 | # Splunk Cluster Master / License Master 149 | ### 150 | function splunk_cm 151 | { 152 | # execute base install and configuration 153 | base 154 | 155 | export RESOURCE="SplunkCM" 156 | printf '%s\t%s\n' "$LOCALIP" 'splunklicense' >> /etc/hosts 157 | hostname splunklicense 158 | 159 | #- for the CM, we can't reference CM_PRIVATEIP in the CloudFormation UserData like 160 | #- we do in the other resources because the CM hasn't been created yet. To keep the 161 | #- syntax consistent across each resource in user_data.sh, export $CM_PRIVATEIP to 162 | #- the CM's local ip address 163 | export CM_PRIVATEIP=$LOCALIP 164 | 165 | # Install license from metadata. 166 | if [ $INSTALL_LICENSE = 1 ]; then 167 | mkdir -p $SPLUNK_HOME/etc/licenses/enterprise/ 168 | chown $SPLUNK_USER:$SPLUNK_USER $SPLUNK_HOME/etc/licenses/enterprise 169 | /opt/aws/bin/cfn-init -v --stack $STACK_NAME --resource $RESOURCE --region $AWS_REGION 170 | fi 171 | 172 | # Increase splunkweb connection timeout with splunkd 173 | mkdir -p $SPLUNK_HOME/etc/apps/base-autogenerated/local 174 | cat >>$SPLUNK_HOME/etc/apps/base-autogenerated/local/web.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/outputs.conf <>$SPLUNK_HOME/etc/system/local/server.conf < /tmp/token 228 | TOKEN=`sed -n 's/\\ttoken=//p' /tmp/token` && rm /tmp/token 229 | 230 | # place generated config into master-apps 231 | mkdir -p $SPLUNK_HOME/etc/master-apps/peer-base-autogenerated/local 232 | mv $SPLUNK_HOME/etc/apps/splunk_httpinput/local/inputs.conf $SPLUNK_HOME/etc/master-apps/peer-base-autogenerated/local 233 | 234 | # peer config 2: enable splunk tcp input 235 | cat >>$SPLUNK_HOME/etc/master-apps/peer-base-autogenerated/local/inputs.conf <>$SPLUNK_HOME/etc/master-apps/_cluster/local/indexes.conf <> $SPLUNK_HOME/etc/slave-apps/_cluster/local/indexes.conf << end 286 | [default] 287 | repFactor = auto 288 | remotePath = volume:remote_store/splunk_db/$_index_name 289 | coldPath=$SPLUNK_DB/$_index_name/colddb 290 | thawedPath=$SPLUNK_DB/$_index_name/thaweddb 291 | end 292 | 293 | cat >>$SPLUNK_HOME/etc/slave-apps/_cluster/local/indexes.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/web.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/server.conf <>$SPLUNK_HOME/etc/system/local/server.conf <> /etc/hosts 386 | hostname "splunksearch-$num" 387 | 388 | # set splunk servername 389 | sudo -u $SPLUNK_USER $SPLUNK_BIN set servername SHC$num 390 | 391 | # Increase splunkweb connection timeout with splunkd 392 | cat >$SPLUNK_HOME/etc/system/local/web.conf <>$SPLUNK_HOME/etc/system/local/server.conf <> /etc/hosts 472 | hostname splunk-shc-deployer 473 | 474 | # Increase splunkweb connection timeout with splunkd 475 | mkdir -p $SPLUNK_HOME/etc/apps/base-autogenerated/local 476 | cat >>$SPLUNK_HOME/etc/apps/base-autogenerated/local/web.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/server.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/outputs.conf <>$SPLUNK_HOME/etc/shcluster/apps/member-base-autogenerated/local/outputs.conf <> /etc/hosts 558 | hostname splunksearch 559 | 560 | # Increase splunkweb connection timeout with splunkd 561 | mkdir -p $SPLUNK_HOME/etc/apps/base-autogenerated/local 562 | cat >>$SPLUNK_HOME/etc/apps/base-autogenerated/local/web.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/outputs.conf <