├── .github └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── LICENSE-SAMPLECODE ├── LICENSE-SUMMARY ├── README.md ├── doc_source ├── ac-managed.md ├── ac-permissions-ref.md ├── ac.md ├── ad.md ├── alerting.md ├── asynchronous-search.md ├── audit-logs.md ├── auto-tune.md ├── bp.md ├── cloudwatch-alarms.md ├── cognito-auth.md ├── cold-storage.md ├── compliance.md ├── configuration-api.md ├── configuration-samples.md ├── configure-client-fluentbit.md ├── configure-client-msk.md ├── configure-client-otel.md ├── configure-client-s3.md ├── configure-client-security-lake.md ├── configure-client.md ├── createanomalydetector-tutorial.md ├── createdomain-configure-slow-logs.md ├── createupdatedomains.md ├── creating-pipeline.md ├── cross-cluster-search.md ├── cross-service-confused-deputy-prevention.md ├── curator.md ├── custom-packages.md ├── customendpoint.md ├── dashboards.md ├── data-protection.md ├── data-streams.md ├── delete-pipeline.md ├── disaster-recovery-resiliency.md ├── encryption-at-rest.md ├── encryption.md ├── features-by-version.md ├── fgac-walkthrough-basic.md ├── fgac-walkthrough-iam.md ├── fgac.md ├── genref.md ├── glossary.md ├── gsg-serverless-cli.md ├── gsg-serverless.md ├── gsg.md ├── gsgcreate-domain.md ├── gsgdeleting.md ├── gsgsearch.md ├── gsgupload-data.md ├── gzip.md ├── handling-errors.md ├── index.md ├── indexing.md ├── infrastructure-security.md ├── ingestion.md ├── integrations.md ├── ism-tutorial.md ├── ism.md ├── knn.md ├── learning-to-rank.md ├── limits.md ├── list-pipeline.md ├── listening-events.md ├── logging-using-cloudtrail.md ├── managedomains-awsresourcetagging.md ├── managedomains-cloudtrailauditing.md ├── managedomains-cloudwatchmetrics.md ├── managedomains-configuration-changes.md ├── managedomains-dedicatedmasternodes.md ├── managedomains-logstash.md ├── managedomains-multiaz.md ├── managedomains-notifications.md ├── managedomains-snapshots.md ├── managing-indices.md ├── migration.md ├── monitoring-cloudwatch.md ├── monitoring-data.md ├── monitoring-events.md ├── monitoring-pipeline-logs.md ├── monitoring-pipeline-metrics.md ├── monitoring-pipelines.md ├── monitoring.md ├── ntn.md ├── observability.md ├── off-peak.md ├── osis-best-practices.md ├── osis-features-overview.md ├── osis-get-started.md ├── osis-getting-started-tutorials.md ├── osis-logging-using-cloudtrail.md ├── osis-sdk.md ├── osis-serverless-get-started.md ├── petabyte-scale.md ├── pipeline--stop-start.md ├── pipeline-collection-access.md ├── pipeline-config-reference.md ├── pipeline-domain-access.md ├── pipeline-security-model.md ├── pipeline-security-overview.md ├── pipeline-security.md ├── pit.md ├── ppl-support.md ├── ppl.md ├── quick-start.md ├── release-notes.md ├── remote-reindex.md ├── rename.md ├── replication.md ├── request-signing.md ├── ri.md ├── rollup.md ├── saml.md ├── samplecode.md ├── search-example.md ├── searching.md ├── security-analytics.md ├── security-iam-ingestion.md ├── security-iam-serverless.md ├── security.md ├── serverless-cfn.md ├── serverless-clients.md ├── serverless-collections.md ├── serverless-data-access.md ├── serverless-encryption.md ├── serverless-genref.md ├── serverless-getting-started.md ├── serverless-manage.md ├── serverless-monitoring-events.md ├── serverless-monitoring.md ├── serverless-network.md ├── serverless-overview.md ├── serverless-saml.md ├── serverless-scaling.md ├── serverless-sdk.md ├── serverless-security.md ├── serverless-service-linked-roles.md ├── serverless-tutorials.md ├── serverless-vector-search.md ├── serverless-vpc.md ├── serverless.md ├── service-software.md ├── setting-up.md ├── sizing-domains.md ├── slr-aos.md ├── slr-osis.md ├── slr.md ├── sns-events.md ├── sql-support.md ├── supported-instance-types.md ├── supported-operations.md ├── supported-plugins.md ├── supported-resources.md ├── tag-collection.md ├── tag-pipeline.md ├── trace-analytics.md ├── transforms.md ├── tutorials.md ├── ultrawarm.md ├── update-pipeline.md ├── use-cases-anomaly-detection.md ├── use-cases-log-aggregation.md ├── use-cases-log-enrichment.md ├── use-cases-metrics-logs.md ├── use-cases-overview.md ├── use-cases-pattern-matching.md ├── use-cases-s3-select.md ├── use-cases-sampling.md ├── use-cases-trace-analytics.md ├── use-cases-trace-metrics.md ├── version-migration.md ├── vpc-interface-endpoints.md ├── vpc.md ├── walkthrough.md └── what-is.md └── sample_code ├── .gitignore └── java ├── aws-request-signing-apache-interceptor ├── LICENSE ├── NOTICE ├── README.md ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── amazonaws │ │ ├── http │ │ └── AwsRequestSigningApacheInterceptor.java │ │ └── samples │ │ ├── BulkIndexDocuments.java │ │ └── IndexDocument.java │ └── test │ └── java │ └── com │ └── amazonaws │ └── http │ └── AwsRequestSigningApacheInterceptorTest.java └── opensearch-java-aws-sdk2-transport ├── LICENSE ├── NOTICE ├── README.md ├── pom.xml └── src └── main └── java └── com └── amazonaws └── samples ├── GetVersion.java ├── IndexData.java └── IndexDocument.java /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | *Issue #, if available:* 2 | 3 | *Description of changes:* 4 | 5 | 6 | By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | md.sh 3 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Guidelines for contributing 2 | 3 | Thank you for your interest in contributing to AWS documentation! We greatly value feedback and contributions from our community. 4 | 5 | Please read through this document before you submit any pull requests or issues. It will help us work together more effectively. 6 | 7 | ## What to expect when you contribute 8 | 9 | When you submit a pull request, our team is notified and will respond as quickly as we can. We'll do our best to work with you to ensure that your pull request adheres to our style and standards. If we merge your pull request, we might make additional edits later for style or clarity. 10 | 11 | The AWS documentation source files on GitHub aren't published directly to the official documentation website. If we merge your pull request, we'll publish your changes to the documentation website as soon as we can, but they won't appear immediately or automatically. 12 | 13 | We look forward to receiving your pull requests for: 14 | 15 | * New content you'd like to contribute (such as new code samples or tutorials) 16 | * Inaccuracies in the content 17 | * Information gaps in the content that need more detail to be complete 18 | * Typos or grammatical errors 19 | * Suggested rewrites that improve clarity and reduce confusion 20 | 21 | **Note:** We all write differently, and you might not like how we've written or organized something currently. We want that feedback. But please be sure that your request for a rewrite is supported by the previous criteria. If it isn't, we might decline to merge it. 22 | 23 | ## How to contribute 24 | 25 | To contribute, send us a pull request. For small changes, such as fixing a typo or adding a link, you can use the [GitHub Edit Button](https://blog.github.com/2011-04-26-forking-with-the-edit-button/). For larger changes: 26 | 27 | 1. [Fork the repository](https://help.github.com/articles/fork-a-repo/). 28 | 2. In your fork, make your change in a branch that's based on this repo's **master** branch. 29 | 3. Commit the change to your fork, using a clear and descriptive commit message. 30 | 4. [Create a pull request](https://help.github.com/articles/creating-a-pull-request-from-a-fork/), answering any questions in the pull request form. 31 | 32 | Before you send us a pull request, please be sure that: 33 | 34 | 1. You're working from the latest source on the **master** branch. 35 | 2. You check [existing open](https://github.com/awsdocs/amazon-opensearch-service-developer-guide/pulls), and [recently closed](https://github.com/awsdocs/amazon-opensearch-service-developer-guide/pulls?q=is%3Apr+is%3Aclosed), pull requests to be sure that someone else hasn't already addressed the problem. 36 | 3. You [create an issue](https://github.com/awsdocs/amazon-opensearch-service-developer-guide/issues/new) before working on a contribution that will take a significant amount of your time. 37 | 38 | For contributions that will take a significant amount of time, [open a new issue](https://github.com/awsdocs/amazon-opensearch-service-developer-guide/issues/new) to pitch your idea before you get started. Explain the problem and describe the content you want to see added to the documentation. Let us know if you'll write it yourself or if you'd like us to help. We'll discuss your proposal with you and let you know whether we're likely to accept it. We don't want you to spend a lot of time on a contribution that might be outside the scope of the documentation or that's already in the works. 39 | 40 | ## Finding contributions to work on 41 | 42 | If you'd like to contribute, but don't have a project in mind, look at the [open issues](https://github.com/awsdocs/amazon-opensearch-service-developer-guide/issues) in this repository for some ideas. Any issues with the [help wanted](https://github.com/awsdocs/amazon-opensearch-service-developer-guide/labels/help%20wanted) or [enhancement](https://github.com/awsdocs/amazon-opensearch-service-developer-guide/labels/enhancement) labels are a great place to start. 43 | 44 | In addition to written content, we really appreciate new examples and code samples for our documentation, such as examples for different platforms or environments, and code samples in additional languages. 45 | 46 | ## Code of conduct 47 | 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). For more information, see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact [opensource-codeofconduct@amazon.com](mailto:opensource-codeofconduct@amazon.com) with any additional questions or comments. 49 | 50 | ## Security issue notifications 51 | 52 | If you discover a potential security issue, please notify AWS Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public issue on GitHub. 53 | 54 | ## Licensing 55 | 56 | See the [LICENSE](https://github.com/awsdocs/amazon-opensearch-service-developer-guide/blob/master/LICENSE) file for this project's licensing. We will ask you to confirm the licensing of your contribution. We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 57 | -------------------------------------------------------------------------------- /LICENSE-SAMPLECODE: -------------------------------------------------------------------------------- 1 | Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this 4 | software and associated documentation files (the "Software"), to deal in the Software 5 | without restriction, including without limitation the rights to use, copy, modify, 6 | merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | permit persons to whom the Software is furnished to do so. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 10 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 11 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 12 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 13 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 14 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | -------------------------------------------------------------------------------- /LICENSE-SUMMARY: -------------------------------------------------------------------------------- 1 | Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | The documentation is made available under the Creative Commons Attribution-ShareAlike 4.0 International License. See the LICENSE file. 4 | 5 | The sample code within this documentation is made available under a modified MIT license. See the LICENSE-SAMPLECODE file. 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Notice 2 | 3 | This repository is archived, read-only, and no longer updated. For more information, see [Retiring the AWS Documentation on GitHub](https://aws.amazon.com/blogs/aws/retiring-the-aws-documentation-on-github/). 4 | 5 | You can find up-to-date AWS technical documentation on the [AWS documentation website](https://docs.aws.amazon.com/), where you can also submit feedback and suggestions for improvement. 6 | -------------------------------------------------------------------------------- /doc_source/ac-permissions-ref.md: -------------------------------------------------------------------------------- 1 | # Amazon OpenSearch Service API permissions reference 2 | 3 | When you set up [access control](ac.md), you write permission policies that you can attach to an IAM identity \(identity\-based policies\)\. For detailed reference information, see the following topics in the *Service Authorization Reference*: 4 | + [Actions, resources, and condition keys for OpenSearch Service](https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazonopensearchservice.html)\. 5 | + [Actions, resources, and condition keys for OpenSearch Ingestion](https://docs.aws.amazon.com/service-authorization/latest/reference/list_opensearchingestionservice.html)\. 6 | 7 | This reference contains information about which API operations can be used in an IAM policy\. It also includes the AWS resource for which you can grant the permissions, and condition keys that you can include for fine\-grained access control\. 8 | 9 | You specify the actions in the policy's `Action` field, the resource value in the policy's `Resource` field, and conditions in the policy's `Condition` field\. To specify an action for OpenSearch Service, use the `es:` prefix followed by the API operation name \(for example, `es:CreateDomain`\)\. To specify an action for OpenSearch Ingestion, use the `osis:` prefix followed by the API operation \(for example, `osis:CreatePipeline`\)\. -------------------------------------------------------------------------------- /doc_source/ad.md: -------------------------------------------------------------------------------- 1 | # Anomaly detection in Amazon OpenSearch Service 2 | 3 | Anomaly detection in Amazon OpenSearch Service automatically detects anomalies in your OpenSearch data in near\-real time by using the Random Cut Forest \(RCF\) algorithm\. RCF is an unsupervised machine learning algorithm that models a sketch of your incoming data stream\. The algorithm computes an `anomaly grade` and `confidence score` value for each incoming data point\. Anomaly detection uses these values to differentiate an anomaly from normal variations in your data\. 4 | 5 | You can pair the anomaly detection plugin with the [Configuring alerts in Amazon OpenSearch Service](alerting.md) plugin to notify you as soon as an anomaly is detected\. 6 | 7 | Anomaly detection is available on domains running any OpenSearch version or Elasticsearch 7\.4 or later\. All instance types support anomaly detection except for `t2.micro` and `t2.small`\. Full documentation for anomaly detection, including detailed steps and API descriptions, is available in the [OpenSearch documentation](https://opensearch.org/docs/latest/monitoring-plugins/ad/index/)\. 8 | 9 | ## 10 | 11 | ### Prerequisites 12 | 13 | Anomaly detection has the following prerequisites: 14 | + Anomaly detection requires OpenSearch or Elasticsearch 7\.4 or later\. 15 | + Anomaly detection only supports [fine\-grained access control](fgac.md) on Elasticsearch versions 7\.9 and later and all versions of OpenSearch\. Prior to Elasticsearch 7\.9, only admin users can create, view, and manage detectors\. 16 | + If your domain uses fine\-grained access control, non\-admin users must be [mapped](fgac.md#fgac-mapping) to the `anomaly_read_access` role in OpenSearch Dashboards in order to view detectors, or `anomaly_full_access` in order to create and manage detectors\. 17 | 18 | ### Getting started with anomaly detection 19 | 20 | To get started, choose **Anomaly Detection** in OpenSearch Dashboards\. 21 | 22 | #### Step 1: Create a detector 23 | 24 | A detector is an individual anomaly detection task\. You can create multiple detectors, and all the detectors can run simultaneously, with each analyzing data from different sources\. 25 | 26 | #### Step 2: Add features to your detector 27 | 28 | A feature is the field in your index that you check for anomalies\. A detector can discover anomalies across one or more features\. You must choose one of the following aggregations for each feature: `average()`, `sum()`, `count()`, `min()`, or `max()`\. 29 | 30 | **Note** 31 | The `count()` aggregation method is only available in OpenSearch and Elasticsearch 7\.7 or later\. For Elasticsearch 7\.4, use a custom expression like the following: 32 | 33 | ``` 34 | { 35 | "aggregation_name": { 36 | "value_count": { 37 | "field": "field_name" 38 | } 39 | } 40 | } 41 | ``` 42 | 43 | The aggregation method determines what constitutes an anomaly\. For example, if you choose `min()`, the detector focuses on finding anomalies based on the minimum values of your feature\. If you choose `average()`, the detector finds anomalies based on the average values of your feature\. You can add a maximum of five features per detector\. 44 | 45 | You can configure the following optional settings \(available in Elasticsearch 7\.7 and later\): 46 | + **Category field** \- Categorize or slice your data with a dimension like IP address, product ID, country code, and so on\. 47 | + **Window size** \- Set the number of aggregation intervals from your data stream to consider in a detection window\. 48 | 49 | After you set up your features, preview sample anomalies and adjust the feature settings if necessary\. 50 | 51 | #### Step 3: Observe the results 52 | 53 | ![\[The following visualizations are available on the anomaly detection dashboard:\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/images/ad.png) 54 | + **Live anomalies** \- displays the live anomaly results for the last 60 intervals\. For example, if the interval is set to 10, it shows the results for the last 600 minutes\. This chart refreshes every 30 seconds\. 55 | + **Anomaly history** \- plots the anomaly grade with the corresponding measure of confidence\. 56 | + **Feature breakdown** \- plots the features based on the aggregation method\. You can vary the date\-time range of the detector\. 57 | + **Anomaly occurrence** \- shows the `Start time`, `End time`, `Data confidence`, and `Anomaly grade` for each anomaly detected\. 58 | 59 | If you set the category field, you see an additional **Heat map** chart that correlates results for anomalous entities\. Choose a filled rectangle to see a more detailed view of the anomaly\. 60 | 61 | #### Step 4: Set up alerts 62 | 63 | To create a monitor to send you notifications when any anomalies are detected, choose **Set up alerts**\. The plugin redirects you to the [https://opensearch.org/docs/monitoring-plugins/alerting/monitors/#create-monitors](https://opensearch.org/docs/monitoring-plugins/alerting/monitors/#create-monitors) page where you can configure an alert\. -------------------------------------------------------------------------------- /doc_source/cloudwatch-alarms.md: -------------------------------------------------------------------------------- 1 | # Recommended CloudWatch alarms for Amazon OpenSearch Service 2 | 3 | CloudWatch alarms perform an action when a CloudWatch metric exceeds a specified value for some amount of time\. For example, you might want AWS to email you if your cluster health status is `red` for longer than one minute\. This section includes some recommended alarms for Amazon OpenSearch Service and how to respond to them\. 4 | 5 | You can automatically deploy these alarms using AWS CloudFormation\. For a sample stack, see the related [GitHub repository](https://github.com/ev2900/OpenSearch_CloudWatch_Alarms)\. 6 | 7 | **Note** 8 | If you deploy the CloudFormation stack, the `KMSKeyError` and `KMSKeyInaccessible` alarms will exists in an `Insufficient Data` state because these metrics only appear if a domain encounters a problem with its encryption key\. 9 | 10 | For more information about configuring alarms, see [Creating Amazon CloudWatch Alarms](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/AlarmThatSendsEmail.html) in the *Amazon CloudWatch User Guide*\. 11 | 12 | [\[See the AWS documentation website for more details\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/cloudwatch-alarms.html) 13 | 14 | **Note** 15 | If you just want to *view* metrics, see [Monitoring OpenSearch cluster metrics with Amazon CloudWatch](managedomains-cloudwatchmetrics.md)\. 16 | 17 | ## Other alarms you might consider 18 | 19 | Consider configuring the following alarms depending on which OpenSearch Service features you regularly use\. 20 | 21 | [\[See the AWS documentation website for more details\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/cloudwatch-alarms.html) -------------------------------------------------------------------------------- /doc_source/compliance.md: -------------------------------------------------------------------------------- 1 | # Compliance validation for Amazon OpenSearch Service 2 | 3 | Third\-party auditors assess the security and compliance of Amazon OpenSearch Service as part of multiple AWS compliance programs\. These programs include SOC, PCI, and HIPAA\. 4 | 5 | If you have compliance requirements, consider using any version of OpenSearch or Elasticsearch 6\.0 or later\. Earlier versions of Elasticsearch don't offer a combination of [encryption of data at rest](encryption-at-rest.md) and [node\-to\-node encryption](ntn.md) and are unlikely to meet your needs\. You might also consider using any version of OpenSearch or Elasticsearch 6\.7 or later if [fine\-grained access control](fgac.md) is important to your use case\. Regardless, choosing a particular OpenSearch or Elasticsearch version when you create a domain does not guarantee compliance\. 6 | 7 | To learn whether an AWS service is within the scope of specific compliance programs, see [AWS services in Scope by Compliance Program](http://aws.amazon.com/compliance/services-in-scope/) and choose the compliance program that you are interested in\. For general information, see [AWS Compliance Programs](http://aws.amazon.com/compliance/programs/)\. 8 | 9 | You can download third\-party audit reports using AWS Artifact\. For more information, see [Downloading Reports in AWS Artifact](https://docs.aws.amazon.com/artifact/latest/ug/downloading-documents.html)\. 10 | 11 | Your compliance responsibility when using AWS services is determined by the sensitivity of your data, your company's compliance objectives, and applicable laws and regulations\. AWS provides the following resources to help with compliance: 12 | + [Security and Compliance Quick Start Guides](http://aws.amazon.com/quickstart/?awsf.quickstart-homepage-filter=categories%23security-identity-compliance) – These deployment guides discuss architectural considerations and provide steps for deploying baseline environments on AWS that are security and compliance focused\. 13 | + [Architecting for HIPAA Security and Compliance on Amazon Web Services](https://docs.aws.amazon.com/whitepapers/latest/architecting-hipaa-security-and-compliance-on-aws/welcome.html) – This whitepaper describes how companies can use AWS to create HIPAA\-eligible applications\. 14 | **Note** 15 | Not all AWS services are HIPAA eligible\. For more information, see the [HIPAA Eligible Services Reference](https://aws.amazon.com/compliance/hipaa-eligible-services-reference/)\. 16 | + [AWS Compliance Resources](http://aws.amazon.com/compliance/resources/) – This collection of workbooks and guides might apply to your industry and location\. 17 | + [Evaluating Resources with Rules](https://docs.aws.amazon.com/config/latest/developerguide/evaluate-config.html) in the *AWS Config Developer Guide* – The AWS Config service assesses how well your resource configurations comply with internal practices, industry guidelines, and regulations\. 18 | + [AWS Security Hub](https://docs.aws.amazon.com/securityhub/latest/userguide/what-is-securityhub.html) – This AWS service provides a comprehensive view of your security state within AWS\. Security Hub uses security controls to evaluate your AWS resources and to check your compliance against security industry standards and best practices\. For a list of supported services and controls, see [Security Hub controls reference](https://docs.aws.amazon.com/securityhub/latest/userguide/securityhub-controls-reference.html)\. 19 | + [AWS Audit Manager](https://docs.aws.amazon.com/audit-manager/latest/userguide/what-is.html) – This AWS service helps you continuously audit your AWS usage to simplify how you manage risk and compliance with regulations and industry standards\. -------------------------------------------------------------------------------- /doc_source/configure-client-fluentbit.md: -------------------------------------------------------------------------------- 1 | # Fluent Bit 2 | 3 | This sample [Fluent Bit configuration file](https://docs.fluentbit.io/manual/pipeline/outputs/http) sends log data from Fluent Bit to an OpenSearch Ingestion pipeline\. For more information about ingesting log data, see [Log Analytics](https://github.com/opensearch-project/data-prepper/blob/main/docs/log_analytics.md) in the Data Prepper documentation\. 4 | 5 | Note the following: 6 | + The `host` value must be your pipeline endpoint\. For example, `pipeline-endpoint.us-east-1.osis.amazonaws.com`\. 7 | + The `aws_service` value must be `osis`\. 8 | + The `aws_role_arn` value is the ARN of the AWS IAM role for the client to assume and use for Signature Version 4 authentication\. 9 | 10 | ``` 11 | [INPUT] 12 | name tail 13 | refresh_interval 5 14 | path /var/log/test.log 15 | read_from_head true 16 | 17 | [OUTPUT] 18 | Name http 19 | Match * 20 | Host pipeline-endpoint.us-east-1.osis.amazonaws.com 21 | Port 443 22 | URI /log/ingest 23 | Format json 24 | aws_auth true 25 | aws_region us-east-1 26 | aws_service osis 27 | aws_role_arn arn:aws:iam::{account-id}:role/ingestion-role 28 | Log_Level trace 29 | tls On 30 | ``` 31 | 32 | You can then configure an OpenSearch Ingestion pipeline like the following, which has HTTP as the source: 33 | 34 | ``` 35 | version: "2" 36 | unaggregated-log-pipeline: 37 | source: 38 | http: 39 | path: "/log/ingest" 40 | processor: 41 | - grok: 42 | match: 43 | log: 44 | - "%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:network_node} %{NOTSPACE:network_host} %{IPORHOST:source_ip}:%{NUMBER:source_port:int} -> %{IPORHOST:destination_ip}:%{NUMBER:destination_port:int} %{GREEDYDATA:details}" 45 | - grok: 46 | match: 47 | details: 48 | - "'%{NOTSPACE:http_method} %{NOTSPACE:http_uri}' %{NOTSPACE:protocol}" 49 | - "TLS%{NOTSPACE:tls_version} %{GREEDYDATA:encryption}" 50 | - "%{NUMBER:status_code:int} %{NUMBER:response_size:int}" 51 | - delete_entries: 52 | with_keys: ["details", "log"] 53 | 54 | sink: 55 | - opensearch: 56 | hosts: ["https://search-domain-endpoint.us-east-1.es.amazonaws.com"] 57 | index: "index_name" 58 | index_type: custom 59 | bulk_size: 20 60 | aws: 61 | # IAM role that the pipeline assumes to access the domain sink 62 | sts_role_arn: "arn:aws:iam::{account-id}:role/pipeline-role" 63 | region: "us-east-1" 64 | ``` -------------------------------------------------------------------------------- /doc_source/configure-client-otel.md: -------------------------------------------------------------------------------- 1 | # OpenTelemetry Collector 2 | 3 | This sample [OpenTelemetry configuration file](https://opentelemetry.io/docs/collector/configuration/) exports trace data from the OpenTelemetry Collector and sends it to an OpenSearch Ingestion pipeline\. For more information about ingesting trace data, see [Trace Analytics](https://github.com/opensearch-project/data-prepper/blob/main/docs/trace_analytics.md) in the Data Prepper documentation\. 4 | 5 | Note the following: 6 | + The `endpoint` value must include your pipeline endpoint\. For example, `https://pipeline-endpoint.us-east-1.osis.amazonaws.com`\. 7 | + The `service` value must be `osis`\. 8 | 9 | ``` 10 | extensions: 11 | sigv4auth: 12 | region: "us-east-1" 13 | service: "osis" 14 | 15 | receivers: 16 | jaeger: 17 | protocols: 18 | grpc: 19 | 20 | exporters: 21 | otlphttp: 22 | traces_endpoint: "https://pipeline-endpoint.us-east-1.osis.amazonaws.com/v1/traces" 23 | auth: 24 | authenticator: sigv4auth 25 | compression: none 26 | 27 | service: 28 | extensions: [sigv4auth] 29 | pipelines: 30 | traces: 31 | receivers: [jaeger] 32 | exporters: [otlphttp] 33 | ``` 34 | 35 | You can then configure an OpenSearch Ingestion pipeline like the following, which specifies the [OTel trace](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sources/otel-trace/) plugin as the source: 36 | 37 | ``` 38 | version: "2" 39 | otel-trace-pipeline: 40 | source: 41 | otel_trace_source: 42 | path: "/v1/traces" 43 | processor: 44 | - trace_peer_forwarder: 45 | sink: 46 | - pipeline: 47 | name: "trace-pipeline" 48 | - pipeline: 49 | name: "service-map-pipeline" 50 | trace-pipeline: 51 | source: 52 | pipeline: 53 | name: "otel-trace-pipeline" 54 | processor: 55 | - otel_traces: 56 | sink: 57 | - opensearch: 58 | hosts: ["https://search-domain-endpoint.us-east-1.es.amazonaws.com"] 59 | index_type: trace-analytics-raw 60 | aws: 61 | # IAM role that OpenSearch Ingestion assumes to access the domain sink 62 | sts_role_arn: "arn:aws:iam::{account-id}:role/pipeline-role" 63 | region: "us-east-1" 64 | 65 | service-map-pipeline: 66 | source: 67 | pipeline: 68 | name: "otel-trace-pipeline" 69 | processor: 70 | - service_map: 71 | sink: 72 | - opensearch: 73 | hosts: ["https://search-domain-endpoint.us-east-1.es.amazonaws.com"] 74 | index_type: trace-analytics-service-map 75 | aws: 76 | # IAM role that the pipeline assumes to access the domain sink 77 | sts_role_arn: "arn:aws:iam::{account-id}:role/pipeline-role" 78 | region: "us-east-1" 79 | ``` 80 | 81 | For another example pipeline, see the **Trace Analytics pipeline** blueprint\. For more information, see [Using blueprints to create a pipeline](creating-pipeline.md#pipeline-blueprint)\. -------------------------------------------------------------------------------- /doc_source/configure-client-security-lake.md: -------------------------------------------------------------------------------- 1 | # Amazon Security Lake 2 | 3 | You can use the [S3 source plugin](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sources/s3/) to ingest data from [Amazon Security Lake](https://docs.aws.amazon.com/security-lake/latest/userguide/what-is-security-lake.html) into your OpenSearch Ingestion pipeline\. Security Lake automatically centralizes security data from AWS environments, on\-premises environments, and SaaS providers into a purpose\-built data lake\. You can create a subscription that replicates data from Security Lake to your OpenSearch Ingestion pipeline, which then writes it to your OpenSearch Service domain or OpenSearch Serverless collection\. 4 | 5 | To configure your pipeline to read from Security Lake, use the Security Lake blueprint named **AWS\-SecurityLakeS3ParquetOCSFPipeline**\. The blueprint includes a default configuration for ingesting Open Cybersecurity Schema Framework \(OCSF\) parquet files from Security Lake\. For more information, see [Using blueprints to create a pipeline](creating-pipeline.md#pipeline-blueprint)\. 6 | 7 | **Topics** 8 | + [Prerequisites](#sl-prereqs) 9 | + [Step 1: Configure the pipeline role](#sl-pipeline-role) 10 | + [Step 2: Create the pipeline](#sl-pipeline) 11 | 12 | ## Prerequisites 13 | 14 | Before you create your OpenSearch Ingestion pipeline, perform the following steps: 15 | + [Enable Security Lake](https://docs.aws.amazon.com/security-lake/latest/userguide/getting-started.html#enable-service)\. 16 | + [Create a subscriber](https://docs.aws.amazon.com/security-lake/latest/userguide/subscriber-data-access.html#create-subscriber-data-access) in Security Lake\. 17 | + Choose the sources that you want to ingest into your pipeline\. 18 | + For **Subscriber credentials**, add the ID of the AWS account where you intend to create the pipeline\. For the external ID, specify `OpenSearchIngestion-{accountid}`\. 19 | + For **Data access method**, choose **S3**\. 20 | + For **Notification details**, choose **SQS queue**\. 21 | 22 | When you create a subscriber, Security Lake automatically creates two inline permissions policies—one for S3 and one for SQS\. The policies take the following format: `AmazonSecurityLake-{12345}-S3` and `AmazonSecurityLake-{12345}-SQS`\. To allow your pipeline to access the subscriber sources, you must associate the required permissions with your pipeline role\. 23 | 24 | ## Step 1: Configure the pipeline role 25 | 26 | Create a new permissions policy in IAM that combines only the required permissions from the two policies that Security Lake automatically created\. The following example policy shows the least privilege required for an OpenSearch Ingestion pipeline to read data from multiple Security Lake sources: 27 | 28 | ``` 29 | { 30 | "Version":"2012-10-17", 31 | "Statement":[ 32 | { 33 | "Effect":"Allow", 34 | "Action":[ 35 | "s3:GetObject" 36 | ], 37 | "Resource":[ 38 | "arn:aws:s3:::aws-security-data-lake-{region}-abcde/aws/LAMBDA_EXECUTION/1.0/*", 39 | "arn:aws:s3:::aws-security-data-lake-{region}-abcde/aws/S3_DATA/1.0/*", 40 | "arn:aws:s3:::aws-security-data-lake-{region}-abcde/aws/VPC_FLOW/1.0/*", 41 | "arn:aws:s3:::aws-security-data-lake-{region}-abcde/aws/ROUTE53/1.0/*", 42 | "arn:aws:s3:::aws-security-data-lake-{region}-abcde/aws/SH_FINDINGS/1.0/*" 43 | ] 44 | }, 45 | { 46 | "Effect":"Allow", 47 | "Action":[ 48 | "sqs:ReceiveMessage", 49 | "sqs:DeleteMessage" 50 | ], 51 | "Resource":[ 52 | "arn:aws:sqs:{region}:{account-id}:AmazonSecurityLake-abcde-Main-Queue" 53 | ] 54 | } 55 | ] 56 | } 57 | ``` 58 | 59 | **Important** 60 | Security Lake doesn’t manage the pipeline role policy for you\. If you add or remove sources from your Security Lake subscription, you must manually update the policy\. Security Lake creates partitions for each log source, so you need to manually add or remove permissions in the pipeline role\. 61 | 62 | You must attach these permissions to the IAM role that you specify in the `sts_role_arn` option within the S3 source plugin configuration, under `sqs`\. 63 | 64 | ``` 65 | version: "2" 66 | source: 67 | s3: 68 | ... 69 | sqs: 70 | queue_url: "https://sqs.{region}.amazonaws.com/{your-account-id}/AmazonSecurityLake-abcde-Main-Queue" 71 | aws: 72 | ... 73 | sts_role_arn: arn:aws:iam::{your-account-id}:role/pipeline-role 74 | processor: 75 | ... 76 | sink: 77 | - opensearch: 78 | ... 79 | ``` 80 | 81 | ## Step 2: Create the pipeline 82 | 83 | After you add the permissions to the pipeline role, use the **AWS\-SecurityLakeS3ParquetOCSFPipeline** blueprint to create the pipeline\. For more information, see [Using blueprints to create a pipeline](creating-pipeline.md#pipeline-blueprint)\. 84 | 85 | You must specify the `queue_url` option within the `s3` source configuration, which is the Amazon SQS queue URL to read from\. To format the URL, locate the **Subscription endpoint** in the subscriber configuration and change `arn:aws:` to `https://`\. For example, `https://sqs.{region}.amazonaws.com/{account-id}/AmazonSecurityLake-abdcef-Main-Queue`\. 86 | 87 | The `sts_role_arn` that you specify within the S3 source configuration must be the ARN of the pipeline role\. -------------------------------------------------------------------------------- /doc_source/createanomalydetector-tutorial.md: -------------------------------------------------------------------------------- 1 | # Tutorial: Detect high CPU usage with anomaly detection 2 | 3 | This tutorial demonstrates how to create an anomaly detector in Amazon OpenSearch Service to detect high CPU usage\. You'll use OpenSearch Dashboards to configure a detector to monitor CPU usage, and generate an alert when your CPU usage rises above a specified threshold\. 4 | 5 | **Note** 6 | These steps apply to the latest version of OpenSearch and might differ slightly for past versions\. 7 | 8 | ## Prerequisites 9 | + You must have an OpenSearch Service domain running Elasticsearch 7\.4 or later, or any OpenSearch version\. 10 | + You must be ingesting application log files into your cluster that contain CPU usage data\. 11 | 12 | ## Step 1: Create a detector 13 | 14 | First, create a detector that identifies anomalies in your CPU usage data\. 15 | 16 | 1. Open the left panel menu in OpenSearch Dashboards and choose **Anomaly Detection**, then choose **Create detector**\. 17 | 18 | 1. Name the detector **high\-cpu\-usage**\. 19 | 20 | 1. For your data source, choose your index that contains CPU usage log files where you want to identify anomalies\. 21 | 22 | 1. Choose the **Timestamp field** from your data\. Optionally, you can add a data filter\. This data filter analyzes only a subset of the data source and reduces the noise from data that's not relevant\. 23 | 24 | 1. Set the **Detector interval** to **2** minutes\. This interval defines the time \(by minute interval\) for the detector to collect the data\. 25 | 26 | 1. In **Window delay**, add a **1\-minute** delay\. This delay adds extra processing time to ensure that all data within the window is present\. 27 | 28 | 1. Choose **Next**\. On the anomaly detection dashboard, under the detector name, choose **Configure model**\. 29 | 30 | 1. For **Feature name**, enter **max\_cpu\_usage**\. For **Feature state**, select **Enable feature**\. 31 | 32 | 1. For **Find anomalies based on**, choose **Field value**\. 33 | 34 | 1. For **Aggregation method**, choose **`max()`**\. 35 | 36 | 1. For **Field**, select the field in your data to check for anomalies\. For example, it might be called `cpu_usage_percentage`\. 37 | 38 | 1. Keep all other settings as their defaults and choose **Next**\. 39 | 40 | 1. Ignore the detector jobs setup and choose **Next**\. 41 | 42 | 1. In the pop\-up window, choose when to start the detector \(automatically or manually\), and then choose **Confirm**\. 43 | 44 | Now that the detector is configured, after it initializes, you will be able to see real\-time results of the CPU usage in the **Real\-time results** section of your detector panel\. The **Live anomalies** section displays any anomalies that occur as data is being ingested in real time\. 45 | 46 | ## Step 2: Configure an alert 47 | 48 | Now that you've created a detector, create a monitor that invokes an alert to send a message to Slack when it detects CPU usage that meets the conditions specified in the detector settings\. You'll receive Slack notifications when data from one or more indexes meets the conditions that invoke the alert\. 49 | 50 | 1. Open the left panel menu in OpenSearch Dashboards and choose **Alerting**, then choose **Create monitor**\. 51 | 52 | 1. Provide a name for the monitor\. 53 | 54 | 1. For **Monitor type**, choose **Per\-query monitor**\. A per\-query monitor runs a specified query and defines the triggers\. 55 | 56 | 1. For **Monitor defining method**, choose **Anomaly detector**, then select the detector that you created in the previous section from the **Detector** dropdown menu\. 57 | 58 | 1. For **Schedule**, choose how often the monitor collects data and how often you receive alerts\. For the purposes of this tutorial, set the schedule to run every **7** minutes\. 59 | 60 | 1. In the **Triggers** section, choose **Add trigger**\. For **Trigger name**, enter **High CPU usage**\. For this tutorial, for **Severity level**, choose **1**, which is the highest level of severity\. 61 | 62 | 1. For **Anomaly grade threshold**, choose **IS ABOVE**\. On the menu under that, choose the grade threshold to apply\. For this tutorial, set the **Anomaly grade** to **0\.7**\. 63 | 64 | 1. For **Anomaly confidence threshold**, choose **IS ABOVE**\. On the menu under that, enter the same number as your Anomaly grade\. For this tutorial, set the **Anomaly confidence threshold** to **0\.7**\. 65 | 66 | 1. In the **Actions** section, choose **Destination**\. In the **Name** field, choose the name of the destination\. On the **Type** menu, choose **Slack**\. In the **Webhook URL** field, enter a webhook URL to receive alerts to\. For more information, see [Sending messages using incoming webhooks](https://api.slack.com/messaging/webhooks)\. 67 | 68 | 1. Choose **Create**\. 69 | 70 | ## Related resources 71 | + [Configuring alerts in Amazon OpenSearch Service](alerting.md) 72 | + [Anomaly detection in Amazon OpenSearch Service](ad.md) 73 | + [Anomaly detection API](https://opensearch.org/docs/latest/monitoring-plugins/ad/api/) -------------------------------------------------------------------------------- /doc_source/cross-service-confused-deputy-prevention.md: -------------------------------------------------------------------------------- 1 | # Cross\-service confused deputy prevention 2 | 3 | The confused deputy problem is a security issue where an entity that doesn't have permission to perform an action can coerce a more\-privileged entity to perform the action\. In AWS, cross\-service impersonation can result in the confused deputy problem\. Cross\-service impersonation can occur when one service \(the *calling service*\) calls another service \(the *called service*\)\. The calling service can be manipulated to use its permissions to act on another customer's resources in a way it should not otherwise have permission to access\. To prevent this, AWS provides tools that help you protect your data for all services with service principals that have been given access to resources in your account\. 4 | 5 | We recommend using the [https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_condition-keys.html#condition-keys-sourcearn](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_condition-keys.html#condition-keys-sourcearn) and [https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_condition-keys.html#condition-keys-sourceaccount](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_condition-keys.html#condition-keys-sourceaccount) global condition context keys in resource policies to limit the permissions that Amazon OpenSearch Service gives another service to the resource\. If the `aws:SourceArn` value does not contain the account ID, such as an Amazon S3 bucket ARN, you must use both global condition context keys to limit permissions\. If you use both global condition context keys and the `aws:SourceArn` value contains the account ID, the `aws:SourceAccount` value and the account in the `aws:SourceArn` value must use the same account ID when used in the same policy statement\. Use `aws:SourceArn` if you want only one resource to be associated with the cross\-service access\. Use `aws:SourceAccount` if you want to allow any resource in that account to be associated with the cross\-service use\. 6 | 7 | The value of `aws:SourceArn` must be the ARN of the OpenSearch Service domain\. 8 | 9 | The most effective way to protect against the confused deputy problem is to use the `aws:SourceArn` global condition context key with the full ARN of the resource\. If you don't know the full ARN of the resource or if you are specifying multiple resources, use the `aws:SourceArn` global context condition key with wildcards \(`*`\) for the unknown portions of the ARN\. For example, `arn:aws:es:*:123456789012:*`\. 10 | 11 | The following example shows how you can use the `aws:SourceArn` and `aws:SourceAccount` global condition context keys in OpenSearch Service to prevent the confused deputy problem\. 12 | 13 | ``` 14 | { 15 | "Version":"2012-10-17", 16 | "Statement":{ 17 | "Sid":"ConfusedDeputyPreventionExamplePolicy", 18 | "Effect":"Allow", 19 | "Principal":{ 20 | "Service":"es.amazonaws.com" 21 | }, 22 | "Action":"sts:AssumeRole", 23 | "Condition":{ 24 | "StringEquals":{ 25 | "aws:SourceAccount":"123456789012" 26 | }, 27 | "ArnLike":{ 28 | "aws:SourceArn":"arn:aws:es:region:123456789012:domain/my-domain" 29 | } 30 | } 31 | } 32 | } 33 | ``` -------------------------------------------------------------------------------- /doc_source/customendpoint.md: -------------------------------------------------------------------------------- 1 | # Creating a custom endpoint for Amazon OpenSearch Service 2 | 3 | Creating a custom endpoint for your Amazon OpenSearch Service domain makes it easier for you to refer to your OpenSearch and OpenSearch Dashboards URLs\. You can include your company's branding or just use a shorter, easier\-to\-remember endpoint than the standard one\. 4 | 5 | If you ever need to switch to a new domain, just update your DNS to point to the new URL and continue using the same endpoint as before\. 6 | 7 | You secure custom endpoints by either generating a certificate in AWS Certificate Manager \(ACM\) or importing one of your own\. 8 | 9 | ## Custom endpoints for new domains 10 | 11 | You can enable a custom endpoint for a new OpenSearch Service domain using the OpenSearch Service console, AWS CLI, or configuration API\. 12 | 13 | **To customize your endpoint \(console\)** 14 | 15 | 1. From the OpenSearch Service console, choose **Create domain** and provide a name for the domain\. 16 | 17 | 1. Under **Custom endpoint**, select **Enable custom endpoint**\. 18 | 19 | 1. For **Custom hostname**, enter your preferred custom endpoint hostname\. The hostname should be a fully qualified domain name \(FQDN\), such as www\.yourdomain\.com or example\.yourdomain\.com\. 20 | **Note** 21 | If you don't have a [wildcard certificate](https://en.wikipedia.org/wiki/Wildcard_certificate) you must obtain a new certificate for your custom endpoint's subdomains\. 22 | 23 | 1. For **AWS certificate**, choose the SSL certificate to use for your domain\. If no certificates are available, you can import one into ACM or use ACM to provision one\. For more information, see [Issuing and Managing Certificates](https://docs.aws.amazon.com/acm/latest/userguide/gs.html) in the *AWS Certificate Manager User Guide*\. 24 | **Note** 25 | The certificate must have the custom endpoint name and be in the same account as your OpenSearch Service domain\. The certificate status should be ISSUED\. 26 | + Follow the rest of the steps to create your domain and choose **Create**\. 27 | + Select the domain when it's finished processing to view your custom endpoint\. 28 | 29 | To use the CLI or configuration API, use the `CreateDomain` and `UpdateDomainConfig` operations\. For more information, see the [AWS CLI Command Reference](https://docs.aws.amazon.com/cli/latest/reference/) and [Amazon OpenSearch Service API Reference](https://docs.aws.amazon.com/opensearch-service/latest/APIReference/Welcome.html)\. 30 | 31 | ## Custom endpoints for existing domains 32 | 33 | To add a custom endpoint to an existing OpenSearch Service domain, choose **Edit** and perform steps 2–4 above\. Editing a domain's custom endpoint triggers a [blue/green deployment](managedomains-configuration-changes.md)\. 34 | 35 | ## Next steps 36 | 37 | After you enable a custom endpoint for your OpenSearch Service domain, you must create a CNAME mapping in Amazon Route 53 \(or your preferred DNS service provider\) to route traffic to the custom endpoint and its subdomains\. Create the CNAME record pointing the custom endpoint to the auto\-generated domain endpoint\. The custom endpoint hostname is the *name* of the CNAME record, and the domain endpoint hostname is the *value* of the CNAME record\. Without this mapping, your custom endpoint won't work\. For steps to create this mapping in Route 53, see [Configuring DNS routing for a new domain ](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/dns-configuring-new-domain.html) and [Creating a hosted zone for a subdomain](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/dns-routing-traffic-for-subdomains.html#dns-routing-traffic-for-subdomains-creating-hosted-zone)\. For other providers, consult their documentation\. 38 | 39 | If you use [SAML authentication for OpenSearch Dashboards](saml.md), you must update your IdP with the new SSO URL\. -------------------------------------------------------------------------------- /doc_source/data-protection.md: -------------------------------------------------------------------------------- 1 | # Data protection in Amazon OpenSearch Service 2 | 3 | The AWS [shared responsibility model](http://aws.amazon.com/compliance/shared-responsibility-model/) applies to data protection in Amazon OpenSearch Service\. As described in this model, AWS is responsible for protecting the global infrastructure that runs all of the AWS Cloud\. You are responsible for maintaining control over your content that is hosted on this infrastructure\. This content includes the security configuration and management tasks for the AWS services that you use\. For more information about data privacy, see the [Data Privacy FAQ](http://aws.amazon.com/compliance/data-privacy-faq)\. For information about data protection in Europe, see the [AWS Shared Responsibility Model and GDPR](http://aws.amazon.com/blogs/security/the-aws-shared-responsibility-model-and-gdpr/) blog post on the *AWS Security Blog*\. 4 | 5 | For data protection purposes, we recommend that you protect AWS account credentials and set up individual users with AWS IAM Identity Center \(successor to AWS Single Sign\-On\) or AWS Identity and Access Management \(IAM\)\. That way, each user is given only the permissions necessary to fulfill their job duties\. We also recommend that you secure your data in the following ways: 6 | + Use multi\-factor authentication \(MFA\) with each account\. 7 | + Use SSL/TLS to communicate with AWS resources\. We require TLS 1\.2 and recommend TLS 1\.3\. 8 | + Set up API and user activity logging with AWS CloudTrail\. 9 | + Use AWS encryption solutions, along with all default security controls within AWS services\. 10 | + Use advanced managed security services such as Amazon Macie, which assists in discovering and securing sensitive data that is stored in Amazon S3\. 11 | + If you require FIPS 140\-2 validated cryptographic modules when accessing AWS through a command line interface or an API, use a FIPS endpoint\. For more information about the available FIPS endpoints, see [Federal Information Processing Standard \(FIPS\) 140\-2](http://aws.amazon.com/compliance/fips/)\. 12 | 13 | We strongly recommend that you never put confidential or sensitive information, such as your customers' email addresses, into tags or free\-form text fields such as a **Name** field\. This includes when you work with OpenSearch Service or other AWS services using the console, API, AWS CLI, or AWS SDKs\. Any data that you enter into tags or free\-form text fields used for names may be used for billing or diagnostic logs\. If you provide a URL to an external server, we strongly recommend that you do not include credentials information in the URL to validate your request to that server\. -------------------------------------------------------------------------------- /doc_source/data-streams.md: -------------------------------------------------------------------------------- 1 | # Managing time\-series data in Amazon OpenSearch Service with data streams 2 | 3 | A typical workflow to manage time\-series data involves multiple steps, such as creating a rollover index alias, defining a write index, and defining common mappings and settings for the backing indices\. 4 | 5 | Data streams in Amazon OpenSearch Service help simplify this initial setup process\. Data streams work out of the box for time\-based data such as application logs that are typically append\-only in nature\. 6 | 7 | Data streams requires OpenSearch 1\.0 or later\. Full documentation for the feature is available in the [OpenSearch documentation](https://opensearch.org/docs/opensearch/data-streams/)\. 8 | 9 | ## Getting started with data streams 10 | 11 | A data stream is internally composed of multiple backing indices\. Search requests are routed to all the backing indices, while indexing requests are routed to the latest write index\. 12 | 13 | ### Step 1: Create an index template 14 | 15 | To create a data stream, you first need to create an index template that configures a set of indexes as a data stream\. The `data_stream` object indicates that it’s a data stream and not a regular index template\. The index pattern matches with the name of the data stream: 16 | 17 | ``` 18 | PUT _index_template/logs-template 19 | { 20 | "index_patterns": [ 21 | "my-data-stream", 22 | "logs-*" 23 | ], 24 | "data_stream": {}, 25 | "priority": 100 26 | } 27 | ``` 28 | 29 | In this case, each ingested document must have an `@timestamp` field\. You can also define your own custom timestamp field as a property in the `data_stream` object: 30 | 31 | ``` 32 | PUT _index_template/logs-template 33 | { 34 | "index_patterns": "my-data-stream", 35 | "data_stream": { 36 | "timestamp_field": { 37 | "name": "request_time" 38 | } 39 | } 40 | } 41 | ``` 42 | 43 | ### Step 2: Create a data stream 44 | 45 | After you create an index template, you can directly start ingesting data without creating a data stream\. 46 | 47 | Because we have a matching index template with a `data_stream` object, OpenSearch automatically creates the data stream: 48 | 49 | ``` 50 | POST logs-staging/_doc 51 | { 52 | "message": "login attempt failed", 53 | "@timestamp": "2013-03-01T00:00:00" 54 | } 55 | ``` 56 | 57 | ### Step 3: Ingest data into the data stream 58 | 59 | To ingest data into a data stream, you can use the regular indexing APIs\. Make sure every document that you index has a timestamp field\. If you try to ingest a document that doesn’t have a timestamp field, you get an error\. 60 | 61 | ``` 62 | POST logs-redis/_doc 63 | { 64 | "message": "login attempt", 65 | "@timestamp": "2013-03-01T00:00:00" 66 | } 67 | ``` 68 | 69 | ### Step 4: Searching a data stream 70 | 71 | You can search a data stream just like you search a regular index or an index alias\. The search operation applies to all of the backing indexes \(all data present in the stream\)\. 72 | 73 | ``` 74 | GET logs-redis/_search 75 | { 76 | "query": { 77 | "match": { 78 | "message": "login" 79 | } 80 | } 81 | } 82 | ``` 83 | 84 | ### Step 5: Rollover a data stream 85 | 86 | You can set up an [Index State Management \(ISM\)](ism.md) policy to automate the rollover process for the data stream\. The ISM policy is applied to the backing indexes at the time of their creation\. When you associate a policy to a data stream, it only affects the future backing indexes of that data stream\. You also don’t need to provide the `rollover_alias` setting, because the ISM policy infers this information from the backing index\. 87 | 88 | **Note** 89 | If you migrate a backing index to [cold storage](cold-storage.md), OpenSearch removes this index from the data stream\. Even if you move the index back to [UltraWarm](ultrawarm.md), the index remains independent and not part of the original data stream\. After an index has been removed from the data stream, searching against the stream won't return any data from the index\. 90 | 91 | **Warning** 92 | The write index for a data stream can't be migrated to cold storage\. If you wish to migrate data in your data stream to cold storage, you must rollover the data stream before migration\. 93 | 94 | ### Step 6: Manage data streams in OpenSearch Dashboards 95 | 96 | To manage data streams from OpenSearch Dashboards, open **OpenSearch Dashboards**, choose **Index Management**, select **Indices** or **Policy managed indices**\. 97 | 98 | ### Step 7: Delete a data stream 99 | 100 | The delete operation first deletes the backing indexes of a data stream and then deletes the data stream itself\. 101 | 102 | To delete a data stream and all of its hidden backing indices: 103 | 104 | ``` 105 | DELETE _data_stream/name_of_data_stream 106 | ``` -------------------------------------------------------------------------------- /doc_source/delete-pipeline.md: -------------------------------------------------------------------------------- 1 | # Deleting Amazon OpenSearch Ingestion pipelines 2 | 3 | You can delete an Amazon OpenSearch Ingestion pipeline using the AWS Management Console, the AWS CLI, or the OpenSearch Ingestion API\. You can't delete a pipeline when has a status of `Creating` or `Updating`\. 4 | 5 | ## Console 6 | 7 | **To delete a pipeline** 8 | 9 | 1. Sign in to the Amazon OpenSearch Service console at [https://console\.aws\.amazon\.com/aos/home](https://console.aws.amazon.com/aos/home)\. 10 | 11 | 1. Choose **Pipelines** in the left navigation pane\. 12 | 13 | 1. Select the pipeline that you want to delete and choose **Delete**\. 14 | 15 | 1. Confirm deletion and choose **Delete**\. 16 | 17 | ## CLI 18 | 19 | To delete a pipeline using the AWS CLI, send a [delete\-pipeline](https://docs.aws.amazon.com/cli/latest/reference/osis/delete-pipeline.html) request: 20 | 21 | ``` 22 | aws osis delete-pipeline --pipeline-name "my-pipeline" 23 | ``` 24 | 25 | ## OpenSearch Ingestion API 26 | 27 | To delete an OpenSearch Ingestion pipeline using the OpenSearch Ingestion API, call the [DeletePipeline](https://docs.aws.amazon.com/opensearch-service/latest/APIReference/API_osis_DeletePipeline.html) operation with the following parameter: 28 | + `PipelineName` – the name of the pipeline\. -------------------------------------------------------------------------------- /doc_source/disaster-recovery-resiliency.md: -------------------------------------------------------------------------------- 1 | # Resilience in Amazon OpenSearch Service 2 | 3 | The AWS global infrastructure is built around AWS Regions and Availability Zones\. AWS Regions provide multiple physically separated and isolated Availability Zones, which are connected with low\-latency, high\-throughput, and highly redundant networking\. With Availability Zones, you can design and operate applications and databases that automatically fail over between Availability Zones without interruption\. Availability Zones are more highly available, fault tolerant, and scalable than traditional single or multiple data center infrastructures\. 4 | 5 | For more information about AWS Regions and Availability Zones, see [AWS Global Infrastructure](http://aws.amazon.com/about-aws/global-infrastructure/)\. 6 | 7 | In addition to the AWS global infrastructure, OpenSearch Service offers several features to help support your data resiliency and backup needs: 8 | + [Multi\-AZ domains and replica shards](managedomains-multiaz.md) 9 | + [Automated and manual snapshots](managedomains-snapshots.md) -------------------------------------------------------------------------------- /doc_source/encryption.md: -------------------------------------------------------------------------------- 1 | # Encryption in Amazon Elasticsearch Service 2 | 3 | These topics cover how to enable encryption of data at rest and node\-to\-node encryption for Amazon Elasticsearch Service domains\. 4 | 5 | **Topics** 6 | + [Encryption of Data at Rest for Amazon Elasticsearch Service](encryption-at-rest.md) 7 | + [Node\-to\-node Encryption for Amazon Elasticsearch Service](ntn.md) -------------------------------------------------------------------------------- /doc_source/features-by-version.md: -------------------------------------------------------------------------------- 1 | # Features by engine version in Amazon OpenSearch Service 2 | 3 | Many OpenSearch Service features have a minimum OpenSearch version requirement or legacy Elasticsearch OSS version requirement\. If you meet the minimum version for a feature, but the feature isn't available on your domain, update your domain's [service software](service-software.md)\. 4 | 5 | [\[See the AWS documentation website for more details\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/features-by-version.html) 6 | 7 | For information about plugins, which enable some of these features and additional functionality, see [Plugins by engine version in Amazon OpenSearch Service](supported-plugins.md)\. For information about the OpenSearch API for each version, see [Supported operations in Amazon OpenSearch Service](supported-operations.md)\. -------------------------------------------------------------------------------- /doc_source/fgac-walkthrough-basic.md: -------------------------------------------------------------------------------- 1 | # Tutorial: Configure a domain with the internal user database and HTTP basic authentication 2 | 3 | This tutorial covers another popular [fine\-grained access control](fgac.md) use case: a master user in the internal user database and HTTP basic authentication for OpenSearch Dashboards\. 4 | 5 | **To get started with fine\-grained access control** 6 | 7 | 1. [Create a domain](createupdatedomains.md) with the following settings: 8 | + OpenSearch 1\.0 or later, or Elasticsearch 7\.9 or later 9 | + Public access 10 | + Fine\-grained access control with a master user in the internal user database \(`TheMasterUser` for the rest of this tutorial\) 11 | + Amazon Cognito authentication for Dashboards *disabled* 12 | + The following access policy: 13 | 14 | ``` 15 | { 16 | "Version": "2012-10-17", 17 | "Statement": [ 18 | { 19 | "Effect": "Allow", 20 | "Principal": { 21 | "AWS": [ 22 | "*" 23 | ] 24 | }, 25 | "Action": [ 26 | "es:ESHttp*" 27 | ], 28 | "Resource": "arn:aws:es:region:account:domain/domain-name/*" 29 | } 30 | ] 31 | } 32 | ``` 33 | + HTTPS required for all traffic to the domain 34 | + Node\-to\-node encryption 35 | + Encryption of data at rest 36 | 37 | 1. Navigate to OpenSearch Dashboards\. 38 | 39 | 1. Sign in using `TheMasterUser`\. 40 | 41 | 1. Choose **Try our sample data**\. 42 | 43 | 1. Add the sample flight data\. 44 | 45 | 1. Choose **Security**, **Internal users**, **Create internal user**\. 46 | 47 | 1. Name the user `new-user` and specify a password\. Then choose **Create**\. 48 | 49 | 1. Choose **Roles**, **Create role**\. 50 | 51 | 1. Name the role `new-role`\. 52 | 53 | 1. For index permissions, specify `dashboards_sample_data_fli*` for the index pattern\. 54 | 55 | 1. For the action group, choose **read**\. 56 | 57 | 1. For **Document level security**, specify the following query: 58 | 59 | ``` 60 | { 61 | "match": { 62 | "FlightDelay": true 63 | } 64 | } 65 | ``` 66 | 67 | 1. For field\-level security, choose **Exclude** and specify `FlightNum`\. 68 | 69 | 1. For **Anonymization**, specify `Dest`\. 70 | 71 | 1. Choose **Create**\. 72 | 73 | 1. Choose **Mapped users**, **Manage mapping**\. Then add `new-user` to **Users** and choose **Map**\. 74 | 75 | 1. Return to the list of roles and choose **opensearch\_dashboards\_user**\. Choose **Mapped users**, **Manage mapping**\. Then add `new-user` to **Users** and choose **Map**\. 76 | 77 | 1. In a new, private browser window, navigate to Dashboards, sign in using `new-user`, and then choose **Explore on my own**\. 78 | 79 | 1. Go to **Dev Tools** and run the default search: 80 | 81 | ``` 82 | GET _search 83 | { 84 | "query": { 85 | "match_all": {} 86 | } 87 | } 88 | ``` 89 | 90 | Note the permissions error\. `new-user` doesn't have permissions to run cluster\-wide searches\. 91 | 92 | 1. Run another search: 93 | 94 | ``` 95 | GET dashboards_sample_data_flights/_search 96 | { 97 | "query": { 98 | "match_all": {} 99 | } 100 | } 101 | ``` 102 | 103 | Note that all matching documents have a `FlightDelay` field of `true`, an anonymized `Dest` field, and no `FlightNum` field\. 104 | 105 | 1. In your original browser window, signed in as `TheMasterUser`, choose **Dev Tools** and perform the same searches\. Note the difference in permissions, number of hits, matching documents, and included fields\. -------------------------------------------------------------------------------- /doc_source/genref.md: -------------------------------------------------------------------------------- 1 | # General reference for Amazon OpenSearch Service 2 | 3 | Amazon OpenSearch Service supports a variety of instances, operations, plugins, and other resources\. 4 | 5 | **Topics** 6 | + [Supported instance types in Amazon OpenSearch Service](supported-instance-types.md) 7 | + [Features by engine version in Amazon OpenSearch Service](features-by-version.md) 8 | + [Plugins by engine version in Amazon OpenSearch Service](supported-plugins.md) 9 | + [Supported operations in Amazon OpenSearch Service](supported-operations.md) 10 | + [Amazon OpenSearch Service quotas](limits.md) 11 | + [Reserved Instances in Amazon OpenSearch Service](ri.md) 12 | + [Other supported resources in Amazon OpenSearch Service](supported-resources.md) -------------------------------------------------------------------------------- /doc_source/glossary.md: -------------------------------------------------------------------------------- 1 | # AWS glossary 2 | 3 | For the latest AWS terminology, see the [AWS glossary](https://docs.aws.amazon.com/glossary/latest/reference/glos-chap.html) in the *AWS Glossary Reference*\. -------------------------------------------------------------------------------- /doc_source/gsgcreate-domain.md: -------------------------------------------------------------------------------- 1 | # Step 1: Create an Amazon OpenSearch Service domain 2 | 3 | **Important** 4 | This is a concise tutorial for configuring a *test * Amazon OpenSearch Service domain\. Do not use this process to create production domains\. For a comprehensive version of the same process, see [Creating and managing Amazon OpenSearch Service domains](createupdatedomains.md)\. 5 | 6 | An OpenSearch Service domain is synonymous with an OpenSearch cluster\. Domains are clusters with the settings, instance types, instance counts, and storage resources that you specify\. You can create an OpenSearch Service domain by using the console, the AWS CLI, or the AWS SDKs\. 7 | 8 | **To create an OpenSearch Service domain using the console** 9 | 10 | 1. Go to [https://aws\.amazon\.com](https://aws.amazon.com) and choose **Sign In to the Console**\. 11 | 12 | 1. Under **Analytics**, choose **Amazon OpenSearch Service**\. 13 | 14 | 1. Choose **Create domain**\. 15 | 16 | 1. Provide a name for the domain\. The examples in this tutorial use the name *movies*\. 17 | 18 | 1. For the domain creation method, choose **Standard create**\. 19 | **Note** 20 | To quickly configure a production domain with best practices, you can choose **Easy create**\. For the development and testing purposes of this tutorial, we'll use **Standard create**\. 21 | 22 | 1. For templates, choose **Dev/test**\. 23 | 24 | 1. For the deployment option, choose **Domain with standby**\. 25 | 26 | 1. For **Version**, choose the latest version\. 27 | 28 | 1. For now, ignore the **Data nodes**, **Warm and cold data storage**, **Dedicated master nodes**, **Snapshot configuration**, and **Custom endpoint** sections\. 29 | 30 | 1. For simplicity in this tutorial, use a public access domain\. Under **Network**, choose **Public access**\. 31 | 32 | 1. In the fine\-grained access control settings, keep the **Enable fine\-grained access control** check box selected\. Select **Create master user** and provide a username and password\. 33 | 34 | 1. For now, ignore the **SAML authentication** and **Amazon Cognito authentication** sections\. 35 | 36 | 1. For **Access policy**, choose **Only use fine\-grained access control**\. In this tutorial, fine\-grained access control handles authentication, not the domain access policy\. 37 | 38 | 1. Ignore the rest of the settings and choose **Create**\. New domains typically take 15–30 minutes to initialize, but can take longer depending on the configuration\. After your domain initializes, select it to open its configuration pane\. Note the domain endpoint under **General information** \(for example, `https://search-my-domain.us-east-1.es.amazonaws.com`\), which you'll use in the next step\. 39 | 40 | **Next**: [Upload data to an OpenSearch Service domain for indexing](gsgupload-data.md) -------------------------------------------------------------------------------- /doc_source/gsgdeleting.md: -------------------------------------------------------------------------------- 1 | # Step 4: Delete an Amazon OpenSearch Service domain 2 | 3 | Because the *movies* domain from this tutorial is for test purposes, make sure to delete it when you're done experimenting to avoid incurring charges\. 4 | 5 | **To delete an OpenSearch Service domain from the console** 6 | 7 | 1. Sign in to the **Amazon OpenSearch Service** console\. 8 | 9 | 1. Under **Domains**, select the **movies** domain\. 10 | 11 | 1. Choose **Delete** and confirm deletion\. 12 | 13 | ## Next steps 14 | 15 | Now that you know how to create a domain and index data, you might want to try some of the following exercises: 16 | + Learn about more advanced options for creating a domain\. For more information, see [Creating and managing Amazon OpenSearch Service domains](createupdatedomains.md)\. 17 | + Discover how to manage the indices in your domain\. For more information, see [Managing indexes in Amazon OpenSearch Service](managing-indices.md)\. 18 | + Try out one of the tutorials for working with Amazon OpenSearch Service\. For more information, see [Amazon OpenSearch Service tutorials](tutorials.md)\. -------------------------------------------------------------------------------- /doc_source/gsgsearch.md: -------------------------------------------------------------------------------- 1 | # Step 3: Search documents in Amazon OpenSearch Service 2 | 3 | To search documents in an Amazon OpenSearch Service domain, use the OpenSearch search API\. Alternatively, you can use [OpenSearch Dashboards](dashboards.md) to search documents in the domain\. 4 | 5 | ## Search documents from the command line 6 | 7 | Run the following command to search the *movies* domain for the word *mars*: 8 | 9 | ``` 10 | curl -XGET -u 'master-user:master-user-password' 'domain-endpoint/movies/_search?q=mars&pretty=true' 11 | ``` 12 | 13 | If you used the bulk data on the previous page, try searching for *rebel* instead\. 14 | 15 | You should see a response similar to the following: 16 | 17 | ``` 18 | { 19 | "took" : 5, 20 | "timed_out" : false, 21 | "_shards" : { 22 | "total" : 5, 23 | "successful" : 5, 24 | "skipped" : 0, 25 | "failed" : 0 26 | }, 27 | "hits" : { 28 | "total" : { 29 | "value" : 1, 30 | "relation" : "eq" 31 | }, 32 | "max_score" : 0.2876821, 33 | "hits" : [ 34 | { 35 | "_index" : "movies", 36 | "_type" : "_doc", 37 | "_id" : "1", 38 | "_score" : 0.2876821, 39 | "_source" : { 40 | "director" : "Burton, Tim", 41 | "genre" : [ 42 | "Comedy", 43 | "Sci-Fi" 44 | ], 45 | "year" : 1996, 46 | "actor" : [ 47 | "Jack Nicholson", 48 | "Pierce Brosnan", 49 | "Sarah Jessica Parker" 50 | ], 51 | "title" : "Mars Attacks!" 52 | } 53 | } 54 | ] 55 | } 56 | } 57 | ``` 58 | 59 | ## Search documents using OpenSearch Dashboards 60 | 61 | OpenSearch Dashboards is a popular open source visualization tool designed to work with OpenSearch\. It provides a helpful user interface for you to search and monitor your indices\. 62 | 63 | **To search documents from an OpenSearch Service domain using Dashboards** 64 | 65 | 1. Navigate to the OpenSearch Dashboards URL for your domain\. You can find the URL on the domain's dashboard in the OpenSearch Service console\. The URL follows this format: 66 | 67 | ``` 68 | domain-endpoint/_dashboards/ 69 | ``` 70 | 71 | 1. Log in using your primary username and password\. 72 | 73 | 1. To use Dashboards, you need to create at least one index pattern\. Dashboards uses these patterns to identify which indexes you want to analyze\. Open the left navigation panel, choose **Stack Management**, choose **Index Patterns**, and then choose **Create index pattern**\. For this tutorial, enter *movies*\. 74 | 75 | 1. Choose **Next step** and then choose **Create index pattern**\. After the pattern is created, you can view the various document fields such as `actor` and `director`\. 76 | 77 | 1. Go back to the **Index Patterns** page and make sure that `movies` is set as the default\. If it's not, select the pattern and choose the star icon to make it the default\. 78 | 79 | 1. To begin searching your data, open the left navigation panel again and choose **Discover**\. 80 | 81 | 1. In the search bar, enter *mars* if you uploaded a single document, or *rebel* if you uploaded multiple documents, and then press **Enter**\. You can try searching other terms, such as actor or director names\. 82 | 83 | **Next**: [Delete a domain ](gsgdeleting.md) -------------------------------------------------------------------------------- /doc_source/gsgupload-data.md: -------------------------------------------------------------------------------- 1 | # Step 2: Upload data to Amazon OpenSearch Service for indexing 2 | 3 | **Important** 4 | This is a concise tutorial for uploading a small amount of test data to Amazon OpenSearch Service\. For more about uploading data in a production domain, see [Indexing data in Amazon OpenSearch Service](indexing.md)\. 5 | 6 | You can upload data to an OpenSearch Service domain using the command line or most programming languages\. 7 | 8 | The following example requests use [curl](https://curl.haxx.se/) \(a common HTTP client\) for brevity and convenience\. Clients like curl can't perform the request signing that's required if your access policies specify IAM users or roles\. To successfully complete this process, you must use fine\-grained access control with a primary username and password like you configured in [Step 1](gsgcreate-domain.md)\. 9 | 10 | You can install curl on Windows and use it from the command prompt, but we recommend a tool like [Cygwin](https://www.cygwin.com/) or the [Windows Subsystem for Linux](https://docs.microsoft.com/en-us/windows/wsl/install-win10)\. macOS and most Linux distributions come with curl preinstalled\. 11 | 12 | ## Option 1: Upload a single document 13 | 14 | Run the following command to add a single document to the *movies* domain: 15 | 16 | ``` 17 | curl -XPUT -u 'master-user:master-user-password' 'domain-endpoint/movies/_doc/1' -d '{"director": "Burton, Tim", "genre": ["Comedy","Sci-Fi"], "year": 1996, "actor": ["Jack Nicholson","Pierce Brosnan","Sarah Jessica Parker"], "title": "Mars Attacks!"}' -H 'Content-Type: application/json' 18 | ``` 19 | 20 | In the command, provide the username and password that you created in [Step 1](gsgcreate-domain.md)\. 21 | 22 | For a detailed explanation of this command and how to make signed requests to OpenSearch Service, see [Indexing data in Amazon OpenSearch Service](indexing.md)\. 23 | 24 | ## Option 2: Upload multiple documents 25 | 26 | **To upload a JSON file that contains multiple documents to an OpenSearch Service domain** 27 | 28 | 1. Create a local file called `bulk_movies.json`\. Paste the following content into the file and add a trailing newline: 29 | 30 | ``` 31 | { "index" : { "_index": "movies", "_id" : "2" } } 32 | {"director": "Frankenheimer, John", "genre": ["Drama", "Mystery", "Thriller", "Crime"], "year": 1962, "actor": ["Lansbury, Angela", "Sinatra, Frank", "Leigh, Janet", "Harvey, Laurence", "Silva, Henry", "Frees, Paul", "Gregory, James", "Bissell, Whit", "McGiver, John", "Parrish, Leslie", "Edwards, James", "Flowers, Bess", "Dhiegh, Khigh", "Payne, Julie", "Kleeb, Helen", "Gray, Joe", "Nalder, Reggie", "Stevens, Bert", "Masters, Michael", "Lowell, Tom"], "title": "The Manchurian Candidate"} 33 | { "index" : { "_index": "movies", "_id" : "3" } } 34 | {"director": "Baird, Stuart", "genre": ["Action", "Crime", "Thriller"], "year": 1998, "actor": ["Downey Jr., Robert", "Jones, Tommy Lee", "Snipes, Wesley", "Pantoliano, Joe", "Jacob, Ir\u00e8ne", "Nelligan, Kate", "Roebuck, Daniel", "Malahide, Patrick", "Richardson, LaTanya", "Wood, Tom", "Kosik, Thomas", "Stellate, Nick", "Minkoff, Robert", "Brown, Spitfire", "Foster, Reese", "Spielbauer, Bruce", "Mukherji, Kevin", "Cray, Ed", "Fordham, David", "Jett, Charlie"], "title": "U.S. Marshals"} 35 | { "index" : { "_index": "movies", "_id" : "4" } } 36 | {"director": "Ray, Nicholas", "genre": ["Drama", "Romance"], "year": 1955, "actor": ["Hopper, Dennis", "Wood, Natalie", "Dean, James", "Mineo, Sal", "Backus, Jim", "Platt, Edward", "Ray, Nicholas", "Hopper, William", "Allen, Corey", "Birch, Paul", "Hudson, Rochelle", "Doran, Ann", "Hicks, Chuck", "Leigh, Nelson", "Williams, Robert", "Wessel, Dick", "Bryar, Paul", "Sessions, Almira", "McMahon, David", "Peters Jr., House"], "title": "Rebel Without a Cause"} 37 | ``` 38 | 39 | 1. Run the following command in the local directory where the file is stored to upload it to the *movies* domain: 40 | 41 | ``` 42 | curl -XPOST -u 'master-user:master-user-password' 'domain-endpoint/_bulk' --data-binary @bulk_movies.json -H 'Content-Type: application/json' 43 | ``` 44 | 45 | For more information about the bulk file format, see [Indexing data in Amazon OpenSearch Service](indexing.md)\. 46 | 47 | **Next**: [Search documents](gsgsearch.md) -------------------------------------------------------------------------------- /doc_source/gzip.md: -------------------------------------------------------------------------------- 1 | # Compressing HTTP requests in Amazon OpenSearch Service 2 | 3 | You can compress HTTP requests and responses in Amazon OpenSearch Service domains using gzip compression\. Gzip compression can help you reduce the size of your documents and lower bandwidth utilization and latency, thereby leading to improved transfer speeds\. 4 | 5 | Gzip compression is supported for all domains running OpenSearch or Elasticsearch 6\.0 or later\. Some OpenSearch clients have built\-in support for gzip compression, and many programming languages have libraries that simplify the process\. 6 | 7 | ## Enabling gzip compression 8 | 9 | Not to be confused with similar OpenSearch settings, `http_compression.enabled` is specific to OpenSearch Service and enables or disables gzip compression on a domain\. Domains running OpenSearch or Elasticsearch 7\.*x* have the gzip compression enabled by default, whereas domains running Elasticsearch 6\.*x* have it disabled by default\. 10 | 11 | To enable gzip compression, send the following request: 12 | 13 | ``` 14 | PUT _cluster/settings 15 | { 16 | "persistent" : { 17 | "http_compression.enabled": true 18 | } 19 | } 20 | ``` 21 | 22 | Requests to `_cluster/settings` must be uncompressed, so you might need to use a separate client or standard HTTP request to update cluster settings\. 23 | 24 | ## Required headers 25 | 26 | When including a gzip\-compressed request body, keep the standard `Content-Type: application/json` header, and add the `Content-Encoding: gzip` header\. To accept a gzip\-compressed response, add the `Accept-Encoding: gzip` header, as well\. If an OpenSearch client supports gzip compression, it likely includes these headers automatically\. 27 | 28 | ## Sample code \(Python 3\) 29 | 30 | The following sample uses [opensearch\-py](https://pypi.org/project/opensearch-py/) to perform the compression and send the request\. This code signs the request using your IAM credentials\. 31 | 32 | ``` 33 | from opensearchpy import OpenSearch, RequestsHttpConnection 34 | from requests_aws4auth import AWS4Auth 35 | import boto3 36 | 37 | host = '' # e.g. my-test-domain.us-east-1.es.amazonaws.com 38 | region = '' # e.g. us-west-1 39 | service = 'es' 40 | credentials = boto3.Session().get_credentials() 41 | awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service, session_token=credentials.token) 42 | 43 | # Create the client. 44 | search = OpenSearch( 45 | hosts = [{'host': host, 'port': 443}], 46 | http_auth = awsauth, 47 | use_ssl = True, 48 | verify_certs = True, 49 | http_compress = True, # enables gzip compression for request bodies 50 | connection_class = RequestsHttpConnection 51 | ) 52 | 53 | document = { 54 | "title": "Moneyball", 55 | "director": "Bennett Miller", 56 | "year": "2011" 57 | } 58 | 59 | # Send the request. 60 | print(search.index(index='movies', id='1', body=document, refresh=True)) 61 | 62 | # print(search.index(index='movies', doc_type='_doc', id='1', body=document, refresh=True)) 63 | ``` 64 | 65 | Alternately, you can specify the proper headers, compress the request body yourself, and use a standard HTTP library like [Requests](https://2.python-requests.org)\. This code signs the request using HTTP basic credentials, which your domain might support if you use [fine\-grained access control](fgac.md)\. 66 | 67 | ``` 68 | import requests 69 | import gzip 70 | import json 71 | 72 | base_url = '' # The domain with https:// and a trailing slash. For example, https://my-test-domain.us-east-1.es.amazonaws.com/ 73 | auth = ('master-user', 'master-user-password') # For testing only. Don't store credentials in code. 74 | 75 | headers = {'Accept-Encoding': 'gzip', 'Content-Type': 'application/json', 76 | 'Content-Encoding': 'gzip'} 77 | 78 | document = { 79 | "title": "Moneyball", 80 | "director": "Bennett Miller", 81 | "year": "2011" 82 | } 83 | 84 | # Compress the document. 85 | compressed_document = gzip.compress(json.dumps(document).encode()) 86 | 87 | # Send the request. 88 | path = 'movies/_doc?refresh=true' 89 | url = base_url + path 90 | response = requests.post(url, auth=auth, headers=headers, data=compressed_document) 91 | print(response.status_code) 92 | print(response.text) 93 | ``` -------------------------------------------------------------------------------- /doc_source/indexing.md: -------------------------------------------------------------------------------- 1 | # Indexing data in Amazon OpenSearch Service 2 | 3 | Because Amazon OpenSearch Service uses a REST API, numerous methods exist for indexing documents\. You can use standard clients like [curl](https://curl.haxx.se/) or any programming language that can send HTTP requests\. To further simplify the process of interacting with it, OpenSearch Service has clients for many programming languages\. Advanced users can skip directly to [Loading streaming data into Amazon OpenSearch Service](integrations.md)\. 4 | 5 | We strongly recommend that you use Amazon OpenSearch Ingestion to ingest data, which is a fully managed data collector built within OpenSearch Service\. For more information, see [Amazon OpenSearch Ingestion](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ingestion.html)\. 6 | 7 | For an introduction to indexing, see the [OpenSearch documentation](https://opensearch.org/docs/opensearch/index-data/)\. 8 | 9 | ## Naming restrictions for indexes 10 | 11 | OpenSearch Service indexes have the following naming restrictions: 12 | + All letters must be lowercase\. 13 | + Index names cannot begin with `_` or `-`\. 14 | + Index names can't contain spaces, commas, `:`, `"`, `*`, `+`, `/`, `\`, `|`, `?`, `#`, `>`, or `<`\. 15 | 16 | Don't include sensitive information in index, type, or document ID names\. OpenSearch Service uses these names in its Uniform Resource Identifiers \(URIs\)\. Servers and applications often log HTTP requests, which can lead to unnecessary data exposure if URIs contain sensitive information: 17 | 18 | ``` 19 | 2018-10-03T23:39:43 198.51.100.14 200 "GET https://opensearch-domain/dr-jane-doe/flu-patients-2018/202-555-0100/ HTTP/1.1" 20 | ``` 21 | 22 | Even if you don't have [permissions](ac.md) to view the associated JSON document, you could infer from this fake log line that one of Dr\. Doe's patients with a phone number of 202\-555\-0100 had the flu in 2018\. 23 | 24 | If OpenSearch Service detects a real or percieved IP address in an index name \(for example, `my-index-12.34.56.78.91`\), it masks the IP address\. A call to `_cat/indices` yields the following response: 25 | 26 | ``` 27 | green open my-index-x.x.x.x.91 soY19tBERoKo71WcEScidw 5 1 0 0 2kb 1kb 28 | ``` 29 | 30 | To prevent unnecessary confusion, avoid including IP addresses in index names\. 31 | 32 | ## Reducing response size 33 | 34 | Responses from the `_index` and `_bulk` APIs contain quite a bit of information\. This information can be useful for troubleshooting requests or for implementing retry logic, but can use considerable bandwidth\. In this example, indexing a 32 byte document results in a 339 byte response \(including headers\): 35 | 36 | ``` 37 | PUT opensearch-domain/more-movies/_doc/1 38 | {"title": "Back to the Future"} 39 | ``` 40 | 41 | **Response** 42 | 43 | ``` 44 | { 45 | "_index": "more-movies", 46 | "_type": "_doc", 47 | "_id": "1", 48 | "_version": 4, 49 | "result": "updated", 50 | "_shards": { 51 | "total": 2, 52 | "successful": 2, 53 | "failed": 0 54 | }, 55 | "_seq_no": 3, 56 | "_primary_term": 1 57 | } 58 | ``` 59 | 60 | This response size might seem minimal, but if you index 1,000,000 documents per day—approximately 11\.5 documents per second—339 bytes per response works out to 10\.17 GB of download traffic per month\. 61 | 62 | If data transfer costs are a concern, use the `filter_path` parameter to reduce the size of the OpenSearch Service response, but be careful not to filter out fields that you need in order to identify or retry failed requests\. These fields vary by client\. The `filter_path` parameter works for all OpenSearch Service REST APIs, but is especially useful with APIs that you call frequently, such as the `_index` and `_bulk` APIs: 63 | 64 | ``` 65 | PUT opensearch-domain/more-movies/_doc/1?filter_path=result,_shards.total 66 | {"title": "Back to the Future"} 67 | ``` 68 | 69 | **Response** 70 | 71 | ``` 72 | { 73 | "result": "updated", 74 | "_shards": { 75 | "total": 2 76 | } 77 | } 78 | ``` 79 | 80 | Instead of including fields, you can exclude fields with a `-` prefix\. `filter_path` also supports wildcards: 81 | 82 | ``` 83 | POST opensearch-domain/_bulk?filter_path=-took,-items.index._* 84 | { "index": { "_index": "more-movies", "_id": "1" } } 85 | {"title": "Back to the Future"} 86 | { "index": { "_index": "more-movies", "_id": "2" } } 87 | {"title": "Spirited Away"} 88 | ``` 89 | 90 | **Response** 91 | 92 | ``` 93 | { 94 | "errors": false, 95 | "items": [ 96 | { 97 | "index": { 98 | "result": "updated", 99 | "status": 200 100 | } 101 | }, 102 | { 103 | "index": { 104 | "result": "updated", 105 | "status": 200 106 | } 107 | } 108 | ] 109 | } 110 | ``` -------------------------------------------------------------------------------- /doc_source/infrastructure-security.md: -------------------------------------------------------------------------------- 1 | # Infrastructure security in Amazon OpenSearch Service 2 | 3 | As a managed service, Amazon OpenSearch Service is protected by AWS global network security\. For information about AWS security services and how AWS protects infrastructure, see [AWS Cloud Security](http://aws.amazon.com/security/)\. To design your AWS environment using the best practices for infrastructure security, see [Infrastructure Protection](https://docs.aws.amazon.com/wellarchitected/latest/security-pillar/infrastructure-protection.html) in *Security Pillar AWS Well‐Architected Framework*\. 4 | 5 | You use AWS published API calls to access OpenSearch Service through the network\. Clients must support the following: 6 | + Transport Layer Security \(TLS\)\. We require TLS 1\.2 and recommend TLS 1\.3\. 7 | + Cipher suites with perfect forward secrecy \(PFS\) such as DHE \(Ephemeral Diffie\-Hellman\) or ECDHE \(Elliptic Curve Ephemeral Diffie\-Hellman\)\. Most modern systems such as Java 7 and later support these modes\. 8 | 9 | Additionally, requests must be signed by using an access key ID and a secret access key that is associated with an IAM principal\. Or you can use the [AWS Security Token Service](https://docs.aws.amazon.com/STS/latest/APIReference/Welcome.html) \(AWS STS\) to generate temporary security credentials to sign requests\. 10 | 11 | You use AWS published API calls to access the OpenSearch Service configuration API through the network\. To configure the minimum required TLS version to accept, specify the `TLSSecurityPolicy` value in the domain endpoint options: 12 | 13 | ``` 14 | aws opensearch update-domain-config --domain-name my-domain --domain-endpoint-options '{"TLSSecurityPolicy": "Policy-Min-TLS-1-2-2019-07"}' 15 | ``` 16 | 17 | For details, see the [AWS CLI command reference](https://docs.aws.amazon.com/cli/latest/reference/opensearch/update-domain-config.html)\. 18 | 19 | Depending on your domain configuration, you might also need to sign requests to the OpenSearch APIs\. For more information, see [Making and signing OpenSearch Service requests](ac.md#managedomains-signing-service-requests)\. 20 | 21 | OpenSearch Service supports public access domains, which can receive requests from any internet\-connected device, and [VPC access domains](vpc.md), which are isolated from the public internet\. -------------------------------------------------------------------------------- /doc_source/list-pipeline.md: -------------------------------------------------------------------------------- 1 | # Viewing Amazon OpenSearch Ingestion pipelines 2 | 3 | You can view the details about an Amazon OpenSearch Ingestion pipeline using the AWS Management Console, the AWS CLI, or the OpenSearch Ingestion API\. 4 | 5 | ## Console 6 | 7 | **To view a pipeline** 8 | 9 | 1. Sign in to the Amazon OpenSearch Service console at [https://console\.aws\.amazon\.com/aos/home](https://console.aws.amazon.com/aos/home)\. 10 | 11 | 1. Choose **Pipelines** in the left navigation pane\. 12 | 13 | 1. \(Optional\) To view pipelines with a particular status, choose **Any status** and select a status to filter by\. 14 | 15 | A pipeline can have the following statuses: 16 | + `Creating` – The pipeline is being created\. 17 | + `Active` – The pipeline is active and ready to ingest data\. 18 | + `Updating` – The pipeline is being updated\. 19 | + `Deleting` – The pipeline is being deleted\. 20 | + `Create failed` – The pipeline could not be created\. 21 | + `Update failed` – The pipeline could not be updated\. 22 | + `Starting` – The pipeline is starting\. 23 | + `Start failed` – The pipeline could not be started\. 24 | + `Stopping` – The pipeline is being stopped\. 25 | + `Stopped` – The pipeline is stopped and can be restarted at any time\. 26 | 27 | You're not billed for Ingestion OCUs when a pipeline is in the `Create failed`, `Creating`, `Deleting`, and `Stopped` states\. 28 | 29 | ## CLI 30 | 31 | To view pipelines using the AWS CLI, send a [list\-pipelines](https://docs.aws.amazon.com/cli/latest/reference/osis/list-pipelines.html) request: 32 | 33 | ``` 34 | aws osis list-pipelines 35 | ``` 36 | 37 | The request returns a list of all existing pipelines: 38 | 39 | ``` 40 | { 41 | "NextToken": null, 42 | "Pipelines": [ 43 | {, 44 | "CreatedAt": 1.671055851E9, 45 | "LastUpdatedAt": 1.671055851E9, 46 | "MaxUnits": 4, 47 | "MinUnits": 2, 48 | "PipelineArn": "arn:aws:osis:us-west-2:123456789012:pipeline/log-pipeline", 49 | "PipelineName": "log-pipeline", 50 | "Status": "ACTIVE", 51 | "StatusReason": { 52 | "Description": "The pipeline is ready to ingest data." 53 | } 54 | }, 55 | "CreatedAt": 1.671055851E9, 56 | "LastUpdatedAt": 1.671055851E9, 57 | "MaxUnits": 2, 58 | "MinUnits": 8, 59 | "PipelineArn": "arn:aws:osis:us-west-2:123456789012:pipeline/another-pipeline", 60 | "PipelineName": "another-pipeline", 61 | "Status": "CREATING", 62 | "StatusReason": { 63 | "Description": "The pipeline is being created. It is not able to ingest data." 64 | } 65 | } 66 | ] 67 | } 68 | ``` 69 | 70 | To get information about a single pipeline, use the [get\-pipeline](https://docs.aws.amazon.com/cli/latest/reference/osis/get-pipeline.html) command: 71 | 72 | ``` 73 | aws osis get-pipeline --pipeline-name "my-pipeline" 74 | ``` 75 | 76 | The request returns configuration information for the specified pipeline: 77 | 78 | ``` 79 | { 80 | "Pipeline": { 81 | "PipelineName": "my-pipeline", 82 | "PipelineArn": "arn:aws:osis:us-east-1:123456789012:pipeline/my-pipeline", 83 | "MinUnits": 9, 84 | "MaxUnits": 10, 85 | "Status": "ACTIVE", 86 | "StatusReason": { 87 | "Description": "The pipeline is ready to ingest data." 88 | }, 89 | "PipelineConfigurationBody": "log-pipeline:\n source:\n http:\n processor:\n - grok:\n match:\nlog: [ '%{COMMONAPACHELOG}' ]\n - date:\n from_time_received: true\n destination: \"@timestamp\"\n sink:\n - opensearch:\n hosts: [ \"https://search-mdp-performance-test-duxkb4qnycd63rpy6svmvyvfpi.us-east-1.es.amazonaws.com\" ]\n index: \"apache_logs\"\n aws_sts_role_arn: \"arn:aws:iam::123456789012:role/my-domain-role\"\n aws_region: \"us-east-1\"\n aws_sigv4: true",, 90 | "CreatedAt": "2022-10-01T15:28:05+00:00", 91 | "LastUpdatedAt": "2022-10-21T21:41:08+00:00", 92 | "IngestEndpointUrls": [ 93 | "my-pipeline-123456789012.us-east-1.osis.amazonaws.com" 94 | ] 95 | } 96 | } 97 | ``` 98 | 99 | ## OpenSearch Ingestion API 100 | 101 | To view OpenSearch Ingestion pipelines using the OpenSearch Ingestion API, call the [ListPipelines](https://docs.aws.amazon.com/opensearch-service/latest/APIReference/API_osis_ListPipelines.html) and [GetPipeline](https://docs.aws.amazon.com/opensearch-service/latest/APIReference/API_osis_GetPipeline.html) operations\. -------------------------------------------------------------------------------- /doc_source/listening-events.md: -------------------------------------------------------------------------------- 1 | # Tutorial: Listening for Amazon OpenSearch Service EventBridge events 2 | 3 | In this tutorial, you set up a simple AWS Lambda function that listens for Amazon OpenSearch Service events and writes them to a CloudWatch Logs log stream\. 4 | 5 | ## Prerequisites 6 | 7 | This tutorial assumes that you have an existing OpenSearch Service domain\. If you haven't created a domain, follow the steps in [Creating and managing Amazon OpenSearch Service domains](createupdatedomains.md) to create one\. 8 | 9 | ## Step 1: Create the Lambda function 10 | 11 | In this procedure, you create a simple Lambda function to serve as a target for OpenSearch Service event messages\. 12 | 13 | **To create a target Lambda function** 14 | 15 | 1. Open the AWS Lambda console at [https://console\.aws\.amazon\.com/lambda/](https://console.aws.amazon.com/lambda/)\. 16 | 17 | 1. Choose **Create function ** and **Author from scratch**\. 18 | 19 | 1. For **Function name**, enter **event\-handler**\. 20 | 21 | 1. For **Runtime**, choose **Python 3\.8**\. 22 | 23 | 1. Choose **Create function**\. 24 | 25 | 1. In the **Function code** section, edit the sample code to match the following example: 26 | 27 | ``` 28 | import json 29 | 30 | def lambda_handler(event, context): 31 | if event["source"] != "aws.es": 32 | raise ValueError("Function only supports input from events with a source type of: aws.es") 33 | 34 | print(json.dumps(event)) 35 | ``` 36 | 37 | This is a simple Python 3\.8 function that prints the events sent by OpenSearch Service\. If everything is configured correctly, at the end of this tutorial, the event details appear in the CloudWatch Logs log stream that's associated with this Lambda function\. 38 | 39 | 1. Choose **Deploy**\. 40 | 41 | ## Step 2: Register an event rule 42 | 43 | In this step, you create an EventBridge rule that captures events from your OpenSearch Service domains\. This rule captures all events within the account where it's defined\. The event messages themselves contain information about the event source, including the domain from which it originated\. You can use this information to filter and sort events programmatically\. 44 | 45 | **To create an EventBridge rule** 46 | 47 | 1. Open the EventBridge console at [https://console\.aws\.amazon\.com/events/](https://console.aws.amazon.com/events/)\. 48 | 49 | 1. Choose **Create rule**\. 50 | 51 | 1. Name the rule **event\-rule**\. 52 | 53 | 1. Choose **Next**\. 54 | 55 | 1. For the event pattern, select **AWS services**, **Amazon OpenSearch Service**, and **All Events**\. This pattern applies across all of your OpenSearch Service domains and to every OpenSearch Service event\. Alternatively, you can create a more specific pattern to filter out some results\. 56 | 57 | 1. Press **Next**\. 58 | 59 | 1. For the target, choose **Lambda function**\. In the function dropdown, choose **event\-handler**\. 60 | 61 | 1. Press **Next**\. 62 | 63 | 1. Skip the tags and press **Next** again\. 64 | 65 | 1. Review the configuration and choose **Create rule**\. 66 | 67 | ## Step 3: Test your configuration 68 | 69 | The next time you receive a notification in the **Notifications** section of the OpenSearch Service console, if everything is configured properly, your Lambda function is triggered and it writes the event data to a CloudWatch Logs log stream for the function\. 70 | 71 | **To test your configuration** 72 | 73 | 1. Open the CloudWatch console at [https://console\.aws\.amazon\.com/cloudwatch/](https://console.aws.amazon.com/cloudwatch/)\. 74 | 75 | 1. On the navigation pane, choose **Logs** and select the log group for your Lambda function \(for example, **/aws/lambda/event\-handler**\)\. 76 | 77 | 1. Select a log stream to view the event data\. -------------------------------------------------------------------------------- /doc_source/managedomains-logstash.md: -------------------------------------------------------------------------------- 1 | # Loading data into Amazon OpenSearch Service with Logstash 2 | 3 | The open source version of Logstash \(Logstash OSS\) provides a convenient way to use the bulk API to upload data into your Amazon OpenSearch Service domain\. The service supports all standard Logstash input plugins, including the Amazon S3 input plugin\. OpenSearch Service supports the [logstash\-output\-opensearch](https://github.com/opensearch-project/logstash-output-opensearch) output plugin, which supports both basic authentication and IAM credentials\. The plugin works with version 8\.1 and lower of Logstash OSS\. 4 | 5 | ## Configuration 6 | 7 | Logstash configuration varies based on the type of authentication your domain uses\. 8 | 9 | No matter which authentication method you use, you must set `ecs_compatibility` to `disabled` in the output section of the configuration file\. Logstash 8\.0 introduced a breaking change where all plugins are run in [ECS compatibility mode by default](https://www.elastic.co/guide/en/logstash/current/ecs-ls.html#_specific_plugin_instance)\. You must override the default value to maintain legacy behavior\. 10 | 11 | ### Fine\-grained access control configuration 12 | 13 | If your OpenSearch Service domain uses [fine\-grained access control](fgac.md) with HTTP basic authentication, configuration is similar to any other OpenSearch cluster\. This example configuration file takes its input from the open source version of Filebeat \(Filebeat OSS\): 14 | 15 | ``` 16 | input { 17 | beats { 18 | port => 5044 19 | } 20 | } 21 | 22 | output { 23 | opensearch { 24 | hosts => "https://domain-endpoint:443" 25 | user => "my-username" 26 | password => "my-password" 27 | index => "logstash-logs-%{+YYYY.MM.dd}" 28 | ecs_compatibility => disabled 29 | ssl_certificate_verification => false 30 | } 31 | } 32 | ``` 33 | 34 | Configuration varies by Beats application and use case, but your Filebeat OSS configuration might look like this: 35 | 36 | ``` 37 | filebeat.inputs: 38 | - type: log 39 | enabled: true 40 | paths: 41 | - /path/to/logs/dir/*.log 42 | filebeat.config.modules: 43 | path: ${path.config}/modules.d/*.yml 44 | reload.enabled: false 45 | setup.ilm.enabled: false 46 | setup.ilm.check_exists: false 47 | setup.template.settings: 48 | index.number_of_shards: 1 49 | output.logstash: 50 | hosts: ["logstash-host:5044"] 51 | ``` 52 | 53 | ### IAM configuration 54 | 55 | If your domain uses an IAM\-based domain access policy or fine\-grained access control with a master user, you must sign all requests to OpenSearch Service using IAM credentials\. The following identity\-based policy grants all HTTP requests to your domain's subresources\. 56 | 57 | ``` 58 | { 59 | "Version": "2012-10-17", 60 | "Statement": [ 61 | { 62 | "Effect": "Allow", 63 | "Action": [ 64 | "es:ESHttp*" 65 | ], 66 | "Resource": "arn:aws:es:region:aws-account-id:domain/domain-name/*" 67 | } 68 | ] 69 | } 70 | ``` 71 | 72 | To set up your Logstash configuration, change your configuration file to use the plugin for its output\. This example configuration file takes its input from files in an S3 bucket: 73 | 74 | ``` 75 | input { 76 | s3 { 77 | bucket => "my-s3-bucket" 78 | region => "us-east-1" 79 | } 80 | } 81 | 82 | output { 83 | opensearch { 84 | hosts => ["domain-endpoint:443"] 85 | auth_type => { 86 | type => 'aws_iam' 87 | aws_access_key_id => 'your-access-key' 88 | aws_secret_access_key => 'your-secret-key' 89 | region => 'us-east-1' 90 | } 91 | index => "logstash-logs-%{+YYYY.MM.dd}" 92 | ecs_compatibility => disabled 93 | } 94 | } 95 | ``` 96 | 97 | If you don't want to provide your IAM credentials within the configuration file, you can export them \(or run `aws configure`\): 98 | 99 | ``` 100 | export AWS_ACCESS_KEY_ID="your-access-key" 101 | export AWS_SECRET_ACCESS_KEY="your-secret-key" 102 | export AWS_SESSION_TOKEN="your-session-token" 103 | ``` 104 | 105 | If your OpenSearch Service domain is in a VPC, the Logstash OSS machine must be able to connect to the VPC and have access to the domain through the VPC security groups\. For more information, see [About access policies on VPC domains](vpc.md#vpc-security)\. -------------------------------------------------------------------------------- /doc_source/managedomains-notifications.md: -------------------------------------------------------------------------------- 1 | # Notifications in Amazon OpenSearch Service 2 | 3 | Notifications in Amazon OpenSearch Service contain important information about the performance and health of your domains\. OpenSearch Service notifies you about service software updates, Auto\-Tune enhancements, cluster health events, and domain errors\. Notifications are available for all versions of OpenSearch and Elasticsearch OSS\. 4 | 5 | You can view notifications in the **Notifications** panel of the OpenSearch Service console\. All notifications for OpenSearch Service are also surfaced in [Amazon EventBridge](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-what-is.html)\. For a full list of notifications and sample events, see [Monitoring OpenSearch Service events with Amazon EventBridge](monitoring-events.md)\. 6 | 7 | ## Getting started with notifications 8 | 9 | Notifications are enabled automatically when you create a domain\. Go to the **Notifications** panel of the OpenSearch Service console to monitor and acknowledge notifications\. Each notification includes information such as the time it was posted, the domain it relates to, a severity and status level, and a brief explanation\. You can view historical notifications for up to 90 days in the console\. 10 | 11 | After accessing the **Notifications** panel or acknowledging a notification, you might receive an error message about not having permissions to perform `es:ListNotifications` or `es:UpdateNotificationStatus`\. To resolve this problem, give your user or role the following permissions in IAM: 12 | 13 | ``` 14 | { 15 | "Version": "2012-10-17", 16 | "Statement": [{ 17 | "Effect": "Allow", 18 | "Action": [ 19 | "es:UpdateNotificationStatus", 20 | "es:ListNotifications" 21 | ], 22 | "Resource": "arn:aws:es:*:123456789012:domain/*" 23 | }] 24 | } 25 | ``` 26 | 27 | The IAM console throws an error \("IAM does not recognize one or more actions\."\) that you can safely ignore\. You can also restrict the `es:UpdateNotificationStatus` action to certain domains\. To learn more, see [Policy element reference](ac.md#ac-reference)\. 28 | 29 | ## Notification severities 30 | 31 | Notifications in OpenSearch Service can be *informational*, which relate to any action you've already taken or the operations of your domain, or *actionable*, which require you to take specific actions such as applying a mandatory security patch\. Each notification has a severity associated with it, which can be `Informational`, `Low`, `Medium`, `High`, or `Critical`\. The following table summarizes each severity: 32 | 33 | 34 | | Severity | Description | Examples | 35 | | --- | --- | --- | 36 | | Informational | Information related to the operation of your domain\. | [\[See the AWS documentation website for more details\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-notifications.html) | 37 | | Low | A recommended action, but has no adverse impact on domain availability or performance if no action is taken\. | [\[See the AWS documentation website for more details\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-notifications.html) | 38 | | Medium | There might be an impact if the recommended action is not taken, but comes with an extended time window for the action to be taken\. | [\[See the AWS documentation website for more details\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-notifications.html) | 39 | | High | Urgent action is required to avoid adverse impact\. | [\[See the AWS documentation website for more details\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-notifications.html) | 40 | | Critical | Immediate action is required to avoid adverse impact, or to recover from it\. | None currently available | 41 | 42 | ## Sample EventBridge event 43 | 44 | The following example shows an OpenSearch Service notification event sent to Amazon EventBridge\. The notification has a severity of `Informational` because the update is optional: 45 | 46 | ``` 47 | { 48 | "version": "0", 49 | "id": "01234567-0123-0123-0123-012345678901", 50 | "detail-type": "Amazon OpenSearch Service Software Update Notification", 51 | "source": "aws.es", 52 | "account": "123456789012", 53 | "time": "2016-11-01T13:12:22Z", 54 | "region": "us-east-1", 55 | "resources": ["arn:aws:es:us-east-1:123456789012:domain/test-domain"], 56 | "detail": { 57 | "event": "Service Software Update", 58 | "status": "Available", 59 | "severity": "Informational", 60 | "description": "Service software update [R20200330-p1] available." 61 | } 62 | } 63 | ``` -------------------------------------------------------------------------------- /doc_source/managing-indices.md: -------------------------------------------------------------------------------- 1 | # Managing indexes in Amazon OpenSearch Service 2 | 3 | After you add data to Amazon OpenSearch Service, you often need to reindex that data, work with index aliases, move an index to more cost\-effective storage, or delete it altogether\. This chapter covers UltraWarm storage, cold storage, and Index State Management\. For information on the OpenSearch index APIs, see the [OpenSearch documentation](https://opensearch.org/docs/opensearch/reindex-data/)\. 4 | 5 | **Topics** 6 | + [UltraWarm storage for Amazon OpenSearch Service](ultrawarm.md) 7 | + [Cold storage for Amazon OpenSearch Service](cold-storage.md) 8 | + [Index State Management in Amazon OpenSearch Service](ism.md) 9 | + [Summarizing indexes in Amazon OpenSearch Service with index rollups](rollup.md) 10 | + [Transforming indexes in Amazon OpenSearch Service](transforms.md) 11 | + [Cross\-cluster replication for Amazon OpenSearch Service](replication.md) 12 | + [Migrating Amazon OpenSearch Service indexes using remote reindex](remote-reindex.md) 13 | + [Managing time\-series data in Amazon OpenSearch Service with data streams](data-streams.md) -------------------------------------------------------------------------------- /doc_source/monitoring-data.md: -------------------------------------------------------------------------------- 1 | # Monitoring data in Amazon OpenSearch Service 2 | 3 | Proactively monitor your data in Amazon OpenSearch Service with alerting and anomaly detection\. Set up alerts to receive notifications when your data exceeds certain thresholds\. Anomaly detection uses machine learning to automatically detect any outliers in your streaming data\. You can pair anomaly detection with alerting to ensure you're notified as soon as an anomaly is detected\. 4 | 5 | **Topics** 6 | + [Configuring alerts in Amazon OpenSearch Service](alerting.md) 7 | + [Anomaly detection in Amazon OpenSearch Service](ad.md) -------------------------------------------------------------------------------- /doc_source/monitoring-pipeline-logs.md: -------------------------------------------------------------------------------- 1 | # Monitoring pipeline logs 2 | 3 | You can enable logging for Amazon OpenSearch Ingestion pipelines to expose error and warning messages raised during pipeline operations and ingestion activity\. OpenSearch Ingestion publishes all logs to *Amazon CloudWatch Logs*\. CloudWatch Logs can monitor information in the log files and notify you when certain thresholds are met\. You can also archive your log data in highly durable storage\. For more information, see the [Amazon CloudWatch Logs User Guide](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/)\. 4 | 5 | Logs from OpenSearch Ingestion might indicate failed processing of requests, authentication errors from the source to the sink, and other warnings that can be helpful for troubleshooting\. For its logs, OpenSearch Ingestion uses the log levels of `INFO`, `WARN`, `ERROR`, and `FATAL`\. We recommend enabling log publishing for all pipelines\. 6 | 7 | ## Permissions required 8 | 9 | In order to enable OpenSearch Ingestion to send logs to CloudWatch Logs, you must be signed in as a user that has certain IAM permissions\. 10 | 11 | You need the following CloudWatch Logs permissions in order to create and update log delivery resources: 12 | 13 | ``` 14 | { 15 | "Version": "2012-10-17", 16 | "Statement": [ 17 | { 18 | "Effect": "Allow", 19 | "Resource": "*", 20 | "Action": [ 21 | "logs:CreateLogDelivery", 22 | "logs:PutResourcePolicy", 23 | "logs:UpdateLogDelivery", 24 | "logs:DeleteLogDelivery", 25 | "logs:DescribeResourcePolicies", 26 | "logs:GetLogDelivery", 27 | "logs:ListLogDeliveries" 28 | ] 29 | } 30 | ] 31 | } 32 | ``` 33 | 34 | ## Enabling log publishing 35 | 36 | You can enable log publishing on existing pipelines, or while creating a pipeline\. For steps to enable log publishing during pipeline creation, see [Creating pipelines](creating-pipeline.md#create-pipeline)\. 37 | 38 | ### Console 39 | 40 | **To enable log publishing on an existing pipeline** 41 | 42 | 1. Sign in to the Amazon OpenSearch Service console at [https://console\.aws\.amazon\.com/aos/home](https://console.aws.amazon.com/aos/home)\. 43 | 44 | 1. Choose **Ingestion** in the left navigation pane and select the pipeline that you want to enable logs for\. 45 | 46 | 1. Choose **Edit log publishing options**\. 47 | 48 | 1. Select **Publish to CloudWatch Logs**\. 49 | 50 | 1. Either create a new log group or select an existing one\. We recommend that you format the name as a path, such as `/aws/vendedlogs/OpenSearchIngestion/pipeline-name/audit-logs`\. This format makes it easier to apply a CloudWatch access policy that grants permissions to all log groups under a specific path such as `/aws/vendedlogs/OpenSearchService/OpenSearchIngestion`\. 51 | **Important** 52 | You must include the prefix `vendedlogs` in the log group name, otherwise creation fails\. 53 | 54 | 1. Choose **Save**\. 55 | 56 | ### CLI 57 | 58 | To enable log publishing using the AWS CLI, send the following request: 59 | 60 | ``` 61 | aws osis update-pipeline \ 62 | --pipeline-name my-pipeline \ 63 | --log-publishing-options IsLoggingEnabled=true,CloudWatchLogDestination={LogGroup="/aws/vendedlogs/OpenSearchIngestion/pipeline-name"} 64 | ``` -------------------------------------------------------------------------------- /doc_source/monitoring-pipelines.md: -------------------------------------------------------------------------------- 1 | # Logging and monitoring Amazon OpenSearch Ingestion with Amazon CloudWatch 2 | 3 | Amazon OpenSearch Ingestion publishes metrics and logs to Amazon CloudWatch\. 4 | 5 | **Topics** 6 | + [Monitoring pipeline logs](monitoring-pipeline-logs.md) 7 | + [Monitoring pipeline metrics](monitoring-pipeline-metrics.md) -------------------------------------------------------------------------------- /doc_source/monitoring.md: -------------------------------------------------------------------------------- 1 | # Monitoring Amazon OpenSearch Service domains 2 | 3 | Monitoring is an important part of maintaining the reliability, availability, and performance of Amazon OpenSearch Service and your other AWS solutions\. AWS provides the following tools to monitor your OpenSearch Service resources, report issues, and take automatic actions when appropriate: 4 | 5 | **Amazon CloudWatch** 6 | Amazon CloudWatch monitors your OpenSearch Service resources in real time\. You can collect and track metrics, create customized dashboards, and set alarms that notify you or take actions when a metric reaches a certain threshold\. For more information, see the [Amazon CloudWatch User Guide](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/)\. 7 | 8 | **Amazon CloudWatch Logs** 9 | Amazon CloudWatch Logs lets you monitor, store, and access your OpenSearch log files\. CloudWatch Logs monitors the information in log files and can notify you when certain thresholds are met\. For more information, see the [Amazon CloudWatch Logs User Guide](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/)\. 10 | 11 | **Amazon EventBridge** 12 | Amazon EventBridge delivers a near real\-time stream of system events that describe changes in your OpenSearch Service domains\. You can create rules that watch for certain events, and trigger automated actions in other AWS services when these events occur\. For more information, see the [Amazon EventBridge User Guide](https://docs.aws.amazon.com/eventbridge/latest/userguide/)\. 13 | 14 | **AWS CloudTrail** 15 | AWS CloudTrail captures configuration API calls made to OpenSearch Service as events\. It can deliver these events to an Amazon S3 bucket that you specify\. Using this information, you can identify which users and accounts made requests, the source IP address from which the requests were made, and when the requests occurred\. For more information, see the [AWS CloudTrail User Guide](https://docs.aws.amazon.com/awscloudtrail/latest/userguide/)\. 16 | 17 | **Topics** 18 | + [Monitoring OpenSearch cluster metrics with Amazon CloudWatch](managedomains-cloudwatchmetrics.md) 19 | + [Monitoring OpenSearch logs with Amazon CloudWatch Logs](createdomain-configure-slow-logs.md) 20 | + [Monitoring audit logs in Amazon OpenSearch Service](audit-logs.md) 21 | + [Monitoring OpenSearch Service events with Amazon EventBridge](monitoring-events.md) 22 | + [Monitoring Amazon OpenSearch Service API calls with AWS CloudTrail](managedomains-cloudtrailauditing.md) -------------------------------------------------------------------------------- /doc_source/ntn.md: -------------------------------------------------------------------------------- 1 | # Node\-to\-node encryption for Amazon OpenSearch Service 2 | 3 | Node\-to\-node encryption provides an additional layer of security on top of the default features of Amazon OpenSearch Service\. 4 | 5 | Each OpenSearch Service domain—regardless of whether the domain uses VPC access—resides within its own, dedicated VPC\. This architecture prevents potential attackers from intercepting traffic between OpenSearch nodes and keeps the cluster secure\. By default, however, traffic within the VPC is unencrypted\. Node\-to\-node encryption enables TLS 1\.2 encryption for all communications within the VPC\. 6 | 7 | If you send data to OpenSearch Service over HTTPS, node\-to\-node encryption helps ensure that your data remains encrypted as OpenSearch distributes \(and redistributes\) it throughout the cluster\. If data arrives unencrypted over HTTP, OpenSearch Service encrypts it after it reaches the cluster\. You can require that all traffic to the domain arrive over HTTPS using the console, AWS CLI, or configuration API\. 8 | 9 | Node\-to\-node encryption is *required* if you enable [fine\-grained access control](fgac.md)\. 10 | 11 | ## Enabling node\-to\-node encryption 12 | 13 | Node\-to\-node encryption on new domains requires any version of OpenSearch, or Elasticsearch 6\.0 or later\. Enabling node\-to\-node encryption on existing domains requires any version of OpenSearch, or Elasticsearch 6\.7 or later\. Choose the existing domain in the AWS console, **Actions**, and **Edit security configuration**\. 14 | 15 | Alternatively, you can use the AWS CLI or configuration API\. For more information, see the [AWS CLI Command Reference](https://docs.aws.amazon.com/cli/latest/reference/) and [OpenSearch Service API reference](https://docs.aws.amazon.com/opensearch-service/latest/APIReference/API_Welcome.html)\. 16 | 17 | ## Disabling node\-to\-node encryption 18 | 19 | After you configure a domain to use node\-to\-node encryption, you can't disable the setting\. Instead, you can take a [manual snapshot](managedomains-snapshots.md) of the encrypted domain, [create another domain](createupdatedomains.md#createdomains), migrate your data, and delete the old domain\. -------------------------------------------------------------------------------- /doc_source/observability.md: -------------------------------------------------------------------------------- 1 | # Observability in Amazon OpenSearch Service 2 | 3 | The default installation of OpenSearch Dashboards for Amazon OpenSearch Service includes the Observability plugin, which you can use to visualize data\-driven events using Piped Processing Language \(PPL\) in order to explore, discover, and query data stored in OpenSearch\. The plugin requires OpenSearch 1\.2 or later\. 4 | 5 | The Observability plugin provides a unified experience for collecting and monitoring metrics, logs, and traces from common data sources\. Data collection and monitoring in one place enables full\-stack, end\-to\-end observability of your entire infrastructure\. Full documentation for the Observability plugin is in the [OpenSearch documentation](https://opensearch.org/docs/latest/observability-plugin/index/)\. 6 | 7 | Everyone's process for exploring data is different\. If you’re new to exploring data and creating visualizations, we recommend trying a workflow like the following: 8 | 9 | ## Explore your data with event analytics 10 | 11 | With event analytics, you can use [Piped Processing Language](https://opensearch.org/docs/latest/observability-plugin/ppl/index) \(PPL\) to build and view different visualizations of your data\. PPL is a query language that lets you use pipe \(`|`\) syntax to explore, discover, and query data stored in OpenSearch\. The PPL editor includes auto\-complete functionality that suggests possible commands and fields for your query\. 12 | 13 | To start, let's say that you're collecting flight data in your OpenSearch Service domain and you want to find out which airline had the most flights arriving in Pittsburgh International Airport last month\. You write the following PPL query: 14 | 15 | ``` 16 | source=opensearch_dashboards_sample_data_flights | 17 | stats count() by Dest, Carrier | 18 | where Dest = "Pittsburgh International Airport" 19 | ``` 20 | 21 | This query pulls data from the index named `opensearch_dashboards_sample_data_flights`\. It then uses the `stats` command to get a total count of flights and groups it according to destination airport and carrier\. Finally, it uses the `where` clause to filter the results to flights arriving in Pittsburgh International Airport\. 22 | 23 | Here's what the data looks like when displayed over the last month: 24 | 25 | ![\[Image NOT FOUND\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/images/o11yflights-pittsburgh.png) 26 | 27 | You can choose the **PPL** button in the query editor to get usage information and examples for each PPL command: 28 | 29 | ![\[Image NOT FOUND\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/images/ppl-ref.png) 30 | 31 | Let's look at a more complex example, which queries for information about flight delays: 32 | 33 | ``` 34 | source=opensearch_dashboards_sample_data_flights | 35 | where FlightDelayMin > 0 | 36 | stats sum(FlightDelayMin) as minimum_delay, count() as total_delayed by Carrier, Dest | 37 | eval avg_delay=minimum_delay / total_delayed | 38 | sort - avg_delay 39 | ``` 40 | 41 | Each command in the query impacts the final output: 42 | + `source=opensearch_dashboards_sample_data_flights` \- pulls data from the same index as the previous example 43 | + `where FlightDelayMin > 0` \- filters the data to flights that were delayed 44 | + `stats sum(FlightDelayMin) as minimum_delay, count() as total_delayed by Carrier` \- for each carrier, gets the total minimum delay time and the total count of delayed flights 45 | + `eval avg_delay=minimum_delay / total_delayed` \- calculates the average delay time for each carrier by dividing the minimum delay time by the total number of delayed flights 46 | + `sort - avg_delay` \- sorts the results by average delay in descending order 47 | 48 | With this query, you can determine that OpenSearch Dashboards Airlines has, on average, fewer delays\. 49 | 50 | ![\[Image NOT FOUND\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/images/o11y-delays.png) 51 | 52 | You can find more sample PPL queries under **Queries and Visualizations** on the **Event analytics** page\. 53 | 54 | ## Create visualizations 55 | 56 | Once you correctly query the data that you're interested in, you can save those queries as visualizations: 57 | 58 | ![\[Image NOT FOUND\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/images/flights-viz.png) 59 | 60 | Then add those visualizations to [operational panels](https://opensearch.org/docs/latest/observability-plugin/operational-panels) to compare different pieces of data\. Leverage [notebooks](https://opensearch.org/docs/latest/observability-plugin/notebooks) to combine different visualizations and code blocks that you can share with team members\. 61 | 62 | ## Dive deeper with Trace Analytics 63 | 64 | [Trace Analytics](trace-analytics.md) provides a way to visualize the flow of events in your OpenSearch data to identify and fix performance problems in distributed applications\. 65 | 66 | ![\[Image NOT FOUND\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/images/ta-dashboards-trace.png) -------------------------------------------------------------------------------- /doc_source/osis-getting-started-tutorials.md: -------------------------------------------------------------------------------- 1 | # Getting started with Amazon OpenSearch Ingestion 2 | 3 | Amazon OpenSearch Ingestion supports ingesting data into managed OpenSearch Service domains and OpenSearch Serverless collections\. The following tutorials walk you through the basic steps to get a pipeline up and running for each of these use cases\. 4 | 5 | **Note** 6 | Pipeline creation will fail if you don't set up the correct permissions\. See [Setting up roles and users in Amazon OpenSearch Ingestion](pipeline-security-overview.md) for a better understanding of the required roles before you create a pipeline\. 7 | 8 | **Topics** 9 | + [Tutorial: Ingesting data into a domain using Amazon OpenSearch Ingestion](osis-get-started.md) 10 | + [Tutorial: Ingesting data into a collection using Amazon OpenSearch Ingestion](osis-serverless-get-started.md) -------------------------------------------------------------------------------- /doc_source/petabyte-scale.md: -------------------------------------------------------------------------------- 1 | # Petabyte scale in Amazon OpenSearch Service 2 | 3 | Amazon OpenSearch Service domains offer attached storage of up to 3 PB\. You can configure a domain with 200 `i3.16xlarge.search` instance types, each with 15 TB of storage\. Because of the sheer difference in scale, recommendations for domains of this size differ from [our general recommendations](bp.md)\. This section discusses considerations for creating domains, costs, storage, and shard size\. 4 | 5 | While this section frequently references the `i3.16xlarge.search` instance types, you can use several other instance types to reach 1 PB of total domain storage\. 6 | 7 | **Creating domains** 8 | Domains of this size exceed the default limit of 40 instances per domain\. To request a service limit increase of up to 200 instances per domain, open a case at the [AWS Support Center](https://console.aws.amazon.com/support/home#/)\. 9 | 10 | **Pricing** 11 | Before creating a domain of this size, check the [Amazon OpenSearch Service pricing](https://aws.amazon.com/opensearch-service/pricing/) page to ensure that the associated costs match your expectations\. Examine [UltraWarm storage for Amazon OpenSearch Service](ultrawarm.md) to see if a hot\-warm architecture fits your use case\. 12 | 13 | **Storage** 14 | The `i3` instance types are designed to provide fast, local non\-volatile memory express \(NVMe\) storage\. Because this local storage tends to offer performance benefits when compared to Amazon Elastic Block Store, EBS volumes are not an option when you select these instance types in OpenSearch Service\. If you prefer EBS storage, use another instance type, such as `r6.12xlarge.search`\. 15 | 16 | **Shard size and count** 17 | A common OpenSearch guideline is not to exceed 50 GB per shard\. Given the number of shards necessary to accommodate large domains and the resources available to `i3.16xlarge.search` instances, we recommend a shard size of 100 GB\. 18 | For example, if you have 450 TB of source data and want one replica, your *minimum* storage requirement is closer to 450 TB \* 2 \* 1\.1 / 0\.95 = 1\.04 PB\. For an explanation of this calculation, see [Calculating storage requirements](sizing-domains.md#bp-storage)\. Although 1\.04 PB / 15 TB = 70 instances, you might select 90 or more `i3.16xlarge.search` instances to give yourself a storage safety net, deal with node failures, and account for some variance in the amount of data over time\. Each instance adds another 20 GiB to your minimum storage requirement, but for disks of this size, those 20 GiB are almost negligible\. 19 | Controlling the number of shards is tricky\. OpenSearch users often rotate indexes on a daily basis and retain data for a week or two\. In this situation, you might find it useful to distinguish between "active" and "inactive" shards\. Active shards are, well, actively being written to or read from\. Inactive shards might service some read requests, but are largely idle\. In general, you should keep the number of active shards below a few thousand\. As the number of active shards approaches 10,000, considerable performance and stability risks emerge\. 20 | To calculate the number of primary shards, use this formula: 450,000 GB \* 1\.1 / 100 GB per shard = 4,950 shards\. Doubling that number to account for replicas is 9,900 shards, which represents a major concern if all shards are active\. But if you rotate indexes and only 1/7th or 1/14th of the shards are active on any given day \(1,414 or 707 shards, respectively\), the cluster might work well\. As always, the most important step of sizing and configuring your domain is to perform representative client testing using a realistic dataset\. -------------------------------------------------------------------------------- /doc_source/pipeline--stop-start.md: -------------------------------------------------------------------------------- 1 | # Stopping and starting Amazon OpenSearch Ingestion pipelines 2 | 3 | Stopping and starting Amazon OpenSearch Ingestion pipelines helps you manage costs for development and test environments\. You can temporarily stop a pipeline instead of setting it up and tearing it down each time that you use the pipeline\. 4 | 5 | **Topics** 6 | + [Overview of stopping and starting an OpenSearch Ingestion pipeline](#pipeline--start-stop-overview) 7 | + [Stopping an OpenSearch Ingestion pipeline](#pipeline--stop) 8 | + [Starting an OpenSearch Ingestion pipeline](#pipeline--start) 9 | 10 | ## Overview of stopping and starting an OpenSearch Ingestion pipeline 11 | 12 | You can stop a pipeline during periods where you don't need to ingest data into it\. You can start the pipeline again anytime you need to use it\. Starting and stopping simplifies the setup and teardown processes for pipelines used for development, testing, or similar activities that don't require continuous availability\. 13 | 14 | While your pipeline is stopped, you aren't charged for any Ingestion OCU hours\. You can still update stopped pipelines, and they receive automatic minor version updates and security patches\. 15 | 16 | Don't use starting and stopping if you need to keep your pipeline running but it has more capacity than you need\. If your pipeline is too costly or not very busy, consider reducing its maximum capacity limits\. For more information, see [Scaling pipelines](ingestion.md#ingestion-scaling)\. 17 | 18 | ## Stopping an OpenSearch Ingestion pipeline 19 | 20 | To use an OpenSearch Ingestion pipeline or perform administration, you always begin with an active pipeline, then stop the pipeline, and then start the pipeline again\. While your pipeline is stopped, you're not charged for Ingestion OCU hours\. 21 | 22 | ### Console 23 | 24 | **To stop a pipeline** 25 | 26 | 1. Sign in to the Amazon OpenSearch Service console at [https://console\.aws\.amazon\.com/aos/home](https://console.aws.amazon.com/aos/home)\. 27 | 28 | 1. In the navigation pane, choose **Pipelines**, and then choose a pipeline\. You can perform the stop operation from this page, or navigate to the details page for the pipeline that you want to stop\. 29 | 30 | 1. For **Actions**, choose **Stop pipeline**\. 31 | 32 | If a pipeline can't be stopped and started, the **Stop pipeline** action isn't available\. 33 | 34 | ### AWS CLI 35 | 36 | To stop a pipeline using the AWS CLI, call the [stop\-pipeline](https://docs.aws.amazon.com/cli/latest/reference/osis/stop-pipeline.html) command with the following parameters: 37 | + `--pipeline-name` – the name of the pipeline\. 38 | 39 | **Example** 40 | 41 | ``` 42 | aws osis stop-pipeline --pipeline-name my-pipeline 43 | ``` 44 | 45 | ### OpenSearch Ingestion API 46 | 47 | To stop a pipeline using the OpenSearch Ingestion API, call the [StopPipeline](https://docs.aws.amazon.com/opensearch-service/latest/APIReference/API_osis_StopPipeline.html) operation with the following parameter: 48 | + `PipelineName` – the name of the pipeline\. 49 | 50 | ## Starting an OpenSearch Ingestion pipeline 51 | 52 | You always start an OpenSearch Ingestion pipeline beginning with a pipeline that's already in the stopped state\. The pipeline keeps its configuration settings such as capacity limits, network settings, and log publishing options\. 53 | 54 | Restarting a pipeline usually takes several minutes\. 55 | 56 | ### Console 57 | 58 | **To start a pipeline** 59 | 60 | 1. Sign in to the Amazon OpenSearch Service console at [https://console\.aws\.amazon\.com/aos/home](https://console.aws.amazon.com/aos/home)\. 61 | 62 | 1. In the navigation pane, choose **Pipelines**, and then choose a pipeline\. You can perform the start operation from this page, or navigate to the details page for the pipeline that you want to start\. 63 | 64 | 1. For **Actions**, choose **Start pipeline**\. 65 | 66 | ### AWS CLI 67 | 68 | To start a pipeline by using the AWS CLI, call the [start\-pipeline](https://docs.aws.amazon.com/cli/latest/reference/osis/start-pipeline.html) command with the following parameters: 69 | + `--pipeline-name` – the name of the pipeline\. 70 | 71 | **Example** 72 | 73 | ``` 74 | aws osis start-pipeline --pipeline-name my-pipeline 75 | ``` 76 | 77 | ### OpenSearch Ingestion API 78 | 79 | To start an OpenSearch Ingestion pipeline using the OpenSearch Ingestion API, call the [StartPipeline](https://docs.aws.amazon.com/opensearch-service/latest/APIReference/API_osis_StartPipeline.html) operation with the following parameter: 80 | + `PipelineName` – the name of the pipeline\. -------------------------------------------------------------------------------- /doc_source/pipeline-security-model.md: -------------------------------------------------------------------------------- 1 | # Security in Amazon OpenSearch Ingestion 2 | 3 | Cloud security at AWS is the highest priority\. As an AWS customer, you benefit from a data center and network architecture that is built to meet the requirements of the most security\-sensitive organizations\. 4 | 5 | Security is a shared responsibility between AWS and you\. The [shared responsibility model](http://aws.amazon.com/compliance/shared-responsibility-model/) describes this as security *of* the cloud and security *in* the cloud: 6 | + **Security of the cloud** – AWS is responsible for protecting the infrastructure that runs AWS services in the AWS Cloud\. AWS also provides you with services that you can use securely\. Third\-party auditors regularly test and verify the effectiveness of our security as part of the [AWS compliance programs](http://aws.amazon.com/compliance/programs/)\. 7 | + **Security in the cloud** – Your responsibility is determined by the AWS service that you use\. You are also responsible for other factors including the sensitivity of your data, your company’s requirements, and applicable laws and regulations\. 8 | 9 | This documentation helps you understand how to apply the shared responsibility model when using OpenSearch Ingestion\. The following topics show you how to configure OpenSearch Ingestion to meet your security and compliance objectives\. You also learn how to use other AWS services that help you to monitor and secure your OpenSearch Ingestion resources\. 10 | 11 | **Topics** 12 | + [Securing Amazon OpenSearch Ingestion pipelines within a VPC](pipeline-security.md) 13 | + [Identity and Access Management for Amazon OpenSearch Ingestion](security-iam-ingestion.md) 14 | + [Logging Amazon OpenSearch Ingestion API calls using AWS CloudTrail](osis-logging-using-cloudtrail.md) -------------------------------------------------------------------------------- /doc_source/ppl-support.md: -------------------------------------------------------------------------------- 1 | # Querying Amazon OpenSearch Service data using Piped Processing Language 2 | 3 | Piped Processing Language \(PPL\) is a query language that lets you use pipe \(`|`\) syntax to query data stored in Amazon OpenSearch Service\. 4 | 5 | The PPL syntax consists of commands delimited by a pipe character \(`|`\) where data flows from left to right through each pipeline\. For example, the PPL syntax to find the number of hosts with HTTP 403 or 503 errors, aggregate them per host, and sort them in the order of impact is as follows: 6 | 7 | ``` 8 | source = dashboards_sample_data_logs | where response='403' or response='503' | stats count(request) as request_count by host, response | sort -request_count 9 | ``` 10 | 11 | PPL requires either OpenSearch or Elasticsearch 7\.9 or later\. Detailed steps and command descriptions are available in the [OpenSearch PPL reference manual](https://github.com/opensearch-project/sql/blob/2.x/docs/user/ppl/index.rst)\. 12 | 13 | ## 14 | 15 | To get started, choose **Query Workbench** in OpenSearch Dashboards and select **PPL**\. Use the `bulk` operation to index some sample data: 16 | 17 | ``` 18 | PUT accounts/_bulk?refresh 19 | {"index":{"_id":"1"}} 20 | {"account_number":1,"balance":39225,"firstname":"Amber","lastname":"Duke","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL"} 21 | {"index":{"_id":"6"}} 22 | {"account_number":6,"balance":5686,"firstname":"Hattie","lastname":"Bond","age":36,"gender":"M","address":"671 Bristol Street","employer":"Netagy","email":"hattiebond@netagy.com","city":"Dante","state":"TN"} 23 | {"index":{"_id":"13"}} 24 | {"account_number":13,"balance":32838,"firstname":"Nanette","lastname":"Bates","age":28,"gender":"F","address":"789 Mady Street","employer":"Quility","city":"Nogal","state":"VA"} 25 | {"index":{"_id":"18"}} 26 | {"account_number":18,"balance":4180,"firstname":"Dale","lastname":"Adams","age":33,"gender":"M","address":"467 Hutchinson Court","email":"daleadams@boink.com","city":"Orick","state":"MD"} 27 | ``` 28 | 29 | The following example returns `firstname` and `lastname` fields for documents in an accounts index with `age` greater than 18: 30 | 31 | ``` 32 | search source=accounts | where age > 18 | fields firstname, lastname 33 | ``` 34 | 35 | 36 | **Sample Response** 37 | 38 | | id | firstname | lastname | 39 | | --- | --- | --- | 40 | | 0 | Amber | Duke | 41 | | 1 | Hattie | Bond | 42 | | 2 | Nanette | Bates | 43 | | 3 | Dale | Adams | 44 | 45 | You can use a complete set of read\-only commands like `search`, `where`, `fields`, `rename`, `dedup`, `stats`, `sort`, `eval`, `head`, `top`, and `rare`\. For descriptions and examples of each command, see the [OpenSearch PPL reference manual](https://github.com/opensearch-project/sql/blob/2.x/docs/user/ppl/index.rst)\. 46 | 47 | The PPL plugin supports all SQL functions, including mathematical, trigonometric, date\-time, string, aggregate, and advanced operators and expressions\. To learn more, see the [OpenSearch PPL reference manual](https://github.com/opensearch-project/sql/blob/2.x/docs/user/ppl/index.rst)\. -------------------------------------------------------------------------------- /doc_source/ppl.md: -------------------------------------------------------------------------------- 1 | # Piped Processing Language 2 | 3 | Piped Processing Language \(PPL\) is a query language that lets you use pipe \(`|`\) syntax to query data stored in Amazon Elasticsearch Service\. 4 | 5 | The PPL syntax consists of commands delimited by a pipe character \(`|`\) where data flows from left to right through each pipeline\. For example, the PPL syntax to find the number of hosts with HTTP 403 or 503 errors, aggregate them per host, and sort them in the order of impact is as follows: 6 | 7 | ``` 8 | source = kibana_sample_data_logs | 9 | where response='403' or response='503' | 10 | stats count(request) as request_count by host, response | 11 | sort -request_count 12 | ``` 13 | 14 | PPL requires Elasticsearch 7\.9 or later\. Full documentation for the feature, including detailed steps and command descriptions, is available in the [Open Distro for Elasticsearch documentation](https://opendistro.github.io/for-elasticsearch-docs/docs/ppl/)\. 15 | 16 | ## 17 | 18 | To get started, choose **Query Workbench** in Kibana and select **PPL**\. Use the `bulk` operation to index some sample data: 19 | 20 | ``` 21 | PUT accounts/_bulk?refresh 22 | {"index":{"_id":"1"}} 23 | {"account_number":1,"balance":39225,"firstname":"Amber","lastname":"Duke","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL"} 24 | {"index":{"_id":"6"}} 25 | {"account_number":6,"balance":5686,"firstname":"Hattie","lastname":"Bond","age":36,"gender":"M","address":"671 Bristol Street","employer":"Netagy","email":"hattiebond@netagy.com","city":"Dante","state":"TN"} 26 | {"index":{"_id":"13"}} 27 | {"account_number":13,"balance":32838,"firstname":"Nanette","lastname":"Bates","age":28,"gender":"F","address":"789 Mady Street","employer":"Quility","city":"Nogal","state":"VA"} 28 | {"index":{"_id":"18"}} 29 | {"account_number":18,"balance":4180,"firstname":"Dale","lastname":"Adams","age":33,"gender":"M","address":"467 Hutchinson Court","email":"daleadams@boink.com","city":"Orick","state":"MD"} 30 | ``` 31 | 32 | The following example returns `firstname` and `lastname` fields for documents in an accounts index with `age` greater than 18: 33 | 34 | ``` 35 | search source=accounts | 36 | where age > 18 | 37 | fields firstname, lastname 38 | ``` 39 | 40 | 41 | **Sample Response** 42 | 43 | | id | firstname | lastname | 44 | | --- | --- | --- | 45 | | 0 | Amber | Duke | 46 | | 1 | Hattie | Bond | 47 | | 2 | Nanette | Bates | 48 | | 3 | Dale | Adams | 49 | 50 | You can use a complete set of read\-only commands like `search`, `where`, `fields`, `rename`, `dedup`, `stats`, `sort`, `eval`, `head`, `top`, and `rare`\. For descriptions and examples of each command, see [Commands](https://opendistro.github.io/for-elasticsearch-docs/docs/ppl/commands/)\. 51 | 52 | The PPL plugin supports all SQL functions, including mathematical, trigonometric, date\-time, string, aggregate, and advanced operators and expressions\. To learn more, seee [SQL Functions](https://opendistro.github.io/for-elasticsearch-docs/docs/sql/functions/)\. -------------------------------------------------------------------------------- /doc_source/rollup.md: -------------------------------------------------------------------------------- 1 | # Summarizing indexes in Amazon OpenSearch Service with index rollups 2 | 3 | Index rollups in Amazon OpenSearch Service let you reduce storage costs by periodically rolling up old data into summarized indices\. 4 | 5 | You pick the fields that interest you and use an index rollup to create a new index with only those fields aggregated into coarser time buckets\. You can store months or years of historical data at a fraction of the cost with the same query performance\. 6 | 7 | Index rollups requires OpenSearch or Elasticsearch 7\.9 or later\. Full documentation for the feature is available in the [OpenSearch documentation](https://opensearch.org/docs/im-plugin/index-rollups/)\. 8 | 9 | ## Creating an index rollup job 10 | 11 | To get started, choose **Index Management** in OpenSearch Dashboards\. Select **Rollup Jobs** and choose **Create rollup job**\. 12 | 13 | ### Step 1: Set up indices 14 | 15 | Set up the source and target indices\. The source index is the one that you want to roll up\. The target index is where the index rollup results are saved\. 16 | 17 | After you create an index rollup job, you can’t change your index selections\. 18 | 19 | ### Step 2: Define aggregations and metrics 20 | 21 | Select the attributes with the aggregations \(terms and histograms\) and metrics \(avg, sum, max, min, and value count\) that you want to roll up\. Make sure you don’t add a lot of highly granular attributes, because you won’t save much space\. 22 | 23 | ### Step 3: Specify schedules 24 | 25 | Specify a schedule to roll up your indexes as it’s being ingested\. The index rollup job is enabled by default\. 26 | 27 | ### Step 4: Review and create 28 | 29 | Review your configuration and select **Create**\. 30 | 31 | ### Step 5: Search the target index 32 | 33 | You can use the standard `_search` API to search the target index\. You can’t access the internal structure of the data in the target index because the plugin automatically rewrites the query in the background to suit the target index\. This is to make sure you can use the same query for the source and target index\. 34 | 35 | To query the target index, set `size` to 0: 36 | 37 | ``` 38 | GET target_index/_search 39 | { 40 | "size": 0, 41 | "query": { 42 | "match_all": {} 43 | }, 44 | "aggs": { 45 | "avg_cpu": { 46 | "avg": { 47 | "field": "cpu_usage" 48 | } 49 | } 50 | } 51 | } 52 | ``` 53 | 54 | **Note** 55 | OpenSearch versions 2\.2 and later support searching multiple rollup indexes in one request\. OpenSearch versions prior to 2\.2 and legacy Elasticsearch OSS versions only support one rollup index per search\. -------------------------------------------------------------------------------- /doc_source/samplecode.md: -------------------------------------------------------------------------------- 1 | # Sample code for Amazon OpenSearch Service 2 | 3 | This chapter contains common sample code for working with Amazon OpenSearch Service: HTTP request signing in a variety of programming languages, compressing HTTP request bodies, and using the AWS SDKs to create domains\. 4 | 5 | **Topics** 6 | + [Elasticsearch client compatibility](#client-compatibility) 7 | + [Compressing HTTP requests in Amazon OpenSearch Service](gzip.md) 8 | + [Using the AWS SDKs to interact with Amazon OpenSearch Service](configuration-samples.md) 9 | 10 | ## Elasticsearch client compatibility 11 | 12 | The latest versions of the Elasticsearch clients might include license or version checks that artificially break compatibility\. The following table includes recommendations around which versions of those clients to use for best compatibility with OpenSearch Service\. 13 | 14 | **Important** 15 | These client versions are out of date and are not updated with the latest dependencies, including Log4j\. We highly recommend using the OpenSearch versions of the clients when possible\. 16 | 17 | 18 | | Client | Recommended version | 19 | | --- | --- | 20 | | Java low\-level REST client | 7\.13\.4 | 21 | | Java high\-level REST client | 7\.13\.4 | 22 | | Python Elasticsearch client | 7\.13\.4 | 23 | | Ruby Elasticsearch client | 7\.13\.3 | 24 | | Node\.js Elasticsearch client | 7\.13\.0 | -------------------------------------------------------------------------------- /doc_source/security.md: -------------------------------------------------------------------------------- 1 | # Security in Amazon OpenSearch Service 2 | 3 | Cloud security at AWS is the highest priority\. As an AWS customer, you benefit from a data center and network architecture that is built to meet the requirements of the most security\-sensitive organizations\. 4 | 5 | Security is a shared responsibility between AWS and you\. The [shared responsibility model](http://aws.amazon.com/compliance/shared-responsibility-model/) describes this as security *of* the cloud and security *in* the cloud: 6 | + **Security of the cloud** – AWS is responsible for protecting the infrastructure that runs AWS services in the AWS Cloud\. AWS also provides you with services that you can use securely\. Third\-party auditors regularly test and verify the effectiveness of our security as part of the [AWS compliance programs](http://aws.amazon.com/compliance/programs/)\. To learn about the compliance programs that apply to Amazon OpenSearch Service, see [AWS Services in Scope by Compliance Program](http://aws.amazon.com/compliance/services-in-scope/)\. 7 | + **Security in the cloud** – Your responsibility is determined by the AWS service that you use\. You are also responsible for other factors including the sensitivity of your data, your company’s requirements, and applicable laws and regulations\. 8 | 9 | This documentation helps you understand how to apply the shared responsibility model when using OpenSearch Service\. The following topics show you how to configure OpenSearch Service to meet your security and compliance objectives\. You also learn how to use other AWS services that help you to monitor and secure your OpenSearch Service resources\. 10 | 11 | **Topics** 12 | + [Data protection in Amazon OpenSearch Service](data-protection.md) 13 | + [Identity and Access Management in Amazon OpenSearch Service](ac.md) 14 | + [Cross\-service confused deputy prevention](cross-service-confused-deputy-prevention.md) 15 | + [Fine\-grained access control in Amazon OpenSearch Service](fgac.md) 16 | + [Compliance validation for Amazon OpenSearch Service](compliance.md) 17 | + [Resilience in Amazon OpenSearch Service](disaster-recovery-resiliency.md) 18 | + [Infrastructure security in Amazon OpenSearch Service](infrastructure-security.md) 19 | + [SAML authentication for OpenSearch Dashboards](saml.md) 20 | + [Configuring Amazon Cognito authentication for OpenSearch Dashboards](cognito-auth.md) 21 | + [Using service\-linked roles for Amazon OpenSearch Service](slr.md) 22 | + [Security Analytics for Amazon OpenSearch Service](security-analytics.md) -------------------------------------------------------------------------------- /doc_source/serverless-cfn.md: -------------------------------------------------------------------------------- 1 | # Using AWS CloudFormation to create Amazon OpenSearch Serverless collections 2 | 3 | You can use AWS CloudFormation to create Amazon OpenSearch Serverless resources such as collections, security policies, and VPC endpoints\. For the comprehensive OpenSearch Serverless CloudFormation reference, see [Amazon OpenSearch Serverless](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/AWS_OpenSearchServerless.html) in the *AWS CloudFormation User Guide*\. 4 | 5 | The following sample CloudFormation template creates a simple data access policy, network policy, and security policy, as well as a matching collection\. It's a good way to get up and running quickly with Amazon OpenSearch Serverless and provision the necessary elements to create and use a collection\. 6 | 7 | **Important** 8 | This example uses public network access, which isn't recommended for production workloads\. We recommend using VPC access to protect your collections\. For more information, see [AWS::OpenSearchServerless::VpcEndpoint](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-opensearchserverless-vpcendpoint.html) and [Access Amazon OpenSearch Serverless using an interface endpoint \(AWS PrivateLink\)](serverless-vpc.md)\. 9 | 10 | ``` 11 | AWSTemplateFormatVersion: 2010-09-09 12 | Description: 'Amazon OpenSearch Serverless template to create an IAM user, encryption policy, data access policy and collection' 13 | Resources: 14 | IAMUSer: 15 | Type: 'AWS::IAM::User' 16 | Properties: 17 | UserName: aossadmin 18 | DataAccessPolicy: 19 | Type: 'AWS::OpenSearchServerless::AccessPolicy' 20 | Properties: 21 | Name: quickstart-access-policy 22 | Type: data 23 | Description: Access policy for quickstart collection 24 | Policy: !Sub >- 25 | [{"Description":"Access for cfn user","Rules":[{"ResourceType":"index","Resource":["index/*/*"],"Permission":["aoss:*"]}, 26 | {"ResourceType":"collection","Resource":["collection/quickstart"],"Permission":["aoss:*"]}], 27 | "Principal":["arn:aws:iam::${AWS::AccountId}:user/aossadmin"]}] 28 | NetworkPolicy: 29 | Type: 'AWS::OpenSearchServerless::SecurityPolicy' 30 | Properties: 31 | Name: quickstart-network-policy 32 | Type: network 33 | Description: Network policy for quickstart collection 34 | Policy: >- 35 | [{"Rules":[{"ResourceType":"collection","Resource":["collection/quickstart"]}, {"ResourceType":"dashboard","Resource":["collection/quickstart"]}],"AllowFromPublic":true}] 36 | EncryptionPolicy: 37 | Type: 'AWS::OpenSearchServerless::SecurityPolicy' 38 | Properties: 39 | Name: quickstart-security-policy 40 | Type: encryption 41 | Description: Encryption policy for quickstart collection 42 | Policy: >- 43 | {"Rules":[{"ResourceType":"collection","Resource":["collection/quickstart"]}],"AWSOwnedKey":true} 44 | Collection: 45 | Type: 'AWS::OpenSearchServerless::Collection' 46 | Properties: 47 | Name: quickstart 48 | Type: TIMESERIES 49 | Description: Collection to holds timeseries data 50 | DependsOn: EncryptionPolicy 51 | Outputs: 52 | IAMUser: 53 | Value: !Ref IAMUSer 54 | DashboardURL: 55 | Value: !GetAtt Collection.DashboardEndpoint 56 | CollectionARN: 57 | Value: !GetAtt Collection.Arn 58 | ``` -------------------------------------------------------------------------------- /doc_source/serverless-collections.md: -------------------------------------------------------------------------------- 1 | # Creating and managing Amazon OpenSearch Serverless collections 2 | 3 | You can create Amazon OpenSearch Serverless collections using the console, the AWS CLI and API, the AWS SDKs, and AWS CloudFormation\. 4 | 5 | **Topics** 6 | + [Creating, listing, and deleting Amazon OpenSearch Serverless collections](serverless-manage.md) 7 | + [Working with vector search collections](serverless-vector-search.md) 8 | + [Using the AWS SDKs to interact with Amazon OpenSearch Serverless](serverless-sdk.md) 9 | + [Using AWS CloudFormation to create Amazon OpenSearch Serverless collections](serverless-cfn.md) -------------------------------------------------------------------------------- /doc_source/serverless-monitoring-events.md: -------------------------------------------------------------------------------- 1 | # Monitoring OpenSearch Serverless events using Amazon EventBridge 2 | 3 | Amazon OpenSearch Service integrates with Amazon EventBridge to notify you of certain events that affect your domains\. Events from AWS services are delivered to EventBridge in near real time\. The same events are also sent to [Amazon CloudWatch Events](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/WhatIsCloudWatchEvents.html), the predecessor of Amazon EventBridge\. You can write rules to indicate which events are of interest to you, and what automated actions to take when an event matches a rule\. Examples of actions that you can automatically activate include the following: 4 | + Invoking an AWS Lambda function 5 | + Invoking an Amazon EC2 Run Command 6 | + Relaying the event to Amazon Kinesis Data Streams 7 | + Activating an AWS Step Functions state machine 8 | + Notifying an Amazon SNS topic or an Amazon SQS queue 9 | 10 | For more information, see [Get started with Amazon EventBridge](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-get-started.html) in the *Amazon EventBridge User Guide*\. 11 | 12 | ## Setting up notifications 13 | 14 | You can use [AWS User Notifications](https://docs.aws.amazon.com/notifications/latest/userguide/what-is-service.html) to receive notifications when an OpenSearch Serverless event occurs\. An event is an indicator of a change in OpenSearch Serverless environment, such as when you reach the maximum limit of your OCU usage\. Amazon EventBridge receives the event and routes a notification to the AWS Management Console Notifications Center and your chosen delivery channels\. You receive a notification when an event matches a rule that you specify\. 15 | 16 | ## OpenSearch Compute Units \(OCU\) events 17 | 18 | OpenSearch Serverless sends events to EventBridge when one of the following OCU\-related events occur\. 19 | 20 | ### OCU usage approaching maximum limit 21 | 22 | OpenSearch Serverless sends this event when your search or index OCU usage reaches 75% of your capacity limit\. Your OCU usage is calculated based on your configured capacity limit and your current OCU consumption\. 23 | 24 | **Example** 25 | 26 | The following is an example event of this type \(search OCU\): 27 | 28 | ``` 29 | { 30 | "version": "0", 31 | "id": "01234567-0123-0123-0123-012345678901", 32 | "detail-type": "OCU Utilization Approaching Max Limit", 33 | "source": "aws.aoss", 34 | "account": "123456789012", 35 | "time": "2016-11-01T13:12:22Z", 36 | "region": "us-east-1", 37 | "resources": ["arn:aws:es:us-east-1:123456789012:domain/test-domain"], 38 | "detail": { 39 | "eventTime" : 1678943345789, 40 | "description": "Your search OCU usage is at 75% and is approaching the configured maximum limit." 41 | } 42 | } 43 | ``` 44 | 45 | The following is an example event of this type \(index OCU\): 46 | 47 | ``` 48 | { 49 | "version": "0", 50 | "id": "01234567-0123-0123-0123-012345678901", 51 | "detail-type": "OCU Utilization Approaching Max Limit", 52 | "source": "aws.aoss", 53 | "account": "123456789012", 54 | "time": "2016-11-01T13:12:22Z", 55 | "region": "us-east-1", 56 | "resources": ["arn:aws:es:us-east-1:123456789012:domain/test-domain"], 57 | "detail": { 58 | "eventTime" : 1678943345789, 59 | "description": "Your indexing OCU usage is at 75% and is approaching the configured maximum limit." 60 | } 61 | ``` 62 | 63 | ### OCU usage reached maximum limit 64 | 65 | OpenSearch Serverless sends this event when your search or index OCU usage reaches 100% of your capacity limit\. Your OCU usage is calculated based on your configured capacity limit and your current OCU consumption\. 66 | 67 | **Example** 68 | 69 | The following is an example event of this type \(search OCU\): 70 | 71 | ``` 72 | { 73 | "version": "0", 74 | "id": "01234567-0123-0123-0123-012345678901", 75 | "detail-type": "OCU Utilization Reached Max Limit", 76 | "source": "aws.aoss", 77 | "account": "123456789012", 78 | "time": "2016-11-01T13:12:22Z", 79 | "region": "us-east-1", 80 | "resources": ["arn:aws:es:us-east-1:123456789012:domain/test-domain"], 81 | "detail": { 82 | "eventTime" : 1678943345789, 83 | "description": "Your search OCU usage has reached the configured maximum limit." 84 | } 85 | } 86 | ``` 87 | 88 | The following is an example event of this type \(index OCU\): 89 | 90 | ``` 91 | { 92 | "version": "0", 93 | "id": "01234567-0123-0123-0123-012345678901", 94 | "detail-type": "OCU Utilization Reached Max Limit", 95 | "source": "aws.aoss", 96 | "account": "123456789012", 97 | "time": "2016-11-01T13:12:22Z", 98 | "region": "us-east-1", 99 | "resources": ["arn:aws:es:us-east-1:123456789012:domain/test-domain"], 100 | "detail": { 101 | "eventTime" : 1678943345789, 102 | "description": "Your indexing OCU usage has reached the configured maximum limit." 103 | } 104 | } 105 | ``` -------------------------------------------------------------------------------- /doc_source/serverless-monitoring.md: -------------------------------------------------------------------------------- 1 | # Monitoring Amazon OpenSearch Serverless 2 | 3 | Monitoring is an important part of maintaining the reliability, availability, and performance of Amazon OpenSearch Serverless and your other AWS solutions\. AWS provides the following monitoring tools to watch OpenSearch Serverless, report when something is wrong, and take automatic actions when appropriate: 4 | + *Amazon CloudWatch* monitors your AWS resources and the applications that you run on AWS in real time\. You can collect and track metrics, create customized dashboards, and set alarms that notify you or take actions when a specified metric reaches a threshold that you specify\. 5 | 6 | For example, you can have CloudWatch track CPU usage or other metrics of your Amazon EC2 instances and automatically launch new instances when needed\. For more information, see the [Amazon CloudWatch User Guide](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/)\. 7 | + *AWS CloudTrail* captures API calls and related events made by or on behalf of your AWS account\. It delivers the log files to an Amazon S3 bucket that you specify\. You can identify which users and accounts called AWS, the source IP address from which the calls were made, and when the calls occurred\. For more information, see the [AWS CloudTrail User Guide](https://docs.aws.amazon.com/awscloudtrail/latest/userguide/)\. 8 | + *Amazon EventBridge* delivers a near real\-time stream of system events that describe changes in your OpenSearch Service domains\. You can create rules that watch for certain events, and trigger automated actions in other AWS services when these events occur\. For more information, see the [Amazon EventBridge User Guide](https://docs.aws.amazon.com/eventbridge/latest/userguide/)\. -------------------------------------------------------------------------------- /doc_source/serverless-tutorials.md: -------------------------------------------------------------------------------- 1 | # Getting started with security in Amazon OpenSearch Serverless 2 | 3 | The following tutorials help you get started using Amazon OpenSearch Serverless\. Both tutorials accomplish the same basic steps, but one uses the console while the other uses the AWS CLI\. 4 | 5 | Note that the use cases in these tutorials are simplified\. The network and security policies are fairly open\. In production workloads, we recommend that you configure more robust security features such as SAML authentication, VPC access, and restrictive data access policies\. 6 | 7 | **Topics** 8 | + [Tutorial: Getting started with security in Amazon OpenSearch Serverless \(console\)](gsg-serverless.md) 9 | + [Tutorial: Getting started with security in Amazon OpenSearch Serverless \(CLI\)](gsg-serverless-cli.md) -------------------------------------------------------------------------------- /doc_source/serverless.md: -------------------------------------------------------------------------------- 1 | # Amazon OpenSearch Serverless 2 | 3 | Amazon OpenSearch Serverless is an on\-demand, auto\-scaling configuration for Amazon OpenSearch Service\. An OpenSearch Serverless *collection* is an OpenSearch cluster that scales compute capacity based on your application's needs\. This contrasts with OpenSearch Service *provisioned OpenSearch domains*, which you manually manage capacity for\. 4 | 5 | OpenSearch Serverless provides a simple, cost\-effective option for infrequent, intermittent, or unpredictable workloads\. It's cost\-effective because it automatically scales compute capacity to match your application's usage\. 6 | 7 | OpenSearch Serverless collections have the same kind of high\-capacity, distributed, and highly available storage volume that is used by provisioned OpenSearch Service domains\. 8 | 9 | OpenSearch Serverless collections are always encrypted\. You can choose the encryption key, but you can't disable encryption\. For more information, see [Encryption in Amazon OpenSearch Serverless](serverless-encryption.md)\. 10 | 11 | **Topics** 12 | + [Benefits](#serverless-benefits) 13 | + [What is Amazon OpenSearch Serverless?](serverless-overview.md) 14 | + [Getting started with Amazon OpenSearch Serverless](serverless-getting-started.md) 15 | + [Creating and managing Amazon OpenSearch Serverless collections](serverless-collections.md) 16 | + [Managing capacity limits for Amazon OpenSearch Serverless](serverless-scaling.md) 17 | + [Ingesting data into Amazon OpenSearch Serverless collections](serverless-clients.md) 18 | + [Overview of security in Amazon OpenSearch Serverless](serverless-security.md) 19 | + [Tagging Amazon OpenSearch Serverless collections](tag-collection.md) 20 | + [Supported operations and plugins in Amazon OpenSearch Serverless](serverless-genref.md) 21 | + [Monitoring Amazon OpenSearch Serverless](serverless-monitoring.md) 22 | 23 | ## Benefits 24 | 25 | OpenSearch Serverless has the following benefits: 26 | + **Simpler than provisioned** – OpenSearch Serverless removes much of the complexity of managing OpenSearch clusters and capacity\. It automatically sizes and tunes your clusters, and takes care of shard and index lifecycle management\. It also manages service software updates and OpenSearch version upgrades\. All updates and upgrades are non\-disruptive\. 27 | + **Cost\-effective** – When you use OpenSearch Serverless, you only pay for the resources that you consume\. This removes the need for upfront provisioning and overprovisioning for peak workloads\. 28 | + **Highly available** – OpenSearch Serverless supports production workloads with redundancy to protect against Availability Zone outages and infrastructure failures\. 29 | + **Scalable** – OpenSearch Serverless automatically scales resources to maintain consistently fast data ingestion rates and query response times\. -------------------------------------------------------------------------------- /doc_source/slr-osis.md: -------------------------------------------------------------------------------- 1 | # Using service\-linked roles to create OpenSearch Ingestion pipelines 2 | 3 | Amazon OpenSearch Ingestion uses AWS Identity and Access Management \(IAM\) [service\-linked roles](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_terms-and-concepts.html#iam-term-service-linked-role)\. A service\-linked role is a unique type of IAM role that is linked directly to OpenSearch Ingestion\. Service\-linked roles are predefined by OpenSearch Ingestion and include all the permissions that the service requires to call other AWS services on your behalf\. 4 | 5 | OpenSearch Ingestion uses the service\-linked role named **AWSServiceRoleForAmazonOpenSearchIngestion**\. The attached policy provides the permissions necessary for the role to create a virtual private cloud \(VPC\) between your account and OpenSearch Ingestion, and to publish CloudWatch metrics to your account\. 6 | 7 | ## Permissions 8 | 9 | The `AWSServiceRoleForAmazonOpenSearchIngestion` service\-linked role trusts the following services to assume the role: 10 | + `osis.amazon.com` 11 | 12 | The role permissions policy named `AmazonOpenSearchIngestionServiceRolePolicy` allows OpenSearch Ingestion to complete the following actions on the specified resources: 13 | + Action: `ec2:DescribeSubnets` on `*` 14 | + Action: `ec2:DescribeSecurityGroups` on `*` 15 | + Action: `ec2:DeleteVpcEndpoints` on `*` 16 | + Action: `ec2:CreateVpcEndpoint` on `*` 17 | + Action: `ec2:DescribeVpcEndpoints` on `*` 18 | + Action: `ec2:CreateTags` on `arn:aws:ec2:*:*:network-interface/*` 19 | + Action: `cloudwatch:PutMetricData` on `cloudwatch:namespace": "AWS/OSIS"` 20 | 21 | You must configure permissions to allow an IAM entity \(such as a user, group, or role\) to create, edit, or delete a service\-linked role\. For more information, see [Service\-linked role permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/using-service-linked-roles.html#service-linked-role-permissions) in the *IAM User Guide*\. 22 | 23 | ## Creating the service\-linked role for OpenSearch Ingestion 24 | 25 | You don't need to manually create a service\-linked role\. When you [create an OpenSearch Ingestion pipeline](creating-pipeline.md#create-pipeline) in the AWS Management Console, the AWS CLI, or the AWS API, OpenSearch Ingestion creates the service\-linked role for you\. 26 | 27 | If you delete this service\-linked role, and then need to create it again, you can use the same process to recreate the role in your account\. When you create an OpenSearch Ingestion pipeline, OpenSearch Ingestion creates the service\-linked role for you again\. 28 | 29 | ## Editing the service\-linked role for OpenSearch Ingestion 30 | 31 | OpenSearch Ingestion does not allow you to edit the `AWSServiceRoleForAmazonOpenSearchIngestion` service\-linked role\. After you create a service\-linked role, you cannot change the name of the role because various entities might reference the role\. However, you can edit the description of the role using IAM\. For more information, see [Editing a service\-linked role](https://docs.aws.amazon.com/IAM/latest/UserGuide/using-service-linked-roles.html#edit-service-linked-role) in the *IAM User Guide*\. 32 | 33 | ## Deleting the service\-linked role for OpenSearch Ingestion 34 | 35 | If you no longer need to use a feature or service that requires a service\-linked role, we recommend that you delete that role\. That way you don't have an unused entity that is not actively monitored or maintained\. However, you must clean up the resources for your service\-linked role before you can manually delete it\. 36 | 37 | ### Cleaning up a service\-linked role 38 | 39 | Before you can use IAM to delete a service\-linked role, you must first delete any resources used by the role\. 40 | 41 | **Note** 42 | If OpenSearch Ingestion is using the role when you try to delete the resources, then the deletion might fail\. If that happens, wait for a few minutes and try the operation again\. 43 | 44 | **To delete OpenSearch Ingestion resources used by the `AWSServiceRoleForAmazonOpenSearchIngestion`** 45 | 46 | 1. Navigate to the Amazon OpenSearch Service console and choose **Ingestion**\. 47 | 48 | 1. Delete all pipelines\. For instructions, see [Deleting Amazon OpenSearch Ingestion pipelines](delete-pipeline.md)\. 49 | 50 | ### Delete the service\-linked role for OpenSearch Ingestion 51 | 52 | You can use the OpenSearch Ingestion console to delete a service\-linked role\. 53 | 54 | **To delete a service\-linked role \(console\)** 55 | 56 | 1. Navigate to the IAM console\. 57 | 58 | 1. Choose **Roles** and search for the **AWSServiceRoleForAmazonOpenSearchIngestion** role\. 59 | 60 | 1. Select the role and choose **Delete**\. -------------------------------------------------------------------------------- /doc_source/slr.md: -------------------------------------------------------------------------------- 1 | # Using service\-linked roles for Amazon OpenSearch Service 2 | 3 | Amazon OpenSearch Service uses AWS Identity and Access Management \(IAM\) [service\-linked roles](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_terms-and-concepts.html#iam-term-service-linked-role)\. A service\-linked role is a unique type of IAM role that is linked directly to OpenSearch Service\. Service\-linked roles are predefined by OpenSearch Service and include all the permissions that the service requires to call other AWS services on your behalf\. 4 | 5 | A service\-linked role makes setting up OpenSearch Service easier because you don’t have to manually add the necessary permissions\. OpenSearch Service defines the permissions of its service\-linked roles, and unless defined otherwise, only OpenSearch Service can assume its roles\. The defined permissions include the trust policy and the permissions policy, and that permissions policy cannot be attached to any other IAM entity\. 6 | 7 | For information about other services that support service\-linked roles, see [AWS services that work with IAM](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html) and look for the services that have **Yes **in the **Service\-linked roles** column\. Choose a **Yes** with a link to view the service\-linked role documentation for that service\. 8 | 9 | **Topics** 10 | + [Using service\-linked roles to create VPC domains](slr-aos.md) 11 | + [Using service\-linked roles to create OpenSearch Serverless collections](serverless-service-linked-roles.md) 12 | + [Using service\-linked roles to create OpenSearch Ingestion pipelines](slr-osis.md) -------------------------------------------------------------------------------- /doc_source/sns-events.md: -------------------------------------------------------------------------------- 1 | # Tutorial: Sending Amazon SNS alerts for available software updates 2 | 3 | In this tutorial, you configure an Amazon EventBridge event rule that captures notifications for available service software updates in Amazon OpenSearch Service and sends you an email notification through Amazon Simple Notification Service \(Amazon SNS\)\. 4 | 5 | ## Prerequisites 6 | 7 | This tutorial assumes that you have an existing OpenSearch Service domain\. If you haven't created a domain, follow the steps in [Creating and managing Amazon OpenSearch Service domains](createupdatedomains.md) to create one\. 8 | 9 | ## Step 1: Create and subscribe to an Amazon SNS topic 10 | 11 | Configure an Amazon SNS topic to serve as an event target for your new event rule\. 12 | 13 | **To create an Amazon SNS target** 14 | 15 | 1. Open the Amazon SNS console at [https://console\.aws\.amazon\.com/sns/v3/home](https://console.aws.amazon.com/sns/v3/home)\. 16 | 17 | 1. Choose **Topics** and **Create topic**\. 18 | 19 | 1. For the job type, choose **Standard**, and name the job **software\-update**\. 20 | 21 | 1. Choose **Create topic**\. 22 | 23 | 1. After the topic is created, choose **Create subscription**\. 24 | 25 | 1. For **Protocol**, choose **Email**\. For **Endpoint**, enter an email address that you currently have access to and choose **Create subscription**\. 26 | 27 | 1. Check your email account and wait to receive a subscription confirmation email message\. When you receive it, choose **Confirm subscription**\. 28 | 29 | ## Step 2: Register an event rule 30 | 31 | Next, register an event rule that captures only service software update events\. 32 | 33 | **To create an event rule** 34 | 35 | 1. Open the EventBridge console at [https://console\.aws\.amazon\.com/events/](https://console.aws.amazon.com/events/)\. 36 | 37 | 1. Choose **Create rule**\. 38 | 39 | 1. Name the rule **softwareupdate\-rule**\. 40 | 41 | 1. Choose **Next**\. 42 | 43 | 1. For the event pattern, select **AWS services**, **Amazon OpenSearch Service**, and **Amazon OpenSearch Service Software Update Notification**\. This pattern matches any service software update event from OpenSearch Service\. For more information about event patterns, see [Amazon EventBridge event patterns](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-event-patterns.html) in the *Amazon EventBridge User Guide*\. 44 | 45 | 1. Optionally, you can filter to only specific severities\. For the severities of each event, see [Service software update events](monitoring-events.md#monitoring-events-sso)\. 46 | 47 | 1. Choose **Next**\. 48 | 49 | 1. For the target, choose **SNS topic** and select **software\-update**\. 50 | 51 | 1. Choose **Next**\. 52 | 53 | 1. Skip the tags and choose **Next**\. 54 | 55 | 1. Review the rule configuration and choose **Create rule**\. 56 | 57 | The next time you receive a notification from OpenSearch Service about an available service software update, if everything is configured properly, Amazon SNS should send you an email alert about the update\. -------------------------------------------------------------------------------- /doc_source/supported-instance-types.md: -------------------------------------------------------------------------------- 1 | # Supported instance types in Amazon OpenSearch Service 2 | 3 | Amazon OpenSearch Service supports the following instance types\. Not all Regions support all instance types\. For availability details, see [Amazon OpenSearch Service pricing](https://aws.amazon.com/opensearch-service/pricing/)\. 4 | 5 | For information about which instance type is appropriate for your use case, see [Sizing Amazon OpenSearch Service domains](sizing-domains.md), [EBS volume size quotas](limits.md#ebsresource), and [Network quotas](limits.md#network-limits)\. 6 | 7 | ## Current generation instance types 8 | 9 | For the best performance, we recommend that you use the following instance types when you create new OpenSearch Service domains\. 10 | 11 | 12 | | Instance type | Instances | Restrictions | 13 | | --- | --- | --- | 14 | | C5 | `c5.large.search` `c5.xlarge.search` `c5.2xlarge.search` `c5.4xlarge.search` `c5.9xlarge.search` `c5.18xlarge.search` | The C5 instance types require Elasticsearch 5\.1 or later or any version of OpenSearch\. | 15 | | C6g | `c6g.large.search` `c6g.xlarge.search` `c6g.2xlarge.search` `c6g.4xlarge.search` `c6g.8xlarge.search` `c6g.12xlarge.search` | [\[See the AWS documentation website for more details\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-instance-types.html) | 16 | | I3 | `i3.large.search` `i3.xlarge.search` `i3.2xlarge.search` `i3.4xlarge.search` `i3.8xlarge.search` `i3.16xlarge.search` | The I3 instance types require Elasticsearch 5\.1 or later or any version of OpenSearch, and do not support EBS storage volumes\. | 17 | | M5 | `m5.large.search` `m5.xlarge.search` `m5.2xlarge.search` `m5.4xlarge.search` `m5.12xlarge.search` | The M5 instance types require Elasticsearch 5\.1 or later or any version of OpenSearch\. | 18 | | M6g | `m6g.large.search` `m6g.xlarge.search` `m6g.2xlarge.search` `m6g.4xlarge.search` `m6g.8xlarge.search` `m6g.12xlarge.search` | [\[See the AWS documentation website for more details\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-instance-types.html) | 19 | | R5 | `r5.large.search` `r5.xlarge.search` `r5.2xlarge.search` `r5.4xlarge.search` `r5.12xlarge.search` | The R5 instance types require Elasticsearch 5\.1 or later or any version of OpenSearch\. | 20 | | R6g | `r6g.large.search` `r6g.xlarge.search` `r6g.2xlarge.search` `r6g.4xlarge.search` `r6g.8xlarge.search` `r6g.12xlarge.search` | [\[See the AWS documentation website for more details\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-instance-types.html) | 21 | | R6gd | `r6gd.large.search` `r6gd.xlarge.search` `r6gd.2xlarge.search` `r6gd.4xlarge.search` `r6gd.8xlarge.search` `r6gd.12xlarge.search` `r6gd.16xlarge.search` | [\[See the AWS documentation website for more details\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-instance-types.html) | 22 | | T3 | `t3.small.search` `t3.medium.search` | [\[See the AWS documentation website for more details\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-instance-types.html) | 23 | 24 | ## Previous generation instance types 25 | 26 | OpenSearch Service offers previous generation instance types for users who have optimized their applications around them and have yet to upgrade\. We encourage you to use current generation instance types to get the best performance, but we continue to support the following previous generation instance types\. 27 | 28 | 29 | | Instance type | Instances | Restrictions | 30 | | --- | --- | --- | 31 | | C4 | `c4.large.search` `c4.xlarge.search` `c4.2xlarge.search` `c4.4xlarge.search` `c4.8xlarge.search` | | 32 | | I2 | `i2.xlarge.search` `i2.2xlarge.search` | | 33 | | M3 | `m3.medium.search` `m3.large.search` `m3.xlarge.search` `m3.2xlarge.search` | [\[See the AWS documentation website for more details\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-instance-types.html) | 34 | | M4 | `m4.large.search` `m4.xlarge.search` `m4.2xlarge.search` `m4.4xlarge.search` `m4.10xlarge.search` | | 35 | | R3 | `r3.large.search` `r3.xlarge.search` `r3.2xlarge.search` `r3.4xlarge.search` `r3.8xlarge.search` | The R3 instance types do not support encryption of data at rest or fine\-grained access control\. | 36 | | R4 | `r4.large.search` `r4.xlarge.search` `r4.2xlarge.search` `r4.4xlarge.search` `r4.8xlarge.search` `r4.16xlarge.search` | | 37 | | T2 | `t2.micro.search` `t2.small.search` `t2.medium.search` | [\[See the AWS documentation website for more details\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-instance-types.html) | 38 | 39 | **Tip** 40 | We often recommend different instance types for [dedicated master nodes](managedomains-dedicatedmasternodes.md) and data nodes\. -------------------------------------------------------------------------------- /doc_source/supported-plugins.md: -------------------------------------------------------------------------------- 1 | # Plugins by engine version in Amazon OpenSearch Service 2 | 3 | Amazon OpenSearch Service domains come prepackaged with plugins from the OpenSearch community\. The service automatically deploys and manages plugins for you, but it deploys different plugins depending on the version of OpenSearch or legacy Elasticsearch OSS you choose for your domain\. 4 | 5 | The following table lists plugins by OpenSearch version, as well as compatible versions of legacy Elasticsearch OSS\. It only includes plugins that you might interact with—it’s not comprehensive\. OpenSearch Service uses additional plugins to enable core service functionality, such as the S3 Repository plugin for snapshots and the [OpenSearch Performance Analyzer](https://opensearch.org/docs/latest/monitoring-plugins/pa/index/) plugin for optimization and monitoring\. For a complete list of all plugins running on your domain, make the following request: 6 | 7 | ``` 8 | GET _cat/plugins?v 9 | ``` 10 | 11 | 12 | **** 13 | [\[See the AWS documentation website for more details\]](http://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-plugins.html) -------------------------------------------------------------------------------- /doc_source/supported-resources.md: -------------------------------------------------------------------------------- 1 | # Other supported resources in Amazon OpenSearch Service 2 | 3 | This topic describes additional resources that Amazon OpenSearch Service supports\. 4 | 5 | **bootstrap\.memory\_lock** 6 | OpenSearch Service enables `bootstrap.memory_lock` in `opensearch.yml`, which locks JVM memory and prevents the operating system from swapping it to disk\. This applies to all supported instance types except for the following: 7 | + `t2.micro.search` 8 | + `t2.small.search` 9 | + `t2.medium.search` 10 | + `t3.small.search` 11 | + `t3.medium.search` 12 | 13 | **Scripting module** 14 | OpenSearch Service supports scripting for Elasticsearch 5\.*x* and later domains\. It does not support scripting for 1\.5 or 2\.3\. 15 | Supported scripting options include the following: 16 | + Painless 17 | + Lucene Expressions 18 | + Mustache 19 | For Elasticsearch 5\.5 and later domains, and all OpenSearch domains, OpenSearch Service supports stored scripts using the `_scripts` endpoint\. Elasticsearch 5\.3 and 5\.1 domains support inline scripts only\. 20 | 21 | **TLS transport** 22 | OpenSearch Service supports HTTP on port 80 and HTTPS over port 443, but does not support TLS transport\. -------------------------------------------------------------------------------- /doc_source/tag-collection.md: -------------------------------------------------------------------------------- 1 | # Tagging Amazon OpenSearch Serverless collections 2 | 3 | Tags let you assign arbitrary information to an Amazon OpenSearch Serverless collection so you can categorize and filter on that information\. A *tag* is a metadata label that you assign or that AWS assigns to an AWS resource\. 4 | 5 | Each tag consists of a *key* and a *value*\. For tags that you assign, you define the key and value\. For example, you might define the key as `stage` and the value for one resource as `test`\. 6 | 7 | With tags, you can do the following: 8 | + Identify and organize your AWS resources\. Many AWS services support tagging, so you can assign the same tag to resources from different services to indicate that the resources are related\. For example, you could assign the same tag to an OpenSearch Serverless collection that you assign to an Amazon OpenSearch Service domain\. 9 | + Track your AWS costs\. You activate these tags on the AWS Billing and Cost Management dashboard\. AWS uses the tags to categorize your costs and deliver a monthly cost allocation report to you\. For more information, see [Use Cost Allocation Tags](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/cost-alloc-tags.html) in the [AWS Billing User Guide](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/)\. 10 | 11 | In OpenSearch Serverless, the primary resource is a collection\. You can use the OpenSearch Service console, the AWS CLI, the OpenSearch Serverless API operations, or the AWS SDKs to add, manage, and remove tags from a collection\. 12 | 13 | ## Permissions required 14 | 15 | OpenSearch Serverless uses the following AWS Identity and Access Management Access Analyzer \(IAM\) permissions for tagging collections: 16 | + `aoss:TagResource` 17 | + `aoss:ListTagsForResource` 18 | + `aoss:UntagResource` 19 | 20 | ## Working with tags \(console\) 21 | 22 | The console is the simplest way to tag a collection\. 23 | 24 | ****To create a tag \(console\)**** 25 | 26 | 1. Sign in to the Amazon OpenSearch Service console at [https://console\.aws\.amazon\.com/aos/home](https://console.aws.amazon.com/aos/home )\. 27 | 28 | 1. Expand **Serverless** in the left navigation pane and choose **Collections**\. 29 | 30 | 1. Select the collection that you want to add tags to, and go to the **Tags** tab\. 31 | 32 | 1. Choose **Manage** and **Add new tag**\. 33 | 34 | 1. Enter a tag key and an optional value\. 35 | 36 | 1. Choose **Save**\. 37 | 38 | To delete a tag, follow the same steps and choose **Remove** on the **Manage tags** page\. 39 | 40 | For more information about using the console to work with tags, see [Tag Editor](https://docs.aws.amazon.com/awsconsolehelpdocs/latest/gsg/tag-editor.html) in the *AWS Management Console Getting Started Guide*\. 41 | 42 | ## Working with tags \(AWS CLI\) 43 | 44 | To tag a collection using the AWS CLI, send a [TagResource](https://docs.aws.amazon.com/opensearch-service/latest/ServerlessAPIReference/API_TagResource.html) request: 45 | 46 | ``` 47 | aws opensearchserverless tag-resource 48 | --resource-arn arn:aws:aoss:us-east-1:123456789012:collection/my-collection 49 | --tags Key=service,Value=aoss Key=source,Value=logs 50 | ``` 51 | 52 | View the existing tags for a collection with the [ListTagsForResource](https://docs.aws.amazon.com/opensearch-service/latest/ServerlessAPIReference/API_ListTagsForResource.html) command: 53 | 54 | ``` 55 | aws opensearchserverless list-tags-for-resource 56 | --resource-arn arn:aws:aoss:us-east-1:123456789012:collection/my-collection 57 | ``` 58 | 59 | Remove tags from a collection using the [UntagResource](https://docs.aws.amazon.com/opensearch-service/latest/ServerlessAPIReference/API_UntagResource.html) command: 60 | 61 | ``` 62 | aws opensearchserverless untag-resource 63 | --resource-arn arn:aws:aoss:us-east-1:123456789012:collection/my-collection 64 | --tag-keys service 65 | ``` -------------------------------------------------------------------------------- /doc_source/tag-pipeline.md: -------------------------------------------------------------------------------- 1 | # Tagging Amazon OpenSearch Ingestion pipelines 2 | 3 | Tags let you assign arbitrary information to an Amazon OpenSearch Ingestion pipeline so you can categorize and filter on that information\. A *tag* is a metadata label that you assign or that AWS assigns to an AWS resource\. Each tag consists of a *key* and a *value*\. For tags that you assign, you define the key and value\. For example, you might define the key as `stage` and the value for one resource as `test`\. 4 | 5 | Tags help you do the following: 6 | + Identify and organize your AWS resources\. Many AWS services support tagging, so you can assign the same tag to resources from different services to indicate that the resources are related\. For example, you could assign the same tag to an OpenSearch Ingestion pipeline that you assign to an Amazon OpenSearch Service domain\. 7 | + Track your AWS costs\. You activate these tags on the AWS Billing and Cost Management dashboard\. AWS uses the tags to categorize your costs and deliver a monthly cost allocation report to you\. For more information, see [Use Cost Allocation Tags](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/cost-alloc-tags.html) in the [AWS Billing User Guide](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/)\. 8 | + Restrict access to pipelines using attribute based access control\. For more information, see [Controlling access based on tag keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_tags.html#access_tags_control-tag-keys) in the IAM User Guide\. 9 | 10 | In OpenSearch Ingestion, the primary resource is a pipeline\. You can use the OpenSearch Service console, the AWS CLI, OpenSearch Ingestion APIs, or the AWS SDKs to add, manage, and remove tags from a pipeline\. 11 | 12 | **Topics** 13 | + [Permissions required](#pipeline-tag-permissions) 14 | + [Working with tags \(console\)](#tag-pipeline-console) 15 | + [Working with tags \(AWS CLI\)](#tag-pipeline-cli) 16 | 17 | ## Permissions required 18 | 19 | OpenSearch Ingestion uses the following AWS Identity and Access Management Access Analyzer \(IAM\) permissions for tagging pipelines: 20 | + `osis:TagResource` 21 | + `osis:ListTagsForResource` 22 | + `osis:UntagResource` 23 | 24 | For more information about each permission, see [Actions, resources, and condition keys for OpenSearch Ingestion](https://docs.aws.amazon.com/service-authorization/latest/reference/list_opensearchingestionservice.html) in the *Service Authorization Reference*\. 25 | 26 | ## Working with tags \(console\) 27 | 28 | The console is the simplest way to tag a pipeline\. 29 | 30 | ****To create a tag**** 31 | 32 | 1. Sign in to the Amazon OpenSearch Service console at [https://console\.aws\.amazon\.com/aos/home](https://console.aws.amazon.com/aos/home )\. 33 | 34 | 1. Choose **Ingestion** on the left navigation pane\. 35 | 36 | 1. Select the pipeline you want to add tags to and go to the **Tags** tab\. 37 | 38 | 1. Choose **Manage** and **Add new tag**\. 39 | 40 | 1. Enter a tag key and an optional value\. 41 | 42 | 1. Choose **Save**\. 43 | 44 | To delete a tag, follow the same steps and choose **Remove** on the **Manage tags** page\. 45 | 46 | For more information about using the console to work with tags, see [Tag Editor](https://docs.aws.amazon.com/awsconsolehelpdocs/latest/gsg/tag-editor.html) in the *AWS Management Console Getting Started Guide*\. 47 | 48 | ## Working with tags \(AWS CLI\) 49 | 50 | To tag a pipeline using the AWS CLI, send a `TagResource` request: 51 | 52 | ``` 53 | aws osis tag-resource 54 | --arn arn:aws:osis:us-east-1:123456789012:pipeline/my-pipeline 55 | --tags Key=service,Value=osis Key=source,Value=otel 56 | ``` 57 | 58 | Remove tags from a pipeline using the `UntagResource` command: 59 | 60 | ``` 61 | aws osis untag-resource 62 | --arn arn:aws:osis:us-east-1:123456789012:pipeline/my-pipeline 63 | --tag-keys service 64 | ``` 65 | 66 | View the existing tags for a pipeline with the `ListTagsForResource` command: 67 | 68 | ``` 69 | aws osis list-tags-for-resource 70 | --arn arn:aws:osis:us-east-1:123456789012:pipeline/my-pipeline 71 | ``` -------------------------------------------------------------------------------- /doc_source/transforms.md: -------------------------------------------------------------------------------- 1 | # Transforming indexes in Amazon OpenSearch Service 2 | 3 | Whereas [index rollup jobs](rollup.md) let you reduce data granularity by rolling up old data into condensed indices, transform jobs let you create a different, summarized view of your data centered around certain fields, so you can visualize or analyze the data in different ways\. 4 | 5 | Index transforms have an OpenSearch Dashboards user interface and REST API\. The feature requires OpenSearch 1\.0 or later\. Full documentation is available in the [OpenSearch documentation](https://opensearch.org/docs/im-plugin/index-transforms/)\. 6 | 7 | ## Creating an index transform job 8 | 9 | If you don’t have any data in your cluster, use the sample flight data within OpenSearch Dashboards to try out transform jobs\. After adding the data, launch OpenSearch Dashboards\. Then choose **Index Management**, **Transform Jobs**, and **Create Transform Job**\. 10 | 11 | ### Step 1: Choose indices 12 | 13 | In the **Indices** section, select the source and target index\. You can either select an existing target index or create a new one by entering a name for it\. 14 | 15 | If you want to transform just a subset of your source index, choose **Add Data Filter**, and use the OpenSearch [query DSL](https://opensearch.org/docs/opensearch/query-dsl/) to specify a subset of your source index\. 16 | 17 | ### Step 2: Choose fields 18 | 19 | After choosing your indices, choose the fields you want to use in your transform job, as well as whether to use groupings or aggregations\. 20 | + You can use groupings to place your data into separate buckets in your transformed index\. For example, if you want to group all of the airport destinations within the sample flight data, group the `DestAirportID` field into a target field of `DestAirportID_terms` field, and you can find the grouped airport IDs in your transformed index after the transform job finishes\. 21 | + On the other hand, aggregations let you perform simple calculations\. For example, you might include an aggregation in your transform job to define a new field of `sum_of_total_ticket_price` that calculates the sum of all airplane tickets\. Then you can analyze the new data in your transformed index\. 22 | 23 | ### Step 3: Specify a schedule 24 | 25 | Transform jobs are enabled by default and run on schedules\. For **transform execution interval**, specify an interval in minutes, hours, or days\. 26 | 27 | ### Step 4: Review and monitor 28 | 29 | Review your configuration and select **Create**\. Then monitor the **Transform job status** column\. 30 | 31 | ### Step 5: Search the target index 32 | 33 | After the job finishes, you can use the standard `_search` API to search the target index\. 34 | 35 | For example, after running a transform job that transforms the flight data based on the `DestAirportID` field, you can run the following request to return all fields that have a value of `SFO`: 36 | 37 | ``` 38 | GET target_index/_search 39 | { 40 | "query": { 41 | "match": { 42 | "DestAirportID_terms" : "SFO" 43 | } 44 | } 45 | } 46 | ``` -------------------------------------------------------------------------------- /doc_source/tutorials.md: -------------------------------------------------------------------------------- 1 | # Amazon OpenSearch Service tutorials 2 | 3 | This chapter includes several start\-to\-finish tutorials for working with Amazon OpenSearch Service, including how to migrate to the service, build a simple search application, and create a visualization in OpenSearch Dashboards\. 4 | 5 | **Topics** 6 | + [Tutorial: Creating and searching for documents in Amazon OpenSearch Service](quick-start.md) 7 | + [Tutorial: Migrating to Amazon OpenSearch Service](migration.md) 8 | + [Tutorial: Creating a search application with Amazon OpenSearch Service](search-example.md) 9 | + [Tutorial: Visualizing customer support calls with OpenSearch Service and OpenSearch Dashboards](walkthrough.md) -------------------------------------------------------------------------------- /doc_source/use-cases-metrics-logs.md: -------------------------------------------------------------------------------- 1 | # Deriving metrics from logs with Amazon OpenSearch Ingestion 2 | 3 | You can use Amazon OpenSearch Ingestion to derive metrics from logs\. The following example pipeline receives incoming logs using the [HTTP source](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sources/http-source/) plugin and the [Grok processor](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/processors/grok/)\. It then uses the [Aggregate processor](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/processors/aggregate/) to extract the metric `bytes` aggregated over a 30\-second window and derives histograms from the results\. 4 | 5 | The overall pipeline contains two sub\-pipelines: 6 | + `apache-log-pipeline-with-metrics` – Receives logs via an HTTP client like FluentBit, extracts important values from the logs by matching the value in the `log` key against the grok common Apache log pattern, and then forwards the grokked logs to both the `log-to-metrics-pipeline` sub\-pipeline and to an OpenSearch index named `logs`\. 7 | + `log-to-metrics-pipeline` – Receives the grokked logs from the `apache-log-pipeline-with-metrics` sub\-pipeline, aggregates the logs and derives histogram metrics of `bytes` based on the values in the `clientip` and `request` keys\. Finally, it sends the histogram metrics to an OpenSearch index named `histogram_metrics`\. 8 | 9 | ``` 10 | version: "2" 11 | apache-log-pipeline-with-metrics: 12 | source: 13 | http: 14 | # Provide the path for ingestion. ${pipelineName} will be replaced with pipeline name configured for this pipeline. 15 | # In this case it would be "/apache-log-pipeline-with-metrics/logs". This will be the FluentBit output URI value. 16 | path: "/${pipelineName}/logs" 17 | processor: 18 | - grok: 19 | match: 20 | log: [ "%{COMMONAPACHELOG_DATATYPED}" ] 21 | sink: 22 | - opensearch: 23 | ... 24 | index: "logs" 25 | - pipeline: 26 | name: "log-to-metrics-pipeline" 27 | 28 | log-to-metrics-pipeline: 29 | source: 30 | pipeline: 31 | name: "apache-log-pipeline-with-metrics" 32 | processor: 33 | - aggregate: 34 | # Specify the required identification keys 35 | identification_keys: ["clientip", "request"] 36 | action: 37 | histogram: 38 | # Specify the appropriate values for each of the following fields 39 | key: "bytes" 40 | record_minmax: true 41 | units: "bytes" 42 | buckets: [0, 25000000, 50000000, 75000000, 100000000] 43 | # Pick the required aggregation period 44 | group_duration: "30s" 45 | sink: 46 | - opensearch: 47 | ... 48 | index: "histogram_metrics" 49 | ``` 50 | 51 | In addition to this example, you can also use the **Log to metric pipeline** blueprint\. For more information about blueprints, see [Using blueprints to create a pipeline](creating-pipeline.md#pipeline-blueprint)\. -------------------------------------------------------------------------------- /doc_source/use-cases-overview.md: -------------------------------------------------------------------------------- 1 | # Use cases for Amazon OpenSearch Ingestion 2 | 3 | This chapter demonstrates some common use cases for Amazon OpenSearch Ingestion\. This list is not exhaustive\. For the full capabilities of each supported plugin, see [Sources](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sources/sources/), [Processors](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/processors/processors/), and [Sinks](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sinks/sinks/) in the Data Prepper documentation\. 4 | 5 | **Topics** 6 | + [Grok pattern matching with Amazon OpenSearch Ingestion](use-cases-pattern-matching.md) 7 | + [Log enrichment with Amazon OpenSearch Ingestion](use-cases-log-enrichment.md) 8 | + [Event aggregation with Amazon OpenSearch Ingestion](use-cases-log-aggregation.md) 9 | + [Deriving metrics from logs with Amazon OpenSearch Ingestion](use-cases-metrics-logs.md) 10 | + [Trace Analytics with Amazon OpenSearch Ingestion](use-cases-trace-analytics.md) 11 | + [Deriving metrics from traces with Amazon OpenSearch Ingestion](use-cases-trace-metrics.md) 12 | + [Anomaly detection with Amazon OpenSearch Ingestion](use-cases-anomaly-detection.md) 13 | + [Sampling with Amazon OpenSearch Ingestion](use-cases-sampling.md) 14 | + [Selective download with Amazon OpenSearch Ingestion](use-cases-s3-select.md) -------------------------------------------------------------------------------- /doc_source/use-cases-s3-select.md: -------------------------------------------------------------------------------- 1 | # Selective download with Amazon OpenSearch Ingestion 2 | 3 | If your pipeline uses an [S3 source](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sources/s3/), you can use SQL expressios to perform filtering and computations on the contents of S3 objects before ingesting them into a pipeline\. 4 | 5 | The `s3_select` option supports objects in Parquet format\. It also works with objects that are compressed with GZIP or BZIP2 \(for CSV and JSON objects only\), and supports columnar compression for Parquet using GZIP and Snappy\. 6 | 7 | The following example pipeline downloads data in incoming S3 objects, encoded in Parquet format: 8 | 9 | ``` 10 | pipeline: 11 | source: 12 | s3: 13 | s3_select: 14 | expression: "select * from s3object s" 15 | input_serialization: parquet 16 | notification_type: "sqs" 17 | ... 18 | ``` 19 | 20 | The following example downloads only the first 1,000 records in the objects: 21 | 22 | ``` 23 | pipeline: 24 | source: 25 | s3: 26 | s3_select: 27 | expression: "select * from s3object s LIMIT 10000" 28 | input_serialization: parquet 29 | notification_type: "sqs" 30 | ... 31 | ``` 32 | 33 | The following example checks for the minimum and maximum value of `data_value` before ingesting events into the pipeline: 34 | 35 | ``` 36 | pipeline: 37 | source: 38 | s3: 39 | s3_select: 40 | expression: "select s.* from s3object s where s.data_value > 200 and s.data_value < 500 " 41 | input_serialization: parquet 42 | notification_type: "sqs" 43 | ... 44 | ``` 45 | 46 | In addition to these examples, you can also use the **S3 select pipeline** blueprint\. For more information about blueprints, see [Using blueprints to create a pipeline](creating-pipeline.md#pipeline-blueprint)\. 47 | 48 | For more information, see the following resources: 49 | + [Filtering and retrieving data using Amazon S3 Select](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/selecting-content-from-objects.html) 50 | + [SQL reference for Amazon S3 Select](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/s3-select-sql-reference.html) -------------------------------------------------------------------------------- /doc_source/use-cases-sampling.md: -------------------------------------------------------------------------------- 1 | # Sampling with Amazon OpenSearch Ingestion 2 | 3 | Amazon OpenSearch Ingestion provides the following sampling capabilities\. In addition to these examples, you can also use the **Apache log sampling** blueprint\. For more information about blueprints, see [Using blueprints to create a pipeline](creating-pipeline.md#pipeline-blueprint)\. 4 | 5 | **Topics** 6 | + [Time sampling](#use-cases-sampling-time) 7 | + [Percentage sampling](#use-cases-sampling-percentage) 8 | + [Tail sampling](#use-cases-sampling) 9 | 10 | ## Time sampling 11 | 12 | You can use the `rate_limiter` action within the [Aggregate processor](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/processors/aggregate/) to limit the number of events that can be processed per second\. You can choose to either drop excess events or carry them forward to the next time period\. 13 | 14 | In this example, only 100 events per second with a status code of `200` are sent to the sink from a given IP address\. It drops all excess events from the configured time window\. 15 | 16 | ``` 17 | ... 18 | processor: 19 | - aggregate: 20 | identification_keys: ["clientip"] 21 | action: 22 | rate_limiter: 23 | events_per_second: 100 24 | when_exceeds: drop 25 | when: "/status == 200" 26 | ... 27 | ``` 28 | 29 | If you instead set the `when_exceeds` option to `block`, the processor will process excess events in the next time window\. 30 | 31 | ## Percentage sampling 32 | 33 | Use the `percent_sampler` action within the Aggregate processor to limit the number of events that are sent to a sink\. All excess events will be dropped\. 34 | 35 | In this example, only 20 percent of events with a status code of `200` are sent to the sink from a given IP address: 36 | 37 | ``` 38 | ... 39 | processor: 40 | - aggregate: 41 | identification_keys: ["clientip"] 42 | duration : 43 | action: 44 | percent_sampler: 45 | percent: 20 46 | when: "/status == 200" 47 | ... 48 | ``` 49 | 50 | ## Tail sampling 51 | 52 | Use the `tail_sampler` action within the Aggregate processor to sample events based on a set of defined policies\. This action waits for an aggregation to complete across different aggregation periods based on the configured wait period\. When an aggregation is complete, and if it matches the specific error condition, it's sent to the sink\. Otherwise, only a configured percentage of events are sent to the sink\. 53 | 54 | The following example pipeline sends all OpenTelemetry traces with an error condition status of `2` to the sink\. It only sends 20% of the traces that don't match this error condition to the sink\. 55 | 56 | ``` 57 | ... 58 | processor: 59 | - aggregate: 60 | identification_keys: ["traceId"] 61 | action: 62 | tail_sampler: 63 | percent: 20 64 | wait_period: "10s" 65 | condition: "/status == 2" 66 | 67 | ... 68 | ``` 69 | 70 | If you set the error condition to `false` or don't include it, only a the configured percentage of events is allowed to pass through, determined by a probabilistic outcome\. 71 | 72 | Because it's difficult to determine exactly when tail sampling should occur, you can use the `wait_period` option to measure the idle time after the last event was received\. -------------------------------------------------------------------------------- /doc_source/use-cases-trace-analytics.md: -------------------------------------------------------------------------------- 1 | # Trace Analytics with Amazon OpenSearch Ingestion 2 | 3 | You can use Amazon OpenSearch Ingestion to collect OpenTelemetry trace data and transform it for use in OpenSearch Service\. The following example pipeline uses three sub\-pipelines to monitor Trace Analytics: `entry-pipeline`, `span-pipeline`, and `service-map-pipeline`\. 4 | 5 | ## OpenTelemetry trace source 6 | 7 | The [Otel trace source](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sources/otel-trace/) plugin accepts trace data from the [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/)\. The plugin follows the [OpenTelemetry Protocol](https://opentelemetry.io/docs/reference/specification/protocol/) and officially supports industry\-standard encryption HTTPS\. 8 | 9 | ## Processors 10 | 11 | You can use the following processors for Trace Analytics: 12 | + [OTel trace](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/processors/otel-trace-raw/) – Receives a collection of span records from the source and performs stateful processing, extraction, and completion of fields\. 13 | + [OTel trace group](https://github.com/opensearch-project/data-prepper/tree/main/data-prepper-plugins/otel-trace-group-processor) – Fills in missing trace group fields in the collection of span records\. 14 | + [Service\-map](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/processors/service-map-stateful/) – Performs preprocessing for trace data and builds metadata to display service\-map dashboards\. 15 | 16 | ## OpenSearch sink 17 | 18 | The [OpenSearch sink](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sinks/opensearch/) plugin provides indexes and index templates that are specific to Trace Analytics\. The following OpenSearch indexes are specific to Trace Analytics: 19 | + `otel-v1-apm-span` – Stores the output from the OTel trace processor\. 20 | + `otel-v1-apm-service-map` – Stores the output from the Service\-map processor\. 21 | 22 | ## Pipeline configuration 23 | 24 | The following example pipeline supports [Observability for OpenSearch Dashboards](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/observability.html)\. The first sub\-pipeline \(`entry-pipeline`\) receives data from the OpenTelemetry Collector and uses two other sub\-pipelines as sinks\. 25 | 26 | The `span-pipeline` sub\-pipeline parses the trace data and enriches and ingests the span documents into a span index\. The `service-map-pipeline` sub\-pipeline aggregates traces into a service map and writes documents to a service map index\. 27 | 28 | ``` 29 | version: "2" 30 | entry-pipeline: 31 | source: 32 | otel_trace_source: 33 | # Provide the path for ingestion. This will be the endpoint URI path in the OpenTelemetry Exporter configuration. 34 | # ${pipelineName} will be replaced with the sub-pipeline name. In this case it would be "/entry-pipeline/v1/traces". 35 | path: "/${pipelineName}/v1/traces" 36 | processor: 37 | - trace_peer_forwarder 38 | sink: 39 | - pipeline: 40 | name: "span-pipeline" 41 | - pipeline: 42 | name: "service-map-pipeline" 43 | 44 | span-pipeline: 45 | source: 46 | pipeline: 47 | name: "entry-pipeline" 48 | processor: 49 | - otel_traces 50 | sink: 51 | - opensearch: 52 | ... 53 | index_type: trace-analytics-raw 54 | 55 | service-map-pipeline: 56 | source: 57 | pipeline: 58 | name: "entry-pipeline" 59 | processor: 60 | - service_map 61 | sink: 62 | - opensearch: 63 | ... 64 | index_type: trace-analytics-service-map 65 | ``` 66 | 67 | You must run the OpenTelemetry Collector in your environment to send data to the ingestion endpoint\. For another example pipeline, see the **Trace Analytics pipeline** blueprint\. For more information, see [Using blueprints to create a pipeline](creating-pipeline.md#pipeline-blueprint)\. -------------------------------------------------------------------------------- /doc_source/use-cases-trace-metrics.md: -------------------------------------------------------------------------------- 1 | # Deriving metrics from traces with Amazon OpenSearch Ingestion 2 | 3 | You can use Amazon OpenSearch Ingestion to derive metrics from OpenTelemetry traces\. The following example pipeline receives incoming traces and extracts a metric called `durationInNanos`, aggregated over a tumbling window of 30 seconds\. It then derives a histogram from the incoming traces\. 4 | 5 | The pipeline contains the following sub\-pipelines: 6 | + `entry-pipeline` – Receives trace data from the OpenTelemetry collector and forwards it to the `trace_to_metrics_pipeline` sub\-pipeline\. 7 | + `trace-to-metrics-pipeline` – Receives the trace data from the `entry-pipeline` sub\-pipeline, aggregates it, and derives a histogram of `durationInNanos` from the traces based on the value of the `serviceName` field\. It then sends the derived metrics to the OpenSearch index called `metrics_for_traces`\. 8 | 9 | ``` 10 | version: "2" 11 | entry-pipeline: 12 | source: 13 | otel_trace_source: 14 | # Provide the path for ingestion. ${pipelineName} will be replaced with sub-pipeline name. 15 | # In this case it would be "/entry-pipeline/v1/traces". This will be endpoint URI path in OpenTelemetry Exporter configuration. 16 | path: "/${pipelineName}/v1/traces" 17 | sink: 18 | - pipeline: 19 | name: "trace-to-metrics-pipeline" 20 | 21 | trace-to-metrics-pipeline: 22 | source: 23 | pipeline: 24 | name: "entry-pipeline" 25 | processor: 26 | - aggregate: 27 | # Pick the required identification keys 28 | identification_keys: ["serviceName"] 29 | action: 30 | histogram: 31 | # Pick the appropriate values for each of the following fields 32 | key: "durationInNanos" 33 | record_minmax: true 34 | units: "seconds" 35 | buckets: [0, 10000000, 50000000, 100000000] 36 | # Specify an aggregation period 37 | group_duration: "30s" 38 | sink: 39 | - opensearch: 40 | ... 41 | index: "metrics_for_traces" 42 | ``` 43 | 44 | For another example pipeline, see the **Trace to metric anomaly pipeline** blueprint\. For more information about blueprints, see [Using blueprints to create a pipeline](creating-pipeline.md#pipeline-blueprint)\. -------------------------------------------------------------------------------- /sample_code/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | 3 | -------------------------------------------------------------------------------- /sample_code/java/aws-request-signing-apache-interceptor/NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | -------------------------------------------------------------------------------- /sample_code/java/aws-request-signing-apache-interceptor/README.md: -------------------------------------------------------------------------------- 1 | To run this sample, modify `host` and `region` in [IndexDocument.java](src/main/java/com/amazonaws/samples/IndexDocument.java) and [BulkIndexDocuments.java](src/main/java/com/amazonaws/samples/BulkIndexDocuments.java), and set AWS credentials (e.g. via `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` variables). 2 | 3 | ``` 4 | mvn compile exec:java -Dexec.mainClass="com.amazonaws.samples.IndexDocument" 5 | mvn compile exec:java -Dexec.mainClass="com.amazonaws.samples.BulkIndexDocuments" 6 | ``` 7 | -------------------------------------------------------------------------------- /sample_code/java/aws-request-signing-apache-interceptor/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.amazonaws 5 | amazon-opensearch-docs-sample 6 | jar 7 | 0.1.0 8 | amazon-opensearch-sample 9 | http://maven.apache.org 10 | 11 | 12 | https://github.com/awsdocs/amazon-opensearch-service-developer-guide/tree/master/sample_code/java/aws-request-signing-apache-interceptor 13 | 14 | 15 | 16 | 17 | Apache License, Version 2.0 18 | https://aws.amazon.com/apache2.0 19 | repo 20 | 21 | 22 | 23 | 24 | 1.8 25 | 1.8 26 | 2.17.4 27 | 28 | 29 | 30 | 31 | org.junit 32 | junit-bom 33 | 5.7.2 34 | pom 35 | import 36 | 37 | 38 | 39 | software.amazon.awssdk 40 | bom 41 | ${aws-java-sdk.version} 42 | pom 43 | import 44 | 45 | 46 | 47 | 48 | 49 | junit 50 | junit 51 | 4.13.2 52 | test 53 | 54 | 55 | org.junit.jupiter 56 | junit-jupiter-api 57 | test 58 | 59 | 60 | org.junit.jupiter 61 | junit-jupiter-engine 62 | test 63 | 64 | 65 | software.amazon.awssdk 66 | auth 67 | 68 | 69 | org.opensearch.client 70 | opensearch-rest-client 71 | 1.3.1 72 | 73 | 74 | org.opensearch.client 75 | opensearch-java 76 | 0.1.0 77 | 78 | 79 | org.opensearch.client 80 | opensearch-rest-high-level-client 81 | 1.3.1 82 | 83 | 84 | com.amazonaws 85 | aws-lambda-java-core 86 | 1.2.1 87 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /sample_code/java/aws-request-signing-apache-interceptor/src/main/java/com/amazonaws/samples/BulkIndexDocuments.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.samples; 2 | 3 | import com.amazonaws.http.AwsRequestSigningApacheInterceptor; 4 | import org.apache.http.HttpHost; 5 | import org.apache.http.HttpRequestInterceptor; 6 | import org.opensearch.OpenSearchStatusException; 7 | import org.opensearch.action.bulk.BulkItemResponse; 8 | import org.opensearch.action.bulk.BulkRequest; 9 | import org.opensearch.action.bulk.BulkResponse; 10 | import org.opensearch.action.index.IndexRequest; 11 | import org.opensearch.action.index.IndexResponse; 12 | import org.opensearch.client.RequestOptions; 13 | import org.opensearch.client.RestClient; 14 | import org.opensearch.client.RestHighLevelClient; 15 | import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; 16 | import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; 17 | import software.amazon.awssdk.auth.signer.Aws4Signer; 18 | 19 | import java.io.IOException; 20 | import java.util.HashMap; 21 | import java.util.Map; 22 | 23 | public class BulkIndexDocuments { 24 | 25 | private static final String serviceName = "es"; 26 | private static final String region = "us-west-2"; 27 | private static final String endpoint = "https://search-...us-west-2.es.amazonaws.com"; 28 | private static final String type = "_doc"; 29 | 30 | static final AwsCredentialsProvider credentialsProvider = DefaultCredentialsProvider.create(); 31 | 32 | public static void main(String[] args) throws IOException { 33 | 34 | RestHighLevelClient client = getOpenSearchClient(); 35 | 36 | try { 37 | String index = "java-client-test-index"; 38 | 39 | // Create a document that simulates a simple log line from a web server 40 | Map document = new HashMap<>(); 41 | document.put("method", "GET"); 42 | document.put("client_ip_address", "123.456.78.90"); 43 | document.put("timestamp", "10/Oct/2000:14:56:14 -0700"); 44 | 45 | System.out.println("Demoing a single index request:"); 46 | String id = "1"; 47 | IndexRequest indexRequest = new IndexRequest(index, type, id).source(document); 48 | IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT); 49 | System.out.println(indexResponse.toString()); 50 | 51 | System.out.println("Demoing a 1 MB bulk request:"); 52 | BulkRequest bulkRequest = new BulkRequest(); 53 | 54 | // Add documents (the simple log line from earlier) to the request until it 55 | // exceeds 1 MB 56 | while (bulkRequest.estimatedSizeInBytes() < 1000000) { 57 | // By not specifying an ID, these documents get auto-assigned IDs 58 | bulkRequest.add(new IndexRequest(index, type).source(document)); 59 | } 60 | 61 | try { 62 | // Send the request and get the response 63 | BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT); 64 | 65 | // Check the response for failures 66 | if (bulkResponse.hasFailures()) { 67 | System.out.println("Encountered failures:"); 68 | for (BulkItemResponse bulkItemResponse : bulkResponse) { 69 | if (bulkItemResponse.isFailed()) { 70 | System.out.println(bulkItemResponse.getFailureMessage()); 71 | } 72 | } 73 | } else { 74 | System.out.println("No failures!"); 75 | // Uncomment these lines for a line-by-line summary 76 | // for (BulkItemResponse bulkItemResponse : bulkResponse) { 77 | // System.out.println(bulkItemResponse.getResponse().toString()); 78 | // } 79 | } 80 | } 81 | 82 | // Usually happens when the request size is too large 83 | catch (OpenSearchStatusException e) { 84 | System.out.println("Encountered exception:"); 85 | System.out.println(e); 86 | } 87 | } finally { 88 | client.close(); 89 | } 90 | } 91 | 92 | // Adds the interceptor to the OpenSearch REST client 93 | public static RestHighLevelClient getOpenSearchClient() { 94 | Aws4Signer signer = Aws4Signer.create(); 95 | HttpRequestInterceptor interceptor = new AwsRequestSigningApacheInterceptor(serviceName, signer, 96 | credentialsProvider, region); 97 | return new RestHighLevelClient(RestClient.builder(HttpHost.create(endpoint)) 98 | .setHttpClientConfigCallback(hacb -> hacb.addInterceptorLast(interceptor))); 99 | } 100 | } -------------------------------------------------------------------------------- /sample_code/java/aws-request-signing-apache-interceptor/src/main/java/com/amazonaws/samples/IndexDocument.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.samples; 2 | 3 | import com.amazonaws.http.AwsRequestSigningApacheInterceptor; 4 | import org.apache.http.HttpEntity; 5 | import org.apache.http.HttpHost; 6 | import org.apache.http.HttpRequestInterceptor; 7 | import org.apache.http.entity.ContentType; 8 | import org.apache.http.nio.entity.NStringEntity; 9 | import org.opensearch.client.Request; 10 | import org.opensearch.client.Response; 11 | import org.opensearch.client.RestClient; 12 | import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; 13 | import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; 14 | import software.amazon.awssdk.auth.signer.Aws4Signer; 15 | import java.io.IOException; 16 | 17 | public class IndexDocument { 18 | 19 | private static String serviceName = "es"; 20 | private static String region = "us-west-2"; 21 | private static String endpoint = "https://search-...us-west-2.es.amazonaws.com"; 22 | 23 | private static String sampleDocument = "{" + "\"title\":\"Walk the Line\"," + "\"director\":\"James Mangold\"," 24 | + "\"year\":\"2005\"}"; 25 | private static String indexingPath = "/my-index/_doc"; 26 | 27 | static final AwsCredentialsProvider credentialsProvider = DefaultCredentialsProvider.create(); 28 | 29 | public static void main(String[] args) throws IOException { 30 | RestClient client = getOpenSearchClient(serviceName, region); 31 | 32 | try { 33 | // Index a document 34 | HttpEntity entity = new NStringEntity(sampleDocument, ContentType.APPLICATION_JSON); 35 | String id = "1"; 36 | Request request = new Request("PUT", indexingPath + "/" + id); 37 | request.setEntity(entity); 38 | 39 | // Using a String instead of an HttpEntity sets Content-Type to application/json 40 | // automatically. 41 | // request.setJsonEntity(sampleDocument); 42 | 43 | Response response = client.performRequest(request); 44 | System.out.println(response.toString()); 45 | } finally { 46 | client.close(); 47 | } 48 | } 49 | 50 | // Adds the interceptor to the OpenSearch REST client 51 | public static RestClient getOpenSearchClient(String serviceName, String region) { 52 | Aws4Signer signer = Aws4Signer.create(); 53 | HttpRequestInterceptor interceptor = new AwsRequestSigningApacheInterceptor(serviceName, signer, 54 | credentialsProvider, region); 55 | return RestClient.builder(HttpHost.create(endpoint)) 56 | .setHttpClientConfigCallback(hacb -> hacb.addInterceptorLast(interceptor)).build(); 57 | } 58 | } -------------------------------------------------------------------------------- /sample_code/java/opensearch-java-aws-sdk2-transport/NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | -------------------------------------------------------------------------------- /sample_code/java/opensearch-java-aws-sdk2-transport/README.md: -------------------------------------------------------------------------------- 1 | To run this sample, modify `host` and `region` in [IndexDocument.java](src/main/java/com/amazonaws/samples/IndexDocument.java) and [BulkIndexDocuments.java](src/main/java/com/amazonaws/samples/BulkIndexDocuments.java), and set AWS credentials (e.g. via `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` variables). 2 | 3 | ``` 4 | mvn compile exec:java -Dexec.mainClass="com.amazonaws.samples.GetVersion" 5 | mvn compile exec:java -Dexec.mainClass="com.amazonaws.samples.IndexDocument" 6 | ``` 7 | -------------------------------------------------------------------------------- /sample_code/java/opensearch-java-aws-sdk2-transport/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | com.amazonaws 4 | amazon-opensearch-docs-sample 5 | jar 6 | 0.1.0 7 | amazon-opensearch-sample 8 | http://maven.apache.org 9 | 10 | 11 | https://github.com/awsdocs/amazon-opensearch-service-developer-guide/tree/master/sample_code/java/opensearch-java-aws-sdk2-transport 12 | 13 | 14 | 15 | 16 | Apache License, Version 2.0 17 | https://aws.amazon.com/apache2.0 18 | repo 19 | 20 | 21 | 22 | 23 | 1.8 24 | 1.8 25 | 2.17.219 26 | 27 | 28 | 29 | 30 | software.amazon.awssdk 31 | bom 32 | ${aws-java-sdk.version} 33 | pom 34 | import 35 | 36 | 37 | 38 | 39 | 40 | software.amazon.awssdk 41 | auth 42 | 43 | 44 | org.opensearch.client 45 | opensearch-java 46 | 2.1.0 47 | 48 | 49 | software.amazon.awssdk 50 | apache-client 51 | 2.17.224 52 | 53 | 54 | -------------------------------------------------------------------------------- /sample_code/java/opensearch-java-aws-sdk2-transport/src/main/java/com/amazonaws/samples/GetVersion.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.samples; 2 | 3 | import java.io.IOException; 4 | 5 | import org.opensearch.client.opensearch.OpenSearchClient; 6 | import org.opensearch.client.opensearch.core.InfoResponse; 7 | import org.opensearch.client.transport.aws.AwsSdk2Transport; 8 | import org.opensearch.client.transport.aws.AwsSdk2TransportOptions; 9 | 10 | import software.amazon.awssdk.http.SdkHttpClient; 11 | import software.amazon.awssdk.http.apache.ApacheHttpClient; 12 | import software.amazon.awssdk.regions.Region; 13 | 14 | public class GetVersion { 15 | 16 | private static Region region = Region.US_WEST_2; 17 | private static final String endpoint = "search-...us-west-2.es.amazonaws.com"; 18 | 19 | public static void main(String[] args) throws IOException { 20 | SdkHttpClient httpClient = ApacheHttpClient.builder().build(); 21 | try { 22 | 23 | OpenSearchClient client = new OpenSearchClient( 24 | new AwsSdk2Transport( 25 | httpClient, 26 | endpoint, 27 | region, 28 | AwsSdk2TransportOptions.builder().build())); 29 | 30 | InfoResponse info = client.info(); 31 | System.out.println(info.version().distribution() + ": " + info.version().number()); 32 | } finally { 33 | httpClient.close(); 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /sample_code/java/opensearch-java-aws-sdk2-transport/src/main/java/com/amazonaws/samples/IndexData.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.samples; 2 | 3 | public class IndexData { 4 | private String firstName; 5 | private String lastName; 6 | 7 | public IndexData() { 8 | 9 | } 10 | 11 | public IndexData(String firstName, String lastName) { 12 | this.firstName = firstName; 13 | this.lastName = lastName; 14 | } 15 | 16 | public String getFirstName() { 17 | return firstName; 18 | } 19 | 20 | public void setFirstName(String firstName) { 21 | this.firstName = firstName; 22 | } 23 | 24 | public String getLastName() { 25 | return lastName; 26 | } 27 | 28 | public void setLastName(String lastName) { 29 | this.lastName = lastName; 30 | } 31 | 32 | @Override 33 | public String toString() { 34 | return String.format("IndexData{first name='%s', last name='%s'}", firstName, lastName); 35 | } 36 | } -------------------------------------------------------------------------------- /sample_code/java/opensearch-java-aws-sdk2-transport/src/main/java/com/amazonaws/samples/IndexDocument.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.samples; 2 | 3 | import java.io.IOException; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | import org.opensearch.client.opensearch.OpenSearchClient; 8 | import org.opensearch.client.opensearch.core.IndexRequest; 9 | import org.opensearch.client.opensearch.core.SearchResponse; 10 | import org.opensearch.client.opensearch.indices.CreateIndexRequest; 11 | import org.opensearch.client.opensearch.indices.DeleteIndexRequest; 12 | import org.opensearch.client.opensearch.indices.IndexSettings; 13 | import org.opensearch.client.opensearch.indices.PutIndicesSettingsRequest; 14 | import org.opensearch.client.transport.aws.AwsSdk2Transport; 15 | import org.opensearch.client.transport.aws.AwsSdk2TransportOptions; 16 | 17 | import software.amazon.awssdk.http.SdkHttpClient; 18 | import software.amazon.awssdk.http.apache.ApacheHttpClient; 19 | import software.amazon.awssdk.regions.Region; 20 | 21 | public class IndexDocument { 22 | 23 | private static Region region = Region.US_WEST_2; 24 | private static final String endpoint = "search-...us-west-2.es.amazonaws.com"; 25 | 26 | public static void main(String[] args) throws IOException, InterruptedException { 27 | SdkHttpClient httpClient = ApacheHttpClient.builder().build(); 28 | try { 29 | 30 | OpenSearchClient client = new OpenSearchClient( 31 | new AwsSdk2Transport( 32 | httpClient, 33 | endpoint, 34 | region, 35 | AwsSdk2TransportOptions.builder().build())); 36 | 37 | // create the index 38 | String index = "sample-index"; 39 | 40 | CreateIndexRequest createIndexRequest = new CreateIndexRequest.Builder().index(index).build(); 41 | client.indices().create(createIndexRequest); 42 | 43 | // add settings to the index 44 | IndexSettings indexSettings = new IndexSettings.Builder().autoExpandReplicas("0-all").build(); 45 | PutIndicesSettingsRequest putSettingsRequest = new PutIndicesSettingsRequest.Builder() 46 | .index(index) 47 | .settings(indexSettings) 48 | .build(); 49 | client.indices().putSettings(putSettingsRequest); 50 | 51 | // index data 52 | IndexData indexData = new IndexData("Bruce", "Willis"); 53 | IndexRequest indexDataRequest = new IndexRequest.Builder() 54 | .index(index) 55 | .id("1") 56 | .document(indexData) 57 | .build(); 58 | client.index(indexDataRequest); 59 | 60 | // alternative using a hash map 61 | Map document = new HashMap<>(); 62 | document.put("firstName", "Michael"); 63 | document.put("lastName", "Douglas"); 64 | IndexRequest documentIndexRequest = new IndexRequest.Builder() 65 | .index(index) 66 | .id("2") 67 | .document(document) 68 | .build(); 69 | client.index(documentIndexRequest); 70 | 71 | // wait for the document to index 72 | Thread.sleep(3000); 73 | 74 | // search for the document 75 | SearchResponse searchResponse = client.search(s -> s.index(index), IndexData.class); 76 | for (int i = 0; i < searchResponse.hits().hits().size(); i++) { 77 | System.out.println(searchResponse.hits().hits().get(i).source().toString()); 78 | } 79 | 80 | // delete the document 81 | client.delete(b -> b.index(index).id("1")); 82 | 83 | // delete the index 84 | DeleteIndexRequest deleteRequest = new DeleteIndexRequest.Builder().index(index).build(); 85 | client.indices().delete(deleteRequest); 86 | 87 | } finally { 88 | httpClient.close(); 89 | } 90 | } 91 | } --------------------------------------------------------------------------------