├── src └── com │ └── amazon │ └── kinesis │ └── streaming │ └── agent │ ├── versionInfo.properties │ ├── package-info.java │ ├── config │ ├── package-info.java │ ├── ConfigurationException.java │ ├── PathConverter.java │ ├── StringConverterWrapper.java │ ├── ConfigurationConverter.java │ ├── BooleanConverter.java │ ├── AgentOptions.java │ └── AgentConfiguration.java │ ├── metrics │ ├── package-info.java │ ├── IMetricsContext.java │ ├── NullMetricsFactory.java │ ├── IMetricsFactory.java │ ├── LogMetricsFactory.java │ ├── NullMetricsScope.java │ ├── ICWMetricsPublisher.java │ ├── CWMetricKey.java │ ├── NestedMetricsScope.java │ ├── CompositeMetricsFactory.java │ ├── CWMetricsScope.java │ ├── CompositeMetricsScope.java │ ├── LogMetricsScope.java │ ├── MetricDatumWithKey.java │ ├── IMetricsScope.java │ ├── DefaultCWMetricsPublisher.java │ ├── AccumulatingMetricsScope.java │ ├── AbstractMetricsScope.java │ ├── Metrics.java │ ├── CWMetricsFactory.java │ ├── MetricAccumulatingQueue.java │ └── CWPublisherRunnable.java │ ├── IHeartbeatProvider.java │ ├── tailing │ ├── checkpoints │ │ ├── package-info.java │ │ ├── FileCheckpoint.java │ │ ├── FileCheckpointStore.java │ │ └── Checkpointer.java │ ├── package-info.java │ ├── FirehoseRecord.java │ ├── ISplitter.java │ ├── IRecord.java │ ├── KinesisParser.java │ ├── FirehoseParser.java │ ├── FirehoseConstants.java │ ├── SingleLineSplitter.java │ ├── KinesisConstants.java │ ├── AbstractSender.java │ ├── BufferSendResult.java │ ├── FileFlowFactory.java │ ├── FileId.java │ ├── KinesisRecord.java │ ├── ISender.java │ ├── RegexSplitter.java │ ├── AbstractRecord.java │ ├── TrackedFileList.java │ ├── SourceFile.java │ ├── RecordBuffer.java │ ├── IParser.java │ ├── FirehoseFileFlow.java │ ├── KinesisFileFlow.java │ └── AsyncPublisherService.java │ ├── CustomLog4jFallbackPatterLayout.java │ ├── Constants.java │ ├── AgentAWSCredentialsProviderChain.java │ ├── HeartbeatService.java │ ├── AgentAWSCredentialsProvider.java │ ├── CustomLog4jFallbackErrorHandler.java │ ├── custom.log4j.xml │ └── ByteBufferInputStream.java ├── support ├── aws-kinesis-agent.cron └── aws-kinesis-agent.sysconfig ├── NOTICE.txt ├── configuration └── release │ └── aws-kinesis-agent.json ├── bin ├── aws-kinesis-agent-babysit ├── start-aws-kinesis-agent └── aws-kinesis-agent ├── LICENSE.txt ├── README.md ├── setup └── pom.xml /src/com/amazon/kinesis/streaming/agent/versionInfo.properties: -------------------------------------------------------------------------------- 1 | version = 1.0 -------------------------------------------------------------------------------- /support/aws-kinesis-agent.cron: -------------------------------------------------------------------------------- 1 | * * * * * root /usr/bin/aws-kinesis-agent-babysit > /dev/null 2>&1 2 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | AWSKinesisStreamingDataAgent 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. -------------------------------------------------------------------------------- /support/aws-kinesis-agent.sysconfig: -------------------------------------------------------------------------------- 1 | # Set AWS credentials for accessing Amazon Kinesis Stream and Amazon Kinesis Firehose 2 | AWS_ACCESS_KEY_ID= 3 | AWS_SECRET_ACCESS_KEY= 4 | AWS_DEFAULT_REGION= -------------------------------------------------------------------------------- /configuration/release/aws-kinesis-agent.json: -------------------------------------------------------------------------------- 1 | { 2 | "cloudwatch.emitMetrics": true, 3 | "kinesis.endpoint": "", 4 | "firehose.endpoint": "", 5 | 6 | "flows": [ 7 | { 8 | "filePattern": "/tmp/app.log*", 9 | "kinesisStream": "yourkinesisstream", 10 | "partitionKeyOption": "RANDOM" 11 | }, 12 | { 13 | "filePattern": "/tmp/app.log*", 14 | "deliveryStream": "yourdeliverystream" 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | 15 | package com.amazon.kinesis.streaming.agent; -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/config/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | /** 15 | * Configuration utilities for the Agent. 16 | */ 17 | package com.amazon.kinesis.streaming.agent.config; -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | /** 15 | * Utilities to collect and emit metrics. 16 | */ 17 | package com.amazon.kinesis.streaming.agent.metrics; -------------------------------------------------------------------------------- /bin/aws-kinesis-agent-babysit: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # The cron-babysit periodically checks the status of the agent, and restart it if stopped abnormally 4 | 5 | DAEMON_NAME=aws-kinesis-agent 6 | PIDFILE=/var/run/$DAEMON_NAME.pid 7 | SERVICE="/sbin/service $DAEMON_NAME" 8 | 9 | function get_agent_pid() { 10 | echo $(ps --ppid $(cat $PIDFILE 2>/dev/null) ho pid 2>/dev/null) 11 | } 12 | 13 | function start_agent() { 14 | $SERVICE restart || exit 1 15 | sleep 3 16 | [[ -n $(get_agent_pid) ]] || exit 1 17 | } 18 | 19 | # Check if PID file exists. 20 | # If it does not, it means either the agent was never started or it was stopped by the user. 21 | [[ -f $PIDFILE ]] || exit 0 22 | 23 | # Check if the child Java process is alive. If not, we should start 24 | [[ -n $(get_agent_pid) ]] || start_agent 25 | 26 | exit 0 -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/IHeartbeatProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent; 15 | 16 | public interface IHeartbeatProvider { 17 | Object heartbeat(AgentContext agent); 18 | } 19 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/checkpoints/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | /** 15 | * Package that implements file checkpointing functionality. 16 | */ 17 | package com.amazon.kinesis.streaming.agent.tailing.checkpoints; -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | /** 15 | * Classes, interfaces and enums that implement the details of file tailing 16 | * logic. 17 | */ 18 | package com.amazon.kinesis.streaming.agent.tailing; -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/IMetricsContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | public interface IMetricsContext { 17 | /** 18 | * @return An instance of {@link IMetricsScope}. 19 | */ 20 | public abstract IMetricsScope beginScope(); 21 | } 22 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/NullMetricsFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | public class NullMetricsFactory implements IMetricsFactory { 17 | @Override 18 | public IMetricsScope createScope() { 19 | return new NullMetricsScope(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/CustomLog4jFallbackPatterLayout.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent; 15 | 16 | import org.apache.log4j.PatternLayout; 17 | 18 | public class CustomLog4jFallbackPatterLayout extends PatternLayout { 19 | @Override 20 | public String getHeader() { 21 | return CustomLog4jFallbackErrorHandler.getErrorHeader(); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/IMetricsFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | /** 17 | * Factory for {@link IMetricsScope} objects. 18 | */ 19 | public interface IMetricsFactory { 20 | /** 21 | * @return a new {@link IMetricsScope} object of the type constructed by 22 | * this factory. 23 | */ 24 | public IMetricsScope createScope(); 25 | } 26 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/LogMetricsFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | /** 17 | * An {@link IMetricsFactory} that creates {@link IMetricsScope} that output 18 | * to the logger. 19 | */ 20 | public class LogMetricsFactory implements IMetricsFactory { 21 | 22 | @Override 23 | public LogMetricsScope createScope() { 24 | return new LogMetricsScope(); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/NullMetricsScope.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | import java.util.Collections; 17 | import java.util.Set; 18 | 19 | import com.amazonaws.services.cloudwatch.model.Dimension; 20 | 21 | public class NullMetricsScope extends AbstractMetricsScope { 22 | @Override 23 | protected Set realGetDimensions() { 24 | return Collections.emptySet(); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/config/ConfigurationException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.config; 15 | 16 | /** 17 | * Raised if an error occurred during configuration. 18 | */ 19 | @SuppressWarnings("serial") 20 | public class ConfigurationException extends RuntimeException { 21 | public ConfigurationException(String message) { 22 | super(message); 23 | } 24 | 25 | public ConfigurationException(String message, Throwable cause) { 26 | super(message, cause); 27 | } 28 | } -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/Constants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent; 15 | 16 | import java.util.concurrent.TimeUnit; 17 | 18 | public class Constants { 19 | public static final char NEW_LINE = '\n'; 20 | public static final long DEFAULT_RETRY_INITIAL_BACKOFF_MILLIS = 100; 21 | public static final long DEFAULT_RETRY_MAX_BACKOFF_MILLIS = 10_000; 22 | public static final int DEFAULT_PUBLISH_QUEUE_CAPACITY = 100; 23 | public static final long DEFAULT_MAX_BUFFER_AGE_MILLIS = TimeUnit.MINUTES.toMillis(1); 24 | public static final long DEFAULT_WAIT_ON_FULL_PUBLISH_QUEUE_MILLIS = TimeUnit.MINUTES.toMillis(1); 25 | public static final long DEFAULT_WAIT_ON_EMPTY_PUBLISH_QUEUE_MILLIS = TimeUnit.MINUTES.toMillis(1); 26 | } 27 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/FirehoseRecord.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import java.nio.ByteBuffer; 17 | 18 | public class FirehoseRecord extends AbstractRecord { 19 | public FirehoseRecord(TrackedFile file, long offset, ByteBuffer data) { 20 | super(file, offset, data); 21 | } 22 | 23 | public FirehoseRecord(TrackedFile file, long offset, byte[] data) { 24 | super(file, offset, data); 25 | } 26 | 27 | @Override 28 | public long lengthWithOverhead() { 29 | return length() + FirehoseConstants.PER_RECORD_OVERHEAD_BYTES; 30 | } 31 | 32 | @Override 33 | protected int getMaxDataSize() { 34 | return FirehoseConstants.MAX_RECORD_SIZE_BYTES; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/ISplitter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import java.nio.ByteBuffer; 17 | 18 | /** 19 | * Encapsulates the logic to split a buffer into records. 20 | * Implementation will define what a record is and how it's delimited. 21 | */ 22 | public interface ISplitter { 23 | /** 24 | * Advances the buffer to the beginning of the next record, or to 25 | * the end of the buffer if a new record was not found. 26 | * 27 | * @param buffer 28 | * @return The position of the next record in the buffer, or {@code -1} 29 | * if the beginning of the record was not found before the end of 30 | * the buffer. 31 | */ 32 | public int locateNextRecord(ByteBuffer buffer); 33 | } 34 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/config/PathConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.config; 15 | 16 | import java.net.URI; 17 | import java.nio.file.Path; 18 | import java.nio.file.Paths; 19 | 20 | import javax.annotation.Nullable; 21 | 22 | import com.google.common.base.Function; 23 | 24 | /** 25 | * Function that converts an object into a {@link Path}. 26 | */ 27 | class PathConverter implements Function { 28 | 29 | @Override 30 | @Nullable 31 | public Path apply(@Nullable Object input) { 32 | if (input != null && !(input instanceof Path)) { 33 | if(input instanceof URI) 34 | return Paths.get((URI)input); 35 | else 36 | return Paths.get(input.toString()); 37 | } else 38 | return (Path) input; 39 | } 40 | } -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/IRecord.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import java.nio.ByteBuffer; 17 | 18 | public interface IRecord { 19 | public ByteBuffer data(); 20 | public long dataLength(); 21 | public long lengthWithOverhead(); 22 | public long length(); 23 | public TrackedFile file(); 24 | public long endOffset(); 25 | public long startOffset(); 26 | 27 | /** 28 | * This method should make sure the truncated record has the appropriate 29 | * terminator (e.g. newline). 30 | */ 31 | public void truncate(); 32 | 33 | /** 34 | * @return A string representation of the data in this record, encoded 35 | * with UTF-8; use for debugging only please. 36 | */ 37 | @Override 38 | public String toString(); 39 | } 40 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/ICWMetricsPublisher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | import java.util.List; 17 | 18 | /** 19 | * An ICWMetricsPublisher is a publisher that contains the logic to publish metrics. 20 | * 21 | * @param is a class that stores information about a MetricDatum. This is useful when wanting 22 | * to compare MetricDatums or aggregate similar MetricDatums. 23 | */ 24 | 25 | public interface ICWMetricsPublisher { 26 | 27 | /** 28 | * Given a list of MetricDatumWithKey, this method extracts the MetricDatum from each 29 | * MetricDatumWithKey and publishes those datums. 30 | * 31 | * @param dataToPublish a list containing all the MetricDatums to publish 32 | */ 33 | 34 | public void publishMetrics(List> dataToPublish); 35 | } 36 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/config/StringConverterWrapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.config; 15 | 16 | import javax.annotation.Nullable; 17 | 18 | import com.google.common.base.Function; 19 | 20 | /** 21 | * Wraps a String -> T function into an 22 | * Object -> T function by calling toString 23 | * on the value. 24 | * 25 | * @param the destination type. 26 | */ 27 | class StringConverterWrapper implements 28 | Function { 29 | private final Function delegate; 30 | 31 | public StringConverterWrapper(Function delegate) { 32 | this.delegate = delegate; 33 | } 34 | 35 | @Override 36 | @Nullable 37 | public T apply(@Nullable Object input) { 38 | return input != null ? this.delegate.apply(input.toString()) : null; 39 | } 40 | } -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/KinesisParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import java.nio.ByteBuffer; 17 | 18 | /** 19 | * Implementation of {@link AbstractParser} specific to Kinesis. 20 | */ 21 | public class KinesisParser extends AbstractParser { 22 | 23 | public KinesisParser(FileFlow flow) { 24 | super(flow); 25 | } 26 | 27 | public KinesisParser(FileFlow flow, int bufferSize) { 28 | super(flow, bufferSize); 29 | } 30 | 31 | @Override 32 | protected synchronized KinesisRecord buildRecord(TrackedFile recordFile, ByteBuffer data, long offset) { 33 | return new KinesisRecord(recordFile, offset, data); 34 | } 35 | 36 | @Override 37 | protected int getMaxRecordSize() { 38 | return KinesisConstants.MAX_RECORD_SIZE_BYTES; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/FirehoseParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import java.nio.ByteBuffer; 17 | 18 | /** 19 | * Implementation of {@link AbstractParser} specific to Firehose. 20 | */ 21 | public class FirehoseParser extends AbstractParser { 22 | 23 | public FirehoseParser(FileFlow flow) { 24 | super(flow); 25 | } 26 | 27 | public FirehoseParser(FileFlow flow, int bufferSize) { 28 | super(flow, bufferSize); 29 | } 30 | 31 | @Override 32 | protected synchronized FirehoseRecord buildRecord(TrackedFile recordFile, ByteBuffer data, long offset) { 33 | return new FirehoseRecord(recordFile, offset, data); 34 | } 35 | 36 | @Override 37 | protected int getMaxRecordSize() { 38 | return FirehoseConstants.MAX_RECORD_SIZE_BYTES; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/FirehoseConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import com.amazon.kinesis.streaming.agent.Constants; 17 | 18 | public final class FirehoseConstants extends Constants { 19 | public static final String DESTINATION_KEY = "deliveryStream"; 20 | 21 | public static final int PER_RECORD_OVERHEAD_BYTES = 0; 22 | public static final int MAX_RECORD_SIZE_BYTES = 40_960; 23 | public static final int PER_BUFFER_OVERHEAD_BYTES = 0; 24 | public static final int MAX_BATCH_PUT_SIZE_RECORDS = 500; 25 | public static final int MAX_BATCH_PUT_SIZE_BYTES = 4 * 1024 * 1024; 26 | public static final int MAX_BUFFER_SIZE_RECORDS = MAX_BATCH_PUT_SIZE_RECORDS; 27 | public static final int MAX_BUFFER_SIZE_BYTES = MAX_BATCH_PUT_SIZE_BYTES; 28 | public static final int DEFAULT_PARSER_BUFFER_SIZE_BYTES = MAX_BATCH_PUT_SIZE_BYTES; 29 | } 30 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/config/ConfigurationConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.config; 15 | 16 | import java.util.Map; 17 | 18 | import javax.annotation.Nullable; 19 | 20 | import com.google.common.base.Function; 21 | 22 | /** 23 | * Function that converts an object into a {@link Configuration}. 24 | */ 25 | class ConfigurationConverter implements Function { 26 | 27 | @SuppressWarnings("unchecked") 28 | @Override 29 | @Nullable 30 | public Configuration apply(@Nullable Object input) { 31 | if (input != null && !(input instanceof Configuration)) { 32 | if(input instanceof Map) 33 | return new Configuration((Map) input); 34 | else 35 | throw new ConfigurationException("Value is not a valid map: " + input.toString()); 36 | } else 37 | return (Configuration) input; 38 | } 39 | } -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/SingleLineSplitter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import java.nio.ByteBuffer; 17 | 18 | import com.amazon.kinesis.streaming.agent.ByteBuffers; 19 | 20 | /** 21 | * Returns one record per line. 22 | * TODO: Limitation of this splitter is that it requires a newline at the end 23 | * of the file, otherwise it will miss the last record. We should be able 24 | * to handle this at the level of the {@link IParser} implementation. 25 | * TODO: Should we parametrize the line delimiter? 26 | */ 27 | public class SingleLineSplitter implements ISplitter { 28 | public static final char LINE_DELIMITER = '\n'; 29 | 30 | @Override 31 | public int locateNextRecord(ByteBuffer buffer) { 32 | // TODO: Skip empty records, commented records, header lines, etc... 33 | // (based on FileFlow configuration?). 34 | return ByteBuffers.advanceBufferToNextLine(buffer); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/AgentAWSCredentialsProviderChain.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent; 15 | 16 | import com.amazon.kinesis.streaming.agent.config.AgentConfiguration; 17 | import com.amazonaws.auth.AWSCredentialsProviderChain; 18 | import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; 19 | import com.amazonaws.auth.InstanceProfileCredentialsProvider; 20 | import com.amazonaws.auth.SystemPropertiesCredentialsProvider; 21 | import com.amazonaws.auth.profile.ProfileCredentialsProvider; 22 | 23 | public class AgentAWSCredentialsProviderChain extends AWSCredentialsProviderChain { 24 | public AgentAWSCredentialsProviderChain(AgentConfiguration config) { 25 | super(new AgentAWSCredentialsProvider(config), 26 | new EnvironmentVariableCredentialsProvider(), 27 | new SystemPropertiesCredentialsProvider(), 28 | new ProfileCredentialsProvider(), 29 | new InstanceProfileCredentialsProvider()); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/KinesisConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import com.amazon.kinesis.streaming.agent.Constants; 17 | 18 | public class KinesisConstants extends Constants { 19 | public static final String DESTINATION_KEY = "kinesisStream"; 20 | public static final String PARTITION_KEY = "partitionKeyOption"; 21 | 22 | public static final int PER_RECORD_OVERHEAD_BYTES = 0; 23 | public static final int MAX_RECORD_SIZE_BYTES = 1024 * 1024; 24 | public static final int PER_BUFFER_OVERHEAD_BYTES = 0; 25 | public static final int MAX_PUT_RECORDS_SIZE_RECORDS = 500; 26 | public static final int MAX_PUT_RECORDS_SIZE_BYTES = 5 * 1024 * 1024; 27 | public static final int MAX_BUFFER_SIZE_RECORDS = MAX_PUT_RECORDS_SIZE_RECORDS; 28 | public static final int MAX_BUFFER_SIZE_BYTES = MAX_PUT_RECORDS_SIZE_BYTES; 29 | public static final int DEFAULT_PARSER_BUFFER_SIZE_BYTES = MAX_BUFFER_SIZE_BYTES; 30 | 31 | public static enum PartitionKeyOption { 32 | RANDOM, 33 | DETERMINISTIC 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/HeartbeatService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent; 15 | 16 | import java.util.concurrent.TimeUnit; 17 | 18 | import lombok.Getter; 19 | 20 | import com.google.common.util.concurrent.AbstractScheduledService; 21 | 22 | public abstract class HeartbeatService extends AbstractScheduledService { 23 | private final long period; 24 | private final TimeUnit periodUnit; 25 | private final AgentContext agent; 26 | @Getter private Object lastResult; 27 | 28 | public HeartbeatService(AgentContext agent, long period, TimeUnit periodUnit) { 29 | super(); 30 | this.period = period; 31 | this.periodUnit = periodUnit; 32 | this.agent = agent; 33 | } 34 | 35 | @Override 36 | protected void runOneIteration() throws Exception { 37 | lastResult = heartbeat(agent); 38 | } 39 | 40 | @Override 41 | protected Scheduler scheduler() { 42 | return Scheduler.newFixedRateSchedule(period, period, periodUnit); 43 | } 44 | 45 | protected abstract Object heartbeat(AgentContext agent); 46 | } 47 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/checkpoints/FileCheckpoint.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing.checkpoints; 15 | 16 | import com.amazon.kinesis.streaming.agent.tailing.FileId; 17 | import com.amazon.kinesis.streaming.agent.tailing.TrackedFile; 18 | import com.google.common.base.Preconditions; 19 | 20 | import lombok.EqualsAndHashCode; 21 | import lombok.Getter; 22 | import lombok.ToString; 23 | 24 | /** 25 | * A checkpoint in a file. 26 | * When doing equality comparison and computing hashcode, only the file ID and 27 | * flow ID are considered, making this class work with files that change in size 28 | * (and potentially path). 29 | */ 30 | @ToString 31 | @EqualsAndHashCode(exclude={"file"}) 32 | public class FileCheckpoint { 33 | @Getter private final TrackedFile file; 34 | @Getter private final long offset; 35 | @Getter private final FileId fileId; 36 | @Getter private final String flowId; 37 | 38 | public FileCheckpoint(TrackedFile file, long offset) { 39 | Preconditions.checkNotNull(file); 40 | Preconditions.checkArgument(offset >= 0, "The offset (%s) must be a non-negative integer", offset); 41 | this.file = file; 42 | this.offset = offset; 43 | fileId = file.getId(); 44 | flowId = file.getFlow().getId(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/CWMetricKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | import java.util.List; 17 | import java.util.Objects; 18 | 19 | import com.amazonaws.services.cloudwatch.model.Dimension; 20 | import com.amazonaws.services.cloudwatch.model.MetricDatum; 21 | 22 | /* 23 | * A representation of a key of a MetricDatum. This class is useful when wanting to compare 24 | * whether 2 keys have the same MetricDatum. This feature will be used in MetricAccumulatingQueue 25 | * where we aggregate metrics across multiple MetricScopes. 26 | */ 27 | public class CWMetricKey { 28 | 29 | private List dimensions; 30 | private String metricName; 31 | 32 | /** 33 | * @param datum data point 34 | */ 35 | 36 | public CWMetricKey(MetricDatum datum) { 37 | this.dimensions = datum.getDimensions(); 38 | this.metricName = datum.getMetricName(); 39 | } 40 | 41 | @Override 42 | public int hashCode() { 43 | return Objects.hash(dimensions, metricName); 44 | } 45 | 46 | @Override 47 | public boolean equals(Object obj) { 48 | if (this == obj) 49 | return true; 50 | if (obj == null) 51 | return false; 52 | if (getClass() != obj.getClass()) 53 | return false; 54 | CWMetricKey other = (CWMetricKey) obj; 55 | return Objects.equals(other.dimensions, dimensions) && Objects.equals(other.metricName, metricName); 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/AbstractSender.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import org.slf4j.Logger; 17 | 18 | import com.amazon.kinesis.streaming.agent.Logging; 19 | 20 | /** 21 | * Base implementation for an {@link ISender}. 22 | * 23 | * @param The record type. 24 | */ 25 | public abstract class AbstractSender implements ISender { 26 | protected final Logger logger; 27 | 28 | public AbstractSender() { 29 | this.logger = Logging.getLogger(getClass()); 30 | } 31 | 32 | @Override 33 | public BufferSendResult sendBuffer(RecordBuffer buffer) { 34 | if (getMaxSendBatchSizeRecords() > 0 && buffer.sizeRecords() > getMaxSendBatchSizeRecords()) { 35 | throw new IllegalArgumentException("Buffer is too large for service call: " + buffer.sizeRecords() + " records vs. allowed maximum of " + getMaxSendBatchSizeRecords()); 36 | } 37 | if (getMaxSendBatchSizeBytes() > 0 && buffer.sizeBytesWithOverhead() > getMaxSendBatchSizeBytes()) { 38 | throw new IllegalArgumentException("Buffer is too large for service call: " + buffer.sizeBytesWithOverhead() + " bytes vs. allowed maximum of " + getMaxSendBatchSizeBytes()); 39 | } 40 | return attemptSend(buffer); 41 | } 42 | 43 | protected abstract long getMaxSendBatchSizeBytes(); 44 | protected abstract int getMaxSendBatchSizeRecords(); 45 | protected abstract BufferSendResult attemptSend(RecordBuffer buffer); 46 | } 47 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/checkpoints/FileCheckpointStore.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing.checkpoints; 15 | 16 | import java.nio.file.Path; 17 | import java.util.List; 18 | import java.util.Map; 19 | 20 | import com.amazon.kinesis.streaming.agent.tailing.FileFlow; 21 | import com.amazon.kinesis.streaming.agent.tailing.TrackedFile; 22 | 23 | /** 24 | * Base interface for a checkpoing store to track progress of file tailers. 25 | */ 26 | public interface FileCheckpointStore { 27 | 28 | /** 29 | * Creates or updates the checkpoint for the given file. 30 | * @param file 31 | * @param offset 32 | * @return The checkpoint that was just created/updated. 33 | */ 34 | public FileCheckpoint saveCheckpoint(TrackedFile file, long offset); 35 | 36 | /** 37 | * 38 | * @param flow 39 | * @param p 40 | * @return The checkpoint for the given path in the given flow, or 41 | * {@code null} if the flow/path combination has no checkpoints in 42 | * the store. 43 | */ 44 | public FileCheckpoint getCheckpointForPath(FileFlow flow, Path p); 45 | 46 | /** 47 | * @param flow 48 | * @return The latest checkpoint for the given flow if any, or {@code null} 49 | * if the flow has no checkpoints in the store. 50 | */ 51 | public FileCheckpoint getCheckpointForFlow(FileFlow flow); 52 | 53 | /** 54 | * Cleans up any resources used up by this store. 55 | */ 56 | public void close(); 57 | 58 | public List> dumpCheckpoints(); 59 | } 60 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/BufferSendResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import lombok.Getter; 17 | import lombok.ToString; 18 | 19 | /** 20 | * The result of a {@link ISender#sendBuffer(RecordBuffer)} call. 21 | * 22 | * @param The record type. 23 | */ 24 | @ToString 25 | public class BufferSendResult { 26 | public static BufferSendResult succeeded(RecordBuffer buffer) { 27 | return new BufferSendResult(Status.SUCCESS, buffer, buffer.sizeRecords()); 28 | } 29 | 30 | public static BufferSendResult succeeded_partially(RecordBuffer retryBuffer, int originalRecordCount) { 31 | return new BufferSendResult(Status.PARTIAL_SUCCESS, retryBuffer, originalRecordCount); 32 | } 33 | 34 | @Getter private final RecordBuffer buffer; 35 | @Getter private final int originalRecordCount; 36 | @Getter private final Status status; 37 | 38 | private BufferSendResult(Status status, RecordBuffer buffer, int originalRecordCount) { 39 | this.buffer = buffer; 40 | this.originalRecordCount = originalRecordCount; 41 | this.status = status; 42 | } 43 | 44 | public int sentRecordCount() { 45 | return status == Status.SUCCESS ? originalRecordCount : (originalRecordCount - buffer.sizeRecords()); 46 | } 47 | 48 | public int remainingRecordCount() { 49 | return status == Status.SUCCESS ? 0 : buffer.sizeRecords(); 50 | } 51 | 52 | public static enum Status { 53 | SUCCESS, 54 | PARTIAL_SUCCESS, 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/NestedMetricsScope.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | import java.util.Set; 17 | 18 | import com.amazonaws.services.cloudwatch.model.Dimension; 19 | import com.amazonaws.services.cloudwatch.model.StandardUnit; 20 | 21 | public class NestedMetricsScope implements IMetricsScope { 22 | private final IMetricsScope delegate; 23 | 24 | public NestedMetricsScope(IMetricsScope delegate) { 25 | this.delegate = delegate; 26 | } 27 | 28 | @Override 29 | public void addData(String name, double value, StandardUnit unit) { 30 | delegate.addData(name, value, unit); 31 | } 32 | 33 | @Override 34 | public void addCount(String name, long amount) { 35 | delegate.addCount(name, amount); 36 | } 37 | 38 | @Override 39 | public void addTimeMillis(String name, long duration) { 40 | delegate.addTimeMillis(name, duration); 41 | } 42 | 43 | @Override 44 | public void addDimension(String name, String value) { 45 | throw new UnsupportedOperationException("Cannot add dimensions for nested metrics."); 46 | } 47 | 48 | @Override 49 | public void commit() { 50 | // TODO: Implement reference counting 51 | } 52 | 53 | @Override 54 | public void cancel() { 55 | throw new UnsupportedOperationException("Cannot cancel nested metrics."); 56 | } 57 | 58 | @Override 59 | public boolean closed() { 60 | return delegate.closed(); 61 | } 62 | 63 | @Override 64 | public Set getDimensions() { 65 | return delegate.getDimensions(); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/CompositeMetricsFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | import java.util.ArrayList; 17 | import java.util.Arrays; 18 | import java.util.Collection; 19 | 20 | import javax.annotation.Nullable; 21 | 22 | import com.google.common.base.Function; 23 | import com.google.common.collect.Collections2; 24 | 25 | /** 26 | * A metrics factory that wraps multiple metrics factories. 27 | */ 28 | public class CompositeMetricsFactory implements IMetricsFactory { 29 | 30 | private final Collection factories; 31 | 32 | /** 33 | * @param factories 34 | */ 35 | public CompositeMetricsFactory(IMetricsFactory... factories) { 36 | this(Arrays.asList(factories)); 37 | } 38 | 39 | /** 40 | * @param factories 41 | */ 42 | public CompositeMetricsFactory(Collection factories) { 43 | this.factories = new ArrayList<>(factories); 44 | } 45 | 46 | /** 47 | * @return a {@link CompositeMetricsScope} containing a scope for each 48 | * of the factories backing this composite. 49 | */ 50 | @Override 51 | public IMetricsScope createScope() { 52 | Collection scopes = Collections2.transform( 53 | this.factories, new Function() { 54 | @Override 55 | @Nullable 56 | public IMetricsScope apply(IMetricsFactory input) { 57 | return input.createScope(); 58 | } 59 | }); 60 | return new CompositeMetricsScope(scopes); 61 | 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/FileFlowFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import com.amazon.kinesis.streaming.agent.AgentContext; 17 | import com.amazon.kinesis.streaming.agent.config.Configuration; 18 | import com.amazon.kinesis.streaming.agent.config.ConfigurationException; 19 | 20 | /** 21 | * Factory to create specific instances of {@link FileFlow} based 22 | * on the configuration type. 23 | * Currently supported implementations: 24 | *
    25 | *
  • {@link FirehoseFileFlow}
    • 26 | *
27 | */ 28 | public class FileFlowFactory { 29 | /** 30 | * 31 | * @param config 32 | * @return 33 | * @throws ConfigurationException If the configuration does not correspond 34 | * to a known {@link FileFlow} type. 35 | */ 36 | public FileFlow getFileFlow(AgentContext context, Configuration config) throws ConfigurationException { 37 | if(config.containsKey(FirehoseConstants.DESTINATION_KEY)) { 38 | return getFirehoseFileflow(context, config); 39 | } 40 | if(config.containsKey(KinesisConstants.DESTINATION_KEY)) { 41 | return getKinesisFileflow(context, config); 42 | } 43 | throw new ConfigurationException("Could not create flow from the given configuration. Could not recognize flow type."); 44 | } 45 | 46 | protected FirehoseFileFlow getFirehoseFileflow(AgentContext context, Configuration config) { 47 | return new FirehoseFileFlow(context, config); 48 | } 49 | 50 | protected KinesisFileFlow getKinesisFileflow(AgentContext context, Configuration config) { 51 | return new KinesisFileFlow(context, config); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/CWMetricsScope.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | 19 | import com.amazonaws.services.cloudwatch.model.MetricDatum; 20 | 21 | public class CWMetricsScope extends AccumulatingMetricsScope implements IMetricsScope { 22 | 23 | private CWPublisherRunnable publisher; 24 | 25 | /** 26 | * Each CWMetricsScope takes a publisher which contains the logic of when to publish metrics. 27 | * 28 | * @param publisher publishing logic 29 | */ 30 | 31 | public CWMetricsScope(CWPublisherRunnable publisher) { 32 | super(); 33 | this.publisher = publisher; 34 | } 35 | 36 | /* 37 | * Once we call this method, all MetricDatums added to the scope will be enqueued to the publisher runnable. 38 | * We enqueue MetricDatumWithKey because the publisher will aggregate similar metrics (i.e. MetricDatum with the 39 | * same metricName) in the background thread. Hence aggregation using MetricDatumWithKey will be especially useful 40 | * when aggregating across multiple MetricScopes. 41 | */ 42 | @Override 43 | protected void realCommit() { 44 | if(!data.isEmpty()) { 45 | List> dataWithKeys = new ArrayList>(); 46 | for (MetricDatum datum : data.values()) { 47 | datum.setDimensions(getDimensions()); 48 | dataWithKeys.add(new MetricDatumWithKey(new CWMetricKey(datum), datum)); 49 | } 50 | publisher.enqueue(dataWithKeys); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/AgentAWSCredentialsProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent; 15 | 16 | import org.slf4j.Logger; 17 | 18 | import com.amazon.kinesis.streaming.agent.config.AgentConfiguration; 19 | import com.amazonaws.AmazonClientException; 20 | import com.amazonaws.auth.AWSCredentialsProvider; 21 | import com.amazonaws.auth.AWSCredentials; 22 | import com.amazonaws.auth.BasicAWSCredentials; 23 | import com.google.common.base.Strings; 24 | 25 | public class AgentAWSCredentialsProvider implements AWSCredentialsProvider { 26 | private static final Logger LOGGER = Logging.getLogger(AgentAWSCredentialsProvider.class); 27 | private final AgentConfiguration config; 28 | 29 | public AgentAWSCredentialsProvider(AgentConfiguration config) { 30 | this.config = config; 31 | } 32 | 33 | public AWSCredentials getCredentials() { 34 | String accessKeyId = config.accessKeyId(); 35 | String secretKey = config.secretKey(); 36 | 37 | if (!Strings.isNullOrEmpty(accessKeyId) && !Strings.isNullOrEmpty(secretKey)) { 38 | LOGGER.debug("Loading credentials from agent config"); 39 | return new BasicAWSCredentials(accessKeyId, secretKey); 40 | } 41 | 42 | throw new AmazonClientException("Unable to load credentials from agent config. Missing entries: " + 43 | (Strings.isNullOrEmpty(accessKeyId) ? AgentConfiguration.CONFIG_ACCESS_KEY : "") + 44 | " " + (Strings.isNullOrEmpty(secretKey) ? AgentConfiguration.CONFIG_SECRET_KEY : "")); 45 | } 46 | 47 | public void refresh() {} 48 | 49 | @Override 50 | public String toString() { 51 | return getClass().getSimpleName(); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/config/BooleanConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.config; 15 | 16 | import javax.annotation.Nullable; 17 | 18 | import com.google.common.base.Function; 19 | 20 | /** 21 | * Function that converts an object into a boolean true or false. 22 | * Following (case insensitive) strings map to true:
    23 | *
  • True
    • 24 | *
  • T
    • 25 | *
  • Yes
    • 26 | *
  • Y
    • 27 | *
  • 1
    • 28 | *
29 | * Following (case insensitive) strings map to false:
    30 | *
  • False
    • 31 | *
  • F
    • 32 | *
  • No
    • 33 | *
  • N
    • 34 | *
  • 0
    • 35 | *
36 | * Everything else raises a {@link ConfigurationException}. 37 | */ 38 | class BooleanConverter implements Function { 39 | 40 | /** 41 | * @throws ConfigurationException if the value could not be converted 42 | * into a Boolean. 43 | */ 44 | @Override 45 | @Nullable 46 | public Boolean apply(@Nullable Object input) { 47 | if (input != null && !(input instanceof Boolean)) { 48 | switch (input.toString().toLowerCase()) { 49 | case "true": 50 | case "yes": 51 | case "t": 52 | case "y": 53 | case "1": 54 | return true; 55 | case "false": 56 | case "no": 57 | case "f": 58 | case "n": 59 | case "0": 60 | return false; 61 | default: 62 | throw new ConfigurationException( 63 | "Cannot convert value to boolean: " 64 | + input.toString()); 65 | 66 | } 67 | } else 68 | return (Boolean) input; 69 | } 70 | 71 | } -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/CompositeMetricsScope.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | import java.util.ArrayList; 17 | import java.util.Arrays; 18 | import java.util.Collection; 19 | import java.util.Collections; 20 | import java.util.List; 21 | import java.util.Set; 22 | 23 | import com.amazonaws.services.cloudwatch.model.Dimension; 24 | import com.amazonaws.services.cloudwatch.model.StandardUnit; 25 | 26 | /** 27 | * A composite {@link IMetricsScope} that delegates all method calls to a list 28 | * of underlying instances. 29 | */ 30 | public class CompositeMetricsScope extends AbstractMetricsScope { 31 | private final List scopes; 32 | 33 | /** 34 | * @param scopes 35 | */ 36 | public CompositeMetricsScope(IMetricsScope... scopes) { 37 | this(Arrays.asList(scopes)); 38 | } 39 | 40 | /** 41 | * @param scopes 42 | */ 43 | public CompositeMetricsScope(Collection scopes) { 44 | this.scopes = new ArrayList<>(scopes); 45 | } 46 | 47 | @Override 48 | protected void realAddData(String name, double value, StandardUnit unit) { 49 | for(IMetricsScope scope : this.scopes) 50 | scope.addData(name, value, unit); 51 | } 52 | 53 | @Override 54 | protected void realAddDimension(String name, String value) { 55 | for(IMetricsScope scope : this.scopes) 56 | scope.addDimension(name, value); 57 | } 58 | 59 | @Override 60 | protected void realCommit() { 61 | for(IMetricsScope scope : this.scopes) 62 | scope.commit(); 63 | } 64 | 65 | @Override 66 | protected void realCancel() { 67 | for(IMetricsScope scope : this.scopes) 68 | scope.cancel(); 69 | } 70 | 71 | @Override 72 | protected Set realGetDimensions() { 73 | return scopes.isEmpty() ? Collections. emptySet() : scopes.get(0).getDimensions(); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/CustomLog4jFallbackErrorHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent; 15 | 16 | import java.text.SimpleDateFormat; 17 | import java.util.Date; 18 | 19 | import org.apache.log4j.spi.LoggingEvent; 20 | import org.apache.log4j.varia.FallbackErrorHandler; 21 | 22 | public class CustomLog4jFallbackErrorHandler extends FallbackErrorHandler { 23 | private static StringBuilder errorHeader = new StringBuilder(); 24 | 25 | /** 26 | * @return A string describing the error that triggered the fallback, or 27 | * {@code null} if no errors occurred. 28 | */ 29 | public static String getErrorHeader() { 30 | return errorHeader.length() == 0 ? null : errorHeader.toString(); 31 | } 32 | 33 | public static String getFallbackLogFile() { 34 | return "/tmp/fallback-aws-kinesis-agent.log"; 35 | } 36 | 37 | @Override 38 | public void error(String message, Exception e, int errorCode, LoggingEvent event) { 39 | errorHeader.append("# ********************************************************************************").append('\n'); 40 | errorHeader.append("# ").append(new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z").format(new Date())).append('\n'); 41 | errorHeader.append("# The following error was reported while initializing logging:").append('\n'); 42 | errorHeader.append("# " + message).append('\n'); 43 | errorHeader.append("# " + e.getClass().getName() + ": " + e.getMessage()).append('\n'); 44 | errorHeader.append("# Please fix the problem and restart the application.").append('\n'); 45 | errorHeader.append("# Logs will be redirected to " + getFallbackLogFile() + " in the meantime.").append('\n'); 46 | errorHeader.append("# ********************************************************************************").append('\n'); 47 | System.err.println(errorHeader.toString()); 48 | super.error(message, e, errorCode, event); 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/LogMetricsScope.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | import org.slf4j.Logger; 17 | 18 | import com.amazon.kinesis.streaming.agent.Logging; 19 | import com.amazonaws.services.cloudwatch.model.Dimension; 20 | import com.amazonaws.services.cloudwatch.model.MetricDatum; 21 | import com.amazonaws.services.cloudwatch.model.StatisticSet; 22 | 23 | /** 24 | * An AccumulatingMetricsScope that outputs via log4j. 25 | */ 26 | public class LogMetricsScope extends AccumulatingMetricsScope { 27 | public static final Logger LOGGER = Logging.getLogger(LogMetricsScope.class); 28 | 29 | public LogMetricsScope() { 30 | super(); 31 | } 32 | 33 | @Override 34 | protected void realCommit() { 35 | if(!data.values().isEmpty()) { 36 | StringBuilder output = new StringBuilder(); 37 | output.append("Metrics:\n"); 38 | 39 | output.append("Dimensions: "); 40 | boolean needsComma = false; 41 | for (Dimension dimension : getDimensions()) { 42 | output.append(String.format("%s[%s: %s]", needsComma ? ", " : "", dimension.getName(), dimension.getValue())); 43 | needsComma = true; 44 | } 45 | output.append("\n"); 46 | 47 | for (MetricDatum datum : data.values()) { 48 | StatisticSet statistics = datum.getStatisticValues(); 49 | output.append(String.format("Name=%50s\tMin=%.2f\tMax=%.2f\tCount=%.2f\tSum=%.2f\tAvg=%.2f\tUnit=%s\n", 50 | datum.getMetricName(), 51 | statistics.getMinimum(), 52 | statistics.getMaximum(), 53 | statistics.getSampleCount(), 54 | statistics.getSum(), 55 | statistics.getSum() / statistics.getSampleCount(), 56 | datum.getUnit())); 57 | } 58 | LOGGER.debug(output.toString()); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/MetricDatumWithKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | import java.util.Objects; 17 | import com.amazonaws.services.cloudwatch.model.MetricDatum; 18 | 19 | /** 20 | * This class is used to store a MetricDatum as well as KeyType which stores specific information about 21 | * that particular MetricDatum. 22 | * 23 | * @param is a class that stores information about a MetricDatum. This is useful 24 | * to compare MetricDatums, aggregate similar MetricDatums or store information about a datum 25 | * that may be relevant to the user (i.e. MetricName, CustomerId, TimeStamp, etc). 26 | * 27 | * Example: 28 | * 29 | * Let SampleMetricKey be a KeyType that takes in the time in which the datum was created. 30 | * 31 | * MetricDatumWithKey sampleDatumWithKey = new MetricDatumWithKey(new 32 | * SampleMetricKey(System.currentTimeMillis()), datum) 33 | * 34 | */ 35 | public class MetricDatumWithKey { 36 | public KeyType key; 37 | public MetricDatum datum; 38 | 39 | /** 40 | * @param key an object that stores relevant information about a MetricDatum (e.g. MetricName, accountId, 41 | * TimeStamp) 42 | * @param datum data point 43 | */ 44 | 45 | public MetricDatumWithKey(KeyType key, MetricDatum datum) { 46 | this.key = key; 47 | this.datum = datum; 48 | } 49 | 50 | @Override 51 | public int hashCode() { 52 | return Objects.hash(key, datum); 53 | } 54 | 55 | @Override 56 | public boolean equals(Object obj) { 57 | if (this == obj) 58 | return true; 59 | if (obj == null) 60 | return false; 61 | if (getClass() != obj.getClass()) 62 | return false; 63 | MetricDatumWithKey other = (MetricDatumWithKey) obj; 64 | return Objects.equals(other.key, key) && Objects.equals(other.datum, datum); 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/FileId.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import java.io.IOException; 17 | import java.nio.file.Files; 18 | import java.nio.file.Path; 19 | import java.nio.file.attribute.BasicFileAttributes; 20 | 21 | import lombok.EqualsAndHashCode; 22 | import lombok.Getter; 23 | 24 | import com.google.common.base.Preconditions; 25 | 26 | /** 27 | * Abstraction of a file identifier (independent from path). Current implementation 28 | * relies on {@link BasicFileAttributes#fileKey()}. 29 | */ 30 | @EqualsAndHashCode 31 | public class FileId { 32 | @Getter private final String id; 33 | 34 | /** 35 | * @see #get(BasicFileAttributes) 36 | */ 37 | public static FileId get(Path file) throws IOException { 38 | if (!Files.exists(file)) { 39 | return null; 40 | } 41 | Preconditions.checkArgument(Files.isRegularFile(file), 42 | "Can only get ID for real files (no directories and symlinks): " 43 | + file); 44 | BasicFileAttributes attr = Files.readAttributes(file, 45 | BasicFileAttributes.class); 46 | return get(attr); 47 | } 48 | 49 | /** 50 | * TODO: this might not be portable as we rely on inner representation of 51 | * the {@link BasicFileAttributes#fileKey()} 52 | * ({@linkplain UnixFileKey http://grepcode.com/file/repository.grepcode.com/java/root/jdk/openjdk/7u40-b43/sun/nio/fs/UnixFileKey.java} 53 | * on Linux), which is not defined canonically anywhere. 54 | * @param attr 55 | * @return 56 | * @throws IOException 57 | */ 58 | public static FileId get(BasicFileAttributes attr) throws IOException { 59 | return new FileId(attr.fileKey().toString()); 60 | } 61 | 62 | public FileId(String id) { 63 | Preconditions.checkNotNull(id); 64 | this.id = id; 65 | } 66 | 67 | @Override 68 | public String toString() { 69 | return this.id; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/IMetricsScope.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | import java.util.Set; 17 | 18 | import com.amazonaws.services.cloudwatch.model.Dimension; 19 | import com.amazonaws.services.cloudwatch.model.StandardUnit; 20 | 21 | /** 22 | * An {@link IMetricsScope} represents a set of metric data that share a set of 23 | * dimensions. {@link IMetricsScope}s know how to output themselves 24 | * (perhaps to disk, perhaps over service calls, etc). 25 | */ 26 | public interface IMetricsScope { 27 | 28 | /** 29 | * Adds a data point to this scope. 30 | * 31 | * @param name data point name 32 | * @param value data point value 33 | * @param unit unit of data point 34 | */ 35 | public void addData(String name, double value, StandardUnit unit); 36 | 37 | /** 38 | * @param name @see {@link #addData(String, double, StandardUnit)} 39 | * @param amount the amount to increment this counter. 40 | */ 41 | public void addCount(String name, long amount); 42 | 43 | /** 44 | * @param name 45 | * @param duration duration of the tiumer in milliseconds 46 | */ 47 | public void addTimeMillis(String name, long duration); 48 | 49 | /** 50 | * Adds a dimension that applies to all metrics in this IMetricsScope. 51 | * 52 | * @param name dimension name 53 | * @param value dimension value 54 | */ 55 | public void addDimension(String name, String value); 56 | 57 | /** 58 | * Flushes the data from this scope and makes it unusable. 59 | */ 60 | public void commit(); 61 | 62 | /** 63 | * Cancels this scope and discards any data. 64 | */ 65 | public void cancel(); 66 | 67 | /** 68 | * @return true if {@link #commit()} or {@link #cancel()} have 69 | * been called on this instance, otherwise false. 70 | */ 71 | public boolean closed(); 72 | 73 | /** 74 | * @return a set of dimensions for an IMetricsScope 75 | */ 76 | public Set getDimensions(); 77 | } 78 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/custom.log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/KinesisRecord.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import java.nio.ByteBuffer; 17 | import java.util.concurrent.ThreadLocalRandom; 18 | 19 | import com.amazon.kinesis.streaming.agent.tailing.KinesisConstants.PartitionKeyOption; 20 | import com.google.common.annotations.VisibleForTesting; 21 | import com.google.common.base.Preconditions; 22 | import com.google.common.hash.Hasher; 23 | import com.google.common.hash.Hashing; 24 | 25 | public class KinesisRecord extends AbstractRecord { 26 | protected final String partitionKey; 27 | 28 | public KinesisRecord(TrackedFile file, long offset, ByteBuffer data) { 29 | super(file, offset, data); 30 | Preconditions.checkNotNull(file); 31 | partitionKey = generatePartitionKey(((KinesisFileFlow)file.getFlow()).getPartitionKeyOption()); 32 | } 33 | 34 | public KinesisRecord(TrackedFile file, long offset, byte[] data) { 35 | super(file, offset, data); 36 | Preconditions.checkNotNull(file); 37 | partitionKey = generatePartitionKey(((KinesisFileFlow)file.getFlow()).getPartitionKeyOption()); 38 | } 39 | 40 | public String partitionKey() { 41 | return partitionKey; 42 | } 43 | 44 | @Override 45 | public long lengthWithOverhead() { 46 | return length() + KinesisConstants.PER_RECORD_OVERHEAD_BYTES; 47 | } 48 | 49 | @Override 50 | public long length() { 51 | return dataLength() + partitionKey.length(); 52 | } 53 | 54 | @Override 55 | protected int getMaxDataSize() { 56 | return KinesisConstants.MAX_RECORD_SIZE_BYTES - partitionKey.length(); 57 | } 58 | 59 | @VisibleForTesting 60 | String generatePartitionKey(PartitionKeyOption option) { 61 | Preconditions.checkNotNull(option); 62 | 63 | if (option == PartitionKeyOption.DETERMINISTIC) { 64 | Hasher hasher = Hashing.md5().newHasher(); 65 | hasher.putBytes(data.array()); 66 | return hasher.hash().toString(); 67 | } 68 | if (option == PartitionKeyOption.RANDOM) 69 | return "" + ThreadLocalRandom.current().nextDouble(1000000); 70 | 71 | return null; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/DefaultCWMetricsPublisher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | 19 | import org.slf4j.Logger; 20 | 21 | import com.amazon.kinesis.streaming.agent.Logging; 22 | import com.amazonaws.AmazonClientException; 23 | import com.amazonaws.services.cloudwatch.AmazonCloudWatch; 24 | import com.amazonaws.services.cloudwatch.model.MetricDatum; 25 | import com.amazonaws.services.cloudwatch.model.PutMetricDataRequest; 26 | 27 | /** 28 | * Default implementation for publishing metrics to CloudWatch. 29 | */ 30 | 31 | public class DefaultCWMetricsPublisher implements ICWMetricsPublisher { 32 | 33 | private static final Logger LOG = Logging.getLogger(CWPublisherRunnable.class); 34 | 35 | // CloudWatch API has a limit of 20 MetricDatums per request 36 | private static final int BATCH_SIZE = 20; 37 | 38 | private final String namespace; 39 | private final AmazonCloudWatch cloudWatchClient; 40 | 41 | public DefaultCWMetricsPublisher(AmazonCloudWatch cloudWatchClient, String namespace) { 42 | this.cloudWatchClient = cloudWatchClient; 43 | this.namespace = namespace; 44 | } 45 | 46 | @Override 47 | public void publishMetrics(List> dataToPublish) { 48 | for (int startIndex = 0; startIndex < dataToPublish.size(); startIndex += BATCH_SIZE) { 49 | int endIndex = Math.min(dataToPublish.size(), startIndex + BATCH_SIZE); 50 | 51 | PutMetricDataRequest request = new PutMetricDataRequest(); 52 | request.setNamespace(namespace); 53 | 54 | List metricData = new ArrayList(); 55 | for (int i = startIndex; i < endIndex; i++) { 56 | MetricDatum metric = dataToPublish.get(i).datum; 57 | if(!metric.getMetricName().startsWith(".")) 58 | metricData.add(dataToPublish.get(i).datum); 59 | } 60 | if(metricData.isEmpty()) 61 | return; 62 | 63 | request.setMetricData(metricData); 64 | try { 65 | cloudWatchClient.putMetricData(request); 66 | 67 | LOG.info(String.format("Successfully published %d datums.", endIndex - startIndex)); 68 | } catch (AmazonClientException e) { 69 | LOG.warn(String.format("Could not publish %d datums to CloudWatch", endIndex - startIndex), e); 70 | } 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/ISender.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import java.util.Map; 17 | 18 | import com.amazon.kinesis.streaming.agent.AgentContext; 19 | 20 | 21 | /** 22 | * Interface for a sender that sends a buffer of records to an implementation- 23 | * specific destination. 24 | * 25 | * @param The record type. 26 | */ 27 | public interface ISender { 28 | /** 29 | * Sends the buffer to the implementation-specific destination. 30 | * The buffer will be modified by this method and after it returns it will 31 | * have all the records that were succeesfully sent to be removed, and what 32 | * remains are records that were not successfully committed to destination. 33 | * More specifically, if the status of the {@link BufferSendResult} 34 | * is: 35 | *
    36 | *
  • {@code SUCEESS}: then the buffer is expected to be empty after the 37 | * call.
    • 38 | *
  • {@code ERROR}: then the buffer is expected to be unchanged after 39 | * the call, and it would contain the same records as before.
    • 40 | *
  • {@code PARTIAL_FAILURE}: then the buffer is expected to have some 41 | * records left in it, but the records that were committed are 42 | * removed.
    • 43 | *
44 | * Note that the {@link RecordBuffer#checkpointFile() buffer.checkpointFile()} 45 | * and {@link RecordBuffer#checkpointOffset() buffer.checkpointOffset()} will 46 | * not be changed in any of the cases above. 47 | * 48 | * These semantics make it safe to retry a buffer by calling this method 49 | * again with the same instance (which is also referenced in 50 | * {@link BufferSendResult#getBuffer() result.getBuffer()}). 51 | * 52 | * @param buffer The buffer to send to destination. Will be modified by this 53 | * call by removing all records that were succeeffully committed to 54 | * the destination. 55 | * @return The result of the send operation. The 56 | * {@link BufferSendResult#getBuffer() result.getBuffer()} is a 57 | * reference to the input parameter. 58 | * @throws IllegalArgumentException 59 | */ 60 | BufferSendResult sendBuffer(RecordBuffer buffer); 61 | 62 | /** 63 | * @return The agent context for this sender. 64 | */ 65 | AgentContext getAgentContext(); 66 | 67 | /** 68 | * @return The destination where the sender is targeting 69 | */ 70 | String getDestination(); 71 | 72 | Map getMetrics(); 73 | 74 | } 75 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/AccumulatingMetricsScope.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | import java.util.HashMap; 17 | import java.util.HashSet; 18 | import java.util.Map; 19 | import java.util.Set; 20 | 21 | import javax.annotation.concurrent.NotThreadSafe; 22 | 23 | import com.amazonaws.services.cloudwatch.model.Dimension; 24 | import com.amazonaws.services.cloudwatch.model.MetricDatum; 25 | import com.amazonaws.services.cloudwatch.model.StandardUnit; 26 | import com.amazonaws.services.cloudwatch.model.StatisticSet; 27 | 28 | /** 29 | * An base implementation of an {@link IMetricsScope} that accumulates data 30 | * from multiple calls to addData with the same name parameter. 31 | * It tracks min, max, sample count, and sum for each named metric. 32 | */ 33 | @NotThreadSafe 34 | public abstract class AccumulatingMetricsScope extends AbstractMetricsScope { 35 | 36 | protected final Set dimensions = new HashSet(); 37 | protected final Map data = new HashMap(); 38 | protected final long startTime = System.currentTimeMillis(); 39 | 40 | @Override 41 | protected void realAddData(String name, double value, StandardUnit unit) { 42 | MetricDatum datum = data.get(name); 43 | if (datum == null) { 44 | data.put(name, 45 | new MetricDatum().withMetricName(name) 46 | .withUnit(unit) 47 | .withStatisticValues(new StatisticSet().withMaximum(value) 48 | .withMinimum(value) 49 | .withSampleCount(1.0) 50 | .withSum(value))); 51 | } else { 52 | if (!datum.getUnit().equals(unit.name())) { 53 | throw new IllegalArgumentException("Cannot add to existing metric with different unit"); 54 | } 55 | StatisticSet statistics = datum.getStatisticValues(); 56 | statistics.setMaximum(Math.max(value, statistics.getMaximum())); 57 | statistics.setMinimum(Math.min(value, statistics.getMinimum())); 58 | statistics.setSampleCount(statistics.getSampleCount() + 1); 59 | statistics.setSum(statistics.getSum() + value); 60 | } 61 | } 62 | 63 | @Override 64 | protected void realAddDimension(String name, String value) { 65 | dimensions.add(new Dimension().withName(name).withValue(value)); 66 | } 67 | 68 | @Override 69 | protected Set realGetDimensions() { 70 | return dimensions; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/checkpoints/Checkpointer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing.checkpoints; 15 | 16 | import org.slf4j.Logger; 17 | 18 | import lombok.Getter; 19 | 20 | import com.amazon.kinesis.streaming.agent.Logging; 21 | import com.amazon.kinesis.streaming.agent.tailing.FileFlow; 22 | import com.amazon.kinesis.streaming.agent.tailing.IRecord; 23 | import com.amazon.kinesis.streaming.agent.tailing.RecordBuffer; 24 | import com.google.common.base.Preconditions; 25 | 26 | /** 27 | * Class that manages checkpoint updates per {@link FileFlow flow}. 28 | * Current implementation only makes sure that for a given {@link FileFlow flow} 29 | * an older checkpoint arriving late does not overwrite a newer checkpoint. 30 | * 31 | * This is not ideal because it could potentially create gaps and lead to data 32 | * loss, but in normal cases this should happen only rarely. 33 | * 34 | * TODO: A future iteration could ensure that checkpoints are created in 35 | * sequence such that no gaps are left. This entails waiting for 36 | * late-arriving buffers, and could lead to data duplication but 37 | * guarantees no data loss. 38 | * 39 | * @param 40 | */ 41 | public class Checkpointer { 42 | private static final Logger LOGGER = Logging.getLogger(Checkpointer.class); 43 | 44 | @Getter private final FileCheckpointStore store; 45 | @Getter private final FileFlow flow; 46 | private long committed = -1; 47 | 48 | public Checkpointer(FileFlow flow, FileCheckpointStore store) { 49 | this.flow = flow; 50 | this.store = store; 51 | } 52 | 53 | /** 54 | * @param buffer The buffer that was sent and that triggered the checkpoint 55 | * update. 56 | * @return The checkpoint that was created, or {@code null} if the buffer 57 | * would overwrite a previous checkpoint. 58 | */ 59 | public synchronized FileCheckpoint saveCheckpoint(RecordBuffer buffer) { 60 | // SANITYCHECK: Can remove when done with debugging 61 | Preconditions.checkArgument(buffer.id() != committed); 62 | // Only store the checkpoint if it has an increasing sequence number 63 | if(buffer.id() > committed) { 64 | committed = buffer.id(); 65 | return store.saveCheckpoint(buffer.checkpointFile(), buffer.checkpointOffset()); 66 | } else { 67 | LOGGER.trace("Buffer {} has lower sequence number than the last committed value {}. " + 68 | "No checkpoints will be updated.", buffer, committed); 69 | // TODO: Add metrics? 70 | return null; 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/RegexSplitter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import java.nio.ByteBuffer; 17 | import java.nio.charset.StandardCharsets; 18 | import java.util.regex.Matcher; 19 | import java.util.regex.Pattern; 20 | 21 | import com.google.common.base.Preconditions; 22 | import com.google.common.base.Strings; 23 | 24 | /** 25 | * Returns one record that splits records based on a regex that matches the beginning of a record 26 | * 27 | */ 28 | public class RegexSplitter implements ISplitter { 29 | public final Pattern startingPattern; 30 | 31 | public RegexSplitter(String startingPattern) { 32 | Preconditions.checkArgument(!Strings.isNullOrEmpty(startingPattern)); 33 | this.startingPattern = Pattern.compile(startingPattern); 34 | } 35 | 36 | public String getPattern() { 37 | return startingPattern.pattern(); 38 | } 39 | 40 | @Override 41 | public int locateNextRecord(ByteBuffer buffer) { 42 | return advanceBufferToNextPattern(buffer); 43 | } 44 | 45 | /** 46 | * Advances the buffer current position to be at the index at the beginning of the 47 | * next pattern, or else the end of the buffer if there is no final pattern. 48 | * 49 | * @return {@code position} of the buffer at the starting index of the next new pattern; 50 | * {@code -1} if the end of the buffer was reached. 51 | */ 52 | private int advanceBufferToNextPattern(ByteBuffer buffer) { 53 | Matcher matcher; 54 | // this marks the position before which we already attempted to match the pattern 55 | int currentLookedPosition = buffer.position(); 56 | boolean firstLine = true; 57 | while (buffer.hasRemaining()) { 58 | // start matching from the current position on a line by line basis 59 | if (buffer.get() == SingleLineSplitter.LINE_DELIMITER) { 60 | // Skip the first line as it must be part of the current record 61 | if (!firstLine) { 62 | String line = new String(buffer.array(), currentLookedPosition, buffer.position() - currentLookedPosition, StandardCharsets.UTF_8); 63 | matcher = startingPattern.matcher(line); 64 | if(matcher.lookingAt()) { 65 | buffer.position(currentLookedPosition); 66 | return currentLookedPosition; 67 | } 68 | } 69 | firstLine = false; 70 | // update the position that we already looked at 71 | currentLookedPosition = buffer.position(); 72 | } 73 | } 74 | 75 | // We've scanned to the end and there is only one complete record in the buffer, set the position to the end 76 | if (!firstLine && buffer.limit() < buffer.capacity()) { 77 | return buffer.position(); 78 | } 79 | 80 | return -1; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/AbstractMetricsScope.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | import java.util.Set; 17 | 18 | import javax.annotation.concurrent.NotThreadSafe; 19 | 20 | import com.amazonaws.services.cloudwatch.model.Dimension; 21 | import com.amazonaws.services.cloudwatch.model.StandardUnit; 22 | import com.google.common.base.Preconditions; 23 | /** 24 | * An base implementation of an {@link IMetricsScope} that accumulates data 25 | * from multiple calls to addData with the same name parameter. 26 | * It tracks min, max, sample count, and sum for each named metric. 27 | */ 28 | @NotThreadSafe 29 | public abstract class AbstractMetricsScope implements IMetricsScope { 30 | private boolean closed = false; 31 | 32 | /** 33 | * Adds data points to the scope {@link IMetricsScope}. 34 | * Multiple calls to this method with the same name will have their data 35 | * accumulated. 36 | * @see IMetricsScope#addData(String, double, StandardUnit) 37 | */ 38 | @Override 39 | public final void addData(String name, double value, StandardUnit unit) { 40 | Preconditions.checkState(!closed, "Scope is already closed."); 41 | realAddData(name, value, unit); 42 | } 43 | protected void realAddData(String name, double value, StandardUnit unit) { } 44 | 45 | @Override 46 | public void addCount(String name, long amount) { 47 | addData(name, amount, StandardUnit.Count); 48 | } 49 | 50 | @Override 51 | public void addTimeMillis(String name, long duration) { 52 | addData(name, duration, StandardUnit.Milliseconds); 53 | } 54 | 55 | @Override 56 | public final void addDimension(String name, String value) { 57 | Preconditions.checkState(!closed, "Scope is already closed."); 58 | realAddDimension(name, value); 59 | } 60 | protected void realAddDimension(String name, String value) { } 61 | 62 | @Override 63 | public final void commit() { 64 | Preconditions.checkState(!closed, "Scope is already closed."); 65 | try { 66 | realCommit(); 67 | } finally { 68 | closed = true; 69 | } 70 | } 71 | 72 | @Override 73 | public final void cancel() { 74 | Preconditions.checkState(!closed, "Scope is already closed."); 75 | try { 76 | realCancel(); 77 | } finally { 78 | closed = true; 79 | } 80 | } 81 | 82 | protected void realCancel() { } 83 | protected void realCommit() { } 84 | 85 | @Override 86 | public boolean closed() { 87 | return closed; 88 | } 89 | 90 | /** 91 | * @return a set of dimensions for an IMetricsScope 92 | */ 93 | @Override 94 | public Set getDimensions() { 95 | Preconditions.checkState(!closed, "Scope is already closed."); 96 | return realGetDimensions(); 97 | } 98 | protected abstract Set realGetDimensions(); 99 | } 100 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/ByteBufferInputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent; 15 | 16 | import java.io.IOException; 17 | import java.io.InputStream; 18 | import java.nio.ByteBuffer; 19 | import java.nio.InvalidMarkException; 20 | 21 | /** 22 | * Adapts a ByteBuffer into an InputStream. 23 | */ 24 | public final class ByteBufferInputStream extends InputStream { 25 | 26 | private static final int MAX_BYTE_UNSIGNED = 0xFF; 27 | private final ByteBuffer buffer; 28 | 29 | /** 30 | * Create a new InputStream that is linked to the provided ByteBuffer. The 31 | * buffer and this stream will share backing data, position, limit, and 32 | * mark, such that reading from this input stream will update the buffer 33 | * position; care should be taken to avoid concurrent reads and 34 | * modifications, as ByteBuffers are not thread-safe. 35 | */ 36 | public ByteBufferInputStream(ByteBuffer buffer) { 37 | this.buffer = buffer; 38 | } 39 | 40 | @Override 41 | public synchronized int read() { 42 | if (!buffer.hasRemaining()) { 43 | return -1; 44 | } 45 | return buffer.get() & MAX_BYTE_UNSIGNED; 46 | } 47 | 48 | @Override 49 | public synchronized int read(byte[] b) { 50 | return this.read(b, 0, b.length); 51 | } 52 | 53 | @Override 54 | public synchronized int read(byte[] b, int offset, int length) { 55 | if (!buffer.hasRemaining()) { 56 | return -1; 57 | } 58 | int ask = Math.min(buffer.remaining(), length); 59 | buffer.get(b, offset, ask); 60 | return ask; 61 | } 62 | 63 | @Override 64 | public boolean markSupported() { 65 | return true; 66 | } 67 | 68 | @Override 69 | public synchronized void mark(int readlimit) { 70 | buffer.mark(); 71 | } 72 | 73 | /** 74 | * Repositions this stream to the position at the time the mark method was 75 | * last called on this input stream. 76 | * 77 | * @throws IOException 78 | * if this stream has not been marked 79 | */ 80 | @Override 81 | public synchronized void reset() throws IOException { 82 | try { 83 | buffer.reset(); 84 | } catch (InvalidMarkException e) { 85 | // Can't call rewind() because it might rewind us past the position 86 | // where this stream began 87 | throw new IOException("No mark set"); 88 | } 89 | } 90 | 91 | @Override 92 | public synchronized int available() { 93 | return buffer.remaining(); 94 | } 95 | 96 | @Override 97 | public synchronized long skip(long n) { 98 | if (n <= 0) { 99 | return 0; 100 | } 101 | // ask must be representable as an int, since buffer.remaining() can't 102 | // be more than Integer.MAX_VALUE 103 | int ask = (int) Math.min(buffer.remaining(), n); 104 | buffer.position(buffer.position() + ask); 105 | return ask; 106 | } 107 | 108 | /** 109 | * Closing a ByteBufferInputStream has no effect. The methods in 110 | * this class can be called after the stream has been closed without 111 | * generating an IOException. 112 | *

113 | */ 114 | @Override 115 | public void close() { 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/Metrics.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | import java.util.Map; 19 | import java.util.regex.Pattern; 20 | 21 | import com.amazon.kinesis.streaming.agent.AgentContext; 22 | import com.amazon.kinesis.streaming.agent.tailing.FirehoseConstants; 23 | 24 | public class Metrics implements IMetricsContext { 25 | // TODO: uncouple from FirehoseConstants 26 | public static final int BYTES_BEHIND_INFO_LEVEL = 5 * FirehoseConstants.DEFAULT_PARSER_BUFFER_SIZE_BYTES; 27 | public static final int BYTES_BEHIND_WARN_LEVEL = 10 * FirehoseConstants.DEFAULT_PARSER_BUFFER_SIZE_BYTES; 28 | 29 | public static final Pattern SENDER_TOTAL_RECORDS_SENT_METRIC = Pattern.compile("^.*Sender.TotalRecordsSent$"); 30 | public static final Pattern PARSER_TOTAL_BYTES_CONSUMED_METRIC = Pattern.compile("^.*Parser.TotalBytesConsumed$"); 31 | public static final Pattern FILE_TAILER_FILES_BEHIND_METRIC = Pattern.compile("^.*FileTailer.FilesBehind$"); 32 | public static final Pattern FILE_TAILER_BYTES_BEHIND_METRIC = Pattern.compile("^.*FileTailer.BytesBehind$"); 33 | public static final Pattern PARSER_TOTAL_RECORDS_PARSED_METRIC = Pattern.compile("^.*Parser.TotalRecordsParsed$"); 34 | 35 | public static final String DESTINATION_DIMENSION = "Destination"; 36 | 37 | private IMetricsFactory factory; 38 | 39 | public Metrics(AgentContext context) { 40 | List factories = new ArrayList<>(); 41 | if (context.logEmitMetrics() && LogMetricsScope.LOGGER.isDebugEnabled()) { 42 | factories.add(new LogMetricsFactory()); 43 | } 44 | if (context.cloudwatchEmitMetrics()) { 45 | final IMetricsFactory cwFactory = new CWMetricsFactory( 46 | context.getCloudWatchClient(), context.cloudwatchNamespace(), 47 | context.cloudwatchMetricsBufferTimeMillis(), 48 | context.cloudwatchMetricsQueueSize()); 49 | Runtime.getRuntime().addShutdownHook(new Thread() { 50 | @Override 51 | public void run() { 52 | // TODO: CWMetricsFactory does not wait for shutdown to 53 | // complete. Need to fix that. 54 | ((CWMetricsFactory) cwFactory).shutdown(); 55 | } 56 | }); 57 | factories.add(cwFactory); 58 | } 59 | 60 | if(factories.size() == 0) { 61 | factory = new NullMetricsFactory(); 62 | } else if(factories.size() == 1) { 63 | factory = factories.get(0); 64 | } else { 65 | factory = new CompositeMetricsFactory(factories); 66 | } 67 | } 68 | 69 | @Override 70 | public IMetricsScope beginScope() { 71 | IMetricsScope scope = factory.createScope(); 72 | return scope; 73 | } 74 | 75 | 76 | @SuppressWarnings("unchecked") 77 | public static T getMetric(Map metrics, Pattern key, T fallback) { 78 | T val = null; 79 | for (String metricKey : metrics.keySet()) { 80 | if (key.matcher(metricKey).matches()) { 81 | val = (T) metrics.get(metricKey); 82 | break; 83 | } 84 | } 85 | if (val != null) 86 | return val; 87 | else 88 | return fallback; 89 | } 90 | 91 | @SuppressWarnings({ "unchecked" }) 92 | public static T getMetric(Map metrics, String key, T fallback) { 93 | T val = (T) metrics.get(key); 94 | if (val != null) 95 | return val; 96 | else 97 | return fallback; 98 | } 99 | } -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/CWMetricsFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | import com.amazonaws.ClientConfiguration; 17 | import com.amazonaws.auth.AWSCredentialsProvider; 18 | import com.amazonaws.services.cloudwatch.AmazonCloudWatch; 19 | import com.amazonaws.services.cloudwatch.AmazonCloudWatchClient; 20 | 21 | /** 22 | * An IMetricsFactory that creates IMetricsScopes that output themselves via CloudWatch. Batches IMetricsScopes together 23 | * to reduce API calls. 24 | */ 25 | public class CWMetricsFactory implements IMetricsFactory { 26 | 27 | /* 28 | * If the CWPublisherRunnable accumulates more than FLUSH_SIZE distinct metrics, it will call CloudWatch 29 | * immediately instead of waiting for the next scheduled call. 30 | */ 31 | private static final int FLUSH_SIZE = 200; 32 | private final CWPublisherRunnable runnable; 33 | private final Thread publicationThread; 34 | 35 | /** 36 | * Constructor. 37 | * 38 | * @param credentialsProvider client credentials for CloudWatch 39 | * @param namespace the namespace under which the metrics will appear in the CloudWatch console 40 | * @param bufferTimeMillis time to buffer metrics before publishing to CloudWatch 41 | * @param maxQueueSize maximum number of metrics that we can have in a queue 42 | */ 43 | public CWMetricsFactory(AWSCredentialsProvider credentialsProvider, 44 | String namespace, 45 | long bufferTimeMillis, 46 | int maxQueueSize) { 47 | this(new AmazonCloudWatchClient(credentialsProvider), namespace, bufferTimeMillis, maxQueueSize); 48 | } 49 | 50 | /** 51 | * Constructor. 52 | * 53 | * @param credentialsProvider client credentials for CloudWatch 54 | * @param clientConfig Configuration to use with the AmazonCloudWatchClient 55 | * @param namespace the namespace under which the metrics will appear in the CloudWatch console 56 | * @param bufferTimeMillis time to buffer metrics before publishing to CloudWatch 57 | * @param maxQueueSize maximum number of metrics that we can have in a queue 58 | */ 59 | public CWMetricsFactory(AWSCredentialsProvider credentialsProvider, 60 | ClientConfiguration clientConfig, 61 | String namespace, 62 | long bufferTimeMillis, 63 | int maxQueueSize) { 64 | this(new AmazonCloudWatchClient(credentialsProvider, clientConfig), namespace, bufferTimeMillis, maxQueueSize); 65 | } 66 | 67 | /** 68 | * Constructor. 69 | * 70 | * @param cloudWatchClient Client used to make CloudWatch requests 71 | * @param namespace the namespace under which the metrics will appear in the CloudWatch console 72 | * @param bufferTimeMillis time to buffer metrics before publishing to CloudWatch 73 | * @param maxQueueSize maximum number of metrics that we can have in a queue 74 | */ 75 | public CWMetricsFactory(AmazonCloudWatch cloudWatchClient, 76 | String namespace, 77 | long bufferTimeMillis, 78 | int maxQueueSize) { 79 | DefaultCWMetricsPublisher metricPublisher = new DefaultCWMetricsPublisher(cloudWatchClient, namespace); 80 | 81 | this.runnable = 82 | new CWPublisherRunnable(metricPublisher, bufferTimeMillis, maxQueueSize, FLUSH_SIZE); 83 | 84 | this.publicationThread = new Thread(runnable); 85 | publicationThread.setName("cw-metrics-publisher"); 86 | publicationThread.start(); 87 | } 88 | 89 | @Override 90 | public IMetricsScope createScope() { 91 | return new CWMetricsScope(runnable); 92 | } 93 | 94 | public void shutdown() { 95 | runnable.shutdown(); 96 | } 97 | 98 | } 99 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/AbstractRecord.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import java.nio.ByteBuffer; 17 | import java.nio.charset.StandardCharsets; 18 | 19 | import com.amazon.kinesis.streaming.agent.ByteBuffers; 20 | import com.google.common.base.Preconditions; 21 | 22 | /** 23 | * Base implementation of an {@link IRecord} interface. 24 | */ 25 | public abstract class AbstractRecord implements IRecord { 26 | protected final ByteBuffer data; 27 | protected final TrackedFile file; 28 | protected final long startOffset; 29 | 30 | public AbstractRecord(TrackedFile file, long offset, ByteBuffer data) { 31 | Preconditions.checkArgument(offset >= 0, 32 | "The offset of a record (%s) must be a non-negative integer (File: %s)", 33 | offset, file); 34 | this.data = data; 35 | this.file = file; 36 | this.startOffset = offset; 37 | } 38 | 39 | public AbstractRecord(TrackedFile file, long offset, byte[] data) { 40 | this(file, offset, ByteBuffer.wrap(data)); 41 | } 42 | 43 | @Override 44 | public long dataLength() { 45 | return data.remaining(); 46 | } 47 | 48 | @Override 49 | public long length() { 50 | return dataLength(); 51 | } 52 | 53 | @Override 54 | public long endOffset() { 55 | return startOffset + dataLength(); 56 | } 57 | 58 | @Override 59 | public long startOffset() { 60 | return startOffset; 61 | } 62 | 63 | @Override 64 | public ByteBuffer data() { 65 | return data; 66 | } 67 | 68 | @Override 69 | public TrackedFile file() { 70 | return file; 71 | } 72 | 73 | @Override 74 | public void truncate() { 75 | if (length() > file.getFlow().getMaxRecordSizeBytes()) { 76 | byte[] terminatorBytes = file.getFlow().getRecordTerminatorBytes(); 77 | int originalPosition = data.position(); 78 | data.limit(originalPosition + getMaxDataSize()); 79 | // go to the position where we want to put the terminator 80 | data.position(originalPosition + getMaxDataSize() - terminatorBytes.length); 81 | // put the terminator 82 | // TODO: 83 | // We might have to handle the case where the last character of the truncated record contains 84 | // multiple bytes. In this case, the terminator itself might not be decoded as intended. 85 | data.put(terminatorBytes); 86 | data.position(originalPosition); 87 | } 88 | } 89 | 90 | /** 91 | * NOTE: Use for debugging only please. 92 | */ 93 | @Override 94 | public String toString() { 95 | StringBuilder sb = new StringBuilder(); 96 | String strData = ByteBuffers.toString(data, StandardCharsets.UTF_8).replace("\n", "\\n").replace(")", "\\)"); 97 | if (strData.length() > 50) 98 | strData = strData.substring(0, 47) + "..."; 99 | sb.append(getClass().getSimpleName()) 100 | .append("(file=").append(file) 101 | .append(",startOffset=").append(startOffset) 102 | .append(",endOffset=").append(endOffset()) 103 | .append(",length=").append(length()) 104 | .append(",lengthIncludingOverhead=").append(lengthWithOverhead()) 105 | .append(",data=(").append(strData).append(")") 106 | .append(")"); 107 | return sb.toString(); 108 | } 109 | 110 | /** 111 | * This size limit is used by {@link AbstractRecord#truncate()} 112 | * to truncate the data of the record to the limit 113 | * 114 | * @return max size of the data blob 115 | */ 116 | protected abstract int getMaxDataSize(); 117 | } 118 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Amazon Software License 2 | 3 | 1. Definitions 4 | "Licensor" means any person or entity that distributes its Work. 5 | 6 | "Software" means the original work of authorship made available under this License. 7 | 8 | "Work" means the Software and any additions to or derivative works of the Software that are made available under this License. 9 | 10 | The terms "reproduce," "reproduction," "derivative works," and "distribution" have the meaning as provided under U.S. copyright law; provided, however, that for the purposes of this License, derivative works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work. 11 | 12 | Works, including the Software, are "made available" under this License by including in or with the Work either (a) a copyright notice referencing the applicability of this License to the Work, or (b) a copy of this License. 13 | 14 | 2. License Grants 15 | 16 | 2.1 Copyright Grant. Subject to the terms and conditions of this License, each Licensor grants to you a perpetual, worldwide, non-exclusive, royalty-free, copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense and distribute its Work and any resulting derivative works in any form. 17 | 18 | 2.2 Patent Grant. Subject to the terms and conditions of this License, each Licensor grants to you a perpetual, worldwide, non-exclusive, royalty-free patent license to make, have made, use, sell, offer for sale, import, and otherwise transfer its Work, in whole or in part. The foregoing license applies only to the patent claims licensable by Licensor that would be infringed by Licensor’s Work (or portion thereof) individually and excluding any combinations with any other materials or technology. 19 | 20 | 3. Limitations 21 | 22 | 3.1 Redistribution. You may reproduce or distribute the Work only if (a) you do so under this License, (b) you include a complete copy of this License with your distribution, and (c) you retain without modification any copyright, patent, trademark, or attribution notices that are present in the Work. 23 | 24 | 3.2 Derivative Works. You may specify that additional or different terms apply to the use, reproduction, and distribution of your derivative works of the Work ("Your Terms") only if (a) Your Terms provide that the use limitation in Section 3.3 applies to your derivative works, and (b) you identify the specific derivative works that are subject to Your Terms. Notwithstanding Your Terms, this License (including the redistribution requirements in Section 3.1) will continue to apply to the Work itself. 25 | 26 | 3.3 Use Limitation. The Work and any derivative works thereof only may be used or intended for use with the web services, computing platforms or applications provided by Amazon.com, Inc. or its affiliates, including Amazon Web Services, Inc. 27 | 28 | 3.4 Patent Claims. If you bring or threaten to bring a patent claim against any Licensor (including any claim, cross-claim or counterclaim in a lawsuit) to enforce any patents that you allege are infringed by any Work, then your rights under this License from such Licensor (including the grants in Sections 2.1 and 2.2) will terminate immediately. 29 | 30 | 3.5 Trademarks. This License does not grant any rights to use any Licensor’s or its affiliates’ names, logos, or trademarks, except as necessary to reproduce the notices described in this License. 31 | 32 | 3.6 Termination. If you violate any term of this License, then your rights under this License (including the grants in Sections 2.1 and 2.2) will terminate immediately. 33 | 34 | 4. Disclaimer of Warranty. 35 | THE WORK IS PROVIDED "AS IS" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WARRANTIES OR CONDITIONS OF M ERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON-INFRINGEMENT. YOU BEAR THE RISK OF UNDERTAKING ANY ACTIVITIES UNDER THIS LICENSE. SOME STATES’ CONSUMER LAWS DO NOT ALLOW EXCLUSION OF AN IMPLIED WARRANTY, SO THIS DISCLAIMER MAY NOT APPLY TO YOU. 36 | 37 | 5. Limitation of Liability. 38 | EXCEPT AS PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE SHALL ANY LICENSOR BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF OR RELATED TO THIS LICENSE, THE USE OR INABILITY TO USE THE WORK (INCLUDING BUT NOT LIMITED TO LOSS OF GOODWILL, BUSINESS INTERRUPTION, LOST PROFITS OR DATA, COMPUTER FAILURE OR MALFUNCTION, OR ANY OTHER COMM ERCIAL DAMAGES OR LOSSES), EVEN IF THE LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/TrackedFileList.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import java.nio.file.Path; 17 | import java.util.AbstractList; 18 | import java.util.ArrayList; 19 | import java.util.Collections; 20 | import java.util.HashSet; 21 | import java.util.List; 22 | import java.util.Set; 23 | 24 | import com.google.common.base.Preconditions; 25 | 26 | /** 27 | * A snapshot of a a collection of files on the filesystem, ordered by last 28 | * modified time (newer-first) as would be retrieved by {@link SourceFile#listFiles(). 29 | */ 30 | public class TrackedFileList extends AbstractList { 31 | 32 | public static TrackedFileList emptyList() { 33 | return new TrackedFileList(Collections. emptyList()); 34 | } 35 | 36 | private final List snapshot; 37 | 38 | public TrackedFileList(List snapshot) { 39 | this.snapshot = new ArrayList<>(validate(snapshot)); 40 | } 41 | 42 | @Override 43 | public TrackedFile get(int index) { 44 | return snapshot.get(index); 45 | } 46 | 47 | @Override 48 | public int size() { 49 | return snapshot.size(); 50 | } 51 | 52 | @Override 53 | public TrackedFileList subList(int fromIndex, int toIndex) { 54 | return new TrackedFileList(snapshot.subList(fromIndex, toIndex)); 55 | } 56 | 57 | public int indexOfPath(Path path) { 58 | int i = 0; 59 | for(TrackedFile f : snapshot) { 60 | if(path.equals(f.getPath())) 61 | return i; 62 | ++i; 63 | } 64 | return -1; 65 | } 66 | 67 | public int indexOfFileId(FileId id) { 68 | int i = 0; 69 | for(TrackedFile f : snapshot) { 70 | if(id.equals(f.getId())) 71 | return i; 72 | ++i; 73 | } 74 | return -1; 75 | } 76 | 77 | /** 78 | * Performs following validations: 79 | *

    80 | *
  1. {@code files} is not {@code null}
    2. 81 | *
  2. No two files have the same path
    3. 82 | *
  3. No two files have the same {@code FileId}
    4. 83 | *
  4. Files are ordered newer-first
    5. 84 | *
85 | * @param files 86 | * @return The input parameter, if all is valid. 87 | * @throws NullPointerException if {@code files} is {@code null}. 88 | * @throws IllegalArgumentException if any of the other conditions are found. 89 | */ 90 | private List validate(List files) { 91 | Preconditions.checkNotNull(files); 92 | // Unique Path 93 | Set seenPaths = new HashSet<>(); 94 | for(TrackedFile f : files) { 95 | Path p = f.getPath().toAbsolutePath(); 96 | Preconditions.checkArgument(!seenPaths.contains(p), 97 | "File with path '" + p + "' shows up multiple times!"); 98 | seenPaths.add(p); 99 | } 100 | // Unique FileId 101 | Set seenIds = new HashSet<>(); 102 | for(TrackedFile f : files) { 103 | Preconditions.checkArgument(!seenIds.contains(f.getId()), 104 | "File with path '" + f.getPath() + "' has an ID " + f.getId() + " that shows up multiple times!"); 105 | seenIds.add(f.getId()); 106 | } 107 | // Order by lastModifiedTime descending 108 | long previousLastModifiedTime = -1; 109 | for(TrackedFile f : files) { 110 | if(previousLastModifiedTime >= 0) { 111 | Preconditions.checkArgument(f.getLastModifiedTime() <= previousLastModifiedTime, 112 | "File with path '" + f.getPath() + "' is older than previous file in the list."); 113 | } 114 | previousLastModifiedTime = f.getLastModifiedTime(); 115 | } 116 | 117 | // All good 118 | return files; 119 | } 120 | 121 | } 122 | -------------------------------------------------------------------------------- /bin/start-aws-kinesis-agent: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | MIN_JAVA_VERSION=1.7 4 | JAVA_START_HEAP=${JAVA_START_HEAP:-32m} 5 | JAVA_MAX_HEAP=${JAVA_MAX_HEAP:-512m} 6 | 7 | function _debug_msg() { 8 | if [[ -n "${_SCRIPT_DEBUG}" ]]; then 9 | echo -e "\e[32mDEBUG: $1\e[39m" >&2 10 | shift 11 | for text in "$@"; do 12 | echo -e "\e[32m ${text}\e[39m" >&2 13 | done 14 | fi 15 | } 16 | 17 | function _error_msg() { 18 | RED_START= 19 | echo -e "\e[31mERROR: $1\e[39m" >&2 20 | shift 21 | for text in "$@"; do 22 | echo -e "\e[31m ${text}\e[39m" >&2 23 | done 24 | } 25 | 26 | function check_java_version() { 27 | local javacmd=$1 28 | local verbose=$2 29 | local javaversion=$($javacmd -version 2>&1 | awk -F '"' '/version/ {print $2}') 30 | if [[ "$javaversion" < "$MIN_JAVA_VERSION" ]]; then 31 | if [[ -n "$verbose" ]]; then 32 | _error_msg "JVM version >= $MIN_JAVA_VERSION is required. Found following JVM:" \ 33 | "\$ $javacmd -version" \ 34 | "$($javacmd -version 2>&1 | sed '2,$s/^/ /g')" 35 | fi 36 | return 1 37 | else 38 | return 0 39 | fi 40 | } 41 | 42 | function get_java_brute_force() { 43 | _debug_msg="Looking for java by brute force..." 44 | candidates=$(find \ 45 | /usr/lib /usr/share/lib /opt/ /usr/ /lib \ 46 | -mindepth 1 -maxdepth 1 \ 47 | -type d \( -name java\* -o -name jre\* -o -name jdk\* \) \ 48 | -printf "%p/bin/java\n%p/jre/bin/java\n%p/jdk/bin/java\n" 2>/dev/null) 49 | for candidate in /usr/bin/java $(echo $candidates | sort -r); do 50 | _debug_msg "Examining java candidate: $candidate" 51 | if [[ -x "$candidate" ]] && check_java_version "$candidate"; then 52 | JAVACMD="$candidate" 53 | _debug_msg "Found JVACMD=$JAVACMD by brute-force" 54 | return 0 55 | fi 56 | done 57 | return 1 58 | } 59 | 60 | function get_java() { 61 | if [[ -n "${JAVACMD}" && -x "${JAVACMD}" ]]; then 62 | _debug_msg "Using environment JAVACMD directly." 63 | elif [[ -n "${JAVA_HOME}" ]]; then 64 | _debug_msg "Used environment JAVA_HOME to set JAVACMD." 65 | JAVACMD="${JAVA_HOME}/bin/java" 66 | elif [[ -n "${JAVA_BINDIR}" && -x "${JAVA_BINDIR}/java" ]]; then 67 | JAVACMD="${JAVA_BINDIR}/java" 68 | _debug_msg "Used JAVA_BINDIR to set JAVACMD" 69 | elif [[ -z "${JAVA_HOME}" && -f /usr/share/java-utils/java-functions ]]; then 70 | # JPackage standard method to set JAVA_HOME 71 | . /usr/share/java-utils/java-functions; set_jvm 72 | _debug_msg "Used JPackage to find JAVA_HOME." 73 | if [[ -x "${JAVA_HOME}/bin/java" ]] \ 74 | && check_java_version "${JAVA_HOME}/bin/java"; then 75 | JAVACMD="${JAVA_HOME}/bin/java" 76 | _debug_msg "Used JAVA_HOME to set JAVACMD." 77 | else 78 | # don't pollute environment with a JAVA_HOME 79 | unset JAVA_HOME 80 | fi 81 | fi 82 | 83 | # if no luck still, see if java is on the path 84 | if [[ -z "${JAVACMD}" || ! -x "${JAVACMD}" ]] \ 85 | || ! check_java_version "${JAVACMD}"; then 86 | JAVACMD=$(which java) 87 | if [[ -n "${JAVACMD}" && -x "${JAVACMD}" ]] \ 88 | && check_java_version "${JAVACMD}"; then 89 | _debug_msg "Used PATH to find JAVACMD." 90 | else 91 | # finally, try brute force search 92 | get_java_brute_force 93 | fi 94 | fi 95 | 96 | # if we were successful, we return 0 else we complain and return 1 97 | if [[ -n "${JAVACMD}" && -x "${JAVACMD}" ]]; then 98 | _debug_msg "Using JAVACMD='$JAVACMD'" 99 | check_java_version $JAVACMD verbose 100 | return $? 101 | else 102 | _error_msg "Couldn't locate a java virtual machine." \ 103 | "Please either set JAVACMD, JAVA_BINDIR or JAVA_HOME " \ 104 | "or make sure java is in your PATH." 105 | return 1 106 | fi 107 | } 108 | 109 | get_java 110 | if [[ $? != 0 ]]; then 111 | exit 1 112 | fi 113 | 114 | set -e 115 | DAEMON_NAME=aws-kinesis-agent 116 | JAVA_DIR="/usr/share/java" 117 | BIN_DIR=$(cd $(dirname "$0") && pwd) 118 | LIB_DIR=$(cd "$BIN_DIR"/../share/$DAEMON_NAME/lib && pwd) 119 | _debug_msg "BIN_DIR=${BIN_DIR}" 120 | _debug_msg "LIB_DIR=${LIB_DIR}" 121 | _debug_msg "PWD=${PWD}" 122 | CLASSPATH="$JAVA_DIR"/*:"$LIB_DIR":$(find "$LIB_DIR" -type f -name \*.jar | paste -s -d:):"$CLASSPATH" 123 | _debug_msg CLASSPATH=$CLASSPATH 124 | set +e 125 | 126 | JVM_ARGS="-server -Xms${JAVA_START_HEAP} -Xmx${JAVA_MAX_HEAP} $JVM_ARGS" 127 | 128 | MAIN_CLASS="com.amazon.kinesis.streaming.agent.Agent" 129 | 130 | exec $JAVACMD $JVM_ARGS \ 131 | -cp "$CLASSPATH" \ 132 | $MAIN_CLASS "$@" 133 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/MetricAccumulatingQueue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | import java.util.ArrayList; 17 | import java.util.HashMap; 18 | import java.util.List; 19 | import java.util.Map; 20 | import java.util.concurrent.BlockingQueue; 21 | import java.util.concurrent.LinkedBlockingQueue; 22 | 23 | import com.amazonaws.services.cloudwatch.model.MetricDatum; 24 | import com.amazonaws.services.cloudwatch.model.StatisticSet; 25 | 26 | /** 27 | * Helper class for accumulating MetricDatums with the same name and dimensions. 28 | * 29 | * @param can be a class or object defined by the user that stores information about a MetricDatum needed 30 | * by the user. 31 | * 32 | * The following is a example of what a KeyType class might look like: 33 | * class SampleKeyType { 34 | * private long timeKeyCreated; 35 | * private MetricDatum datum; 36 | * public SampleKeyType(long timeKeyCreated, MetricDatum datum){ 37 | * this.timeKeyCreated = timeKeyCreated; 38 | * this.datum = datum; 39 | * } 40 | * } 41 | */ 42 | public class MetricAccumulatingQueue { 43 | 44 | // Queue is for first in first out behavior 45 | private BlockingQueue> queue; 46 | // Map is for constant time lookup by key 47 | private Map map; 48 | 49 | public MetricAccumulatingQueue(int maxQueueSize) { 50 | queue = new LinkedBlockingQueue>(maxQueueSize); 51 | map = new HashMap(); 52 | } 53 | 54 | /** 55 | * @param maxItems number of items to remove from the queue. 56 | * @return a list of MetricDatums that are no longer contained within the queue or map. 57 | */ 58 | public synchronized List> drain(int maxItems) { 59 | List> drainedItems = new ArrayList>(maxItems); 60 | 61 | queue.drainTo(drainedItems, maxItems); 62 | 63 | for (MetricDatumWithKey datumWithKey : drainedItems) { 64 | map.remove(datumWithKey.key); 65 | } 66 | 67 | return drainedItems; 68 | } 69 | 70 | public synchronized boolean isEmpty() { 71 | return queue.isEmpty(); 72 | } 73 | 74 | public synchronized int size() { 75 | return queue.size(); 76 | } 77 | 78 | /** 79 | * We use a queue and a map in this method. The reason for this is because, the queue will keep our metrics in 80 | * FIFO order and the map will provide us with constant time lookup to get the appropriate MetricDatum. 81 | * 82 | * @param key metric key to be inserted into queue 83 | * @param datum metric to be inserted into queue 84 | * @return a boolean depending on whether the datum was inserted into the queue 85 | */ 86 | public synchronized boolean offer(KeyType key, MetricDatum datum) { 87 | MetricDatum old = map.get(key); 88 | if (old == null) { 89 | boolean offered = queue.offer(new MetricDatumWithKey(key, datum)); 90 | if (offered) { 91 | map.put(key, datum); 92 | } 93 | 94 | return offered; 95 | } else { 96 | accumulate(old, datum); 97 | return true; 98 | } 99 | } 100 | 101 | private void accumulate(MetricDatum oldDatum, MetricDatum newDatum) { 102 | if (!oldDatum.getUnit().equals(newDatum.getUnit())) { 103 | throw new IllegalArgumentException("Unit mismatch for datum named " + oldDatum.getMetricName()); 104 | } 105 | 106 | StatisticSet oldStats = oldDatum.getStatisticValues(); 107 | StatisticSet newStats = newDatum.getStatisticValues(); 108 | 109 | oldStats.setSampleCount(oldStats.getSampleCount() + newStats.getSampleCount()); 110 | oldStats.setMaximum(Math.max(oldStats.getMaximum(), newStats.getMaximum())); 111 | oldStats.setMinimum(Math.min(oldStats.getMinimum(), newStats.getMinimum())); 112 | oldStats.setSum(oldStats.getSum() + newStats.getSum()); 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/config/AgentOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.config; 15 | 16 | import java.io.File; 17 | 18 | import lombok.Getter; 19 | 20 | import org.apache.commons.lang3.ArrayUtils; 21 | 22 | import com.beust.jcommander.IParameterValidator; 23 | import com.beust.jcommander.JCommander; 24 | import com.beust.jcommander.Parameter; 25 | import com.beust.jcommander.ParameterException; 26 | import com.beust.jcommander.Parameters; 27 | import com.google.common.base.Joiner; 28 | import com.google.common.collect.Range; 29 | 30 | @Parameters(separators = "=") 31 | public class AgentOptions { 32 | 33 | private static final String DEFAULT_CONFIG_FILE = "/etc/aws-kinesis/agent.json"; 34 | private static final String[] VALID_LOG_LEVELS = { "TRACE", "DEBUG", "INFO", "WARN", "ERROR" }; 35 | 36 | @Parameter(names = { "--configuration", "-c" }, description = "Path to the configuration file for the agent.", validateWith = FileReadableValidator.class) 37 | @Getter String configFile = DEFAULT_CONFIG_FILE; 38 | 39 | @Parameter(names = { "--log-file", "-l" }, description = "Path to the agent's log file.") 40 | @Getter String logFile = null; 41 | 42 | @Parameter(names = { "--log-level", "-L" }, description = "Log level. Can be one of: TRACE,DEBUG,INFO,WARN,ERROR.", validateWith = LogLevelValidator.class) 43 | @Getter String logLevel = null; 44 | 45 | @Parameter(names = { "--help", "-h" }, help = true, description = "Display this help message") 46 | Boolean help; 47 | 48 | public static AgentOptions parse(String[] args) { 49 | AgentOptions opts = new AgentOptions(); 50 | JCommander jc = new JCommander(opts); 51 | jc.setProgramName("aws-kinesis-agent"); 52 | try { 53 | jc.parse(args); 54 | } catch (ParameterException e) { 55 | System.err.println(e.getMessage()); 56 | jc.usage(); 57 | System.exit(1); 58 | } 59 | if (Boolean.TRUE.equals(opts.help)) { 60 | jc.usage(); 61 | System.exit(0); 62 | } 63 | return opts; 64 | } 65 | 66 | public static class LongRangeValidator implements IParameterValidator { 67 | private final Range range; 68 | 69 | public LongRangeValidator(Range range) { 70 | this.range = range; 71 | } 72 | 73 | @Override 74 | public void validate(String name, String value) 75 | throws ParameterException { 76 | try { 77 | long longVal = Long.parseLong(value); 78 | if (!this.range.contains(longVal)) 79 | throw new ParameterException("Parameter " + name 80 | + " should be within the range " 81 | + this.range.toString() + ". Value " + value 82 | + " is not valid."); 83 | } catch (NumberFormatException e) { 84 | throw new ParameterException("Parameter " + name 85 | + " is not a valid number: " + value); 86 | } 87 | 88 | } 89 | } 90 | 91 | public static class LogLevelValidator implements IParameterValidator { 92 | @Override 93 | public void validate(String name, String value) 94 | throws ParameterException { 95 | if (ArrayUtils.indexOf(VALID_LOG_LEVELS, value) < 0) 96 | throw new ParameterException("Valid values for parameter " 97 | + name + " are: " 98 | + Joiner.on(",").join(VALID_LOG_LEVELS) + ". Value " 99 | + value + " is not valid."); 100 | } 101 | 102 | } 103 | 104 | public static class FileReadableValidator implements IParameterValidator { 105 | @Override 106 | public void validate(String name, String value) 107 | throws ParameterException { 108 | File f = new File(value); 109 | if (!f.exists()) { 110 | throw new ParameterException("Parameter " + name 111 | + " points to a file that doesn't exist: " + value); 112 | } 113 | if (!f.canRead()) { 114 | throw new ParameterException("Parameter " + name 115 | + " points to a file that's not accessible: " + value); 116 | } 117 | 118 | } 119 | 120 | } 121 | } -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/SourceFile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import java.io.IOException; 17 | import java.nio.file.DirectoryStream; 18 | import java.nio.file.FileSystems; 19 | import java.nio.file.Files; 20 | import java.nio.file.Path; 21 | import java.nio.file.PathMatcher; 22 | import java.util.ArrayList; 23 | import java.util.Collections; 24 | import java.util.List; 25 | 26 | import lombok.EqualsAndHashCode; 27 | import lombok.Getter; 28 | 29 | import com.google.common.base.Preconditions; 30 | 31 | /** 32 | * Specification of the file(s) to be tailed. 33 | */ 34 | @EqualsAndHashCode(exclude = { "pathMatcher" }) 35 | public class SourceFile { 36 | @Getter private final FileFlow flow; 37 | @Getter private final Path directory; 38 | @Getter private final Path filePattern; 39 | private final PathMatcher pathMatcher; 40 | 41 | public SourceFile(FileFlow flow, String filePattern) { 42 | this.flow = flow; 43 | // fileName 44 | Preconditions.checkArgument(!filePattern.endsWith("/"), "File name component is empty!"); 45 | Path filePath = FileSystems.getDefault().getPath(filePattern); 46 | // TODO: this does not handle globs in directory component: e.g. /opt/*/logs/app.log, /opt/**/app.log* 47 | this.directory = filePath.getParent(); 48 | validateDirectory(this.directory); 49 | this.filePattern = filePath.getFileName(); 50 | this.pathMatcher = FileSystems.getDefault().getPathMatcher("glob:" + this.filePattern.toString()); 51 | } 52 | 53 | /** 54 | * @return List of {@link Path} objects contained in the given directory 55 | * and that match the file name pattern, sorted by {@code lastModifiedTime} 56 | * descending (newest at the top). An empty list is returned if 57 | * {@link #directory} does not exist, or if there are no files 58 | * that match the pattern. 59 | * @throws IOException If there was an error reading the directory or getting 60 | * the {@code lastModifiedTime} of a directory. Note that if 61 | * the {@link #directory} doesn't exist no exception will be thrown 62 | * but an empty list is returned instead. 63 | */ 64 | public TrackedFileList listFiles() throws IOException { 65 | if(!Files.exists(this.directory)) 66 | return TrackedFileList.emptyList(); 67 | 68 | List files = new ArrayList<>(); 69 | try (DirectoryStream directoryStream = Files.newDirectoryStream(this.directory)) { 70 | for (Path p : directoryStream) { 71 | if (this.pathMatcher.matches(p.getFileName())) { 72 | files.add(new TrackedFile(flow, p)); 73 | } 74 | } 75 | } 76 | // sort the files by decsending last modified time and return 77 | Collections.sort(files, new TrackedFile.NewestFirstComparator()); 78 | return new TrackedFileList(files); 79 | } 80 | 81 | /** 82 | * @return The number of files on the file system that match the given input 83 | * pattern. More lightweight than {@link #listFiles()}. 84 | * @throws IOException If there was an error reading the directory or getting 85 | * the {@code lastModifiedTime} of a directory. Note that if 86 | * the {@link #directory} doesn't exist no exception will be thrown 87 | * but {@code 0} is returned instead. 88 | */ 89 | public int countFiles() throws IOException { 90 | int count = 0; 91 | if(Files.exists(this.directory)) { 92 | try (DirectoryStream directoryStream = Files.newDirectoryStream(this.directory)) { 93 | for (Path p : directoryStream) { 94 | if (this.pathMatcher.matches(p.getFileName())) { 95 | ++count; 96 | } 97 | } 98 | } 99 | } 100 | return count; 101 | } 102 | 103 | @Override 104 | public String toString() { 105 | return this.directory + "/" + this.filePattern; 106 | } 107 | 108 | /** 109 | * Performs basic validation on the directory parameter making sure it fits 110 | * within the supported functionality of this class. 111 | * @param dir 112 | */ 113 | private void validateDirectory(Path dir) { 114 | Preconditions.checkArgument(dir != null, "Directory component is empty!"); 115 | // TODO: validate that the directory component has no glob characters 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/RecordBuffer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import java.util.ArrayList; 17 | import java.util.Iterator; 18 | import java.util.List; 19 | import java.util.concurrent.atomic.AtomicLong; 20 | 21 | import javax.annotation.concurrent.NotThreadSafe; 22 | 23 | import com.google.common.annotations.VisibleForTesting; 24 | import com.google.common.primitives.Ints; 25 | 26 | /** 27 | * An record buffer. 28 | * 29 | * @param The record type. 30 | */ 31 | @NotThreadSafe 32 | public class RecordBuffer implements Iterable { 33 | private static final int DEFAULT_INITIAL_CAPACITY = 500; 34 | private static final AtomicLong NEXT_BUFFER_ID = new AtomicLong(1); 35 | 36 | protected final FileFlow flow; 37 | protected final List records; 38 | 39 | protected long timestamp = -1; 40 | /** Cumulative size of records including any per-record overhead. */ 41 | protected long currentSizeBytes = 0; 42 | protected IRecord lastRecord; 43 | protected final long id; 44 | 45 | public RecordBuffer(FileFlow flow) { 46 | this.flow = flow; 47 | this.records = new ArrayList(DEFAULT_INITIAL_CAPACITY); 48 | this.id = NEXT_BUFFER_ID.incrementAndGet(); 49 | } 50 | 51 | /** 52 | * Adds a record to the buffer. 53 | * @param record 54 | */ 55 | public void add(R record) { 56 | records.add(record); 57 | lastRecord = record; 58 | currentSizeBytes += record.lengthWithOverhead(); 59 | if(timestamp < 0) { 60 | timestamp = System.currentTimeMillis(); 61 | } 62 | } 63 | 64 | /** 65 | * @return A unique sequence number for this buffer. 66 | */ 67 | public long id() { 68 | return id; 69 | } 70 | 71 | public FileFlow flow() { 72 | return flow; 73 | } 74 | 75 | public long timestamp() { 76 | return timestamp; 77 | } 78 | 79 | public long age() { 80 | return age(System.currentTimeMillis()); 81 | } 82 | 83 | @VisibleForTesting 84 | long age(long at) { 85 | return timestamp > 0 ? (at - timestamp) : 0; 86 | } 87 | 88 | public int sizeRecords() { 89 | return records.size(); 90 | } 91 | 92 | public long sizeBytes() { 93 | return currentSizeBytes; 94 | } 95 | 96 | public long sizeBytesWithOverhead() { 97 | return currentSizeBytes + flow.getPerBufferOverheadBytes(); 98 | } 99 | 100 | public boolean isEmpty() { 101 | return records.isEmpty(); 102 | } 103 | 104 | public TrackedFile checkpointFile() { 105 | return lastRecord.file(); 106 | } 107 | 108 | public long checkpointOffset() { 109 | return lastRecord.endOffset(); 110 | } 111 | 112 | /** 113 | * Removes the records at the specified indices. 114 | * @param itemsToRemoveSorted Indices of the records to be removed. 115 | * @return A buffer identical to the current one with the specified records 116 | * removed. 117 | */ 118 | public RecordBuffer remove(int... itemsToRemoveSorted) { 119 | return remove(Ints.asList(itemsToRemoveSorted)); 120 | } 121 | 122 | /** 123 | * Removes the records at the specified indices. 124 | * @param itemsToRemoveSorted Indices of the records to be removed. 125 | * @return A buffer identical to the current one with the specified records 126 | * removed. 127 | */ 128 | public RecordBuffer remove(List itemsToRemoveSorted) { 129 | if(!itemsToRemoveSorted.isEmpty()) { 130 | Iterator toRemoveIt = itemsToRemoveSorted.iterator(); 131 | int toRemoveIndex = toRemoveIt.next(); 132 | int newIndex = toRemoveIndex; 133 | for(int originalIndex = toRemoveIndex; originalIndex < records.size(); ++originalIndex) { 134 | if(originalIndex == toRemoveIndex) { 135 | currentSizeBytes -= records.get(toRemoveIndex).length(); 136 | toRemoveIndex = toRemoveIt.hasNext() ? toRemoveIt.next() : -1; 137 | } else { 138 | records.set(newIndex++, records.get(originalIndex)); 139 | } 140 | } 141 | // Shrink list to include only non-null elements 142 | records.subList(newIndex, records.size()).clear(); 143 | } 144 | return this; 145 | } 146 | 147 | @Override 148 | public Iterator iterator() { 149 | return records.iterator(); 150 | } 151 | 152 | /** 153 | * NOTE: Use for debugging only please. 154 | */ 155 | @Override 156 | public String toString() { 157 | StringBuilder sb = new StringBuilder(); 158 | sb.append(getClass().getSimpleName()) 159 | .append("(id=").append(id) 160 | .append(",records=").append(sizeRecords()) 161 | .append(",bytes=").append(sizeBytes()) 162 | .append(")"); 163 | return sb.toString(); 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Amazon Kinesis Agent 2 | 3 | The **Amazon Kinesis Agent** is a stand-alone Java software application that offers an easier way to collect and ingest data into [Amazon Kinesis][kinesis] services, including [Amazon Kinesis Streams][kinesis-stream] and [Amazon Kinesis Firehose][kinesis-firehose]. 4 | 5 | * [Amazon Kinesis details][kinesis] 6 | * [Forum][kinesis-forum] 7 | * [Issues][kinesis-agent-issues] 8 | 9 | ## Features 10 | 11 | * Monitors file patterns and sends new data records to delivery streams 12 | * Handles file rotation, checkpointing, and retry upon failure 13 | * Delivers all data in a reliable, timely, and simpler manner 14 | * Emits Amazon CloudWatch metrics to help you better monitor and troubleshoot the streaming process 15 | 16 | ## Getting started 17 | 18 | 1. **Sign up for AWS** — Before you begin, you need an AWS account. For more information about creating an AWS account and retrieving your AWS credentials, see [AWS Account and Credentials][docs-signup] in the AWS SDK for Java Developer Guide. 19 | 2. **Sign up for Amazon Kinesis** — Go to the Amazon Kinesis console to sign up for the service and create a Amazon Kinesis stream or Firehose delivery stream. For more information, see [Create an Amazon Kinesis Stream][kinesis-stream-create] in the *Amazon Kinesis Streams Developer Guide* or [Create an Amazon Kinesis Firehose Delivery Stream][kinesis-firehose-create] in the *Amazon Kinesis Firehose Developer Guide*. 20 | 3. **Minimum requirements** — To start the Amazon Kinesis Agent, you need **Java 1.7+**. 21 | 4. **Using the Amazon Kinesis Agent** — For more information about using the Amazon Kinesis Agent to deliver data to Streams and Firehose, see [Writing to Amazon Kinesis with Agents][kinesis-stream-agent-guide] and [Writing to Delivery Streams with Agents][kinesis-firehose-agent-guide]. 22 | 23 | ## Installing Amazon Kinesis Agent 24 | After you've downloaded the code from GitHub, you can install the Amazon Kinesis Agent with the following command: 25 | 26 | ```sh 27 | sudo ./setup --install 28 | ``` 29 | 30 | This setup script downloads all the dependencies and bootstraps the environment for running the Java program. 31 | 32 | ## Configuring and starting Amazon Kinesis Agent 33 | After the agent is installed, the configuration file can be found in `/etc/aws-kinesis/agent.json`. You need to modify this configuration file to set the data destinations and AWS credentials, and to point the agent to the files to push. After you complete the configuration, you can make the agent start automatically at system startup with the following command: 34 | 35 | ```sh 36 | sudo chkconfig aws-kinesis-agent on 37 | ``` 38 | 39 | If you do not want the agent running at system startup, turn it off with the following command: 40 | 41 | ```sh 42 | sudo chkconfig aws-kinesis-agent off 43 | ``` 44 | 45 | To start the agent manually, use the following command: 46 | 47 | ```sh 48 | sudo service aws-kinesis-agent start 49 | ``` 50 | 51 | You can make sure the agent is running with the following command: 52 | 53 | ```sh 54 | sudo service aws-kinesis-agent status 55 | ``` 56 | 57 | You may see messages such as `aws-kinesis-agent (pid [PID]) is running...` 58 | 59 | To stop the agent, use the following command: 60 | 61 | ```sh 62 | sudo service aws-kinesis-agent stop 63 | ``` 64 | 65 | ## Viewing the Amazon Kinesis Agent log file 66 | The agent writes its logs to `/var/log/aws-kinesis-agent/aws-kinesis-agent.log` 67 | 68 | ## Uninstalling Amazon Kinesis Agent 69 | To uninstall the agent, use the following command: 70 | 71 | ```sh 72 | sudo ./setup --uninstall 73 | ``` 74 | 75 | ## Building from Source 76 | 77 | The installation done by the setup script is only tested on **Red Hat Enterprise Linux** version 7 or later and **Amazon Linux AMI** version 2015.09 or later. For other distributions or platforms, you can build the Java project with the following command: 78 | 79 | ```sh 80 | sudo ./setup --build 81 | ``` 82 | 83 | or by using Ant target as you would build any Java program: 84 | 85 | ```sh 86 | ant [-Dbuild.dependencies=DEPENDENCY_DIR] 87 | ``` 88 | 89 | If you use Ant command, you need to download all the dependencies listed in pom.xml before building the Java program. **DEPENDENCY_DIR** is the directory where you download and store the dependencies. 90 | By default, the Amazon Kinesis Agent reads the configuration file from /etc/aws-kinesis/agent.json. You need to create such a file if it does not already exist. A sample configuration can be found at ./configuration/release/aws-kinesis-agent.json 91 | 92 | To start the program, use the following command: 93 | 94 | ```sh 95 | java -cp CLASSPATH "com.amazon.kinesis.streaming.agent.Agent" 96 | ``` 97 | 98 | **CLASSPATH** is the classpath to your dependencies and the target JAR file that you built from the step above. 99 | 100 | ## Release Notes 101 | ### Release 1.0 (October 7, 2015) 102 | * This is the first release. 103 | 104 | [kinesis]: http://aws.amazon.com/kinesis 105 | [kinesis-forum]: http://developer.amazonwebservices.com/connect/forum.jspa?forumID=169 106 | [kinesis-agent-issues]: https://github.com/awslabs/amazon-kinesis-agent/issues 107 | [docs-signup]: http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-setup.html 108 | [kinesis-stream]: https://aws.amazon.com/kinesis/streams/ 109 | [kinesis-firehose]: https://aws.amazon.com/kinesis/firehose/ 110 | [kinesis-stream-create]: http://docs.aws.amazon.com/kinesis/latest/dev/kinesis-using-sdk-java-create-stream.html 111 | [kinesis-firehose-create]: http://docs.aws.amazon.com/firehose/latest/dev/basic-create.html 112 | [kinesis-stream-agent-guide]: http://docs.aws.amazon.com/kinesis/latest/dev/writing-with-agents.html 113 | [kinesis-firehose-agent-guide]: http://docs.aws.amazon.com/firehose/latest/dev/writing-with-agents.html -------------------------------------------------------------------------------- /bin/aws-kinesis-agent: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | ### BEGIN INIT INFO 4 | # Provides: aws-kinesis-agent 5 | # Required-Start: $network 6 | # Required-Stop: $network 7 | # Short-Description: Daemon for Amazon Kinesis Agent. 8 | # Description: This script is responsible for running Amazon Kinesis Agent 9 | # as a daemon. 10 | ### END INIT INFO 11 | 12 | # chkconfig: - 30 70 13 | # description: This script is responsible for running Amazon Kinesis Agent as a daemon. 14 | # 15 | # config: /etc/aws-kinesis/agent.json 16 | # config: /etc/sysconfig/aws-kinesis-agent 17 | 18 | # get init functions: checkpid(), status(), echo_success(), echo_failure() 19 | . /etc/init.d/functions 20 | 21 | # load any configs/environment from /etc/sysconfig/ 22 | [[ -f /etc/sysconfig/$DAEMON_NAME ]] && . /etc/sysconfig/$DAEMON_NAME 23 | 24 | # Log that something succeeded 25 | success() { 26 | [ "$BOOTUP" != "verbose" -a -z "${LSB:-}" ] && echo_success 27 | echo $1 28 | return 0 29 | } 30 | 31 | # Log that something failed 32 | failure() { 33 | local rc=$? 34 | [ "$BOOTUP" != "verbose" -a -z "${LSB:-}" ] && echo_failure 35 | echo $1 36 | return $rc 37 | } 38 | 39 | #set -x 40 | DAEMON_NAME=aws-kinesis-agent 41 | AGENT_USER=aws-kinesis-agent-user 42 | PIDFILE=/var/run/$DAEMON_NAME.pid 43 | STATE_HOME=/var/run/$DAEMON_NAME 44 | LOG_DIR=/var/log/$DAEMON_NAME 45 | [[ -d $STATE_HOME ]] || mkdir -p $STATE_HOME 46 | MUTEXFILE=$STATE_HOME/mutex 47 | SHUTDOWN_TIME=11 #10 second default value in AgentConfiguration.java, +1 second buffer 48 | 49 | INITLOGFILE=/tmp/$DAEMON_NAME.`date '+%Y%m%d%H%M%S'`.initlog 50 | 51 | # This script is in /etc/rc.d/init.d/ and the executable is in /usr/bin 52 | BASEDIR=${BASEDIR%/} 53 | DAEMON_EXEC=$BASEDIR/usr/bin/start-$DAEMON_NAME 54 | SYSCONFIG_FILE=/etc/sysconfig/$DAEMON_NAME 55 | RETVAL=0 56 | 57 | do_start () { 58 | # if lock file is there or if PID file is there and process is alive, return 59 | if [[ -e $PIDFILE ]] && checkpid `cat $PIDFILE`; then 60 | echo "$DAEMON_NAME already running. Use 'restart' command to restart it." 61 | return 62 | fi 63 | 64 | [ -f $SYSCONFIG_FILE ] && . $SYSCONFIG_FILE 65 | export AWS_ACCESS_KEY_ID 66 | export AWS_SECRET_ACCESS_KEY 67 | export AWS_DEFAULT_REGION 68 | 69 | ( 70 | flock -w 10 -x 9 71 | DAEMON_NAME=$DAEMON_NAME nohup runuser $AGENT_USER -s /bin/sh -c "$DAEMON_EXEC $@" > $INITLOGFILE 2>&1 & 72 | 73 | pid=$! 74 | echo $pid > $PIDFILE 75 | 76 | # wait a bit and make sure process still there 77 | sleep 2 78 | checkpid $pid 79 | RETVAL=$? 80 | # create the lock file 81 | if [[ $RETVAL != 0 && -s $INITLOGFILE ]]; then 82 | echo "Initialization logs can be found in $INITLOGFILE" >&2 83 | cat $INITLOGFILE >&2 84 | else 85 | rm -f $INITLOGFILE 86 | fi 87 | 88 | # output status message 89 | [[ $RETVAL == 0 ]] && success "$DAEMON_NAME startup" || failure "$DAEMON_NAME startup" 90 | 91 | ) 9>$MUTEXFILE 92 | RETVAL=$? 93 | rm -f $MUTEXFILE 94 | return $RETVAL 95 | } 96 | 97 | get_pids() { 98 | # e.g. start-daemon -> [s]tart-daemon 99 | # this serves to exclude the grep command itself from matching process list 100 | PROCESS_PATTERN=`basename $DAEMON_EXEC | sed 's/\([a-zA-Z0-9]\)/[\1]/'` 101 | ps axo pid,args | grep "$PROCESS_PATTERN" 102 | } 103 | 104 | do_stop () { 105 | ( 106 | flock -w 10 -x 9 107 | ppids=`get_pids | awk '{print $1}'` 108 | if [[ $? == 0 ]]; then 109 | for pid in $ppids; do 110 | pkill -TERM -P $pid > /dev/null 2>&1 111 | done 112 | 113 | i=0 114 | while [[ $i -lt $SHUTDOWN_TIME ]] && get_pids > /dev/null; do 115 | sleep 1 116 | (( i = i + 1 )) 117 | done 118 | 119 | ppids=`get_pids | awk '{print $1}'` 120 | if [[ $? == 0 ]]; then 121 | for pid in $ppids; do 122 | pkill -KILL -P $pid > /dev/null 2>&1 123 | done 124 | fi 125 | procList=`get_pids` 126 | if [ $? != 0 ]; then 127 | RETVAL=0 128 | else 129 | RETVAL=1 130 | fi 131 | fi 132 | 133 | # finally validate that the pid in the PIDFILE is gone 134 | if [[ $RETVAL == 0 && -e $PIDFILE ]] && checkpid `cat $PIDFILE`; then 135 | RETVAL=1 136 | fi 137 | 138 | # all OK? cleanup 139 | [[ $RETVAL == 0 ]] && rm -f $PIDFILE 140 | 141 | # print status message 142 | [[ $RETVAL == 0 ]] && success "$DAEMON_NAME shutdown" || failure "$DAEMON_NAME shutdown" 143 | ) 9>$MUTEXFILE 144 | RETVAL=$? 145 | rm -f $MUTEXFILE 146 | return $RETVAL 147 | } 148 | 149 | do_status () { 150 | status -p $PIDFILE $DAEMON_NAME 151 | } 152 | 153 | do_restart () { 154 | do_stop && do_start 155 | } 156 | 157 | do_condrestart () { 158 | if do_status >&/dev/null; then 159 | do_restart 160 | fi 161 | } 162 | 163 | do_install () { 164 | local dest_dir=/etc/aws-kinesis 165 | local src_dir=$BASEDIR/etc/aws-kinesis 166 | local configfile=agent.json 167 | if [[ "$src_dir" != "$dest_dir" ]]; then 168 | mkdir -p "$dest_dir" 169 | if [[ -f "$dest_dir/$configfile" ]]; then 170 | echo "Configuration file found at: $dest_dir/$configfile" 171 | else 172 | cp "$src_dir/$configfile" "$dest_dir/" 173 | echo "Configuration file installed at: $dest_dir/$configfile" 174 | fi 175 | fi 176 | echo "Configuration details:" 177 | cat "$dest_dir/$configfile" 178 | echo "" 179 | echo "Amazon Kinesis Streaming Data Agent is installed at $BASEDIR" 180 | echo "" 181 | echo "To start the $DAEMON_NAME service, run:" 182 | echo " sudo service $DAEMON_NAME start" 183 | echo "To stop the $DAEMON_NAME service, run:" 184 | echo " sudo service $DAEMON_NAME stop" 185 | echo "To check the status of the $DAEMON_NAME service, run:" 186 | echo " sudo service $DAEMON_NAME status" 187 | echo "" 188 | echo "$DAEMON_NAME log file will be found at: $LOG_DIR" 189 | } 190 | 191 | command=$1 192 | shift 193 | case "$command" in 194 | start) 195 | do_start "$@" 196 | ;; 197 | stop) 198 | do_stop 199 | ;; 200 | restart) 201 | do_restart "$@" 202 | ;; 203 | condrestart) 204 | do_condrestart "$@" 205 | ;; 206 | status) 207 | do_status 208 | ;; 209 | install) 210 | do_install 211 | ;; 212 | *) 213 | echo "Usage: $0 {start|stop|restart|condrestart|status|install}" 214 | exit 1 215 | ;; 216 | esac 217 | exit $RETVAL -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/IParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import java.util.Map; 17 | 18 | /** 19 | * The inteface for a record parser that defines the protocol or parsing records 20 | * from an open channel. 21 | * 22 | * To start reading from a file channel, the user of this interface needs to call 23 | * {@link #setCurrentFile(TrackedFile, boolean) setCurrentFile} with an open file. 24 | * This file can be called at a later time as needed as the underlying file rotates. 25 | * After that, successive calls to {@link #readRecord()} will yield the successive 26 | * records read from the file channel. When no more complete records can be 27 | * read from the underlying channel, {@code null} will be returned. The caller 28 | * is then responsible to figure out what's the reason (e.g. simple EOF or 29 | * rotation happened), and either keep reading from the same file or switching 30 | * to another file by calling {@code setCurrentFile(TrackedFile, boolean) setCurrentFile} 31 | * again. 32 | * 33 | * @param The record type that this parser recognizes and produces. 34 | */ 35 | public interface IParser { 36 | /** 37 | * Sets up the parser to read records from the given {@code file}, and 38 | * advances to the initial position specified in the 39 | * {@link FileFlow#getInitialPosition()} configuration parameters. 40 | * Any existing parsing state is discarded (e.g. buffered data from a 41 | * previous file). 42 | * 43 | * If {@code file == null}, this is equivalent to calling {@link #stopParsing()}. 44 | * 45 | * @param file {@code null} or a file to parse data from. The file should have 46 | * @{code file.isOpen() == true}, and its current offset is 47 | * assumed to be at the beginning of a record (assumed but not 48 | * explicitly checked). 49 | * @return {@code true} if the new file is opened successfully, or 50 | * {@code false} if {@code file == null} or if an error occurred 51 | * while opening the new file. If {@code false} is returned, it 52 | * means that parsing was stopped. 53 | */ 54 | boolean startParsingFile(TrackedFile file); 55 | 56 | /** 57 | * Sets up the parser to read records from the given {@code file}, assuming 58 | * no discontinuity in the data. Any buffered data from a previous 59 | * file will be preserved. 60 | * Call this method if the new file is the "same" as the current file 61 | * or if the current file was successfully parsed to its end. 62 | * 63 | * If there's no file currently being parsed, this method is identical to 64 | * {@link #switchParsingToFile(TrackedFile)}. 65 | * 66 | * If {@code file == null}, this is equivalent to calling {@link #stopParsing()}. 67 | * @param file {@code null} or a file to continue parsing data from. 68 | * If there's already a file being parsed, continuity is assumed 69 | * and is not explicitly checked. 70 | * @return {@code true} if the new file is opened successfully, or 71 | * {@code false} if {@code file == null} or if an error occurred 72 | * while opening the new file. If {@code false} is returned, it 73 | * means that parsing was stopped. 74 | */ 75 | boolean continueParsingWithFile(TrackedFile file); 76 | 77 | /** 78 | * Sets up the parser to read records from the given {@code file}, assuming 79 | * there's no continuity in the data between the current file (if any) and 80 | * the new file. Any buffered data from a previous file will be 81 | * discarded. 82 | * 83 | * If {@code file == null}, this is equivalent to calling {@link #stopParsing()}. 84 | * 85 | * @param file {@code null} or a file to start parsing data from. 86 | * @return {@code true} if the new file is opened successfully, or 87 | * {@code false} if {@code file == null} or if an error occurred 88 | * while opening the new file. If {@code false} is returned, it 89 | * means that parsing was stopped. 90 | */ 91 | boolean switchParsingToFile(TrackedFile file); 92 | 93 | 94 | /** 95 | * @return {@code true} if there's a file currently being parsed, and 96 | * {@code false} otherwise. 97 | */ 98 | boolean isParsing(); 99 | 100 | /** 101 | * Forcibly stops parsing the current file (if any), discarding any buffered 102 | * data. Subsequent calls to {@link #readRecord()} will return {@code null}. 103 | * @param reason message describing the reason we're stopping. 104 | * @return {@code true} if there was a file being parsed, and {@code false} 105 | * otherwise. 106 | */ 107 | boolean stopParsing(String reason); 108 | 109 | /** 110 | * @return The current open file being parsed by this instance. 111 | */ 112 | public TrackedFile getCurrentFile(); 113 | 114 | /** 115 | * @return The number of bytes remaining in the internal buffer. 116 | */ 117 | public int bufferedBytesRemaining(); 118 | 119 | /** 120 | * NOTE: A {@code null} return should indicate to the caller that we reached 121 | * the end of file for the underlying channel. It's up to the caller to 122 | * decide what to do: whether to wait for more data to be written to 123 | * current file or to open a new file (in case of rotation e.g.). 124 | * @return The next record read from the current open file, or 125 | * {@code null} if no COMPLETE records could be read from the 126 | * underlying channel at this time. 127 | */ 128 | public R readRecord(); 129 | 130 | /** 131 | * @return {@code true} if the parser has read all bytes found in the 132 | * current file, and {@code false} otherwise. The return value might 133 | * change over time if more data is written to the file. 134 | */ 135 | public boolean isAtEndOfCurrentFile(); 136 | 137 | public Map getMetrics(); 138 | } 139 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/FirehoseFileFlow.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import java.io.IOException; 17 | import java.nio.charset.StandardCharsets; 18 | import java.util.concurrent.ExecutorService; 19 | import java.util.concurrent.TimeUnit; 20 | 21 | import lombok.Getter; 22 | import lombok.ToString; 23 | 24 | import com.amazon.kinesis.streaming.agent.AgentContext; 25 | import com.amazon.kinesis.streaming.agent.config.Configuration; 26 | import com.amazon.kinesis.streaming.agent.config.ConfigurationException; 27 | import com.amazon.kinesis.streaming.agent.tailing.checkpoints.FileCheckpointStore; 28 | import com.google.common.base.Strings; 29 | import com.google.common.collect.Range; 30 | /** 31 | * An implementation of a {@link FileFlow} where the destination is a firehose. 32 | */ 33 | @ToString(callSuper=true) 34 | public class FirehoseFileFlow extends FileFlow { 35 | public static final Range VALID_MAX_BUFFER_AGE_RANGE_MILLIS = Range.closed( 36 | TimeUnit.SECONDS.toMillis(1), TimeUnit.MINUTES.toMillis(15)); 37 | public static final Range VALID_MAX_BUFFER_SIZE_RECORDS_RANGE = Range.closed(1, FirehoseConstants.MAX_BUFFER_SIZE_RECORDS); 38 | public static final Range VALID_MAX_BUFFER_SIZE_BYTES_RANGE = Range.closed(1, FirehoseConstants.MAX_BUFFER_SIZE_BYTES); 39 | public static final Range VALID_WAIT_ON_FULL_PUBLISH_QUEUE_MILLIS_RANGE = Range.closed( 40 | TimeUnit.SECONDS.toMillis(1), TimeUnit.MINUTES.toMillis(15)); 41 | public static final Range VALID_WAIT_ON_EMPTY_PUBLISH_QUEUE_MILLIS_RANGE = Range.closed( 42 | TimeUnit.SECONDS.toMillis(1), TimeUnit.MINUTES.toMillis(15)); 43 | 44 | @Getter protected final String id; 45 | @Getter protected final String destination; 46 | 47 | public FirehoseFileFlow(AgentContext context, Configuration config) { 48 | super(context, config); 49 | destination = readString(FirehoseConstants.DESTINATION_KEY); 50 | id = "fh:" + destination + ":" + sourceFile.toString(); 51 | } 52 | 53 | @Override 54 | public int getPerRecordOverheadBytes() { 55 | return FirehoseConstants.PER_RECORD_OVERHEAD_BYTES; 56 | } 57 | 58 | @Override 59 | public int getMaxRecordSizeBytes() { 60 | return FirehoseConstants.MAX_RECORD_SIZE_BYTES; 61 | } 62 | 63 | @Override 64 | public int getPerBufferOverheadBytes() { 65 | return FirehoseConstants.PER_BUFFER_OVERHEAD_BYTES; 66 | } 67 | 68 | @Override 69 | protected FileTailer createNewTailer( 70 | FileCheckpointStore checkpoints, 71 | ExecutorService sendingExecutor) throws IOException { 72 | SourceFileTracker fileTracker = buildSourceFileTracker(); 73 | AsyncPublisherService publisher = getPublisher(checkpoints, sendingExecutor); 74 | return new FileTailer( 75 | agentContext, this, fileTracker, 76 | publisher, buildParser(), checkpoints); 77 | } 78 | 79 | @Override 80 | protected SourceFileTracker buildSourceFileTracker() throws IOException { 81 | return new SourceFileTracker(agentContext, this); 82 | } 83 | 84 | @Override 85 | protected AsyncPublisherService getPublisher( 86 | FileCheckpointStore checkpoints, 87 | ExecutorService sendingExecutor) { 88 | return new AsyncPublisherService<>(agentContext, this, checkpoints, 89 | buildSender(), sendingExecutor); 90 | } 91 | 92 | @Override 93 | protected IParser buildParser() { 94 | return new FirehoseParser(this, getParserBufferSize()); 95 | } 96 | 97 | @Override 98 | protected ISender buildSender() { 99 | return new FirehoseSender(agentContext, this); 100 | } 101 | 102 | @Override 103 | public int getParserBufferSize() { 104 | return FirehoseConstants.DEFAULT_PARSER_BUFFER_SIZE_BYTES; 105 | } 106 | 107 | @Override 108 | protected Range getWaitOnEmptyPublishQueueMillisValidRange() { 109 | return VALID_WAIT_ON_EMPTY_PUBLISH_QUEUE_MILLIS_RANGE; 110 | } 111 | 112 | @Override 113 | protected long getDefaultWaitOnEmptyPublishQueueMillis() { 114 | return FirehoseConstants.DEFAULT_WAIT_ON_EMPTY_PUBLISH_QUEUE_MILLIS; 115 | } 116 | 117 | @Override 118 | protected Range getWaitOnPublishQueueMillisValidRange() { 119 | return VALID_WAIT_ON_FULL_PUBLISH_QUEUE_MILLIS_RANGE; 120 | } 121 | 122 | @Override 123 | protected long getDefaultWaitOnPublishQueueMillis() { 124 | return FirehoseConstants.DEFAULT_WAIT_ON_FULL_PUBLISH_QUEUE_MILLIS; 125 | } 126 | 127 | @Override 128 | protected Range getMaxBufferSizeBytesValidRange() { 129 | return VALID_MAX_BUFFER_SIZE_BYTES_RANGE; 130 | } 131 | 132 | @Override 133 | protected int getDefaultMaxBufferSizeBytes() { 134 | return FirehoseConstants.MAX_BUFFER_SIZE_BYTES; 135 | } 136 | 137 | @Override 138 | protected Range getBufferSizeRecordsValidRange() { 139 | return VALID_MAX_BUFFER_SIZE_RECORDS_RANGE; 140 | } 141 | 142 | @Override 143 | protected int getDefaultBufferSizeRecords() { 144 | return FirehoseConstants.MAX_BUFFER_SIZE_RECORDS; 145 | } 146 | 147 | @Override 148 | protected Range getMaxBufferAgeMillisValidRange() { 149 | return VALID_MAX_BUFFER_AGE_RANGE_MILLIS; 150 | } 151 | 152 | @Override 153 | protected long getDefaultMaxBufferAgeMillis() { 154 | return FirehoseConstants.DEFAULT_MAX_BUFFER_AGE_MILLIS; 155 | } 156 | 157 | @Override 158 | public long getDefaultRetryInitialBackoffMillis() { 159 | return FirehoseConstants.DEFAULT_RETRY_INITIAL_BACKOFF_MILLIS; 160 | } 161 | 162 | @Override 163 | public long getDefaultRetryMaxBackoffMillis() { 164 | return FirehoseConstants.DEFAULT_RETRY_MAX_BACKOFF_MILLIS; 165 | } 166 | 167 | @Override 168 | public int getDefaultPublishQueueCapacity() { 169 | return FirehoseConstants.DEFAULT_PUBLISH_QUEUE_CAPACITY; 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /setup: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | # Make sure only root can run our script 4 | if [ "$(id -u)" != "0" ]; then 5 | echo "This script must be run as root" >&2 6 | exit 1 7 | fi 8 | 9 | daemon_name=aws-kinesis-agent 10 | agent_user_name=aws-kinesis-agent-user 11 | bin_dir=/usr/bin 12 | cron_dir=/etc/cron.d 13 | sysconfig_dir=/etc/sysconfig 14 | config_dir=/etc/aws-kinesis 15 | jar_dir=/usr/share/${daemon_name}/lib 16 | init_dir=/etc/rc.d/init.d 17 | dependencies_dir=./dependencies 18 | log_dir=/var/log/${daemon_name} 19 | state_dir=/var/run/${daemon_name} 20 | agent_service=${init_dir}/${daemon_name} 21 | 22 | usage() { 23 | echo "Usage:" 24 | echo "To install Kinesis Agent, type:" 25 | echo " sudo /setup --install" 26 | echo "To uninstall Kinesis Agent, type:" 27 | echo " sudo /setup --uninstall" 28 | } 29 | 30 | fail() { 31 | echo $1 >&2 32 | exit 1 33 | } 34 | 35 | download_jar() { 36 | group_id=$1 37 | artifact_id=$2 38 | version=$3 39 | prefix="http://search.maven.org/remotecontent?filepath=" 40 | path=${1//.//} 41 | jar_name=$artifact_id-$version.jar 42 | url=${prefix}${path}/${artifact_id}/${version}/${jar_name} 43 | [[ -f ${dependencies_dir}/${jar_name} ]] || wget -P ${dependencies_dir} $url 44 | } 45 | 46 | download_dependencies() { 47 | install -d ${dependencies_dir} 48 | 49 | echo "Downloading dependencies ..." 50 | aws_java_sdk_version="1.10.26" 51 | 52 | remote_mvn_pkg="com.amazonaws:aws-java-sdk-core:${aws_java_sdk_version} \ 53 | com.amazonaws:aws-java-sdk-kinesis:${aws_java_sdk_version} \ 54 | com.amazonaws:aws-java-sdk-cloudwatch:${aws_java_sdk_version} \ 55 | com.fasterxml.jackson.core:jackson-annotations:2.6.3 \ 56 | com.fasterxml.jackson.core:jackson-core:2.6.3 \ 57 | com.fasterxml.jackson.core:jackson-databind:2.6.3 \ 58 | com.google.code.findbugs:jsr305:3.0.1 \ 59 | com.google.guava:guava:18.0 \ 60 | org.apache.httpcomponents:httpclient:4.5.1 \ 61 | org.apache.httpcomponents:httpclient-cache:4.5.1 \ 62 | org.apache.httpcomponents:httpmime:4.5.1 \ 63 | org.apache.httpcomponents:httpcore:4.4.3 \ 64 | org.apache.httpcomponents:httpcore-ab:4.4.3 \ 65 | org.apache.httpcomponents:httpcore-nio:4.4.3 \ 66 | commons-cli:commons-cli:1.2 \ 67 | commons-codec:commons-codec:1.6 \ 68 | commons-logging:commons-logging-adapters:1.1 \ 69 | commons-logging:commons-logging-api:1.1 \ 70 | org.apache.commons:commons-lang3:3.4 \ 71 | log4j:log4j:1.2.17 \ 72 | org.slf4j:slf4j-api:1.7.12 \ 73 | org.slf4j:slf4j-log4j12:1.7.12 \ 74 | com.beust:jcommander:1.48 \ 75 | org.xerial:sqlite-jdbc:3.8.11.2 \ 76 | joda-time:joda-time:2.8.2 \ 77 | org.projectlombok:lombok:1.16.6" 78 | 79 | for package in ${remote_mvn_pkg} 80 | do 81 | download_jar $(echo $package | tr : " ") 82 | done 83 | } 84 | 85 | do_uninstall () { 86 | echo "Uninstalling $daemon_name ..." 87 | # stop the service if it's running 88 | if [ -f $agent_service ]; then 89 | echo "Stopping $daemon_name..." 90 | $agent_service stop 91 | fi 92 | 93 | # remove the service from system services 94 | chkconfig --list $daemon_name > /dev/null 2>&1 95 | if [ $? == 0 ]; then 96 | echo "Removing $daemon_name from system services..." 97 | sudo chkconfig --del $daemon_name 98 | fi 99 | 100 | # remove the jars and config files 101 | rm -rf ${state_dir} 102 | rm -rf ${log_dir} 103 | rm -rf ${jar_dir} 104 | rm -f ${agent_service} 105 | rm -f ${bin_dir}/start-${daemon_name} 106 | 107 | # remove the user for starting the agent 108 | user_id=$(id -u $agent_user_name > /dev/null 2>&1) 109 | [[ $? == 0 ]] && userdel $agent_user_name 110 | 111 | # remove sysconfig 112 | rm -f ${sysconfig_dir}/${daemon_name} 113 | 114 | # remove cron job 115 | rm -f ${bin_dir}/${daemon_name}-babysit 116 | rm -f ${cron_dir}/${daemon_name} 117 | 118 | return 0 119 | } 120 | 121 | do_build () { 122 | download_dependencies 123 | 124 | ant || fail "Failed to build the Java project" 125 | } 126 | 127 | do_install () { 128 | do_uninstall 129 | 130 | echo "Installing Kinesis Agent ..." 131 | 132 | do_build 133 | 134 | install -d ${config_dir} 135 | install -d ${jar_dir} 136 | install -d ${init_dir} 137 | install -d ${cron_dir} 138 | install -d ${sysconfig_dir} 139 | install -d ${log_dir} 140 | install -d ${state_dir} 141 | install -m755 ./bin/start-${daemon_name} ${bin_dir} 142 | install -m755 ./bin/${daemon_name}-babysit ${bin_dir} 143 | install -m644 ./${dependencies_dir}/* ${jar_dir} 144 | install -m644 ./ant_build/lib/* ${jar_dir} 145 | install -m755 ./bin/${daemon_name} ${init_dir} 146 | install -m644 ./support/${daemon_name}.cron ${cron_dir}/${daemon_name} 147 | install -m644 ./support/${daemon_name}.sysconfig ${sysconfig_dir}/${daemon_name} 148 | 149 | # add a user for starting the agent 150 | user_id=$(id -u $agent_user_name > /dev/null 2>&1) 151 | [[ $? == 0 ]] || useradd -c "Streaming Data Agent" -r $agent_user_name 152 | usermod -L $agent_user_name 153 | 154 | # change the owner of log files and checkpoint files to the user 155 | chown -R $agent_user_name $state_dir 156 | chown -R $agent_user_name $log_dir 157 | 158 | config_file=${config_dir}/agent.json 159 | [[ -f ${config_file} ]] || install -m644 ./configuration/release/${daemon_name}.json ${config_file} 160 | echo "Configuration file installed at: ${config_file}" 161 | echo "Configuration details:" 162 | cat "${config_file}" 163 | 164 | /sbin/chkconfig --add $daemon_name 165 | 166 | echo "Amazon Kinesis Agent is installed successfully." 167 | echo "To start the $daemon_name service, run:" 168 | echo " sudo service $daemon_name start" 169 | echo "To stop the $daemon_name service, run:" 170 | echo " sudo service $daemon_name stop" 171 | echo "To check the status of the $daemon_name service, run:" 172 | echo " sudo service $daemon_name status" 173 | echo "" 174 | echo "$daemon_name log file will be found at: $log_dir" 175 | 176 | echo "To make the agent automatically start at system startup, type:" 177 | echo " sudo chkconfig $daemon_name on" 178 | echo "" 179 | echo "Your installation has completed!" 180 | } 181 | 182 | COMMAND=$1 183 | case "$COMMAND" in 184 | --build) 185 | shift 186 | do_build "$@" 187 | ;; 188 | --install) 189 | shift 190 | do_install "$@" 191 | ;; 192 | --uninstall) 193 | shift 194 | do_uninstall "$@" 195 | echo "Kinesis Agent has been uninstalled" 196 | ;; 197 | *) 198 | usage 199 | ;; 200 | esac -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/KinesisFileFlow.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import java.io.IOException; 17 | import java.nio.charset.StandardCharsets; 18 | import java.util.concurrent.ExecutorService; 19 | import java.util.concurrent.TimeUnit; 20 | 21 | import lombok.Getter; 22 | import lombok.ToString; 23 | 24 | import com.amazon.kinesis.streaming.agent.AgentContext; 25 | import com.amazon.kinesis.streaming.agent.config.Configuration; 26 | import com.amazon.kinesis.streaming.agent.config.ConfigurationException; 27 | import com.amazon.kinesis.streaming.agent.tailing.KinesisConstants.PartitionKeyOption; 28 | import com.amazon.kinesis.streaming.agent.tailing.checkpoints.FileCheckpointStore; 29 | import com.google.common.base.Strings; 30 | import com.google.common.collect.Range; 31 | /** 32 | * An implementation of a {@link FileFlow} where the destination is a kinesis stream. 33 | */ 34 | @ToString(callSuper=true) 35 | public class KinesisFileFlow extends FileFlow { 36 | public static final Range VALID_MAX_BUFFER_AGE_RANGE_MILLIS = Range.closed( 37 | TimeUnit.SECONDS.toMillis(1), TimeUnit.MINUTES.toMillis(15)); 38 | public static final Range VALID_MAX_BUFFER_SIZE_RECORDS_RANGE = Range.closed(1, KinesisConstants.MAX_BUFFER_SIZE_RECORDS); 39 | public static final Range VALID_MAX_BUFFER_SIZE_BYTES_RANGE = Range.closed(1, KinesisConstants.MAX_BUFFER_SIZE_BYTES); 40 | public static final Range VALID_WAIT_ON_FULL_PUBLISH_QUEUE_MILLIS_RANGE = Range.closed( 41 | TimeUnit.SECONDS.toMillis(1), TimeUnit.MINUTES.toMillis(15)); 42 | public static final Range VALID_WAIT_ON_EMPTY_PUBLISH_QUEUE_MILLIS_RANGE = Range.closed( 43 | TimeUnit.SECONDS.toMillis(1), TimeUnit.MINUTES.toMillis(15)); 44 | 45 | @Getter protected final String id; 46 | @Getter protected final String destination; 47 | @Getter protected final PartitionKeyOption partitionKeyOption; 48 | 49 | public KinesisFileFlow(AgentContext context, Configuration config) { 50 | super(context, config); 51 | destination = readString(KinesisConstants.DESTINATION_KEY); 52 | id = "kinesis:" + destination + ":" + sourceFile.toString(); 53 | partitionKeyOption = readEnum(PartitionKeyOption.class, KinesisConstants.PARTITION_KEY, PartitionKeyOption.RANDOM); 54 | } 55 | 56 | @Override 57 | public int getPerRecordOverheadBytes() { 58 | return KinesisConstants.PER_RECORD_OVERHEAD_BYTES; 59 | } 60 | 61 | @Override 62 | public int getMaxRecordSizeBytes() { 63 | return KinesisConstants.MAX_RECORD_SIZE_BYTES; 64 | } 65 | 66 | @Override 67 | public int getPerBufferOverheadBytes() { 68 | return KinesisConstants.PER_BUFFER_OVERHEAD_BYTES; 69 | } 70 | 71 | @Override 72 | public FileTailer createNewTailer( 73 | FileCheckpointStore checkpoints, 74 | ExecutorService sendingExecutor) throws IOException { 75 | SourceFileTracker fileTracker = buildSourceFileTracker(); 76 | AsyncPublisherService publisher = getPublisher(checkpoints, sendingExecutor); 77 | return new FileTailer( 78 | agentContext, this, fileTracker, 79 | publisher, buildParser(), checkpoints); 80 | } 81 | 82 | @Override 83 | protected SourceFileTracker buildSourceFileTracker() throws IOException { 84 | return new SourceFileTracker(agentContext, this); 85 | } 86 | 87 | @Override 88 | protected AsyncPublisherService getPublisher( 89 | FileCheckpointStore checkpoints, 90 | ExecutorService sendingExecutor) { 91 | return new AsyncPublisherService<>(agentContext, this, checkpoints, 92 | buildSender(), sendingExecutor); 93 | } 94 | 95 | @Override 96 | protected IParser buildParser() { 97 | return new KinesisParser(this, getParserBufferSize()); 98 | } 99 | 100 | @Override 101 | protected ISender buildSender() { 102 | return new KinesisSender(agentContext, this); 103 | } 104 | 105 | @Override 106 | public int getParserBufferSize() { 107 | return KinesisConstants.DEFAULT_PARSER_BUFFER_SIZE_BYTES; 108 | } 109 | 110 | @Override 111 | protected Range getWaitOnEmptyPublishQueueMillisValidRange() { 112 | return VALID_WAIT_ON_EMPTY_PUBLISH_QUEUE_MILLIS_RANGE; 113 | } 114 | 115 | @Override 116 | protected long getDefaultWaitOnEmptyPublishQueueMillis() { 117 | return KinesisConstants.DEFAULT_WAIT_ON_EMPTY_PUBLISH_QUEUE_MILLIS; 118 | } 119 | 120 | @Override 121 | protected Range getWaitOnPublishQueueMillisValidRange() { 122 | return VALID_WAIT_ON_FULL_PUBLISH_QUEUE_MILLIS_RANGE; 123 | } 124 | 125 | @Override 126 | protected long getDefaultWaitOnPublishQueueMillis() { 127 | return KinesisConstants.DEFAULT_WAIT_ON_FULL_PUBLISH_QUEUE_MILLIS; 128 | } 129 | 130 | @Override 131 | protected Range getMaxBufferSizeBytesValidRange() { 132 | return VALID_MAX_BUFFER_SIZE_BYTES_RANGE; 133 | } 134 | 135 | @Override 136 | protected int getDefaultMaxBufferSizeBytes() { 137 | return KinesisConstants.MAX_BUFFER_SIZE_BYTES; 138 | } 139 | 140 | @Override 141 | protected Range getBufferSizeRecordsValidRange() { 142 | return VALID_MAX_BUFFER_SIZE_RECORDS_RANGE; 143 | } 144 | 145 | @Override 146 | protected int getDefaultBufferSizeRecords() { 147 | return KinesisConstants.MAX_BUFFER_SIZE_RECORDS; 148 | } 149 | 150 | @Override 151 | protected Range getMaxBufferAgeMillisValidRange() { 152 | return VALID_MAX_BUFFER_AGE_RANGE_MILLIS; 153 | } 154 | 155 | @Override 156 | protected long getDefaultMaxBufferAgeMillis() { 157 | return KinesisConstants.DEFAULT_MAX_BUFFER_AGE_MILLIS; 158 | } 159 | 160 | @Override 161 | public long getDefaultRetryInitialBackoffMillis() { 162 | return KinesisConstants.DEFAULT_RETRY_INITIAL_BACKOFF_MILLIS; 163 | } 164 | 165 | @Override 166 | public long getDefaultRetryMaxBackoffMillis() { 167 | return KinesisConstants.DEFAULT_RETRY_MAX_BACKOFF_MILLIS; 168 | } 169 | 170 | @Override 171 | public int getDefaultPublishQueueCapacity() { 172 | return KinesisConstants.DEFAULT_PUBLISH_QUEUE_CAPACITY; 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/tailing/AsyncPublisherService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.tailing; 15 | 16 | import java.util.Map; 17 | import java.util.concurrent.ExecutorService; 18 | import java.util.concurrent.TimeUnit; 19 | 20 | import org.slf4j.Logger; 21 | 22 | import com.amazon.kinesis.streaming.agent.AgentContext; 23 | import com.amazon.kinesis.streaming.agent.IHeartbeatProvider; 24 | import com.amazon.kinesis.streaming.agent.Logging; 25 | import com.amazon.kinesis.streaming.agent.tailing.checkpoints.Checkpointer; 26 | import com.amazon.kinesis.streaming.agent.tailing.checkpoints.FileCheckpointStore; 27 | import com.google.common.annotations.VisibleForTesting; 28 | import com.google.common.base.Preconditions; 29 | import com.google.common.util.concurrent.AbstractExecutionThreadService; 30 | 31 | /** 32 | * A publisher that buffers records into an {@link PublishingQueue}, and makes 33 | * the send requests asynchronously. 34 | * 35 | * @param The record type. 36 | */ 37 | public final class AsyncPublisherService 38 | extends AbstractExecutionThreadService 39 | implements IHeartbeatProvider { 40 | private static final int NO_TIMEOUT = -1; 41 | private static final long SHUTDOWN_MARGIN_MILLIS = 500; 42 | private static final Logger LOGGER = Logging.getLogger(AsyncPublisherService.class); 43 | 44 | private final AsyncPublisher publisher; 45 | private Thread serviceThread; 46 | 47 | /** 48 | * 49 | * @param agentContext 50 | * @param flow 51 | * @param checkpoints 52 | * @param sender 53 | * @param sendingExecutor The executor that will run the async send 54 | * requests. 55 | */ 56 | public AsyncPublisherService( 57 | AgentContext agentContext, 58 | FileFlow flow, 59 | FileCheckpointStore checkpoints, 60 | ISender sender, 61 | ExecutorService sendingExecutor) { 62 | this.publisher = new AsyncPublisher(agentContext, flow, checkpoints, sender, sendingExecutor); 63 | } 64 | 65 | public boolean publishRecord(R record) { 66 | return publisher.publishRecord(record); 67 | } 68 | 69 | @Override 70 | protected void run() throws Exception { 71 | serviceThread = Thread.currentThread(); 72 | LOGGER.trace("{}: Main loop started", serviceName()); 73 | do { 74 | runOnce(); 75 | } while (isRunning()); 76 | } 77 | 78 | protected void runOnce() { 79 | publisher.backoff(); 80 | if (isRunning()) { 81 | publisher.sendNextBufferAsync(true); 82 | } 83 | } 84 | 85 | public void flush() { 86 | publisher.flush(); 87 | } 88 | 89 | @VisibleForTesting 90 | PublishingQueue queue() { 91 | return publisher.queue; 92 | } 93 | 94 | @VisibleForTesting 95 | Checkpointer checkpointer() { 96 | return publisher.checkpointer; 97 | } 98 | 99 | @Override 100 | protected String serviceName() { 101 | return publisher.name(); 102 | } 103 | 104 | @Override 105 | protected void startUp() throws Exception { 106 | LOGGER.debug("{}: Starting up...", serviceName()); 107 | super.startUp(); 108 | } 109 | 110 | @Override 111 | protected void shutDown() throws Exception { 112 | LOGGER.debug("{}: Shutting down...", serviceName()); 113 | super.shutDown(); 114 | } 115 | 116 | @Override 117 | protected void triggerShutdown() { 118 | super.triggerShutdown(); 119 | // At this time, isRunning() will return false so no more records can be published 120 | LOGGER.debug("{}: Shutdown triggered...", serviceName()); 121 | publisher.close(); 122 | 123 | if (serviceThread != null) 124 | serviceThread.interrupt(); 125 | 126 | // Give the senders some time to complete before cancelling everything 127 | LOGGER.trace("{}: Shutdown timeout: {}ms", serviceName(), getShutdownTimeoutMillis()); 128 | waitForIdle(getShutdownTimeoutMillis(), TimeUnit.MILLISECONDS); 129 | } 130 | 131 | protected long getShutdownTimeoutMillis() { 132 | return publisher.agentContext.shutdownTimeoutMillis() - SHUTDOWN_MARGIN_MILLIS; 133 | } 134 | 135 | /** 136 | * Initializes and starts the publisher. Cannot invoke {@link #publishRecord(IRecord)} 137 | * before calling this method. 138 | */ 139 | public void startPublisher() { 140 | Preconditions.checkState(!isRunning(), "%s: Publisher already running.", serviceName()); 141 | startAsync(); 142 | awaitRunning(); 143 | } 144 | 145 | /** 146 | * Terminates the publisher and performs any cleanup. The implementation 147 | * should make sure that any pending data is sent by the time this method 148 | * returns. After this method returns, {@link #isIdle()} should always 149 | * return {@code true}, and {@link #isRunning()} will always return 150 | * {@code false}. 151 | * 152 | * Cannot call {@link #publishRecord(IRecord)} after calling this method. 153 | */ 154 | public void stopPublisher() { 155 | stopPublisherAsync(); 156 | awaitTerminated(); 157 | } 158 | 159 | /** 160 | * Starts the termination of the publisher in an asynchronous thread and 161 | * returns immediately. After this method returns {@link #isRunning()} will 162 | * always return {@code false}. 163 | * 164 | * Cannot call {@link #publishRecord(IRecord)} after calling this method. 165 | */ 166 | public void stopPublisherAsync() { 167 | LOGGER.debug("{}: Stopping...", serviceName()); 168 | Preconditions.checkState(isRunning(), "%s: Publisher already stopped.", serviceName()); 169 | stopAsync(); 170 | } 171 | 172 | public boolean isIdle() { 173 | return publisher.isIdle(); 174 | } 175 | 176 | /** 177 | * Wait indefinitely for the publisher to reach idle state. 178 | */ 179 | public void waitForIdle() { 180 | waitForIdle(NO_TIMEOUT, TimeUnit.MILLISECONDS); 181 | } 182 | 183 | public boolean waitForIdle(long timeout, TimeUnit unit) { 184 | return publisher.waitForIdle(timeout, unit); 185 | } 186 | 187 | @Override 188 | public Object heartbeat(AgentContext agent) { 189 | return publisher.heartbeat(agent); 190 | } 191 | 192 | // Use for debugging only please. 193 | @Override 194 | public String toString() { 195 | StringBuilder sb = new StringBuilder(); 196 | sb.append(getClass().getSimpleName()) 197 | .append("(isRunning=").append(isRunning()) 198 | .append(",publisher=").append(publisher) 199 | .append(")"); 200 | return sb.toString(); 201 | } 202 | 203 | public Map getMetrics() { 204 | return publisher.getMetrics(); 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/metrics/CWPublisherRunnable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.metrics; 15 | 16 | import java.util.Collection; 17 | import java.util.List; 18 | import java.util.Random; 19 | 20 | import org.slf4j.Logger; 21 | 22 | import com.amazon.kinesis.streaming.agent.Logging; 23 | 24 | /** 25 | * A CWPublisherRunnable contains the logic of when to publish metrics. 26 | * 27 | * @param 28 | */ 29 | 30 | public class CWPublisherRunnable implements Runnable { 31 | 32 | private static final Logger LOG = Logging.getLogger(CWPublisherRunnable.class); 33 | 34 | private final ICWMetricsPublisher metricsPublisher; 35 | private final MetricAccumulatingQueue queue; 36 | private final long bufferTimeMillis; 37 | 38 | /* 39 | * Number of metrics that will cause us to flush. 40 | */ 41 | private int flushSize; 42 | private boolean shuttingDown = false; 43 | private boolean shutdown = false; 44 | private long lastFlushTime = Long.MAX_VALUE; 45 | private int maxJitter; 46 | 47 | private Random rand = new Random(); 48 | private int nextJitterValueToUse = 0; 49 | 50 | /** 51 | * Constructor. 52 | * 53 | * @param metricsPublisher publishes metrics 54 | * @param bufferTimeMillis time between publishing metrics 55 | * @param maxQueueSize max size of metrics to publish 56 | * @param batchSize size of batch that can be published at a time 57 | */ 58 | 59 | public CWPublisherRunnable(ICWMetricsPublisher metricsPublisher, 60 | long bufferTimeMillis, 61 | int maxQueueSize, 62 | int batchSize) { 63 | this(metricsPublisher, bufferTimeMillis, maxQueueSize, batchSize, 0); 64 | } 65 | 66 | public CWPublisherRunnable(ICWMetricsPublisher metricsPublisher, 67 | long bufferTimeMillis, 68 | int maxQueueSize, 69 | int batchSize, 70 | int maxJitter) { 71 | if (LOG.isDebugEnabled()) { 72 | LOG.debug("Constructing CWPublisherRunnable with maxBufferTimeMillis {} maxQueueSize {} batchSize {} maxJitter {}", 73 | bufferTimeMillis, 74 | maxQueueSize, 75 | batchSize, 76 | maxJitter); 77 | } 78 | 79 | this.metricsPublisher = metricsPublisher; 80 | this.bufferTimeMillis = bufferTimeMillis; 81 | this.queue = new MetricAccumulatingQueue(maxQueueSize); 82 | this.flushSize = batchSize; 83 | this.maxJitter = maxJitter; 84 | } 85 | 86 | @Override 87 | public void run() { 88 | while (!shutdown) { 89 | try { 90 | runOnce(); 91 | } catch (Throwable t) { 92 | LOG.error("Encountered throwable in CWPublisherRunable", t); 93 | } 94 | } 95 | 96 | LOG.info("CWPublication thread finished."); 97 | } 98 | 99 | /** 100 | * Exposed for testing purposes. 101 | */ 102 | public void runOnce() { 103 | List> dataToPublish = null; 104 | synchronized (queue) { 105 | /* 106 | * We should send if: 107 | * 108 | * it's been maxBufferTimeMillis since our last send 109 | * or if the queue contains > batchSize elements 110 | * or if we're shutting down 111 | */ 112 | long timeSinceFlush = Math.max(0, getTime() - lastFlushTime); 113 | if (timeSinceFlush >= bufferTimeMillis || queue.size() >= flushSize || shuttingDown) { 114 | dataToPublish = queue.drain(flushSize); 115 | LOG.debug("Drained {} datums from queue", dataToPublish.size()); 116 | 117 | if (shuttingDown) { 118 | LOG.debug("Shutting down with {} datums left on the queue", queue.size()); 119 | 120 | // If we're shutting down, we successfully shut down only when the queue is empty. 121 | shutdown = queue.isEmpty(); 122 | } 123 | } else { 124 | long waitTime = bufferTimeMillis - timeSinceFlush; 125 | LOG.debug("Waiting up to {}ms for {} more datums to appear.", waitTime, flushSize 126 | - queue.size()); 127 | 128 | try { 129 | // Wait for enqueues for up to maxBufferTimeMillis. 130 | queue.wait(waitTime); 131 | } catch (InterruptedException e) { 132 | } 133 | } 134 | } 135 | 136 | if (dataToPublish != null) { 137 | try { 138 | metricsPublisher.publishMetrics(dataToPublish); 139 | } catch (Throwable t) { 140 | LOG.error("Caught exception thrown by metrics Publisher in CWPublisherRunnable", t); 141 | } 142 | // Changing the value of lastFlushTime will change the time when metrics are flushed next. 143 | lastFlushTime = getTime() + nextJitterValueToUse; 144 | if (maxJitter != 0) { 145 | // nextJittervalueToUse will be a value between (-maxJitter,+maxJitter) 146 | nextJitterValueToUse = maxJitter - rand.nextInt(2 * maxJitter); 147 | } 148 | } 149 | } 150 | 151 | /** 152 | * Overrideable for testing purposes. 153 | */ 154 | protected long getTime() { 155 | return System.currentTimeMillis(); 156 | } 157 | 158 | public void shutdown() { 159 | LOG.info("Shutting down CWPublication thread."); 160 | synchronized (queue) { 161 | shuttingDown = true; 162 | queue.notify(); 163 | } 164 | } 165 | 166 | public boolean isShutdown() { 167 | return shutdown; 168 | } 169 | 170 | /** 171 | * Enqueues metric data for publication. 172 | * 173 | * @param data collection of MetricDatum to enqueue 174 | */ 175 | public void enqueue(Collection> data) { 176 | synchronized (queue) { 177 | if (shuttingDown) { 178 | LOG.warn("Dropping metrics {} because CWPublisherRunnable is shutting down.", data); 179 | return; 180 | } 181 | 182 | if (LOG.isDebugEnabled()) { 183 | LOG.debug("Enqueueing {} datums for publication", data.size()); 184 | } 185 | 186 | for (MetricDatumWithKey datumWithKey : data) { 187 | if (!queue.offer(datumWithKey.key, datumWithKey.datum)) { 188 | LOG.warn("Metrics queue full - dropping metric " + datumWithKey.datum); 189 | } 190 | } 191 | 192 | // If this is the first enqueue, start buffering from now. 193 | if (lastFlushTime == Long.MAX_VALUE) { 194 | lastFlushTime = getTime(); 195 | } 196 | 197 | queue.notify(); 198 | } 199 | } 200 | } 201 | -------------------------------------------------------------------------------- /src/com/amazon/kinesis/streaming/agent/config/AgentConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Amazon Software License (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/asl/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and limitations under the License. 13 | */ 14 | package com.amazon.kinesis.streaming.agent.config; 15 | 16 | import java.nio.file.Path; 17 | import java.nio.file.Paths; 18 | import java.util.Map; 19 | 20 | import com.amazonaws.ClientConfiguration; 21 | 22 | public class AgentConfiguration extends Configuration { 23 | static final String[] VALID_LOG_LEVELS = { "TRACE", "DEBUG", "INFO", "WARN", "ERROR" }; 24 | static final long DEFAULT_SENDING_THREADS_KEEPALIVE_MILLIS = 60_000L; 25 | static final int DEFAULT_SENDING_THREADS_MAX_QUEUE_SIZE = 100; 26 | static final String DEFAULT_CHECKPOINTS_FILE = "/var/run/aws-kinesis-agent/checkpoints"; 27 | static final int DEFAULT_MAX_SENDING_THREADS_PER_CORE = 12; 28 | 29 | static final int DEFAULT_CW_QUEUE_SIZE = 10_000; 30 | static final boolean DEFAULT_CW_EMIT_METRICS = true; 31 | static final long DEFAULT_CW_BUFFER_TIME_MILLIS = 30_000L; 32 | static final boolean DEFAULT_LOG_EMIT_METRICS = false; 33 | static final boolean DEFAULT_LOG_EMIT_INTERNAL_METRICS = false; 34 | static final int DEFAULT_LOG_STATUS_REPORTING_PERIOD_SECONDS = 30; 35 | static final int DEFAULT_CHECKPOINT_TTL_DAYS = 7; 36 | 37 | // NOTE: If changing the default make sure to change SHUTDOWN_TIME variable in `bin/aws-kinesis-agent` as well... 38 | static final long DEFAULT_SHUTDOWN_TIMEOUT_MILLIS = 10_000L; 39 | 40 | private static final String CW_DEFAULT_NAMESPACE = "AWSKinesisAgent"; 41 | public static final String CONFIG_ACCESS_KEY = "awsAccessKeyId"; 42 | public static final String CONFIG_SECRET_KEY = "awsSecretAccessKey"; 43 | public static final String SHUTDOWN_TIMEOUT_MILLIS_KEY = "shutdownTimeoutMillis"; 44 | public static final String ENDPOINT_KEY = "endpoint"; 45 | static final String LOG_FILE_KEY = "log.file"; 46 | static final String LOG_LEVEL_KEY = "log.level"; 47 | static final String LOG_MAX_BACKUP_INDEX_KEY = "log.maxBackupIndex"; 48 | static final String LOG_MAX_FILE_SIZE_KEY = "log.maxFileSize"; 49 | 50 | public AgentConfiguration(Map config) { 51 | super(config); 52 | } 53 | 54 | public long shutdownTimeoutMillis() { 55 | return readLong(SHUTDOWN_TIMEOUT_MILLIS_KEY, DEFAULT_SHUTDOWN_TIMEOUT_MILLIS); 56 | } 57 | 58 | public String accessKeyId() { 59 | return this.readString(CONFIG_ACCESS_KEY, null); 60 | } 61 | 62 | public String secretKey() { 63 | return this.readString(CONFIG_SECRET_KEY, null); 64 | } 65 | 66 | public AgentConfiguration(Configuration config) { 67 | this(config.getConfigMap()); 68 | } 69 | 70 | public String logLevel() { 71 | return readString(LOG_LEVEL_KEY, null); 72 | } 73 | 74 | public Path logFile() { 75 | return readPath(LOG_FILE_KEY, null); 76 | } 77 | 78 | public int logMaxBackupIndex() { 79 | return readInteger(LOG_MAX_BACKUP_INDEX_KEY, -1); 80 | } 81 | 82 | public long logMaxFileSize() { 83 | return readLong(LOG_MAX_FILE_SIZE_KEY, -1L); 84 | } 85 | 86 | public boolean logEmitMetrics() { 87 | return this.readBoolean("log.emitMetrics", 88 | DEFAULT_LOG_EMIT_METRICS); 89 | } 90 | 91 | public boolean logEmitInternalMetrics() { 92 | return this.readBoolean("log.emitInternalMetrics", 93 | DEFAULT_LOG_EMIT_INTERNAL_METRICS); 94 | } 95 | 96 | public int logStatusReportingPeriodSeconds() { 97 | return this.readInteger("log.statusReportingPeriodSeconds", 98 | DEFAULT_LOG_STATUS_REPORTING_PERIOD_SECONDS); 99 | } 100 | 101 | public boolean cloudwatchEmitMetrics() { 102 | return this.readBoolean("cloudwatch.emitMetrics", 103 | DEFAULT_CW_EMIT_METRICS); 104 | } 105 | 106 | public long cloudwatchMetricsBufferTimeMillis() { 107 | return this.readLong( 108 | "cloudwatch.metricsBufferTimeMillis", 109 | DEFAULT_CW_BUFFER_TIME_MILLIS); 110 | } 111 | 112 | public String cloudwatchNamespace() { 113 | return this.readString("cloudwatch.namespace", CW_DEFAULT_NAMESPACE); 114 | } 115 | 116 | public int cloudwatchMetricsQueueSize() { 117 | return this.readInteger("cloudwatch.metricsQueueSize", 118 | DEFAULT_CW_QUEUE_SIZE); 119 | } 120 | 121 | public int maxSendingThreads() { 122 | return this.readInteger("maxSendingThreads", 123 | defaultMaxSendingThreads()); 124 | } 125 | 126 | public long sendingThreadsKeepAliveMillis() { 127 | return this.readLong("sendingThreadsKeepAliveMillis", 128 | DEFAULT_SENDING_THREADS_KEEPALIVE_MILLIS); 129 | } 130 | 131 | public int sendingThreadsMaxQueueSize() { 132 | return this.readInteger("sendingThreadsMaxQueueSize", 133 | DEFAULT_SENDING_THREADS_MAX_QUEUE_SIZE); 134 | } 135 | 136 | public int maxSendingThreadsPerCore() { 137 | return readInteger("maxSendingThreadsPerCore", DEFAULT_MAX_SENDING_THREADS_PER_CORE); 138 | } 139 | 140 | public int defaultMaxSendingThreads() { 141 | return Math.max(2, 142 | maxSendingThreadsPerCore() * Runtime.getRuntime().availableProcessors()); 143 | } 144 | 145 | public int maxConnections() { 146 | return readInteger("maxConnections", defaultMaxConnections()); 147 | } 148 | 149 | public int defaultMaxConnections() { 150 | return Math.max(ClientConfiguration.DEFAULT_MAX_CONNECTIONS, maxSendingThreads()); 151 | } 152 | 153 | public int connectionTimeoutMillis() { 154 | return readInteger("connectionTimeoutMillis", ClientConfiguration.DEFAULT_CONNECTION_TIMEOUT); 155 | } 156 | 157 | public long connectionTTLMillis() { 158 | return readLong("connectionTTLMillis", ClientConfiguration.DEFAULT_CONNECTION_TTL); 159 | } 160 | 161 | public int socketTimeoutMillis() { 162 | return readInteger("socketTimeoutMillis", ClientConfiguration.DEFAULT_SOCKET_TIMEOUT); 163 | } 164 | 165 | public boolean useTcpKeepAlive() { 166 | return readBoolean("useTcpKeepAlive", ClientConfiguration.DEFAULT_TCP_KEEP_ALIVE); 167 | } 168 | 169 | public boolean useHttpGzip() { 170 | return readBoolean("useHttpGzip", ClientConfiguration.DEFAULT_USE_GZIP); 171 | } 172 | 173 | public Path checkpointFile() { 174 | return this.readPath("checkpointFile", 175 | Paths.get(DEFAULT_CHECKPOINTS_FILE)); 176 | } 177 | 178 | public int checkpointTimeToLiveDays() { 179 | return this.readInteger("checkpointTimeToLiveDays", 180 | DEFAULT_CHECKPOINT_TTL_DAYS); 181 | } 182 | 183 | public String kinesisEndpoint() { 184 | return this.readString("kinesis." + ENDPOINT_KEY, null); 185 | } 186 | 187 | public String firehoseEndpoint() { 188 | return this.readString("firehose." + ENDPOINT_KEY, null); 189 | } 190 | 191 | public String cloudwatchEndpoint() { 192 | return this.readString("cloudwatch." + ENDPOINT_KEY, null); 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 4.0.0 5 | com.amazonaws 6 | amazon-kinesis-agent 7 | jar 8 | Amazon Kinesis Agent 9 | 1.0 10 | Amazon Kinesis Agent runs on customer hosts and continuously monitors a set of log files and sends new data to the Amazon Kinesis Stream and Amazon Kinesis Firehose services in near-real-time. 11 | https://aws.amazon.com/kinesis 12 | 13 | 14 | https://github.com/awslabs/amazon-kinesis-agent.git 15 | 16 | 17 | 18 | 19 | Amazon Software License 20 | https://aws.amazon.com/asl 21 | repo 22 | 23 | 24 | 25 | 26 | 1.10.26 27 | 28 | 29 | 30 | 31 | com.amazonaws 32 | aws-java-sdk-core 33 | ${aws-java-sdk.version} 34 | 35 | 36 | com.amazonaws 37 | aws-java-sdk-kinesis 38 | ${aws-java-sdk.version} 39 | 40 | 41 | com.amazonaws 42 | aws-java-sdk-cloudwatch 43 | ${aws-java-sdk.version} 44 | 45 | 46 | com.fasterxml.jackson.core 47 | jackson-annotations 48 | 2.6.3 49 | 50 | 51 | com.fasterxml.jackson.core 52 | jackson-core 53 | 2.6.3 54 | 55 | 56 | com.fasterxml.jackson.core 57 | jackson-databind 58 | 2.6.3 59 | 60 | 61 | com.google.code.findbugs 62 | jsr305 63 | 3.0.1 64 | 65 | 66 | com.google.guava 67 | guava 68 | 18.0 69 | 70 | 71 | org.apache.httpcomponents 72 | httpclient 73 | 4.5.1 74 | 75 | 76 | org.apache.httpcomponents 77 | httpclient-cache 78 | 4.5.1 79 | 80 | 81 | org.apache.httpcomponents 82 | httpmime 83 | 4.5.1 84 | 85 | 86 | org.apache.httpcomponents 87 | httpcore 88 | 4.4.3 89 | 90 | 91 | org.apache.httpcomponents 92 | httpcore-ab 93 | 4.4.3 94 | 95 | 96 | org.apache.httpcomponents 97 | httpcore-nio 98 | 4.4.3 99 | 100 | 101 | commons-cli 102 | commons-cli 103 | 1.2 104 | 105 | 106 | commons-codec 107 | commons-codec 108 | 1.6 109 | 110 | 111 | commons-logging 112 | commons-logging-adapters 113 | 1.1 114 | 115 | 116 | commons-logging 117 | commons-logging-api 118 | 1.1 119 | 120 | 121 | org.apache.commons 122 | commons-lang3 123 | 3.4 124 | 125 | 126 | log4j 127 | log4j 128 | 1.2.17 129 | 130 | 131 | org.slf4j 132 | slf4j-api 133 | 1.7.12 134 | 135 | 136 | org.slf4j 137 | slf4j-log4j12 138 | 1.7.12 139 | 140 | 141 | com.beust 142 | jcommander 143 | 1.48 144 | 145 | 146 | org.xerial 147 | sqlite-jdbc 148 | 3.8.11.2 149 | 150 | 151 | joda-time 152 | joda-time 153 | 2.8.2 154 | 155 | 156 | org.projectlombok 157 | lombok 158 | 1.16.6 159 | 160 | 161 | 162 | 163 | 164 | amazonwebservices 165 | Amazon Web Services 166 | https://aws.amazon.com 167 | 168 | developer 169 | 170 | 171 | 172 | 173 | 174 | target 175 | target/classes 176 | ${project.artifactId}-${project.version} 177 | src 178 | src 179 | 180 | 181 | 182 | org.apache.maven.plugins 183 | maven-compiler-plugin 184 | 3.2 185 | 186 | 1.7 187 | 1.7 188 | UTF-8 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | org.apache.maven.plugins 197 | maven-gpg-plugin 198 | 1.5 199 | 200 | 201 | sign-artifacts 202 | verify 203 | 204 | sign 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | --------------------------------------------------------------------------------