├── .asf.yaml ├── .gitattributes ├── .github └── workflows │ ├── build.yml │ ├── codeql-analysis.yml │ └── develocity-publish-build-scans.yaml ├── .gitignore ├── .mvn └── wrapper │ └── maven-wrapper.properties ├── .travis.yml.sav ├── CHANGELOG ├── CONTRIBUTING.md ├── DEVNOTES ├── LICENSE ├── NOTICE ├── README.md ├── RELEASE-NOTES ├── bin ├── flume-ng ├── flume-ng.cmd └── flume-ng.ps1 ├── build-support ├── pom.xml └── src │ └── main │ └── resources │ └── config │ └── checkstyle │ ├── checkstyle-suppressions.xml │ └── checkstyle.xml ├── conf ├── flume-conf.properties.template ├── flume-env.ps1.template ├── flume-env.sh.template └── log4j2.xml ├── dev-docs ├── DevelopersQuickHackSheet.md ├── HowToCommit.md ├── HowToRelease.md └── UpdateLicenses.md ├── dev-support ├── create-source-archive.sh ├── generate-source-release.sh ├── includes.sh ├── license-diff.sh ├── sign-checksum-artifact.sh └── test-patch.py ├── doap_Flume.rdf ├── flume-bom └── pom.xml ├── flume-ng-auth ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── flume │ │ ├── api │ │ ├── SecureRpcClientFactory.java │ │ └── SecureThriftRpcClient.java │ │ └── auth │ │ ├── FlumeAuthenticationUtil.java │ │ ├── FlumeAuthenticator.java │ │ ├── KerberosAuthenticator.java │ │ ├── KerberosUser.java │ │ ├── PrivilegedExecutor.java │ │ ├── SecurityException.java │ │ ├── SimpleAuthenticator.java │ │ └── UGIExecutor.java │ └── test │ └── java │ └── org │ └── apache │ └── flume │ └── auth │ └── TestFlumeAuthenticator.java ├── flume-ng-channels ├── flume-file-channel │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── org │ │ │ │ └── apache │ │ │ │ └── flume │ │ │ │ └── channel │ │ │ │ └── file │ │ │ │ ├── BadCheckpointException.java │ │ │ │ ├── CheckpointRebuilder.java │ │ │ │ ├── Commit.java │ │ │ │ ├── CorruptEventException.java │ │ │ │ ├── EventQueueBackingStore.java │ │ │ │ ├── EventQueueBackingStoreFactory.java │ │ │ │ ├── EventQueueBackingStoreFile.java │ │ │ │ ├── EventQueueBackingStoreFileV2.java │ │ │ │ ├── EventQueueBackingStoreFileV3.java │ │ │ │ ├── EventUtils.java │ │ │ │ ├── FileChannel.java │ │ │ │ ├── FileChannelConfiguration.java │ │ │ │ ├── FlumeEvent.java │ │ │ │ ├── FlumeEventPointer.java │ │ │ │ ├── FlumeEventQueue.java │ │ │ │ ├── Log.java │ │ │ │ ├── LogFile.java │ │ │ │ ├── LogFileFactory.java │ │ │ │ ├── LogFileRetryableIOException.java │ │ │ │ ├── LogFileV2.java │ │ │ │ ├── LogFileV3.java │ │ │ │ ├── LogRecord.java │ │ │ │ ├── LogUtils.java │ │ │ │ ├── NoopRecordException.java │ │ │ │ ├── Pair.java │ │ │ │ ├── Put.java │ │ │ │ ├── ReplayHandler.java │ │ │ │ ├── Rollback.java │ │ │ │ ├── Serialization.java │ │ │ │ ├── Take.java │ │ │ │ ├── TransactionEventRecord.java │ │ │ │ ├── TransactionIDOracle.java │ │ │ │ ├── Writable.java │ │ │ │ ├── WritableUtils.java │ │ │ │ ├── WriteOrderOracle.java │ │ │ │ ├── encryption │ │ │ │ ├── AESCTRNoPaddingProvider.java │ │ │ │ ├── CipherProvider.java │ │ │ │ ├── CipherProviderFactory.java │ │ │ │ ├── CipherProviderType.java │ │ │ │ ├── DecryptionFailureException.java │ │ │ │ ├── EncryptionConfiguration.java │ │ │ │ ├── JCEFileKeyProvider.java │ │ │ │ ├── KeyProvider.java │ │ │ │ ├── KeyProviderFactory.java │ │ │ │ └── KeyProviderType.java │ │ │ │ └── instrumentation │ │ │ │ ├── FileChannelCounter.java │ │ │ │ └── FileChannelCounterMBean.java │ │ └── proto │ │ │ └── filechannel.proto │ │ └── test │ │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── flume │ │ │ └── channel │ │ │ └── file │ │ │ ├── CountingSinkRunner.java │ │ │ ├── CountingSourceRunner.java │ │ │ ├── TestCheckpoint.java │ │ │ ├── TestCheckpointRebuilder.java │ │ │ ├── TestEventQueueBackingStoreFactory.java │ │ │ ├── TestEventUtils.java │ │ │ ├── TestFileChannel.java │ │ │ ├── TestFileChannelBase.java │ │ │ ├── TestFileChannelErrorMetrics.java │ │ │ ├── TestFileChannelFormatRegression.java │ │ │ ├── TestFileChannelRestart.java │ │ │ ├── TestFileChannelRollback.java │ │ │ ├── TestFlumeEvent.java │ │ │ ├── TestFlumeEventPointer.java │ │ │ ├── TestFlumeEventQueue.java │ │ │ ├── TestIntegration.java │ │ │ ├── TestLog.java │ │ │ ├── TestLogFile.java │ │ │ ├── TestLogRecord.java │ │ │ ├── TestTransactionEventRecordV2.java │ │ │ ├── TestTransactionEventRecordV3.java │ │ │ ├── TestTransactionIDOracle.java │ │ │ ├── TestUtils.java │ │ │ ├── TestWriteOrderOracle.java │ │ │ └── encryption │ │ │ ├── CipherProviderTestSuite.java │ │ │ ├── EncryptionTestUtils.java │ │ │ ├── TestAESCTRNoPaddingProvider.java │ │ │ ├── TestFileChannelEncryption.java │ │ │ └── TestJCEFileKeyProvider.java │ │ └── resources │ │ ├── fileformat-v2-checkpoint.gz │ │ ├── fileformat-v2-log-1.gz │ │ ├── fileformat-v2-log-2.gz │ │ ├── fileformat-v2-log-3.gz │ │ ├── fileformat-v2-pre-FLUME-1432-checkpoint.gz │ │ ├── fileformat-v2-pre-FLUME-1432-log-1.gz │ │ ├── fileformat-v2-pre-FLUME-1432-log-2.gz │ │ ├── fileformat-v2-pre-FLUME-1432-log-3.gz │ │ ├── ibm-test.keystore │ │ ├── log4j2.xml │ │ └── sun-test.keystore ├── flume-spillable-memory-channel │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── flume │ │ │ └── channel │ │ │ └── SpillableMemoryChannel.java │ │ └── test │ │ └── java │ │ └── org │ │ └── apache │ │ └── flume │ │ └── channel │ │ └── TestSpillableMemoryChannel.java └── pom.xml ├── flume-ng-clients ├── flume-ng-log4jappender │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── assembly │ │ │ └── descriptor.xml │ │ ├── java │ │ │ └── org │ │ │ │ └── apache │ │ │ │ └── flume │ │ │ │ └── clients │ │ │ │ └── log4jappender │ │ │ │ ├── LoadBalancingLog4jAppender.java │ │ │ │ ├── Log4jAppender.java │ │ │ │ └── Log4jAvroHeaders.java │ │ └── resources-for-jar-with-deps │ │ │ └── META-INF │ │ │ ├── LICENSE │ │ │ └── NOTICE │ │ └── test │ │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── flume │ │ │ └── clients │ │ │ └── log4jappender │ │ │ ├── TestLoadBalancingLog4jAppender.java │ │ │ ├── TestLog4jAppender.java │ │ │ └── TestLog4jAppenderWithAvro.java │ │ └── resources │ │ ├── flume-loadbalancing-backoff-log4jtest.properties │ │ ├── flume-loadbalancing-rnd-log4jtest.properties │ │ ├── flume-loadbalancinglog4jtest.properties │ │ ├── flume-log4jtest-avro-generic.properties │ │ ├── flume-log4jtest-avro-reflect.properties │ │ ├── flume-log4jtest.properties │ │ └── myrecord.avsc └── pom.xml ├── flume-ng-configfilters ├── flume-ng-config-filter-api │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── flume │ │ └── configfilter │ │ ├── AbstractConfigFilter.java │ │ └── ConfigFilter.java ├── flume-ng-environment-variable-config-filter │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── flume │ │ │ └── configfilter │ │ │ └── EnvironmentVariableConfigFilter.java │ │ └── test │ │ └── java │ │ └── org │ │ └── apache │ │ └── flume │ │ └── configfilter │ │ └── TestEnvironmentVariableConfigFilter.java ├── flume-ng-external-process-config-filter │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── flume │ │ │ └── configfilter │ │ │ └── ExternalProcessConfigFilter.java │ │ └── test │ │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── flume │ │ │ └── configfilter │ │ │ └── TestExternalProcessConfigFilter.java │ │ └── resources │ │ ├── test.sh │ │ └── test_error.sh └── pom.xml ├── flume-ng-configuration ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── flume │ │ ├── Context.java │ │ └── conf │ │ ├── BasicConfigurationConstants.java │ │ ├── ComponentConfiguration.java │ │ ├── ComponentConfigurationFactory.java │ │ ├── ComponentWithClassName.java │ │ ├── ConfigFilterFactory.java │ │ ├── ConfigurationException.java │ │ ├── FlumeConfiguration.java │ │ ├── FlumeConfigurationError.java │ │ ├── FlumeConfigurationErrorType.java │ │ ├── LogPrivacyUtil.java │ │ ├── channel │ │ ├── ChannelConfiguration.java │ │ ├── ChannelSelectorConfiguration.java │ │ ├── ChannelSelectorType.java │ │ └── ChannelType.java │ │ ├── configfilter │ │ ├── ConfigFilterConfiguration.java │ │ └── ConfigFilterType.java │ │ ├── sink │ │ ├── SinkConfiguration.java │ │ ├── SinkGroupConfiguration.java │ │ ├── SinkProcessorConfiguration.java │ │ ├── SinkProcessorType.java │ │ └── SinkType.java │ │ └── source │ │ ├── SourceConfiguration.java │ │ └── SourceType.java │ └── test │ └── java │ └── org │ └── apache │ └── flume │ └── conf │ ├── TestAgentConfiguration.java │ ├── TestFlumeConfiguration.java │ ├── TestFlumeConfigurationConfigFilter.java │ ├── channel │ └── MemoryChannelConfiguration.java │ ├── configfilter │ ├── EnvironmentVariableConfigFilterConfiguration.java │ └── MockConfigFilter.java │ ├── sink │ └── NullSinkConfiguration.java │ └── source │ ├── TestSourceConfiguration.java │ └── jms │ └── JMSSourceConfiguration.java ├── flume-ng-core ├── pom.xml ├── scripts │ ├── saveVersion.ps1 │ └── saveVersion.sh └── src │ ├── main │ ├── avro │ │ └── TransferStateFileMeta.avsc │ └── java │ │ └── org │ │ └── apache │ │ └── flume │ │ ├── Channel.java │ │ ├── ChannelException.java │ │ ├── ChannelFactory.java │ │ ├── ChannelFullException.java │ │ ├── ChannelSelector.java │ │ ├── Clock.java │ │ ├── Constants.java │ │ ├── CounterGroup.java │ │ ├── EventDrivenSource.java │ │ ├── NamedComponent.java │ │ ├── PollableSource.java │ │ ├── Sink.java │ │ ├── SinkFactory.java │ │ ├── SinkProcessor.java │ │ ├── SinkRunner.java │ │ ├── Source.java │ │ ├── SourceFactory.java │ │ ├── SourceRunner.java │ │ ├── SystemClock.java │ │ ├── Transaction.java │ │ ├── VersionAnnotation.java │ │ ├── annotations │ │ ├── Disposable.java │ │ ├── InterfaceAudience.java │ │ ├── InterfaceStability.java │ │ └── Recyclable.java │ │ ├── channel │ │ ├── AbstractChannel.java │ │ ├── AbstractChannelSelector.java │ │ ├── BasicChannelSemantics.java │ │ ├── BasicTransactionSemantics.java │ │ ├── ChannelProcessor.java │ │ ├── ChannelSelectorFactory.java │ │ ├── ChannelUtils.java │ │ ├── DefaultChannelFactory.java │ │ ├── LoadBalancingChannelSelector.java │ │ ├── MemoryChannel.java │ │ ├── MultiplexingChannelSelector.java │ │ ├── PseudoTxnMemoryChannel.java │ │ └── ReplicatingChannelSelector.java │ │ ├── client │ │ └── avro │ │ │ ├── AvroCLIClient.java │ │ │ ├── EventReader.java │ │ │ ├── ReliableEventReader.java │ │ │ ├── ReliableSpoolingFileEventReader.java │ │ │ └── SimpleTextLineEventReader.java │ │ ├── conf │ │ ├── BatchSizeSupported.java │ │ ├── Configurable.java │ │ ├── ConfigurableComponent.java │ │ ├── Configurables.java │ │ └── TransactionCapacitySupported.java │ │ ├── event │ │ └── EventHelper.java │ │ ├── formatter │ │ └── output │ │ │ ├── BucketPath.java │ │ │ ├── DefaultPathManager.java │ │ │ ├── EventFormatter.java │ │ │ ├── PathManager.java │ │ │ ├── PathManagerFactory.java │ │ │ ├── PathManagerType.java │ │ │ ├── RollTimePathManager.java │ │ │ └── TextDelimitedOutputFormatter.java │ │ ├── instrumentation │ │ ├── ChannelCounter.java │ │ ├── ChannelCounterMBean.java │ │ ├── ChannelProcessorCounter.java │ │ ├── GangliaServer.java │ │ ├── MonitorService.java │ │ ├── MonitoredCounterGroup.java │ │ ├── MonitoringType.java │ │ ├── SinkCounter.java │ │ ├── SinkCounterMBean.java │ │ ├── SinkProcessorCounter.java │ │ ├── SourceCounter.java │ │ ├── SourceCounterMBean.java │ │ ├── http │ │ │ ├── HTTPMetricsServer.java │ │ │ └── PrometheusHTTPMetricsServer.java │ │ ├── kafka │ │ │ ├── KafkaChannelCounter.java │ │ │ ├── KafkaChannelCounterMBean.java │ │ │ ├── KafkaSinkCounter.java │ │ │ ├── KafkaSinkCounterMBean.java │ │ │ ├── KafkaSourceCounter.java │ │ │ └── KafkaSourceCounterMBean.java │ │ └── util │ │ │ └── JMXPollUtil.java │ │ ├── interceptor │ │ ├── HostInterceptor.java │ │ ├── Interceptor.java │ │ ├── InterceptorBuilderFactory.java │ │ ├── InterceptorChain.java │ │ ├── InterceptorType.java │ │ ├── RegexExtractorInterceptor.java │ │ ├── RegexExtractorInterceptorMillisSerializer.java │ │ ├── RegexExtractorInterceptorPassThroughSerializer.java │ │ ├── RegexExtractorInterceptorSerializer.java │ │ ├── RegexFilteringInterceptor.java │ │ ├── RemoveHeaderInterceptor.java │ │ ├── SearchAndReplaceInterceptor.java │ │ ├── StaticInterceptor.java │ │ └── TimestampInterceptor.java │ │ ├── lifecycle │ │ ├── LifecycleAware.java │ │ ├── LifecycleController.java │ │ ├── LifecycleException.java │ │ ├── LifecycleState.java │ │ └── LifecycleSupervisor.java │ │ ├── netty │ │ └── filter │ │ │ └── PatternRule.java │ │ ├── serialization │ │ ├── AbstractAvroEventSerializer.java │ │ ├── AvroEventDeserializer.java │ │ ├── AvroEventSerializerConfigurationConstants.java │ │ ├── BodyTextEventSerializer.java │ │ ├── DecodeErrorPolicy.java │ │ ├── DurablePositionTracker.java │ │ ├── EventDeserializer.java │ │ ├── EventDeserializerFactory.java │ │ ├── EventDeserializerType.java │ │ ├── EventSerDe.java │ │ ├── EventSerializer.java │ │ ├── EventSerializerFactory.java │ │ ├── EventSerializerType.java │ │ ├── FlumeEventAvroEventSerializer.java │ │ ├── HeaderAndBodyTextEventSerializer.java │ │ ├── LengthMeasurable.java │ │ ├── LineDeserializer.java │ │ ├── PositionTracker.java │ │ ├── RemoteMarkable.java │ │ ├── Resettable.java │ │ ├── ResettableFileInputStream.java │ │ ├── ResettableInputStream.java │ │ └── Seekable.java │ │ ├── sink │ │ ├── AbstractRpcSink.java │ │ ├── AbstractSingleSinkProcessor.java │ │ ├── AbstractSink.java │ │ ├── AbstractSinkProcessor.java │ │ ├── AbstractSinkSelector.java │ │ ├── AvroSink.java │ │ ├── DefaultSinkFactory.java │ │ ├── DefaultSinkProcessor.java │ │ ├── FailoverSinkProcessor.java │ │ ├── LoadBalancingSinkProcessor.java │ │ ├── LoggerSink.java │ │ ├── NullSink.java │ │ ├── RollingFileSink.java │ │ ├── SinkGroup.java │ │ ├── SinkProcessorFactory.java │ │ └── ThriftSink.java │ │ ├── source │ │ ├── AbstractEventDrivenSource.java │ │ ├── AbstractPollableSource.java │ │ ├── AbstractSource.java │ │ ├── AvroSource.java │ │ ├── BasicSourceSemantics.java │ │ ├── DefaultSourceFactory.java │ │ ├── EventDrivenSourceRunner.java │ │ ├── ExecSource.java │ │ ├── ExecSourceConfigurationConstants.java │ │ ├── MultiportSyslogTCPSource.java │ │ ├── NetcatSource.java │ │ ├── NetcatSourceConfigurationConstants.java │ │ ├── NetcatUdpSource.java │ │ ├── PollableSourceConstants.java │ │ ├── PollableSourceRunner.java │ │ ├── SequenceGeneratorSource.java │ │ ├── SpoolDirectorySource.java │ │ ├── SpoolDirectorySourceConfigurationConstants.java │ │ ├── SslContextAwareAbstractSource.java │ │ ├── StressSource.java │ │ ├── SyslogParser.java │ │ ├── SyslogSourceConfigurationConstants.java │ │ ├── SyslogTcpSource.java │ │ ├── SyslogUDPSource.java │ │ ├── SyslogUtils.java │ │ ├── ThriftSource.java │ │ ├── http │ │ │ ├── BLOBHandler.java │ │ │ ├── HTTPBadRequestException.java │ │ │ ├── HTTPSource.java │ │ │ ├── HTTPSourceConfigurationConstants.java │ │ │ ├── HTTPSourceHandler.java │ │ │ └── JSONHandler.java │ │ └── shaded │ │ │ └── guava │ │ │ ├── RateLimiter.java │ │ │ ├── SmoothRateLimiter.java │ │ │ ├── Stopwatch.java │ │ │ └── Uninterruptibles.java │ │ └── tools │ │ ├── DirectMemoryUtils.java │ │ ├── FlumeBeanConfigurator.java │ │ ├── GetJavaProperty.java │ │ ├── HTTPServerConstraintUtil.java │ │ ├── PlatformDetect.java │ │ ├── TimestampRoundDownUtil.java │ │ └── VersionInfo.java │ └── test │ ├── java │ └── org │ │ └── apache │ │ └── flume │ │ ├── TestContext.java │ │ ├── TestCounterGroup.java │ │ ├── channel │ │ ├── AbstractBasicChannelSemanticsTest.java │ │ ├── MockChannel.java │ │ ├── MockEvent.java │ │ ├── TestBasicChannelSemantics.java │ │ ├── TestChannelProcessor.java │ │ ├── TestChannelUtils.java │ │ ├── TestLoadBalancingChannelSelector.java │ │ ├── TestMemoryChannel.java │ │ ├── TestMemoryChannelConcurrency.java │ │ ├── TestMemoryChannelTransaction.java │ │ ├── TestMultiplexingChannelSelector.java │ │ └── TestReplicatingChannelSelector.java │ │ ├── client │ │ └── avro │ │ │ ├── TestBufferedLineReader.java │ │ │ ├── TestReliableSpoolingFileEventReader.java │ │ │ └── TestSpoolingFileLineReader.java │ │ ├── event │ │ └── TestEventHelper.java │ │ ├── formatter │ │ └── output │ │ │ └── TestBucketPath.java │ │ ├── instrumentation │ │ ├── TestMonitoredCounterGroup.java │ │ ├── http │ │ │ ├── BaseHTTPMetricsTest.java │ │ │ ├── TestHTTPMetricsServer.java │ │ │ └── TestPrometheusMetricsServer.java │ │ ├── kafka │ │ │ └── KafkaSourceCounterTest.java │ │ └── util │ │ │ ├── JMXTestUtils.java │ │ │ └── TestJMXPollUtil.java │ │ ├── interceptor │ │ ├── CensoringInterceptor.java │ │ ├── RemoveHeaderInterceptorTest.java │ │ ├── TestCensoringInterceptor.java │ │ ├── TestHostInterceptor.java │ │ ├── TestRegexExtractorInterceptor.java │ │ ├── TestRegexExtractorInterceptorMillisSerializer.java │ │ ├── TestRegexExtractorInterceptorPassThroughSerializer.java │ │ ├── TestRegexFilteringInterceptor.java │ │ ├── TestSearchAndReplaceInterceptor.java │ │ ├── TestStaticInterceptor.java │ │ └── TestTimestampInterceptor.java │ │ ├── lifecycle │ │ ├── TestLifecycleController.java │ │ └── TestLifecycleSupervisor.java │ │ ├── serialization │ │ ├── ResettableTestStringInputStream.java │ │ ├── SyslogAvroEventSerializer.java │ │ ├── TestAvroEventDeserializer.java │ │ ├── TestBodyTextEventSerializer.java │ │ ├── TestDurablePositionTracker.java │ │ ├── TestFlumeEventAvroEventSerializer.java │ │ ├── TestHeaderAndBodyTextEventSerializer.java │ │ ├── TestLineDeserializer.java │ │ ├── TestResettableFileInputStream.java │ │ ├── TestSyslogAvroEventSerializer.java │ │ └── TransientPositionTracker.java │ │ ├── sink │ │ ├── FixedOrderSelector.java │ │ ├── SinkProcessorFactoryTest.java │ │ ├── TestAvroSink.java │ │ ├── TestDefaultSinkFactory.java │ │ ├── TestFailoverSinkProcessor.java │ │ ├── TestLoadBalancingSinkProcessor.java │ │ ├── TestLoggerSink.java │ │ ├── TestRollingFileSink.java │ │ └── TestThriftSink.java │ │ ├── source │ │ ├── MockSource.java │ │ ├── TestAbstractPollableSource.java │ │ ├── TestAvroSource.java │ │ ├── TestBasicSourceSemantics.java │ │ ├── TestDefaultSourceFactory.java │ │ ├── TestExecSource.java │ │ ├── TestMultiportSyslogTCPSource.java │ │ ├── TestNetcatSource.java │ │ ├── TestNetcatUdpSource.java │ │ ├── TestPollableSourceRunner.java │ │ ├── TestSequenceGeneratorSource.java │ │ ├── TestSpoolDirectorySource.java │ │ ├── TestStressSource.java │ │ ├── TestSyslogParser.java │ │ ├── TestSyslogTcpSource.java │ │ ├── TestSyslogUdpSource.java │ │ ├── TestSyslogUtils.java │ │ ├── TestThriftSource.java │ │ ├── http │ │ │ ├── FlumeHttpServletRequestWrapper.java │ │ │ ├── TestBLOBHandler.java │ │ │ ├── TestHTTPSource.java │ │ │ └── TestJSONHandler.java │ │ └── shaded │ │ │ └── guava │ │ │ └── TestRateLimiter.java │ │ └── tools │ │ ├── TestFlumeConfigurator.java │ │ ├── TestTimestampRoundDownUtil.java │ │ └── TestVersionInfo.java │ └── resources │ ├── TestResettableFileInputStream_1.avro │ ├── TestResettableFileInputStream_1.truncated.avro │ ├── certs │ ├── gencerts.sh │ ├── rootca.conf │ └── server.conf │ ├── jettykeystore │ ├── keystorefile.jks │ ├── log4j2.xml │ ├── server.flume-crt.pem │ ├── server.flume-keystore.p12 │ ├── server.flume.pem │ ├── server.p12 │ ├── syslog_event.avsc │ ├── test_command.ps1 │ ├── test_command.txt │ ├── truststore.jks │ └── truststorefile.jks ├── flume-ng-dist ├── pom.xml └── src │ └── main │ ├── appended-resources │ └── META-INF │ │ ├── LICENSE │ │ └── NOTICE │ └── assembly │ ├── bin.xml │ └── src.xml ├── flume-ng-embedded-agent ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── flume │ │ └── agent │ │ └── embedded │ │ ├── EmbeddedAgent.java │ │ ├── EmbeddedAgentConfiguration.java │ │ ├── EmbeddedSource.java │ │ ├── MaterializedConfigurationProvider.java │ │ ├── MemoryConfigurationProvider.java │ │ └── package-info.java │ └── test │ ├── java │ └── org │ │ └── apache │ │ └── flume │ │ └── agent │ │ └── embedded │ │ ├── TestEmbeddedAgent.java │ │ ├── TestEmbeddedAgentConfiguration.java │ │ ├── TestEmbeddedAgentEmbeddedSource.java │ │ └── TestEmbeddedAgentState.java │ └── resources │ └── log4j2.xml ├── flume-ng-node ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── flume │ │ │ └── node │ │ │ ├── AbstractConfigurationProvider.java │ │ │ ├── AbstractZooKeeperConfigurationProvider.java │ │ │ ├── Application.java │ │ │ ├── ClasspathConfigurationSource.java │ │ │ ├── ClasspathConfigurationSourceFactory.java │ │ │ ├── ConfigurationProvider.java │ │ │ ├── ConfigurationSource.java │ │ │ ├── ConfigurationSourceFactory.java │ │ │ ├── EnvVarResolverProperties.java │ │ │ ├── FileConfigurationSource.java │ │ │ ├── FileConfigurationSourceFactory.java │ │ │ ├── HttpConfigurationSource.java │ │ │ ├── HttpConfigurationSourceFactory.java │ │ │ ├── Initializable.java │ │ │ ├── MapResolver.java │ │ │ ├── MaterializedConfiguration.java │ │ │ ├── PollingPropertiesFileConfigurationProvider.java │ │ │ ├── PollingZooKeeperConfigurationProvider.java │ │ │ ├── PropertiesFileConfigurationProvider.java │ │ │ ├── SimpleMaterializedConfiguration.java │ │ │ ├── StaticZooKeeperConfigurationProvider.java │ │ │ ├── UriConfigurationProvider.java │ │ │ └── net │ │ │ ├── AuthorizationProvider.java │ │ │ ├── BasicAuthorizationProvider.java │ │ │ ├── LaxHostnameVerifier.java │ │ │ └── UrlConnectionFactory.java │ └── resources │ │ └── META-INF │ │ └── services │ │ └── org.apache.flume.node.ConfigurationSourceFactory │ └── test │ ├── java │ └── org │ │ └── apache │ │ └── flume │ │ ├── node │ │ ├── TestAbstractConfigurationProvider.java │ │ ├── TestAbstractZooKeeperConfigurationProvider.java │ │ ├── TestApplication.java │ │ ├── TestClasspathConfigurationSource.java │ │ ├── TestEnvLookup.java │ │ ├── TestEnvVarResolverProperties.java │ │ ├── TestHttpConfigurationSource.java │ │ ├── TestMapResolver.java │ │ ├── TestOverrideFile.java │ │ ├── TestPollingPropertiesFileConfigurationProvider.java │ │ ├── TestPollingZooKeeperConfigurationProvider.java │ │ ├── TestPropertiesFileConfigurationProvider.java │ │ ├── TestRecursiveLookup.java │ │ ├── TestStaticZooKeeperConfigurationProvider.java │ │ └── lookup │ │ │ └── TestLookup.java │ │ ├── sink │ │ └── NullInitSink.java │ │ └── source │ │ ├── EventProcessor.java │ │ ├── LocalSource.java │ │ └── TestNetcatSource.java │ └── resources │ ├── flume-conf-init.properties │ ├── flume-conf-override.properties │ ├── flume-conf-with-envLookup.properties │ ├── flume-conf-with-envvars.properties │ ├── flume-conf-with-recursiveLookup.properties │ ├── flume-conf.properties │ ├── flume-conf.properties.2786 │ ├── log4j2.xml │ ├── map-resolver.properties │ └── test-lookups.properties ├── flume-ng-sdk ├── pom.xml └── src │ ├── main │ ├── avro │ │ └── flume.avdl │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── flume │ │ │ ├── Event.java │ │ │ ├── EventDeliveryException.java │ │ │ ├── FlumeException.java │ │ │ ├── api │ │ │ ├── AbstractRpcClient.java │ │ │ ├── FailoverRpcClient.java │ │ │ ├── HostInfo.java │ │ │ ├── LoadBalancingRpcClient.java │ │ │ ├── NettyAvroRpcClient.java │ │ │ ├── RpcClient.java │ │ │ ├── RpcClientConfigurationConstants.java │ │ │ ├── RpcClientFactory.java │ │ │ ├── SSLContextAwareAbstractRpcClient.java │ │ │ └── ThriftRpcClient.java │ │ │ ├── event │ │ │ ├── EventBuilder.java │ │ │ ├── JSONEvent.java │ │ │ └── SimpleEvent.java │ │ │ ├── thrift │ │ │ ├── Status.java │ │ │ ├── ThriftFlumeEvent.java │ │ │ └── ThriftSourceProtocol.java │ │ │ └── util │ │ │ ├── OrderSelector.java │ │ │ ├── RandomOrderSelector.java │ │ │ ├── RoundRobinOrderSelector.java │ │ │ ├── SSLUtil.java │ │ │ └── SpecificOrderIterator.java │ └── thrift │ │ ├── aslv2 │ │ └── flume.thrift │ └── test │ ├── java │ └── org │ │ └── apache │ │ └── flume │ │ ├── api │ │ ├── RpcTestUtils.java │ │ ├── TestFailoverRpcClient.java │ │ ├── TestLoadBalancingRpcClient.java │ │ ├── TestNettyAvroRpcClient.java │ │ ├── TestRpcClientFactory.java │ │ ├── TestThriftRpcClient.java │ │ └── ThriftTestingSource.java │ │ ├── event │ │ └── TestEventBuilder.java │ │ └── util │ │ ├── AbstractSSLUtilListTest.java │ │ ├── AbstractSSLUtilTest.java │ │ ├── SSLUtilExcludeCipherSuitesTest.java │ │ ├── SSLUtilExcludeProtocolsTest.java │ │ ├── SSLUtilIncludeCipherSuitesTest.java │ │ ├── SSLUtilIncludeProtocolsTest.java │ │ ├── SSLUtilKeystorePasswordTest.java │ │ ├── SSLUtilKeystorePathTest.java │ │ ├── SSLUtilKeystoreTypeTest.java │ │ ├── SSLUtilKeystoreTypeWithDefaultTest.java │ │ ├── SSLUtilTruststorePasswordTest.java │ │ ├── SSLUtilTruststorePathTest.java │ │ ├── SSLUtilTruststoreTypeTest.java │ │ └── SSLUtilTruststoreTypeWithDefaultTest.java │ └── resources │ └── log4j2.xml ├── flume-ng-sinks ├── flume-http-sink │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── flume │ │ │ └── sink │ │ │ └── http │ │ │ ├── HttpSink.java │ │ │ └── package-info.java │ │ └── test │ │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── flume │ │ │ └── sink │ │ │ └── http │ │ │ ├── TestHttpSink.java │ │ │ └── TestHttpSinkIT.java │ │ └── resources │ │ └── log4j2.xml ├── flume-irc-sink │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── flume │ │ │ └── sink │ │ │ └── irc │ │ │ └── IRCSink.java │ │ └── test │ │ └── java │ │ └── org │ │ └── apache │ │ └── flume │ │ └── sink │ │ └── irc │ │ └── TestIRCSink.java ├── flume-ng-morphline-solr-sink │ ├── README.md │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── flume │ │ │ └── sink │ │ │ └── solr │ │ │ └── morphline │ │ │ ├── BlobDeserializer.java │ │ │ ├── BlobHandler.java │ │ │ ├── MorphlineHandler.java │ │ │ ├── MorphlineHandlerImpl.java │ │ │ ├── MorphlineInterceptor.java │ │ │ ├── MorphlineSink.java │ │ │ ├── MorphlineSolrSink.java │ │ │ └── UUIDInterceptor.java │ │ └── test │ │ ├── appended-resources │ │ └── META-INF │ │ │ └── LICENSE │ │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── flume │ │ │ └── sink │ │ │ └── solr │ │ │ └── morphline │ │ │ ├── EmbeddedSource.java │ │ │ ├── FlumeHttpServletRequestWrapper.java │ │ │ ├── ResettableTestStringInputStream.java │ │ │ ├── TestBlobDeserializer.java │ │ │ ├── TestBlobHandler.java │ │ │ ├── TestEnvironment.java │ │ │ ├── TestMorphlineInterceptor.java │ │ │ ├── TestMorphlineSolrSink.java │ │ │ └── TestUUIDInterceptor.java │ │ └── resources │ │ ├── custom-mimetypes.xml │ │ ├── log4j2.xml │ │ ├── solr │ │ └── collection1 │ │ │ └── conf │ │ │ ├── currency.xml │ │ │ ├── elevate.xml │ │ │ ├── lang │ │ │ ├── contractions_ca.txt │ │ │ ├── contractions_fr.txt │ │ │ ├── contractions_ga.txt │ │ │ ├── contractions_it.txt │ │ │ ├── hyphenations_ga.txt │ │ │ ├── stemdict_nl.txt │ │ │ ├── stoptags_ja.txt │ │ │ ├── stopwords_ar.txt │ │ │ ├── stopwords_bg.txt │ │ │ ├── stopwords_ca.txt │ │ │ ├── stopwords_cz.txt │ │ │ ├── stopwords_da.txt │ │ │ ├── stopwords_de.txt │ │ │ ├── stopwords_el.txt │ │ │ ├── stopwords_en.txt │ │ │ ├── stopwords_es.txt │ │ │ ├── stopwords_eu.txt │ │ │ ├── stopwords_fa.txt │ │ │ ├── stopwords_fi.txt │ │ │ ├── stopwords_fr.txt │ │ │ ├── stopwords_ga.txt │ │ │ ├── stopwords_gl.txt │ │ │ ├── stopwords_hi.txt │ │ │ ├── stopwords_hu.txt │ │ │ ├── stopwords_hy.txt │ │ │ ├── stopwords_id.txt │ │ │ ├── stopwords_it.txt │ │ │ ├── stopwords_ja.txt │ │ │ ├── stopwords_lv.txt │ │ │ ├── stopwords_nl.txt │ │ │ ├── stopwords_no.txt │ │ │ ├── stopwords_pt.txt │ │ │ ├── stopwords_ro.txt │ │ │ ├── stopwords_ru.txt │ │ │ ├── stopwords_sv.txt │ │ │ ├── stopwords_th.txt │ │ │ ├── stopwords_tr.txt │ │ │ └── userdict_ja.txt │ │ │ ├── protwords.txt │ │ │ ├── schema.xml │ │ │ ├── solrconfig.xml │ │ │ ├── stopwords.txt │ │ │ └── synonyms.txt │ │ ├── test-documents │ │ ├── NullHeader.docx │ │ ├── boilerplate.html │ │ ├── cars.csv │ │ ├── cars.csv.gz │ │ ├── cars.ssv │ │ ├── cars.tar.gz │ │ ├── cars.tsv │ │ ├── complex.mbox │ │ ├── multiline-sessions.log │ │ ├── multiline-stacktrace-expected-long-event.log │ │ ├── multiline-stacktrace.log │ │ ├── non-length-delimited-20130430-234145-tweets.json.gz │ │ ├── rsstest.rss │ │ ├── sample-statuses-20120906-141433 │ │ ├── sample-statuses-20120906-141433-medium.avro │ │ ├── sample-statuses-20120906-141433-subschema.avsc │ │ ├── sample-statuses-20120906-141433.avro │ │ ├── sample-statuses-20120906-141433.avsc │ │ ├── sample-statuses-20120906-141433.bz2 │ │ ├── sample-statuses-20120906-141433.gz │ │ ├── test-documents.7z │ │ ├── test-documents.cpio │ │ ├── test-documents.tar │ │ ├── test-documents.tbz2 │ │ ├── test-documents.tgz │ │ ├── test-documents.zip │ │ ├── test-outlook.msg │ │ ├── test-zip-of-zip.zip │ │ ├── testAIFF.aif │ │ ├── testBMP.bmp │ │ ├── testBMPfp.txt │ │ ├── testDITA.dita │ │ ├── testEMLX.emlx │ │ ├── testEXCEL.xls │ │ ├── testEXCEL.xlsx │ │ ├── testFLAC.flac │ │ ├── testFLAC.oga │ │ ├── testFLV.flv │ │ ├── testGIF.gif │ │ ├── testJAR.jar │ │ ├── testJPEG_EXIF.jpg │ │ ├── testJPEG_EXIF.jpg.gz │ │ ├── testJPEG_EXIF.jpg.tar.gz │ │ ├── testKML.kml │ │ ├── testKeynote.key │ │ ├── testMP3i18n.mp3 │ │ ├── testMP4.m4a │ │ ├── testNumbers.numbers │ │ ├── testPDF.pdf │ │ ├── testPNG.png │ │ ├── testPPM.ppm │ │ ├── testPPT_various.ppt │ │ ├── testPPT_various.pptx │ │ ├── testPSD.psd │ │ ├── testPages.pages │ │ ├── testRDF.rdf │ │ ├── testRFC822 │ │ ├── testRTFVarious.rtf │ │ ├── testSVG.svg │ │ ├── testTIFF.tif │ │ ├── testTrueType.ttf │ │ ├── testVISIO.vsd │ │ ├── testVORBIS.ogg │ │ ├── testWAR.war │ │ ├── testWAV.wav │ │ ├── testWINMAIL.dat │ │ ├── testWMA.wma │ │ ├── testWMF.wmf │ │ ├── testWMV.wmv │ │ ├── testWORD_various.doc │ │ ├── testWindows-x86-32.exe │ │ └── testXML.xml │ │ └── test-morphlines │ │ ├── grokIfNotMatchDropRecord.conf │ │ ├── ifDetectMimeType.conf │ │ ├── noOperation.conf │ │ ├── readClob.conf │ │ └── solrCellDocumentTypes.conf └── pom.xml ├── flume-ng-sources ├── flume-jms-source │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── flume │ │ │ └── source │ │ │ └── jms │ │ │ ├── DefaultJMSMessageConverter.java │ │ │ ├── InitialContextFactory.java │ │ │ ├── JMSDestinationLocator.java │ │ │ ├── JMSDestinationType.java │ │ │ ├── JMSMessageConsumer.java │ │ │ ├── JMSMessageConverter.java │ │ │ ├── JMSSource.java │ │ │ └── JMSSourceConfiguration.java │ │ └── test │ │ └── java │ │ └── org │ │ └── apache │ │ └── flume │ │ └── source │ │ └── jms │ │ ├── JMSMessageConsumerTestBase.java │ │ ├── TestDefaultJMSMessageConverter.java │ │ ├── TestIntegrationActiveMQ.java │ │ ├── TestJMSMessageConsumer.java │ │ ├── TestJMSSource.java │ │ └── TestJMSSourceCreation.java ├── flume-scribe-source │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── org │ │ │ │ └── apache │ │ │ │ └── flume │ │ │ │ └── source │ │ │ │ └── scribe │ │ │ │ ├── LogEntry.java │ │ │ │ ├── ResultCode.java │ │ │ │ ├── Scribe.java │ │ │ │ └── ScribeSource.java │ │ └── thrift │ │ │ ├── aslv2 │ │ │ └── scribe-source.thrift │ │ └── test │ │ └── java │ │ └── org │ │ └── apache │ │ └── flume │ │ └── source │ │ └── scribe │ │ └── TestScribeSource.java ├── flume-taildir-source │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── flume │ │ │ └── source │ │ │ └── taildir │ │ │ ├── ReliableTaildirEventReader.java │ │ │ ├── TailFile.java │ │ │ ├── TaildirMatcher.java │ │ │ ├── TaildirSource.java │ │ │ └── TaildirSourceConfigurationConstants.java │ │ └── test │ │ └── java │ │ └── org │ │ └── apache │ │ └── flume │ │ └── source │ │ └── taildir │ │ ├── TestTaildirEventReader.java │ │ ├── TestTaildirMatcher.java │ │ └── TestTaildirSource.java └── pom.xml ├── flume-ng-tests ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── flume │ │ └── Dummy.java │ └── test │ ├── java │ └── org │ │ └── apache │ │ └── flume │ │ └── test │ │ ├── agent │ │ ├── TestConfig.java │ │ ├── TestFileChannel.java │ │ ├── TestRpcClient.java │ │ ├── TestRpcClientCommunicationFailure.java │ │ ├── TestSpooldirSource.java │ │ └── TestSyslogSource.java │ │ └── util │ │ ├── StagedInstall.java │ │ └── SyslogAgent.java │ └── resources │ ├── log4j2.xml │ └── rpc-client-test.properties ├── flume-parent └── pom.xml ├── flume-tools ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── flume │ │ └── tools │ │ ├── EventValidator.java │ │ ├── FileChannelIntegrityTool.java │ │ ├── FlumeTool.java │ │ ├── FlumeToolType.java │ │ └── FlumeToolsMain.java │ └── test │ └── java │ └── org │ └── apache │ └── flume │ └── tools │ └── TestFileChannelIntegrityTool.java ├── mvnw ├── mvnw.cmd └── pom.xml /.gitattributes: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # All text files with LF line endings 17 | * text=auto eol=lf 18 | # PS1 files are Windows specific 19 | *.ps1 eol=crlf 20 | # Maven Wrapper cmd script 21 | /mvnw.cmd eol=crlf 22 | # Maven Wrapper need LF line endings 23 | /.mvn/wrapper/maven-wrapper.properties eol=lf 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Lines that start with '#' are comments. 2 | *~ 3 | *.diff 4 | *# 5 | .classpath 6 | .project 7 | .settings 8 | bin/flume-env.sh 9 | conf/flume-site.xml 10 | bin/.settings 11 | .eclipse 12 | pmd_report.html 13 | */bin 14 | target 15 | patchprocess 16 | derby.log 17 | .idea 18 | *.iml 19 | nb-configuration.xml 20 | .DS_Store 21 | /.mvn/wrapper/maven-wrapper.jar 22 | **/metastore_db 23 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.6/apache-maven-3.9.6-bin.zip 18 | wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar 19 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Apache Flume 2 | Copyright 2011-2022 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | 7 | Portions of this software were developed at 8 | Cloudera, Inc. (http://www.cloudera.com/), copyright 2009-2011. 9 | -------------------------------------------------------------------------------- /bin/flume-ng.cmd: -------------------------------------------------------------------------------- 1 | @REM Licensed to the Apache Software Foundation (ASF) under one 2 | @REM or more contributor license agreements. See the NOTICE file 3 | @REM distributed with this work for additional information 4 | @REM regarding copyright ownership. The ASF licenses this file 5 | @REM to you under the Apache License, Version 2.0 (the 6 | @REM "License"); you may not use this file except in compliance 7 | @REM with the License. You may obtain a copy of the License at 8 | @REM 9 | @REM http://www.apache.org/licenses/LICENSE-2.0 10 | @REM 11 | @REM Unless required by applicable law or agreed to in writing, 12 | @REM software distributed under the License is distributed on an 13 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | @REM KIND, either express or implied. See the License for the 15 | @REM specific language governing permissions and limitations 16 | @REM under the License. 17 | 18 | powershell.exe -NoProfile -InputFormat none -ExecutionPolicy unrestricted -File %~dp0flume-ng.ps1 %* 19 | 20 | -------------------------------------------------------------------------------- /conf/flume-env.ps1.template: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | # Give Flume more memory and pre-allocate, enable remote monitoring via JMX 19 | $JAVA_OPTS="-Xms100m -Xmx200m -Dcom.sun.management.jmxremote" 20 | 21 | # Let Flume write raw event data and configuration information to its log files for debugging 22 | # purposes. Enabling these flags is not recommended in production, 23 | # as it may result in logging sensitive user information or encryption secrets. 24 | # $JAVA_OPTS="$JAVA_OPTS -Dorg.apache.flume.log.rawdata=true -Dorg.apache.flume.log.printconfig=true " 25 | 26 | # Foll. classpath will be included in Flume's classpath. 27 | # Note that the Flume conf directory is always included in the classpath. 28 | $FLUME_CLASSPATH="" # Example: "path1;path2;path3" 29 | -------------------------------------------------------------------------------- /dev-support/includes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | ################################################################################ 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | ################################################################################ 20 | # Utilities for dev-support scripts. 21 | ################################################################################ 22 | 23 | # Print an error message and exit. 24 | error() { 25 | echo $1 1>&2 26 | exit 1 27 | } 28 | 29 | # Searches the PATH for each command name passed, and returns the path of the 30 | # first one found. 31 | find_in_path() { 32 | for COMMAND in "$@"; do 33 | FOUND=$(which $COMMAND) 34 | if [ -n "$FOUND" ]; then 35 | echo "$FOUND" 36 | return 37 | fi 38 | done 39 | error "Cannot find $1. Please install $1 to continue." 40 | } 41 | -------------------------------------------------------------------------------- /dev-support/license-diff.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ################################################################################ 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | ################################################################################ 20 | # Script to compare LICENSE file content and distributed jars 21 | # use after flume-ng-dist target has been generated (mvn clean package) 22 | ################################################################################ 23 | PROJECT_ROOT="$(cd $(dirname $0)/..; pwd)" 24 | 25 | echo "LICENSE file <---------> jar files" 26 | diff -y <(grep .jar: ${PROJECT_ROOT}/LICENSE | sed -E 's/:|\s+//g' | sort) <(ls ${PROJECT_ROOT}/flume-ng-dist/target/apache-flume-*-bin/apache-flume-*-bin/lib/*.jar |xargs -n1 basename |grep -v flume | sed -E 's/(.+)-([^-]+).jar/\1-.jar/' | uniq | sort) 27 | -------------------------------------------------------------------------------- /flume-ng-auth/src/main/java/org/apache/flume/auth/SecurityException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume.auth; 19 | 20 | /** 21 | * SecurityException thrown in the Flume security module 22 | */ 23 | public class SecurityException extends RuntimeException { 24 | public SecurityException(String message) { 25 | super(message); 26 | } 27 | 28 | public SecurityException(String message, Throwable cause) { 29 | super(message, cause); 30 | } 31 | 32 | public SecurityException(Throwable cause) { 33 | super(cause); 34 | } 35 | } 36 | 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/BadCheckpointException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flume.channel.file; 20 | 21 | import org.apache.flume.FlumeException; 22 | 23 | /** 24 | * Exception thrown when the checkpoint directory contains invalid data, 25 | * probably due to the channel stopping while the checkpoint was written. 26 | */ 27 | public class BadCheckpointException extends FlumeException { 28 | private static final long serialVersionUID = -5038652693746472779L; 29 | 30 | public BadCheckpointException(String msg) { 31 | super(msg); 32 | } 33 | 34 | public BadCheckpointException(String msg, Throwable t) { 35 | super(msg, t); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/CorruptEventException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.channel.file; 20 | 21 | public class CorruptEventException extends Exception { 22 | private static final long serialVersionUID = -2986946303540798416L; 23 | 24 | public CorruptEventException() { 25 | super(); 26 | } 27 | 28 | public CorruptEventException(String msg) { 29 | super(msg); 30 | } 31 | 32 | public CorruptEventException(String msg, Throwable th) { 33 | super(msg, th); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/EventUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.flume.channel.file; 21 | 22 | import org.apache.flume.Event; 23 | 24 | /** 25 | * 26 | */ 27 | public class EventUtils { 28 | 29 | /** 30 | * Returns the Event encapsulated by a Put wrapper 31 | * 32 | * @param transactionEventRecord TransactionEvent 33 | * @return Event if Put instance is present, null otherwise 34 | */ 35 | public static Event getEventFromTransactionEvent(TransactionEventRecord transactionEventRecord) { 36 | if (transactionEventRecord instanceof Put) { 37 | return ((Put)transactionEventRecord).getEvent(); 38 | } 39 | return null; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileRetryableIOException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.channel.file; 20 | 21 | import java.io.IOException; 22 | 23 | public class LogFileRetryableIOException extends IOException { 24 | private static final long serialVersionUID = -2747112999806160431L; 25 | 26 | public LogFileRetryableIOException() { 27 | super(); 28 | } 29 | 30 | public LogFileRetryableIOException(String msg) { 31 | super(msg); 32 | } 33 | 34 | public LogFileRetryableIOException(String msg, Throwable t) { 35 | super(msg, t); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/NoopRecordException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.channel.file; 20 | 21 | public class NoopRecordException extends Exception { 22 | private static final long serialVersionUID = -7394180633208889738L; 23 | 24 | public NoopRecordException() { 25 | super(); 26 | } 27 | 28 | public NoopRecordException(String msg) { 29 | super(msg); 30 | } 31 | 32 | public NoopRecordException(String msg, Throwable th) { 33 | super(msg, th); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Pair.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.channel.file; 20 | 21 | class Pair { 22 | private final L left; 23 | private final R right; 24 | 25 | Pair(L l, R r) { 26 | left = l; 27 | right = r; 28 | } 29 | 30 | L getLeft() { 31 | return left; 32 | } 33 | R getRight() { 34 | return right; 35 | } 36 | 37 | static Pair of(L left, R right) { 38 | return new Pair(left, right); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionIDOracle.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.channel.file; 20 | 21 | import java.util.concurrent.atomic.AtomicLong; 22 | 23 | public final class TransactionIDOracle { 24 | 25 | private TransactionIDOracle() { 26 | } 27 | 28 | private static final AtomicLong TRANSACTION_ID = 29 | new AtomicLong(System.currentTimeMillis()); 30 | 31 | public static void setSeed(long highest) { 32 | long previous; 33 | while (highest > (previous = TRANSACTION_ID.get())) { 34 | TRANSACTION_ID.compareAndSet(previous, highest); 35 | } 36 | } 37 | 38 | public static long next() { 39 | return TRANSACTION_ID.incrementAndGet(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/WriteOrderOracle.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.channel.file; 20 | 21 | import java.util.concurrent.atomic.AtomicLong; 22 | 23 | public final class WriteOrderOracle { 24 | 25 | private WriteOrderOracle() { 26 | } 27 | 28 | private static final AtomicLong WRITER_ORDERER = 29 | new AtomicLong(System.currentTimeMillis()); 30 | 31 | public static void setSeed(long highest) { 32 | long previous; 33 | while (highest > (previous = WRITER_ORDERER.get())) { 34 | WRITER_ORDERER.compareAndSet(previous, highest); 35 | } 36 | } 37 | 38 | public static long next() { 39 | return WRITER_ORDERER.incrementAndGet(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/encryption/CipherProviderType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.channel.file.encryption; 20 | 21 | public enum CipherProviderType { 22 | AESCTRNOPADDING(AESCTRNoPaddingProvider.class), 23 | OTHER(null); 24 | 25 | private final Class providerClass; 26 | 27 | CipherProviderType(Class providerClass) { 28 | this.providerClass = providerClass; 29 | } 30 | 31 | public Class getProviderClass() { 32 | return providerClass; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/encryption/DecryptionFailureException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.channel.file.encryption; 20 | 21 | import org.apache.flume.FlumeException; 22 | 23 | /** 24 | * Exception that is thrown when the channel is unable to decrypt an even 25 | * read from the channel. 26 | */ 27 | public class DecryptionFailureException extends FlumeException { 28 | private static final long serialVersionUID = 6646810195384793646L; 29 | 30 | public DecryptionFailureException(String msg) { 31 | super(msg); 32 | } 33 | 34 | public DecryptionFailureException(String msg, Throwable th) { 35 | super(msg, th); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/encryption/KeyProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.channel.file.encryption; 20 | 21 | import java.security.Key; 22 | 23 | import org.apache.flume.Context; 24 | 25 | public abstract class KeyProvider { 26 | 27 | /** 28 | * Returns a non-null Key 29 | */ 30 | public abstract Key getKey(String alias); 31 | 32 | public interface Builder { 33 | public abstract KeyProvider build(Context context); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/encryption/KeyProviderType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.channel.file.encryption; 20 | 21 | public enum KeyProviderType { 22 | JCEKSFILE(JCEFileKeyProvider.Builder.class), 23 | OTHER(null); 24 | 25 | private final Class keyProviderClass; 26 | 27 | KeyProviderType(Class keyStoreProviderClass) { 28 | this.keyProviderClass = keyStoreProviderClass; 29 | } 30 | 31 | public Class getBuilderClass() { 32 | return keyProviderClass; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestTransactionIDOracle.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.channel.file; 20 | 21 | import junit.framework.Assert; 22 | 23 | import org.junit.Test; 24 | 25 | public class TestTransactionIDOracle { 26 | 27 | 28 | @Test 29 | public void testSetSeed() { 30 | long current = TransactionIDOracle.next(); 31 | current += Integer.MAX_VALUE; 32 | TransactionIDOracle.setSeed(current); 33 | Assert.assertTrue(TransactionIDOracle.next() > System.currentTimeMillis()); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestWriteOrderOracle.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.channel.file; 20 | 21 | import junit.framework.Assert; 22 | 23 | import org.junit.Test; 24 | 25 | public class TestWriteOrderOracle { 26 | 27 | 28 | @Test 29 | public void testSetSeed() { 30 | long current = WriteOrderOracle.next(); 31 | current += Integer.MAX_VALUE; 32 | WriteOrderOracle.setSeed(current); 33 | Assert.assertTrue(WriteOrderOracle.next() > System.currentTimeMillis()); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/test/resources/fileformat-v2-checkpoint.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-channels/flume-file-channel/src/test/resources/fileformat-v2-checkpoint.gz -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/test/resources/fileformat-v2-log-1.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-channels/flume-file-channel/src/test/resources/fileformat-v2-log-1.gz -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/test/resources/fileformat-v2-log-2.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-channels/flume-file-channel/src/test/resources/fileformat-v2-log-2.gz -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/test/resources/fileformat-v2-log-3.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-channels/flume-file-channel/src/test/resources/fileformat-v2-log-3.gz -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/test/resources/fileformat-v2-pre-FLUME-1432-checkpoint.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-channels/flume-file-channel/src/test/resources/fileformat-v2-pre-FLUME-1432-checkpoint.gz -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/test/resources/fileformat-v2-pre-FLUME-1432-log-1.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-channels/flume-file-channel/src/test/resources/fileformat-v2-pre-FLUME-1432-log-1.gz -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/test/resources/fileformat-v2-pre-FLUME-1432-log-2.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-channels/flume-file-channel/src/test/resources/fileformat-v2-pre-FLUME-1432-log-2.gz -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/test/resources/fileformat-v2-pre-FLUME-1432-log-3.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-channels/flume-file-channel/src/test/resources/fileformat-v2-pre-FLUME-1432-log-3.gz -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/test/resources/ibm-test.keystore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-channels/flume-file-channel/src/test/resources/ibm-test.keystore -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /flume-ng-channels/flume-file-channel/src/test/resources/sun-test.keystore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-channels/flume-file-channel/src/test/resources/sun-test.keystore -------------------------------------------------------------------------------- /flume-ng-channels/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 20 | 21 | 4.0.0 22 | 23 | 24 | flume-parent 25 | org.apache.flume 26 | 1.11.1-SNAPSHOT 27 | ../flume-parent/pom.xml 28 | 29 | 30 | flume-ng-channels 31 | Flume NG Channels 32 | pom 33 | 34 | 35 | flume-file-channel 36 | flume-spillable-memory-channel 37 | 38 | 39 | -------------------------------------------------------------------------------- /flume-ng-clients/flume-ng-log4jappender/src/test/resources/flume-loadbalancing-backoff-log4jtest.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | log4j.appender.out2 = org.apache.flume.clients.log4jappender.LoadBalancingLog4jAppender 18 | #log4j.appender.out2.Hosts = Set from java source 19 | log4j.appender.out2.Selector = ROUND_ROBIN 20 | log4j.appender.out2.MaxBackoff = 30000 21 | log4j.logger.org.apache.flume.clients.log4jappender = DEBUG,out2 -------------------------------------------------------------------------------- /flume-ng-clients/flume-ng-log4jappender/src/test/resources/flume-loadbalancing-rnd-log4jtest.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | log4j.appender.out2 = org.apache.flume.clients.log4jappender.LoadBalancingLog4jAppender 18 | #log4j.appender.out2.Hosts = Set from java source 19 | log4j.appender.out2.Selector = RANDOM 20 | log4j.logger.org.apache.flume.clients.log4jappender = DEBUG,out2 -------------------------------------------------------------------------------- /flume-ng-clients/flume-ng-log4jappender/src/test/resources/flume-loadbalancinglog4jtest.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | log4j.appender.out2 = org.apache.flume.clients.log4jappender.LoadBalancingLog4jAppender 18 | #log4j.appender.out2.Hosts = Set from java source 19 | log4j.logger.org.apache.flume.clients.log4jappender = DEBUG,out2 -------------------------------------------------------------------------------- /flume-ng-clients/flume-ng-log4jappender/src/test/resources/flume-log4jtest-avro-generic.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | log4j.appender.out2 = org.apache.flume.clients.log4jappender.Log4jAppender 18 | #log4j.appender.out2.Port = Set from java source 19 | log4j.appender.out2.Hostname = localhost 20 | log4j.appender.out2.AvroSchemaUrl = file:///tmp/myrecord.avsc 21 | log4j.logger.org.apache.flume.clients.log4jappender = DEBUG,out2 -------------------------------------------------------------------------------- /flume-ng-clients/flume-ng-log4jappender/src/test/resources/flume-log4jtest-avro-reflect.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | log4j.appender.out2 = org.apache.flume.clients.log4jappender.Log4jAppender 18 | #log4j.appender.out2.Port = Set from java source 19 | log4j.appender.out2.Hostname = localhost 20 | log4j.appender.out2.AvroReflectionEnabled = true 21 | log4j.logger.org.apache.flume.clients.log4jappender = DEBUG,out2 -------------------------------------------------------------------------------- /flume-ng-clients/flume-ng-log4jappender/src/test/resources/flume-log4jtest.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | log4j.appender.out2 = org.apache.flume.clients.log4jappender.Log4jAppender 18 | #log4j.appender.out2.Port = Set from java source 19 | log4j.appender.out2.Hostname = localhost 20 | log4j.logger.org.apache.flume.clients.log4jappender = DEBUG,out2 -------------------------------------------------------------------------------- /flume-ng-clients/flume-ng-log4jappender/src/test/resources/myrecord.avsc: -------------------------------------------------------------------------------- 1 | {"type":"record","name":"myrecord","fields":[{"name":"message","type":"string"}]} -------------------------------------------------------------------------------- /flume-ng-clients/pom.xml: -------------------------------------------------------------------------------- 1 | 17 | 19 | 4.0.0 20 | 21 | flume-parent 22 | org.apache.flume 23 | 1.11.1-SNAPSHOT 24 | ../flume-parent/pom.xml 25 | 26 | flume-ng-clients 27 | pom 28 | Flume NG Clients 29 | All flume NG clients will come under this module 30 | 31 | flume-ng-log4jappender 32 | 33 | 34 | -------------------------------------------------------------------------------- /flume-ng-configfilters/flume-ng-config-filter-api/pom.xml: -------------------------------------------------------------------------------- 1 | 17 | 20 | 21 | flume-ng-configfilters 22 | org.apache.flume 23 | 1.11.1-SNAPSHOT 24 | 25 | 4.0.0 26 | 27 | 28 | org.apache.flume.configfilter.api 29 | 30 | 31 | flume-ng-config-filter-api 32 | Flume NG Config Filters API 33 | 34 | -------------------------------------------------------------------------------- /flume-ng-configfilters/flume-ng-config-filter-api/src/main/java/org/apache/flume/configfilter/AbstractConfigFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.flume.configfilter; 18 | 19 | /** 20 | * A base implementation of the common methods for Configuration filters 21 | */ 22 | public abstract class AbstractConfigFilter implements ConfigFilter { 23 | 24 | private String name; 25 | 26 | @Override 27 | public void setName(String name) { 28 | this.name = name; 29 | } 30 | 31 | @Override 32 | public String getName() { 33 | return name; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /flume-ng-configfilters/flume-ng-environment-variable-config-filter/src/main/java/org/apache/flume/configfilter/EnvironmentVariableConfigFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.configfilter; 20 | 21 | import java.util.Map; 22 | 23 | public class EnvironmentVariableConfigFilter extends AbstractConfigFilter { 24 | 25 | @Override 26 | public String filter(String key) { 27 | return System.getenv(key); 28 | } 29 | 30 | @Override 31 | public void initializeWithConfiguration(Map configuration) { 32 | //NO-OP 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /flume-ng-configfilters/flume-ng-external-process-config-filter/src/test/resources/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ $1 = "my_password_key" ]; then 4 | echo "filtered"; 5 | fi 6 | 7 | if [ $1 = "my_password_key2" ]; then 8 | echo "filtered2"; 9 | fi 10 | 11 | exit 0 -------------------------------------------------------------------------------- /flume-ng-configfilters/flume-ng-external-process-config-filter/src/test/resources/test_error.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo "Error message from stderr" 1>&2 3 | echo "Error message from stdout" 4 | 5 | return 123 -------------------------------------------------------------------------------- /flume-ng-configuration/src/main/java/org/apache/flume/conf/ComponentWithClassName.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.flume.conf; 18 | 19 | public interface ComponentWithClassName { 20 | String getClassName(); 21 | } 22 | -------------------------------------------------------------------------------- /flume-ng-configuration/src/main/java/org/apache/flume/conf/ConfigurationException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.conf; 20 | 21 | import org.apache.flume.FlumeException; 22 | 23 | public class ConfigurationException extends FlumeException { 24 | 25 | /** 26 | * 27 | */ 28 | private static final long serialVersionUID = 1L; 29 | 30 | public ConfigurationException(String arg0) { 31 | super(arg0); 32 | } 33 | 34 | public ConfigurationException(Throwable arg0) { 35 | super(arg0); 36 | } 37 | 38 | public ConfigurationException(String arg0, Throwable arg1) { 39 | super(arg0, arg1); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /flume-ng-configuration/src/main/java/org/apache/flume/conf/configfilter/ConfigFilterType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.flume.conf.configfilter; 18 | 19 | import org.apache.flume.conf.ComponentWithClassName; 20 | 21 | public enum ConfigFilterType implements ComponentWithClassName { 22 | OTHER(null), 23 | ENV("org.apache.flume.configfilter.EnvironmentVariableConfigFilter"), 24 | HADOOP("org.apache.flume.configfilter.HadoopCredentialStoreConfigFilter"), 25 | EXTERNAL("org.apache.flume.configfilter.ExternalProcessConfigFilter"); 26 | 27 | private final String className; 28 | 29 | ConfigFilterType(String className) { 30 | this.className = className; 31 | } 32 | 33 | @Override 34 | public String getClassName() { 35 | return className; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /flume-ng-configuration/src/test/java/org/apache/flume/conf/channel/MemoryChannelConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.flume.conf.channel; 18 | 19 | /** 20 | * This is a mock to avoid the circular dependency in tests 21 | * TODO fix wrong dependency directions in the project config should not depend on an implementation 22 | */ 23 | public class MemoryChannelConfiguration extends ChannelConfiguration { 24 | public MemoryChannelConfiguration(String componentName) { 25 | super(componentName); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /flume-ng-configuration/src/test/java/org/apache/flume/conf/configfilter/EnvironmentVariableConfigFilterConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.flume.conf.configfilter; 18 | 19 | /** 20 | * This is a mock to avoid the circular dependency in tests 21 | * TODO fix wrong dependency directions in the project config should not depend on an implementation 22 | */ 23 | public class EnvironmentVariableConfigFilterConfiguration extends ConfigFilterConfiguration { 24 | public EnvironmentVariableConfigFilterConfiguration(String componentName) { 25 | super(componentName); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /flume-ng-configuration/src/test/java/org/apache/flume/conf/configfilter/MockConfigFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.flume.conf.configfilter; 18 | 19 | import org.apache.flume.configfilter.AbstractConfigFilter; 20 | 21 | import java.util.Map; 22 | 23 | public class MockConfigFilter extends AbstractConfigFilter { 24 | @Override 25 | public String filter(String key) { 26 | 27 | if (key.equals("null")) { 28 | return null; 29 | } 30 | 31 | if (key.equals("throw")) { 32 | throw new IllegalStateException("Test exception"); 33 | } 34 | 35 | return "filtered_" + key; 36 | } 37 | 38 | @Override 39 | public void initializeWithConfiguration(Map configuration) { 40 | 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /flume-ng-configuration/src/test/java/org/apache/flume/conf/sink/NullSinkConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.flume.conf.sink; 18 | 19 | /** 20 | * This is a mock to avoid the circular dependency in tests 21 | * TODO fix wrong dependency directions in the project config should not depend on an implementation 22 | */ 23 | public class NullSinkConfiguration extends SinkConfiguration { 24 | public NullSinkConfiguration(String componentName) { 25 | super(componentName); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /flume-ng-configuration/src/test/java/org/apache/flume/conf/source/TestSourceConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.flume.conf.source; 18 | 19 | import org.apache.flume.Context; 20 | import org.apache.flume.conf.ConfigurationException; 21 | import org.junit.Test; 22 | 23 | public class TestSourceConfiguration { 24 | 25 | /** 26 | * Test fails without FLUME-1847 27 | */ 28 | @Test(expected = ConfigurationException.class) 29 | public void testFLUME1847() throws Exception { 30 | Context context = new Context(); 31 | context.put("type", "something"); 32 | SourceConfiguration sourceConfig = new SourceConfiguration("src"); 33 | sourceConfig.configure(context); 34 | 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /flume-ng-configuration/src/test/java/org/apache/flume/conf/source/jms/JMSSourceConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.flume.conf.source.jms; 18 | 19 | import org.apache.flume.conf.source.SourceConfiguration; 20 | 21 | /** 22 | * This is a mock to avoid the circular dependency in tests 23 | * TODO fix wrong dependency directions in the project config should not depend on an implementation 24 | */ 25 | public class JMSSourceConfiguration extends SourceConfiguration { 26 | public JMSSourceConfiguration(String componentName) { 27 | super(componentName); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/avro/TransferStateFileMeta.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "org.apache.flume.serialization", 3 | "type": "record", 4 | "name": "TransferStateFileMeta", 5 | "fields": [ 6 | {"name": "offset", "type": "long"} 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/ChannelFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume; 19 | 20 | public interface ChannelFactory { 21 | 22 | Channel create(String name, String type) throws FlumeException; 23 | 24 | Class getClass(String type) throws FlumeException; 25 | } 26 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/ChannelFullException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flume; 20 | 21 | public class ChannelFullException extends ChannelException { 22 | 23 | private static final long serialVersionUID = -8098141359417449525L; 24 | 25 | /** 26 | * @param message the exception message 27 | */ 28 | public ChannelFullException(String message) { 29 | super(message); 30 | } 31 | 32 | /** 33 | * @param ex the causal exception 34 | */ 35 | public ChannelFullException(Throwable ex) { 36 | super(ex); 37 | } 38 | 39 | /** 40 | * @param message the exception message 41 | * @param ex the causal exception 42 | */ 43 | public ChannelFullException(String message, Throwable ex) { 44 | super(message, ex); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/Clock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume; 20 | 21 | /** 22 | * Facade for System.currentTimeMillis for Testing 23 | */ 24 | public interface Clock { 25 | long currentTimeMillis(); 26 | } 27 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/Constants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume; 20 | 21 | public final class Constants { 22 | 23 | /** 24 | * Disables the fail-fast startup behavior. This would be used in the 25 | * scenario where the agent is expected to start, but the config 26 | * file be populated at a later point in time. 27 | */ 28 | public static final String SYSPROP_CALLED_FROM_SERVICE 29 | = "flume.called.from.service"; 30 | 31 | private Constants() { 32 | // disable explicit object creation 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/EventDrivenSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.flume; 21 | 22 | /** 23 | * A {@link Source} that does not need an external driver to poll for 24 | * {@linkplain Event events} to ingest; it provides its own event-driven 25 | * mechanism to invoke event processing. 26 | */ 27 | public interface EventDrivenSource extends Source { 28 | 29 | } 30 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/NamedComponent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume; 20 | 21 | import org.apache.flume.annotations.InterfaceAudience; 22 | import org.apache.flume.annotations.InterfaceStability; 23 | 24 | /** 25 | * Enables a component to be tagged with a name so that it can be referred 26 | * to uniquely within the configuration system. 27 | */ 28 | @InterfaceAudience.Public 29 | @InterfaceStability.Stable 30 | public interface NamedComponent { 31 | 32 | public void setName(String name); 33 | 34 | public String getName(); 35 | 36 | } 37 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/SinkFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.flume; 21 | 22 | public interface SinkFactory { 23 | Sink create(String name, String type) 24 | throws FlumeException; 25 | 26 | Class getClass(String type) 27 | throws FlumeException; 28 | } 29 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/SourceFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.flume; 21 | 22 | public interface SourceFactory { 23 | 24 | Source create(String sourceName, String type) 25 | throws FlumeException; 26 | 27 | Class getClass(String type) 28 | throws FlumeException; 29 | } 30 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/SystemClock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume; 20 | 21 | /** 22 | * Default implementation of Clock which uses System 23 | */ 24 | public class SystemClock implements Clock { 25 | 26 | public long currentTimeMillis() { 27 | return System.currentTimeMillis(); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/annotations/Disposable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.flume.annotations; 21 | 22 | import java.lang.annotation.Target; 23 | import java.lang.annotation.Retention; 24 | 25 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 26 | import static java.lang.annotation.ElementType.TYPE; 27 | 28 | @Target({ TYPE }) @Retention(RUNTIME) 29 | public @interface Disposable {} 30 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/annotations/Recyclable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.flume.annotations; 21 | 22 | import java.lang.annotation.Target; 23 | import java.lang.annotation.Retention; 24 | 25 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 26 | import static java.lang.annotation.ElementType.TYPE; 27 | 28 | @Target({ TYPE }) @Retention(RUNTIME) 29 | public @interface Recyclable {} 30 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/conf/BatchSizeSupported.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume.conf; 19 | 20 | /** 21 | * This interface indicates that a component does batching and the batch size 22 | * is publicly available. 23 | * 24 | */ 25 | public interface BatchSizeSupported { 26 | 27 | /** 28 | * Returns the batch size 29 | */ 30 | long getBatchSize(); 31 | 32 | } 33 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/conf/ConfigurableComponent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | 18 | package org.apache.flume.conf; 19 | 20 | public interface ConfigurableComponent { 21 | 22 | public void configure(ComponentConfiguration conf); 23 | 24 | } 25 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/conf/TransactionCapacitySupported.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume.conf; 19 | 20 | /** 21 | * This interface indicates that a component has a transaction capacity 22 | * and it is publicly available. 23 | * 24 | */ 25 | public interface TransactionCapacitySupported { 26 | 27 | /** 28 | * Returns the transaction capacity 29 | */ 30 | long getTransactionCapacity(); 31 | 32 | } 33 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/formatter/output/EventFormatter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.flume.formatter.output; 21 | 22 | import org.apache.flume.Event; 23 | 24 | @Deprecated 25 | public interface EventFormatter { 26 | 27 | public byte[] format(Event event); 28 | 29 | } 30 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/formatter/output/TextDelimitedOutputFormatter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.flume.formatter.output; 21 | 22 | import org.apache.flume.Event; 23 | 24 | @Deprecated 25 | public class TextDelimitedOutputFormatter implements EventFormatter { 26 | 27 | @Override 28 | public byte[] format(Event event) { 29 | String body = event.getBody().length > 0 ? new String(event.getBody()) : ""; 30 | 31 | return (body + "\n").getBytes(); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/instrumentation/ChannelProcessorCounter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.instrumentation; 20 | 21 | public class ChannelProcessorCounter extends MonitoredCounterGroup { 22 | 23 | protected ChannelProcessorCounter(String name) { 24 | super(MonitoredCounterGroup.Type.CHANNEL_PROCESSOR, name); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/instrumentation/MonitorService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.instrumentation; 20 | 21 | import org.apache.flume.conf.Configurable; 22 | 23 | /** 24 | * Interface that any monitoring service should implement. If the monitor 25 | * service is to be started up when Flume starts, it should implement this 26 | * and the class name should be passed in during startup, with any additional 27 | * context it requires. 28 | */ 29 | public interface MonitorService extends Configurable { 30 | 31 | public void start(); 32 | 33 | public void stop(); 34 | 35 | } 36 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/instrumentation/MonitoringType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.flume.instrumentation; 21 | 22 | /** 23 | * Enum for Monitoring types. 24 | */ 25 | public enum MonitoringType { 26 | OTHER(null), 27 | GANGLIA(org.apache.flume.instrumentation.GangliaServer.class), 28 | HTTP(org.apache.flume.instrumentation.http.HTTPMetricsServer.class), 29 | PROMETHEUS(org.apache.flume.instrumentation.http.PrometheusHTTPMetricsServer.class); 30 | 31 | private Class monitoringClass; 32 | 33 | private MonitoringType(Class klass) { 34 | this.monitoringClass = klass; 35 | } 36 | 37 | public Class getMonitorClass() { 38 | return this.monitoringClass; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/instrumentation/SinkProcessorCounter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.instrumentation; 20 | 21 | public class SinkProcessorCounter extends MonitoredCounterGroup { 22 | 23 | protected SinkProcessorCounter(String name) { 24 | super(MonitoredCounterGroup.Type.SINK_PROCESSOR, name); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/instrumentation/kafka/KafkaChannelCounterMBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume.instrumentation.kafka; 19 | 20 | public interface KafkaChannelCounterMBean { 21 | 22 | long getKafkaEventGetTimer(); 23 | 24 | long getKafkaEventSendTimer(); 25 | 26 | long getKafkaCommitTimer(); 27 | 28 | long getRollbackCount(); 29 | 30 | long getChannelSize(); 31 | 32 | long getEventPutAttemptCount(); 33 | 34 | long getEventTakeAttemptCount(); 35 | 36 | long getEventPutSuccessCount(); 37 | 38 | long getEventTakeSuccessCount(); 39 | 40 | long getStartTime(); 41 | 42 | long getStopTime(); 43 | 44 | long getChannelCapacity(); 45 | 46 | String getType(); 47 | 48 | double getChannelFillPercentage(); 49 | 50 | } 51 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/instrumentation/kafka/KafkaSinkCounterMBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume.instrumentation.kafka; 19 | 20 | public interface KafkaSinkCounterMBean { 21 | 22 | long getKafkaEventSendTimer(); 23 | 24 | long getRollbackCount(); 25 | 26 | long getConnectionCreatedCount(); 27 | 28 | long getConnectionClosedCount(); 29 | 30 | long getConnectionFailedCount(); 31 | 32 | long getBatchEmptyCount(); 33 | 34 | long getBatchUnderflowCount(); 35 | 36 | long getBatchCompleteCount(); 37 | 38 | long getEventDrainAttemptCount(); 39 | 40 | long getEventDrainSuccessCount(); 41 | 42 | long getStartTime(); 43 | 44 | long getStopTime(); 45 | 46 | String getType(); 47 | } 48 | 49 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/instrumentation/kafka/KafkaSourceCounterMBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume.instrumentation.kafka; 19 | 20 | public interface KafkaSourceCounterMBean { 21 | 22 | long getKafkaEventGetTimer(); 23 | 24 | long getKafkaCommitTimer(); 25 | 26 | long getKafkaEmptyCount(); 27 | 28 | long getEventReceivedCount(); 29 | 30 | long getEventAcceptedCount(); 31 | 32 | long getAppendReceivedCount(); 33 | 34 | long getAppendAcceptedCount(); 35 | 36 | long getAppendBatchReceivedCount(); 37 | 38 | long getAppendBatchAcceptedCount(); 39 | 40 | long getStartTime(); 41 | 42 | long getStopTime(); 43 | 44 | String getType(); 45 | 46 | long getOpenConnectionCount(); 47 | } 48 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/interceptor/RegexExtractorInterceptorPassThroughSerializer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume.interceptor; 19 | 20 | import org.apache.flume.Context; 21 | import org.apache.flume.conf.ComponentConfiguration; 22 | 23 | /** 24 | * Serializer that simply returns the passed in value 25 | */ 26 | public class RegexExtractorInterceptorPassThroughSerializer implements 27 | RegexExtractorInterceptorSerializer { 28 | 29 | @Override 30 | public String serialize(String value) { 31 | return value; 32 | } 33 | 34 | @Override 35 | public void configure(Context context) { 36 | // NO-OP... 37 | } 38 | 39 | @Override 40 | public void configure(ComponentConfiguration conf) { 41 | // NO-OP... 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/interceptor/RegexExtractorInterceptorSerializer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume.interceptor; 19 | 20 | import org.apache.flume.conf.Configurable; 21 | import org.apache.flume.conf.ConfigurableComponent; 22 | 23 | /** 24 | * Serializer for serializing groups matched by the 25 | * {@link RegexExtractorInterceptor} 26 | */ 27 | public interface RegexExtractorInterceptorSerializer extends Configurable, 28 | ConfigurableComponent { 29 | 30 | /** 31 | * @param value 32 | * The value extracted by the {@link RegexExtractorInterceptor} 33 | * @return The serialized version of the specified value 34 | */ 35 | String serialize(String value); 36 | 37 | } 38 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/lifecycle/LifecycleException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.flume.lifecycle; 21 | 22 | public class LifecycleException extends Exception { 23 | 24 | private static final long serialVersionUID = 4689000562519155240L; 25 | 26 | public LifecycleException() { 27 | super(); 28 | } 29 | 30 | public LifecycleException(String message) { 31 | super(message); 32 | } 33 | 34 | public LifecycleException(String message, Throwable t) { 35 | super(message, t); 36 | } 37 | 38 | public LifecycleException(Throwable t) { 39 | super(t); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/lifecycle/LifecycleState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.flume.lifecycle; 21 | 22 | public enum LifecycleState { 23 | IDLE, START, STOP, ERROR; 24 | 25 | public static final LifecycleState[] START_OR_ERROR = new LifecycleState[] { 26 | START, ERROR }; 27 | public static final LifecycleState[] STOP_OR_ERROR = new LifecycleState[] { 28 | STOP, ERROR }; 29 | 30 | } 31 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/serialization/DecodeErrorPolicy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.flume.serialization; 21 | 22 | import org.apache.flume.annotations.InterfaceAudience; 23 | import org.apache.flume.annotations.InterfaceStability; 24 | 25 | @InterfaceAudience.Private 26 | @InterfaceStability.Evolving 27 | public enum DecodeErrorPolicy { 28 | FAIL, 29 | REPLACE, 30 | IGNORE 31 | } 32 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/serialization/LengthMeasurable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume.serialization; 19 | 20 | import java.io.IOException; 21 | 22 | public interface LengthMeasurable { 23 | 24 | /** returns the total length of the stream or file */ 25 | long length() throws IOException; 26 | 27 | } 28 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/serialization/PositionTracker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume.serialization; 19 | 20 | import org.apache.flume.annotations.InterfaceAudience; 21 | import org.apache.flume.annotations.InterfaceStability; 22 | 23 | import java.io.Closeable; 24 | import java.io.IOException; 25 | 26 | /** 27 | * Defines an interface for tracking the offset position in a target file. 28 | */ 29 | @InterfaceAudience.Private 30 | @InterfaceStability.Evolving 31 | public interface PositionTracker extends Closeable { 32 | void storePosition(long position) throws IOException; 33 | long getPosition(); 34 | String getTarget(); 35 | void close() throws IOException; 36 | } 37 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/serialization/RemoteMarkable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume.serialization; 19 | 20 | import java.io.IOException; 21 | 22 | /** Allows for calling mark() without a seek() */ 23 | public interface RemoteMarkable { 24 | 25 | /** 26 | * Indicate that the specified position should be returned to in the case of 27 | * {@link Resettable#reset()} being called. 28 | * @throws java.io.IOException 29 | */ 30 | void markPosition(long position) throws IOException; 31 | 32 | /** 33 | * Return the saved mark position without moving the mark pointer. 34 | * @throws IOException 35 | */ 36 | long getMarkPosition() throws IOException; 37 | 38 | } 39 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/serialization/Seekable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume.serialization; 19 | 20 | import java.io.IOException; 21 | 22 | public interface Seekable { 23 | void seek(long position) throws IOException; 24 | long tell() throws IOException; 25 | } 26 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/source/AbstractEventDrivenSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.source; 20 | 21 | import org.apache.flume.EventDrivenSource; 22 | import org.apache.flume.annotations.InterfaceAudience; 23 | import org.apache.flume.annotations.InterfaceStability; 24 | 25 | /** 26 | * Base class which ensures sub-classes will inherit all the properties 27 | * of BasicSourceSemantics. Adds no additional functionality and is provided 28 | * for completeness sake. 29 | */ 30 | @InterfaceAudience.Public 31 | @InterfaceStability.Evolving 32 | public abstract class AbstractEventDrivenSource 33 | extends BasicSourceSemantics 34 | implements EventDrivenSource { 35 | public AbstractEventDrivenSource() { 36 | super(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/source/PollableSourceConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.flume.source; 21 | 22 | public class PollableSourceConstants { 23 | 24 | public static final String BACKOFF_SLEEP_INCREMENT = "backoffSleepIncrement"; 25 | public static final String MAX_BACKOFF_SLEEP = "maxBackoffSleep"; 26 | public static final long DEFAULT_BACKOFF_SLEEP_INCREMENT = 1000; 27 | public static final long DEFAULT_MAX_BACKOFF_SLEEP = 5000; 28 | } -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/tools/GetJavaProperty.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume.tools; 19 | 20 | /** 21 | * A generic way for querying Java properties. 22 | */ 23 | public class GetJavaProperty { 24 | public static void main(String[] args) { 25 | if (args.length == 0) { 26 | for (Object prop : System.getProperties().keySet()) { 27 | System.out.println(prop + "=" + System.getProperty((String)prop, "")); 28 | } 29 | } else { 30 | for (String prop : args) { 31 | System.out.println(prop + "=" + System.getProperty(prop, "")); 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /flume-ng-core/src/main/java/org/apache/flume/tools/PlatformDetect.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume.tools; 19 | 20 | import java.util.Locale; 21 | 22 | /** 23 | * Utilities for platform & operating system detection 24 | */ 25 | public class PlatformDetect { 26 | 27 | /** 28 | * Detects whether we are running under Microsoft Windows. 29 | * @return true if and only if we are running on a Windows system. 30 | */ 31 | public static boolean isWindows() { 32 | String os = System.getProperty("os.name"); 33 | boolean isWin = (os.toLowerCase(Locale.ENGLISH).indexOf("win") >= 0); 34 | return isWin; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /flume-ng-core/src/test/java/org/apache/flume/interceptor/TestRegexExtractorInterceptorPassThroughSerializer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume.interceptor; 19 | 20 | import junit.framework.Assert; 21 | 22 | import org.apache.flume.Context; 23 | import org.junit.Test; 24 | 25 | public class TestRegexExtractorInterceptorPassThroughSerializer { 26 | 27 | @Test 28 | public void shouldReturnSameValue() { 29 | RegexExtractorInterceptorPassThroughSerializer fixture = 30 | new RegexExtractorInterceptorPassThroughSerializer(); 31 | fixture.configure(new Context()); 32 | String input = "testing (1,2,3,4)"; 33 | Assert.assertEquals(input, fixture.serialize(input)); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /flume-ng-core/src/test/resources/TestResettableFileInputStream_1.avro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-core/src/test/resources/TestResettableFileInputStream_1.avro -------------------------------------------------------------------------------- /flume-ng-core/src/test/resources/TestResettableFileInputStream_1.truncated.avro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-core/src/test/resources/TestResettableFileInputStream_1.truncated.avro -------------------------------------------------------------------------------- /flume-ng-core/src/test/resources/certs/rootca.conf: -------------------------------------------------------------------------------- 1 | [ req ] 2 | distinguished_name = CA_DN 3 | prompt = no 4 | output_password = password 5 | default_bits = 2048 6 | 7 | [ CA_DN ] 8 | C = US 9 | CN = flume-ca 10 | -------------------------------------------------------------------------------- /flume-ng-core/src/test/resources/certs/server.conf: -------------------------------------------------------------------------------- 1 | [ req ] 2 | distinguished_name = CA_DN 3 | prompt = no 4 | output_password = password 5 | default_bits = 2048 6 | 7 | [ CA_DN ] 8 | C = US 9 | CN = server.flume 10 | -------------------------------------------------------------------------------- /flume-ng-core/src/test/resources/jettykeystore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-core/src/test/resources/jettykeystore -------------------------------------------------------------------------------- /flume-ng-core/src/test/resources/keystorefile.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-core/src/test/resources/keystorefile.jks -------------------------------------------------------------------------------- /flume-ng-core/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /flume-ng-core/src/test/resources/server.flume-crt.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICwjCCAaoCCQD8AQyla5FbDjANBgkqhkiG9w0BAQsFADAgMQswCQYDVQQGEwJV 3 | UzERMA8GA1UEAwwIZmx1bWUtY2EwIBcNMjMwMzIyMDAyODM0WhgPMjA1MzAzMjQw 4 | MDI4MzRaMCQxCzAJBgNVBAYTAlVTMRUwEwYDVQQDEwxzZXJ2ZXIuZmx1bWUwggEi 5 | MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEEAUShlX2lWDYiEgcZL1JzWFw 6 | auNkdxOGo1DtD3YaZdrU1GS2UwspLm0qcSFEF1Sx6uVTABdgjZGnxJLSTJkabVu/ 7 | iMuP+EQrE/4AWJoyuuaYMSG0EGeP+mttTtLHYyt/k9NJkABYxFZkNqdDuo+lF/Vs 8 | QdW3icym0vqMwiIvE+VKw9j3F+zFZizfx6MiBH0uuNrXHFCNUg52/cbeyiXO1mks 9 | yruOV+PF8/44zAepjLWiJgp7Wo6ejXmLvR+k68RwB5V7fXrzPYueM9GQDmLffkdO 10 | ZhrcafiRFGlzWSmC820Eb2b5+cVnm96XAlUXE5ao5o58oMmcufMnJ5k2UbFJAgMB 11 | AAEwDQYJKoZIhvcNAQELBQADggEBAHBGpqraT39aw/HVrJdmpsw8CwSmdiir+NYk 12 | 5PprbIKAyf/P/9ObKcqesO8d8CQZVvzzm+Ok2rgcALDIl/TbbAVUPizIrN4AiH+Z 13 | BPOqDFF4taWkw73iMDiq61QS8SpJIOxxmL8PsK5eefuABrpumnVgW5X9BT/uMIqW 14 | NOwiyII3NVvtlErcdAL/ZTYWc3S8CEWVRc88ZpIBSLB4/tqQbPM+m+ZtYMSKi3Sh 15 | ugOjonPuteeQqu6R7HRYOajepKdGe048Moq90v0IrDI8v+rbezLFpEWOnG0fUDEq 16 | LfA9l7e/q1ukXRW4ccJWZWXLrZbEbX5hJeTlyYhHciwB5jfueMM= 17 | -----END CERTIFICATE----- 18 | -------------------------------------------------------------------------------- /flume-ng-core/src/test/resources/server.flume-keystore.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-core/src/test/resources/server.flume-keystore.p12 -------------------------------------------------------------------------------- /flume-ng-core/src/test/resources/server.flume.pem: -------------------------------------------------------------------------------- 1 | Bag Attributes 2 | friendlyName: server 3 | localKeyID: 54 69 6D 65 20 31 36 37 39 34 34 34 39 31 33 33 33 39 4 | subject=/C=US/CN=server.flume 5 | issuer=/C=US/CN=server.flume 6 | -----BEGIN CERTIFICATE----- 7 | MIIC6TCCAdGgAwIBAgIEMlDgqzANBgkqhkiG9w0BAQsFADAkMQswCQYDVQQGEwJV 8 | UzEVMBMGA1UEAxMMc2VydmVyLmZsdW1lMCAXDTIzMDMyMjAwMjgzM1oYDzIwNTMw 9 | MzI0MDAyODMzWjAkMQswCQYDVQQGEwJVUzEVMBMGA1UEAxMMc2VydmVyLmZsdW1l 10 | MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxBAFEoZV9pVg2IhIHGS9 11 | Sc1hcGrjZHcThqNQ7Q92GmXa1NRktlMLKS5tKnEhRBdUserlUwAXYI2Rp8SS0kyZ 12 | Gm1bv4jLj/hEKxP+AFiaMrrmmDEhtBBnj/prbU7Sx2Mrf5PTSZAAWMRWZDanQ7qP 13 | pRf1bEHVt4nMptL6jMIiLxPlSsPY9xfsxWYs38ejIgR9Lrja1xxQjVIOdv3G3sol 14 | ztZpLMq7jlfjxfP+OMwHqYy1oiYKe1qOno15i70fpOvEcAeVe3168z2LnjPRkA5i 15 | 335HTmYa3Gn4kRRpc1kpgvNtBG9m+fnFZ5velwJVFxOWqOaOfKDJnLnzJyeZNlGx 16 | SQIDAQABoyEwHzAdBgNVHQ4EFgQUClD6FZ+qPIFrtBaz0swRTtWt1WEwDQYJKoZI 17 | hvcNAQELBQADggEBALuX9E+tRWvvA9uULj9Iq+k9iUNMQzkmyzXGu7hY46ZU9lx+ 18 | fNnLZq82zz9rHq8IhK4HsLIsPCLfNeXwG/TNR4zUHKI53lzkburxgu76soMUDbHX 19 | 8udyUgrs0YjQcppw6IOOmlZtgNeF2nu7jeoXrCaA07yXzehAqukHv7glWaV3oORc 20 | rDkZvHfJ2G7hPbUYYIeouJsbG9rNukNPOY9JEYGFYzDxZ8hlFt7Lp/icbdpjFGDV 21 | tkMtPpVz59B47j/Kk/k5zxaDLnD42svL8GByyM5UxvAqAlYnfMKiZqXfY0JbCpMC 22 | e9Z5xOyt9F8NLFyjRsmBlJD61LuLb8hAZK/Ho70= 23 | -----END CERTIFICATE----- 24 | -------------------------------------------------------------------------------- /flume-ng-core/src/test/resources/server.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-core/src/test/resources/server.p12 -------------------------------------------------------------------------------- /flume-ng-core/src/test/resources/syslog_event.avsc: -------------------------------------------------------------------------------- 1 | { "type": "record", "name": "SyslogEvent", "namespace": "org.apache.flume.test", 2 | "fields": [ 3 | { "name": "facility", "type": "int" }, 4 | { "name": "severity", "type": "int" }, 5 | { "name": "timestamp", "type": "long" }, 6 | { "name": "hostname", "type": "string" }, 7 | { "name": "message", "type": "string" } 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /flume-ng-core/src/test/resources/test_command.ps1: -------------------------------------------------------------------------------- 1 | " \"'1'\", \"\"\"2\"\"\",\"`\3\", \"\4\" " 2 | foreach ($i in 1..5) { $i } 3 | 1..5 | %{$_ +1 } -------------------------------------------------------------------------------- /flume-ng-core/src/test/resources/test_command.txt: -------------------------------------------------------------------------------- 1 | echo "'1'"; echo "\"2\""; echo "\\3"; echo "\4" 2 | for i in {1..5}; do echo $i; done 3 | for i in `seq 5`; do echo $i; done | awk ' { print $1 + 1 } ' 4 | -------------------------------------------------------------------------------- /flume-ng-core/src/test/resources/truststore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-core/src/test/resources/truststore.jks -------------------------------------------------------------------------------- /flume-ng-core/src/test/resources/truststorefile.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-core/src/test/resources/truststorefile.jks -------------------------------------------------------------------------------- /flume-ng-embedded-agent/src/main/java/org/apache/flume/agent/embedded/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | /** 20 | * This package provides Flume users the ability to embed simple agents 21 | * in applications. For specific and up to date information, please see 22 | * the Flume Developer Guide. 23 | */ 24 | package org.apache.flume.agent.embedded; 25 | -------------------------------------------------------------------------------- /flume-ng-embedded-agent/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /flume-ng-node/src/main/java/org/apache/flume/node/ConfigurationProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.flume.node; 21 | 22 | public interface ConfigurationProvider { 23 | MaterializedConfiguration getConfiguration(); 24 | } 25 | -------------------------------------------------------------------------------- /flume-ng-node/src/main/java/org/apache/flume/node/Initializable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache license, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the license for the specific language governing permissions and 15 | * limitations under the license. 16 | */ 17 | package org.apache.flume.node; 18 | 19 | /** 20 | * An interface implmmented by components that need access after all components have been created but before 21 | * any have been started. 22 | */ 23 | public interface Initializable { 24 | 25 | /** 26 | * Called to initialize the component. 27 | * @param configuration the materialized configuration. 28 | */ 29 | void initialize(MaterializedConfiguration configuration); 30 | } 31 | -------------------------------------------------------------------------------- /flume-ng-node/src/main/java/org/apache/flume/node/PollingPropertiesFileConfigurationProvider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume.node; 19 | 20 | import java.io.File; 21 | 22 | import com.google.common.collect.Lists; 23 | import com.google.common.eventbus.EventBus; 24 | 25 | /** 26 | * @deprecated Use UriConfigurationProvider instead. 27 | */ 28 | @Deprecated 29 | public class PollingPropertiesFileConfigurationProvider extends UriConfigurationProvider { 30 | 31 | public PollingPropertiesFileConfigurationProvider(String agentName, File file, EventBus eventBus, 32 | int interval) { 33 | super(agentName, Lists.newArrayList(new FileConfigurationSource(file.toURI())), null, 34 | eventBus, interval); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /flume-ng-node/src/main/java/org/apache/flume/node/net/AuthorizationProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache license, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the license for the specific language governing permissions and 15 | * limitations under the license. 16 | */ 17 | package org.apache.flume.node.net; 18 | 19 | import java.net.URLConnection; 20 | 21 | /** 22 | * Interface to be implemented to add an Authorization header to an HTTP request. 23 | */ 24 | public interface AuthorizationProvider { 25 | 26 | void addAuthorization(URLConnection urlConnection); 27 | } 28 | -------------------------------------------------------------------------------- /flume-ng-node/src/main/java/org/apache/flume/node/net/LaxHostnameVerifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache license, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the license for the specific language governing permissions and 15 | * limitations under the license. 16 | */ 17 | package org.apache.flume.node.net; 18 | 19 | import javax.net.ssl.HostnameVerifier; 20 | import javax.net.ssl.SSLSession; 21 | 22 | /** 23 | * An HostnameVerifier which accepts everything. 24 | */ 25 | public final class LaxHostnameVerifier implements HostnameVerifier { 26 | /** 27 | * Singleton instance. 28 | */ 29 | public static final HostnameVerifier INSTANCE = new LaxHostnameVerifier(); 30 | 31 | private LaxHostnameVerifier() { 32 | } 33 | 34 | @Override 35 | public boolean verify(final String s, final SSLSession sslSession) { 36 | return true; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /flume-ng-node/src/main/resources/META-INF/services/org.apache.flume.node.ConfigurationSourceFactory: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache license, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the license for the specific language governing permissions and 14 | # limitations under the license. 15 | org.apache.flume.node.ClasspathConfigurationSourceFactory 16 | org.apache.flume.node.FileConfigurationSourceFactory 17 | org.apache.flume.node.HttpConfigurationSourceFactory -------------------------------------------------------------------------------- /flume-ng-node/src/test/java/org/apache/flume/node/lookup/TestLookup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache license, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the license for the specific language governing permissions and 15 | * limitations under the license. 16 | */ 17 | package org.apache.flume.node.lookup; 18 | 19 | import org.apache.commons.text.lookup.StringLookup; 20 | 21 | /** 22 | * Test Lookup. 23 | */ 24 | public class TestLookup implements StringLookup { 25 | @Override 26 | public String lookup(String key) { 27 | return key; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /flume-ng-node/src/test/java/org/apache/flume/source/EventProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.source; 20 | 21 | import org.apache.flume.Event; 22 | 23 | /** 24 | * Interface indicating processEvent is implemented. 25 | */ 26 | public interface EventProcessor { 27 | /** 28 | * When implemented causes the event to be handled by the component. 29 | * @param event The Flume event. 30 | */ 31 | default void processEvent(Event event) { 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /flume-ng-node/src/test/resources/flume-conf-override.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | PROD_BIND=192.168.12.110 20 | DEV_BIND=192.168.13.101 21 | 22 | -------------------------------------------------------------------------------- /flume-ng-node/src/test/resources/flume-conf-with-envLookup.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | a1.sources = r1 20 | a1.sources.r1.type = netcat 21 | a1.sources.r1.bind = 0.0.0.0 22 | a1.sources.r1.port = ${env:NC_PORT} 23 | a1.sources.r1.channels = c1 24 | 25 | a1.channels = c1 26 | a1.channels.c1.type = memory 27 | a1.channels.c1.capacity = 10000 28 | a1.channels.c1.transactionCapacity = 10000 29 | a1.channels.c1.byteCapacityBufferPercentage = 20 30 | a1.channels.c1.byteCapacity = 800000 31 | 32 | a1.channels = c1 33 | a1.sinks = k1 34 | a1.sinks.k1.type = logger 35 | a1.sinks.k1.channel = c1 36 | -------------------------------------------------------------------------------- /flume-ng-node/src/test/resources/flume-conf-with-envvars.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | a1.sources = r1 20 | a1.sources.r1.type = netcat 21 | a1.sources.r1.bind = 0.0.0.0 22 | a1.sources.r1.port = ${NC_PORT} 23 | a1.sources.r1.channels = c1 24 | 25 | a1.channels = c1 26 | a1.channels.c1.type = memory 27 | a1.channels.c1.capacity = 10000 28 | a1.channels.c1.transactionCapacity = 10000 29 | a1.channels.c1.byteCapacityBufferPercentage = 20 30 | a1.channels.c1.byteCapacity = 800000 31 | 32 | a1.channels = c1 33 | a1.sinks = k1 34 | a1.sinks.k1.type = logger 35 | a1.sinks.k1.channel = c1 36 | -------------------------------------------------------------------------------- /flume-ng-node/src/test/resources/flume-conf-with-recursiveLookup.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | PROD_BIND=192.168.10.110 20 | DEV_BIND=192.168.11.101 21 | a1.sources = r1 22 | a1.sources.r1.type = netcat 23 | a1.sources.r1.bind = ${${sys:env}_BIND} 24 | a1.sources.r1.port = 6667 25 | a1.sources.r1.channels = c1 26 | 27 | a1.channels = c1 28 | a1.channels.c1.type = memory 29 | a1.channels.c1.capacity = 10000 30 | a1.channels.c1.transactionCapacity = 10000 31 | a1.channels.c1.byteCapacityBufferPercentage = 20 32 | a1.channels.c1.byteCapacity = 800000 33 | 34 | a1.channels = c1 35 | a1.sinks = k1 36 | a1.sinks.k1.type = logger 37 | a1.sinks.k1.channel = c1 38 | -------------------------------------------------------------------------------- /flume-ng-node/src/test/resources/flume-conf.properties.2786: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | # 20 | # Flume Configuration for testing FLUME-2786 21 | # 22 | 23 | test.sources = source1 24 | test.channels = channel1 25 | test.sinks = sink1 26 | 27 | test.sources.source1.type = seq 28 | test.sources.source1.totalEvents = 10000 29 | test.sources.source1.channels = channel1 30 | 31 | test.channels.channel1.type = memory 32 | test.channels.channel1.capacity = 10000 33 | 34 | test.sinks.sink1.type = null 35 | test.sinks.sink1.channel = channel1 36 | -------------------------------------------------------------------------------- /flume-ng-node/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /flume-ng-node/src/test/resources/map-resolver.properties: -------------------------------------------------------------------------------- 1 | name = ${sys:name} 2 | const = ${const:org.apache.flume.node.TestMapResolver.TEST_CONST} 3 | version = ${java:version} 4 | test = ${test:Value} -------------------------------------------------------------------------------- /flume-ng-node/src/test/resources/test-lookups.properties: -------------------------------------------------------------------------------- 1 | sys = org.apache.commons.text.lookup.DefaultStringLookup.SYSTEM_PROPERTIES 2 | env = org.apache.commons.text.lookup.DefaultStringLookup.ENVIRONMENT 3 | java = org.apache.commons.text.lookup.DefaultStringLookup.JAVA 4 | test = org.apache.flume.node.lookup.TestLookup -------------------------------------------------------------------------------- /flume-ng-sdk/src/main/avro/flume.avdl: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | @namespace("org.apache.flume.source.avro") 21 | 22 | protocol AvroSourceProtocol { 23 | 24 | enum Status { 25 | OK, FAILED, UNKNOWN 26 | } 27 | 28 | record AvroFlumeEvent { 29 | map headers; 30 | bytes body; 31 | } 32 | 33 | Status append( AvroFlumeEvent event ); 34 | 35 | Status appendBatch( array events ); 36 | 37 | } 38 | -------------------------------------------------------------------------------- /flume-ng-sdk/src/main/java/org/apache/flume/EventDeliveryException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.flume; 21 | 22 | /** 23 | * An event delivery exception is raised whenever an {@link Event} fails to 24 | * reach at least one of its intended (next-hop) destinations. 25 | */ 26 | public class EventDeliveryException extends Exception { 27 | 28 | private static final long serialVersionUID = 1102327497549834945L; 29 | 30 | public EventDeliveryException() { 31 | super(); 32 | } 33 | 34 | public EventDeliveryException(String message) { 35 | super(message); 36 | } 37 | 38 | public EventDeliveryException(String message, Throwable t) { 39 | super(message, t); 40 | } 41 | 42 | public EventDeliveryException(Throwable t) { 43 | super(t); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /flume-ng-sdk/src/main/java/org/apache/flume/FlumeException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume; 19 | 20 | /** 21 | * Base class of all flume exceptions. 22 | */ 23 | public class FlumeException extends RuntimeException { 24 | 25 | private static final long serialVersionUID = 1L; 26 | 27 | public FlumeException(String msg) { 28 | super(msg); 29 | } 30 | 31 | public FlumeException(String msg, Throwable th) { 32 | super(msg, th); 33 | } 34 | 35 | public FlumeException(Throwable th) { 36 | super(th); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /flume-ng-sdk/src/main/thrift/aslv2: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | -------------------------------------------------------------------------------- /flume-ng-sdk/src/main/thrift/flume.thrift: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | namespace java org.apache.flume.thrift 21 | 22 | struct ThriftFlumeEvent { 23 | 1: required map headers, 24 | 2: required binary body, 25 | } 26 | 27 | enum Status { 28 | OK, 29 | FAILED, 30 | ERROR, 31 | UNKNOWN 32 | } 33 | 34 | service ThriftSourceProtocol { 35 | Status append(1: ThriftFlumeEvent event), 36 | Status appendBatch(1: list events), 37 | } 38 | 39 | -------------------------------------------------------------------------------- /flume-ng-sdk/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-http-sink/src/main/java/org/apache/flume/sink/http/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | /** 20 | * This package provides an HTTP sink for Flume so that events can be sent out 21 | * to a target HTTP endpoint. 22 | */ 23 | package org.apache.flume.sink.http; 24 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-http-sink/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/README.md: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Flume Morphline Solr Sink 19 | 20 | This module contains a Flume Morphline Solr Sink that extracts search documents from Flume events, transforms them and loads them in Near Real Time into Apache Solr, typically a SolrCloud. This sink is intended to be used alongside the HdfsSink. It is designed to process not just structured data, but also arbitrary raw data, including data from many heterogeneous data sources. 21 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/java/org/apache/flume/sink/solr/morphline/TestEnvironment.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.flume.sink.solr.morphline; 18 | 19 | import java.net.UnknownHostException; 20 | 21 | import org.junit.Test; 22 | 23 | import org.kitesdk.morphline.solr.EnvironmentTest; 24 | 25 | /** Print and verify some info about the environment in which the unit tests are running */ 26 | public class TestEnvironment extends EnvironmentTest { 27 | 28 | @Test 29 | public void testEnvironment() throws UnknownHostException { 30 | super.testEnvironment(); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/solr/collection1/conf/elevate.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/solr/collection1/conf/lang/contractions_ca.txt: -------------------------------------------------------------------------------- 1 | # Set of Catalan contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | d 4 | l 5 | m 6 | n 7 | s 8 | t 9 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/solr/collection1/conf/lang/contractions_fr.txt: -------------------------------------------------------------------------------- 1 | # Set of French contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | l 4 | m 5 | t 6 | qu 7 | n 8 | s 9 | j 10 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/solr/collection1/conf/lang/contractions_ga.txt: -------------------------------------------------------------------------------- 1 | # Set of Irish contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | d 4 | m 5 | b 6 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/solr/collection1/conf/lang/contractions_it.txt: -------------------------------------------------------------------------------- 1 | # Set of Italian contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | c 4 | l 5 | all 6 | dall 7 | dell 8 | nell 9 | sull 10 | coll 11 | pell 12 | gl 13 | agl 14 | dagl 15 | degl 16 | negl 17 | sugl 18 | un 19 | m 20 | t 21 | s 22 | v 23 | d 24 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/solr/collection1/conf/lang/hyphenations_ga.txt: -------------------------------------------------------------------------------- 1 | # Set of Irish hyphenations for StopFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | h 4 | n 5 | t 6 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/solr/collection1/conf/lang/stemdict_nl.txt: -------------------------------------------------------------------------------- 1 | # Set of overrides for the dutch stemmer 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | fiets fiets 4 | bromfiets bromfiets 5 | ei eier 6 | kind kinder 7 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/solr/collection1/conf/lang/stopwords_el.txt: -------------------------------------------------------------------------------- 1 | # Lucene Greek Stopwords list 2 | # Note: by default this file is used after GreekLowerCaseFilter, 3 | # so when modifying this file use 'σ' instead of 'ς' 4 | ο 5 | η 6 | το 7 | οι 8 | τα 9 | του 10 | τησ 11 | των 12 | τον 13 | την 14 | και 15 | κι 16 | κ 17 | ειμαι 18 | εισαι 19 | ειναι 20 | ειμαστε 21 | ειστε 22 | στο 23 | στον 24 | στη 25 | στην 26 | μα 27 | αλλα 28 | απο 29 | για 30 | προσ 31 | με 32 | σε 33 | ωσ 34 | παρα 35 | αντι 36 | κατα 37 | μετα 38 | θα 39 | να 40 | δε 41 | δεν 42 | μη 43 | μην 44 | επι 45 | ενω 46 | εαν 47 | αν 48 | τοτε 49 | που 50 | πωσ 51 | ποιοσ 52 | ποια 53 | ποιο 54 | ποιοι 55 | ποιεσ 56 | ποιων 57 | ποιουσ 58 | αυτοσ 59 | αυτη 60 | αυτο 61 | αυτοι 62 | αυτων 63 | αυτουσ 64 | αυτεσ 65 | αυτα 66 | εκεινοσ 67 | εκεινη 68 | εκεινο 69 | εκεινοι 70 | εκεινεσ 71 | εκεινα 72 | εκεινων 73 | εκεινουσ 74 | οπωσ 75 | ομωσ 76 | ισωσ 77 | οσο 78 | οτι 79 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/solr/collection1/conf/lang/stopwords_en.txt: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # a couple of test stopwords to test that the words are really being 17 | # configured from this file: 18 | stopworda 19 | stopwordb 20 | 21 | # Standard english stop words taken from Lucene's StopAnalyzer 22 | a 23 | an 24 | and 25 | are 26 | as 27 | at 28 | be 29 | but 30 | by 31 | for 32 | if 33 | in 34 | into 35 | is 36 | it 37 | no 38 | not 39 | of 40 | on 41 | or 42 | such 43 | that 44 | the 45 | their 46 | then 47 | there 48 | these 49 | they 50 | this 51 | to 52 | was 53 | will 54 | with 55 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/solr/collection1/conf/lang/stopwords_eu.txt: -------------------------------------------------------------------------------- 1 | # example set of basque stopwords 2 | al 3 | anitz 4 | arabera 5 | asko 6 | baina 7 | bat 8 | batean 9 | batek 10 | bati 11 | batzuei 12 | batzuek 13 | batzuetan 14 | batzuk 15 | bera 16 | beraiek 17 | berau 18 | berauek 19 | bere 20 | berori 21 | beroriek 22 | beste 23 | bezala 24 | da 25 | dago 26 | dira 27 | ditu 28 | du 29 | dute 30 | edo 31 | egin 32 | ere 33 | eta 34 | eurak 35 | ez 36 | gainera 37 | gu 38 | gutxi 39 | guzti 40 | haiei 41 | haiek 42 | haietan 43 | hainbeste 44 | hala 45 | han 46 | handik 47 | hango 48 | hara 49 | hari 50 | hark 51 | hartan 52 | hau 53 | hauei 54 | hauek 55 | hauetan 56 | hemen 57 | hemendik 58 | hemengo 59 | hi 60 | hona 61 | honek 62 | honela 63 | honetan 64 | honi 65 | hor 66 | hori 67 | horiei 68 | horiek 69 | horietan 70 | horko 71 | horra 72 | horrek 73 | horrela 74 | horretan 75 | horri 76 | hortik 77 | hura 78 | izan 79 | ni 80 | noiz 81 | nola 82 | non 83 | nondik 84 | nongo 85 | nor 86 | nora 87 | ze 88 | zein 89 | zen 90 | zenbait 91 | zenbat 92 | zer 93 | zergatik 94 | ziren 95 | zituen 96 | zu 97 | zuek 98 | zuen 99 | zuten 100 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/solr/collection1/conf/lang/stopwords_ga.txt: -------------------------------------------------------------------------------- 1 | 2 | a 3 | ach 4 | ag 5 | agus 6 | an 7 | aon 8 | ar 9 | arna 10 | as 11 | b' 12 | ba 13 | beirt 14 | bhúr 15 | caoga 16 | ceathair 17 | ceathrar 18 | chomh 19 | chtó 20 | chuig 21 | chun 22 | cois 23 | céad 24 | cúig 25 | cúigear 26 | d' 27 | daichead 28 | dar 29 | de 30 | deich 31 | deichniúr 32 | den 33 | dhá 34 | do 35 | don 36 | dtí 37 | dá 38 | dár 39 | dó 40 | faoi 41 | faoin 42 | faoina 43 | faoinár 44 | fara 45 | fiche 46 | gach 47 | gan 48 | go 49 | gur 50 | haon 51 | hocht 52 | i 53 | iad 54 | idir 55 | in 56 | ina 57 | ins 58 | inár 59 | is 60 | le 61 | leis 62 | lena 63 | lenár 64 | m' 65 | mar 66 | mo 67 | mé 68 | na 69 | nach 70 | naoi 71 | naonúr 72 | ná 73 | ní 74 | níor 75 | nó 76 | nócha 77 | ocht 78 | ochtar 79 | os 80 | roimh 81 | sa 82 | seacht 83 | seachtar 84 | seachtó 85 | seasca 86 | seisear 87 | siad 88 | sibh 89 | sinn 90 | sna 91 | sé 92 | sí 93 | tar 94 | thar 95 | thú 96 | triúr 97 | trí 98 | trína 99 | trínár 100 | tríocha 101 | tú 102 | um 103 | ár 104 | é 105 | éis 106 | í 107 | ó 108 | ón 109 | óna 110 | ónár 111 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/solr/collection1/conf/lang/stopwords_hy.txt: -------------------------------------------------------------------------------- 1 | # example set of Armenian stopwords. 2 | այդ 3 | այլ 4 | այն 5 | այս 6 | դու 7 | դուք 8 | եմ 9 | են 10 | ենք 11 | ես 12 | եք 13 | է 14 | էի 15 | էին 16 | էինք 17 | էիր 18 | էիք 19 | էր 20 | ըստ 21 | թ 22 | ի 23 | ին 24 | իսկ 25 | իր 26 | կամ 27 | համար 28 | հետ 29 | հետո 30 | մենք 31 | մեջ 32 | մի 33 | ն 34 | նա 35 | նաև 36 | նրա 37 | նրանք 38 | որ 39 | որը 40 | որոնք 41 | որպես 42 | ու 43 | ում 44 | պիտի 45 | վրա 46 | և 47 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/solr/collection1/conf/lang/stopwords_th.txt: -------------------------------------------------------------------------------- 1 | # Thai stopwords from: 2 | # "Opinion Detection in Thai Political News Columns 3 | # Based on Subjectivity Analysis" 4 | # Khampol Sukhum, Supot Nitsuwat, and Choochart Haruechaiyasak 5 | ไว้ 6 | ไม่ 7 | ไป 8 | ได้ 9 | ให้ 10 | ใน 11 | โดย 12 | แห่ง 13 | แล้ว 14 | และ 15 | แรก 16 | แบบ 17 | แต่ 18 | เอง 19 | เห็น 20 | เลย 21 | เริ่ม 22 | เรา 23 | เมื่อ 24 | เพื่อ 25 | เพราะ 26 | เป็นการ 27 | เป็น 28 | เปิดเผย 29 | เปิด 30 | เนื่องจาก 31 | เดียวกัน 32 | เดียว 33 | เช่น 34 | เฉพาะ 35 | เคย 36 | เข้า 37 | เขา 38 | อีก 39 | อาจ 40 | อะไร 41 | ออก 42 | อย่าง 43 | อยู่ 44 | อยาก 45 | หาก 46 | หลาย 47 | หลังจาก 48 | หลัง 49 | หรือ 50 | หนึ่ง 51 | ส่วน 52 | ส่ง 53 | สุด 54 | สําหรับ 55 | ว่า 56 | วัน 57 | ลง 58 | ร่วม 59 | ราย 60 | รับ 61 | ระหว่าง 62 | รวม 63 | ยัง 64 | มี 65 | มาก 66 | มา 67 | พร้อม 68 | พบ 69 | ผ่าน 70 | ผล 71 | บาง 72 | น่า 73 | นี้ 74 | นํา 75 | นั้น 76 | นัก 77 | นอกจาก 78 | ทุก 79 | ที่สุด 80 | ที่ 81 | ทําให้ 82 | ทํา 83 | ทาง 84 | ทั้งนี้ 85 | ทั้ง 86 | ถ้า 87 | ถูก 88 | ถึง 89 | ต้อง 90 | ต่างๆ 91 | ต่าง 92 | ต่อ 93 | ตาม 94 | ตั้งแต่ 95 | ตั้ง 96 | ด้าน 97 | ด้วย 98 | ดัง 99 | ซึ่ง 100 | ช่วง 101 | จึง 102 | จาก 103 | จัด 104 | จะ 105 | คือ 106 | ความ 107 | ครั้ง 108 | คง 109 | ขึ้น 110 | ของ 111 | ขอ 112 | ขณะ 113 | ก่อน 114 | ก็ 115 | การ 116 | กับ 117 | กัน 118 | กว่า 119 | กล่าว 120 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/solr/collection1/conf/lang/userdict_ja.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This is a sample user dictionary for Kuromoji (JapaneseTokenizer) 3 | # 4 | # Add entries to this file in order to override the statistical model in terms 5 | # of segmentation, readings and part-of-speech tags. Notice that entries do 6 | # not have weights since they are always used when found. This is by-design 7 | # in order to maximize ease-of-use. 8 | # 9 | # Entries are defined using the following CSV format: 10 | # , ... , ... , 11 | # 12 | # Notice that a single half-width space separates tokens and readings, and 13 | # that the number tokens and readings must match exactly. 14 | # 15 | # Also notice that multiple entries with the same is undefined. 16 | # 17 | # Whitespace only lines are ignored. Comments are not allowed on entry lines. 18 | # 19 | 20 | # Custom segmentation for kanji compounds 21 | 日本経済新聞,日本 経済 新聞,ニホン ケイザイ シンブン,カスタム名詞 22 | 関西国際空港,関西 国際 空港,カンサイ コクサイ クウコウ,カスタム名詞 23 | 24 | # Custom segmentation for compound katakana 25 | トートバッグ,トート バッグ,トート バッグ,かずカナ名詞 26 | ショルダーバッグ,ショルダー バッグ,ショルダー バッグ,かずカナ名詞 27 | 28 | # Custom reading for former sumo wrestler 29 | 朝青龍,朝青龍,アサショウリュウ,カスタム人名 30 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/solr/collection1/conf/protwords.txt: -------------------------------------------------------------------------------- 1 | # The ASF licenses this file to You under the Apache License, Version 2.0 2 | # (the "License"); you may not use this file except in compliance with 3 | # the License. You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | #----------------------------------------------------------------------- 14 | # Use a protected word file to protect against the stemmer reducing two 15 | # unrelated words to the same base word. 16 | 17 | # Some non-words that normally won't be encountered, 18 | # just to test that they won't be stemmed. 19 | dontstems 20 | zwhacky 21 | 22 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/solr/collection1/conf/stopwords.txt: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/solr/collection1/conf/synonyms.txt: -------------------------------------------------------------------------------- 1 | # The ASF licenses this file to You under the Apache License, Version 2.0 2 | # (the "License"); you may not use this file except in compliance with 3 | # the License. You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | #----------------------------------------------------------------------- 14 | #some test synonym mappings unlikely to appear in real input text 15 | aaafoo => aaabar 16 | bbbfoo => bbbfoo bbbbar 17 | cccfoo => cccbar cccbaz 18 | fooaaa,baraaa,bazaaa 19 | 20 | # Some synonym groups specific to this example 21 | GB,gib,gigabyte,gigabytes 22 | MB,mib,megabyte,megabytes 23 | Television, Televisions, TV, TVs 24 | #notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming 25 | #after us won't split it into two words. 26 | 27 | # Synonym mappings can be used for spelling correction too 28 | pixima => pixma 29 | 30 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/NullHeader.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/NullHeader.docx -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/boilerplate.html: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | Title 8 | 9 | 10 | 11 | 12 | 13 | 21 | 22 |
14 | 15 | 16 | 17 | 18 | 19 |
boilerplatetext
20 |
23 | 24 |

This is the real meat of the page, 25 | and represents the text we want. 26 | It has lots of juicy content. 27 | 28 | We assume that it won't get filtered out. 29 | And that all of the lines will be in the 30 | output. 31 |

32 | 33 |

34 | Here's another paragraph of text. 35 | This is the end of the text. 36 |

37 | 38 |

footer

39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/cars.csv: -------------------------------------------------------------------------------- 1 | Age,Color,Extras,Type,Used 2 | 2,blue,GPS,"Gas, with electric","" 3 | 10,green,"Labeled ""Vintage, 1913""",,yes 4 | 100,red,"Labeled ""Vintage 1913""",yes 5 | 5,orange,none,"This is a 6 | multi, line text",no -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/cars.csv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/cars.csv.gz -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/cars.ssv: -------------------------------------------------------------------------------- 1 | Age Color Extras Type Used 2 | 2 blue GPS "Gas, with electric" "" 3 | 10 green "Labeled ""Vintage, 1913""" yes 4 | 100 red "Labeled ""Vintage 1913""" yes 5 | 5 orange none "This is a 6 | multi, line text" no -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/cars.tsv: -------------------------------------------------------------------------------- 1 | Age Color Extras Type Used 2 | 2 blue GPS "Gas with electric" "" 3 | 10 green "Labeled ""Vintage 1913""" yes 4 | 100 red "Labeled ""Vintage 1913""" yes 5 | 5 orange none "This is a 6 | multi line text" no -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/multiline-sessions.log: -------------------------------------------------------------------------------- 1 | Started GET /foo 2 | Foo Started GET as HTML 3 | Completed 401 Unauthorized in 0ms 4 | 5 | 6 | Started GET /bar 7 | Bar as HTML 8 | Completed 200 OK in 339ms 9 | Started GET /baz 10 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/non-length-delimited-20130430-234145-tweets.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/non-length-delimited-20130430-234145-tweets.json.gz -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/sample-statuses-20120906-141433-medium.avro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/sample-statuses-20120906-141433-medium.avro -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/sample-statuses-20120906-141433-subschema.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type" : "record", 3 | "name" : "Doc", 4 | "doc" : "adoc", 5 | "fields" : [ { 6 | "name" : "id", 7 | "type" : "string" 8 | }, { 9 | "name" : "text", 10 | "type" : [ "string", "null" ] 11 | } ] 12 | } 13 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/sample-statuses-20120906-141433.avro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/sample-statuses-20120906-141433.avro -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/sample-statuses-20120906-141433.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type" : "record", 3 | "name" : "Doc", 4 | "doc" : "adoc", 5 | "fields" : [ { 6 | "name" : "id", 7 | "type" : "string" 8 | }, { 9 | "name" : "user_friends_count", 10 | "type" : [ "int", "null" ] 11 | }, { 12 | "name" : "user_location", 13 | "type" : [ "string", "null" ] 14 | }, { 15 | "name" : "user_description", 16 | "type" : [ "string", "null" ] 17 | }, { 18 | "name" : "user_statuses_count", 19 | "type" : [ "int", "null" ] 20 | }, { 21 | "name" : "user_followers_count", 22 | "type" : [ "int", "null" ] 23 | }, { 24 | "name" : "user_name", 25 | "type" : [ "string", "null" ] 26 | }, { 27 | "name" : "user_screen_name", 28 | "type" : [ "string", "null" ] 29 | }, { 30 | "name" : "created_at", 31 | "type" : [ "string", "null" ] 32 | }, { 33 | "name" : "text", 34 | "type" : [ "string", "null" ] 35 | }, { 36 | "name" : "retweet_count", 37 | "type" : [ "int", "null" ] 38 | }, { 39 | "name" : "retweeted", 40 | "type" : [ "boolean", "null" ] 41 | }, { 42 | "name" : "in_reply_to_user_id", 43 | "type" : [ "long", "null" ] 44 | }, { 45 | "name" : "source", 46 | "type" : [ "string", "null" ] 47 | }, { 48 | "name" : "in_reply_to_status_id", 49 | "type" : [ "long", "null" ] 50 | }, { 51 | "name" : "media_url_https", 52 | "type" : [ "string", "null" ] 53 | }, { 54 | "name" : "expanded_url", 55 | "type" : [ "string", "null" ] 56 | } ] 57 | } 58 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/sample-statuses-20120906-141433.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/sample-statuses-20120906-141433.bz2 -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/sample-statuses-20120906-141433.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/sample-statuses-20120906-141433.gz -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/test-documents.7z: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/test-documents.7z -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/test-documents.cpio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/test-documents.cpio -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/test-documents.tar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/test-documents.tar -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/test-documents.tbz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/test-documents.tbz2 -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/test-documents.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/test-documents.tgz -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/test-documents.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/test-documents.zip -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/test-outlook.msg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/test-outlook.msg -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/test-zip-of-zip.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/test-zip-of-zip.zip -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testAIFF.aif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testAIFF.aif -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testBMP.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testBMP.bmp -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testEXCEL.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testEXCEL.xls -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testEXCEL.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testEXCEL.xlsx -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testFLAC.flac: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testFLAC.flac -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testFLAC.oga: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testFLAC.oga -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testFLV.flv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testFLV.flv -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testGIF.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testGIF.gif -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testJAR.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testJAR.jar -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testJPEG_EXIF.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testJPEG_EXIF.jpg -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testJPEG_EXIF.jpg.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testJPEG_EXIF.jpg.gz -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testJPEG_EXIF.jpg.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testJPEG_EXIF.jpg.tar.gz -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testKeynote.key: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testKeynote.key -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testMP3i18n.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testMP3i18n.mp3 -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testMP4.m4a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testMP4.m4a -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testNumbers.numbers: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testNumbers.numbers -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testPDF.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testPDF.pdf -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testPNG.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testPNG.png -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testPPM.ppm: -------------------------------------------------------------------------------- 1 | P3 2 | 1 1 3 | 255 4 | 0 0 0 -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testPPT_various.ppt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testPPT_various.ppt -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testPPT_various.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testPPT_various.pptx -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testPSD.psd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testPSD.psd -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testPages.pages: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testPages.pages -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testRDF.rdf: -------------------------------------------------------------------------------- 1 | 2 | 18 | 20 | 23 | 24 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testSVG.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | Test SVG image 6 | 7 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testTIFF.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testTIFF.tif -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testTrueType.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testTrueType.ttf -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testVISIO.vsd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testVISIO.vsd -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testVORBIS.ogg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testVORBIS.ogg -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testWAR.war: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testWAR.war -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testWAV.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testWAV.wav -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testWINMAIL.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testWINMAIL.dat -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testWMA.wma: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testWMA.wma -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testWMF.wmf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testWMF.wmf -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testWMV.wmv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testWMV.wmv -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testWORD_various.doc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testWORD_various.doc -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testWindows-x86-32.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/logging-flume/ff739dbe659ec5fa3fa92dcf2e4339c115673e7c/flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-documents/testWindows-x86-32.exe -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-morphlines/noOperation.conf: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | morphlines : [ 19 | { 20 | id : morphline1 21 | importCommands : ["org.kitesdk.**"] 22 | 23 | commands : [ 24 | { logDebug { format : "output record: {}", args : ["@{}"] } } 25 | ] 26 | } 27 | ] 28 | -------------------------------------------------------------------------------- /flume-ng-sinks/flume-ng-morphline-solr-sink/src/test/resources/test-morphlines/readClob.conf: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | morphlines : [ 19 | { 20 | id : morphline1 21 | importCommands : ["org.kitesdk.**", "org.apache.solr.**"] 22 | 23 | commands : [ 24 | { 25 | readClob { 26 | charset : UTF-8 27 | } 28 | } 29 | { logDebug { format : "output record: {}", args : ["@{}"] } } 30 | ] 31 | } 32 | ] 33 | -------------------------------------------------------------------------------- /flume-ng-sources/flume-jms-source/src/main/java/org/apache/flume/source/jms/InitialContextFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flume.source.jms; 19 | 20 | import java.util.Properties; 21 | 22 | import javax.naming.InitialContext; 23 | import javax.naming.NamingException; 24 | 25 | public class InitialContextFactory { 26 | 27 | public InitialContext create(Properties properties) throws NamingException { 28 | return new InitialContext(properties); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /flume-ng-sources/flume-jms-source/src/main/java/org/apache/flume/source/jms/JMSDestinationLocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.source.jms; 20 | 21 | public enum JMSDestinationLocator { 22 | JNDI, CDI 23 | } -------------------------------------------------------------------------------- /flume-ng-sources/flume-jms-source/src/main/java/org/apache/flume/source/jms/JMSDestinationType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.source.jms; 20 | 21 | public enum JMSDestinationType { 22 | QUEUE(), TOPIC(), 23 | } -------------------------------------------------------------------------------- /flume-ng-sources/flume-scribe-source/src/main/thrift/aslv2: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | -------------------------------------------------------------------------------- /flume-ng-sources/flume-scribe-source/src/main/thrift/scribe-source.thrift: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | 21 | namespace java org.apache.flume.source.scribe 22 | 23 | enum ResultCode { 24 | OK,TRY_LATER 25 | } 26 | 27 | struct LogEntry { 28 | 1: string category, 29 | 2: string message 30 | } 31 | 32 | service Scribe { 33 | ResultCode Log(1: list messages); 34 | } -------------------------------------------------------------------------------- /flume-ng-tests/src/main/java/org/apache/flume/Dummy.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flume; 20 | 21 | /** 22 | * Placeholder class so the git mirror does not leave out the src/main/java dir 23 | */ 24 | public class Dummy { 25 | 26 | public Dummy() { 27 | throw new UnsupportedOperationException("Dummy class meant for use"); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /flume-ng-tests/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /flume-ng-tests/src/test/resources/rpc-client-test.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | rpccagent.sources = src1 19 | rpccagent.channels = ch1 20 | 21 | rpccagent.sources.src1.type = avro 22 | rpccagent.sources.src1.bind = 127.0.0.1 23 | #rpccagent.sources.src1.port = Set from java source 24 | rpccagent.sources.src1.channels = ch1 25 | 26 | rpccagent.channels.ch1.type = memory 27 | rpccagent.channels.ch1.capacity = 100 28 | -------------------------------------------------------------------------------- /flume-tools/src/main/java/org/apache/flume/tools/FlumeTool.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.tools; 20 | 21 | public interface FlumeTool { 22 | 23 | public void run(String[] args) throws Exception; 24 | } 25 | -------------------------------------------------------------------------------- /flume-tools/src/main/java/org/apache/flume/tools/FlumeToolType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.flume.tools; 20 | 21 | import java.util.Locale; 22 | 23 | public enum FlumeToolType { 24 | FCINTEGRITYTOOL(FileChannelIntegrityTool.class); 25 | 26 | private final Class klass; 27 | private FlumeToolType(Class klass) { 28 | this.klass = klass; 29 | } 30 | 31 | public Class getClassInstance() { 32 | return this.klass; 33 | } 34 | 35 | public static String getNames() { 36 | StringBuilder builder = new StringBuilder(); 37 | for (FlumeToolType type: values()) { 38 | builder.append(type.name().toLowerCase(Locale.ENGLISH) + "\n"); 39 | } 40 | return builder.toString(); 41 | } 42 | } 43 | --------------------------------------------------------------------------------