├── .gitignore ├── LICENSE ├── README.md ├── TODO.md ├── bin └── jlogstash.sh ├── core ├── .gitignore ├── metrics.md ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ ├── JlogstashMain.java │ │ │ ├── OptionsProcessor.java │ │ │ ├── annotation │ │ │ ├── Path.java │ │ │ ├── Required.java │ │ │ ├── Url.java │ │ │ └── plugin │ │ │ │ ├── AnnotationInterface.java │ │ │ │ ├── PathPlugin.java │ │ │ │ ├── RequiredPlugin.java │ │ │ │ └── UrlPlugin.java │ │ │ ├── assembly │ │ │ ├── AssemblyPipeline.java │ │ │ ├── CmdLineParams.java │ │ │ ├── ShutDownHook.java │ │ │ ├── pthread │ │ │ │ ├── FilterThread.java │ │ │ │ ├── InputThread.java │ │ │ │ └── OutputThread.java │ │ │ └── qlist │ │ │ │ ├── FilterQueueList.java │ │ │ │ ├── OutPutQueueList.java │ │ │ │ └── QueueList.java │ │ │ ├── callback │ │ │ ├── ClassLoaderCallBack.java │ │ │ └── ClassLoaderCallBackMethod.java │ │ │ ├── classloader │ │ │ ├── JARClassLoader.java │ │ │ └── JARClassLoaderFactory.java │ │ │ ├── configs │ │ │ ├── Config.java │ │ │ ├── ConfigObject.java │ │ │ └── YamlConfig.java │ │ │ ├── date │ │ │ ├── DateParser.java │ │ │ ├── FormatParser.java │ │ │ ├── ISODateParser.java │ │ │ ├── UnixMSParser.java │ │ │ └── UnixParser.java │ │ │ ├── decoder │ │ │ ├── IDecode.java │ │ │ ├── JsonDecoder.java │ │ │ ├── JsonMessageDecoder.java │ │ │ ├── MultilineDecoder.java │ │ │ └── PlainDecoder.java │ │ │ ├── exception │ │ │ ├── ExceptionUtil.java │ │ │ ├── LogstashException.java │ │ │ ├── PathException.java │ │ │ ├── RequiredException.java │ │ │ └── UrlException.java │ │ │ ├── factory │ │ │ ├── FilterFactory.java │ │ │ ├── InputFactory.java │ │ │ ├── InstanceFactory.java │ │ │ ├── LogstashThreadFactory.java │ │ │ └── OutputFactory.java │ │ │ ├── filters │ │ │ ├── BaseFilter.java │ │ │ ├── FilterProxy.java │ │ │ └── IBaseFilter.java │ │ │ ├── inputs │ │ │ ├── BaseInput.java │ │ │ ├── IBaseInput.java │ │ │ └── InputProxy.java │ │ │ ├── log │ │ │ ├── Log4jComponent.java │ │ │ ├── LogComponent.java │ │ │ └── LogbackComponent.java │ │ │ ├── metrics │ │ │ ├── JlogstashMetric.java │ │ │ ├── MetricNames.java │ │ │ ├── MetricRegistry.java │ │ │ ├── MetricRegistryImpl.java │ │ │ ├── ViewUpdater.java │ │ │ ├── base │ │ │ │ ├── CharacterFilter.java │ │ │ │ ├── Counter.java │ │ │ │ ├── Gauge.java │ │ │ │ ├── Meter.java │ │ │ │ ├── MeterView.java │ │ │ │ ├── Metric.java │ │ │ │ ├── MetricConfig.java │ │ │ │ ├── MetricGroup.java │ │ │ │ ├── SimpleCounter.java │ │ │ │ ├── View.java │ │ │ │ └── reporter │ │ │ │ │ ├── MetricReporter.java │ │ │ │ │ └── Scheduled.java │ │ │ ├── groups │ │ │ │ ├── AbstractMetricGroup.java │ │ │ │ ├── ComponentMetricGroup.java │ │ │ │ ├── FrontMetricGroup.java │ │ │ │ ├── GenericMetricGroup.java │ │ │ │ ├── JlogstashJobMetricGroup.java │ │ │ │ ├── PipelineInputMetricGroup.java │ │ │ │ ├── PipelineOutputMetricGroup.java │ │ │ │ └── ProxyMetricGroup.java │ │ │ ├── promethues │ │ │ │ ├── AbstractPrometheusReporter.java │ │ │ │ ├── PrometheusPushGatewayReporter.java │ │ │ │ └── PrometheusReporter.java │ │ │ ├── scope │ │ │ │ ├── JlogstashJobScopeFormat.java │ │ │ │ ├── PipelineScopeFormat.java │ │ │ │ ├── ScopeFormat.java │ │ │ │ └── ScopeFormats.java │ │ │ └── util │ │ │ │ ├── AbstractID.java │ │ │ │ ├── MetricUtils.java │ │ │ │ ├── NetUtils.java │ │ │ │ ├── Preconditions.java │ │ │ │ └── UnionIterator.java │ │ │ ├── monitor │ │ │ ├── MonitorInfo.java │ │ │ └── MonitorService.java │ │ │ ├── outputs │ │ │ ├── BaseOutput.java │ │ │ ├── IBaseOutput.java │ │ │ └── OutputProxy.java │ │ │ ├── property │ │ │ └── SystemProperty.java │ │ │ ├── render │ │ │ ├── Formatter.java │ │ │ ├── FreeMarkerRender.java │ │ │ └── TemplateRender.java │ │ │ └── utils │ │ │ ├── Base64Util.java │ │ │ ├── BasePluginUtil.java │ │ │ ├── ExceptionUtil.java │ │ │ ├── GZipUtil.java │ │ │ ├── LocalIpAddressUtil.java │ │ │ ├── Package.java │ │ │ └── Public.java │ └── resources │ │ ├── patterns │ │ ├── firewalls │ │ ├── grok-patterns │ │ ├── haproxy │ │ ├── java │ │ ├── junos │ │ ├── linux-syslog │ │ ├── mcollective │ │ ├── mcollective-patterns │ │ ├── mongodb │ │ ├── nagios │ │ ├── postgresql │ │ ├── redis │ │ └── ruby │ │ └── ua_parser │ │ └── regexes.yaml │ └── test │ └── java │ └── com │ └── dtstack │ └── jlogstash │ ├── Test.java │ ├── TestFilterPlugin.java │ ├── TestInputPlugin.java │ └── TestOutputPlugin.java ├── example.yml ├── index.html ├── lib └── README.MD ├── params.json ├── pipeline ├── filters │ ├── README.md │ ├── add │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── filters │ │ │ └── Add.java │ ├── dateiso8601 │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── filters │ │ │ └── DateISO8601.java │ ├── geoip2 │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── filters │ │ │ └── GeoIP2.java │ ├── grok │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── com │ │ │ │ └── dtstack │ │ │ │ └── jlogstash │ │ │ │ └── filters │ │ │ │ ├── JGrok.java │ │ │ │ └── PatternRead.java │ │ │ └── resources │ │ │ └── pattern │ ├── gsub │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── dtstack │ │ │ │ └── jlogstash │ │ │ │ └── filters │ │ │ │ └── Gsub.java │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── logstash │ │ │ └── filter │ │ │ └── gsub │ │ │ └── AppTest.java │ ├── ipip │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── dtstack │ │ │ │ │ └── jlogstash │ │ │ │ │ └── filters │ │ │ │ │ ├── IP.java │ │ │ │ │ └── IpIp.java │ │ │ └── resources │ │ │ │ └── 17monipdb.dat │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── logstash │ │ │ └── filter │ │ │ └── ipip │ │ │ └── AppTest.java │ ├── java │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── com │ │ │ │ └── dtstack │ │ │ │ └── jlogstash │ │ │ │ ├── exception │ │ │ │ └── InitializeException.java │ │ │ │ └── filters │ │ │ │ └── Java.java │ │ │ └── resources │ │ │ └── code.ftl │ ├── json │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── filters │ │ │ └── Json.java │ ├── kv │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── filters │ │ │ └── KV.java │ ├── lowercase │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── filters │ │ │ └── Lowercase.java │ ├── performance │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── filters │ │ │ └── Performance.java │ ├── pom.xml │ ├── remove │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── filters │ │ │ └── Remove.java │ ├── rename │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── filters │ │ │ └── Rename.java │ ├── replace │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── filters │ │ │ └── Replace.java │ ├── split │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── filters │ │ │ └── Split.java │ ├── splitStr │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── filters │ │ │ └── SplitStr.java │ ├── trim │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── filters │ │ │ └── Trim.java │ ├── ua │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ ├── filters │ │ │ └── UA.java │ │ │ └── parser │ │ │ ├── CachingParser.java │ │ │ ├── Client.java │ │ │ ├── Device.java │ │ │ ├── DeviceParser.java │ │ │ ├── OS.java │ │ │ ├── OSParser.java │ │ │ ├── Parser.java │ │ │ ├── UserAgent.java │ │ │ ├── UserAgentParser.java │ │ │ └── UserAgentUtil.java │ ├── uppercase │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── filters │ │ │ └── Uppercase.java │ └── urldecode │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── dtstack │ │ └── jlogstash │ │ └── filters │ │ └── URLDecode.java ├── inputs │ ├── README.md │ ├── beats │ │ ├── .gitignore │ │ ├── jlogstash-java-docs │ │ │ ├── allclasses-frame.html │ │ │ ├── allclasses-noframe.html │ │ │ ├── com │ │ │ │ └── dtstack │ │ │ │ │ └── jlogstash │ │ │ │ │ └── inputs │ │ │ │ │ ├── Batch.html │ │ │ │ │ ├── BatchEncoder.html │ │ │ │ │ ├── Beats.MessageListener.html │ │ │ │ │ ├── Beats.html │ │ │ │ │ ├── BeatsHandler.html │ │ │ │ │ ├── BeatsParser.html │ │ │ │ │ ├── CompressedBatchEncoder.html │ │ │ │ │ ├── IMessageListener.html │ │ │ │ │ ├── JsonUtils.html │ │ │ │ │ ├── Message.html │ │ │ │ │ ├── PrivateKeyConverter.html │ │ │ │ │ ├── Protocol.html │ │ │ │ │ ├── Server.html │ │ │ │ │ ├── SslSimpleBuilder.html │ │ │ │ │ ├── class-use │ │ │ │ │ ├── Batch.html │ │ │ │ │ ├── BatchEncoder.html │ │ │ │ │ ├── Beats.MessageListener.html │ │ │ │ │ ├── Beats.html │ │ │ │ │ ├── BeatsHandler.html │ │ │ │ │ ├── BeatsParser.html │ │ │ │ │ ├── CompressedBatchEncoder.html │ │ │ │ │ ├── IMessageListener.html │ │ │ │ │ ├── JsonUtils.html │ │ │ │ │ ├── Message.html │ │ │ │ │ ├── PrivateKeyConverter.html │ │ │ │ │ ├── Protocol.html │ │ │ │ │ ├── Server.html │ │ │ │ │ └── SslSimpleBuilder.html │ │ │ │ │ ├── package-frame.html │ │ │ │ │ ├── package-summary.html │ │ │ │ │ ├── package-tree.html │ │ │ │ │ └── package-use.html │ │ │ ├── constant-values.html │ │ │ ├── deprecated-list.html │ │ │ ├── help-doc.html │ │ │ ├── index-all.html │ │ │ ├── index.html │ │ │ ├── overview-tree.html │ │ │ ├── package-list │ │ │ ├── script.js │ │ │ ├── serialized-form.html │ │ │ └── stylesheet.css │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── inputs │ │ │ ├── Batch.java │ │ │ ├── BatchEncoder.java │ │ │ ├── Beats.java │ │ │ ├── BeatsHandler.java │ │ │ ├── BeatsParser.java │ │ │ ├── CompressedBatchEncoder.java │ │ │ ├── IMessageListener.java │ │ │ ├── JsonUtils.java │ │ │ ├── Message.java │ │ │ ├── PrivateKeyConverter.java │ │ │ ├── Protocol.java │ │ │ ├── Server.java │ │ │ └── SslSimpleBuilder.java │ ├── binlog │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── com │ │ │ │ ├── alibaba │ │ │ │ └── otter │ │ │ │ │ └── canal │ │ │ │ │ └── filter │ │ │ │ │ └── PatternUtils.java │ │ │ │ ├── dtstack │ │ │ │ └── jlogstash │ │ │ │ │ └── inputs │ │ │ │ │ ├── Binlog.java │ │ │ │ │ ├── BinlogEventSink.java │ │ │ │ │ ├── BinlogJournalValidator.java │ │ │ │ │ ├── BinlogPosUtil.java │ │ │ │ │ └── BinlogPositionManager.java │ │ │ │ └── google │ │ │ │ └── common │ │ │ │ └── collect │ │ │ │ ├── MapMakerHelper.java │ │ │ │ └── MigrateMap.java │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── TestMysqlDistributeTable.java │ ├── elasticsearch │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ ├── exception │ │ │ └── InitializeException.java │ │ │ └── inputs │ │ │ ├── Elasticsearch.java │ │ │ ├── ElasticsearchSourceConfig.java │ │ │ └── EsClientHelper.java │ ├── file │ │ ├── .gitignore │ │ ├── jlogstash-java-docs │ │ │ ├── allclasses-frame.html │ │ │ ├── allclasses-noframe.html │ │ │ ├── com │ │ │ │ └── dtstack │ │ │ │ │ └── jlogstash │ │ │ │ │ └── inputs │ │ │ │ │ ├── File.html │ │ │ │ │ ├── IReader.html │ │ │ │ │ ├── ReadFactory.html │ │ │ │ │ ├── ReadLineUtil.html │ │ │ │ │ ├── ReadRarFile.html │ │ │ │ │ ├── ReadTarFile.html │ │ │ │ │ ├── ReadZipFile.html │ │ │ │ │ ├── class-use │ │ │ │ │ ├── File.html │ │ │ │ │ ├── IReader.html │ │ │ │ │ ├── ReadFactory.html │ │ │ │ │ ├── ReadLineUtil.html │ │ │ │ │ ├── ReadRarFile.html │ │ │ │ │ ├── ReadTarFile.html │ │ │ │ │ └── ReadZipFile.html │ │ │ │ │ ├── package-frame.html │ │ │ │ │ ├── package-summary.html │ │ │ │ │ ├── package-tree.html │ │ │ │ │ └── package-use.html │ │ │ ├── constant-values.html │ │ │ ├── deprecated-list.html │ │ │ ├── help-doc.html │ │ │ ├── index-all.html │ │ │ ├── index.html │ │ │ ├── overview-tree.html │ │ │ ├── package-list │ │ │ ├── script.js │ │ │ ├── serialized-form.html │ │ │ └── stylesheet.css │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── dtstack │ │ │ │ └── jlogstash │ │ │ │ └── inputs │ │ │ │ ├── File.java │ │ │ │ ├── IReader.java │ │ │ │ ├── ReadFactory.java │ │ │ │ ├── ReadLineUtil.java │ │ │ │ ├── ReadRarFile.java │ │ │ │ ├── ReadTarFile.java │ │ │ │ └── ReadZipFile.java │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── logstash │ │ │ └── input │ │ │ └── fileinput │ │ │ └── Test.java │ ├── jdbc │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ ├── exception │ │ │ └── InitializeException.java │ │ │ ├── inputs │ │ │ ├── ConnectionConfig.java │ │ │ ├── Jdbc.java │ │ │ └── JdbcClientHelper.java │ │ │ └── util │ │ │ ├── BlobClobUtil.java │ │ │ └── IOUtils.java │ ├── kafka09 │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── dtstack │ │ │ │ └── jlogstash │ │ │ │ └── inputs │ │ │ │ ├── IKafkaChg.java │ │ │ │ ├── Kafka09.java │ │ │ │ └── MonitorCluster.java │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── logstash │ │ │ └── input │ │ │ └── kafka │ │ │ └── AppTest.java │ ├── kafka10 │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── inputs │ │ │ ├── Kafka10.java │ │ │ └── kafka │ │ │ └── JKafkaConsumer.java │ ├── kafkadistributed │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ ├── distributed │ │ │ ├── BrokerNode.java │ │ │ ├── BrokersNode.java │ │ │ ├── DownReblance.java │ │ │ ├── HearBeat.java │ │ │ ├── HeartBeatCheck.java │ │ │ ├── MasterCheck.java │ │ │ ├── RouteSelect.java │ │ │ ├── UpReblance.java │ │ │ ├── ZkDistributed.java │ │ │ ├── http │ │ │ │ ├── cilent │ │ │ │ │ ├── HttpClient.java │ │ │ │ │ └── LogstashHttpClient.java │ │ │ │ ├── common │ │ │ │ │ ├── HttpCommon.java │ │ │ │ │ └── Urls.java │ │ │ │ └── server │ │ │ │ │ ├── GetLogPoolNoCompleteData.java │ │ │ │ │ ├── ImmediatelyLoadNodeDataHandler.java │ │ │ │ │ ├── ImmediatelyLogPoolDataHandler.java │ │ │ │ │ ├── LogstashHttpServer.java │ │ │ │ │ ├── ManuallyLogPoolDataHandler.java │ │ │ │ │ ├── PostHandler.java │ │ │ │ │ └── callback │ │ │ │ │ ├── ApiCallback.java │ │ │ │ │ ├── ApiCallbackMethod.java │ │ │ │ │ ├── ApiResult.java │ │ │ │ │ └── RequestContext.java │ │ │ ├── logmerge │ │ │ │ ├── ClusterLog.java │ │ │ │ ├── CompletedLog.java │ │ │ │ ├── IPreLog.java │ │ │ │ ├── IPreLogx.java │ │ │ │ ├── LogDeleWatcher.java │ │ │ │ ├── LogPool.java │ │ │ │ └── LogTypeConstant.java │ │ │ ├── netty │ │ │ │ ├── client │ │ │ │ │ └── NettySend.java │ │ │ │ └── server │ │ │ │ │ └── NettyRev.java │ │ │ └── util │ │ │ │ ├── CountUtil.java │ │ │ │ └── RouteUtil.java │ │ │ ├── inputs │ │ │ ├── IKafkaChg.java │ │ │ ├── KafkaDistributed.java │ │ │ └── MonitorCluster.java │ │ │ └── jdistributed │ │ │ └── gclog │ │ │ ├── CMSLogPattern.java │ │ │ ├── CMSPreLogInfo.java │ │ │ └── GCTypeConstant.java │ ├── mongodb │ │ ├── .gitignore │ │ ├── jlogstash-java-docs │ │ │ ├── allclasses-frame.html │ │ │ ├── allclasses-noframe.html │ │ │ ├── com │ │ │ │ └── dtstack │ │ │ │ │ └── jlogstash │ │ │ │ │ ├── constans │ │ │ │ │ ├── ConfigConstans.html │ │ │ │ │ ├── class-use │ │ │ │ │ │ └── ConfigConstans.html │ │ │ │ │ ├── package-frame.html │ │ │ │ │ ├── package-summary.html │ │ │ │ │ ├── package-tree.html │ │ │ │ │ └── package-use.html │ │ │ │ │ └── inputs │ │ │ │ │ ├── MongoDB.html │ │ │ │ │ ├── MongoDBTest.html │ │ │ │ │ ├── Scheduler.html │ │ │ │ │ ├── SimpleScheduler.html │ │ │ │ │ ├── StartTime.html │ │ │ │ │ ├── Task.html │ │ │ │ │ ├── class-use │ │ │ │ │ ├── MongoDB.html │ │ │ │ │ ├── MongoDBTest.html │ │ │ │ │ ├── Scheduler.html │ │ │ │ │ ├── SimpleScheduler.html │ │ │ │ │ ├── StartTime.html │ │ │ │ │ └── Task.html │ │ │ │ │ ├── package-frame.html │ │ │ │ │ ├── package-summary.html │ │ │ │ │ ├── package-tree.html │ │ │ │ │ └── package-use.html │ │ │ ├── constant-values.html │ │ │ ├── deprecated-list.html │ │ │ ├── help-doc.html │ │ │ ├── index-all.html │ │ │ ├── index.html │ │ │ ├── overview-frame.html │ │ │ ├── overview-summary.html │ │ │ ├── overview-tree.html │ │ │ ├── package-list │ │ │ ├── script.js │ │ │ ├── serialized-form.html │ │ │ └── stylesheet.css │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ ├── constans │ │ │ └── ConfigConstans.java │ │ │ └── inputs │ │ │ ├── MongoDB.java │ │ │ ├── MongoDBTest.java │ │ │ ├── Scheduler.java │ │ │ ├── SimpleScheduler.java │ │ │ ├── StartTime.java │ │ │ └── Task.java │ ├── netty │ │ ├── .gitignore │ │ ├── jlogstash-java-docs │ │ │ ├── allclasses-frame.html │ │ │ ├── allclasses-noframe.html │ │ │ ├── com │ │ │ │ └── dtstack │ │ │ │ │ └── jlogstash │ │ │ │ │ └── inputs │ │ │ │ │ ├── Netty.html │ │ │ │ │ ├── WhiteIpTask.html │ │ │ │ │ ├── class-use │ │ │ │ │ ├── Netty.html │ │ │ │ │ └── WhiteIpTask.html │ │ │ │ │ ├── package-frame.html │ │ │ │ │ ├── package-summary.html │ │ │ │ │ ├── package-tree.html │ │ │ │ │ └── package-use.html │ │ │ ├── constant-values.html │ │ │ ├── deprecated-list.html │ │ │ ├── help-doc.html │ │ │ ├── index-all.html │ │ │ ├── index.html │ │ │ ├── overview-tree.html │ │ │ ├── package-list │ │ │ ├── script.js │ │ │ ├── serialized-form.html │ │ │ └── stylesheet.css │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── inputs │ │ │ ├── Netty.java │ │ │ └── WhiteIpTask.java │ ├── pom.xml │ ├── redis │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── dtstack │ │ │ │ └── jlogstash │ │ │ │ ├── inputs │ │ │ │ ├── ChannelListener.java │ │ │ │ └── Redis.java │ │ │ │ └── utils │ │ │ │ ├── DataType.java │ │ │ │ └── JedisPoolUtils.java │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ ├── AppTest.java │ │ │ └── TestPubSub.java │ ├── stdin │ │ ├── .gitignore │ │ ├── jlogstash-java-docs │ │ │ ├── allclasses-frame.html │ │ │ ├── allclasses-noframe.html │ │ │ ├── com │ │ │ │ └── dtstack │ │ │ │ │ └── jlogstash │ │ │ │ │ └── inputs │ │ │ │ │ ├── Stdin.html │ │ │ │ │ ├── class-use │ │ │ │ │ └── Stdin.html │ │ │ │ │ ├── package-frame.html │ │ │ │ │ ├── package-summary.html │ │ │ │ │ ├── package-tree.html │ │ │ │ │ └── package-use.html │ │ │ ├── constant-values.html │ │ │ ├── deprecated-list.html │ │ │ ├── help-doc.html │ │ │ ├── index-all.html │ │ │ ├── index.html │ │ │ ├── overview-tree.html │ │ │ ├── package-list │ │ │ ├── script.js │ │ │ ├── serialized-form.html │ │ │ └── stylesheet.css │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── inputs │ │ │ └── Stdin.java │ └── tcp │ │ ├── .gitignore │ │ ├── jlogstash-java-docs │ │ ├── allclasses-frame.html │ │ ├── allclasses-noframe.html │ │ ├── com │ │ │ └── dtstack │ │ │ │ └── jlogstash │ │ │ │ └── inputs │ │ │ │ ├── Tcp.MinaBizHandler.html │ │ │ │ ├── Tcp.html │ │ │ │ ├── class-use │ │ │ │ ├── Tcp.MinaBizHandler.html │ │ │ │ └── Tcp.html │ │ │ │ ├── package-frame.html │ │ │ │ ├── package-summary.html │ │ │ │ ├── package-tree.html │ │ │ │ └── package-use.html │ │ ├── constant-values.html │ │ ├── deprecated-list.html │ │ ├── help-doc.html │ │ ├── index-all.html │ │ ├── index.html │ │ ├── overview-tree.html │ │ ├── package-list │ │ ├── script.js │ │ ├── serialized-form.html │ │ └── stylesheet.css │ │ ├── pom.xml │ │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── inputs │ │ │ └── Tcp.java │ │ └── test │ │ └── java │ │ └── com │ │ └── dtstack │ │ └── jogstash │ │ └── input │ │ ├── ClientMessageHandler.java │ │ └── MyClient.java └── outputs │ ├── README.md │ ├── elasticsearch │ ├── .gitignore │ ├── jlogstash-java-docs │ │ ├── allclasses-frame.html │ │ ├── allclasses-noframe.html │ │ ├── com │ │ │ └── dtstack │ │ │ │ └── jlogstash │ │ │ │ └── outputs │ │ │ │ ├── Elasticsearch.html │ │ │ │ ├── class-use │ │ │ │ └── Elasticsearch.html │ │ │ │ ├── package-frame.html │ │ │ │ ├── package-summary.html │ │ │ │ ├── package-tree.html │ │ │ │ └── package-use.html │ │ ├── constant-values.html │ │ ├── deprecated-list.html │ │ ├── help-doc.html │ │ ├── index-all.html │ │ ├── index.html │ │ ├── overview-tree.html │ │ ├── package-list │ │ ├── script.js │ │ ├── serialized-form.html │ │ └── stylesheet.css │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── dtstack │ │ └── jlogstash │ │ └── outputs │ │ └── Elasticsearch.java │ ├── elasticsearch5 │ ├── .gitignore │ ├── jlogstash-java-docs │ │ ├── allclasses-frame.html │ │ ├── allclasses-noframe.html │ │ ├── com │ │ │ └── dtstack │ │ │ │ └── jlogstash │ │ │ │ └── outputs │ │ │ │ ├── Elasticsearch5.html │ │ │ │ ├── class-use │ │ │ │ └── Elasticsearch5.html │ │ │ │ ├── package-frame.html │ │ │ │ ├── package-summary.html │ │ │ │ ├── package-tree.html │ │ │ │ └── package-use.html │ │ ├── constant-values.html │ │ ├── deprecated-list.html │ │ ├── help-doc.html │ │ ├── index-all.html │ │ ├── index.html │ │ ├── overview-tree.html │ │ ├── package-list │ │ ├── script.js │ │ ├── serialized-form.html │ │ └── stylesheet.css │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── outputs │ │ │ └── Elasticsearch5.java │ │ └── test │ │ └── java │ │ └── com │ │ └── dtstack │ │ └── jlogstash │ │ └── output │ │ └── Test.java │ ├── elasticsearch6 │ ├── .gitignore │ ├── jlogstash-java-docs │ │ ├── allclasses-frame.html │ │ ├── allclasses-noframe.html │ │ ├── com │ │ │ └── dtstack │ │ │ │ └── jlogstash │ │ │ │ └── outputs │ │ │ │ ├── Elasticsearch6.html │ │ │ │ ├── class-use │ │ │ │ └── Elasticsearch6.html │ │ │ │ ├── package-frame.html │ │ │ │ ├── package-summary.html │ │ │ │ ├── package-tree.html │ │ │ │ └── package-use.html │ │ ├── constant-values.html │ │ ├── deprecated-list.html │ │ ├── help-doc.html │ │ ├── index-all.html │ │ ├── index.html │ │ ├── overview-tree.html │ │ ├── package-list │ │ ├── script.js │ │ ├── serialized-form.html │ │ └── stylesheet.css │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── dtstack │ │ └── jlogstash │ │ └── outputs │ │ └── Elasticsearch6.java │ ├── file │ ├── .gitignore │ ├── jlogstash-java-docs │ │ ├── allclasses-frame.html │ │ ├── allclasses-noframe.html │ │ ├── com │ │ │ └── dtstack │ │ │ │ └── jlogstash │ │ │ │ └── outputs │ │ │ │ ├── File.html │ │ │ │ ├── class-use │ │ │ │ └── File.html │ │ │ │ ├── package-frame.html │ │ │ │ ├── package-summary.html │ │ │ │ ├── package-tree.html │ │ │ │ └── package-use.html │ │ ├── constant-values.html │ │ ├── deprecated-list.html │ │ ├── help-doc.html │ │ ├── index-all.html │ │ ├── index.html │ │ ├── overview-tree.html │ │ ├── package-list │ │ ├── script.js │ │ ├── serialized-form.html │ │ └── stylesheet.css │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── dtstack │ │ └── jlogstash │ │ └── outputs │ │ └── File.java │ ├── hdfs │ ├── .gitignore │ ├── jlogstash-java-docs │ │ ├── allclasses-frame.html │ │ ├── allclasses-noframe.html │ │ ├── com │ │ │ └── dtstack │ │ │ │ └── jlogstash │ │ │ │ ├── format │ │ │ │ ├── CompressEnum.html │ │ │ │ ├── HdfsOutputFormat.html │ │ │ │ ├── HdfsUtil.html │ │ │ │ ├── ModeEnum.html │ │ │ │ ├── OutputFormat.html │ │ │ │ ├── StoreEnum.html │ │ │ │ ├── class-use │ │ │ │ │ ├── CompressEnum.html │ │ │ │ │ ├── HdfsOutputFormat.html │ │ │ │ │ ├── HdfsUtil.html │ │ │ │ │ ├── ModeEnum.html │ │ │ │ │ ├── OutputFormat.html │ │ │ │ │ └── StoreEnum.html │ │ │ │ ├── package-frame.html │ │ │ │ ├── package-summary.html │ │ │ │ ├── package-tree.html │ │ │ │ ├── package-use.html │ │ │ │ └── util │ │ │ │ │ ├── ClassUtil.html │ │ │ │ │ ├── DateUtil.html │ │ │ │ │ ├── HostUtil.html │ │ │ │ │ ├── class-use │ │ │ │ │ ├── ClassUtil.html │ │ │ │ │ ├── DateUtil.html │ │ │ │ │ └── HostUtil.html │ │ │ │ │ ├── package-frame.html │ │ │ │ │ ├── package-summary.html │ │ │ │ │ ├── package-tree.html │ │ │ │ │ └── package-use.html │ │ │ │ └── outputs │ │ │ │ ├── Hdfs.html │ │ │ │ ├── class-use │ │ │ │ └── Hdfs.html │ │ │ │ ├── package-frame.html │ │ │ │ ├── package-summary.html │ │ │ │ ├── package-tree.html │ │ │ │ └── package-use.html │ │ ├── constant-values.html │ │ ├── deprecated-list.html │ │ ├── help-doc.html │ │ ├── index-all.html │ │ ├── index.html │ │ ├── org │ │ │ └── apache │ │ │ │ └── hadoop │ │ │ │ └── mapred │ │ │ │ ├── TextOutputFormatBak.LineRecordWriter.html │ │ │ │ ├── TextOutputFormatBak.html │ │ │ │ ├── class-use │ │ │ │ ├── TextOutputFormatBak.LineRecordWriter.html │ │ │ │ └── TextOutputFormatBak.html │ │ │ │ ├── package-frame.html │ │ │ │ ├── package-summary.html │ │ │ │ ├── package-tree.html │ │ │ │ └── package-use.html │ │ ├── overview-frame.html │ │ ├── overview-summary.html │ │ ├── overview-tree.html │ │ ├── package-list │ │ ├── script.js │ │ ├── serialized-form.html │ │ └── stylesheet.css │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ ├── com │ │ └── dtstack │ │ │ └── jlogstash │ │ │ ├── format │ │ │ ├── CompressEnum.java │ │ │ ├── HdfsOutputFormat.java │ │ │ ├── HdfsUtil.java │ │ │ ├── ModeEnum.java │ │ │ ├── OutputFormat.java │ │ │ ├── StoreEnum.java │ │ │ ├── plugin │ │ │ │ ├── HdfsOrcOutputFormat.java │ │ │ │ └── HdfsTextOutputFormat.java │ │ │ └── util │ │ │ │ ├── ClassUtil.java │ │ │ │ ├── DateUtil.java │ │ │ │ ├── HdfsConverter.java │ │ │ │ └── HostUtil.java │ │ │ └── outputs │ │ │ └── Hdfs.java │ │ └── org │ │ └── apache │ │ └── hadoop │ │ └── mapred │ │ └── TextOutputFormatBak.java │ ├── hive │ ├── .gitignore │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ ├── com │ │ │ └── dtstack │ │ │ │ └── jlogstash │ │ │ │ ├── format │ │ │ │ ├── CompressEnum.java │ │ │ │ ├── HiveOutputFormat.java │ │ │ │ ├── ModeEnum.java │ │ │ │ ├── OutputFormat.java │ │ │ │ ├── StoreEnum.java │ │ │ │ ├── TableInfo.java │ │ │ │ ├── TimePartitionFormat.java │ │ │ │ ├── dirty │ │ │ │ │ └── DirtyDataManager.java │ │ │ │ ├── plugin │ │ │ │ │ ├── HiveOrcOutputFormat.java │ │ │ │ │ └── HiveTextOutputFormat.java │ │ │ │ └── util │ │ │ │ │ ├── AddressUtil.java │ │ │ │ │ ├── DBUtil.java │ │ │ │ │ ├── DateUtil.java │ │ │ │ │ ├── HiveUtil.java │ │ │ │ │ ├── HostUtil.java │ │ │ │ │ ├── PathConverterUtil.java │ │ │ │ │ └── RetryUtil.java │ │ │ │ └── outputs │ │ │ │ └── Hive.java │ │ │ └── org │ │ │ └── apache │ │ │ ├── hadoop │ │ │ ├── hive │ │ │ │ └── ql │ │ │ │ │ └── io │ │ │ │ │ └── orc │ │ │ │ │ └── OrcOutputFormatBak.java │ │ │ └── mapred │ │ │ │ └── TextOutputFormatBak.java │ │ │ └── hive │ │ │ └── jdbc │ │ │ └── HiveConnection.java │ │ └── test │ │ └── java │ │ └── com │ │ └── dtstack │ │ └── jlogstash │ │ └── TestHive.java │ ├── kafka │ ├── .gitignore │ ├── jlogstash-java-docs │ │ ├── allclasses-frame.html │ │ ├── allclasses-noframe.html │ │ ├── com │ │ │ └── dtstack │ │ │ │ └── jlogstash │ │ │ │ └── outputs │ │ │ │ ├── Kafka.html │ │ │ │ ├── class-use │ │ │ │ └── Kafka.html │ │ │ │ ├── package-frame.html │ │ │ │ ├── package-summary.html │ │ │ │ ├── package-tree.html │ │ │ │ └── package-use.html │ │ ├── constant-values.html │ │ ├── deprecated-list.html │ │ ├── help-doc.html │ │ ├── index-all.html │ │ ├── index.html │ │ ├── overview-tree.html │ │ ├── package-list │ │ ├── script.js │ │ ├── serialized-form.html │ │ └── stylesheet.css │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── dtstack │ │ └── jlogstash │ │ └── outputs │ │ └── Kafka.java │ ├── kafka10 │ ├── .gitignore │ ├── jlogstash-java-docs │ │ ├── allclasses-frame.html │ │ ├── allclasses-noframe.html │ │ ├── com │ │ │ └── dtstack │ │ │ │ └── jlogstash │ │ │ │ └── outputs │ │ │ │ ├── Kafka10.html │ │ │ │ ├── class-use │ │ │ │ └── Kafka10.html │ │ │ │ ├── kafka │ │ │ │ ├── JKafkaProducer.html │ │ │ │ ├── MonitorCluster.html │ │ │ │ ├── NamedThreadFactory.html │ │ │ │ ├── class-use │ │ │ │ │ ├── JKafkaProducer.html │ │ │ │ │ ├── MonitorCluster.html │ │ │ │ │ └── NamedThreadFactory.html │ │ │ │ ├── package-frame.html │ │ │ │ ├── package-summary.html │ │ │ │ ├── package-tree.html │ │ │ │ └── package-use.html │ │ │ │ ├── package-frame.html │ │ │ │ ├── package-summary.html │ │ │ │ ├── package-tree.html │ │ │ │ └── package-use.html │ │ ├── constant-values.html │ │ ├── deprecated-list.html │ │ ├── help-doc.html │ │ ├── index-all.html │ │ ├── index.html │ │ ├── overview-frame.html │ │ ├── overview-summary.html │ │ ├── overview-tree.html │ │ ├── package-list │ │ ├── script.js │ │ ├── serialized-form.html │ │ └── stylesheet.css │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── dtstack │ │ └── jlogstash │ │ └── outputs │ │ ├── Kafka10.java │ │ └── kafka │ │ ├── JKafkaProducer.java │ │ ├── MonitorCluster.java │ │ └── NamedThreadFactory.java │ ├── kafka11 │ ├── .gitignore │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── dtstack │ │ └── jlogstash │ │ └── outputs │ │ ├── Kafka11.java │ │ └── kafka │ │ ├── JKafkaProducer.java │ │ ├── MonitorCluster.java │ │ └── NamedThreadFactory.java │ ├── netty │ ├── .gitignore │ ├── jlogstash-java-docs │ │ ├── allclasses-frame.html │ │ ├── allclasses-noframe.html │ │ ├── com │ │ │ └── dtstack │ │ │ │ └── jlogstash │ │ │ │ └── outputs │ │ │ │ ├── Netty.html │ │ │ │ ├── class-use │ │ │ │ └── Netty.html │ │ │ │ ├── package-frame.html │ │ │ │ ├── package-summary.html │ │ │ │ ├── package-tree.html │ │ │ │ ├── package-use.html │ │ │ │ └── util │ │ │ │ ├── LocalIpAddressUtil.html │ │ │ │ ├── class-use │ │ │ │ └── LocalIpAddressUtil.html │ │ │ │ ├── flow │ │ │ │ ├── FlowControlShiper.html │ │ │ │ ├── Threshold.html │ │ │ │ ├── class-use │ │ │ │ │ ├── FlowControlShiper.html │ │ │ │ │ └── Threshold.html │ │ │ │ ├── package-frame.html │ │ │ │ ├── package-summary.html │ │ │ │ ├── package-tree.html │ │ │ │ └── package-use.html │ │ │ │ ├── package-frame.html │ │ │ │ ├── package-summary.html │ │ │ │ ├── package-tree.html │ │ │ │ └── package-use.html │ │ ├── constant-values.html │ │ ├── deprecated-list.html │ │ ├── help-doc.html │ │ ├── index-all.html │ │ ├── index.html │ │ ├── overview-frame.html │ │ ├── overview-summary.html │ │ ├── overview-tree.html │ │ ├── package-list │ │ ├── script.js │ │ ├── serialized-form.html │ │ └── stylesheet.css │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── dtstack │ │ └── jlogstash │ │ └── outputs │ │ ├── Netty.java │ │ └── util │ │ ├── LocalIpAddressUtil.java │ │ └── flow │ │ ├── FlowControlShiper.java │ │ └── Threshold.java │ ├── odps │ ├── .gitignore │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── dtstack │ │ └── jlogstash │ │ └── outputs │ │ └── OutOdps.java │ ├── performance │ ├── .gitignore │ ├── jlogstash-java-docs │ │ ├── allclasses-frame.html │ │ ├── allclasses-noframe.html │ │ ├── com │ │ │ └── dtstack │ │ │ │ └── jlogstash │ │ │ │ └── outputs │ │ │ │ ├── Performance.html │ │ │ │ ├── class-use │ │ │ │ └── Performance.html │ │ │ │ ├── package-frame.html │ │ │ │ ├── package-summary.html │ │ │ │ ├── package-tree.html │ │ │ │ └── package-use.html │ │ ├── constant-values.html │ │ ├── deprecated-list.html │ │ ├── help-doc.html │ │ ├── index-all.html │ │ ├── index.html │ │ ├── overview-tree.html │ │ ├── package-list │ │ ├── script.js │ │ ├── serialized-form.html │ │ └── stylesheet.css │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── dtstack │ │ └── jlogstash │ │ └── outputs │ │ └── Performance.java │ ├── pom.xml │ └── stdout │ ├── .gitignore │ ├── jlogstash-java-docs │ ├── allclasses-frame.html │ ├── allclasses-noframe.html │ ├── com │ │ └── dtstack │ │ │ └── jlogstash │ │ │ └── outputs │ │ │ ├── Stdout.html │ │ │ ├── class-use │ │ │ └── Stdout.html │ │ │ ├── package-frame.html │ │ │ ├── package-summary.html │ │ │ ├── package-tree.html │ │ │ └── package-use.html │ ├── constant-values.html │ ├── deprecated-list.html │ ├── help-doc.html │ ├── index-all.html │ ├── index.html │ ├── overview-tree.html │ ├── package-list │ ├── script.js │ ├── serialized-form.html │ └── stylesheet.css │ ├── pom.xml │ └── src │ └── main │ └── java │ └── com │ └── dtstack │ └── jlogstash │ └── outputs │ └── Stdout.java ├── plugin ├── README.MD ├── filter │ └── README.MD ├── input │ └── README.MD └── output │ └── README.MD ├── pom.xml ├── stylesheets ├── github-light.css ├── normalize.css └── stylesheet.css └── test ├── binlog_hdfs_orc.json ├── binlog_hdfs_text.json ├── binlog_hive_orc.json ├── binlog_hive_orc2.json └── binlog_hive_text.json /.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | .idea/ 8 | /plugin/ 9 | *.iml 10 | RemoteSystemsTempFiles/ 11 | logs/ 12 | conf/ 13 | dependency-reduced-pom.xml 14 | jlogstash.iml 15 | lib/ 16 | *_output 17 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | 技术架构模型调整 2 | -------------------------------------------------------------------------------- /bin/jlogstash.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | HO_HEAP_SIZE="${HO_HEAP_SIZE:=300m}" 4 | 5 | unset CDPATH 6 | export basedir=$(cd `dirname $0`/..; pwd) 7 | 8 | JAVA_OPTS="$JAVA_OPTS -Xmx${HO_HEAP_SIZE}" 9 | 10 | JAVA_OPTS="$JAVA_OPTS -Xms${HO_HEAP_SIZE}" 11 | 12 | JAVA_OPTS="$JAVA_OPTS -server" 13 | 14 | JAVA_OPTS="$JAVA_OPTS -Xloggc:../logs/jlogstash.gc" 15 | 16 | JAVA_OPTS="$JAVA_OPTS -XX:HeapDumpPath=../logs/heapdump.hprof" 17 | 18 | 19 | #-XX:MaxDirectMemorySize=16M According to owner memory 20 | JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+HeapDumpOnOutOfMemoryError -XX:+DisableExplicitGC -Dfile.encoding=UTF-8 -Djna.nosys=true -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps" 21 | 22 | #Comment to speed up starting time 23 | #JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" 24 | 25 | exec java $JAVA_OPTS -cp $basedir/lib/* com.dtstack.jlogstash.JlogstashMain "$@" 26 | -------------------------------------------------------------------------------- /core/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | .idea/ 8 | plugin/ 9 | *.iml 10 | RemoteSystemsTempFiles/ 11 | logs/ 12 | conf/ 13 | dependency-reduced-pom.xml 14 | jlogstash.iml 15 | lib/ 16 | 17 | !/src/main/java/com/dtstack/jlogstash/annotation/plugin/ 18 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/annotation/Path.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.annotation; 19 | 20 | import java.lang.annotation.ElementType; 21 | import java.lang.annotation.Retention; 22 | import java.lang.annotation.RetentionPolicy; 23 | import java.lang.annotation.Target; 24 | 25 | /** 26 | * 27 | * Reason: TODO ADD REASON(可选) 28 | * Date: 2016年8月31日 下午1:24:34 29 | * Company: www.dtstack.com 30 | * @author sishu.yss 31 | * 32 | */ 33 | @Target({ElementType.FIELD}) 34 | @Retention(RetentionPolicy.RUNTIME) 35 | public @interface Path { 36 | 37 | } 38 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/annotation/Required.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.annotation; 19 | 20 | import java.lang.annotation.ElementType; 21 | import java.lang.annotation.Retention; 22 | import java.lang.annotation.RetentionPolicy; 23 | import java.lang.annotation.Target; 24 | 25 | /** 26 | * 27 | * Reason: TODO ADD REASON(可选) 28 | * Date: 2016年8月31日 下午1:24:39 29 | * Company: www.dtstack.com 30 | * @author sishu.yss 31 | * 32 | */ 33 | @Target({ElementType.FIELD}) 34 | @Retention(RetentionPolicy.RUNTIME) 35 | public @interface Required { 36 | boolean required() default false; 37 | } 38 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/annotation/Url.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.annotation; 19 | 20 | import java.lang.annotation.ElementType; 21 | import java.lang.annotation.Retention; 22 | import java.lang.annotation.RetentionPolicy; 23 | import java.lang.annotation.Target; 24 | 25 | 26 | /** 27 | * 28 | * Reason: TODO ADD REASON(可选) 29 | * Date: 2016年9月07日 下午1:24:57 30 | * Company: www.dtstack.com 31 | * @author sishu.yss 32 | * 33 | */ 34 | @Target({ElementType.FIELD}) 35 | @Retention(RetentionPolicy.RUNTIME) 36 | public @interface Url { 37 | 38 | } 39 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/annotation/plugin/AnnotationInterface.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.annotation.plugin; 2 | 3 | import java.lang.reflect.Field; 4 | 5 | 6 | /** 7 | * 8 | * Reason: TODO ADD REASON(可选) 9 | * Date: 2016年8月31日 下午1:24:50 10 | * Company: www.dtstack.com 11 | * @author sishu.yss 12 | * 13 | */ 14 | public interface AnnotationInterface { 15 | 16 | public void required(Field field,Object obj) throws Exception; 17 | 18 | } -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/annotation/plugin/RequiredPlugin.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.annotation.plugin; 19 | 20 | import java.lang.reflect.Field; 21 | 22 | import com.dtstack.jlogstash.annotation.Required; 23 | import com.dtstack.jlogstash.exception.RequiredException; 24 | 25 | /** 26 | * 27 | * Reason: TODO ADD REASON(可选) 28 | * Date: 2016年8月31日 下午1:25:03 29 | * Company: www.dtstack.com 30 | * @author sishu.yss 31 | * 32 | */ 33 | public class RequiredPlugin implements AnnotationInterface{ 34 | 35 | @Override 36 | public void required(Field field,Object obj) throws Exception{ 37 | // TODO Auto-generated method stub 38 | Required required = field.getAnnotation(Required.class); 39 | if(required.required()&&obj==null){ 40 | throw new RequiredException(field.getName()); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/callback/ClassLoaderCallBack.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.callback; 2 | 3 | /** 4 | * Created by sishu.yss on 2017/8/28. 5 | */ 6 | @FunctionalInterface 7 | public interface ClassLoaderCallBack { 8 | 9 | T execute() throws Exception; 10 | 11 | } 12 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/callback/ClassLoaderCallBackMethod.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.callback; 2 | 3 | /** 4 | * Created by sishu.yss on 2017/8/28. 5 | */ 6 | public class ClassLoaderCallBackMethod { 7 | 8 | public static M callbackAndReset(ClassLoaderCallBack classLoaderCallBack, ClassLoader toSetClassLoader, boolean reset) throws Exception { 9 | 10 | ClassLoader oldClassLoader = Thread.currentThread().getContextClassLoader(); 11 | Thread.currentThread().setContextClassLoader(toSetClassLoader); 12 | M result = classLoaderCallBack.execute(); 13 | if(reset){ 14 | Thread.currentThread().setContextClassLoader(oldClassLoader); 15 | } 16 | return result; 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/configs/Config.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.configs; 19 | 20 | /** 21 | * 22 | * Reason: TODO ADD REASON(可选) 23 | * Date: 2016年8月31日 下午1:25:40 24 | * Company: www.dtstack.com 25 | * @author sishu.yss 26 | * 27 | */ 28 | public interface Config { 29 | 30 | public ConfigObject parse(String conf)throws Exception; 31 | 32 | } 33 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/date/DateParser.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.date; 19 | 20 | import org.joda.time.DateTime; 21 | 22 | 23 | /** 24 | * 25 | * Reason: TODO ADD REASON(可选) 26 | * Date: 2016年8月31日 下午1:28:23 27 | * Company: www.dtstack.com 28 | * @author sishu.yss 29 | * 30 | */ 31 | public interface DateParser { 32 | public DateTime parse(String input); 33 | } 34 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/date/UnixMSParser.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.date; 19 | 20 | import org.joda.time.DateTime; 21 | 22 | 23 | /** 24 | * 25 | * Reason: TODO ADD REASON(可选) 26 | * Date: 2016年8月31日 下午1:28:47 27 | * Company: www.dtstack.com 28 | * @author sishu.yss 29 | * 30 | */ 31 | public class UnixMSParser implements DateParser { 32 | 33 | @Override 34 | public DateTime parse(String input) { 35 | // TODO Auto-generated method stub 36 | return new DateTime(Long.parseLong(input)); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/date/UnixParser.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.date; 19 | 20 | import org.joda.time.DateTime; 21 | 22 | /** 23 | * 24 | * Reason: TODO ADD REASON(可选) 25 | * Date: 2016年8月31日 下午1:28:51 26 | * Company: www.dtstack.com 27 | * @author sishu.yss 28 | * 29 | */ 30 | public class UnixParser implements DateParser { 31 | 32 | @Override 33 | public DateTime parse(String input) { 34 | return new DateTime((long) (Double.parseDouble(input) * 1000)); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/decoder/IDecode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.decoder; 19 | 20 | import java.util.Map; 21 | 22 | /** 23 | * 24 | * Reason: TODO ADD REASON(可选) 25 | * Date: 2016年8月31日 下午1:25:51 26 | * Company: www.dtstack.com 27 | * @author sishu.yss 28 | * 29 | */ 30 | public interface IDecode { 31 | 32 | public Map decode(String message); 33 | 34 | public Map decode(String message, String identify); 35 | 36 | } 37 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/decoder/PlainDecoder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.decoder; 19 | 20 | import java.util.HashMap; 21 | import java.util.Map; 22 | 23 | 24 | /** 25 | * 26 | * Reason: TODO ADD REASON(可选) 27 | * Date: 2016年8月31日 下午1:26:00 28 | * Company: www.dtstack.com 29 | * @author sishu.yss 30 | * 31 | */ 32 | public class PlainDecoder implements IDecode { 33 | 34 | @SuppressWarnings("serial") 35 | @Override 36 | public Map decode(final String message) { 37 | HashMap event = new HashMap() { 38 | { 39 | put("message", message); 40 | } 41 | }; 42 | return event; 43 | } 44 | 45 | @Override 46 | public Map decode(String message, String identify) { 47 | // TODO Auto-generated method stub 48 | return null; 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/exception/LogstashException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.exception; 19 | 20 | /** 21 | * 22 | * Reason: TODO ADD REASON(可选) 23 | * Date: 2016年8月31日 下午1:26:07 24 | * Company: www.dtstack.com 25 | * @author sishu.yss 26 | * 27 | */ 28 | @SuppressWarnings("serial") 29 | public class LogstashException extends Exception{ 30 | 31 | public LogstashException(String message){ 32 | super(message); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/exception/PathException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.exception; 19 | 20 | /** 21 | * 22 | * Reason: TODO ADD REASON(可选) 23 | * Date: 2016年8月31日 下午1:26:12 24 | * Company: www.dtstack.com 25 | * @author sishu.yss 26 | * 27 | */ 28 | @SuppressWarnings("serial") 29 | public class PathException extends LogstashException{ 30 | 31 | private static String message = "Path: %s format is error"; 32 | 33 | public PathException(String name){ 34 | super(String.format(message, name)); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/exception/RequiredException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.exception; 19 | 20 | 21 | /** 22 | * 23 | * Reason: TODO ADD REASON(可选) 24 | * Date: 2016年8月31日 下午1:26:18 25 | * Company: www.dtstack.com 26 | * @author sishu.yss 27 | * 28 | */ 29 | @SuppressWarnings("serial") 30 | public class RequiredException extends LogstashException{ 31 | 32 | private static String message = "param: %s is not empty"; 33 | 34 | public RequiredException(String param){ 35 | super(String.format(message, param)); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/exception/UrlException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.exception; 19 | 20 | 21 | /** 22 | * 23 | * Reason: TODO ADD REASON(可选) 24 | * Date: 2016年9月07日 下午1:24:57 25 | * Company: www.dtstack.com 26 | * @author sishu.yss 27 | * 28 | */ 29 | @SuppressWarnings("serial") 30 | public class UrlException extends LogstashException{ 31 | 32 | private static String message = "Url: %s format is error"; 33 | 34 | 35 | public UrlException(String name) { 36 | // TODO Auto-generated constructor stub 37 | super(String.format(message, name)); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/filters/IBaseFilter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.filters; 19 | 20 | import java.util.Map; 21 | 22 | /** 23 | * company: www.dtstack.com 24 | * author: toutian 25 | * create: 2019/5/29 26 | */ 27 | public interface IBaseFilter extends Cloneable { 28 | 29 | void prepare(); 30 | 31 | Map process(Map event); 32 | 33 | void postProcess(Map event, boolean ifSuccess); 34 | } 35 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/inputs/IBaseInput.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.inputs; 19 | 20 | /** 21 | * company: www.dtstack.com 22 | * author: toutian 23 | * create: 2019/5/29 24 | */ 25 | public interface IBaseInput extends Cloneable { 26 | 27 | void prepare(); 28 | 29 | void emit(); 30 | 31 | void release(); 32 | } 33 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/log/LogComponent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.log; 19 | 20 | import org.apache.commons.lang3.StringUtils; 21 | import com.dtstack.jlogstash.assembly.CmdLineParams; 22 | 23 | /** 24 | * 25 | * Reason: TODO ADD REASON(可选) 26 | * Date: 2016年8月31日 下午1:27:21 27 | * Company: www.dtstack.com 28 | * @author sishu.yss 29 | * 30 | */ 31 | public abstract class LogComponent { 32 | 33 | public void setupLogger() {} 34 | 35 | protected String checkFile(){ 36 | String logfile = CmdLineParams.getLogFilePath(); 37 | if(StringUtils.isBlank(logfile)){ 38 | return String.format("%s/%s", System.getProperty("user.dir"),"logs/jlogstash.log"); 39 | } 40 | return logfile; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/metrics/base/CharacterFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.jlogstash.metrics.base; 20 | 21 | /** 22 | * copy from https://github.com/apache/flink 23 | * 24 | * Interface for a character filter function. The filter function is given a string which the filter 25 | * can transform. The returned string is the transformation result. 26 | */ 27 | public interface CharacterFilter { 28 | 29 | /** 30 | * Filter the given string and generate a resulting string from it. 31 | * 32 | *

For example, one implementation could filter out invalid characters from the input string. 33 | * 34 | * @param input Input string 35 | * @return Filtered result string 36 | */ 37 | String filterCharacters(String input); 38 | } 39 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/metrics/base/Counter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.jlogstash.metrics.base; 20 | 21 | /** 22 | * copy from https://github.com/apache/flink 23 | * 24 | * A Counter is a {@link Metric} that measures a count. 25 | */ 26 | public interface Counter extends Metric { 27 | 28 | /** 29 | * Increment the current count by 1. 30 | */ 31 | void inc(); 32 | 33 | /** 34 | * Increment the current count by the given value. 35 | * 36 | * @param n value to increment the current count by 37 | */ 38 | void inc(long n); 39 | 40 | /** 41 | * Decrement the current count by 1. 42 | */ 43 | void dec(); 44 | 45 | /** 46 | * Decrement the current count by the given value. 47 | * 48 | * @param n value to decrement the current count by 49 | */ 50 | void dec(long n); 51 | 52 | /** 53 | * Returns the current count. 54 | * 55 | * @return current count 56 | */ 57 | long getCount(); 58 | } 59 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/metrics/base/Gauge.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.jlogstash.metrics.base; 20 | 21 | /** 22 | * copy from https://github.com/apache/flink 23 | * 24 | * A Gauge is a {@link Metric} that calculates a specific value at a point in time. 25 | */ 26 | public interface Gauge extends Metric { 27 | 28 | /** 29 | * Calculates and returns the measured value. 30 | * 31 | * @return calculated value 32 | */ 33 | T getValue(); 34 | } 35 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/metrics/base/Meter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.jlogstash.metrics.base; 20 | 21 | /** 22 | * copy from https://github.com/apache/flink 23 | * 24 | * Metric for measuring throughput. 25 | */ 26 | public interface Meter extends Metric { 27 | 28 | /** 29 | * Mark occurrence of an event. 30 | */ 31 | void markEvent(); 32 | 33 | /** 34 | * Mark occurrence of multiple events. 35 | * 36 | * @param n number of events occurred 37 | */ 38 | void markEvent(long n); 39 | 40 | /** 41 | * Returns the current rate of events per second. 42 | * 43 | * @return current rate of events per second 44 | */ 45 | double getRate(); 46 | 47 | /** 48 | * Get number of events marked on the meter. 49 | * 50 | * @return number of events marked on the meter 51 | */ 52 | long getCount(); 53 | } 54 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/metrics/base/Metric.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.jlogstash.metrics.base; 20 | 21 | /** 22 | * 23 | * copy from https://github.com/apache/flink 24 | * 25 | * Common super interface for all metrics. 26 | */ 27 | public interface Metric { 28 | } 29 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/metrics/base/View.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.jlogstash.metrics.base; 20 | 21 | /** 22 | * 23 | * copy from https://github.com/apache/flink 24 | * 25 | * An interface for metrics which should be updated in regular intervals by a background thread. 26 | */ 27 | public interface View { 28 | /** The interval in which metrics are updated. */ 29 | int UPDATE_INTERVAL_SECONDS = 5; 30 | 31 | /** 32 | * This method will be called regularly to update the metric. 33 | */ 34 | void update(); 35 | } 36 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/metrics/base/reporter/Scheduled.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.jlogstash.metrics.base.reporter; 20 | 21 | /** 22 | * copy from https://github.com/apache/flink 23 | * 24 | * Interface for reporters that actively send out data periodically. 25 | */ 26 | public interface Scheduled { 27 | 28 | /** 29 | * Report the current measurements. This method is called periodically by the 30 | * metrics registry that uses the reporter. 31 | */ 32 | void report(); 33 | } 34 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/metrics/scope/JlogstashJobScopeFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.jlogstash.metrics.scope; 20 | 21 | public class JlogstashJobScopeFormat extends ScopeFormat { 22 | 23 | public JlogstashJobScopeFormat(String format) { 24 | super(format, null, new String[] { 25 | SCOPE_HOST, 26 | SCOPE_JOB_NAME 27 | }); 28 | } 29 | 30 | public String[] formatScope(String hostname, String jobName) { 31 | final String[] template = copyTemplate(); 32 | final String[] values = { hostname, jobName}; 33 | return bindVariables(template, values); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/metrics/scope/PipelineScopeFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.jlogstash.metrics.scope; 20 | 21 | 22 | public class PipelineScopeFormat extends ScopeFormat { 23 | 24 | public PipelineScopeFormat(String format) { 25 | super(format, null, new String[]{ 26 | SCOPE_HOST, 27 | SCOPE_PLUGINE_TYPE, 28 | SCOPE_PLUGINE_NAME, 29 | SCOPE_JOB_NAME 30 | }); 31 | } 32 | 33 | public String[] formatScope(String hostname, String pluginType, String pluginName, String jobName) { 34 | final String[] template = copyTemplate(); 35 | final String[] values = {hostname, pluginType, pluginName, jobName}; 36 | return bindVariables(template, values); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/outputs/IBaseOutput.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.jlogstash.outputs; 20 | 21 | import java.util.Map; 22 | import java.util.concurrent.atomic.AtomicInteger; 23 | 24 | /** 25 | * company: www.dtstack.com 26 | * author: toutian 27 | * create: 2019/5/29 28 | */ 29 | public interface IBaseOutput extends Cloneable { 30 | 31 | void prepare(); 32 | 33 | void process(Map event); 34 | 35 | void release(); 36 | 37 | 38 | AtomicInteger getAto(); 39 | 40 | boolean isConsistency(); 41 | 42 | boolean dealFailedMsg(); 43 | 44 | void addFailedMsg(Object msg); 45 | 46 | void sendFailedMsg(Object msg); 47 | 48 | } 49 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/render/TemplateRender.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.render; 19 | 20 | import java.util.Map; 21 | 22 | 23 | /** 24 | * 25 | *copy from hangout(https://github.com/childe/hangout) 26 | * 27 | */ 28 | public abstract class TemplateRender { 29 | 30 | public abstract String render(Map event); 31 | 32 | public abstract String render(String template, Map event); 33 | } 34 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/jlogstash/utils/Package.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.utils; 19 | 20 | import org.apache.commons.lang3.StringUtils; 21 | 22 | import com.dtstack.jlogstash.property.SystemProperty; 23 | 24 | 25 | /** 26 | * 27 | * Reason: TODO ADD REASON(可选) 28 | * Date: 2016年8月31日 下午1:28:43 29 | * Company: www.dtstack.com 30 | * @author sishu.yss 31 | * 32 | */ 33 | public class Package { 34 | private static String point ="."; 35 | 36 | public static String getRealClassName(String name,String key){ 37 | if(StringUtils.isBlank(name)){ 38 | return null; 39 | } 40 | if(name.indexOf(point)>=0){ 41 | return name; 42 | } 43 | return SystemProperty.getSystemProperty(key)+point+name; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /core/src/main/resources/patterns/java: -------------------------------------------------------------------------------- 1 | JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$_]+ 2 | JAVAFILE (?:[A-Za-z0-9_. -]+) 3 | JAVASTACKTRACEPART at %{JAVACLASS:class}\.%{WORD:method}\(%{JAVAFILE:file}:%{NUMBER:line}\) 4 | -------------------------------------------------------------------------------- /core/src/main/resources/patterns/junos: -------------------------------------------------------------------------------- 1 | # JUNOS 11.4 RT_FLOW patterns 2 | RT_FLOW_EVENT (RT_FLOW_SESSION_CREATE|RT_FLOW_SESSION_CLOSE|RT_FLOW_SESSION_DENY) 3 | 4 | RT_FLOW1 %{RT_FLOW_EVENT:event}: %{GREEDYDATA:close-reason}: %{IP:src-ip}/%{DATA:src-port}->%{IP:dst-ip}/%{DATA:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{DATA:nat-src-port}->%{IP:nat-dst-ip}/%{DATA:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} \d+\(%{DATA:sent}\) \d+\(%{DATA:received}\) %{INT:elapsed-time} .* 5 | 6 | RT_FLOW2 %{RT_FLOW_EVENT:event}: session created %{IP:src-ip}/%{DATA:src-port}->%{IP:dst-ip}/%{DATA:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{DATA:nat-src-port}->%{IP:nat-dst-ip}/%{DATA:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} .* 7 | 8 | RT_FLOW3 %{RT_FLOW_EVENT:event}: session denied %{IP:src-ip}/%{DATA:src-port}->%{IP:dst-ip}/%{DATA:dst-port} %{DATA:service} %{INT:protocol-id}\(\d\) %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} .* 9 | 10 | -------------------------------------------------------------------------------- /core/src/main/resources/patterns/linux-syslog: -------------------------------------------------------------------------------- 1 | SYSLOGBASE2 (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: 2 | SYSLOGPAMSESSION %{SYSLOGBASE} (?=%{GREEDYDATA:message})%{WORD:pam_module}\(%{DATA:pam_caller}\): session %{WORD:pam_session_state} for user %{USERNAME:username}(?: by %{GREEDYDATA:pam_by})? 3 | 4 | CRON_ACTION [A-Z ]+ 5 | CRONLOG %{SYSLOGBASE} \(%{USER:user}\) %{CRON_ACTION:action} \(%{DATA:message}\) 6 | 7 | SYSLOGLINE %{SYSLOGBASE2} %{GREEDYDATA:message} 8 | 9 | # IETF 5424 syslog(8) format (see http://www.rfc-editor.org/info/rfc5424) 10 | SYSLOG5424PRI <%{NONNEGINT:syslog5424_pri}> 11 | SYSLOG5424SD \[%{DATA}\]+ 12 | SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{HOSTNAME:syslog5424_host}|-) +(?:%{WORD:syslog5424_app}|-) +(?:%{WORD:syslog5424_proc}|-) +(?:%{WORD:syslog5424_msgid}|-) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|) 13 | 14 | SYSLOG5424LINE %{SYSLOG5424BASE} +%{GREEDYDATA:syslog5424_msg} 15 | -------------------------------------------------------------------------------- /core/src/main/resources/patterns/mcollective: -------------------------------------------------------------------------------- 1 | MCOLLECTIVEAUDIT %{TIMESTAMP_ISO8601:timestamp}: 2 | -------------------------------------------------------------------------------- /core/src/main/resources/patterns/mcollective-patterns: -------------------------------------------------------------------------------- 1 | # Remember, these can be multi-line events. 2 | MCOLLECTIVE ., \[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:pid}\]%{SPACE}%{LOGLEVEL:event_level} 3 | 4 | MCOLLECTIVEAUDIT %{TIMESTAMP_ISO8601:timestamp}: 5 | -------------------------------------------------------------------------------- /core/src/main/resources/patterns/mongodb: -------------------------------------------------------------------------------- 1 | MONGO_LOG %{SYSLOGTIMESTAMP:timestamp} \[%{WORD:component}\] %{GREEDYDATA:message} 2 | MONGO_QUERY \{ (?<={ ).*(?= } ntoreturn:) \} 3 | MONGO_SLOWQUERY %{WORD} %{MONGO_WORDDASH:database}\.%{MONGO_WORDDASH:collection} %{WORD}: %{MONGO_QUERY:query} %{WORD}:%{NONNEGINT:ntoreturn} %{WORD}:%{NONNEGINT:ntoskip} %{WORD}:%{NONNEGINT:nscanned}.*nreturned:%{NONNEGINT:nreturned}..+ (?[0-9]+)ms 4 | MONGO_WORDDASH \b[\w-]+\b 5 | -------------------------------------------------------------------------------- /core/src/main/resources/patterns/postgresql: -------------------------------------------------------------------------------- 1 | # Default postgresql pg_log format pattern 2 | POSTGRESQL %{DATESTAMP:timestamp} %{TZ} %{DATA:user_id} %{GREEDYDATA:connection_id} %{POSINT:pid} 3 | 4 | -------------------------------------------------------------------------------- /core/src/main/resources/patterns/redis: -------------------------------------------------------------------------------- 1 | REDISTIMESTAMP %{MONTHDAY} %{MONTH} %{TIME} 2 | REDISLOG \[%{POSINT:pid}\] %{REDISTIMESTAMP:timestamp} \* 3 | 4 | -------------------------------------------------------------------------------- /core/src/main/resources/patterns/ruby: -------------------------------------------------------------------------------- 1 | RUBY_LOGLEVEL (?:DEBUG|FATAL|ERROR|WARN|INFO) 2 | RUBY_LOGGER [DFEWI], \[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:pid}\] *%{RUBY_LOGLEVEL:loglevel} -- +%{DATA:progname}: %{GREEDYDATA:message} 3 | -------------------------------------------------------------------------------- /core/src/test/java/com/dtstack/jlogstash/Test.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash; 2 | 3 | import java.io.IOException; 4 | 5 | /** 6 | * Created by sishu.yss on 2018/11/17. 7 | */ 8 | public class Test { 9 | 10 | public static void main(String[] args) throws IOException { 11 | String[] cmd = { "sh", "-c", "ls > /Users/sishuyss/1.txt" }; 12 | Runtime.getRuntime().exec(cmd); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /core/src/test/java/com/dtstack/jlogstash/TestFilterPlugin.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash; 2 | 3 | import java.util.Map; 4 | 5 | import com.dtstack.jlogstash.filters.BaseFilter; 6 | 7 | public class TestFilterPlugin extends BaseFilter{ 8 | 9 | public TestFilterPlugin(Map config) { 10 | super(config); 11 | // TODO Auto-generated constructor stub 12 | } 13 | 14 | /** 15 | * 16 | */ 17 | private static final long serialVersionUID = 1L; 18 | 19 | /** 20 | * 插件初始化准备工作 21 | */ 22 | @Override 23 | public void prepare() { 24 | // TODO Auto-generated method stub 25 | 26 | } 27 | 28 | /*** 29 | * event 事件过滤工作 30 | */ 31 | @Override 32 | protected Map filter(Map event) { 33 | // TODO Auto-generated method stub 34 | return null; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /core/src/test/java/com/dtstack/jlogstash/TestInputPlugin.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash; 2 | 3 | import java.util.Map; 4 | 5 | import com.dtstack.jlogstash.inputs.BaseInput; 6 | 7 | public class TestInputPlugin extends BaseInput{ 8 | 9 | public TestInputPlugin(Map config) { 10 | super(config); 11 | // TODO Auto-generated constructor stub 12 | } 13 | 14 | /** 15 | * 16 | */ 17 | private static final long serialVersionUID = 1L; 18 | 19 | /** 20 | * 插件初始胡准备工作 21 | */ 22 | @Override 23 | public void prepare() { 24 | // TODO Auto-generated method stub 25 | 26 | } 27 | 28 | /** 29 | * event事件处理工作 30 | */ 31 | @Override 32 | public void emit() { 33 | // TODO Auto-generated method stub 34 | 35 | } 36 | 37 | /** 38 | * jvm 退出资源释放工作 39 | */ 40 | @Override 41 | public void release() { 42 | // TODO Auto-generated method stub 43 | 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /core/src/test/java/com/dtstack/jlogstash/TestOutputPlugin.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash; 2 | 3 | import java.util.Map; 4 | 5 | import com.dtstack.jlogstash.outputs.BaseOutput; 6 | 7 | 8 | /** 9 | * output plugin 样列 10 | * @author sishu.yss 11 | * 12 | */ 13 | public class TestOutputPlugin extends BaseOutput{ 14 | 15 | public TestOutputPlugin(Map config) { 16 | super(config); 17 | // TODO Auto-generated constructor stub 18 | } 19 | 20 | /** 21 | * 22 | */ 23 | private static final long serialVersionUID = 1L; 24 | 25 | /** 26 | * 插件初始化准备工作 27 | */ 28 | @Override 29 | public void prepare() { 30 | // TODO Auto-generated method stub 31 | 32 | } 33 | 34 | /** 35 | * 事件逻辑处理 36 | */ 37 | @Override 38 | protected void emit(Map event) { 39 | // TODO Auto-generated method stub 40 | 41 | } 42 | 43 | /** 44 | * jvm 退出时插件所有释放的资源 45 | */ 46 | @Override 47 | public void release(){} 48 | 49 | /** 50 | * 数据失败时候处理的逻辑 51 | */ 52 | @Override 53 | public void sendFailedMsg(Object msg){} 54 | 55 | } 56 | -------------------------------------------------------------------------------- /lib/README.MD: -------------------------------------------------------------------------------- 1 | jlogstash 核心代码 2 | -------------------------------------------------------------------------------- /pipeline/filters/README.md: -------------------------------------------------------------------------------- 1 | # Add: 2 | 3 | fields: 必填 map结构({"host":"hostname","ip":"%{ip}%"}) 4 | 5 | hostname: 本生是event里的属性，则就会取event.hostname里的值,没有就是原声的字符串hostname, 6 | 7 | %{ip}% 这样就表示调用内置函数获取本机ip 8 | 9 | 现在的内置函数有hostname，timestamp，ip 10 | 11 | # DateISO8601: 12 | 13 | match: 必填 map结构（{"timestamp":{"srcFormat":"dd/MMM/yyyy:HH:mm:ss Z","target":"timestamp","locale":"en"}}） 14 | 15 | # Remove: 16 | 17 | fields:必填 list结构 18 | 19 | removeNULL:默认值false ，是否删除null或空字符串字段 20 | 21 | # Rename: 22 | 23 | fields:必填 map结构{"oldname":"newname"} 24 | 25 | # IpIp: 26 | 27 | source: 默认值 clientip 需要解析的ip 28 | 29 | target: 默认值 ipip 30 | 31 | size: 默认值 5000 32 | 33 | # UA: 34 | 35 | source:必填 需要解析属性 36 | 37 | # JGrok: 38 | 39 | srcs:必填 list 结构，需要通过jgrok解析的字段（["ip"]） 40 | 41 | patterns:必填 map结构，自定义的正则表达式，{"ip":"(?\[0-9A-B\.])"} 如果:grok自带的已经有了,正则表达式不需要写，列如:{"%{COMBINEDAPACHELOG}":""} 42 | 43 | # Json: 44 | 45 | fields: 必填 map 结构 example {"messgae":"messgae1"} 源属性是message 目标属性message1，没有目标属性可以为“” 46 | 47 | # Java: 48 | code: 必填，String类型 。 49 | 50 | 示例： 51 | ``` 52 | filters: 53 | - Java: 54 | code: ' 55 | Object name = event.get("NAME"); 56 | event.put("XM", name); 57 | ' 58 | ``` 59 | 60 | # Performance: 61 | 62 | interval: 数据刷入文件的间隔时间，默认30秒 63 | 64 | timeZone: 时区 默认UTC 65 | 66 | path: 文件路径（home/admin/jlogserver/logs/srsyslog-performance-%{+YYYY.MM.dd}.txt）必填 67 | -------------------------------------------------------------------------------- /pipeline/filters/add/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/filters/dateiso8601/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/filters/geoip2/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/filters/grok/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | .idea/ 8 | -------------------------------------------------------------------------------- /pipeline/filters/gsub/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/filters/gsub/src/test/java/com/dtstack/logstash/filter/gsub/AppTest.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.logstash.filter.gsub; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /pipeline/filters/ipip/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/filters/ipip/src/main/resources/17monipdb.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/jlogstash/abe9fb4439c68ffa1341d4f8976d33afb6922f66/pipeline/filters/ipip/src/main/resources/17monipdb.dat -------------------------------------------------------------------------------- /pipeline/filters/ipip/src/test/java/com/dtstack/logstash/filter/ipip/AppTest.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.logstash.filter.ipip; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /pipeline/filters/java/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | -------------------------------------------------------------------------------- /pipeline/filters/java/src/main/java/com/dtstack/jlogstash/exception/InitializeException.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.exception; 2 | 3 | /** 4 | * @author zxb 5 | * @version 1.0.0 6 | * 2017年03月23日 16:02 7 | * @since Jdk1.6 8 | */ 9 | public class InitializeException extends RuntimeException { 10 | public InitializeException() { 11 | super(); 12 | } 13 | 14 | public InitializeException(String message) { 15 | super(message); 16 | } 17 | 18 | public InitializeException(String message, Throwable cause) { 19 | super(message, cause); 20 | } 21 | 22 | public InitializeException(Throwable cause) { 23 | super(cause); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /pipeline/filters/java/src/main/resources/code.ftl: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.filters; 2 | 3 | import java.util.*; 4 | import java.lang.*; 5 | 6 | public class JavaCodeUtil { 7 | 8 | public Map filter(Map event) { 9 | ${code} 10 | return event; 11 | } 12 | } -------------------------------------------------------------------------------- /pipeline/filters/json/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/filters/kv/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/filters/lowercase/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/filters/lowercase/src/main/java/com/dtstack/jlogstash/filters/Lowercase.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.filters; 19 | 20 | import java.util.List; 21 | import java.util.Map; 22 | import com.dtstack.jlogstash.annotation.Required; 23 | 24 | /** 25 | * 26 | * Reason: TODO ADD REASON(可选) 27 | * Date: 2016年8月31日 下午1:54:14 28 | * Company: www.dtstack.com 29 | * @author sishu.yss 30 | * 31 | */ 32 | public class Lowercase extends BaseFilter { 33 | public Lowercase(Map config) { 34 | super(config); 35 | } 36 | 37 | @Required(required=true) 38 | private List fields; 39 | 40 | public void prepare() { 41 | } 42 | 43 | @Override 44 | protected Map filter(Map event) { 45 | for (String field : fields) { 46 | if (event.containsKey(field)) { 47 | event.put(field, ((String) event.get(field)).toLowerCase()); 48 | } 49 | } 50 | return event; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /pipeline/filters/performance/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | -------------------------------------------------------------------------------- /pipeline/filters/remove/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/filters/rename/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/filters/replace/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/filters/split/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | *.iml 8 | -------------------------------------------------------------------------------- /pipeline/filters/splitStr/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | *.iml 8 | -------------------------------------------------------------------------------- /pipeline/filters/trim/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/filters/trim/src/main/java/com/dtstack/jlogstash/filters/Trim.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.filters; 19 | 20 | import java.util.List; 21 | import java.util.Map; 22 | 23 | import com.dtstack.jlogstash.annotation.Required; 24 | 25 | 26 | /** 27 | * 28 | * Reason: TODO ADD REASON(可选) 29 | * Date: 2016年8月31日 下午1:55:04 30 | * Company: www.dtstack.com 31 | * @author sishu.yss 32 | * 33 | */ 34 | public class Trim extends BaseFilter { 35 | public Trim(Map config) { 36 | super(config); 37 | } 38 | 39 | @Required(required=true) 40 | private static List fields; 41 | 42 | public void prepare() { 43 | } 44 | 45 | @Override 46 | protected Map filter(final Map event) { 47 | for (String field : fields) { 48 | if (event.containsKey(field)) { 49 | event.put(field, ((String) event.remove(field)).trim()); 50 | } 51 | } 52 | return event; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /pipeline/filters/ua/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/filters/ua/src/main/java/com/dtstack/jlogstash/parser/Device.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2012 Twitter, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.dtstack.jlogstash.parser; 18 | 19 | import java.util.Map; 20 | 21 | /** 22 | * Device parsed data class 23 | * 24 | * @author Steve Jiang (@sjiang) 25 | */ 26 | public class Device { 27 | public final String family; 28 | 29 | public Device(String family) { 30 | this.family = family; 31 | } 32 | 33 | public static Device fromMap(Map m) { 34 | return new Device((String) m.get("family")); 35 | } 36 | 37 | @Override 38 | public boolean equals(Object other) { 39 | if (other == this) return true; 40 | if (!(other instanceof Device)) return false; 41 | 42 | Device o = (Device) other; 43 | return (this.family != null && this.family.equals(o.family)) || this.family == o.family; 44 | } 45 | 46 | @Override 47 | public int hashCode() { 48 | return family == null ? 0 : family.hashCode(); 49 | } 50 | 51 | @Override 52 | public String toString() { 53 | return String.format("{family: %s}", 54 | family == null ? null : '"' + family + '"'); 55 | } 56 | } -------------------------------------------------------------------------------- /pipeline/filters/uppercase/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/filters/uppercase/src/main/java/com/dtstack/jlogstash/filters/Uppercase.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.filters; 19 | 20 | import java.util.List; 21 | import java.util.Map; 22 | import com.dtstack.jlogstash.annotation.Required; 23 | 24 | 25 | /** 26 | * 27 | * Reason: TODO ADD REASON(可选) 28 | * Date: 2016年8月31日 下午1:55:24 29 | * Company: www.dtstack.com 30 | * @author sishu.yss 31 | * 32 | */ 33 | public class Uppercase extends BaseFilter { 34 | public Uppercase(Map config) { 35 | super(config); 36 | } 37 | 38 | @Required(required=true) 39 | private static List fields; 40 | 41 | public void prepare() { 42 | } 43 | 44 | @Override 45 | protected Map filter(Map event) { 46 | for (String field : fields) { 47 | if (event.containsKey(field)) { 48 | event.put(field, ((String) event.get(field)).toUpperCase()); 49 | } 50 | } 51 | return event; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /pipeline/filters/urldecode/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/inputs/beats/.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | *.iml 3 | /target/ 4 | target/* 5 | .project 6 | .settings 7 | .classpath 8 | .metadata/ 9 | -------------------------------------------------------------------------------- /pipeline/inputs/beats/jlogstash-java-docs/package-list: -------------------------------------------------------------------------------- 1 | com.dtstack.jlogstash.inputs 2 | -------------------------------------------------------------------------------- /pipeline/inputs/beats/jlogstash-java-docs/script.js: -------------------------------------------------------------------------------- 1 | function show(type) 2 | { 3 | count = 0; 4 | for (var key in methods) { 5 | var row = document.getElementById(key); 6 | if ((methods[key] & type) != 0) { 7 | row.style.display = ''; 8 | row.className = (count++ % 2) ? rowColor : altColor; 9 | } 10 | else 11 | row.style.display = 'none'; 12 | } 13 | updateTabs(type); 14 | } 15 | 16 | function updateTabs(type) 17 | { 18 | for (var value in tabs) { 19 | var sNode = document.getElementById(tabs[value][0]); 20 | var spanNode = sNode.firstChild; 21 | if (value == type) { 22 | sNode.className = activeTableTab; 23 | spanNode.innerHTML = tabs[value][1]; 24 | } 25 | else { 26 | sNode.className = tableTab; 27 | spanNode.innerHTML = "" + tabs[value][1] + ""; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pipeline/inputs/beats/src/main/java/com/dtstack/jlogstash/inputs/Batch.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.inputs; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | /** 7 | * 打包协议版本、窗口、消息集。 8 | * copy from https://github.com/elastic/java-lumber 9 | * 10 | */ 11 | public class Batch { 12 | private byte protocol = Protocol.VERSION_2; 13 | private int windowSize; 14 | private List messages = new ArrayList(); 15 | 16 | public List getMessages() { 17 | return this.messages; 18 | } 19 | 20 | public void addMessage(Message message) { 21 | message.setBatch(this); 22 | this.messages.add(message); 23 | } 24 | 25 | public int size() { 26 | return this.messages.size(); 27 | } 28 | 29 | public void setWindowSize(int windowSize) { 30 | this.windowSize = windowSize; 31 | } 32 | 33 | public int getWindowSize() { 34 | return this.windowSize; 35 | } 36 | 37 | public boolean isEmpty() { 38 | if(0 == this.messages.size()) { 39 | return true; 40 | } else { 41 | return false; 42 | } 43 | } 44 | 45 | public byte getProtocol() { 46 | return protocol; 47 | } 48 | 49 | public void setProtocol(byte protocol) { 50 | this.protocol = protocol; 51 | } 52 | } -------------------------------------------------------------------------------- /pipeline/inputs/beats/src/main/java/com/dtstack/jlogstash/inputs/CompressedBatchEncoder.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.inputs; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | 5 | import io.netty.buffer.ByteBuf; 6 | import io.netty.buffer.ByteBufOutputStream; 7 | import io.netty.channel.ChannelHandlerContext; 8 | 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | import java.io.IOException; 12 | import java.io.OutputStream; 13 | import java.util.zip.Deflater; 14 | import java.util.zip.DeflaterOutputStream; 15 | 16 | /** 17 | * copy from https://github.com/elastic/java-lumber 18 | * 19 | */ 20 | public class CompressedBatchEncoder extends BatchEncoder { 21 | private final static Logger logger = LoggerFactory.getLogger(BatchEncoder.class.getName()); 22 | 23 | @Override 24 | protected ByteBuf getPayload(ChannelHandlerContext ctx, Batch batch) throws IOException { 25 | ByteBuf payload = super.getPayload(ctx, batch); 26 | 27 | Deflater deflater = new Deflater(); 28 | ByteBufOutputStream output = new ByteBufOutputStream(ctx.alloc().buffer()); 29 | DeflaterOutputStream outputDeflater = new DeflaterOutputStream(output, deflater); 30 | 31 | byte[] chunk = new byte[payload.readableBytes()]; 32 | payload.readBytes(chunk); 33 | outputDeflater.write(chunk); 34 | outputDeflater.close(); 35 | 36 | ByteBuf content = ctx.alloc().buffer(); 37 | content.writeByte(batch.getProtocol()); 38 | content.writeByte('C'); 39 | 40 | 41 | content.writeInt(output.writtenBytes()); 42 | content.writeBytes(output.buffer()); 43 | 44 | return content; 45 | } 46 | } -------------------------------------------------------------------------------- /pipeline/inputs/beats/src/main/java/com/dtstack/jlogstash/inputs/IMessageListener.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.inputs; 2 | 3 | import io.netty.channel.ChannelHandlerContext; 4 | 5 | /** 6 | * copy from https://github.com/elastic/java-lumber 7 | * 8 | */ 9 | public interface IMessageListener { 10 | public void onNewMessage(ChannelHandlerContext ctx, Message message); 11 | public void onNewConnection(ChannelHandlerContext ctx); 12 | public void onConnectionClose(ChannelHandlerContext ctx); 13 | public void onException(ChannelHandlerContext ctx,Throwable cause); 14 | } -------------------------------------------------------------------------------- /pipeline/inputs/beats/src/main/java/com/dtstack/jlogstash/inputs/JsonUtils.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.inputs; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | //import com.fasterxml.jackson.module.afterburner.AfterburnerModule; 5 | 6 | /** 7 | * 8 | * @author sishu.yss 9 | * 10 | */ 11 | public class JsonUtils { 12 | 13 | public final static ObjectMapper mapper = new ObjectMapper(); 14 | 15 | // public final static ObjectMapper mapper = new ObjectMapper().registerModule(new AfterburnerModule()); 16 | } 17 | -------------------------------------------------------------------------------- /pipeline/inputs/beats/src/main/java/com/dtstack/jlogstash/inputs/Protocol.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.inputs; 2 | 3 | 4 | /** 5 | * copy from https://github.com/elastic/java-lumber 6 | * 7 | */ 8 | public class Protocol { 9 | public static final byte VERSION_1 = '1'; 10 | public static final byte VERSION_2 = '2'; 11 | 12 | public static final byte CODE_WINDOW_SIZE = 'W'; 13 | public static final byte CODE_JSON_FRAME = 'J'; 14 | public static final byte CODE_COMPRESSED_FRAME = 'C'; 15 | public static final byte CODE_FRAME = 'D'; 16 | 17 | public static boolean isVersion2(byte versionRead) { 18 | if(Protocol.VERSION_2 == versionRead){ 19 | return true; 20 | } else { 21 | return false; 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /pipeline/inputs/binlog/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | 8 | 9 | -------------------------------------------------------------------------------- /pipeline/inputs/binlog/src/main/java/com/google/common/collect/MigrateMap.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.google.common.collect; 19 | 20 | import java.util.concurrent.ConcurrentMap; 21 | 22 | import com.google.common.base.Function; 23 | 24 | public class MigrateMap { 25 | 26 | @SuppressWarnings("deprecation") 27 | public static ConcurrentMap makeComputingMap(MapMaker maker, Function computingFunction) { 28 | return MapMakerHelper.makeComputingMap(maker, computingFunction); 29 | } 30 | 31 | @SuppressWarnings("deprecation") 32 | public static ConcurrentMap makeComputingMap(Function computingFunction) { 33 | return MapMakerHelper.makeComputingMap(new MapMaker(), computingFunction); 34 | } 35 | 36 | } -------------------------------------------------------------------------------- /pipeline/inputs/elasticsearch/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | 8 | 9 | -------------------------------------------------------------------------------- /pipeline/inputs/elasticsearch/src/main/java/com/dtstack/jlogstash/exception/InitializeException.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.exception; 2 | 3 | /** 4 | * @author zxb 5 | * @version 1.0.0 6 | * 2017年03月23日 16:02 7 | * @since Jdk1.6 8 | */ 9 | public class InitializeException extends RuntimeException { 10 | public InitializeException() { 11 | super(); 12 | } 13 | 14 | public InitializeException(String message) { 15 | super(message); 16 | } 17 | 18 | public InitializeException(String message, Throwable cause) { 19 | super(message, cause); 20 | } 21 | 22 | public InitializeException(Throwable cause) { 23 | super(cause); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /pipeline/inputs/elasticsearch/src/main/java/com/dtstack/jlogstash/inputs/ElasticsearchSourceConfig.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.inputs; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * ES数据源配置 7 | * 8 | * @author zxb 9 | * @version 1.0.0 10 | * 2017年03月24日 09:06 11 | * @since Jdk1.6 12 | */ 13 | public class ElasticsearchSourceConfig { 14 | 15 | private String cluster; 16 | 17 | private List hosts; 18 | 19 | private boolean sniff; 20 | 21 | public ElasticsearchSourceConfig(String cluster, List hosts, boolean sniff) { 22 | this.cluster = cluster; 23 | this.hosts = hosts; 24 | this.sniff = sniff; 25 | } 26 | 27 | public ElasticsearchSourceConfig() { 28 | } 29 | 30 | public String getCluster() { 31 | return cluster; 32 | } 33 | 34 | public void setCluster(String cluster) { 35 | this.cluster = cluster; 36 | } 37 | 38 | public List getHosts() { 39 | return hosts; 40 | } 41 | 42 | public void setHosts(List hosts) { 43 | this.hosts = hosts; 44 | } 45 | 46 | public boolean isSniff() { 47 | return sniff; 48 | } 49 | 50 | public void setSniff(boolean sniff) { 51 | this.sniff = sniff; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /pipeline/inputs/file/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/inputs/file/jlogstash-java-docs/allclasses-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (file 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 24 |
25 | 26 | 27 | -------------------------------------------------------------------------------- /pipeline/inputs/file/jlogstash-java-docs/allclasses-noframe.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (file 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 24 |
25 | 26 | 27 | -------------------------------------------------------------------------------- /pipeline/inputs/file/jlogstash-java-docs/package-list: -------------------------------------------------------------------------------- 1 | com.dtstack.jlogstash.inputs 2 | -------------------------------------------------------------------------------- /pipeline/inputs/file/jlogstash-java-docs/script.js: -------------------------------------------------------------------------------- 1 | function show(type) 2 | { 3 | count = 0; 4 | for (var key in methods) { 5 | var row = document.getElementById(key); 6 | if ((methods[key] & type) != 0) { 7 | row.style.display = ''; 8 | row.className = (count++ % 2) ? rowColor : altColor; 9 | } 10 | else 11 | row.style.display = 'none'; 12 | } 13 | updateTabs(type); 14 | } 15 | 16 | function updateTabs(type) 17 | { 18 | for (var value in tabs) { 19 | var sNode = document.getElementById(tabs[value][0]); 20 | var spanNode = sNode.firstChild; 21 | if (value == type) { 22 | sNode.className = activeTableTab; 23 | spanNode.innerHTML = tabs[value][1]; 24 | } 25 | else { 26 | sNode.className = tableTab; 27 | spanNode.innerHTML = "" + tabs[value][1] + ""; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pipeline/inputs/file/src/test/java/com/dtstack/logstash/input/fileinput/Test.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.logstash.input.fileinput; 2 | 3 | import java.io.IOException; 4 | import java.util.ArrayList; 5 | import java.util.HashMap; 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | import com.dtstack.jlogstash.inputs.File; 10 | import com.dtstack.jlogstash.inputs.ReadLineUtil; 11 | 12 | public class Test { 13 | 14 | public void test(){ 15 | Map config = new HashMap(); 16 | config.put("codec", "plain"); 17 | List path = new ArrayList(); 18 | path.add("E:\\controller.log"); 19 | path.add("E:\\server.log"); 20 | 21 | File file = new File(config); 22 | // File.path = path; 23 | file.prepare(); 24 | file.emit(); 25 | } 26 | 27 | public void testReadLine() throws Exception{ 28 | java.io.File file = new java.io.File("D:\\testReadLine.txt"); 29 | ReadLineUtil readLineUtil = new ReadLineUtil(file, "UTF-8", 0); 30 | String line = null; 31 | while((line = readLineUtil.readLine()) != null){ 32 | System.out.println(line); 33 | } 34 | 35 | System.out.println(readLineUtil.getCurrBufPos()); 36 | 37 | } 38 | 39 | public static void main(String[] args) throws IOException { 40 | // Test test = new Test(); 41 | // test.testReadLine(); 42 | System.out.println(Long.MAX_VALUE/1024/1024/1024/1024/1024); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /pipeline/inputs/jdbc/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/inputs/jdbc/src/main/java/com/dtstack/jlogstash/exception/InitializeException.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.exception; 2 | 3 | /** 4 | * @author zxb 5 | * @version 1.0.0 6 | * 2017年03月23日 16:02 7 | * @since Jdk1.6 8 | */ 9 | public class InitializeException extends RuntimeException { 10 | public InitializeException() { 11 | super(); 12 | } 13 | 14 | public InitializeException(String message) { 15 | super(message); 16 | } 17 | 18 | public InitializeException(String message, Throwable cause) { 19 | super(message, cause); 20 | } 21 | 22 | public InitializeException(Throwable cause) { 23 | super(cause); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /pipeline/inputs/jdbc/src/main/java/com/dtstack/jlogstash/inputs/ConnectionConfig.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.inputs; 2 | 3 | /** 4 | * @author zxb 5 | * @version 1.0.0 6 | * 2017年03月23日 15:47 7 | * @since Jdk1.6 8 | */ 9 | public class ConnectionConfig { 10 | 11 | private String jdbc_connection_string; 12 | 13 | private String jdbc_driver_class; 14 | 15 | private String jdbc_driver_library; 16 | 17 | private String jdbc_user; 18 | 19 | private String jdbc_password; 20 | 21 | public String getJdbc_connection_string() { 22 | return jdbc_connection_string; 23 | } 24 | 25 | public void setJdbc_connection_string(String jdbc_connection_string) { 26 | this.jdbc_connection_string = jdbc_connection_string; 27 | } 28 | 29 | public String getJdbc_driver_class() { 30 | return jdbc_driver_class; 31 | } 32 | 33 | public void setJdbc_driver_class(String jdbc_driver_class) { 34 | this.jdbc_driver_class = jdbc_driver_class; 35 | } 36 | 37 | public String getJdbc_driver_library() { 38 | return jdbc_driver_library; 39 | } 40 | 41 | public void setJdbc_driver_library(String jdbc_driver_library) { 42 | this.jdbc_driver_library = jdbc_driver_library; 43 | } 44 | 45 | public String getJdbc_user() { 46 | return jdbc_user; 47 | } 48 | 49 | public void setJdbc_user(String jdbc_user) { 50 | this.jdbc_user = jdbc_user; 51 | } 52 | 53 | public String getJdbc_password() { 54 | return jdbc_password; 55 | } 56 | 57 | public void setJdbc_password(String jdbc_password) { 58 | this.jdbc_password = jdbc_password; 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /pipeline/inputs/jdbc/src/main/java/com/dtstack/jlogstash/util/IOUtils.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.util; 2 | 3 | import java.io.Closeable; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.io.OutputStream; 7 | 8 | /** 9 | * @author zxb 10 | * @version 1.0.0 11 | * 2017年03月24日 22:10 12 | * @since Jdk1.6 13 | */ 14 | public class IOUtils { 15 | 16 | public static long copy(InputStream input, OutputStream output, byte[] buffer) throws IOException { 17 | long count; 18 | int n; 19 | for(count = 0L; -1 != (n = input.read(buffer)); count += (long)n) { 20 | output.write(buffer, 0, n); 21 | } 22 | 23 | return count; 24 | } 25 | 26 | public static void closeQuietly(Closeable... closeables) { 27 | if(closeables != null) { 28 | Closeable[] arr = closeables; 29 | int len = closeables.length; 30 | 31 | for(int i = 0; i < len; i++) { 32 | Closeable closeable = arr[i]; 33 | closeQuietly(closeable); 34 | } 35 | } 36 | } 37 | 38 | public static void closeQuietly(Closeable closeable) { 39 | try { 40 | if(closeable != null) { 41 | closeable.close(); 42 | } 43 | } catch (IOException e) { 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /pipeline/inputs/kafka09/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | 8 | 9 | -------------------------------------------------------------------------------- /pipeline/inputs/kafka09/src/main/java/com/dtstack/jlogstash/inputs/IKafkaChg.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.inputs; 19 | 20 | public interface IKafkaChg { 21 | 22 | /** 23 | * 某个topic对应的分区发生变化,或者消耗的客户端数量发生变化 24 | * @param topicName 25 | * @param consumers 26 | * @param partitions 27 | */ 28 | void onInfoChgTrigger(String topicName, int consumers, int partitions); 29 | 30 | /** 31 | * kafka集群挂掉 32 | */ 33 | void onClusterShutDown(); 34 | 35 | } 36 | -------------------------------------------------------------------------------- /pipeline/inputs/kafka09/src/test/java/com/dtstack/logstash/input/kafka/AppTest.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.logstash.input.kafka; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /pipeline/inputs/kafka10/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | *.iml 8 | .idea/ 9 | -------------------------------------------------------------------------------- /pipeline/inputs/kafkadistributed/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | .settings 3 | .classpath 4 | .project 5 | .idea 6 | kafkadistributed.iml 7 | -------------------------------------------------------------------------------- /pipeline/inputs/kafkadistributed/src/main/java/com/dtstack/jlogstash/distributed/BrokersNode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.distributed; 19 | 20 | 21 | /** 22 | * 23 | * Reason: TODO ADD REASON(可选) 24 | * Date: 2016年12月28日 下午1:16:37 25 | * Company: www.dtstack.com 26 | * @author sishu.yss 27 | * 28 | */ 29 | public class BrokersNode { 30 | 31 | private String master; 32 | 33 | public String getMaster() { 34 | return master; 35 | } 36 | 37 | public void setMaster(String master) { 38 | this.master = master; 39 | } 40 | 41 | public static BrokersNode initBrokersNode(){ 42 | return new BrokersNode(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /pipeline/inputs/kafkadistributed/src/main/java/com/dtstack/jlogstash/distributed/http/common/HttpCommon.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.distributed.http.common; 19 | 20 | /** 21 | * 22 | * Reason: TODO ADD REASON(可选) 23 | * Date: 2017年1月2日 下午1:16:37 24 | * Company: www.dtstack.com 25 | * @author sishu.yss 26 | * 27 | */ 28 | public class HttpCommon { 29 | 30 | public static Object[] getUrlPort(String address){ 31 | Object[] object =new Object[2]; 32 | String[] la = address.split(":"); 33 | object[0] = la[0].trim(); 34 | object[1] = Integer.parseInt(la[1].trim())+1; 35 | return object; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /pipeline/inputs/kafkadistributed/src/main/java/com/dtstack/jlogstash/distributed/http/common/Urls.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.distributed.http.common; 19 | 20 | /** 21 | * 22 | * Reason: TODO ADD REASON(可选) 23 | * Date: 2017年1月2日 下午1:16:37 24 | * Company: www.dtstack.com 25 | * @author sishu.yss 26 | * 27 | */ 28 | public class Urls { 29 | 30 | public final static String ROOT = "/distributed"; 31 | 32 | public final static String LOADNODEDATA = ROOT+"/loadNodeData"; 33 | 34 | public final static String LOGPOOLDATA = ROOT+"/logPoolData"; 35 | 36 | public final static String MANUALLYDATA = ROOT+"/manuallyData"; 37 | 38 | public final static String NOCOMPLETEDATA = ROOT+"/logPoolNoCompleteData"; 39 | 40 | } 41 | -------------------------------------------------------------------------------- /pipeline/inputs/kafkadistributed/src/main/java/com/dtstack/jlogstash/distributed/http/server/callback/ApiCallback.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.distributed.http.server.callback; 19 | 20 | 21 | /** 22 | * 23 | * Reason: TODO ADD REASON(可选) 24 | * Date: 2016年12月30日 下午1:16:37 25 | * Company: www.dtstack.com 26 | * @author sishu.yss 27 | * 28 | */ 29 | public interface ApiCallback { 30 | 31 | void execute(ApiResult apiResult) throws Exception; 32 | } 33 | -------------------------------------------------------------------------------- /pipeline/inputs/kafkadistributed/src/main/java/com/dtstack/jlogstash/distributed/logmerge/LogDeleWatcher.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.distributed.logmerge; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import java.util.concurrent.Callable; 6 | import java.util.concurrent.ExecutorService; 7 | import java.util.concurrent.Executors; 8 | 9 | /** 10 | * Reason: 11 | * Date: 2017/1/13 12 | * Company: www.dtstack.com 13 | * 14 | * @ahthor xuchao 15 | */ 16 | 17 | public class LogDeleWatcher implements Callable { 18 | 19 | private static final Logger logger = LoggerFactory.getLogger(LogDeleWatcher.class); 20 | 21 | private boolean isRunning = false; 22 | 23 | private ExecutorService executorService; 24 | 25 | private LogPool logPool; 26 | 27 | /**删除数据检查间隔时间*/ 28 | private static int SLEEP_TIME = 1 * 60 * 1000; 29 | 30 | public LogDeleWatcher(LogPool logPool){ 31 | this.logPool = logPool; 32 | } 33 | 34 | @Override 35 | public Object call() throws Exception { 36 | while (isRunning){ 37 | Thread.sleep(SLEEP_TIME); 38 | logger.warn("begin to watch time out log..."); 39 | logPool.deleteLog(); 40 | } 41 | return null; 42 | } 43 | 44 | public void startup(){ 45 | this.isRunning = true; 46 | executorService = Executors.newSingleThreadExecutor(); 47 | executorService.submit(this); 48 | logger.info("log pool delete watcher is start up success, SLEEP_TIME:{}", SLEEP_TIME); 49 | } 50 | 51 | public void shutdown(){ 52 | this.isRunning = false; 53 | executorService.shutdown(); 54 | logger.info("log pool delete watcher is shutdown"); 55 | } 56 | 57 | 58 | } 59 | -------------------------------------------------------------------------------- /pipeline/inputs/kafkadistributed/src/main/java/com/dtstack/jlogstash/distributed/logmerge/LogTypeConstant.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.distributed.logmerge; 19 | 20 | /** 21 | * 22 | * 日志类型常量 23 | * Date: 2017/1/4 24 | * Company: www.dtstack.com 25 | * @ahthor xuchao 26 | */ 27 | 28 | public class LogTypeConstant { 29 | 30 | public static final String CMS18 = "cms18_gclog"; 31 | } 32 | -------------------------------------------------------------------------------- /pipeline/inputs/kafkadistributed/src/main/java/com/dtstack/jlogstash/distributed/util/CountUtil.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.distributed.util; 2 | 3 | public class CountUtil { 4 | 5 | public static boolean count(int index,int multiples){ 6 | return index%multiples==0; 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /pipeline/inputs/kafkadistributed/src/main/java/com/dtstack/jlogstash/inputs/IKafkaChg.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.dtstack.jlogstash.inputs; 19 | 20 | public interface IKafkaChg { 21 | 22 | /** 23 | * 某个topic对应的分区发生变化,或者消耗的客户端数量发生变化 24 | * @param topicName 25 | * @param consumers 26 | * @param partitions 27 | */ 28 | public void onInfoChgTrigger(String topicName, int consumers, int partitions); 29 | 30 | /** 31 | * kafka集群挂掉 32 | */ 33 | public void onClusterShutDown(); 34 | 35 | } 36 | -------------------------------------------------------------------------------- /pipeline/inputs/kafkadistributed/src/main/java/com/dtstack/jlogstash/jdistributed/gclog/GCTypeConstant.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.jdistributed.gclog; 2 | 3 | /** 4 | * Reason: 5 | * Date: 2017/1/10 6 | * Company: www.dtstack.com 7 | * 8 | * @ahthor xuchao 9 | */ 10 | 11 | public class GCTypeConstant { 12 | 13 | public static final String CMS_LOG_TYPE = "cmsgc"; 14 | 15 | public static final String YOUNG_LOG_TYPE = "younggc"; 16 | 17 | public static final String GC_BEGIN = "gcbegin"; 18 | } 19 | -------------------------------------------------------------------------------- /pipeline/inputs/mongodb/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | 8 | 9 | -------------------------------------------------------------------------------- /pipeline/inputs/mongodb/jlogstash-java-docs/allclasses-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (mongodb 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 24 |
25 | 26 | 27 | -------------------------------------------------------------------------------- /pipeline/inputs/mongodb/jlogstash-java-docs/allclasses-noframe.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (mongodb 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 24 |
25 | 26 | 27 | -------------------------------------------------------------------------------- /pipeline/inputs/mongodb/jlogstash-java-docs/com/dtstack/jlogstash/constans/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.constans (mongodb 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.constans

14 |
15 |

类

16 | 19 |
20 | 21 | 22 | -------------------------------------------------------------------------------- /pipeline/inputs/mongodb/jlogstash-java-docs/com/dtstack/jlogstash/inputs/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.inputs (mongodb 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.inputs

14 |
15 |

接口

16 | 19 |

类

20 | 27 |
28 | 29 | 30 | -------------------------------------------------------------------------------- /pipeline/inputs/mongodb/jlogstash-java-docs/overview-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 概览列表 (mongodb 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |
所有类
14 |
15 |

程序包

16 | 20 |
21 |

 

22 | 23 | 24 | -------------------------------------------------------------------------------- /pipeline/inputs/mongodb/jlogstash-java-docs/package-list: -------------------------------------------------------------------------------- 1 | com.dtstack.jlogstash.constans 2 | com.dtstack.jlogstash.inputs 3 | -------------------------------------------------------------------------------- /pipeline/inputs/mongodb/jlogstash-java-docs/script.js: -------------------------------------------------------------------------------- 1 | function show(type) 2 | { 3 | count = 0; 4 | for (var key in methods) { 5 | var row = document.getElementById(key); 6 | if ((methods[key] & type) != 0) { 7 | row.style.display = ''; 8 | row.className = (count++ % 2) ? rowColor : altColor; 9 | } 10 | else 11 | row.style.display = 'none'; 12 | } 13 | updateTabs(type); 14 | } 15 | 16 | function updateTabs(type) 17 | { 18 | for (var value in tabs) { 19 | var sNode = document.getElementById(tabs[value][0]); 20 | var spanNode = sNode.firstChild; 21 | if (value == type) { 22 | sNode.className = activeTableTab; 23 | spanNode.innerHTML = tabs[value][1]; 24 | } 25 | else { 26 | sNode.className = tableTab; 27 | spanNode.innerHTML = "" + tabs[value][1] + ""; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pipeline/inputs/mongodb/src/main/java/com/dtstack/jlogstash/constans/ConfigConstans.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.constans; 2 | 3 | /** 4 | * @author zxb 5 | * @version 1.0.0 6 | * 2017年03月23日 09:19 7 | * @since Jdk1.6 8 | */ 9 | public class ConfigConstans { 10 | 11 | public static final String DEFAULT_META_FILE_PATH = System.getProperty("java.io.tmpdir") + System.getProperty("line.separator") + ".last_run_metadata_path"; 12 | 13 | public static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; 14 | 15 | private ConfigConstans() { 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /pipeline/inputs/mongodb/src/main/java/com/dtstack/jlogstash/inputs/Scheduler.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.inputs; 2 | 3 | /** 4 | * 任务调度器 5 | * 6 | * @author zxb 7 | * @version 1.0.0 8 | * 2017年03月22日 14:51 9 | * @since Jdk1.6 10 | */ 11 | public abstract class Scheduler { 12 | 13 | protected Task task; 14 | 15 | /** 16 | * 初始化构造 17 | * 18 | * @param task 19 | */ 20 | public Scheduler(Task task) { 21 | this.task = task; 22 | } 23 | 24 | /** 25 | * 开始调度 26 | */ 27 | public abstract void start(); 28 | 29 | /** 30 | * 结束调度 31 | */ 32 | public abstract void stop(); 33 | 34 | /** 35 | * 获取本次执行与下次执行的时间间隔 36 | * 37 | * @return 38 | */ 39 | public abstract Long getNextDuration(); 40 | } 41 | -------------------------------------------------------------------------------- /pipeline/inputs/mongodb/src/main/java/com/dtstack/jlogstash/inputs/SimpleScheduler.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.inputs; 2 | 3 | /** 4 | * 简单调度器，按频率调度 5 | * 6 | * @author zxb 7 | * @version 1.0.0 8 | * 2017年03月22日 15:26 9 | * @since Jdk1.6 10 | */ 11 | public class SimpleScheduler extends Scheduler { 12 | 13 | /** 14 | * 间隔频率 15 | */ 16 | private Long interval; 17 | 18 | /** 19 | * 是否停止 20 | */ 21 | private volatile boolean stop = false; 22 | 23 | public SimpleScheduler(Task task) { 24 | super(task); 25 | } 26 | 27 | public SimpleScheduler(Task task, Long interval) { 28 | super(task); 29 | this.interval = interval; 30 | } 31 | 32 | @Override 33 | public void start() { 34 | while (!stop) { 35 | task.execute(); 36 | 37 | // interval为空时直接退出 38 | if (interval == null) { 39 | break; 40 | } 41 | 42 | try { 43 | Thread.sleep(interval); 44 | } catch (InterruptedException e) { 45 | e.printStackTrace(); 46 | } 47 | } 48 | } 49 | 50 | @Override 51 | public void stop() { 52 | stop = true; 53 | } 54 | 55 | @Override 56 | public Long getNextDuration() { 57 | return interval; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /pipeline/inputs/mongodb/src/main/java/com/dtstack/jlogstash/inputs/Task.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.inputs; 2 | 3 | /** 4 | * 任务接口 5 | * @author zxb 6 | * @version 1.0.0 7 | * 2017年03月22日 15:20 8 | * @since Jdk1.6 9 | */ 10 | public interface Task { 11 | 12 | public void execute(); 13 | } 14 | -------------------------------------------------------------------------------- /pipeline/inputs/netty/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/inputs/netty/jlogstash-java-docs/allclasses-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (netty 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 19 |
20 | 21 | 22 | -------------------------------------------------------------------------------- /pipeline/inputs/netty/jlogstash-java-docs/allclasses-noframe.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (netty 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 19 |
20 | 21 | 22 | -------------------------------------------------------------------------------- /pipeline/inputs/netty/jlogstash-java-docs/com/dtstack/jlogstash/inputs/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.inputs (netty 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.inputs

14 |
15 |

类

16 | 20 |
21 | 22 | 23 | -------------------------------------------------------------------------------- /pipeline/inputs/netty/jlogstash-java-docs/package-list: -------------------------------------------------------------------------------- 1 | com.dtstack.jlogstash.inputs 2 | -------------------------------------------------------------------------------- /pipeline/inputs/netty/jlogstash-java-docs/script.js: -------------------------------------------------------------------------------- 1 | function show(type) 2 | { 3 | count = 0; 4 | for (var key in methods) { 5 | var row = document.getElementById(key); 6 | if ((methods[key] & type) != 0) { 7 | row.style.display = ''; 8 | row.className = (count++ % 2) ? rowColor : altColor; 9 | } 10 | else 11 | row.style.display = 'none'; 12 | } 13 | updateTabs(type); 14 | } 15 | 16 | function updateTabs(type) 17 | { 18 | for (var value in tabs) { 19 | var sNode = document.getElementById(tabs[value][0]); 20 | var spanNode = sNode.firstChild; 21 | if (value == type) { 22 | sNode.className = activeTableTab; 23 | spanNode.innerHTML = tabs[value][1]; 24 | } 25 | else { 26 | sNode.className = tableTab; 27 | spanNode.innerHTML = "" + tabs[value][1] + ""; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pipeline/inputs/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | com.dtstack.jlogstash 7 | jlogstash-all 8 | 1.0.0 9 | ../../pom.xml 10 | 11 | 4.0.0 12 | input.plugin 13 | pom 14 | 15 | beats 16 | elasticsearch 17 | file 18 | jdbc 19 | kafka09 20 | kafka10 21 | kafkadistributed 22 | mongodb 23 | netty 24 | redis 25 | stdin 26 | tcp 27 | binlog 28 | 29 | 30 | 31 | 32 | 33 | com.dtstack.jlogstash 34 | jlogstash 35 | 1.0.0 36 | provided 37 | 38 | 39 | junit 40 | junit 41 | 4.13.1 42 | test 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /pipeline/inputs/redis/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ -------------------------------------------------------------------------------- /pipeline/inputs/redis/src/main/java/com/dtstack/jlogstash/inputs/ChannelListener.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.inputs; 2 | 3 | import redis.clients.jedis.JedisPubSub; 4 | 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | 8 | /** 9 | * Created by Administrator on 2017/6/19. 10 | */ 11 | public class ChannelListener extends JedisPubSub { 12 | private Redis redis; 13 | 14 | public ChannelListener(Redis redis){ 15 | this.redis = redis; 16 | } 17 | 18 | @Override 19 | public void onMessage(String channel, String message) { 20 | Map map = new HashMap(); 21 | map.put(channel,message); 22 | redis.process(map); 23 | } 24 | 25 | @Override 26 | public void onPMessage(String pattern, String channel, String message) { 27 | Map map = new HashMap(); 28 | map.put(channel,message); 29 | redis.process(map); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /pipeline/inputs/redis/src/main/java/com/dtstack/jlogstash/utils/DataType.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.utils; 2 | 3 | public enum DataType { 4 | LIST("list") 5 | ,STRING("string") 6 | ,SET("set") 7 | ,HASH("hash") 8 | ,SORTEDSET("sorted_set") 9 | ,CHANNEL("channel") 10 | ,CHANNEL_PATTERN("channel_pattern"); 11 | 12 | private String dataType; 13 | 14 | DataType(String dataType){ 15 | this.dataType = dataType; 16 | } 17 | 18 | public String getDataType() { 19 | return dataType; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /pipeline/inputs/redis/src/test/java/com/dtstack/jlogstash/TestPubSub.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash; 2 | 3 | import redis.clients.jedis.JedisPubSub; 4 | 5 | /** 6 | * Created by Administrator on 2017/6/16. 7 | */ 8 | public class TestPubSub extends JedisPubSub { 9 | 10 | @Override 11 | public void onMessage(String channel, String message) { 12 | System.out.println("onMessage--"+"channel:"+channel+","+"message:"+message); 13 | } 14 | 15 | @Override 16 | public void onPMessage(String pattern, String channel, String message) { 17 | System.out.println("onPMessage--"+"pattern:"+pattern+","+"channel:"+channel+","+"message:"+message); 18 | } 19 | 20 | @Override 21 | public void onSubscribe(String channel, int subscribedChannels) { 22 | System.out.println("onSubscribe--"+"channel:"+channel+","+"subscribedChannels:"+subscribedChannels); 23 | } 24 | 25 | 26 | @Override 27 | public void onUnsubscribe(String channel, int subscribedChannels) { 28 | System.out.println("onUnsubscribe--"+"channel:"+channel+","+"subscribedChannels:"+subscribedChannels); 29 | } 30 | 31 | @Override 32 | public void onPUnsubscribe(String pattern, int subscribedChannels) { 33 | System.out.println("onPUnsubscribe--"+"pattern:"+pattern+","+"subscribedChannels:"+subscribedChannels); 34 | } 35 | 36 | @Override 37 | public void onPSubscribe(String pattern, int subscribedChannels) { 38 | System.out.println("onPSubscribe--"+"pattern:"+pattern+","+"subscribedChannels:"+subscribedChannels); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /pipeline/inputs/stdin/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/inputs/stdin/jlogstash-java-docs/allclasses-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (stdin 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /pipeline/inputs/stdin/jlogstash-java-docs/allclasses-noframe.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (stdin 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /pipeline/inputs/stdin/jlogstash-java-docs/com/dtstack/jlogstash/inputs/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.inputs (stdin 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.inputs

14 |
15 |

类

16 | 19 |
20 | 21 | 22 | -------------------------------------------------------------------------------- /pipeline/inputs/stdin/jlogstash-java-docs/package-list: -------------------------------------------------------------------------------- 1 | com.dtstack.jlogstash.inputs 2 | -------------------------------------------------------------------------------- /pipeline/inputs/stdin/jlogstash-java-docs/script.js: -------------------------------------------------------------------------------- 1 | function show(type) 2 | { 3 | count = 0; 4 | for (var key in methods) { 5 | var row = document.getElementById(key); 6 | if ((methods[key] & type) != 0) { 7 | row.style.display = ''; 8 | row.className = (count++ % 2) ? rowColor : altColor; 9 | } 10 | else 11 | row.style.display = 'none'; 12 | } 13 | updateTabs(type); 14 | } 15 | 16 | function updateTabs(type) 17 | { 18 | for (var value in tabs) { 19 | var sNode = document.getElementById(tabs[value][0]); 20 | var spanNode = sNode.firstChild; 21 | if (value == type) { 22 | sNode.className = activeTableTab; 23 | spanNode.innerHTML = tabs[value][1]; 24 | } 25 | else { 26 | sNode.className = tableTab; 27 | spanNode.innerHTML = "" + tabs[value][1] + ""; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pipeline/inputs/tcp/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/inputs/tcp/jlogstash-java-docs/allclasses-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (tcp 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 |
    16 |
  • Tcp
    • 17 |
18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /pipeline/inputs/tcp/jlogstash-java-docs/allclasses-noframe.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (tcp 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 |
    16 |
  • Tcp
    • 17 |
18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /pipeline/inputs/tcp/jlogstash-java-docs/com/dtstack/jlogstash/inputs/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.inputs (tcp 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.inputs

14 |
15 |

类

16 |
    17 |
  • Tcp
    • 18 |
19 |
20 | 21 | 22 | -------------------------------------------------------------------------------- /pipeline/inputs/tcp/jlogstash-java-docs/package-list: -------------------------------------------------------------------------------- 1 | com.dtstack.jlogstash.inputs 2 | -------------------------------------------------------------------------------- /pipeline/inputs/tcp/jlogstash-java-docs/script.js: -------------------------------------------------------------------------------- 1 | function show(type) 2 | { 3 | count = 0; 4 | for (var key in methods) { 5 | var row = document.getElementById(key); 6 | if ((methods[key] & type) != 0) { 7 | row.style.display = ''; 8 | row.className = (count++ % 2) ? rowColor : altColor; 9 | } 10 | else 11 | row.style.display = 'none'; 12 | } 13 | updateTabs(type); 14 | } 15 | 16 | function updateTabs(type) 17 | { 18 | for (var value in tabs) { 19 | var sNode = document.getElementById(tabs[value][0]); 20 | var spanNode = sNode.firstChild; 21 | if (value == type) { 22 | sNode.className = activeTableTab; 23 | spanNode.innerHTML = tabs[value][1]; 24 | } 25 | else { 26 | sNode.className = tableTab; 27 | spanNode.innerHTML = "" + tabs[value][1] + ""; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pipeline/inputs/tcp/src/test/java/com/dtstack/jogstash/input/ClientMessageHandler.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jogstash.input; 2 | 3 | import org.apache.log4j.Logger; 4 | import org.apache.mina.core.service.IoHandlerAdapter; 5 | import org.apache.mina.core.session.IoSession; 6 | 7 | /** 8 | * 9 | * Reason: TODO ADD REASON(可选) 10 | * Date: 2016年8月31日 下午1:17:48 11 | * Company: www.dtstack.com 12 | * @author sishu.yss 13 | * 14 | */ 15 | public class ClientMessageHandler extends IoHandlerAdapter { 16 | 17 | private static Logger logger = Logger.getLogger(ClientMessageHandler.class); 18 | 19 | public void messageReceived(IoSession session, Object message) 20 | throws Exception { 21 | String msg = message.toString(); 22 | System.out.println("客户端接收到的信息为：" + msg); 23 | } 24 | 25 | @Override 26 | public void exceptionCaught(IoSession session, Throwable cause) 27 | throws Exception { 28 | System.out.println(cause); 29 | logger.info("客户端发生异常..." + cause); 30 | } 31 | } -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch/jlogstash-java-docs/allclasses-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (elasticsearch 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch/jlogstash-java-docs/allclasses-noframe.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (elasticsearch 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch/jlogstash-java-docs/com/dtstack/jlogstash/outputs/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.outputs (elasticsearch 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.outputs

14 |
15 |

类

16 | 19 |
20 | 21 | 22 | -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch/jlogstash-java-docs/package-list: -------------------------------------------------------------------------------- 1 | com.dtstack.jlogstash.outputs 2 | -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch/jlogstash-java-docs/script.js: -------------------------------------------------------------------------------- 1 | function show(type) 2 | { 3 | count = 0; 4 | for (var key in methods) { 5 | var row = document.getElementById(key); 6 | if ((methods[key] & type) != 0) { 7 | row.style.display = ''; 8 | row.className = (count++ % 2) ? rowColor : altColor; 9 | } 10 | else 11 | row.style.display = 'none'; 12 | } 13 | updateTabs(type); 14 | } 15 | 16 | function updateTabs(type) 17 | { 18 | for (var value in tabs) { 19 | var sNode = document.getElementById(tabs[value][0]); 20 | var spanNode = sNode.firstChild; 21 | if (value == type) { 22 | sNode.className = activeTableTab; 23 | spanNode.innerHTML = tabs[value][1]; 24 | } 25 | else { 26 | sNode.className = tableTab; 27 | spanNode.innerHTML = "" + tabs[value][1] + ""; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch5/.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | *.iml 3 | /target/ 4 | target/* 5 | .project 6 | .settings 7 | .classpath 8 | .metadata/ 9 | -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch5/jlogstash-java-docs/allclasses-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (elasticsearch5 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch5/jlogstash-java-docs/allclasses-noframe.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (elasticsearch5 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch5/jlogstash-java-docs/com/dtstack/jlogstash/outputs/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.outputs (elasticsearch5 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.outputs

14 |
15 |

类

16 | 19 |
20 | 21 | 22 | -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch5/jlogstash-java-docs/package-list: -------------------------------------------------------------------------------- 1 | com.dtstack.jlogstash.outputs 2 | -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch5/jlogstash-java-docs/script.js: -------------------------------------------------------------------------------- 1 | function show(type) 2 | { 3 | count = 0; 4 | for (var key in methods) { 5 | var row = document.getElementById(key); 6 | if ((methods[key] & type) != 0) { 7 | row.style.display = ''; 8 | row.className = (count++ % 2) ? rowColor : altColor; 9 | } 10 | else 11 | row.style.display = 'none'; 12 | } 13 | updateTabs(type); 14 | } 15 | 16 | function updateTabs(type) 17 | { 18 | for (var value in tabs) { 19 | var sNode = document.getElementById(tabs[value][0]); 20 | var spanNode = sNode.firstChild; 21 | if (value == type) { 22 | sNode.className = activeTableTab; 23 | spanNode.innerHTML = tabs[value][1]; 24 | } 25 | else { 26 | sNode.className = tableTab; 27 | spanNode.innerHTML = "" + tabs[value][1] + ""; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch5/src/test/java/com/dtstack/jlogstash/output/Test.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.output; 2 | 3 | import java.io.IOException; 4 | import java.util.Map; 5 | 6 | import com.dtstack.jlogstash.outputs.Elasticsearch5; 7 | import com.dtstack.jlogstash.render.Formatter; 8 | import com.dtstack.jlogstash.render.FreeMarkerRender; 9 | import com.dtstack.jlogstash.render.TemplateRender; 10 | import com.google.common.collect.Lists; 11 | import com.google.common.collect.Maps; 12 | 13 | public class Test { 14 | 15 | public static void main(String[] args) throws Exception{ 16 | // Map event = Maps.newHashMap(); 17 | // event.put("id",12); 18 | // event.put("name", "yxp"); 19 | // Elasticsearch5.cluster="daily_dtstack"; 20 | // Elasticsearch5.documentId="${id}"; 21 | // Elasticsearch5.hosts = Lists.newArrayList("121.43.170.183:9300"); 22 | // Elasticsearch5.index="dt_ysq_log"; 23 | // Elasticsearch5 elasticsearch5 = new Elasticsearch5(Maps.newHashMap()); 24 | // elasticsearch5.prepare(); 25 | // elasticsearch5.emit(event); 26 | // Thread.sleep(30000); 27 | 28 | test(); 29 | } 30 | 31 | public static void test() throws IOException{ 32 | 33 | Map event = Maps.newConcurrentMap(); 34 | event.put("ysq", 123); 35 | // 36 | // TemplateRender idRender = new FreeMarkerRender("${ysq}"); 37 | // 38 | // System.out.println(idRender.render(event)); 39 | 40 | String _index = Formatter.format(event, "logs"); 41 | System.out.println(_index); 42 | System.out.println(Formatter.isFormat(_index)); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch6/.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | *.iml 3 | /target/ 4 | target/* 5 | .project 6 | .settings 7 | .classpath 8 | .metadata/ 9 | -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch6/jlogstash-java-docs/allclasses-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (output.elasticsearch6 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch6/jlogstash-java-docs/allclasses-noframe.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (output.elasticsearch6 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch6/jlogstash-java-docs/com/dtstack/jlogstash/outputs/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.outputs (output.elasticsearch6 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.outputs

14 |
15 |

类

16 | 19 |
20 | 21 | 22 | -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch6/jlogstash-java-docs/package-list: -------------------------------------------------------------------------------- 1 | com.dtstack.jlogstash.outputs 2 | -------------------------------------------------------------------------------- /pipeline/outputs/elasticsearch6/jlogstash-java-docs/script.js: -------------------------------------------------------------------------------- 1 | function show(type) 2 | { 3 | count = 0; 4 | for (var key in methods) { 5 | var row = document.getElementById(key); 6 | if ((methods[key] & type) != 0) { 7 | row.style.display = ''; 8 | row.className = (count++ % 2) ? rowColor : altColor; 9 | } 10 | else 11 | row.style.display = 'none'; 12 | } 13 | updateTabs(type); 14 | } 15 | 16 | function updateTabs(type) 17 | { 18 | for (var value in tabs) { 19 | var sNode = document.getElementById(tabs[value][0]); 20 | var spanNode = sNode.firstChild; 21 | if (value == type) { 22 | sNode.className = activeTableTab; 23 | spanNode.innerHTML = tabs[value][1]; 24 | } 25 | else { 26 | sNode.className = tableTab; 27 | spanNode.innerHTML = "" + tabs[value][1] + ""; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pipeline/outputs/file/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/outputs/file/jlogstash-java-docs/allclasses-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (file 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /pipeline/outputs/file/jlogstash-java-docs/allclasses-noframe.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (file 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /pipeline/outputs/file/jlogstash-java-docs/com/dtstack/jlogstash/outputs/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.outputs (file 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.outputs

14 |
15 |

类

16 | 19 |
20 | 21 | 22 | -------------------------------------------------------------------------------- /pipeline/outputs/file/jlogstash-java-docs/package-list: -------------------------------------------------------------------------------- 1 | com.dtstack.jlogstash.outputs 2 | -------------------------------------------------------------------------------- /pipeline/outputs/file/jlogstash-java-docs/script.js: -------------------------------------------------------------------------------- 1 | function show(type) 2 | { 3 | count = 0; 4 | for (var key in methods) { 5 | var row = document.getElementById(key); 6 | if ((methods[key] & type) != 0) { 7 | row.style.display = ''; 8 | row.className = (count++ % 2) ? rowColor : altColor; 9 | } 10 | else 11 | row.style.display = 'none'; 12 | } 13 | updateTabs(type); 14 | } 15 | 16 | function updateTabs(type) 17 | { 18 | for (var value in tabs) { 19 | var sNode = document.getElementById(tabs[value][0]); 20 | var spanNode = sNode.firstChild; 21 | if (value == type) { 22 | sNode.className = activeTableTab; 23 | spanNode.innerHTML = tabs[value][1]; 24 | } 25 | else { 26 | sNode.className = tableTab; 27 | spanNode.innerHTML = "" + tabs[value][1] + ""; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pipeline/outputs/hdfs/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | *.iml 8 | -------------------------------------------------------------------------------- /pipeline/outputs/hdfs/jlogstash-java-docs/com/dtstack/jlogstash/format/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.format (hdfs 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.format

14 |
15 |

接口

16 | 19 |

类

20 | 24 |

枚举

25 | 30 |
31 | 32 | 33 | -------------------------------------------------------------------------------- /pipeline/outputs/hdfs/jlogstash-java-docs/com/dtstack/jlogstash/format/util/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.format.util (hdfs 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.format.util

14 |
15 |

类

16 | 21 |
22 | 23 | 24 | -------------------------------------------------------------------------------- /pipeline/outputs/hdfs/jlogstash-java-docs/com/dtstack/jlogstash/outputs/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.outputs (hdfs 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.outputs

14 |
15 |

类

16 | 19 |
20 | 21 | 22 | -------------------------------------------------------------------------------- /pipeline/outputs/hdfs/jlogstash-java-docs/org/apache/hadoop/mapred/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | org.apache.hadoop.mapred (hdfs 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

org.apache.hadoop.mapred

14 |
15 |

类

16 | 20 |
21 | 22 | 23 | -------------------------------------------------------------------------------- /pipeline/outputs/hdfs/jlogstash-java-docs/overview-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 概览列表 (hdfs 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |
所有类
14 |
15 |

程序包

16 | 23 |
24 |

 

25 | 26 | 27 | -------------------------------------------------------------------------------- /pipeline/outputs/hdfs/jlogstash-java-docs/package-list: -------------------------------------------------------------------------------- 1 | com.dtstack.jlogstash.format 2 | com.dtstack.jlogstash.format.plugin 3 | com.dtstack.jlogstash.format.util 4 | com.dtstack.jlogstash.outputs 5 | org.apache.hadoop.mapred 6 | -------------------------------------------------------------------------------- /pipeline/outputs/hdfs/jlogstash-java-docs/script.js: -------------------------------------------------------------------------------- 1 | function show(type) 2 | { 3 | count = 0; 4 | for (var key in methods) { 5 | var row = document.getElementById(key); 6 | if ((methods[key] & type) != 0) { 7 | row.style.display = ''; 8 | row.className = (count++ % 2) ? rowColor : altColor; 9 | } 10 | else 11 | row.style.display = 'none'; 12 | } 13 | updateTabs(type); 14 | } 15 | 16 | function updateTabs(type) 17 | { 18 | for (var value in tabs) { 19 | var sNode = document.getElementById(tabs[value][0]); 20 | var spanNode = sNode.firstChild; 21 | if (value == type) { 22 | sNode.className = activeTableTab; 23 | spanNode.innerHTML = tabs[value][1]; 24 | } 25 | else { 26 | sNode.className = tableTab; 27 | spanNode.innerHTML = "" + tabs[value][1] + ""; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pipeline/outputs/hdfs/src/main/java/com/dtstack/jlogstash/format/CompressEnum.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.format; 2 | 3 | /** 4 | * 5 | * @author sishu.yss 6 | * 7 | */ 8 | public enum CompressEnum { 9 | 10 | NONE,GZIP,BZIP2,SNAPPY; 11 | 12 | } 13 | -------------------------------------------------------------------------------- /pipeline/outputs/hdfs/src/main/java/com/dtstack/jlogstash/format/ModeEnum.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.format; 2 | 3 | /** 4 | * 5 | * @author sishu.yss 6 | * 7 | */ 8 | public enum ModeEnum { 9 | 10 | APPEND,NONCONFLICT; 11 | 12 | } 13 | -------------------------------------------------------------------------------- /pipeline/outputs/hdfs/src/main/java/com/dtstack/jlogstash/format/OutputFormat.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.format; 2 | 3 | import java.io.IOException; 4 | import java.io.Serializable; 5 | import java.util.Map; 6 | 7 | /** 8 | * 9 | * @author sishu.yss 10 | * 11 | */ 12 | public interface OutputFormat extends Serializable { 13 | 14 | 15 | void configure(); 16 | 17 | 18 | void open() throws IOException; 19 | 20 | 21 | void writeRecord(Map row) throws IOException; 22 | 23 | 24 | void close() throws IOException; 25 | } 26 | 27 | -------------------------------------------------------------------------------- /pipeline/outputs/hdfs/src/main/java/com/dtstack/jlogstash/format/StoreEnum.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.format; 2 | 3 | /** 4 | * 5 | * @author sishu.yss 6 | * 7 | */ 8 | public enum StoreEnum { 9 | 10 | TEXT,ORC; 11 | 12 | public static String listStore(){ 13 | StringBuilder sb = new StringBuilder(); 14 | for (StoreEnum store : StoreEnum.values()) { 15 | sb.append(store.name()); 16 | sb.append(","); 17 | } 18 | sb.setCharAt(sb.length() - 1,' '); 19 | return sb.toString(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /pipeline/outputs/hdfs/src/main/java/com/dtstack/jlogstash/format/util/HostUtil.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.format.util; 2 | 3 | import java.net.InetAddress; 4 | import java.net.UnknownHostException; 5 | 6 | /** 7 | * 8 | * @author sishu.yss 9 | * 10 | */ 11 | public class HostUtil { 12 | 13 | public static String getHostNameForLiunx() { 14 | try { 15 | return (InetAddress.getLocalHost()).getHostName(); 16 | } catch (UnknownHostException uhe) { 17 | String host = uhe.getMessage(); // host = "hostname: hostname" 18 | if (host != null) { 19 | int colon = host.indexOf(':'); 20 | if (colon > 0) { 21 | return host.substring(0, colon); 22 | } 23 | } 24 | return "UnknownHost"; 25 | } 26 | } 27 | 28 | 29 | public static String getHostName() { 30 | if (System.getenv("COMPUTERNAME") != null) { 31 | return System.getenv("COMPUTERNAME"); 32 | } else { 33 | return getHostNameForLiunx(); 34 | } 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /pipeline/outputs/hive/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | *.iml 8 | -------------------------------------------------------------------------------- /pipeline/outputs/hive/src/main/java/com/dtstack/jlogstash/format/CompressEnum.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.jlogstash.format; 20 | 21 | 22 | /** 23 | * @author haisi 24 | */ 25 | 26 | public enum CompressEnum { 27 | 28 | NONE,GZIP,BZIP2,SNAPPY; 29 | 30 | } 31 | -------------------------------------------------------------------------------- /pipeline/outputs/hive/src/main/java/com/dtstack/jlogstash/format/ModeEnum.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.jlogstash.format; 20 | 21 | 22 | /** 23 | * @author haisi 24 | */ 25 | 26 | public enum ModeEnum { 27 | 28 | APPEND,OVERWRITE; 29 | 30 | } 31 | -------------------------------------------------------------------------------- /pipeline/outputs/hive/src/main/java/com/dtstack/jlogstash/format/OutputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.jlogstash.format; 20 | 21 | import java.io.IOException; 22 | import java.io.Serializable; 23 | import java.util.Map; 24 | 25 | 26 | /** 27 | * @author haisi 28 | */ 29 | public interface OutputFormat extends Serializable { 30 | 31 | 32 | void configure(); 33 | 34 | 35 | void open() throws IOException; 36 | 37 | 38 | Object[] convert2Record(Map row) throws Exception; 39 | 40 | void writeRecord(Object[] record) throws Exception; 41 | 42 | 43 | void close() throws IOException; 44 | } 45 | 46 | -------------------------------------------------------------------------------- /pipeline/outputs/hive/src/main/java/com/dtstack/jlogstash/format/StoreEnum.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.jlogstash.format; 20 | 21 | 22 | /** 23 | * @author haisi 24 | */ 25 | 26 | public enum StoreEnum { 27 | 28 | TEXT,ORC; 29 | 30 | public static String listStore(){ 31 | StringBuilder sb = new StringBuilder(); 32 | for (StoreEnum store : StoreEnum.values()) { 33 | sb.append(store.name()); 34 | sb.append(","); 35 | } 36 | sb.setCharAt(sb.length() - 1,' '); 37 | return sb.toString(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /pipeline/outputs/kafka/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/outputs/kafka/jlogstash-java-docs/allclasses-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (kafka 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /pipeline/outputs/kafka/jlogstash-java-docs/allclasses-noframe.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (kafka 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /pipeline/outputs/kafka/jlogstash-java-docs/com/dtstack/jlogstash/outputs/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.outputs (kafka 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.outputs

14 |
15 |

类

16 | 19 |
20 | 21 | 22 | -------------------------------------------------------------------------------- /pipeline/outputs/kafka/jlogstash-java-docs/package-list: -------------------------------------------------------------------------------- 1 | com.dtstack.jlogstash.outputs 2 | -------------------------------------------------------------------------------- /pipeline/outputs/kafka/jlogstash-java-docs/script.js: -------------------------------------------------------------------------------- 1 | function show(type) 2 | { 3 | count = 0; 4 | for (var key in methods) { 5 | var row = document.getElementById(key); 6 | if ((methods[key] & type) != 0) { 7 | row.style.display = ''; 8 | row.className = (count++ % 2) ? rowColor : altColor; 9 | } 10 | else 11 | row.style.display = 'none'; 12 | } 13 | updateTabs(type); 14 | } 15 | 16 | function updateTabs(type) 17 | { 18 | for (var value in tabs) { 19 | var sNode = document.getElementById(tabs[value][0]); 20 | var spanNode = sNode.firstChild; 21 | if (value == type) { 22 | sNode.className = activeTableTab; 23 | spanNode.innerHTML = tabs[value][1]; 24 | } 25 | else { 26 | sNode.className = tableTab; 27 | spanNode.innerHTML = "" + tabs[value][1] + ""; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pipeline/outputs/kafka10/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | *.iml 8 | -------------------------------------------------------------------------------- /pipeline/outputs/kafka10/jlogstash-java-docs/allclasses-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (kafka10 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 21 |
22 | 23 | 24 | -------------------------------------------------------------------------------- /pipeline/outputs/kafka10/jlogstash-java-docs/allclasses-noframe.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (kafka10 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 21 |
22 | 23 | 24 | -------------------------------------------------------------------------------- /pipeline/outputs/kafka10/jlogstash-java-docs/com/dtstack/jlogstash/outputs/kafka/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.outputs.kafka (kafka10 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.outputs.kafka

14 |
15 |

类

16 | 21 |
22 | 23 | 24 | -------------------------------------------------------------------------------- /pipeline/outputs/kafka10/jlogstash-java-docs/com/dtstack/jlogstash/outputs/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.outputs (kafka10 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.outputs

14 |
15 |

类

16 | 19 |
20 | 21 | 22 | -------------------------------------------------------------------------------- /pipeline/outputs/kafka10/jlogstash-java-docs/overview-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 概览列表 (kafka10 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |
所有类
14 |
15 |

程序包

16 | 20 |
21 |

 

22 | 23 | 24 | -------------------------------------------------------------------------------- /pipeline/outputs/kafka10/jlogstash-java-docs/package-list: -------------------------------------------------------------------------------- 1 | com.dtstack.jlogstash.outputs 2 | com.dtstack.jlogstash.outputs.kafka 3 | -------------------------------------------------------------------------------- /pipeline/outputs/kafka10/jlogstash-java-docs/script.js: -------------------------------------------------------------------------------- 1 | function show(type) 2 | { 3 | count = 0; 4 | for (var key in methods) { 5 | var row = document.getElementById(key); 6 | if ((methods[key] & type) != 0) { 7 | row.style.display = ''; 8 | row.className = (count++ % 2) ? rowColor : altColor; 9 | } 10 | else 11 | row.style.display = 'none'; 12 | } 13 | updateTabs(type); 14 | } 15 | 16 | function updateTabs(type) 17 | { 18 | for (var value in tabs) { 19 | var sNode = document.getElementById(tabs[value][0]); 20 | var spanNode = sNode.firstChild; 21 | if (value == type) { 22 | sNode.className = activeTableTab; 23 | spanNode.innerHTML = tabs[value][1]; 24 | } 25 | else { 26 | sNode.className = tableTab; 27 | spanNode.innerHTML = "" + tabs[value][1] + ""; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pipeline/outputs/kafka10/src/main/java/com/dtstack/jlogstash/outputs/kafka/MonitorCluster.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.outputs.kafka; 2 | 3 | import java.util.List; 4 | import java.util.concurrent.ThreadFactory; 5 | import java.util.concurrent.atomic.AtomicBoolean; 6 | import java.util.concurrent.atomic.AtomicInteger; 7 | 8 | import org.I0Itec.zkclient.ZkClient; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | public class MonitorCluster implements Runnable { 13 | 14 | private static final Logger logger = LoggerFactory.getLogger(MonitorCluster.class); 15 | 16 | private String zkAddress = "localhost:2181"; 17 | 18 | private ZkClient zkClient; 19 | 20 | private volatile AtomicBoolean isKafkaHealth; 21 | 22 | public MonitorCluster(String zkAddress, AtomicBoolean isKafkaHealth){ 23 | this.zkAddress = zkAddress; 24 | this.zkClient = new ZkClient(this.zkAddress); 25 | this.isKafkaHealth = isKafkaHealth; 26 | } 27 | 28 | 29 | @Override 30 | public void run() { 31 | 32 | if(healthcheck()) { 33 | isKafkaHealth.set(true); 34 | } else { 35 | isKafkaHealth.set(false); 36 | } 37 | } 38 | 39 | public boolean healthcheck(){ 40 | 41 | try { 42 | String brokersIdsPath = getBrokersIds(); 43 | List brokersList = zkClient.getChildren(brokersIdsPath); 44 | if(brokersList == null || brokersList.size() == 0) { 45 | logger.warn("kafka healthcheck, brokersList is empty"); 46 | return false; 47 | } 48 | logger.debug("kafka healthcheck, brokersList is not empty"); 49 | return true; 50 | } catch(Exception e) { 51 | logger.error("kafka healthcheck", e); 52 | return false; 53 | } 54 | } 55 | 56 | public String getBrokersIds(){ 57 | return "/brokers/ids"; 58 | } 59 | 60 | 61 | } -------------------------------------------------------------------------------- /pipeline/outputs/kafka10/src/main/java/com/dtstack/jlogstash/outputs/kafka/NamedThreadFactory.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.outputs.kafka; 2 | 3 | import java.util.concurrent.ThreadFactory; 4 | import java.util.concurrent.atomic.AtomicInteger; 5 | 6 | /** 7 | * Created by daguan on 18/9/3. 8 | */ 9 | public class NamedThreadFactory implements ThreadFactory { 10 | 11 | private static final AtomicInteger poolNumber = new AtomicInteger(1); 12 | 13 | final AtomicInteger threadNumber = new AtomicInteger(1); 14 | final ThreadGroup group; 15 | final String namePrefix; 16 | final boolean isDaemon; 17 | 18 | public NamedThreadFactory() { 19 | this("pool"); 20 | } 21 | 22 | public NamedThreadFactory(String name) { 23 | this(name, false); 24 | } 25 | 26 | public NamedThreadFactory(String preffix, boolean daemon) { 27 | SecurityManager s = System.getSecurityManager(); 28 | group = (s != null) ? s.getThreadGroup() : Thread.currentThread() 29 | .getThreadGroup(); 30 | namePrefix = preffix + "-" + poolNumber.getAndIncrement() + "-thread-"; 31 | isDaemon = daemon; 32 | } 33 | 34 | @Override 35 | public Thread newThread(Runnable r) { 36 | Thread t = new Thread(group, r, namePrefix 37 | + threadNumber.getAndIncrement(), 0); 38 | t.setDaemon(isDaemon); 39 | if (t.getPriority() != Thread.NORM_PRIORITY) { 40 | t.setPriority(Thread.NORM_PRIORITY); 41 | } 42 | return t; 43 | } 44 | 45 | public static void main(String[] args) { 46 | 47 | } 48 | 49 | } -------------------------------------------------------------------------------- /pipeline/outputs/kafka11/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | *.iml 8 | -------------------------------------------------------------------------------- /pipeline/outputs/kafka11/src/main/java/com/dtstack/jlogstash/outputs/kafka/NamedThreadFactory.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.outputs.kafka; 2 | 3 | import java.util.concurrent.ThreadFactory; 4 | import java.util.concurrent.atomic.AtomicInteger; 5 | 6 | /** 7 | * @author: haisi 8 | * @date 2019-06-25 10:44 9 | */ 10 | public class NamedThreadFactory implements ThreadFactory { 11 | 12 | public static final AtomicInteger poolNumber = new AtomicInteger(1); 13 | 14 | final AtomicInteger threadNumber = new AtomicInteger(1); 15 | final ThreadGroup group; 16 | final String namePrefix; 17 | final boolean isDaemon; 18 | 19 | 20 | public NamedThreadFactory(){ 21 | this("pool"); 22 | } 23 | 24 | public NamedThreadFactory(String name) { 25 | this(name,false); 26 | } 27 | 28 | public NamedThreadFactory(String preffix, boolean daemon){ 29 | SecurityManager s = System.getSecurityManager(); 30 | group = (s!=null) ? s.getThreadGroup() : Thread.currentThread() 31 | .getThreadGroup(); 32 | namePrefix = preffix + "-"+ poolNumber.getAndIncrement()+"-thread-"; 33 | isDaemon = daemon; 34 | } 35 | 36 | @Override 37 | public Thread newThread(Runnable r) { 38 | Thread t = new Thread(group,r,namePrefix+threadNumber.getAndIncrement(),0); 39 | t.setDaemon(this.isDaemon); 40 | if (t.getPriority()!=Thread.NORM_PRIORITY){ 41 | t.setPriority(Thread.NORM_PRIORITY); 42 | } 43 | return t; 44 | } 45 | 46 | public static void main(String[] args) { 47 | 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /pipeline/outputs/netty/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | .idea 8 | output.netty.iml 9 | -------------------------------------------------------------------------------- /pipeline/outputs/netty/jlogstash-java-docs/allclasses-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (netty 1.0.1 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 21 |
22 | 23 | 24 | -------------------------------------------------------------------------------- /pipeline/outputs/netty/jlogstash-java-docs/allclasses-noframe.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (netty 1.0.1 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 21 |
22 | 23 | 24 | -------------------------------------------------------------------------------- /pipeline/outputs/netty/jlogstash-java-docs/com/dtstack/jlogstash/outputs/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.outputs (netty 1.0.1 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.outputs

14 |
15 |

类

16 | 19 |
20 | 21 | 22 | -------------------------------------------------------------------------------- /pipeline/outputs/netty/jlogstash-java-docs/com/dtstack/jlogstash/outputs/util/flow/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.outputs.util.flow (netty 1.0.1 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.outputs.util.flow

14 |
15 |

类

16 | 20 |
21 | 22 | 23 | -------------------------------------------------------------------------------- /pipeline/outputs/netty/jlogstash-java-docs/com/dtstack/jlogstash/outputs/util/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.outputs.util (netty 1.0.1 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.outputs.util

14 |
15 |

类

16 | 19 |
20 | 21 | 22 | -------------------------------------------------------------------------------- /pipeline/outputs/netty/jlogstash-java-docs/overview-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 概览列表 (netty 1.0.1 API) 8 | 9 | 10 | 11 | 12 | 13 |
所有类
14 |
15 |

程序包

16 | 21 |
22 |

 

23 | 24 | 25 | -------------------------------------------------------------------------------- /pipeline/outputs/netty/jlogstash-java-docs/package-list: -------------------------------------------------------------------------------- 1 | com.dtstack.jlogstash.outputs 2 | com.dtstack.jlogstash.outputs.util 3 | com.dtstack.jlogstash.outputs.util.flow 4 | -------------------------------------------------------------------------------- /pipeline/outputs/netty/jlogstash-java-docs/script.js: -------------------------------------------------------------------------------- 1 | function show(type) 2 | { 3 | count = 0; 4 | for (var key in methods) { 5 | var row = document.getElementById(key); 6 | if ((methods[key] & type) != 0) { 7 | row.style.display = ''; 8 | row.className = (count++ % 2) ? rowColor : altColor; 9 | } 10 | else 11 | row.style.display = 'none'; 12 | } 13 | updateTabs(type); 14 | } 15 | 16 | function updateTabs(type) 17 | { 18 | for (var value in tabs) { 19 | var sNode = document.getElementById(tabs[value][0]); 20 | var spanNode = sNode.firstChild; 21 | if (value == type) { 22 | sNode.className = activeTableTab; 23 | spanNode.innerHTML = tabs[value][1]; 24 | } 25 | else { 26 | sNode.className = tableTab; 27 | spanNode.innerHTML = "" + tabs[value][1] + ""; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pipeline/outputs/netty/src/main/java/com/dtstack/jlogstash/outputs/util/flow/FlowControlShiper.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.jlogstash.outputs.util.flow; 2 | 3 | import com.google.common.util.concurrent.RateLimiter; 4 | 5 | /** 6 | * Created by daguan on 17/6/15. 7 | */ 8 | public class FlowControlShiper { 9 | 10 | private RateLimiter rateLimiter; 11 | 12 | public FlowControlShiper(Threshold threshold) { 13 | rateLimiter = RateLimiter.create(threshold.getValue()); 14 | } 15 | 16 | public double acquire() { 17 | return rateLimiter.acquire(); 18 | } 19 | 20 | public double acquire(int permits) { 21 | return rateLimiter.acquire(permits); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /pipeline/outputs/odps/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/outputs/performance/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/outputs/performance/jlogstash-java-docs/allclasses-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (performance 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /pipeline/outputs/performance/jlogstash-java-docs/allclasses-noframe.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (performance 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /pipeline/outputs/performance/jlogstash-java-docs/com/dtstack/jlogstash/outputs/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.outputs (performance 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.outputs

14 |
15 |

类

16 | 19 |
20 | 21 | 22 | -------------------------------------------------------------------------------- /pipeline/outputs/performance/jlogstash-java-docs/package-list: -------------------------------------------------------------------------------- 1 | com.dtstack.jlogstash.outputs 2 | -------------------------------------------------------------------------------- /pipeline/outputs/performance/jlogstash-java-docs/script.js: -------------------------------------------------------------------------------- 1 | function show(type) 2 | { 3 | count = 0; 4 | for (var key in methods) { 5 | var row = document.getElementById(key); 6 | if ((methods[key] & type) != 0) { 7 | row.style.display = ''; 8 | row.className = (count++ % 2) ? rowColor : altColor; 9 | } 10 | else 11 | row.style.display = 'none'; 12 | } 13 | updateTabs(type); 14 | } 15 | 16 | function updateTabs(type) 17 | { 18 | for (var value in tabs) { 19 | var sNode = document.getElementById(tabs[value][0]); 20 | var spanNode = sNode.firstChild; 21 | if (value == type) { 22 | sNode.className = activeTableTab; 23 | spanNode.innerHTML = tabs[value][1]; 24 | } 25 | else { 26 | sNode.className = tableTab; 27 | spanNode.innerHTML = "" + tabs[value][1] + ""; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pipeline/outputs/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | com.dtstack.jlogstash 7 | jlogstash-all 8 | 1.0.0 9 | ../../pom.xml 10 | 11 | 4.0.0 12 | output.plugin 13 | pom 14 | 15 | elasticsearch 16 | elasticsearch5 17 | elasticsearch6 18 | file 19 | hdfs 20 | kafka 21 | kafka10 22 | kafka11 23 | netty 24 | odps 25 | performance 26 | stdout 27 | hive 28 | 29 | 30 | 31 | 32 | com.dtstack.jlogstash 33 | jlogstash 34 | 1.0.0 35 | provided 36 | 37 | 38 | junit 39 | junit 40 | 4.13.1 41 | test 42 | 43 | 44 | -------------------------------------------------------------------------------- /pipeline/outputs/stdout/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | target/* 3 | .project 4 | .settings 5 | .classpath 6 | .metadata/ 7 | -------------------------------------------------------------------------------- /pipeline/outputs/stdout/jlogstash-java-docs/allclasses-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (stdout 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /pipeline/outputs/stdout/jlogstash-java-docs/allclasses-noframe.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 所有类 (stdout 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

所有类

14 |
15 | 18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /pipeline/outputs/stdout/jlogstash-java-docs/com/dtstack/jlogstash/outputs/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.dtstack.jlogstash.outputs (stdout 1.0.0 API) 8 | 9 | 10 | 11 | 12 | 13 |

com.dtstack.jlogstash.outputs

14 |
15 |

类

16 | 19 |
20 | 21 | 22 | -------------------------------------------------------------------------------- /pipeline/outputs/stdout/jlogstash-java-docs/package-list: -------------------------------------------------------------------------------- 1 | com.dtstack.jlogstash.outputs 2 | -------------------------------------------------------------------------------- /pipeline/outputs/stdout/jlogstash-java-docs/script.js: -------------------------------------------------------------------------------- 1 | function show(type) 2 | { 3 | count = 0; 4 | for (var key in methods) { 5 | var row = document.getElementById(key); 6 | if ((methods[key] & type) != 0) { 7 | row.style.display = ''; 8 | row.className = (count++ % 2) ? rowColor : altColor; 9 | } 10 | else 11 | row.style.display = 'none'; 12 | } 13 | updateTabs(type); 14 | } 15 | 16 | function updateTabs(type) 17 | { 18 | for (var value in tabs) { 19 | var sNode = document.getElementById(tabs[value][0]); 20 | var spanNode = sNode.firstChild; 21 | if (value == type) { 22 | sNode.className = activeTableTab; 23 | spanNode.innerHTML = tabs[value][1]; 24 | } 25 | else { 26 | sNode.className = tableTab; 27 | spanNode.innerHTML = "" + tabs[value][1] + ""; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /plugin/README.MD: -------------------------------------------------------------------------------- 1 | jlogstash 插件代码 2 | -------------------------------------------------------------------------------- /plugin/filter/README.MD: -------------------------------------------------------------------------------- 1 | filter 插件代码jar 2 | -------------------------------------------------------------------------------- /plugin/input/README.MD: -------------------------------------------------------------------------------- 1 | input 插件代码jar 2 | -------------------------------------------------------------------------------- /plugin/output/README.MD: -------------------------------------------------------------------------------- 1 | output 插件代码jar 2 | -------------------------------------------------------------------------------- /test/binlog_hdfs_text.json: -------------------------------------------------------------------------------- 1 | { 2 | "outputs" : [ { 3 | "Kafka11" : { 4 | "bootstrapServers" : "localhost:9092", 5 | "topic": "dt_all_log", 6 | "zkAddress":"localhost:2181" 7 | } 8 | } ], 9 | "inputs" : [ { 10 | "Binlog" : { 11 | "password" : "abc123", 12 | "cat" : "insert,update,delete", 13 | "hadoopConfigMap": { 14 | "dfs.ha.namenodes.ns1": "nn1,nn2", 15 | "fs.defaultFS": "hdfs://ns1", 16 | "dfs.namenode.rpc-address.ns1.nn2": "node002:9000", 17 | "dfs.client.failover.proxy.provider.ns1": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider", 18 | "dfs.namenode.rpc-address.ns1.nn1": "node001:9000", 19 | "dfs.nameservices": "ns1", 20 | "fs.hdfs.impl.disable.cache": true, 21 | "fs.hdfs.impl": "org.apache.hadoop.hdfs.DistributedFileSystem" 22 | }, 23 | "jdbcUrl" : "jdbc:mysql://172.16.10.45:3306/test", 24 | "host": "172.16.10.45", 25 | "start" : { }, 26 | "table" : [ "date_test", "es_sink", "inserttype", "myresult", "pv1", "sales", "stmresult", "subtable3", "subtable4", "test_text", "testmuyun", "time_test", "totalsales", "user_list", "user_sink", "user_source", "xcpv1", "xctime", "ysq_order_fact20_carbon_3" ], 27 | "username" : "dtstack" 28 | } 29 | } ] 30 | } --------------------------------------------------------------------------------