├── README.md ├── code ├── log_analyse │ ├── .idea │ │ ├── artifacts │ │ │ └── log_analyse_war_exploded.xml │ │ ├── compiler.xml │ │ ├── misc.xml │ │ ├── modules.xml │ │ └── workspace.xml │ ├── log_analyse.iml │ ├── out │ │ ├── artifacts │ │ │ └── log_analyse_war_exploded │ │ │ │ ├── WEB-INF │ │ │ │ ├── classes │ │ │ │ │ └── com │ │ │ │ │ │ └── msb │ │ │ │ │ │ ├── client │ │ │ │ │ │ ├── AnalyticsEngineSDK.class │ │ │ │ │ │ ├── SendDataMonitor$1.class │ │ │ │ │ │ ├── SendDataMonitor$HttpRequestUtil.class │ │ │ │ │ │ └── SendDataMonitor.class │ │ │ │ │ │ └── test │ │ │ │ │ │ └── Test.class │ │ │ │ └── web.xml │ │ │ │ ├── demo.jsp │ │ │ │ ├── demo2.jsp │ │ │ │ ├── demo3.jsp │ │ │ │ ├── demo4.jsp │ │ │ │ ├── index.jsp │ │ │ │ └── js │ │ │ │ └── analytics.js │ │ └── production │ │ │ └── log_analyse │ │ │ └── com │ │ │ └── msb │ │ │ ├── client │ │ │ ├── AnalyticsEngineSDK.class │ │ │ ├── SendDataMonitor$1.class │ │ │ ├── SendDataMonitor$HttpRequestUtil.class │ │ │ └── SendDataMonitor.class │ │ │ └── test │ │ │ └── Test.class │ ├── src │ │ └── com │ │ │ └── msb │ │ │ ├── client │ │ │ ├── AnalyticsEngineSDK.java │ │ │ └── SendDataMonitor.java │ │ │ └── test │ │ │ └── Test.java │ └── web │ │ ├── WEB-INF │ │ └── web.xml │ │ ├── demo.jsp │ │ ├── demo2.jsp │ │ ├── demo3.jsp │ │ ├── demo4.jsp │ │ ├── index.jsp │ │ └── js │ │ └── analytics.js ├── log_analyse_dataclean │ ├── .idea │ │ ├── compiler.xml │ │ ├── encodings.xml │ │ ├── misc.xml │ │ └── workspace.xml │ ├── log_analyse_dataclean.iml │ ├── pom.xml │ ├── src │ │ └── main │ │ │ ├── java │ │ │ ├── com │ │ │ │ └── mashibing │ │ │ │ │ ├── common │ │ │ │ │ ├── DateEnum.java │ │ │ │ │ ├── EventLogConstants.java │ │ │ │ │ ├── GlobalConstants.java │ │ │ │ │ └── KpiType.java │ │ │ │ │ ├── etl │ │ │ │ │ ├── mr │ │ │ │ │ │ └── ald │ │ │ │ │ │ │ ├── AnalyserLogDataMapper.java │ │ │ │ │ │ │ └── AnalyserLogDataRunner.java │ │ │ │ │ └── util │ │ │ │ │ │ ├── IPSeekerExt.java │ │ │ │ │ │ ├── LoggerUtil.java │ │ │ │ │ │ ├── UserAgentUtil.java │ │ │ │ │ │ └── ip │ │ │ │ │ │ └── IPSeeker.java │ │ │ │ │ └── util │ │ │ │ │ ├── EJob.java │ │ │ │ │ ├── JdbcManager.java │ │ │ │ │ └── TimeUtil.java │ │ │ └── test │ │ │ │ ├── TestDataMaker.java │ │ │ │ ├── TestIPSeeker.java │ │ │ │ ├── TestIPSeekerExt.java │ │ │ │ ├── TestLoggerUtil.java │ │ │ │ └── TestUserAgentUtil.java │ │ │ └── resource │ │ │ ├── core-site.xml │ │ │ ├── hdfs-site.xml │ │ │ ├── log4j.properties │ │ │ ├── mapred-site.xml │ │ │ ├── output-collector.xml │ │ │ ├── qqwry.dat │ │ │ ├── query-mapping.xml │ │ │ ├── transformer-env.xml │ │ │ └── yarn-site.xml │ └── target │ │ └── classes │ │ ├── com │ │ └── mashibing │ │ │ ├── common │ │ │ ├── DateEnum.class │ │ │ ├── EventLogConstants$EventEnum.class │ │ │ ├── EventLogConstants.class │ │ │ ├── GlobalConstants.class │ │ │ └── KpiType.class │ │ │ ├── etl │ │ │ ├── mr │ │ │ │ └── ald │ │ │ │ │ ├── AnalyserLogDataMapper$1.class │ │ │ │ │ ├── AnalyserLogDataMapper.class │ │ │ │ │ └── AnalyserLogDataRunner.class │ │ │ └── util │ │ │ │ ├── IPSeekerExt$RegionInfo.class │ │ │ │ ├── IPSeekerExt.class │ │ │ │ ├── LoggerUtil.class │ │ │ │ ├── UserAgentUtil$UserAgentInfo.class │ │ │ │ ├── UserAgentUtil.class │ │ │ │ └── ip │ │ │ │ ├── IPSeeker$IPEntry.class │ │ │ │ ├── IPSeeker$IPLocation.class │ │ │ │ ├── IPSeeker$IPSeekerUtils.class │ │ │ │ └── IPSeeker.class │ │ │ └── util │ │ │ ├── EJob$1.class │ │ │ ├── EJob.class │ │ │ ├── JdbcManager.class │ │ │ └── TimeUtil.class │ │ ├── core-site.xml │ │ ├── hdfs-site.xml │ │ ├── log4j.properties │ │ ├── mapred-site.xml │ │ ├── output-collector.xml │ │ ├── qqwry.dat │ │ ├── query-mapping.xml │ │ ├── test │ │ ├── Sex.class │ │ ├── TestIPSeek.class │ │ └── TestUserAgent.class │ │ ├── transformer-env.xml │ │ └── yarn-site.xml ├── log_analyse_full │ ├── .idea │ │ ├── compiler.xml │ │ ├── encodings.xml │ │ ├── misc.xml │ │ └── workspace.xml │ ├── log_analyse_full.iml │ ├── pom.xml │ ├── src │ │ └── main │ │ │ ├── java │ │ │ └── com │ │ │ │ └── mashibing │ │ │ │ ├── common │ │ │ │ ├── DateEnum.java │ │ │ │ ├── EventLogConstants.java │ │ │ │ ├── GlobalConstants.java │ │ │ │ └── KpiType.java │ │ │ │ ├── etl │ │ │ │ ├── mr │ │ │ │ │ └── ald │ │ │ │ │ │ ├── AnalyserLogDataMapper.java │ │ │ │ │ │ └── AnalyserLogDataRunner.java │ │ │ │ └── util │ │ │ │ │ ├── IPSeekerExt.java │ │ │ │ │ ├── LoggerUtil.java │ │ │ │ │ ├── UserAgentUtil.java │ │ │ │ │ └── ip │ │ │ │ │ └── IPSeeker.java │ │ │ │ ├── transformer │ │ │ │ ├── hive │ │ │ │ │ ├── DateDimensionUDF.java │ │ │ │ │ └── vd │ │ │ │ │ │ ├── statsViewDepth.hql │ │ │ │ │ │ └── view_depth_run.sh │ │ │ │ ├── model │ │ │ │ │ ├── dim │ │ │ │ │ │ ├── StatsCommonDimension.java │ │ │ │ │ │ ├── StatsDimension.java │ │ │ │ │ │ ├── StatsInboundBounceDimension.java │ │ │ │ │ │ ├── StatsInboundDimension.java │ │ │ │ │ │ ├── StatsLocationDimension.java │ │ │ │ │ │ ├── StatsUserDimension.java │ │ │ │ │ │ └── base │ │ │ │ │ │ │ ├── BaseDimension.java │ │ │ │ │ │ │ ├── BrowserDimension.java │ │ │ │ │ │ │ ├── DateDimension.java │ │ │ │ │ │ │ ├── InboundDimension.java │ │ │ │ │ │ │ ├── KpiDimension.java │ │ │ │ │ │ │ ├── LocationDimension.java │ │ │ │ │ │ │ └── PlatformDimension.java │ │ │ │ │ └── value │ │ │ │ │ │ ├── BaseStatsValueWritable.java │ │ │ │ │ │ ├── map │ │ │ │ │ │ ├── TextsOutputValue.java │ │ │ │ │ │ └── TimeOutputValue.java │ │ │ │ │ │ └── reduce │ │ │ │ │ │ ├── InboundBounceReduceValue.java │ │ │ │ │ │ ├── InboundReduceValue.java │ │ │ │ │ │ ├── LocationReducerOutputValue.java │ │ │ │ │ │ └── MapWritableValue.java │ │ │ │ ├── mr │ │ │ │ │ ├── IOutputCollector.java │ │ │ │ │ ├── TransformerBaseMapper.java │ │ │ │ │ ├── TransformerBaseRunner.java │ │ │ │ │ ├── TransformerOutputFormat.java │ │ │ │ │ ├── am │ │ │ │ │ │ ├── ActiveMemberCollector.java │ │ │ │ │ │ ├── ActiveMemberMapper.java │ │ │ │ │ │ ├── ActiveMemberReducer.java │ │ │ │ │ │ ├── ActiveMemberRunner.java │ │ │ │ │ │ └── package-info.java │ │ │ │ │ ├── au │ │ │ │ │ │ ├── ActiveUserBrowserCollector.java │ │ │ │ │ │ ├── ActiveUserCollector.java │ │ │ │ │ │ ├── ActiveUserCombine.java │ │ │ │ │ │ ├── ActiveUserMapper.java │ │ │ │ │ │ ├── ActiveUserReducer.java │ │ │ │ │ │ ├── ActiveUserRunner.java │ │ │ │ │ │ └── HourlyActiveUserCollector.java │ │ │ │ │ ├── inbound │ │ │ │ │ │ ├── InboundCollector.java │ │ │ │ │ │ ├── InboundMapper.java │ │ │ │ │ │ ├── InboundReducer.java │ │ │ │ │ │ ├── InboundRunner.java │ │ │ │ │ │ ├── bounce │ │ │ │ │ │ │ ├── InboundBounceCollector.java │ │ │ │ │ │ │ ├── InboundBounceMapper.java │ │ │ │ │ │ │ ├── InboundBounceReducer.java │ │ │ │ │ │ │ ├── InboundBounceRunner.java │ │ │ │ │ │ │ └── InboundBounceSecondSort.java │ │ │ │ │ │ └── package-info.java │ │ │ │ │ ├── location │ │ │ │ │ │ ├── LocationCollector.java │ │ │ │ │ │ ├── LocationMapper.java │ │ │ │ │ │ ├── LocationReducer.java │ │ │ │ │ │ ├── LocationRunner.java │ │ │ │ │ │ └── package-info.java │ │ │ │ │ ├── nm │ │ │ │ │ │ ├── NewMemberCollector.java │ │ │ │ │ │ ├── NewMemberMapper.java │ │ │ │ │ │ ├── NewMemberReducer.java │ │ │ │ │ │ ├── NewMemberRunner.java │ │ │ │ │ │ └── package-info.java │ │ │ │ │ ├── nu │ │ │ │ │ │ ├── NewInstallUserMapper.java │ │ │ │ │ │ ├── NewInstallUserReducer.java │ │ │ │ │ │ ├── NewInstallUserRunner.java │ │ │ │ │ │ ├── StatsDeviceBrowserNewInstallUserCollector.java │ │ │ │ │ │ └── StatsUserNewInstallUserCollector.java │ │ │ │ │ ├── pv │ │ │ │ │ │ ├── PageViewCollector.java │ │ │ │ │ │ ├── PageViewMapper.java │ │ │ │ │ │ ├── PageViewReducer.java │ │ │ │ │ │ └── PageViewRunner.java │ │ │ │ │ └── sessions │ │ │ │ │ │ ├── SessionsCollector.java │ │ │ │ │ │ ├── SessionsMapper.java │ │ │ │ │ │ ├── SessionsReducer.java │ │ │ │ │ │ ├── SessionsRunner.java │ │ │ │ │ │ └── package-info.java │ │ │ │ ├── service │ │ │ │ │ ├── IDimensionConverter.java │ │ │ │ │ └── impl │ │ │ │ │ │ ├── DimensionConverterImpl.java │ │ │ │ │ │ └── InboundDimensionService.java │ │ │ │ └── util │ │ │ │ │ ├── MemberUtil.java │ │ │ │ │ ├── TimeChain.java │ │ │ │ │ └── UrlUtil.java │ │ │ │ └── util │ │ │ │ ├── EJob.java │ │ │ │ ├── JdbcManager.java │ │ │ │ └── TimeUtil.java │ │ │ └── resource │ │ │ ├── core-site.xml │ │ │ ├── hdfs-site.xml │ │ │ ├── log4j.properties │ │ │ ├── mapred-site.xml │ │ │ ├── output-collector.xml │ │ │ ├── qqwry.dat │ │ │ ├── query-mapping.xml │ │ │ ├── transformer-env.xml │ │ │ └── yarn-site.xml │ └── target │ │ └── classes │ │ ├── com │ │ └── mashibing │ │ │ ├── common │ │ │ ├── DateEnum.class │ │ │ ├── EventLogConstants$EventEnum.class │ │ │ ├── EventLogConstants.class │ │ │ ├── GlobalConstants.class │ │ │ └── KpiType.class │ │ │ ├── etl │ │ │ ├── mr │ │ │ │ └── ald │ │ │ │ │ ├── AnalyserLogDataMapper$1.class │ │ │ │ │ ├── AnalyserLogDataMapper.class │ │ │ │ │ └── AnalyserLogDataRunner.class │ │ │ └── util │ │ │ │ ├── IPSeekerExt$RegionInfo.class │ │ │ │ ├── IPSeekerExt.class │ │ │ │ ├── LoggerUtil.class │ │ │ │ ├── UserAgentUtil$UserAgentInfo.class │ │ │ │ ├── UserAgentUtil.class │ │ │ │ └── ip │ │ │ │ ├── IPSeeker$IPEntry.class │ │ │ │ ├── IPSeeker$IPLocation.class │ │ │ │ ├── IPSeeker$IPSeekerUtils.class │ │ │ │ └── IPSeeker.class │ │ │ ├── transformer │ │ │ ├── hive │ │ │ │ ├── DateDimensionUDF.class │ │ │ │ └── vd │ │ │ │ │ ├── statsViewDepth.hql │ │ │ │ │ └── view_depth_run.sh │ │ │ ├── model │ │ │ │ ├── dim │ │ │ │ │ ├── StatsCommonDimension.class │ │ │ │ │ ├── StatsDimension.class │ │ │ │ │ ├── StatsInboundBounceDimension.class │ │ │ │ │ ├── StatsInboundDimension.class │ │ │ │ │ ├── StatsLocationDimension.class │ │ │ │ │ ├── StatsUserDimension.class │ │ │ │ │ └── base │ │ │ │ │ │ ├── BaseDimension.class │ │ │ │ │ │ ├── BrowserDimension.class │ │ │ │ │ │ ├── DateDimension.class │ │ │ │ │ │ ├── InboundDimension.class │ │ │ │ │ │ ├── KpiDimension.class │ │ │ │ │ │ ├── LocationDimension.class │ │ │ │ │ │ └── PlatformDimension.class │ │ │ │ └── value │ │ │ │ │ ├── BaseStatsValueWritable.class │ │ │ │ │ ├── map │ │ │ │ │ ├── TextsOutputValue.class │ │ │ │ │ └── TimeOutputValue.class │ │ │ │ │ └── reduce │ │ │ │ │ ├── InboundBounceReduceValue.class │ │ │ │ │ ├── InboundReduceValue.class │ │ │ │ │ ├── LocationReducerOutputValue.class │ │ │ │ │ └── MapWritableValue.class │ │ │ ├── mr │ │ │ │ ├── IOutputCollector.class │ │ │ │ ├── TransformerBaseMapper.class │ │ │ │ ├── TransformerBaseRunner.class │ │ │ │ ├── TransformerOutputFormat$TransformerRecordWriter.class │ │ │ │ ├── TransformerOutputFormat.class │ │ │ │ ├── am │ │ │ │ │ ├── ActiveMemberCollector.class │ │ │ │ │ ├── ActiveMemberMapper.class │ │ │ │ │ ├── ActiveMemberReducer.class │ │ │ │ │ └── ActiveMemberRunner.class │ │ │ │ ├── au │ │ │ │ │ ├── ActiveUserBrowserCollector.class │ │ │ │ │ ├── ActiveUserCollector.class │ │ │ │ │ ├── ActiveUserCombine.class │ │ │ │ │ ├── ActiveUserMapper.class │ │ │ │ │ ├── ActiveUserReducer.class │ │ │ │ │ ├── ActiveUserRunner$ActiveUserPartitioner.class │ │ │ │ │ ├── ActiveUserRunner.class │ │ │ │ │ └── HourlyActiveUserCollector.class │ │ │ │ ├── inbound │ │ │ │ │ ├── InboundCollector.class │ │ │ │ │ ├── InboundMapper.class │ │ │ │ │ ├── InboundReducer.class │ │ │ │ │ ├── InboundRunner.class │ │ │ │ │ └── bounce │ │ │ │ │ │ ├── InboundBounceCollector.class │ │ │ │ │ │ ├── InboundBounceMapper.class │ │ │ │ │ │ ├── InboundBounceReducer.class │ │ │ │ │ │ ├── InboundBounceRunner.class │ │ │ │ │ │ ├── InboundBounceSecondSort$InboundBounceGroupingComparator.class │ │ │ │ │ │ ├── InboundBounceSecondSort$InboundBouncePartitioner.class │ │ │ │ │ │ └── InboundBounceSecondSort.class │ │ │ │ ├── location │ │ │ │ │ ├── LocationCollector.class │ │ │ │ │ ├── LocationMapper.class │ │ │ │ │ ├── LocationReducer.class │ │ │ │ │ └── LocationRunner.class │ │ │ │ ├── nm │ │ │ │ │ ├── NewMemberCollector$1.class │ │ │ │ │ ├── NewMemberCollector.class │ │ │ │ │ ├── NewMemberMapper.class │ │ │ │ │ ├── NewMemberReducer.class │ │ │ │ │ └── NewMemberRunner.class │ │ │ │ ├── nu │ │ │ │ │ ├── NewInstallUserMapper.class │ │ │ │ │ ├── NewInstallUserReducer.class │ │ │ │ │ ├── NewInstallUserRunner.class │ │ │ │ │ ├── StatsDeviceBrowserNewInstallUserCollector.class │ │ │ │ │ └── StatsUserNewInstallUserCollector.class │ │ │ │ ├── pv │ │ │ │ │ ├── PageViewCollector.class │ │ │ │ │ ├── PageViewMapper.class │ │ │ │ │ ├── PageViewReducer.class │ │ │ │ │ └── PageViewRunner.class │ │ │ │ └── sessions │ │ │ │ │ ├── SessionsCollector$1.class │ │ │ │ │ ├── SessionsCollector.class │ │ │ │ │ ├── SessionsMapper.class │ │ │ │ │ ├── SessionsReducer.class │ │ │ │ │ └── SessionsRunner.class │ │ │ ├── service │ │ │ │ ├── IDimensionConverter.class │ │ │ │ └── impl │ │ │ │ │ ├── DimensionConverterImpl$1.class │ │ │ │ │ ├── DimensionConverterImpl.class │ │ │ │ │ └── InboundDimensionService.class │ │ │ └── util │ │ │ │ ├── MemberUtil$1.class │ │ │ │ ├── MemberUtil.class │ │ │ │ ├── TimeChain.class │ │ │ │ └── UrlUtil.class │ │ │ └── util │ │ │ ├── EJob$1.class │ │ │ ├── EJob.class │ │ │ ├── JdbcManager.class │ │ │ └── TimeUtil.class │ │ ├── core-site.xml │ │ ├── hdfs-site.xml │ │ ├── log4j.properties │ │ ├── mapred-site.xml │ │ ├── output-collector.xml │ │ ├── qqwry.dat │ │ ├── query-mapping.xml │ │ ├── transformer-env.xml │ │ └── yarn-site.xml └── log_analyse_mr │ ├── .idea │ ├── compiler.xml │ ├── encodings.xml │ ├── misc.xml │ └── workspace.xml │ ├── log_analyse_mr.iml │ ├── pom.xml │ ├── src │ └── main │ │ ├── java │ │ └── com │ │ │ └── mashibing │ │ │ ├── common │ │ │ ├── DateEnum.java │ │ │ ├── EventLogConstants.java │ │ │ ├── GlobalConstants.java │ │ │ └── KpiType.java │ │ │ ├── etl │ │ │ ├── mr │ │ │ │ └── ald │ │ │ │ │ ├── AnalyserLogDataMapper.java │ │ │ │ │ └── AnalyserLogDataRunner.java │ │ │ └── util │ │ │ │ ├── IPSeekerExt.java │ │ │ │ ├── LoggerUtil.java │ │ │ │ ├── UserAgentUtil.java │ │ │ │ └── ip │ │ │ │ └── IPSeeker.java │ │ │ ├── transformer │ │ │ ├── model │ │ │ │ ├── dim │ │ │ │ │ ├── StatsCommonDimension.java │ │ │ │ │ ├── StatsDimension.java │ │ │ │ │ ├── StatsUserDimension.java │ │ │ │ │ └── base │ │ │ │ │ │ ├── BaseDimension.java │ │ │ │ │ │ ├── BrowserDimension.java │ │ │ │ │ │ ├── DateDimension.java │ │ │ │ │ │ ├── KpiDimension.java │ │ │ │ │ │ └── PlatformDimension.java │ │ │ │ └── value │ │ │ │ │ ├── BaseStatsValueWritable.java │ │ │ │ │ ├── map │ │ │ │ │ └── TimeOutputValue.java │ │ │ │ │ └── reduce │ │ │ │ │ └── MapWritableValue.java │ │ │ ├── mr │ │ │ │ ├── IOutputCollector.java │ │ │ │ ├── TransformerOutputFormat.java │ │ │ │ ├── au │ │ │ │ │ ├── ActiveUserBrowserCollector.java │ │ │ │ │ ├── ActiveUserCollector.java │ │ │ │ │ ├── ActiveUserMapper.java │ │ │ │ │ ├── ActiveUserReducer.java │ │ │ │ │ └── ActiveUserRunner.java │ │ │ │ └── nu │ │ │ │ │ ├── NewInstallUserMapper.java │ │ │ │ │ ├── NewInstallUserReducer.java │ │ │ │ │ ├── NewInstallUserRunner.java │ │ │ │ │ ├── StatsDeviceBrowserNewInstallUserCollector.java │ │ │ │ │ └── StatsUserNewInstallUserCollector.java │ │ │ └── service │ │ │ │ ├── IDimensionConverter.java │ │ │ │ └── impl │ │ │ │ └── DimensionConverterImpl.java │ │ │ └── util │ │ │ ├── EJob.java │ │ │ ├── JdbcManager.java │ │ │ └── TimeUtil.java │ │ └── resource │ │ ├── core-site.xml │ │ ├── hdfs-site.xml │ │ ├── log4j.properties │ │ ├── mapred-site.xml │ │ ├── output-collector.xml │ │ ├── qqwry.dat │ │ ├── query-mapping.xml │ │ ├── transformer-env.xml │ │ └── yarn-site.xml │ └── target │ └── classes │ ├── com │ └── mashibing │ │ ├── common │ │ ├── DateEnum.class │ │ ├── EventLogConstants$EventEnum.class │ │ ├── EventLogConstants.class │ │ ├── GlobalConstants.class │ │ └── KpiType.class │ │ ├── etl │ │ ├── mr │ │ │ └── ald │ │ │ │ ├── AnalyserLogDataMapper$1.class │ │ │ │ ├── AnalyserLogDataMapper.class │ │ │ │ └── AnalyserLogDataRunner.class │ │ └── util │ │ │ ├── IPSeekerExt$RegionInfo.class │ │ │ ├── IPSeekerExt.class │ │ │ ├── LoggerUtil.class │ │ │ ├── UserAgentUtil$UserAgentInfo.class │ │ │ ├── UserAgentUtil.class │ │ │ └── ip │ │ │ ├── IPSeeker$IPEntry.class │ │ │ ├── IPSeeker$IPLocation.class │ │ │ ├── IPSeeker$IPSeekerUtils.class │ │ │ └── IPSeeker.class │ │ ├── transformer │ │ ├── model │ │ │ ├── dim │ │ │ │ ├── StatsCommonDimension.class │ │ │ │ ├── StatsDimension.class │ │ │ │ ├── StatsUserDimension.class │ │ │ │ └── base │ │ │ │ │ ├── BaseDimension.class │ │ │ │ │ ├── BrowserDimension.class │ │ │ │ │ ├── DateDimension.class │ │ │ │ │ ├── KpiDimension.class │ │ │ │ │ └── PlatformDimension.class │ │ │ └── value │ │ │ │ ├── BaseStatsValueWritable.class │ │ │ │ ├── map │ │ │ │ └── TimeOutputValue.class │ │ │ │ └── reduce │ │ │ │ └── MapWritableValue.class │ │ ├── mr │ │ │ ├── IOutputCollector.class │ │ │ ├── TransformerOutputFormat$TransformerRecordWriter.class │ │ │ ├── TransformerOutputFormat.class │ │ │ ├── au │ │ │ │ ├── ActiveUserBrowserCollector.class │ │ │ │ ├── ActiveUserCollector.class │ │ │ │ ├── ActiveUserMapper.class │ │ │ │ ├── ActiveUserReducer.class │ │ │ │ └── ActiveUserRunner.class │ │ │ └── nu │ │ │ │ ├── NewInstallUserMapper.class │ │ │ │ ├── NewInstallUserReducer.class │ │ │ │ ├── NewInstallUserRunner.class │ │ │ │ ├── StatsDeviceBrowserNewInstallUserCollector.class │ │ │ │ └── StatsUserNewInstallUserCollector.class │ │ └── service │ │ │ ├── IDimensionConverter.class │ │ │ └── impl │ │ │ ├── DimensionConverterImpl$1.class │ │ │ └── DimensionConverterImpl.class │ │ └── util │ │ ├── EJob$1.class │ │ ├── EJob.class │ │ ├── JdbcManager.class │ │ └── TimeUtil.class │ ├── core-site.xml │ ├── hdfs-site.xml │ ├── log4j.properties │ ├── mapred-site.xml │ ├── output-collector.xml │ ├── qqwry.dat │ ├── query-mapping.xml │ ├── transformer-env.xml │ └── yarn-site.xml └── note ├── JS SDK需求文档&设计文档.doc ├── Java SDK需求文档&设计文档 .doc ├── mysql_表设计.sql ├── nginx.conf ├── option-flume单节点案例 ├── option2-flume两台连接node02配置 ├── option3-flume两台连接node03配置 ├── option4 ├── option5 ├── option6 ├── project ├── sqoop ├── option └── option2 ├── statsOrder.hql ├── statsViewDepth.hql ├── view_depth_run.sh ├── 代码设计思路 ├── 思路 ├── 数据流图.png ├── 最终展示界面需求文档&设计文档.doc └── 配置Nginx服务.txt /README.md: -------------------------------------------------------------------------------- 1 | Hadoop_Project详细课程列表 2 | 3 | 第一节课 项目需求讲解及数据源分析 2019年8月8日 20:00-22:00 4 | 5 | ​ 腾讯课堂课程编号26 6 | 7 | ​ 知识点列表： 8 | 9 | 1、项目需求讲解 10 | 2、项目数据源分析 11 | 12 | -------------------------------------------------------------------------------- /code/log_analyse/.idea/artifacts/log_analyse_war_exploded.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | $PROJECT_DIR$/out/artifacts/log_analyse_war_exploded 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /code/log_analyse/.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /code/log_analyse/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /code/log_analyse/.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /code/log_analyse/log_analyse.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /code/log_analyse/out/artifacts/log_analyse_war_exploded/WEB-INF/classes/com/msb/client/AnalyticsEngineSDK.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse/out/artifacts/log_analyse_war_exploded/WEB-INF/classes/com/msb/client/AnalyticsEngineSDK.class -------------------------------------------------------------------------------- /code/log_analyse/out/artifacts/log_analyse_war_exploded/WEB-INF/classes/com/msb/client/SendDataMonitor$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse/out/artifacts/log_analyse_war_exploded/WEB-INF/classes/com/msb/client/SendDataMonitor$1.class -------------------------------------------------------------------------------- /code/log_analyse/out/artifacts/log_analyse_war_exploded/WEB-INF/classes/com/msb/client/SendDataMonitor$HttpRequestUtil.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse/out/artifacts/log_analyse_war_exploded/WEB-INF/classes/com/msb/client/SendDataMonitor$HttpRequestUtil.class -------------------------------------------------------------------------------- /code/log_analyse/out/artifacts/log_analyse_war_exploded/WEB-INF/classes/com/msb/client/SendDataMonitor.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse/out/artifacts/log_analyse_war_exploded/WEB-INF/classes/com/msb/client/SendDataMonitor.class -------------------------------------------------------------------------------- /code/log_analyse/out/artifacts/log_analyse_war_exploded/WEB-INF/classes/com/msb/test/Test.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse/out/artifacts/log_analyse_war_exploded/WEB-INF/classes/com/msb/test/Test.class -------------------------------------------------------------------------------- /code/log_analyse/out/artifacts/log_analyse_war_exploded/WEB-INF/web.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | log_analyse 7 | 8 | index.html 9 | index.htm 10 | index.jsp 11 | default.html 12 | default.htm 13 | default.jsp 14 | 15 | -------------------------------------------------------------------------------- /code/log_analyse/out/artifacts/log_analyse_war_exploded/demo.jsp: -------------------------------------------------------------------------------- 1 | <%@ page contentType="text/html; charset=utf-8" pageEncoding="utf-8"%> 2 | 3 | 4 | 5 | 6 | 测试页面1 7 | 8 | 9 | 10 | 测试页面1
11 | 跳转到: 12 | demo 13 | demo2 14 | demo3 15 | demo4 16 | 17 | -------------------------------------------------------------------------------- /code/log_analyse/out/artifacts/log_analyse_war_exploded/demo2.jsp: -------------------------------------------------------------------------------- 1 | <%@ page contentType="text/html; charset=utf-8" pageEncoding="utf-8"%> 2 | 3 | 4 | 5 | 6 | 测试页面2 7 | 8 | 9 | 10 | 测试页面2 11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 | 跳转到: 19 | demo 20 | demo2 21 | demo3 22 | demo4 23 | 24 | -------------------------------------------------------------------------------- /code/log_analyse/out/artifacts/log_analyse_war_exploded/demo3.jsp: -------------------------------------------------------------------------------- 1 | <%@ page contentType="text/html; charset=utf-8" pageEncoding="utf-8"%> 2 | 3 | 4 | 5 | 6 | 测试页面3 7 | 8 | 9 | 10 | 测试页面3
11 |
12 |
13 |
14 |
15 |
16 |
17 | 跳转到: 18 | demo 19 | demo2 20 | demo3 21 | demo4 22 | 23 | -------------------------------------------------------------------------------- /code/log_analyse/out/artifacts/log_analyse_war_exploded/demo4.jsp: -------------------------------------------------------------------------------- 1 | <%@ page contentType="text/html; charset=utf-8" pageEncoding="utf-8"%> 2 | 3 | 4 | 5 | 6 | 测试页面4 7 | 23 | 24 | 25 | 测试页面4
26 | 在本页面设置memberid为zhangsan
27 | 跳转到: 28 | demo 29 | demo2 30 | demo3 31 | demo4 32 | 33 | -------------------------------------------------------------------------------- /code/log_analyse/out/artifacts/log_analyse_war_exploded/index.jsp: -------------------------------------------------------------------------------- 1 | <%-- 2 | Created by IntelliJ IDEA. 3 | User: seanzhou 4 | Date: 2019/8/8 5 | Time: 18:45 6 | To change this template use File | Settings | File Templates. 7 | --%> 8 | <%@ page contentType="text/html;charset=UTF-8" language="java" %> 9 | 10 | 11 | $Title$ 12 | 13 | 14 | $END$ 15 | 16 | 17 | -------------------------------------------------------------------------------- /code/log_analyse/out/production/log_analyse/com/msb/client/AnalyticsEngineSDK.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse/out/production/log_analyse/com/msb/client/AnalyticsEngineSDK.class -------------------------------------------------------------------------------- /code/log_analyse/out/production/log_analyse/com/msb/client/SendDataMonitor$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse/out/production/log_analyse/com/msb/client/SendDataMonitor$1.class -------------------------------------------------------------------------------- /code/log_analyse/out/production/log_analyse/com/msb/client/SendDataMonitor$HttpRequestUtil.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse/out/production/log_analyse/com/msb/client/SendDataMonitor$HttpRequestUtil.class -------------------------------------------------------------------------------- /code/log_analyse/out/production/log_analyse/com/msb/client/SendDataMonitor.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse/out/production/log_analyse/com/msb/client/SendDataMonitor.class -------------------------------------------------------------------------------- /code/log_analyse/out/production/log_analyse/com/msb/test/Test.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse/out/production/log_analyse/com/msb/test/Test.class -------------------------------------------------------------------------------- /code/log_analyse/src/com/msb/test/Test.java: -------------------------------------------------------------------------------- 1 | package com.msb.test; 2 | 3 | import com.msb.client.AnalyticsEngineSDK; 4 | public class Test { 5 | public static void main(String[] args) { 6 | AnalyticsEngineSDK.onChargeSuccess("orderid123", "zhangsan"); 7 | AnalyticsEngineSDK.onChargeRefund("orderid456", "lisi"); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /code/log_analyse/web/WEB-INF/web.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | log_analyse 7 | 8 | index.html 9 | index.htm 10 | index.jsp 11 | default.html 12 | default.htm 13 | default.jsp 14 | 15 | -------------------------------------------------------------------------------- /code/log_analyse/web/demo.jsp: -------------------------------------------------------------------------------- 1 | <%@ page contentType="text/html; charset=utf-8" pageEncoding="utf-8"%> 2 | 3 | 4 | 5 | 6 | 测试页面1 7 | 8 | 9 | 10 | 测试页面1
11 | 跳转到: 12 | demo 13 | demo2 14 | demo3 15 | demo4 16 | 17 | -------------------------------------------------------------------------------- /code/log_analyse/web/demo2.jsp: -------------------------------------------------------------------------------- 1 | <%@ page contentType="text/html; charset=utf-8" pageEncoding="utf-8"%> 2 | 3 | 4 | 5 | 6 | 测试页面2 7 | 8 | 9 | 10 | 测试页面2 11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 | 跳转到: 19 | demo 20 | demo2 21 | demo3 22 | demo4 23 | 24 | -------------------------------------------------------------------------------- /code/log_analyse/web/demo3.jsp: -------------------------------------------------------------------------------- 1 | <%@ page contentType="text/html; charset=utf-8" pageEncoding="utf-8"%> 2 | 3 | 4 | 5 | 6 | 测试页面3 7 | 8 | 9 | 10 | 测试页面3
11 |
12 |
13 |
14 |
15 |
16 |
17 | 跳转到: 18 | demo 19 | demo2 20 | demo3 21 | demo4 22 | 23 | -------------------------------------------------------------------------------- /code/log_analyse/web/demo4.jsp: -------------------------------------------------------------------------------- 1 | <%@ page contentType="text/html; charset=utf-8" pageEncoding="utf-8"%> 2 | 3 | 4 | 5 | 6 | 测试页面4 7 | 23 | 24 | 25 | 测试页面4
26 | 在本页面设置memberid为zhangsan
27 | 跳转到: 28 | demo 29 | demo2 30 | demo3 31 | demo4 32 | 33 | -------------------------------------------------------------------------------- /code/log_analyse/web/index.jsp: -------------------------------------------------------------------------------- 1 | <%-- 2 | Created by IntelliJ IDEA. 3 | User: seanzhou 4 | Date: 2019/8/8 5 | Time: 18:45 6 | To change this template use File | Settings | File Templates. 7 | --%> 8 | <%@ page contentType="text/html;charset=UTF-8" language="java" %> 9 | 10 | 11 | $Title$ 12 | 13 | 14 | $END$ 15 | 16 | 17 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/log_analyse_dataclean.iml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/src/main/java/com/mashibing/common/DateEnum.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.common; 2 | 3 | /** 4 | * 日期类型枚举类 5 | * 6 | * @author 马士兵教育 7 | * 8 | */ 9 | public enum DateEnum { 10 | YEAR("year"), SEASON("season"), MONTH("month"), WEEK("week"), DAY("day"), HOUR( 11 | "hour"); 12 | 13 | public final String name; 14 | 15 | private DateEnum(String name) { 16 | this.name = name; 17 | } 18 | 19 | /** 20 | * 根据属性name的值获取对应的type对象 21 | * 22 | * @param name 23 | * @return 24 | */ 25 | public static DateEnum valueOfName(String name) { 26 | for (DateEnum type : values()) { 27 | if (type.name.equals(name)) { 28 | return type; 29 | } 30 | } 31 | return null; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/src/main/java/com/mashibing/common/GlobalConstants.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.common; 2 | 3 | /** 4 | * 全局常量类 5 | * 6 | * @author 马士兵教育 7 | * 8 | */ 9 | public class GlobalConstants { 10 | 11 | /** 12 | * 一天的毫秒数 13 | */ 14 | public static final int DAY_OF_MILLISECONDS = 86400000; 15 | 16 | /** 17 | * 定义的运行时间变量名 18 | */ 19 | public static final String RUNNING_DATE_PARAMES = "RUNNING_DATE"; 20 | 21 | /** 22 | * 默认值 23 | */ 24 | public static final String DEFAULT_VALUE = "unknown"; 25 | 26 | /** 27 | * 维度信息表中指定全部列值 28 | */ 29 | public static final String VALUE_OF_ALL = "all"; 30 | 31 | 32 | /** 33 | * 定义的output collector的前缀 34 | */ 35 | public static final String OUTPUT_COLLECTOR_KEY_PREFIX = "collector_"; 36 | 37 | /** 38 | * 指定连接表配置为report 39 | */ 40 | public static final String WAREHOUSE_OF_REPORT = "report"; 41 | 42 | /** 43 | * 批量执行的key 44 | */ 45 | public static final String JDBC_BATCH_NUMBER = "mysql.batch.number"; 46 | 47 | /** 48 | * 默认批量大小 49 | */ 50 | public static final String DEFAULT_JDBC_BATCH_NUMBER = "500"; 51 | 52 | /** 53 | * driver 名称 54 | */ 55 | public static final String JDBC_DRIVER = "mysql.%s.driver"; 56 | 57 | /** 58 | * JDBC URL 59 | */ 60 | public static final String JDBC_URL = "mysql.%s.url"; 61 | 62 | /** 63 | * username名称 64 | */ 65 | public static final String JDBC_USERNAME = "mysql.%s.username"; 66 | 67 | /** 68 | * password名称 69 | */ 70 | public static final String JDBC_PASSWORD = "mysql.%s.password"; 71 | 72 | } 73 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/src/main/java/com/mashibing/common/KpiType.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.common; 2 | 3 | /** 4 | * 统计kpi的名称枚举类 5 | * 6 | * @author 马士兵教育 7 | * 8 | */ 9 | public enum KpiType { 10 | 11 | NEW_INSTALL_USER("new_install_user"), // 统计新用户的kpi 12 | BROWSER_NEW_INSTALL_USER("browser_new_install_user"), // 统计浏览器维度的新用户kpi 13 | ACTIVE_USER("active_user"), // 统计活跃用户kpi 14 | BROWSER_ACTIVE_USER("browser_active_user"), // 统计浏览器维度的活跃用户kpi 15 | ; 16 | 17 | public final String name; 18 | 19 | private KpiType(String name) { 20 | this.name = name; 21 | } 22 | 23 | /** 24 | * 根据kpiType的名称字符串值，获取对应的kpitype枚举对象 25 | * 26 | * @param name 27 | * @return 28 | */ 29 | public static KpiType valueOfName(String name) { 30 | for (KpiType type : values()) { 31 | if (type.name.equals(name)) { 32 | return type; 33 | } 34 | } 35 | throw new RuntimeException("指定的name不属于该KpiType枚举类：" + name); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/src/main/java/com/mashibing/util/JdbcManager.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.util; 2 | 3 | import java.sql.Connection; 4 | import java.sql.DriverManager; 5 | import java.sql.SQLException; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | 9 | import com.mashibing.common.GlobalConstants; 10 | 11 | /** 12 | * jdbc管理 13 | * 14 | * @author 马士兵教育 15 | * 16 | */ 17 | public class JdbcManager { 18 | /** 19 | * 根据配置获取获取关系型数据库的jdbc连接 20 | * 21 | * @param conf 22 | * hadoop配置信息 23 | * @param flag 24 | * 区分不同数据源的标志位 25 | * @return 26 | * @throws SQLException 27 | */ 28 | public static Connection getConnection(Configuration conf, String flag) throws SQLException { 29 | String driverStr = String.format(GlobalConstants.JDBC_DRIVER, flag); 30 | String urlStr = String.format(GlobalConstants.JDBC_URL, flag); 31 | String usernameStr = String.format(GlobalConstants.JDBC_USERNAME, flag); 32 | String passwordStr = String.format(GlobalConstants.JDBC_PASSWORD, flag); 33 | 34 | String driverClass = conf.get(driverStr); 35 | String url = conf.get(urlStr); 36 | String username = conf.get(usernameStr); 37 | String password = conf.get(passwordStr); 38 | try { 39 | Class.forName(driverClass); 40 | } catch (ClassNotFoundException e) { 41 | // nothing 42 | } 43 | return DriverManager.getConnection(url, username, password); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/src/main/java/test/TestIPSeeker.java: -------------------------------------------------------------------------------- 1 | package test; 2 | 3 | import com.mashibing.etl.util.ip.IPSeeker; 4 | 5 | public class TestIPSeeker { 6 | public static void main(String[] args) { 7 | IPSeeker ipSeeker = IPSeeker.getInstance(); 8 | System.out.println(ipSeeker.getCountry("120.197.87.216")); 9 | System.out.println(ipSeeker.getCountry("192.168.1.14")); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/src/main/java/test/TestIPSeekerExt.java: -------------------------------------------------------------------------------- 1 | package test; 2 | 3 | import java.util.List; 4 | 5 | import com.mashibing.etl.util.IPSeekerExt; 6 | import com.mashibing.etl.util.IPSeekerExt.RegionInfo; 7 | 8 | 9 | public class TestIPSeekerExt { 10 | public static void main(String[] args) { 11 | IPSeekerExt ipSeekerExt = new IPSeekerExt(); 12 | RegionInfo info = ipSeekerExt.analyticIp("114.114.114.114"); 13 | System.out.println(info); 14 | 15 | List ips = ipSeekerExt.getAllIp(); 16 | for (String ip : ips) { 17 | System.out.println(ip + " --- " + ipSeekerExt.analyticIp(ip)); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/src/main/java/test/TestLoggerUtil.java: -------------------------------------------------------------------------------- 1 | package test; 2 | 3 | import com.mashibing.etl.util.IPSeekerExt; 4 | import com.mashibing.etl.util.LoggerUtil; 5 | 6 | public class TestLoggerUtil { 7 | public static void main(String[] args) { 8 | String log = "192.168.100.102^A1449411239.595^A192.168.239.8^A/log.gif?c_time=1449411240818&oid=orderid456&u_mid=zhangsan&pl=java_server&en=e_cr&sdk=jdk&ver=1"; 9 | log = "192.168.100.102^A1449587515.394^A192.168.239.8^A/log.gif?en=e_pv&p_url=http%3A%2F%2Flocalhost%3A8080%2Fbf_track_jssdk%2Fdemo2.jsp&p_ref=http%3A%2F%2Flocalhost%3A8080%2Fbf_track_jssdk%2Fdemo.jsp&tt=%E6%B5%8B%E8%AF%95%E9%A1%B5%E9%9D%A22&ver=1&pl=website&sdk=js&u_ud=948AB94A-E1A5-4EED-BBB8-CEDB74B8B4D0&u_sd=9EF5D22F-5CCD-4290-AFCA-641672988F73&c_time=1449587517241&l=zh-CN&b_iev=Mozilla%2F5.0%20(Windows%20NT%206.1%3B%20WOW64)%20AppleWebKit%2F537.36%20(KHTML%2C%20like%20Gecko)%20Chrome%2F46.0.2490.71%20Safari%2F537.36&b_rst=1280*768"; 10 | System.out.println(LoggerUtil.handleLog(log)); 11 | System.out.println(IPSeekerExt.getInstance().getCountry("192.168.100.102")); 12 | System.out.println(IPSeekerExt.getInstance().getArea("192.168.100.102")); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/src/main/java/test/TestUserAgentUtil.java: -------------------------------------------------------------------------------- 1 | package test; 2 | 3 | import com.mashibing.etl.util.UserAgentUtil; 4 | import com.mashibing.etl.util.UserAgentUtil.UserAgentInfo; 5 | 6 | 7 | public class TestUserAgentUtil { 8 | public static void main(String[] args) { 9 | String userAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36"; 10 | userAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; GWX:QUALIFIED; rv:11.0) like Gecko"; 11 | userAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0"; 12 | UserAgentInfo info = UserAgentUtil.analyticUserAgent(userAgent); 13 | System.out.println(info); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/src/main/resource/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | fs.defaultFS 22 | hdfs://mycluster 23 | 24 | 25 | ha.zookeeper.quorum 26 | node02:2181,node03:2181,node04:2181 27 | 28 | 29 | hadoop.proxyuser.root.groups 30 | * 31 | 32 | 33 | hadoop.proxyuser.root.hosts 34 | * 35 | 36 | 37 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/src/main/resource/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=info,console 2 | 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.layout=org.apache.log4j.SimpleLayout -------------------------------------------------------------------------------- /code/log_analyse_dataclean/src/main/resource/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | mapreduce.framework.name 23 | yarn 24 | 25 | 26 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/src/main/resource/output-collector.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | collector_new_install_user 6 | com.mashibing.transformer.mr.nu.StatsUserNewInstallUserCollector 7 | 8 | 9 | 10 | collector_browser_new_install_user 11 | com.mashibing.transformer.mr.nu.StatsDeviceBrowserNewInstallUserCollector 12 | 13 | 14 | collector_active_user 15 | com.mashibing.transformer.mr.au.ActiveUserCollector 16 | 17 | 18 | 19 | collector_browser_active_user 20 | com.mashibing.transformer.mr.au.ActiveUserBrowserCollector 21 | 22 | 23 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/src/main/resource/qqwry.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/src/main/resource/qqwry.dat -------------------------------------------------------------------------------- /code/log_analyse_dataclean/src/main/resource/query-mapping.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | new_install_user 6 | 7 | INSERT INTO `stats_user`( 8 | `platform_dimension_id`, 9 | `date_dimension_id`, 10 | `new_install_users`, 11 | `created`) 12 | VALUES(?, ?, ?, ?) ON DUPLICATE KEY UPDATE `new_install_users` = ? 13 | 14 | 15 | 16 | 17 | browser_new_install_user 18 | 19 | INSERT INTO `stats_device_browser`( 20 | `platform_dimension_id`, 21 | `date_dimension_id`, 22 | `browser_dimension_id`, 23 | `new_install_users`, 24 | `created`) 25 | VALUES(?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE `new_install_users` = ? 26 | 27 | 28 | 29 | 30 | active_user 31 | 32 | INSERT INTO `stats_user`( 33 | `platform_dimension_id`, 34 | `date_dimension_id`, 35 | `active_users`, 36 | `created`) 37 | VALUES(?, ?, ?, ?) ON DUPLICATE KEY UPDATE `active_users` = ? 38 | 39 | 40 | 41 | 42 | browser_active_user 43 | 44 | INSERT INTO `stats_device_browser`( 45 | `platform_dimension_id`, 46 | `date_dimension_id`, 47 | `browser_dimension_id`, 48 | `active_users`, 49 | `created` 50 | ) VALUES(?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE `active_users` = ? 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/src/main/resource/transformer-env.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | mysql.batch.number 6 | 10 7 | 8 | 9 | 10 | mysql.report.driver 11 | com.mysql.jdbc.Driver 12 | 13 | 14 | 15 | mysql.report.url 16 | jdbc:mysql://node12:3306/result_db?useUnicode=true&characterEncoding=utf8 17 | 18 | 19 | 20 | mysql.report.username 21 | root 22 | 23 | 24 | 25 | mysql.report.password 26 | 123456 27 | 28 | 29 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/src/main/resource/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 15 | 16 | 17 | 18 | 19 | 20 | yarn.nodemanager.aux-services 21 | mapreduce_shuffle 22 | 23 | 24 | yarn.resourcemanager.ha.enabled 25 | true 26 | 27 | 28 | yarn.resourcemanager.zk-address 29 | node02:2181,node03:2181,node04:2181 30 | 31 | 32 | yarn.resourcemanager.cluster-id 33 | mashibing 34 | 35 | 36 | yarn.resourcemanager.ha.rm-ids 37 | rm1,rm2 38 | 39 | 40 | yarn.resourcemanager.hostname.rm1 41 | node03 42 | 43 | 44 | yarn.resourcemanager.hostname.rm2 45 | node04 46 | 47 | 48 | yarn.log-aggregation-enable 49 | true 50 | 51 | 52 | yarn.log-aggregation.retain-seconds 53 | 604800 54 | 55 | 56 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/common/DateEnum.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/common/DateEnum.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/common/EventLogConstants$EventEnum.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/common/EventLogConstants$EventEnum.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/common/EventLogConstants.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/common/EventLogConstants.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/common/GlobalConstants.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/common/GlobalConstants.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/common/KpiType.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/common/KpiType.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/etl/mr/ald/AnalyserLogDataMapper$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/etl/mr/ald/AnalyserLogDataMapper$1.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/etl/mr/ald/AnalyserLogDataMapper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/etl/mr/ald/AnalyserLogDataMapper.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/etl/mr/ald/AnalyserLogDataRunner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/etl/mr/ald/AnalyserLogDataRunner.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/etl/util/IPSeekerExt$RegionInfo.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/etl/util/IPSeekerExt$RegionInfo.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/etl/util/IPSeekerExt.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/etl/util/IPSeekerExt.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/etl/util/LoggerUtil.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/etl/util/LoggerUtil.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/etl/util/UserAgentUtil$UserAgentInfo.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/etl/util/UserAgentUtil$UserAgentInfo.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/etl/util/UserAgentUtil.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/etl/util/UserAgentUtil.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/etl/util/ip/IPSeeker$IPEntry.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/etl/util/ip/IPSeeker$IPEntry.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/etl/util/ip/IPSeeker$IPLocation.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/etl/util/ip/IPSeeker$IPLocation.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/etl/util/ip/IPSeeker$IPSeekerUtils.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/etl/util/ip/IPSeeker$IPSeekerUtils.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/etl/util/ip/IPSeeker.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/etl/util/ip/IPSeeker.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/util/EJob$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/util/EJob$1.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/util/EJob.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/util/EJob.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/util/JdbcManager.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/util/JdbcManager.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/com/mashibing/util/TimeUtil.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/com/mashibing/util/TimeUtil.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | fs.defaultFS 22 | hdfs://mycluster 23 | 24 | 25 | ha.zookeeper.quorum 26 | node02:2181,node03:2181,node04:2181 27 | 28 | 29 | hadoop.proxyuser.root.groups 30 | * 31 | 32 | 33 | hadoop.proxyuser.root.hosts 34 | * 35 | 36 | 37 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=info,console 2 | 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.layout=org.apache.log4j.SimpleLayout -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | mapreduce.framework.name 23 | yarn 24 | 25 | 26 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/output-collector.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | collector_new_install_user 6 | com.mashibing.transformer.mr.nu.StatsUserNewInstallUserCollector 7 | 8 | 9 | 10 | collector_browser_new_install_user 11 | com.mashibing.transformer.mr.nu.StatsDeviceBrowserNewInstallUserCollector 12 | 13 | 14 | collector_active_user 15 | com.mashibing.transformer.mr.au.ActiveUserCollector 16 | 17 | 18 | 19 | collector_browser_active_user 20 | com.mashibing.transformer.mr.au.ActiveUserBrowserCollector 21 | 22 | 23 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/qqwry.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/qqwry.dat -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/query-mapping.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | new_install_user 6 | 7 | INSERT INTO `stats_user`( 8 | `platform_dimension_id`, 9 | `date_dimension_id`, 10 | `new_install_users`, 11 | `created`) 12 | VALUES(?, ?, ?, ?) ON DUPLICATE KEY UPDATE `new_install_users` = ? 13 | 14 | 15 | 16 | 17 | browser_new_install_user 18 | 19 | INSERT INTO `stats_device_browser`( 20 | `platform_dimension_id`, 21 | `date_dimension_id`, 22 | `browser_dimension_id`, 23 | `new_install_users`, 24 | `created`) 25 | VALUES(?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE `new_install_users` = ? 26 | 27 | 28 | 29 | 30 | active_user 31 | 32 | INSERT INTO `stats_user`( 33 | `platform_dimension_id`, 34 | `date_dimension_id`, 35 | `active_users`, 36 | `created`) 37 | VALUES(?, ?, ?, ?) ON DUPLICATE KEY UPDATE `active_users` = ? 38 | 39 | 40 | 41 | 42 | browser_active_user 43 | 44 | INSERT INTO `stats_device_browser`( 45 | `platform_dimension_id`, 46 | `date_dimension_id`, 47 | `browser_dimension_id`, 48 | `active_users`, 49 | `created` 50 | ) VALUES(?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE `active_users` = ? 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/test/Sex.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/test/Sex.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/test/TestIPSeek.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/test/TestIPSeek.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/test/TestUserAgent.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_dataclean/target/classes/test/TestUserAgent.class -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/transformer-env.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | mysql.batch.number 6 | 10 7 | 8 | 9 | 10 | mysql.report.driver 11 | com.mysql.jdbc.Driver 12 | 13 | 14 | 15 | mysql.report.url 16 | jdbc:mysql://node12:3306/result_db?useUnicode=true&characterEncoding=utf8 17 | 18 | 19 | 20 | mysql.report.username 21 | root 22 | 23 | 24 | 25 | mysql.report.password 26 | 123456 27 | 28 | 29 | -------------------------------------------------------------------------------- /code/log_analyse_dataclean/target/classes/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 15 | 16 | 17 | 18 | 19 | 20 | yarn.nodemanager.aux-services 21 | mapreduce_shuffle 22 | 23 | 24 | yarn.resourcemanager.ha.enabled 25 | true 26 | 27 | 28 | yarn.resourcemanager.zk-address 29 | node02:2181,node03:2181,node04:2181 30 | 31 | 32 | yarn.resourcemanager.cluster-id 33 | mashibing 34 | 35 | 36 | yarn.resourcemanager.ha.rm-ids 37 | rm1,rm2 38 | 39 | 40 | yarn.resourcemanager.hostname.rm1 41 | node03 42 | 43 | 44 | yarn.resourcemanager.hostname.rm2 45 | node04 46 | 47 | 48 | yarn.log-aggregation-enable 49 | true 50 | 51 | 52 | yarn.log-aggregation.retain-seconds 53 | 604800 54 | 55 | 56 | -------------------------------------------------------------------------------- /code/log_analyse_full/.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /code/log_analyse_full/.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /code/log_analyse_full/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /code/log_analyse_full/log_analyse_full.iml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/common/DateEnum.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.common; 2 | 3 | /** 4 | * 日期类型枚举类 5 | * 6 | * @author 马士兵教育 7 | * 8 | */ 9 | public enum DateEnum { 10 | YEAR("year"), SEASON("season"), MONTH("month"), WEEK("week"), DAY("day"), HOUR("hour"); 11 | 12 | public final String name; 13 | 14 | private DateEnum(String name) { 15 | this.name = name; 16 | } 17 | 18 | /** 19 | * 根据属性name的值获取对应的type对象 20 | * 21 | * @param name 22 | * @return 23 | */ 24 | public static DateEnum valueOfName(String name) { 25 | for (DateEnum type : values()) { 26 | if (type.name.equals(name)) { 27 | return type; 28 | } 29 | } 30 | return null; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/common/GlobalConstants.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.common; 2 | 3 | /** 4 | * 全局常量类 5 | * 6 | * @author 马士兵教育 7 | * 8 | */ 9 | public class GlobalConstants { 10 | /** 11 | * 一天的毫秒数 12 | */ 13 | public static final int DAY_OF_MILLISECONDS = 86400000; 14 | /** 15 | * 定义的运行时间变量名 16 | */ 17 | public static final String RUNNING_DATE_PARAMES = "RUNNING_DATE"; 18 | /** 19 | * 默认值 20 | */ 21 | public static final String DEFAULT_VALUE = "unknown"; 22 | /** 23 | * 维度信息表中指定全部列值 24 | */ 25 | public static final String VALUE_OF_ALL = "all"; 26 | 27 | /** 28 | * 定义的output collector的前缀 29 | */ 30 | public static final String OUTPUT_COLLECTOR_KEY_PREFIX = "collector_"; 31 | 32 | /** 33 | * 指定连接表配置为report 34 | */ 35 | public static final String WAREHOUSE_OF_REPORT = "report"; 36 | 37 | /** 38 | * 批量执行的key 39 | */ 40 | public static final String JDBC_BATCH_NUMBER = "mysql.batch.number"; 41 | 42 | /** 43 | * 默认批量大小 44 | */ 45 | public static final String DEFAULT_JDBC_BATCH_NUMBER = "500"; 46 | 47 | /** 48 | * driver 名称 49 | */ 50 | public static final String JDBC_DRIVER = "mysql.%s.driver"; 51 | 52 | /** 53 | * JDBC URL 54 | */ 55 | public static final String JDBC_URL = "mysql.%s.url"; 56 | 57 | /** 58 | * username名称 59 | */ 60 | public static final String JDBC_USERNAME = "mysql.%s.username"; 61 | 62 | /** 63 | * password名称 64 | */ 65 | public static final String JDBC_PASSWORD = "mysql.%s.password"; 66 | 67 | } 68 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/common/KpiType.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.common; 2 | 3 | /** 4 | * 统计kpi的名称枚举类 5 | * 6 | * @author 马士兵教育 7 | * 8 | */ 9 | public enum KpiType { 10 | 11 | NEW_INSTALL_USER("new_install_user"), // 统计新用户的kpi 12 | BROWSER_NEW_INSTALL_USER("browser_new_install_user"), // 统计浏览器维度的新用户kpi 13 | ACTIVE_USER("active_user"), // 统计活跃用户kpi 14 | BROWSER_ACTIVE_USER("browser_active_user"), // 统计浏览器维度的活跃用户kpi 15 | ACTIVE_MEMBER("active_member"), // 统计活跃会员kpi 16 | BROWSER_ACTIVE_MEMBER("browser_active_member"), // 统计浏览器维度的活跃会员kpi 17 | NEW_MEMBER("new_member"), // 统计新增会员kpi 18 | BROWSER_NEW_MEMBER("browser_new_member"), // 统计浏览器维度新增会员kpi 19 | INSERT_MEMBER_INFO("insert_member_info"), // 插入会员信息kpi 20 | SESSIONS("sessions"), // 统计会话kpi 21 | BROWSER_SESSIONS("browser_sessions"), // 统计浏览器维度会话kpi 22 | HOURLY_ACTIVE_USER("hourly_active_user"), // 按小时统计活跃用户kpi 23 | HOURLY_SESSIONS("hourly_sessions"), // 按小时统计会话格式kpi 24 | HOURLY_SESSIONS_LENGTH("hourly_sessions_length"), // 按小时统计会话长度kpi 25 | WEBSITE_PAGEVIEW("website_pageview"), // 统计浏览器维度的pv kpi 26 | LOCATION("location"), // 统计地域信息维度的kpi 27 | INBOUND("inbound"), // 统计inbound维度信息的kpi 28 | INBOUND_BOUNCE("inbound_bounce"), //统计inbound维度下的跳出会话kpi 29 | ; 30 | 31 | public final String name; 32 | 33 | private KpiType(String name) { 34 | this.name = name; 35 | } 36 | 37 | /** 38 | * 根据kpiType的名称字符串值，获取对应的kpitype枚举对象 39 | * 40 | * @param name 41 | * @return 42 | */ 43 | public static KpiType valueOfName(String name) { 44 | for (KpiType type : values()) { 45 | if (type.name.equals(name)) { 46 | return type; 47 | } 48 | } 49 | throw new RuntimeException("指定的name不属于该KpiType枚举类：" + name); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/hive/DateDimensionUDF.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.hive; 2 | 3 | 4 | import java.io.IOException; 5 | 6 | import org.apache.hadoop.hive.ql.exec.UDF; 7 | import org.apache.hadoop.io.IntWritable; 8 | import org.apache.hadoop.io.Text; 9 | 10 | import com.mashibing.common.DateEnum; 11 | import com.mashibing.transformer.model.dim.base.DateDimension; 12 | import com.mashibing.transformer.service.IDimensionConverter; 13 | import com.mashibing.transformer.service.impl.DimensionConverterImpl; 14 | import com.mashibing.util.TimeUtil; 15 | 16 | /** 17 | * 操作日期dimension 相关的udf 18 | * 19 | * @author root 20 | * 21 | */ 22 | public class DateDimensionUDF extends UDF { 23 | private IDimensionConverter converter = new DimensionConverterImpl(); 24 | 25 | 26 | 27 | /** 28 | * 根据给定的日期（格式为:yyyy-MM-dd）至返回id 29 | * 30 | * @param day 31 | * @return 32 | */ 33 | public IntWritable evaluate(Text day) { 34 | DateDimension dimension = DateDimension.buildDate(TimeUtil.parseString2Long(day.toString()), DateEnum.DAY); 35 | try { 36 | int id = this.converter.getDimensionIdByValue(dimension); 37 | return new IntWritable(id); 38 | } catch (IOException e) { 39 | throw new RuntimeException("获取id异常"); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/model/dim/StatsDimension.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.model.dim; 2 | 3 | import com.mashibing.transformer.model.dim.base.BaseDimension; 4 | 5 | public abstract class StatsDimension extends BaseDimension { 6 | // nothing 7 | } 8 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/model/dim/base/BaseDimension.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.model.dim.base; 2 | 3 | import org.apache.hadoop.io.WritableComparable; 4 | 5 | /** 6 | * 顶级的维度信息类 7 | * 8 | * @author 马士兵教育 9 | * 10 | */ 11 | public abstract class BaseDimension implements WritableComparable{ 12 | // nothing 13 | } 14 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/model/value/BaseStatsValueWritable.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.model.value; 2 | 3 | import com.mashibing.common.KpiType; 4 | import org.apache.hadoop.io.Writable; 5 | 6 | import com.mashibing.common.KpiType; 7 | 8 | /** 9 | * 自定义顶级的输出value父类 10 | * 11 | * @author 马士兵教育 12 | * 13 | */ 14 | public abstract class BaseStatsValueWritable implements Writable { 15 | /** 16 | * 获取当前value对应的kpi值 17 | * 18 | * @return 19 | */ 20 | public abstract KpiType getKpi(); 21 | } 22 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/model/value/map/TimeOutputValue.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.model.value.map; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | 7 | import com.mashibing.common.KpiType; 8 | import com.mashibing.transformer.model.value.BaseStatsValueWritable; 9 | 10 | public class TimeOutputValue extends BaseStatsValueWritable { 11 | private String id; // id 12 | private long time; // 时间戳 13 | 14 | public String getId() { 15 | return id; 16 | } 17 | 18 | public void setId(String id) { 19 | this.id = id; 20 | } 21 | 22 | public long getTime() { 23 | return time; 24 | } 25 | 26 | public void setTime(long time) { 27 | this.time = time; 28 | } 29 | 30 | @Override 31 | public void write(DataOutput out) throws IOException { 32 | out.writeUTF(this.id); 33 | out.writeLong(this.time); 34 | } 35 | 36 | @Override 37 | public void readFields(DataInput in) throws IOException { 38 | this.id = in.readUTF(); 39 | this.time = in.readLong(); 40 | } 41 | 42 | @Override 43 | public KpiType getKpi() { 44 | // TODO Auto-generated method stub 45 | return null; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/model/value/reduce/InboundBounceReduceValue.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.model.value.reduce; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | 7 | import org.apache.hadoop.io.WritableUtils; 8 | 9 | import com.mashibing.common.KpiType; 10 | import com.mashibing.transformer.model.value.BaseStatsValueWritable; 11 | 12 | public class InboundBounceReduceValue extends BaseStatsValueWritable { 13 | private KpiType kpi; 14 | private int bounceNumber; 15 | 16 | public InboundBounceReduceValue() { 17 | super(); 18 | } 19 | 20 | public InboundBounceReduceValue(int bounceNumber) { 21 | super(); 22 | this.bounceNumber = bounceNumber; 23 | } 24 | 25 | public int getBounceNumber() { 26 | return bounceNumber; 27 | } 28 | 29 | public void setBounceNumber(int bounceNumber) { 30 | this.bounceNumber = bounceNumber; 31 | } 32 | 33 | public void setKpi(KpiType kpi) { 34 | this.kpi = kpi; 35 | } 36 | 37 | /** 38 | * 自增1 39 | */ 40 | public void incrBounceNum() { 41 | this.bounceNumber = this.bounceNumber + 1; 42 | } 43 | 44 | @Override 45 | public void write(DataOutput out) throws IOException { 46 | out.writeInt(this.bounceNumber); 47 | WritableUtils.writeEnum(out, this.kpi); 48 | } 49 | 50 | @Override 51 | public void readFields(DataInput in) throws IOException { 52 | this.bounceNumber = in.readInt(); 53 | this.kpi = WritableUtils.readEnum(in, KpiType.class); 54 | } 55 | 56 | @Override 57 | public KpiType getKpi() { 58 | return kpi; 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/model/value/reduce/InboundReduceValue.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.model.value.reduce; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | 7 | import org.apache.hadoop.io.WritableUtils; 8 | 9 | import com.mashibing.common.KpiType; 10 | import com.mashibing.transformer.model.value.BaseStatsValueWritable; 11 | 12 | public class InboundReduceValue extends BaseStatsValueWritable { 13 | private KpiType kpi; 14 | private int uvs; 15 | private int visit; 16 | 17 | public int getUvs() { 18 | return uvs; 19 | } 20 | 21 | public void setUvs(int uvs) { 22 | this.uvs = uvs; 23 | } 24 | 25 | public int getVisit() { 26 | return visit; 27 | } 28 | 29 | public void setVisit(int visit) { 30 | this.visit = visit; 31 | } 32 | 33 | public void setKpi(KpiType kpi) { 34 | this.kpi = kpi; 35 | } 36 | 37 | @Override 38 | public void write(DataOutput out) throws IOException { 39 | out.writeInt(this.uvs); 40 | out.writeInt(this.visit); 41 | WritableUtils.writeEnum(out, this.kpi); 42 | } 43 | 44 | @Override 45 | public void readFields(DataInput in) throws IOException { 46 | this.uvs = in.readInt(); 47 | this.visit = in.readInt(); 48 | this.kpi = WritableUtils.readEnum(in, KpiType.class); 49 | } 50 | 51 | @Override 52 | public KpiType getKpi() { 53 | return this.kpi; 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/model/value/reduce/MapWritableValue.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.model.value.reduce; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | 7 | import org.apache.hadoop.io.MapWritable; 8 | import org.apache.hadoop.io.WritableUtils; 9 | 10 | import com.mashibing.common.KpiType; 11 | import com.mashibing.transformer.model.value.BaseStatsValueWritable; 12 | 13 | public class MapWritableValue extends BaseStatsValueWritable { 14 | private MapWritable value = new MapWritable(); 15 | private KpiType kpi; 16 | 17 | public MapWritableValue() { 18 | super(); 19 | } 20 | 21 | public MapWritableValue(MapWritable value, KpiType kpi) { 22 | super(); 23 | this.value = value; 24 | this.kpi = kpi; 25 | } 26 | 27 | public MapWritable getValue() { 28 | return value; 29 | } 30 | 31 | public void setValue(MapWritable value) { 32 | this.value = value; 33 | } 34 | 35 | public void setKpi(KpiType kpi) { 36 | this.kpi = kpi; 37 | } 38 | 39 | @Override 40 | public void write(DataOutput out) throws IOException { 41 | this.value.write(out); 42 | WritableUtils.writeEnum(out, this.kpi); 43 | } 44 | 45 | @Override 46 | public void readFields(DataInput in) throws IOException { 47 | this.value.readFields(in); 48 | this.kpi = WritableUtils.readEnum(in, KpiType.class); 49 | } 50 | 51 | @Override 52 | public KpiType getKpi() { 53 | return this.kpi; 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/mr/IOutputCollector.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.mr; 2 | 3 | import java.io.IOException; 4 | import java.sql.PreparedStatement; 5 | import java.sql.SQLException; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | 9 | import com.mashibing.transformer.model.dim.base.BaseDimension; 10 | import com.mashibing.transformer.model.value.BaseStatsValueWritable; 11 | import com.mashibing.transformer.service.IDimensionConverter; 12 | 13 | /** 14 | * 自定义的配合自定义output进行具体sql输出的类 15 | * 16 | * @author 马士兵教育 17 | * 18 | */ 19 | public interface IOutputCollector { 20 | 21 | /** 22 | * 具体执行统计数据插入的方法 23 | * 24 | * @param conf 25 | * @param key 26 | * @param value 27 | * @param pstmt 28 | * @param converter 29 | * @throws SQLException 30 | * @throws IOException 31 | */ 32 | public void collect(Configuration conf, BaseDimension key, BaseStatsValueWritable value, PreparedStatement pstmt, IDimensionConverter converter) throws SQLException, IOException; 33 | } 34 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/mr/am/ActiveMemberReducer.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.mr.am; 2 | 3 | import java.io.IOException; 4 | import java.util.HashSet; 5 | import java.util.Set; 6 | 7 | import org.apache.hadoop.io.IntWritable; 8 | import org.apache.hadoop.io.MapWritable; 9 | import org.apache.hadoop.mapreduce.Reducer; 10 | 11 | import com.mashibing.common.KpiType; 12 | import com.mashibing.transformer.model.dim.StatsUserDimension; 13 | import com.mashibing.transformer.model.value.map.TimeOutputValue; 14 | import com.mashibing.transformer.model.value.reduce.MapWritableValue; 15 | 16 | /** 17 | * mapreduce程序中计算active member的reduce类
18 | * 其实就是计算每一组中去重memberid的一个个数 19 | * 20 | * @author 马士兵教育 21 | * 22 | */ 23 | public class ActiveMemberReducer extends Reducer { 24 | private Set unique = new HashSet(); 25 | private MapWritableValue outputValue = new MapWritableValue(); 26 | private MapWritable map = new MapWritable(); 27 | 28 | @Override 29 | protected void reduce(StatsUserDimension key, Iterable values, Context context) throws IOException, InterruptedException { 30 | try { 31 | // 将memberid添加到set集合中去，方便进行统计memberid的去重个数 32 | for (TimeOutputValue value : values) { 33 | this.unique.add(value.getId()); 34 | } 35 | 36 | // 设置kpi 37 | this.outputValue.setKpi(KpiType.valueOfName(key.getStatsCommon().getKpi().getKpiName())); 38 | // 设置value 39 | this.map.put(new IntWritable(-1), new IntWritable(this.unique.size())); 40 | this.outputValue.setValue(this.map); 41 | 42 | // 进行输出 43 | context.write(key, this.outputValue); 44 | } finally { 45 | // 清空操作 46 | this.unique.clear(); 47 | } 48 | 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/mr/am/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 统计active_members，规则：计算u_mid的去重个数(在各个不同维度上) 3 | * 4 | * @author 马士兵教育 5 | * 6 | */ 7 | package com.mashibing.transformer.mr.am; -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/mr/au/ActiveUserBrowserCollector.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.mr.au; 2 | 3 | import java.io.IOException; 4 | import java.sql.PreparedStatement; 5 | import java.sql.SQLException; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | import org.apache.hadoop.io.IntWritable; 9 | 10 | import com.mashibing.common.GlobalConstants; 11 | import com.mashibing.transformer.model.dim.StatsUserDimension; 12 | import com.mashibing.transformer.model.dim.base.BaseDimension; 13 | import com.mashibing.transformer.model.value.BaseStatsValueWritable; 14 | import com.mashibing.transformer.model.value.reduce.MapWritableValue; 15 | import com.mashibing.transformer.mr.IOutputCollector; 16 | import com.mashibing.transformer.service.IDimensionConverter; 17 | 18 | public class ActiveUserBrowserCollector implements IOutputCollector { 19 | 20 | @Override 21 | public void collect(Configuration conf, BaseDimension key, BaseStatsValueWritable value, PreparedStatement pstmt, IDimensionConverter converter) throws SQLException, IOException { 22 | // 进行强制后获取对应的值 23 | StatsUserDimension statsUser = (StatsUserDimension) key; 24 | IntWritable activeUserValue = (IntWritable) ((MapWritableValue) value).getValue().get(new IntWritable(-1)); 25 | 26 | // 进行参数设置 27 | int i = 0; 28 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUser.getStatsCommon().getPlatform())); 29 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUser.getStatsCommon().getDate())); 30 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUser.getBrowser())); 31 | pstmt.setInt(++i, activeUserValue.get()); 32 | pstmt.setString(++i, conf.get(GlobalConstants.RUNNING_DATE_PARAMES)); 33 | pstmt.setInt(++i, activeUserValue.get()); 34 | 35 | // 添加到batch中 36 | pstmt.addBatch(); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/mr/au/ActiveUserCollector.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.mr.au; 2 | 3 | import java.io.IOException; 4 | import java.sql.PreparedStatement; 5 | import java.sql.SQLException; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | import org.apache.hadoop.io.IntWritable; 9 | 10 | import com.mashibing.common.GlobalConstants; 11 | import com.mashibing.transformer.model.dim.StatsUserDimension; 12 | import com.mashibing.transformer.model.dim.base.BaseDimension; 13 | import com.mashibing.transformer.model.value.BaseStatsValueWritable; 14 | import com.mashibing.transformer.model.value.reduce.MapWritableValue; 15 | import com.mashibing.transformer.mr.IOutputCollector; 16 | import com.mashibing.transformer.service.IDimensionConverter; 17 | 18 | public class ActiveUserCollector implements IOutputCollector { 19 | 20 | @Override 21 | public void collect(Configuration conf, BaseDimension key, BaseStatsValueWritable value, PreparedStatement pstmt, IDimensionConverter converter) throws SQLException, IOException { 22 | // 进行强制后获取对应的值 23 | StatsUserDimension statsUser = (StatsUserDimension) key; 24 | IntWritable activeUserValue = (IntWritable) ((MapWritableValue) value).getValue().get(new IntWritable(-1)); 25 | 26 | // 进行参数设置 27 | int i = 0; 28 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUser.getStatsCommon().getPlatform())); 29 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUser.getStatsCommon().getDate())); 30 | pstmt.setInt(++i, activeUserValue.get()); 31 | pstmt.setString(++i, conf.get(GlobalConstants.RUNNING_DATE_PARAMES)); 32 | pstmt.setInt(++i, activeUserValue.get()); 33 | 34 | // 添加到batch中 35 | pstmt.addBatch(); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/mr/au/ActiveUserCombine.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.mr.au; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.mapreduce.Reducer; 6 | 7 | import com.mashibing.transformer.model.dim.StatsUserDimension; 8 | import com.mashibing.transformer.model.value.map.TimeOutputValue; 9 | 10 | /** 11 | * combine类 12 | * 13 | * @author 马士兵教育 14 | * 15 | */ 16 | public class ActiveUserCombine extends Reducer { 17 | @Override 18 | protected void reduce(StatsUserDimension key, Iterable values, Context context) throws IOException, InterruptedException { 19 | for (TimeOutputValue tov : values) { 20 | context.write(key, tov); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/mr/inbound/InboundCollector.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.mr.inbound; 2 | 3 | import java.io.IOException; 4 | import java.sql.PreparedStatement; 5 | import java.sql.SQLException; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | 9 | import com.mashibing.common.GlobalConstants; 10 | import com.mashibing.transformer.model.dim.StatsInboundDimension; 11 | import com.mashibing.transformer.model.dim.base.BaseDimension; 12 | import com.mashibing.transformer.model.value.BaseStatsValueWritable; 13 | import com.mashibing.transformer.model.value.reduce.InboundReduceValue; 14 | import com.mashibing.transformer.mr.IOutputCollector; 15 | import com.mashibing.transformer.service.IDimensionConverter; 16 | 17 | public class InboundCollector implements IOutputCollector { 18 | 19 | @Override 20 | public void collect(Configuration conf, BaseDimension key, BaseStatsValueWritable value, PreparedStatement pstmt, IDimensionConverter converter) throws SQLException, IOException { 21 | StatsInboundDimension inboundDimension = (StatsInboundDimension) key; 22 | InboundReduceValue inboundReduceValue = (InboundReduceValue) value; 23 | 24 | int i = 0; 25 | pstmt.setInt(++i, converter.getDimensionIdByValue(inboundDimension.getStatsCommon().getPlatform())); 26 | pstmt.setInt(++i, converter.getDimensionIdByValue(inboundDimension.getStatsCommon().getDate())); 27 | pstmt.setInt(++i, inboundDimension.getInbound().getId()); // 直接设置，在mapper类中已经设置 28 | pstmt.setInt(++i, inboundReduceValue.getUvs()); 29 | pstmt.setInt(++i, inboundReduceValue.getVisit()); 30 | pstmt.setString(++i, conf.get(GlobalConstants.RUNNING_DATE_PARAMES)); 31 | pstmt.setInt(++i, inboundReduceValue.getUvs()); 32 | pstmt.setInt(++i, inboundReduceValue.getVisit()); 33 | 34 | pstmt.addBatch(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/mr/inbound/InboundReducer.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.mr.inbound; 2 | 3 | import java.io.IOException; 4 | import java.util.HashSet; 5 | import java.util.Set; 6 | 7 | import org.apache.hadoop.mapreduce.Reducer; 8 | 9 | import com.mashibing.common.KpiType; 10 | import com.mashibing.transformer.model.dim.StatsInboundDimension; 11 | import com.mashibing.transformer.model.value.map.TextsOutputValue; 12 | import com.mashibing.transformer.model.value.reduce.InboundReduceValue; 13 | 14 | /** 15 | * 计算reducer类 16 | * 17 | * @author 马士兵教育 18 | * 19 | */ 20 | public class InboundReducer extends Reducer { 21 | private Set uvs = new HashSet(); 22 | private Set visits = new HashSet(); 23 | private InboundReduceValue outputValue = new InboundReduceValue(); 24 | 25 | @Override 26 | protected void reduce(StatsInboundDimension key, Iterable values, Context context) throws IOException, InterruptedException { 27 | try { 28 | for (TextsOutputValue value : values) { 29 | this.uvs.add(value.getUuid()); 30 | this.visits.add(value.getSid()); 31 | } 32 | 33 | this.outputValue.setKpi(KpiType.valueOfName(key.getStatsCommon().getKpi().getKpiName())); 34 | this.outputValue.setUvs(this.uvs.size()); 35 | this.outputValue.setVisit(this.visits.size()); 36 | context.write(key, this.outputValue); 37 | } finally { 38 | // 清空操作 39 | this.uvs.clear(); 40 | this.visits.clear(); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/mr/inbound/bounce/InboundBounceCollector.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.mr.inbound.bounce; 2 | 3 | import java.io.IOException; 4 | import java.sql.PreparedStatement; 5 | import java.sql.SQLException; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | 9 | import com.mashibing.common.GlobalConstants; 10 | import com.mashibing.transformer.model.dim.StatsInboundDimension; 11 | import com.mashibing.transformer.model.dim.base.BaseDimension; 12 | import com.mashibing.transformer.model.value.BaseStatsValueWritable; 13 | import com.mashibing.transformer.model.value.reduce.InboundBounceReduceValue; 14 | import com.mashibing.transformer.mr.IOutputCollector; 15 | import com.mashibing.transformer.service.IDimensionConverter; 16 | 17 | public class InboundBounceCollector implements IOutputCollector { 18 | 19 | @Override 20 | public void collect(Configuration conf, BaseDimension key, BaseStatsValueWritable value, PreparedStatement pstmt, IDimensionConverter converter) throws SQLException, IOException { 21 | StatsInboundDimension inboundDimension = (StatsInboundDimension) key; 22 | InboundBounceReduceValue inboundReduceValue = (InboundBounceReduceValue) value; 23 | 24 | int i = 0; 25 | pstmt.setInt(++i, converter.getDimensionIdByValue(inboundDimension.getStatsCommon().getPlatform())); 26 | pstmt.setInt(++i, converter.getDimensionIdByValue(inboundDimension.getStatsCommon().getDate())); 27 | pstmt.setInt(++i, inboundDimension.getInbound().getId()); // 直接设置，在mapper类中已经设置 28 | pstmt.setInt(++i, inboundReduceValue.getBounceNumber()); 29 | pstmt.setString(++i, conf.get(GlobalConstants.RUNNING_DATE_PARAMES)); 30 | pstmt.setInt(++i, inboundReduceValue.getBounceNumber()); 31 | 32 | pstmt.addBatch(); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/mr/inbound/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 计算统计inbund相关指标信息，包含活跃用户，总会话和跳出会话 3 | * 4 | * @author 马士兵教育 5 | * 6 | */ 7 | package com.mashibing.transformer.mr.inbound; -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/mr/location/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 统计location相关的指标信息，包括活跃用户、总会话个数、跳出会话个数。 3 | * 4 | * @author 马士兵教育 5 | * 6 | */ 7 | package com.mashibing.transformer.mr.location; -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/mr/nm/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 统计new member&total member信息 3 | * @author 马士兵教育 4 | * 5 | */ 6 | package com.mashibing.transformer.mr.nm; -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/mr/nu/StatsDeviceBrowserNewInstallUserCollector.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.mr.nu; 2 | 3 | import java.io.IOException; 4 | import java.sql.PreparedStatement; 5 | import java.sql.SQLException; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | import org.apache.hadoop.io.IntWritable; 9 | 10 | import com.mashibing.common.GlobalConstants; 11 | import com.mashibing.transformer.model.dim.StatsUserDimension; 12 | import com.mashibing.transformer.model.dim.base.BaseDimension; 13 | import com.mashibing.transformer.model.value.BaseStatsValueWritable; 14 | import com.mashibing.transformer.model.value.reduce.MapWritableValue; 15 | import com.mashibing.transformer.mr.IOutputCollector; 16 | import com.mashibing.transformer.service.IDimensionConverter; 17 | 18 | public class StatsDeviceBrowserNewInstallUserCollector implements IOutputCollector { 19 | 20 | @Override 21 | public void collect(Configuration conf, BaseDimension key, BaseStatsValueWritable value, PreparedStatement pstmt, IDimensionConverter converter) throws SQLException, IOException { 22 | StatsUserDimension statsUserDimension = (StatsUserDimension) key; 23 | MapWritableValue mapWritableValue = (MapWritableValue) value; 24 | IntWritable newInstallUsers = (IntWritable) mapWritableValue.getValue().get(new IntWritable(-1)); 25 | 26 | int i = 0; 27 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUserDimension.getStatsCommon().getPlatform())); 28 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUserDimension.getStatsCommon().getDate())); 29 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUserDimension.getBrowser())); 30 | pstmt.setInt(++i, newInstallUsers.get()); 31 | pstmt.setString(++i, conf.get(GlobalConstants.RUNNING_DATE_PARAMES)); 32 | pstmt.setInt(++i, newInstallUsers.get()); 33 | pstmt.addBatch(); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/mr/nu/StatsUserNewInstallUserCollector.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.mr.nu; 2 | 3 | import java.io.IOException; 4 | import java.sql.PreparedStatement; 5 | import java.sql.SQLException; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | import org.apache.hadoop.io.IntWritable; 9 | 10 | import com.mashibing.common.GlobalConstants; 11 | import com.mashibing.transformer.model.dim.StatsUserDimension; 12 | import com.mashibing.transformer.model.dim.base.BaseDimension; 13 | import com.mashibing.transformer.model.value.BaseStatsValueWritable; 14 | import com.mashibing.transformer.model.value.reduce.MapWritableValue; 15 | import com.mashibing.transformer.mr.IOutputCollector; 16 | import com.mashibing.transformer.service.IDimensionConverter; 17 | 18 | public class StatsUserNewInstallUserCollector implements IOutputCollector { 19 | 20 | @Override 21 | public void collect(Configuration conf, BaseDimension key, BaseStatsValueWritable value, PreparedStatement pstmt, IDimensionConverter converter) throws SQLException, IOException { 22 | StatsUserDimension statsUserDimension = (StatsUserDimension) key; 23 | MapWritableValue mapWritableValue = (MapWritableValue) value; 24 | IntWritable newInstallUsers = (IntWritable) mapWritableValue.getValue().get(new IntWritable(-1)); 25 | 26 | int i = 0; 27 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUserDimension.getStatsCommon().getPlatform())); 28 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUserDimension.getStatsCommon().getDate())); 29 | pstmt.setInt(++i, newInstallUsers.get()); 30 | pstmt.setString(++i, conf.get(GlobalConstants.RUNNING_DATE_PARAMES)); 31 | pstmt.setInt(++i, newInstallUsers.get()); 32 | pstmt.addBatch(); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/mr/pv/PageViewCollector.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.mr.pv; 2 | 3 | import java.io.IOException; 4 | import java.sql.PreparedStatement; 5 | import java.sql.SQLException; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | import org.apache.hadoop.io.IntWritable; 9 | 10 | import com.mashibing.common.GlobalConstants; 11 | import com.mashibing.transformer.model.dim.StatsUserDimension; 12 | import com.mashibing.transformer.model.dim.base.BaseDimension; 13 | import com.mashibing.transformer.model.value.BaseStatsValueWritable; 14 | import com.mashibing.transformer.model.value.reduce.MapWritableValue; 15 | import com.mashibing.transformer.mr.IOutputCollector; 16 | import com.mashibing.transformer.service.IDimensionConverter; 17 | 18 | /** 19 | * 定义pageview计算的具体输出代码 20 | * 21 | * @author 马士兵教育 22 | * 23 | */ 24 | public class PageViewCollector implements IOutputCollector { 25 | 26 | @Override 27 | public void collect(Configuration conf, BaseDimension key, BaseStatsValueWritable value, PreparedStatement pstmt, IDimensionConverter converter) throws SQLException, IOException { 28 | StatsUserDimension statsUser = (StatsUserDimension) key; 29 | int pv = ((IntWritable) ((MapWritableValue) value).getValue().get(new IntWritable(-1))).get(); 30 | 31 | // 参数设置 32 | int i = 0; 33 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUser.getStatsCommon().getPlatform())); 34 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUser.getStatsCommon().getDate())); 35 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUser.getBrowser())); 36 | pstmt.setInt(++i, pv); 37 | pstmt.setString(++i, conf.get(GlobalConstants.RUNNING_DATE_PARAMES)); 38 | pstmt.setInt(++i, pv); 39 | 40 | // 加入batch 41 | pstmt.addBatch(); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/mr/pv/PageViewReducer.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.mr.pv; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.IntWritable; 6 | import org.apache.hadoop.io.MapWritable; 7 | import org.apache.hadoop.io.NullWritable; 8 | import org.apache.hadoop.mapreduce.Reducer; 9 | 10 | import com.mashibing.common.KpiType; 11 | import com.mashibing.transformer.model.dim.StatsUserDimension; 12 | import com.mashibing.transformer.model.value.reduce.MapWritableValue; 13 | 14 | /** 15 | * 统计website的page view数量的reducer类
16 | * 不涉及到去重，直接统计输入到reducer的同一组key中，value的个数。 17 | * 18 | * @author 马士兵教育 19 | * 20 | */ 21 | public class PageViewReducer extends Reducer { 22 | private MapWritableValue mapWritableValue = new MapWritableValue(); 23 | private MapWritable map = new MapWritable(); 24 | 25 | @SuppressWarnings("unused") 26 | @Override 27 | protected void reduce(StatsUserDimension key, Iterable values, Context context) throws IOException, InterruptedException { 28 | int pvCount = 0; 29 | for (NullWritable value : values) { 30 | // pv++，每一条数据算一个pv，不涉及到去重 31 | pvCount++; 32 | } 33 | 34 | // 填充value 35 | this.map.put(new IntWritable(-1), new IntWritable(pvCount)); 36 | this.mapWritableValue.setValue(this.map); 37 | 38 | // 填充kpi 39 | this.mapWritableValue.setKpi(KpiType.valueOfName(key.getStatsCommon().getKpi().getKpiName())); 40 | 41 | // 输出 42 | context.write(key, this.mapWritableValue); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/mr/sessions/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 计算会话个数&会话长度 3 | * 4 | * @author 马士兵教育 5 | * 6 | */ 7 | package com.mashibing.transformer.mr.sessions; -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/service/IDimensionConverter.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.service; 2 | 3 | import java.io.IOException; 4 | 5 | import com.mashibing.transformer.model.dim.base.BaseDimension; 6 | 7 | /** 8 | * 提供专门操作dimension表的接口 9 | * 10 | * @author 马士兵教育 11 | * 12 | */ 13 | public interface IDimensionConverter { 14 | /** 15 | * 根据dimension的value值获取id
16 | * 如果数据库中有，那么直接返回。如果没有，那么进行插入后返回新的id值 17 | * 18 | * @param dimension 19 | * @return 20 | * @throws IOException 21 | */ 22 | public int getDimensionIdByValue(BaseDimension dimension) throws IOException; 23 | } 24 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/java/com/mashibing/transformer/util/UrlUtil.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.util; 2 | 3 | import java.net.MalformedURLException; 4 | import java.net.URL; 5 | 6 | /** 7 | * url工具类 8 | * 9 | * @author 马士兵教育 10 | * 11 | */ 12 | public class UrlUtil { 13 | /** 14 | * 判断指定的host是否是一个有效的外链host，如果不是，那么直接返回false，如果是，返回true。 15 | * 16 | * @param host 17 | * @return 18 | */ 19 | public static boolean isValidateInboundHost(String host) { 20 | if ("www.sxt.com".equals(host) || "www.isxt.com".equals(host)) { 21 | return false; 22 | } 23 | return true; 24 | } 25 | 26 | /** 27 | * 获取指定url字符串中的host 28 | * 29 | * @param url 30 | * @return 31 | * @throws MalformedURLException 32 | */ 33 | public static String getHost(String url) throws MalformedURLException { 34 | URL u = getURL(url); 35 | return u.getHost(); 36 | } 37 | 38 | /** 39 | * 根据字符串url创建一个URL对象 40 | * 41 | * @param url 42 | * @return 43 | * @throws MalformedURLException 44 | */ 45 | public static URL getURL(String url) throws MalformedURLException { 46 | url = url.trim(); 47 | if (!(url.startsWith("http:") || url.startsWith("https:"))) { 48 | url = "http://" + url; 49 | } 50 | return new URL(url); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/resource/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | fs.defaultFS 22 | hdfs://mycluster 23 | 24 | 25 | ha.zookeeper.quorum 26 | node02:2181,node03:2181,node04:2181 27 | 28 | 29 | hadoop.proxyuser.root.groups 30 | * 31 | 32 | 33 | hadoop.proxyuser.root.hosts 34 | * 35 | 36 | 37 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/resource/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=info,console 2 | 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.layout=org.apache.log4j.SimpleLayout -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/resource/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | mapreduce.framework.name 23 | yarn 24 | 25 | 26 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/resource/output-collector.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | collector_new_install_user 6 | com.mashibing.transformer.mr.nu.StatsUserNewInstallUserCollector 7 | 8 | 9 | 10 | collector_browser_new_install_user 11 | com.mashibing.transformer.mr.nu.StatsDeviceBrowserNewInstallUserCollector 12 | 13 | 14 | collector_active_user 15 | com.mashibing.transformer.mr.au.ActiveUserCollector 16 | 17 | 18 | 19 | collector_browser_active_user 20 | com.mashibing.transformer.mr.au.ActiveUserBrowserCollector 21 | 22 | 23 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/resource/qqwry.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/src/main/resource/qqwry.dat -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/resource/query-mapping.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | new_install_user 6 | 7 | INSERT INTO `stats_user`( 8 | `platform_dimension_id`, 9 | `date_dimension_id`, 10 | `new_install_users`, 11 | `created`) 12 | VALUES(?, ?, ?, ?) ON DUPLICATE KEY UPDATE `new_install_users` = ? 13 | 14 | 15 | 16 | 17 | browser_new_install_user 18 | 19 | INSERT INTO `stats_device_browser`( 20 | `platform_dimension_id`, 21 | `date_dimension_id`, 22 | `browser_dimension_id`, 23 | `new_install_users`, 24 | `created`) 25 | VALUES(?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE `new_install_users` = ? 26 | 27 | 28 | 29 | 30 | active_user 31 | 32 | INSERT INTO `stats_user`( 33 | `platform_dimension_id`, 34 | `date_dimension_id`, 35 | `active_users`, 36 | `created`) 37 | VALUES(?, ?, ?, ?) ON DUPLICATE KEY UPDATE `active_users` = ? 38 | 39 | 40 | 41 | 42 | browser_active_user 43 | 44 | INSERT INTO `stats_device_browser`( 45 | `platform_dimension_id`, 46 | `date_dimension_id`, 47 | `browser_dimension_id`, 48 | `active_users`, 49 | `created` 50 | ) VALUES(?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE `active_users` = ? 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/resource/transformer-env.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | mysql.batch.number 6 | 10 7 | 8 | 9 | 10 | mysql.report.driver 11 | com.mysql.jdbc.Driver 12 | 13 | 14 | 15 | mysql.report.url 16 | jdbc:mysql://node01:3306/result_db?useUnicode=true&characterEncoding=utf8 17 | 18 | 19 | 20 | mysql.report.username 21 | root 22 | 23 | 24 | 25 | mysql.report.password 26 | 123 27 | 28 | 29 | -------------------------------------------------------------------------------- /code/log_analyse_full/src/main/resource/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 15 | 16 | 17 | 18 | 19 | 20 | yarn.nodemanager.aux-services 21 | mapreduce_shuffle 22 | 23 | 24 | yarn.resourcemanager.ha.enabled 25 | true 26 | 27 | 28 | yarn.resourcemanager.zk-address 29 | node02:2181,node03:2181,node04:2181 30 | 31 | 32 | yarn.resourcemanager.cluster-id 33 | mashibing 34 | 35 | 36 | yarn.resourcemanager.ha.rm-ids 37 | rm1,rm2 38 | 39 | 40 | yarn.resourcemanager.hostname.rm1 41 | node03 42 | 43 | 44 | yarn.resourcemanager.hostname.rm2 45 | node04 46 | 47 | 48 | yarn.log-aggregation-enable 49 | true 50 | 51 | 52 | yarn.log-aggregation.retain-seconds 53 | 604800 54 | 55 | 56 | -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/common/DateEnum.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/common/DateEnum.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/common/EventLogConstants$EventEnum.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/common/EventLogConstants$EventEnum.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/common/EventLogConstants.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/common/EventLogConstants.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/common/GlobalConstants.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/common/GlobalConstants.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/common/KpiType.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/common/KpiType.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/etl/mr/ald/AnalyserLogDataMapper$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/etl/mr/ald/AnalyserLogDataMapper$1.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/etl/mr/ald/AnalyserLogDataMapper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/etl/mr/ald/AnalyserLogDataMapper.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/etl/mr/ald/AnalyserLogDataRunner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/etl/mr/ald/AnalyserLogDataRunner.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/etl/util/IPSeekerExt$RegionInfo.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/etl/util/IPSeekerExt$RegionInfo.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/etl/util/IPSeekerExt.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/etl/util/IPSeekerExt.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/etl/util/LoggerUtil.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/etl/util/LoggerUtil.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/etl/util/UserAgentUtil$UserAgentInfo.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/etl/util/UserAgentUtil$UserAgentInfo.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/etl/util/UserAgentUtil.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/etl/util/UserAgentUtil.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/etl/util/ip/IPSeeker$IPEntry.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/etl/util/ip/IPSeeker$IPEntry.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/etl/util/ip/IPSeeker$IPLocation.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/etl/util/ip/IPSeeker$IPLocation.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/etl/util/ip/IPSeeker$IPSeekerUtils.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/etl/util/ip/IPSeeker$IPSeekerUtils.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/etl/util/ip/IPSeeker.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/etl/util/ip/IPSeeker.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/hive/DateDimensionUDF.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/hive/DateDimensionUDF.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/StatsCommonDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/StatsCommonDimension.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/StatsDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/StatsDimension.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/StatsInboundBounceDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/StatsInboundBounceDimension.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/StatsInboundDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/StatsInboundDimension.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/StatsLocationDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/StatsLocationDimension.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/StatsUserDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/StatsUserDimension.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/base/BaseDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/base/BaseDimension.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/base/BrowserDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/base/BrowserDimension.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/base/DateDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/base/DateDimension.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/base/InboundDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/base/InboundDimension.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/base/KpiDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/base/KpiDimension.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/base/LocationDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/base/LocationDimension.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/base/PlatformDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/dim/base/PlatformDimension.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/value/BaseStatsValueWritable.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/value/BaseStatsValueWritable.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/value/map/TextsOutputValue.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/value/map/TextsOutputValue.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/value/map/TimeOutputValue.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/value/map/TimeOutputValue.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/value/reduce/InboundBounceReduceValue.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/value/reduce/InboundBounceReduceValue.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/value/reduce/InboundReduceValue.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/value/reduce/InboundReduceValue.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/value/reduce/LocationReducerOutputValue.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/value/reduce/LocationReducerOutputValue.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/model/value/reduce/MapWritableValue.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/model/value/reduce/MapWritableValue.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/IOutputCollector.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/IOutputCollector.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/TransformerBaseMapper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/TransformerBaseMapper.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/TransformerBaseRunner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/TransformerBaseRunner.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/TransformerOutputFormat$TransformerRecordWriter.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/TransformerOutputFormat$TransformerRecordWriter.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/TransformerOutputFormat.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/TransformerOutputFormat.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/am/ActiveMemberCollector.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/am/ActiveMemberCollector.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/am/ActiveMemberMapper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/am/ActiveMemberMapper.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/am/ActiveMemberReducer.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/am/ActiveMemberReducer.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/am/ActiveMemberRunner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/am/ActiveMemberRunner.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/au/ActiveUserBrowserCollector.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/au/ActiveUserBrowserCollector.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/au/ActiveUserCollector.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/au/ActiveUserCollector.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/au/ActiveUserCombine.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/au/ActiveUserCombine.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/au/ActiveUserMapper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/au/ActiveUserMapper.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/au/ActiveUserReducer.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/au/ActiveUserReducer.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/au/ActiveUserRunner$ActiveUserPartitioner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/au/ActiveUserRunner$ActiveUserPartitioner.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/au/ActiveUserRunner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/au/ActiveUserRunner.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/au/HourlyActiveUserCollector.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/au/HourlyActiveUserCollector.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/InboundCollector.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/InboundCollector.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/InboundMapper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/InboundMapper.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/InboundReducer.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/InboundReducer.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/InboundRunner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/InboundRunner.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/bounce/InboundBounceCollector.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/bounce/InboundBounceCollector.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/bounce/InboundBounceMapper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/bounce/InboundBounceMapper.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/bounce/InboundBounceReducer.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/bounce/InboundBounceReducer.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/bounce/InboundBounceRunner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/bounce/InboundBounceRunner.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/bounce/InboundBounceSecondSort$InboundBounceGroupingComparator.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/bounce/InboundBounceSecondSort$InboundBounceGroupingComparator.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/bounce/InboundBounceSecondSort$InboundBouncePartitioner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/bounce/InboundBounceSecondSort$InboundBouncePartitioner.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/bounce/InboundBounceSecondSort.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/inbound/bounce/InboundBounceSecondSort.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/location/LocationCollector.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/location/LocationCollector.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/location/LocationMapper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/location/LocationMapper.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/location/LocationReducer.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/location/LocationReducer.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/location/LocationRunner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/location/LocationRunner.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nm/NewMemberCollector$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nm/NewMemberCollector$1.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nm/NewMemberCollector.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nm/NewMemberCollector.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nm/NewMemberMapper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nm/NewMemberMapper.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nm/NewMemberReducer.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nm/NewMemberReducer.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nm/NewMemberRunner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nm/NewMemberRunner.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nu/NewInstallUserMapper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nu/NewInstallUserMapper.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nu/NewInstallUserReducer.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nu/NewInstallUserReducer.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nu/NewInstallUserRunner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nu/NewInstallUserRunner.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nu/StatsDeviceBrowserNewInstallUserCollector.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nu/StatsDeviceBrowserNewInstallUserCollector.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nu/StatsUserNewInstallUserCollector.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/nu/StatsUserNewInstallUserCollector.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/pv/PageViewCollector.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/pv/PageViewCollector.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/pv/PageViewMapper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/pv/PageViewMapper.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/pv/PageViewReducer.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/pv/PageViewReducer.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/pv/PageViewRunner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/pv/PageViewRunner.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/sessions/SessionsCollector$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/sessions/SessionsCollector$1.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/sessions/SessionsCollector.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/sessions/SessionsCollector.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/sessions/SessionsMapper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/sessions/SessionsMapper.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/sessions/SessionsReducer.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/sessions/SessionsReducer.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/mr/sessions/SessionsRunner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/mr/sessions/SessionsRunner.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/service/IDimensionConverter.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/service/IDimensionConverter.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/service/impl/DimensionConverterImpl$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/service/impl/DimensionConverterImpl$1.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/service/impl/DimensionConverterImpl.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/service/impl/DimensionConverterImpl.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/service/impl/InboundDimensionService.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/service/impl/InboundDimensionService.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/util/MemberUtil$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/util/MemberUtil$1.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/util/MemberUtil.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/util/MemberUtil.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/util/TimeChain.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/util/TimeChain.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/transformer/util/UrlUtil.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/transformer/util/UrlUtil.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/util/EJob$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/util/EJob$1.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/util/EJob.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/util/EJob.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/util/JdbcManager.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/util/JdbcManager.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/com/mashibing/util/TimeUtil.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/com/mashibing/util/TimeUtil.class -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | fs.defaultFS 22 | hdfs://mycluster 23 | 24 | 25 | ha.zookeeper.quorum 26 | node02:2181,node03:2181,node04:2181 27 | 28 | 29 | hadoop.proxyuser.root.groups 30 | * 31 | 32 | 33 | hadoop.proxyuser.root.hosts 34 | * 35 | 36 | 37 | -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=info,console 2 | 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.layout=org.apache.log4j.SimpleLayout -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | mapreduce.framework.name 23 | yarn 24 | 25 | 26 | -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/output-collector.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | collector_new_install_user 6 | com.mashibing.transformer.mr.nu.StatsUserNewInstallUserCollector 7 | 8 | 9 | 10 | collector_browser_new_install_user 11 | com.mashibing.transformer.mr.nu.StatsDeviceBrowserNewInstallUserCollector 12 | 13 | 14 | collector_active_user 15 | com.mashibing.transformer.mr.au.ActiveUserCollector 16 | 17 | 18 | 19 | collector_browser_active_user 20 | com.mashibing.transformer.mr.au.ActiveUserBrowserCollector 21 | 22 | 23 | -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/qqwry.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_full/target/classes/qqwry.dat -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/query-mapping.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | new_install_user 6 | 7 | INSERT INTO `stats_user`( 8 | `platform_dimension_id`, 9 | `date_dimension_id`, 10 | `new_install_users`, 11 | `created`) 12 | VALUES(?, ?, ?, ?) ON DUPLICATE KEY UPDATE `new_install_users` = ? 13 | 14 | 15 | 16 | 17 | browser_new_install_user 18 | 19 | INSERT INTO `stats_device_browser`( 20 | `platform_dimension_id`, 21 | `date_dimension_id`, 22 | `browser_dimension_id`, 23 | `new_install_users`, 24 | `created`) 25 | VALUES(?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE `new_install_users` = ? 26 | 27 | 28 | 29 | 30 | active_user 31 | 32 | INSERT INTO `stats_user`( 33 | `platform_dimension_id`, 34 | `date_dimension_id`, 35 | `active_users`, 36 | `created`) 37 | VALUES(?, ?, ?, ?) ON DUPLICATE KEY UPDATE `active_users` = ? 38 | 39 | 40 | 41 | 42 | browser_active_user 43 | 44 | INSERT INTO `stats_device_browser`( 45 | `platform_dimension_id`, 46 | `date_dimension_id`, 47 | `browser_dimension_id`, 48 | `active_users`, 49 | `created` 50 | ) VALUES(?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE `active_users` = ? 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/transformer-env.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | mysql.batch.number 6 | 10 7 | 8 | 9 | 10 | mysql.report.driver 11 | com.mysql.jdbc.Driver 12 | 13 | 14 | 15 | mysql.report.url 16 | jdbc:mysql://node01:3306/result_db?useUnicode=true&characterEncoding=utf8 17 | 18 | 19 | 20 | mysql.report.username 21 | root 22 | 23 | 24 | 25 | mysql.report.password 26 | 123 27 | 28 | 29 | -------------------------------------------------------------------------------- /code/log_analyse_full/target/classes/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 15 | 16 | 17 | 18 | 19 | 20 | yarn.nodemanager.aux-services 21 | mapreduce_shuffle 22 | 23 | 24 | yarn.resourcemanager.ha.enabled 25 | true 26 | 27 | 28 | yarn.resourcemanager.zk-address 29 | node02:2181,node03:2181,node04:2181 30 | 31 | 32 | yarn.resourcemanager.cluster-id 33 | mashibing 34 | 35 | 36 | yarn.resourcemanager.ha.rm-ids 37 | rm1,rm2 38 | 39 | 40 | yarn.resourcemanager.hostname.rm1 41 | node03 42 | 43 | 44 | yarn.resourcemanager.hostname.rm2 45 | node04 46 | 47 | 48 | yarn.log-aggregation-enable 49 | true 50 | 51 | 52 | yarn.log-aggregation.retain-seconds 53 | 604800 54 | 55 | 56 | -------------------------------------------------------------------------------- /code/log_analyse_mr/.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /code/log_analyse_mr/.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /code/log_analyse_mr/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /code/log_analyse_mr/log_analyse_mr.iml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/java/com/mashibing/common/DateEnum.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.common; 2 | 3 | /** 4 | * 日期类型枚举类 5 | * 6 | * @author 马士兵教育 7 | * 8 | */ 9 | public enum DateEnum { 10 | YEAR("year"), SEASON("season"), MONTH("month"), WEEK("week"), DAY("day"), HOUR( 11 | "hour"); 12 | 13 | public final String name; 14 | 15 | private DateEnum(String name) { 16 | this.name = name; 17 | } 18 | 19 | /** 20 | * 根据属性name的值获取对应的type对象 21 | * 22 | * @param name 23 | * @return 24 | */ 25 | public static DateEnum valueOfName(String name) { 26 | for (DateEnum type : values()) { 27 | if (type.name.equals(name)) { 28 | return type; 29 | } 30 | } 31 | return null; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/java/com/mashibing/common/GlobalConstants.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.common; 2 | 3 | /** 4 | * 全局常量类 5 | * 6 | * @author 马士兵教育 7 | * 8 | */ 9 | public class GlobalConstants { 10 | 11 | /** 12 | * 一天的毫秒数 13 | */ 14 | public static final int DAY_OF_MILLISECONDS = 86400000; 15 | 16 | /** 17 | * 定义的运行时间变量名 18 | */ 19 | public static final String RUNNING_DATE_PARAMES = "RUNNING_DATE"; 20 | 21 | /** 22 | * 默认值 23 | */ 24 | public static final String DEFAULT_VALUE = "unknown"; 25 | 26 | /** 27 | * 维度信息表中指定全部列值 28 | */ 29 | public static final String VALUE_OF_ALL = "all"; 30 | 31 | 32 | /** 33 | * 定义的output collector的前缀 34 | */ 35 | public static final String OUTPUT_COLLECTOR_KEY_PREFIX = "collector_"; 36 | 37 | /** 38 | * 指定连接表配置为report 39 | */ 40 | public static final String WAREHOUSE_OF_REPORT = "report"; 41 | 42 | /** 43 | * 批量执行的key 44 | */ 45 | public static final String JDBC_BATCH_NUMBER = "mysql.batch.number"; 46 | 47 | /** 48 | * 默认批量大小 49 | */ 50 | public static final String DEFAULT_JDBC_BATCH_NUMBER = "500"; 51 | 52 | /** 53 | * driver 名称 54 | */ 55 | public static final String JDBC_DRIVER = "mysql.%s.driver"; 56 | 57 | /** 58 | * JDBC URL 59 | */ 60 | public static final String JDBC_URL = "mysql.%s.url"; 61 | 62 | /** 63 | * username名称 64 | */ 65 | public static final String JDBC_USERNAME = "mysql.%s.username"; 66 | 67 | /** 68 | * password名称 69 | */ 70 | public static final String JDBC_PASSWORD = "mysql.%s.password"; 71 | 72 | } 73 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/java/com/mashibing/common/KpiType.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.common; 2 | 3 | /** 4 | * 统计kpi的名称枚举类 5 | * 6 | * @author 马士兵教育 7 | * 8 | */ 9 | public enum KpiType { 10 | 11 | NEW_INSTALL_USER("new_install_user"), // 统计新用户的kpi 12 | BROWSER_NEW_INSTALL_USER("browser_new_install_user"), // 统计浏览器维度的新用户kpi 13 | ACTIVE_USER("active_user"), // 统计活跃用户kpi 14 | BROWSER_ACTIVE_USER("browser_active_user"), // 统计浏览器维度的活跃用户kpi 15 | ; 16 | 17 | public final String name; 18 | 19 | private KpiType(String name) { 20 | this.name = name; 21 | } 22 | 23 | /** 24 | * 根据kpiType的名称字符串值，获取对应的kpitype枚举对象 25 | * 26 | * @param name 27 | * @return 28 | */ 29 | public static KpiType valueOfName(String name) { 30 | for (KpiType type : values()) { 31 | if (type.name.equals(name)) { 32 | return type; 33 | } 34 | } 35 | throw new RuntimeException("指定的name不属于该KpiType枚举类：" + name); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/java/com/mashibing/transformer/model/dim/StatsDimension.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.model.dim; 2 | 3 | import com.mashibing.transformer.model.dim.base.BaseDimension; 4 | 5 | public abstract class StatsDimension extends BaseDimension { 6 | // nothing 7 | } 8 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/java/com/mashibing/transformer/model/dim/base/BaseDimension.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.model.dim.base; 2 | 3 | import org.apache.hadoop.io.WritableComparable; 4 | 5 | /** 6 | * 顶级的维度信息类 7 | * 8 | * @author 马士兵教育 9 | * 10 | */ 11 | public abstract class BaseDimension implements WritableComparable{ 12 | // nothing 13 | } 14 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/java/com/mashibing/transformer/model/value/BaseStatsValueWritable.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.model.value; 2 | 3 | import com.mashibing.common.KpiType; 4 | import org.apache.hadoop.io.Writable; 5 | 6 | import com.mashibing.common.KpiType; 7 | 8 | /** 9 | * 自定义顶级的输出value父类 10 | * 11 | * @author 马士兵教育 12 | * 13 | */ 14 | public abstract class BaseStatsValueWritable implements Writable { 15 | /** 16 | * 获取当前value对应的kpi值 17 | * 18 | * @return 19 | */ 20 | public abstract KpiType getKpi(); 21 | } 22 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/java/com/mashibing/transformer/model/value/map/TimeOutputValue.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.model.value.map; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | 7 | import com.mashibing.common.KpiType; 8 | import com.mashibing.transformer.model.value.BaseStatsValueWritable; 9 | 10 | public class TimeOutputValue extends BaseStatsValueWritable { 11 | private String id; // id 12 | private long time; // 时间戳 13 | 14 | public String getId() { 15 | return id; 16 | } 17 | 18 | public void setId(String id) { 19 | this.id = id; 20 | } 21 | 22 | public long getTime() { 23 | return time; 24 | } 25 | 26 | public void setTime(long time) { 27 | this.time = time; 28 | } 29 | 30 | @Override 31 | public void write(DataOutput out) throws IOException { 32 | out.writeUTF(this.id); 33 | out.writeLong(this.time); 34 | } 35 | 36 | @Override 37 | public void readFields(DataInput in) throws IOException { 38 | this.id = in.readUTF(); 39 | this.time = in.readLong(); 40 | } 41 | 42 | @Override 43 | public KpiType getKpi() { 44 | // TODO Auto-generated method stub 45 | return null; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/java/com/mashibing/transformer/model/value/reduce/MapWritableValue.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.model.value.reduce; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | 7 | import org.apache.hadoop.io.MapWritable; 8 | import org.apache.hadoop.io.WritableUtils; 9 | 10 | import com.mashibing.common.KpiType; 11 | import com.mashibing.transformer.model.value.BaseStatsValueWritable; 12 | 13 | public class MapWritableValue extends BaseStatsValueWritable { 14 | /** 15 | * 16 | */ 17 | private MapWritable value = new MapWritable();//即将插入数据库表中的一行记录 18 | private KpiType kpi; 19 | 20 | public MapWritableValue() { 21 | super(); 22 | } 23 | 24 | public MapWritableValue(MapWritable value, KpiType kpi) { 25 | super(); 26 | this.value = value; 27 | this.kpi = kpi; 28 | } 29 | 30 | public MapWritable getValue() { 31 | return value; 32 | } 33 | 34 | public void setValue(MapWritable value) { 35 | this.value = value; 36 | } 37 | 38 | public void setKpi(KpiType kpi) { 39 | this.kpi = kpi; 40 | } 41 | 42 | @Override 43 | public void write(DataOutput out) throws IOException { 44 | this.value.write(out); 45 | WritableUtils.writeEnum(out, this.kpi); 46 | } 47 | 48 | @Override 49 | public void readFields(DataInput in) throws IOException { 50 | this.value.readFields(in); 51 | this.kpi = WritableUtils.readEnum(in, KpiType.class); 52 | } 53 | 54 | @Override 55 | public KpiType getKpi() { 56 | return this.kpi; 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/java/com/mashibing/transformer/mr/IOutputCollector.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.mr; 2 | 3 | import java.io.IOException; 4 | import java.sql.PreparedStatement; 5 | import java.sql.SQLException; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | 9 | import com.mashibing.transformer.model.dim.base.BaseDimension; 10 | import com.mashibing.transformer.model.value.BaseStatsValueWritable; 11 | import com.mashibing.transformer.service.IDimensionConverter; 12 | 13 | /** 14 | * 自定义的配合自定义output进行具体sql输出的类 15 | * 16 | * @author 马士兵教育 17 | * 18 | */ 19 | public interface IOutputCollector { 20 | 21 | /** 22 | * 具体执行统计数据插入的方法 23 | * 24 | * @param conf 25 | * @param key 26 | * @param value 27 | * @param pstmt 28 | * @param converter 29 | * @throws SQLException 30 | * @throws IOException 31 | */ 32 | public void collect(Configuration conf, BaseDimension key, BaseStatsValueWritable value, PreparedStatement pstmt, IDimensionConverter converter) throws SQLException, IOException; 33 | } 34 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/java/com/mashibing/transformer/mr/au/ActiveUserBrowserCollector.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.mr.au; 2 | 3 | import java.io.IOException; 4 | import java.sql.PreparedStatement; 5 | import java.sql.SQLException; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | import org.apache.hadoop.io.IntWritable; 9 | 10 | import com.mashibing.common.GlobalConstants; 11 | import com.mashibing.transformer.model.dim.StatsUserDimension; 12 | import com.mashibing.transformer.model.dim.base.BaseDimension; 13 | import com.mashibing.transformer.model.value.BaseStatsValueWritable; 14 | import com.mashibing.transformer.model.value.reduce.MapWritableValue; 15 | import com.mashibing.transformer.mr.IOutputCollector; 16 | import com.mashibing.transformer.service.IDimensionConverter; 17 | 18 | /** 19 | * 20 | * @author 马士兵教育 21 | * 22 | */ 23 | public class ActiveUserBrowserCollector implements IOutputCollector { 24 | 25 | @Override 26 | public void collect(Configuration conf, BaseDimension key, BaseStatsValueWritable value, PreparedStatement pstmt, IDimensionConverter converter) throws SQLException, IOException { 27 | // 进行强制后获取对应的值 28 | StatsUserDimension statsUser = (StatsUserDimension) key; 29 | IntWritable activeUserValue = (IntWritable) ((MapWritableValue) value).getValue().get(new IntWritable(-1)); 30 | 31 | // 进行参数设置 32 | int i = 0; 33 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUser.getStatsCommon().getPlatform())); 34 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUser.getStatsCommon().getDate())); 35 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUser.getBrowser())); 36 | pstmt.setInt(++i, activeUserValue.get()); 37 | pstmt.setString(++i, conf.get(GlobalConstants.RUNNING_DATE_PARAMES)); 38 | pstmt.setInt(++i, activeUserValue.get()); 39 | 40 | // 添加到batch中 41 | pstmt.addBatch(); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/java/com/mashibing/transformer/mr/au/ActiveUserCollector.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.mr.au; 2 | 3 | import java.io.IOException; 4 | import java.sql.PreparedStatement; 5 | import java.sql.SQLException; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | import org.apache.hadoop.io.IntWritable; 9 | 10 | import com.mashibing.common.GlobalConstants; 11 | import com.mashibing.transformer.model.dim.StatsUserDimension; 12 | import com.mashibing.transformer.model.dim.base.BaseDimension; 13 | import com.mashibing.transformer.model.value.BaseStatsValueWritable; 14 | import com.mashibing.transformer.model.value.reduce.MapWritableValue; 15 | import com.mashibing.transformer.mr.IOutputCollector; 16 | import com.mashibing.transformer.service.IDimensionConverter; 17 | 18 | /** 19 | * 20 | * @author 马士兵教育 21 | * 22 | */ 23 | public class ActiveUserCollector implements IOutputCollector { 24 | 25 | @Override 26 | public void collect(Configuration conf, BaseDimension key, BaseStatsValueWritable value, 27 | PreparedStatement pstmt, IDimensionConverter converter) throws SQLException, IOException { 28 | // 进行强制后获取对应的值 29 | StatsUserDimension statsUser = (StatsUserDimension) key; 30 | IntWritable activeUserValue = (IntWritable) ((MapWritableValue) value).getValue().get(new IntWritable(-1)); 31 | 32 | // 进行参数设置 33 | int i = 0; 34 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUser.getStatsCommon().getPlatform())); 35 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUser.getStatsCommon().getDate())); 36 | pstmt.setInt(++i, activeUserValue.get()); 37 | pstmt.setString(++i, conf.get(GlobalConstants.RUNNING_DATE_PARAMES)); 38 | pstmt.setInt(++i, activeUserValue.get()); 39 | 40 | // 添加到batch中 41 | pstmt.addBatch(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/java/com/mashibing/transformer/mr/au/ActiveUserReducer.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.mr.au; 2 | 3 | import java.io.IOException; 4 | import java.util.HashSet; 5 | import java.util.Set; 6 | 7 | import org.apache.hadoop.io.IntWritable; 8 | import org.apache.hadoop.io.MapWritable; 9 | import org.apache.hadoop.mapreduce.Reducer; 10 | 11 | import com.mashibing.common.KpiType; 12 | import com.mashibing.transformer.model.dim.StatsUserDimension; 13 | import com.mashibing.transformer.model.value.map.TimeOutputValue; 14 | import com.mashibing.transformer.model.value.reduce.MapWritableValue; 15 | 16 | /** 17 | * 统计active user， 其实就是计算本一组中这个uuid的个数 18 | * 19 | * @author 马士兵教育 20 | * 21 | */ 22 | public class ActiveUserReducer extends 23 | Reducer { 24 | 25 | private Set unique = new HashSet(); 26 | private MapWritableValue outputValue = new MapWritableValue(); 27 | private MapWritable map = new MapWritable(); 28 | 29 | @Override 30 | protected void reduce(StatsUserDimension key, Iterable values, Context context) 31 | throws IOException, InterruptedException { 32 | try { 33 | // 将uuid添加到set集合中去，方便进行统计uuid的去重个数 34 | for (TimeOutputValue value : values) { 35 | this.unique.add(value.getId()); 36 | } 37 | 38 | // 设置kpi 39 | this.outputValue.setKpi(KpiType.valueOfName(key.getStatsCommon().getKpi().getKpiName())); 40 | // 设置value 41 | this.map.put(new IntWritable(-1), new IntWritable(this.unique.size())); 42 | this.outputValue.setValue(this.map); 43 | 44 | // 进行输出 45 | context.write(key, this.outputValue); 46 | } finally { 47 | // 清空操作 48 | this.unique.clear(); 49 | } 50 | 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/java/com/mashibing/transformer/mr/nu/StatsUserNewInstallUserCollector.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.mr.nu; 2 | 3 | import java.io.IOException; 4 | import java.sql.PreparedStatement; 5 | import java.sql.SQLException; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | import org.apache.hadoop.io.IntWritable; 9 | 10 | import com.mashibing.common.GlobalConstants; 11 | import com.mashibing.transformer.model.dim.StatsUserDimension; 12 | import com.mashibing.transformer.model.dim.base.BaseDimension; 13 | import com.mashibing.transformer.model.value.BaseStatsValueWritable; 14 | import com.mashibing.transformer.model.value.reduce.MapWritableValue; 15 | import com.mashibing.transformer.mr.IOutputCollector; 16 | import com.mashibing.transformer.service.IDimensionConverter; 17 | 18 | public class StatsUserNewInstallUserCollector implements IOutputCollector { 19 | 20 | @Override 21 | public void collect(Configuration conf, BaseDimension key, BaseStatsValueWritable value, 22 | PreparedStatement pstmt, IDimensionConverter converter) 23 | throws SQLException, IOException { 24 | StatsUserDimension statsUserDimension = (StatsUserDimension) key; 25 | MapWritableValue mapWritableValue = (MapWritableValue) value; 26 | IntWritable newInstallUsers = (IntWritable) mapWritableValue.getValue().get(new IntWritable(-1)); 27 | 28 | int i = 0; 29 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUserDimension.getStatsCommon().getPlatform())); 30 | pstmt.setInt(++i, converter.getDimensionIdByValue(statsUserDimension.getStatsCommon().getDate())); 31 | pstmt.setInt(++i, newInstallUsers.get()); 32 | pstmt.setString(++i, conf.get(GlobalConstants.RUNNING_DATE_PARAMES)); 33 | pstmt.setInt(++i, newInstallUsers.get()); 34 | pstmt.addBatch();//往批处理放入数据 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/java/com/mashibing/transformer/service/IDimensionConverter.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.transformer.service; 2 | 3 | import java.io.IOException; 4 | 5 | import com.mashibing.transformer.model.dim.base.BaseDimension; 6 | 7 | /** 8 | * 提供专门操作（从关系型数据库中查询、插入）dimension表的接口 9 | * 10 | * @author 马士兵教育 11 | * 12 | */ 13 | public interface IDimensionConverter { 14 | /** 15 | * 根据dimension的value值获取id
16 | * 如果数据库中有，那么直接返回。如果没有，那么进行插入后返回新的id值 17 | * 18 | * @param dimension 19 | * @return 20 | * @throws IOException 21 | */ 22 | public int getDimensionIdByValue(BaseDimension dimension) throws IOException; 23 | } 24 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/java/com/mashibing/util/JdbcManager.java: -------------------------------------------------------------------------------- 1 | package com.mashibing.util; 2 | 3 | import java.sql.Connection; 4 | import java.sql.DriverManager; 5 | import java.sql.SQLException; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | 9 | import com.mashibing.common.GlobalConstants; 10 | 11 | /** 12 | * jdbc管理 13 | * 14 | * @author 马士兵教育 15 | * 16 | */ 17 | public class JdbcManager { 18 | /** 19 | * 根据配置获取获取关系型数据库的jdbc连接 20 | * 21 | * @param conf 22 | * hadoop配置信息 23 | * @param flag 24 | * 区分不同数据源的标志位 25 | * @return 26 | * @throws SQLException 27 | */ 28 | public static Connection getConnection(Configuration conf, String flag) throws SQLException { 29 | String driverStr = String.format(GlobalConstants.JDBC_DRIVER, flag); 30 | String urlStr = String.format(GlobalConstants.JDBC_URL, flag); 31 | String usernameStr = String.format(GlobalConstants.JDBC_USERNAME, flag); 32 | String passwordStr = String.format(GlobalConstants.JDBC_PASSWORD, flag); 33 | 34 | String driverClass = conf.get(driverStr); 35 | String url = conf.get(urlStr); 36 | String username = conf.get(usernameStr); 37 | String password = conf.get(passwordStr); 38 | try { 39 | Class.forName(driverClass); 40 | } catch (ClassNotFoundException e) { 41 | // nothing 42 | } 43 | return DriverManager.getConnection(url, username, password); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/resource/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | fs.defaultFS 22 | hdfs://mycluster 23 | 24 | 25 | ha.zookeeper.quorum 26 | node02:2181,node03:2181,node04:2181 27 | 28 | 29 | hadoop.proxyuser.root.groups 30 | * 31 | 32 | 33 | hadoop.proxyuser.root.hosts 34 | * 35 | 36 | 37 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/resource/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=info,console 2 | 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.layout=org.apache.log4j.SimpleLayout -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/resource/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | mapreduce.framework.name 23 | yarn 24 | 25 | 26 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/resource/output-collector.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | collector_new_install_user 6 | com.mashibing.transformer.mr.nu.StatsUserNewInstallUserCollector 7 | 8 | 9 | 10 | collector_browser_new_install_user 11 | com.mashibing.transformer.mr.nu.StatsDeviceBrowserNewInstallUserCollector 12 | 13 | 14 | collector_active_user 15 | com.mashibing.transformer.mr.au.ActiveUserCollector 16 | 17 | 18 | 19 | collector_browser_active_user 20 | com.mashibing.transformer.mr.au.ActiveUserBrowserCollector 21 | 22 | 23 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/resource/qqwry.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/src/main/resource/qqwry.dat -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/resource/query-mapping.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | new_install_user 6 | 7 | INSERT INTO `stats_user`( 8 | `platform_dimension_id`, 9 | `date_dimension_id`, 10 | `new_install_users`, 11 | `created`) 12 | VALUES(?, ?, ?, ?) ON DUPLICATE KEY UPDATE `new_install_users` = ? 13 | 14 | 15 | 16 | 17 | browser_new_install_user 18 | 19 | INSERT INTO `stats_device_browser`( 20 | `platform_dimension_id`, 21 | `date_dimension_id`, 22 | `browser_dimension_id`, 23 | `new_install_users`, 24 | `created`) 25 | VALUES(?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE `new_install_users` = ? 26 | 27 | 28 | 29 | 30 | active_user 31 | 32 | INSERT INTO `stats_user`( 33 | `platform_dimension_id`, 34 | `date_dimension_id`, 35 | `active_users`, 36 | `created`) 37 | VALUES(?, ?, ?, ?) ON DUPLICATE KEY UPDATE `active_users` = ? 38 | 39 | 40 | 41 | 42 | browser_active_user 43 | 44 | INSERT INTO `stats_device_browser`( 45 | `platform_dimension_id`, 46 | `date_dimension_id`, 47 | `browser_dimension_id`, 48 | `active_users`, 49 | `created` 50 | ) VALUES(?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE `active_users` = ? 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/resource/transformer-env.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | mysql.batch.number 6 | 10 7 | 8 | 9 | 10 | mysql.report.driver 11 | com.mysql.jdbc.Driver 12 | 13 | 14 | 15 | mysql.report.url 16 | jdbc:mysql://node01:3306/result_db?useUnicode=true&characterEncoding=utf8 17 | 18 | 19 | 20 | mysql.report.username 21 | root 22 | 23 | 24 | 25 | mysql.report.password 26 | 123 27 | 28 | 29 | -------------------------------------------------------------------------------- /code/log_analyse_mr/src/main/resource/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 15 | 16 | 17 | 18 | 19 | 20 | yarn.nodemanager.aux-services 21 | mapreduce_shuffle 22 | 23 | 24 | yarn.resourcemanager.ha.enabled 25 | true 26 | 27 | 28 | yarn.resourcemanager.zk-address 29 | node02:2181,node03:2181,node04:2181 30 | 31 | 32 | yarn.resourcemanager.cluster-id 33 | mashibing 34 | 35 | 36 | yarn.resourcemanager.ha.rm-ids 37 | rm1,rm2 38 | 39 | 40 | yarn.resourcemanager.hostname.rm1 41 | node03 42 | 43 | 44 | yarn.resourcemanager.hostname.rm2 45 | node04 46 | 47 | 48 | yarn.log-aggregation-enable 49 | true 50 | 51 | 52 | yarn.log-aggregation.retain-seconds 53 | 604800 54 | 55 | 56 | -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/common/DateEnum.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/common/DateEnum.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/common/EventLogConstants$EventEnum.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/common/EventLogConstants$EventEnum.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/common/EventLogConstants.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/common/EventLogConstants.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/common/GlobalConstants.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/common/GlobalConstants.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/common/KpiType.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/common/KpiType.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/etl/mr/ald/AnalyserLogDataMapper$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/etl/mr/ald/AnalyserLogDataMapper$1.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/etl/mr/ald/AnalyserLogDataMapper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/etl/mr/ald/AnalyserLogDataMapper.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/etl/mr/ald/AnalyserLogDataRunner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/etl/mr/ald/AnalyserLogDataRunner.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/etl/util/IPSeekerExt$RegionInfo.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/etl/util/IPSeekerExt$RegionInfo.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/etl/util/IPSeekerExt.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/etl/util/IPSeekerExt.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/etl/util/LoggerUtil.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/etl/util/LoggerUtil.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/etl/util/UserAgentUtil$UserAgentInfo.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/etl/util/UserAgentUtil$UserAgentInfo.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/etl/util/UserAgentUtil.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/etl/util/UserAgentUtil.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/etl/util/ip/IPSeeker$IPEntry.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/etl/util/ip/IPSeeker$IPEntry.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/etl/util/ip/IPSeeker$IPLocation.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/etl/util/ip/IPSeeker$IPLocation.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/etl/util/ip/IPSeeker$IPSeekerUtils.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/etl/util/ip/IPSeeker$IPSeekerUtils.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/etl/util/ip/IPSeeker.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/etl/util/ip/IPSeeker.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/model/dim/StatsCommonDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/model/dim/StatsCommonDimension.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/model/dim/StatsDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/model/dim/StatsDimension.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/model/dim/StatsUserDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/model/dim/StatsUserDimension.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/model/dim/base/BaseDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/model/dim/base/BaseDimension.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/model/dim/base/BrowserDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/model/dim/base/BrowserDimension.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/model/dim/base/DateDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/model/dim/base/DateDimension.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/model/dim/base/KpiDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/model/dim/base/KpiDimension.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/model/dim/base/PlatformDimension.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/model/dim/base/PlatformDimension.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/model/value/BaseStatsValueWritable.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/model/value/BaseStatsValueWritable.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/model/value/map/TimeOutputValue.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/model/value/map/TimeOutputValue.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/model/value/reduce/MapWritableValue.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/model/value/reduce/MapWritableValue.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/IOutputCollector.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/IOutputCollector.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/TransformerOutputFormat$TransformerRecordWriter.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/TransformerOutputFormat$TransformerRecordWriter.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/TransformerOutputFormat.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/TransformerOutputFormat.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/au/ActiveUserBrowserCollector.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/au/ActiveUserBrowserCollector.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/au/ActiveUserCollector.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/au/ActiveUserCollector.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/au/ActiveUserMapper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/au/ActiveUserMapper.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/au/ActiveUserReducer.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/au/ActiveUserReducer.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/au/ActiveUserRunner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/au/ActiveUserRunner.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/nu/NewInstallUserMapper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/nu/NewInstallUserMapper.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/nu/NewInstallUserReducer.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/nu/NewInstallUserReducer.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/nu/NewInstallUserRunner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/nu/NewInstallUserRunner.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/nu/StatsDeviceBrowserNewInstallUserCollector.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/nu/StatsDeviceBrowserNewInstallUserCollector.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/nu/StatsUserNewInstallUserCollector.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/mr/nu/StatsUserNewInstallUserCollector.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/service/IDimensionConverter.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/service/IDimensionConverter.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/service/impl/DimensionConverterImpl$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/service/impl/DimensionConverterImpl$1.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/transformer/service/impl/DimensionConverterImpl.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/transformer/service/impl/DimensionConverterImpl.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/util/EJob$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/util/EJob$1.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/util/EJob.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/util/EJob.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/util/JdbcManager.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/util/JdbcManager.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/com/mashibing/util/TimeUtil.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/com/mashibing/util/TimeUtil.class -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | fs.defaultFS 22 | hdfs://mycluster 23 | 24 | 25 | ha.zookeeper.quorum 26 | node02:2181,node03:2181,node04:2181 27 | 28 | 29 | hadoop.proxyuser.root.groups 30 | * 31 | 32 | 33 | hadoop.proxyuser.root.hosts 34 | * 35 | 36 | 37 | -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=info,console 2 | 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.layout=org.apache.log4j.SimpleLayout -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | mapreduce.framework.name 23 | yarn 24 | 25 | 26 | -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/output-collector.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | collector_new_install_user 6 | com.mashibing.transformer.mr.nu.StatsUserNewInstallUserCollector 7 | 8 | 9 | 10 | collector_browser_new_install_user 11 | com.mashibing.transformer.mr.nu.StatsDeviceBrowserNewInstallUserCollector 12 | 13 | 14 | collector_active_user 15 | com.mashibing.transformer.mr.au.ActiveUserCollector 16 | 17 | 18 | 19 | collector_browser_active_user 20 | com.mashibing.transformer.mr.au.ActiveUserBrowserCollector 21 | 22 | 23 | -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/qqwry.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/code/log_analyse_mr/target/classes/qqwry.dat -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/query-mapping.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | new_install_user 6 | 7 | INSERT INTO `stats_user`( 8 | `platform_dimension_id`, 9 | `date_dimension_id`, 10 | `new_install_users`, 11 | `created`) 12 | VALUES(?, ?, ?, ?) ON DUPLICATE KEY UPDATE `new_install_users` = ? 13 | 14 | 15 | 16 | 17 | browser_new_install_user 18 | 19 | INSERT INTO `stats_device_browser`( 20 | `platform_dimension_id`, 21 | `date_dimension_id`, 22 | `browser_dimension_id`, 23 | `new_install_users`, 24 | `created`) 25 | VALUES(?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE `new_install_users` = ? 26 | 27 | 28 | 29 | 30 | active_user 31 | 32 | INSERT INTO `stats_user`( 33 | `platform_dimension_id`, 34 | `date_dimension_id`, 35 | `active_users`, 36 | `created`) 37 | VALUES(?, ?, ?, ?) ON DUPLICATE KEY UPDATE `active_users` = ? 38 | 39 | 40 | 41 | 42 | browser_active_user 43 | 44 | INSERT INTO `stats_device_browser`( 45 | `platform_dimension_id`, 46 | `date_dimension_id`, 47 | `browser_dimension_id`, 48 | `active_users`, 49 | `created` 50 | ) VALUES(?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE `active_users` = ? 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/transformer-env.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | mysql.batch.number 6 | 10 7 | 8 | 9 | 10 | mysql.report.driver 11 | com.mysql.jdbc.Driver 12 | 13 | 14 | 15 | mysql.report.url 16 | jdbc:mysql://node01:3306/result_db?useUnicode=true&characterEncoding=utf8 17 | 18 | 19 | 20 | mysql.report.username 21 | root 22 | 23 | 24 | 25 | mysql.report.password 26 | 123 27 | 28 | 29 | -------------------------------------------------------------------------------- /code/log_analyse_mr/target/classes/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 15 | 16 | 17 | 18 | 19 | 20 | yarn.nodemanager.aux-services 21 | mapreduce_shuffle 22 | 23 | 24 | yarn.resourcemanager.ha.enabled 25 | true 26 | 27 | 28 | yarn.resourcemanager.zk-address 29 | node02:2181,node03:2181,node04:2181 30 | 31 | 32 | yarn.resourcemanager.cluster-id 33 | mashibing 34 | 35 | 36 | yarn.resourcemanager.ha.rm-ids 37 | rm1,rm2 38 | 39 | 40 | yarn.resourcemanager.hostname.rm1 41 | node03 42 | 43 | 44 | yarn.resourcemanager.hostname.rm2 45 | node04 46 | 47 | 48 | yarn.log-aggregation-enable 49 | true 50 | 51 | 52 | yarn.log-aggregation.retain-seconds 53 | 604800 54 | 55 | 56 | -------------------------------------------------------------------------------- /note/JS SDK需求文档&设计文档.doc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/note/JS SDK需求文档&设计文档.doc -------------------------------------------------------------------------------- /note/Java SDK需求文档&设计文档 .doc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/note/Java SDK需求文档&设计文档 .doc -------------------------------------------------------------------------------- /note/option-flume单节点案例: -------------------------------------------------------------------------------- 1 | # example.conf: A single-node Flume configuration 2 | 3 | # Name the components on this agent 4 | a1.sources = r1 5 | a1.sinks = k1 6 | a1.channels = c1 7 | 8 | # Describe/configure the source 9 | a1.sources.r1.type = netcat 10 | a1.sources.r1.bind = node02 11 | a1.sources.r1.port = 44444 12 | 13 | # Describe the sink 14 | a1.sinks.k1.type = logger 15 | 16 | # Use a channel which buffers events in memory 17 | a1.channels.c1.type = memory 18 | a1.channels.c1.capacity = 1000 19 | a1.channels.c1.transactionCapacity = 100 20 | 21 | # Bind the source and sink to the channel 22 | a1.sources.r1.channels = c1 23 | a1.sinks.k1.channel = c1 24 | 25 | flume-ng agent --conf-file option --name a1 -Dflume.root.logger=INFO,console -------------------------------------------------------------------------------- /note/option2-flume两台连接node02配置: -------------------------------------------------------------------------------- 1 | # node02 2 | 3 | # Name the components on this agent 4 | a1.sources = r1 5 | a1.sinks = k1 6 | a1.channels = c1 7 | 8 | # Describe/configure the source 9 | a1.sources.r1.type = netcat 10 | a1.sources.r1.bind = node02 11 | a1.sources.r1.port = 44444 12 | 13 | # Describe the sink 14 | a1.sinks.k1.type = avro 15 | a1.sinks.k1.hostname = node03 16 | a1.sinks.k1.port = 10086 17 | 18 | # Use a channel which buffers events in memory 19 | a1.channels.c1.type = memory 20 | a1.channels.c1.capacity = 1000 21 | a1.channels.c1.transactionCapacity = 100 22 | 23 | # Bind the source and sink to the channel 24 | a1.sources.r1.channels = c1 25 | a1.sinks.k1.channel = c1 26 | 27 | flume-ng agent --conf-file option2 --name a1 -Dflume.root.logger=INFO,console -------------------------------------------------------------------------------- /note/option3-flume两台连接node03配置: -------------------------------------------------------------------------------- 1 | # node03 2 | 3 | # Name the components on this agent 4 | a1.sources = r1 5 | a1.sinks = k1 6 | a1.channels = c1 7 | 8 | # Describe/configure the source 9 | a1.sources.r1.type = avro 10 | a1.sources.r1.bind = node03 11 | a1.sources.r1.port = 10086 12 | 13 | # Describe the sink 14 | a1.sinks.k1.type = logger 15 | 16 | # Use a channel which buffers events in memory 17 | a1.channels.c1.type = memory 18 | a1.channels.c1.capacity = 1000 19 | a1.channels.c1.transactionCapacity = 100 20 | 21 | # Bind the source and sink to the channel 22 | a1.sources.r1.channels = c1 23 | a1.sinks.k1.channel = c1 24 | 25 | flume-ng agent --conf-file option3 --name a1 -Dflume.root.logger=INFO,console -------------------------------------------------------------------------------- /note/option4: -------------------------------------------------------------------------------- 1 | # example.conf: A single-node Flume configuration 2 | 3 | # Name the components on this agent 4 | a1.sources = r1 5 | a1.sinks = k1 6 | a1.channels = c1 7 | 8 | # Describe/configure the source 9 | a1.sources.r1.type = exec 10 | a1.sources.r1.command = tail -F /root/log 11 | 12 | # Describe the sink 13 | a1.sinks.k1.type = logger 14 | 15 | # Use a channel which buffers events in memory 16 | a1.channels.c1.type = memory 17 | a1.channels.c1.capacity = 1000 18 | a1.channels.c1.transactionCapacity = 100 19 | 20 | # Bind the source and sink to the channel 21 | a1.sources.r1.channels = c1 22 | a1.sinks.k1.channel = c1 23 | 24 | -------------------------------------------------------------------------------- /note/option5: -------------------------------------------------------------------------------- 1 | # example.conf: A single-node Flume configuration 2 | 3 | # Name the components on this agent 4 | a1.sources = r1 5 | a1.sinks = k1 6 | a1.channels = c1 7 | 8 | # Describe/configure the source 9 | a1.sources.r1.type = spooldir 10 | a1.sources.r1.spoolDir = /root/data/ 11 | a1.sources.r1.fileSuffix=.msb 12 | a1.sources.r1.fileHeader = true 13 | 14 | # Describe the sink 15 | a1.sinks.k1.type = logger 16 | 17 | # Use a channel which buffers events in memory 18 | a1.channels.c1.type = memory 19 | a1.channels.c1.capacity = 1000 20 | a1.channels.c1.transactionCapacity = 100 21 | 22 | # Bind the source and sink to the channel 23 | a1.sources.r1.channels = c1 24 | a1.sinks.k1.channel = c1 25 | -------------------------------------------------------------------------------- /note/option6: -------------------------------------------------------------------------------- 1 | # example.conf: A single-node Flume configuration 2 | 3 | # Name the components on this agent 4 | a1.sources = r1 5 | a1.sinks = k1 6 | a1.channels = c1 7 | 8 | # Describe/configure the source 9 | a1.sources.r1.type = netcat 10 | a1.sources.r1.bind = node02 11 | a1.sources.r1.port = 44444 12 | 13 | # Describe the sink 14 | a1.sinks.k1.type = hdfs 15 | a1.sinks.k1.hdfs.path = /flume/events/%y-%m-%d/%H%M/%S 16 | a1.sinks.k1.hdfs.filePrefix = events- 17 | a1.sinks.k1.hdfs.round = true 18 | a1.sinks.k1.hdfs.roundValue = 10 19 | a1.sinks.k1.hdfs.roundUnit = second 20 | a1.sinks.k1.hdfs.useLocalTimeStamp = true 21 | 22 | # Use a channel which buffers events in memory 23 | a1.channels.c1.type = memory 24 | a1.channels.c1.capacity = 1000 25 | a1.channels.c1.transactionCapacity = 100 26 | 27 | # Bind the source and sink to the channel 28 | a1.sources.r1.channels = c1 29 | a1.sinks.k1.channel = c1 30 | -------------------------------------------------------------------------------- /note/project: -------------------------------------------------------------------------------- 1 | # example.conf: A single-node Flume configuration 2 | 3 | # Name the components on this agent 4 | a1.sources = r1 5 | a1.sinks = k1 6 | a1.channels = c1 7 | 8 | # Describe/configure the source 9 | a1.sources.r1.type = exec 10 | a1.sources.r1.command = tail -F /opt/data/access.log 11 | 12 | # Describe the sink 13 | a1.sinks.k1.type = hdfs 14 | a1.sinks.k1.hdfs.path = /project/%Y%m%d 15 | a1.sinks.k1.hdfs.filePrefix = log- 16 | a1.sinks.k1.hdfs.rollInterval = 0 17 | a1.sinks.k1.hdfs.rollSize = 10240 18 | a1.sinks.k1.hdfs.rollCount = 0 19 | a1.sinks.k1.hdfs.idleTimeout = 30 20 | a1.sinks.k1.hdfs.fileType = DataStream 21 | a1.sinks.k1.hdfs.callTimeout = 60000 22 | a1.sinks.k1.hdfs.useLocalTimeStamp = true 23 | 24 | # Use a channel which buffers events in memory 25 | a1.channels.c1.type = memory 26 | a1.channels.c1.capacity = 1000 27 | a1.channels.c1.transactionCapacity = 100 28 | 29 | # Bind the source and sink to the channel 30 | a1.sources.r1.channels = c1 31 | a1.sinks.k1.channel = c1 32 | -------------------------------------------------------------------------------- /note/sqoop/option: -------------------------------------------------------------------------------- 1 | import 2 | --connect 3 | jdbc:mysql://node01:3306/result_db 4 | --username 5 | root 6 | --password 7 | 123 8 | --as-textfile 9 | --columns 10 | id,browser_name,browser_version 11 | --delete-target-dir 12 | -m 13 | 1 14 | --table 15 | dimension_browser 16 | --target-dir 17 | /sqoop 18 | -------------------------------------------------------------------------------- /note/sqoop/option2: -------------------------------------------------------------------------------- 1 | import 2 | --connect 3 | jdbc:mysql://node01:3306/result_db 4 | --username 5 | root 6 | --password 7 | 123 8 | --as-textfile 9 | --delete-target-dir 10 | -m 11 | 1 12 | --target-dir 13 | /sqoop 14 | -e 15 | select * from dimension_browser where id > 20 and $CONDITIONS 16 | -------------------------------------------------------------------------------- /note/思路: -------------------------------------------------------------------------------- 1 | 输出格式化类的实现： 2 | outputformat： 3 | getRecordWriter() 4 | 得到一个RecordWriter的对象，此对象一般需要自己实现 5 | RecordWriter： 6 | close() 7 | write() 8 | 9 | 整体思路： 10 | 用户基本信息模块中包含新增用户： 11 | 浏览器分析模块中也包含新增用户： 12 | 思考：如何将两个MR合并成同一个MR？ 13 | MR的原语是： 14 | map:将数据变成K_V键值对的组合 15 | 项目中： 16 | key: 17 | 维度组合 18 | value： 19 | 存储值得自定义对象（uuid,time） 20 | reduce:将相同的key合并为一组 21 | 项目中：需要考虑去重 22 | key: 23 | 维度组合 24 | value： 25 | 保存的是具体的值以及向哪张表插入数据 26 | 27 | map: 28 | 1、首先从hbase中获取需要的字段信息（6） 29 | 2、根据取出的字段值创建单一的维度对象 30 | 3、根据不同的模块进行单一维度对象的组合 31 | 4、将维度对象组合和value进行输出 32 | reduce: 33 | 1、创建去重集合 34 | 2、将迭代器中的值进行添加去重操作 35 | 3、设置向哪张表中添加数据 36 | 4、直接向mysql中插入数据 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /note/数据流图.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/note/数据流图.png -------------------------------------------------------------------------------- /note/最终展示界面需求文档&设计文档.doc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/note/最终展示界面需求文档&设计文档.doc -------------------------------------------------------------------------------- /note/配置Nginx服务.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msbbigdata/hadoop_project/a3ae85c6c65f0a29d7d1b695219469c7d0daa2b1/note/配置Nginx服务.txt --------------------------------------------------------------------------------