├── kanalony-core ├── project │ ├── build.properties │ └── plugins.sbt ├── src │ ├── main │ │ ├── scala │ │ │ └── com │ │ │ │ └── kaltura │ │ │ │ └── core │ │ │ │ ├── urls │ │ │ │ ├── UrlParts.scala │ │ │ │ └── UrlParser.scala │ │ │ │ ├── ip2location │ │ │ │ ├── Location.scala │ │ │ │ └── LocationResolver.scala │ │ │ │ ├── logging │ │ │ │ └── MetaLog.scala │ │ │ │ ├── sessions │ │ │ │ └── IPartnerSecretStore.scala │ │ │ │ ├── userAgent │ │ │ │ ├── UserAgent.scala │ │ │ │ ├── enums │ │ │ │ │ └── Device.scala │ │ │ │ └── UserAgentResolver.scala │ │ │ │ ├── utils │ │ │ │ ├── ReadableDateUnits.scala │ │ │ │ ├── ReadableTimeUnits.scala │ │ │ │ ├── CollectionOps.scala │ │ │ │ ├── KalturaAPI.scala │ │ │ │ └── ConfigurationManager.scala │ │ │ │ ├── cassandra │ │ │ │ └── ClusterManager.scala │ │ │ │ └── streaming │ │ │ │ └── StreamManager.scala │ │ └── java │ │ │ └── com │ │ │ └── kaltura │ │ │ └── core │ │ │ └── ip2location │ │ │ ├── Coordinate.java │ │ │ ├── Utils.java │ │ │ ├── DbAttribtues.java │ │ │ ├── SerializableIP2LocationReader.java │ │ │ ├── city2Location │ │ │ └── CityCoordinatesCreator.java │ │ │ ├── Ip2LocationReader.java │ │ │ └── country2location │ │ │ └── CountryLocator.java │ └── test │ │ └── scala │ │ └── com │ │ └── kaltura │ │ ├── utils │ │ └── CollectionsHelperSpec.scala │ │ ├── ip2location │ │ └── IP2LocationSpec.scala │ │ └── core │ │ ├── urls │ │ └── UrlParserSpec.scala │ │ └── userAgent │ │ └── UserAgentSpec.scala └── build.sbt ├── kanalony-model ├── project │ ├── plugins.sbt │ └── build.properties ├── src │ ├── main │ │ └── scala │ │ │ └── com │ │ │ └── kaltura │ │ │ └── model │ │ │ ├── entities │ │ │ ├── Entry.scala │ │ │ ├── EntryMetadata.scala │ │ │ ├── Partner.scala │ │ │ └── PlayerEventTypes.scala │ │ │ ├── events │ │ │ ├── AccessLogRow.scala │ │ │ ├── RawPlayerEvent.scala │ │ │ ├── EnrichedPlayerEvent.scala │ │ │ └── PlayerEventParser.scala │ │ │ ├── dao │ │ │ ├── PartnerDAO.scala │ │ │ ├── EntryMetadataDAO.scala │ │ │ ├── DAOBase.scala │ │ │ └── EntryDAO.scala │ │ │ └── cache │ │ │ ├── CacheBase.scala │ │ │ ├── EntryMetadataCache.scala │ │ │ ├── EntryCache.scala │ │ │ └── PartnerCache.scala │ └── test │ │ └── scala-2.10 │ │ └── DAOTests.scala └── build.sbt ├── kanalony-aggregations ├── project │ ├── build.properties │ └── plugins.sbt ├── src │ └── main │ │ ├── resources │ │ └── com.kaltura.aggregations │ │ └── scala │ │ └── com │ │ └── kaltura │ │ └── aggregations │ │ ├── IAggregate.scala │ │ ├── IAggregateHourly.scala │ │ ├── IAggregateMinutely.scala │ │ ├── IAggregateTenSecs.scala │ │ ├── AggregationByPartner.scala │ │ ├── AggregationByDomain.scala │ │ ├── AggregationByCustomVar3.scala │ │ ├── AggregationByDevice.scala │ │ ├── AggregationByCustomVar1.scala │ │ ├── AggregationByCustomVar2.scala │ │ ├── AggregationByEntry.scala │ │ ├── AggregationByBrowser.scala │ │ ├── AggregationByCountry.scala │ │ ├── AggregationByCountryBrowser.scala │ │ ├── AggregationByApplication.scala │ │ ├── AggregationByOperatingSystem.scala │ │ ├── AggregationByCountryOperatingSystem.scala │ │ ├── AggregationByCustomVar1CustomVar2.scala │ │ └── AggregationByPlaybackContext.scala ├── tasks │ ├── create_jar.sh │ ├── deploy_driver_on_aws.sh │ └── upload_driver.sh └── build.sbt ├── kanalony-enrichment ├── project │ ├── build.properties │ └── plugins.sbt ├── tasks │ ├── create_jar.sh │ ├── deploy_driver_on_aws.sh │ └── upload_driver.sh ├── src │ ├── main │ │ ├── resources │ │ │ └── config.properties.template │ │ └── scala │ │ │ └── com │ │ │ └── kaltura │ │ │ └── enrichment │ │ │ ├── IEnrich.scala │ │ │ ├── EnrichByEntry.scala │ │ │ ├── EnrichByPartner.scala │ │ │ └── EventsFilter.scala │ └── test │ │ └── scala │ │ └── org │ │ └── apache │ │ └── spark │ │ └── SparkFunSuite.scala └── build.sbt ├── kanalony-storage-access ├── project │ ├── build.properties │ └── plugins.sbt ├── src │ └── main │ │ └── scala │ │ ├── IConnectorFactory.scala │ │ ├── kanalony │ │ └── storage │ │ │ ├── utils │ │ │ └── fs.scala │ │ │ └── generator │ │ │ ├── ColumnNames.scala │ │ │ ├── IColumnDefinition.scala │ │ │ ├── DbClientFactoryGenerator.scala │ │ │ ├── Driver.scala │ │ │ └── EntityClassGenerator.scala │ │ └── ConnectorFactory.scala └── build.sbt ├── kanalony-storage-logic ├── project │ ├── build.properties │ └── plugins.sbt ├── src │ ├── main │ │ └── scala │ │ │ └── kanalony │ │ │ └── storage │ │ │ └── logic │ │ │ ├── IQueryLocator.scala │ │ │ ├── queries │ │ │ ├── DailyMaxQuery.scala │ │ │ ├── DailyCountQuery.scala │ │ │ ├── EstimatedMinutesWatchedQuery.scala │ │ │ ├── AverageTimeViewedQuery.scala │ │ │ ├── PlayRatioQuery.scala │ │ │ └── AverageViewDropOffQuery.scala │ │ │ ├── ComputedDimensions.scala │ │ │ └── ComputedMetrics.scala │ └── test │ │ └── scala │ │ └── kanalony │ │ └── storage │ │ └── logic │ │ ├── EstimatedMinutesWatchedQueryTests.scala │ │ ├── PlayRatioQueryTests.scala │ │ ├── AverageTimeViewedQueryTests.scala │ │ ├── AverageViewDropOffQueryTests.scala │ │ ├── DailyCountQueryTests.scala │ │ └── DailyMaxQueryTests.scala └── build.sbt ├── kanalony-storage-logic-core ├── project │ ├── build.properties │ └── plugins.sbt ├── src │ └── main │ │ └── scala │ │ ├── kanalony │ │ └── storage │ │ │ └── logic │ │ │ ├── QueryNotSupportedException.scala │ │ │ ├── queries │ │ │ └── model │ │ │ │ ├── PagerDefinition.scala │ │ │ │ ├── OrderDefinition.scala │ │ │ │ ├── IDimensionConstraint.scala │ │ │ │ ├── QueryConstraint.scala │ │ │ │ ├── OrderDirection.scala │ │ │ │ ├── IQueryDimensionDefinition.scala │ │ │ │ ├── QueryDimensionDefinition.scala │ │ │ │ ├── IDimensionDefinition.scala │ │ │ │ ├── DimensionDefinition.scala │ │ │ │ ├── DimensionConstraintDeclaration.scala │ │ │ │ ├── DimensionUnconstrained.scala │ │ │ │ ├── DimensionEqualityConstraint.scala │ │ │ │ └── DimensionRangeConstraint.scala │ │ │ ├── QueryResult.scala │ │ │ ├── IQueryResult.scala │ │ │ ├── IYearlyPartitionedQueryParams.scala │ │ │ ├── IQuery.scala │ │ │ ├── ComputedQueryFactory.scala │ │ │ ├── IDailyPartitionedQueryParams.scala │ │ │ ├── QueryParams.scala │ │ │ ├── Driver.scala │ │ │ ├── IMonthlyPartitionedQueryParams.scala │ │ │ └── Dimensions.scala │ │ ├── queryGenerator │ │ ├── IRowBreaker.scala │ │ ├── DayPartitioner.scala │ │ ├── YearPartitioner.scala │ │ ├── MonthPartitioner.scala │ │ ├── RowBreakerFactory.scala │ │ ├── QueryListGenerator.scala │ │ ├── ColumnExtendedDefinition.scala │ │ └── ParamsTypeGenerator.scala │ │ └── IUserAccessQuery.scala └── build.sbt ├── kanalony-storage-access-generated ├── project │ ├── build.properties │ └── plugins.sbt ├── src │ └── main │ │ └── scala │ │ └── kanalony │ │ └── storage │ │ └── generated │ │ └── EntryTableAccessor.scala └── build.sbt ├── kanalony-storage-logic-generated ├── project │ ├── build.properties │ └── plugins.sbt ├── build.sbt └── src │ └── main │ └── scala │ └── kanalony │ └── storage │ └── logic │ └── generated │ ├── HourlyAggQuery.scala │ ├── TensecsAggQuery.scala │ └── MinutelyAggQuery.scala ├── kanalony-web-server ├── .gitignore ├── app │ ├── model │ │ ├── Pager.scala │ │ ├── ErrorResponse.scala │ │ ├── EqualityFilter.scala │ │ ├── AnalyticsResponse.scala │ │ ├── AnalyticsRequest.scala │ │ └── Implicits.scala │ ├── controllers │ │ ├── FormattedResponse.scala │ │ ├── InvalidMetricsException.scala │ │ ├── InvalidOrderByException.scala │ │ ├── MetricNotSuppliedException.scala │ │ ├── InvalidDimensionException.scala │ │ ├── IResponseFormatter.scala │ │ ├── ResponseFormatterFactory.scala │ │ ├── CsvResponseFormatter.scala │ │ └── JsonResponseFormatter.scala │ └── Global.scala ├── conf │ ├── routes │ ├── logback.xml │ └── application.conf ├── project │ └── plugins.sbt ├── build.sbt └── test │ └── load │ └── DataApiLoadTest.scala ├── schema ├── kafka │ └── 20160303_0953 │ │ ├── 1_create_player_events_topic.sh │ │ └── 2_create_enriched_player_events_topic.sh └── cassandra │ ├── 20160329_1400 │ └── kanalony_agg_keyspace.cql │ └── 20160303_0953 │ └── 1_create_enrichment_cache_keyspace.cql ├── kanalony-logstash ├── src │ └── test │ │ └── scala │ │ ├── utils │ │ ├── KafkaDriver.scala │ │ └── KafkaConsumer.scala │ │ └── load │ │ └── EventParsingLoadTest.scala ├── build.sbt └── config │ ├── logstash_nginx_log_input.conf.local │ └── logstash_nginx_log_input.conf.staging ├── kanalony-receiver ├── kanalony.js ├── package.json ├── lib │ ├── configurationUtil.js │ ├── kafkaProducer.js │ ├── timeUֹtil.js │ ├── eventsRelay.js │ └── logger.js └── server.js ├── .gitignore ├── config └── config.properties.template └── README.md /kanalony-core/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 0.13.8 -------------------------------------------------------------------------------- /kanalony-core/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | logLevel := Level.Warn -------------------------------------------------------------------------------- /kanalony-model/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | logLevel := Level.Warn -------------------------------------------------------------------------------- /kanalony-model/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 0.13.8 -------------------------------------------------------------------------------- /kanalony-aggregations/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 0.13.8 -------------------------------------------------------------------------------- /kanalony-enrichment/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 0.13.8 -------------------------------------------------------------------------------- /kanalony-storage-access/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 0.13.8 -------------------------------------------------------------------------------- /kanalony-storage-access/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | logLevel := Level.Warn -------------------------------------------------------------------------------- /kanalony-storage-logic/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 0.13.8 -------------------------------------------------------------------------------- /kanalony-storage-logic-core/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 0.13.8 -------------------------------------------------------------------------------- /kanalony-storage-access-generated/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 0.13.8 -------------------------------------------------------------------------------- /kanalony-storage-access-generated/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | logLevel := Level.Warn -------------------------------------------------------------------------------- /kanalony-storage-logic-generated/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 0.13.8 -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/resources/com.kaltura.aggregations: -------------------------------------------------------------------------------- 1 | com.kaltura.aggregations.EntryAggregation 2 | -------------------------------------------------------------------------------- /kanalony-aggregations/tasks/create_jar.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | sbt 'set test in assembly := {}' clean assembly -------------------------------------------------------------------------------- /kanalony-enrichment/tasks/create_jar.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | sbt 'set test in assembly := {}' clean assembly -------------------------------------------------------------------------------- /kanalony-enrichment/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | logLevel := Level.Warn 2 | 3 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.12.0") -------------------------------------------------------------------------------- /kanalony-storage-logic-core/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | logLevel := Level.Warn 2 | 3 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.3.5") -------------------------------------------------------------------------------- /kanalony-storage-logic-generated/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | logLevel := Level.Warn 2 | 3 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.3.5") -------------------------------------------------------------------------------- /kanalony-storage-logic/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | logLevel := Level.Warn 2 | 3 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.3.5") 4 | 5 | -------------------------------------------------------------------------------- /kanalony-web-server/.gitignore: -------------------------------------------------------------------------------- 1 | logs 2 | target 3 | /.idea 4 | /.idea_modules 5 | /.classpath 6 | /.project 7 | /.settings 8 | /RUNNING_PID 9 | -------------------------------------------------------------------------------- /kanalony-web-server/app/model/Pager.scala: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | /** 4 | * Created by orlylampert on 11/21/16. 5 | */ 6 | case class Pager(size: Int, index: Int) 7 | 8 | -------------------------------------------------------------------------------- /kanalony-core/src/main/scala/com/kaltura/core/urls/UrlParts.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.urls 2 | 3 | case class UrlParts(domain:String, canonicalUrl:String, originalUrl:String) 4 | -------------------------------------------------------------------------------- /schema/kafka/20160303_0953/1_create_player_events_topic.sh: -------------------------------------------------------------------------------- 1 | /opt/kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 2 --partitions 10 --topic player-events 2 | -------------------------------------------------------------------------------- /kanalony-web-server/app/model/ErrorResponse.scala: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | /** 4 | * Created by elad.benedict on 5/2/2016. 5 | */ 6 | 7 | case class ErrorResponse(kind : String, data : String) 8 | -------------------------------------------------------------------------------- /kanalony-web-server/app/model/EqualityFilter.scala: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | /** 4 | * Created by elad.benedict on 2/21/2016. 5 | */ 6 | case class EqualityFilter(dimension : String, values : List[String]) 7 | -------------------------------------------------------------------------------- /schema/cassandra/20160329_1400/kanalony_agg_keyspace.cql: -------------------------------------------------------------------------------- 1 | CREATE KEYSPACE IF NOT EXISTS kanalony_agg WITH replication = {'class': 'NetworkTopologyStrategy', 'us-west-2':'2'} AND durable_writes = true; 2 | -------------------------------------------------------------------------------- /schema/kafka/20160303_0953/2_create_enriched_player_events_topic.sh: -------------------------------------------------------------------------------- 1 | /opt/kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 2 --partitions 10 --topic enriched-player-events 2 | -------------------------------------------------------------------------------- /kanalony-web-server/app/controllers/FormattedResponse.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | /** 4 | * Created by elad.benedict on 2/24/2016. 5 | */ 6 | 7 | case class FormattedResponse(data : String, mime : String) 8 | -------------------------------------------------------------------------------- /kanalony-web-server/app/model/AnalyticsResponse.scala: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | /** 4 | * Created by elad.benedict on 2/21/2016. 5 | */ 6 | case class AnalyticsResponse(headers: List[String], data : List[List[String]]) 7 | -------------------------------------------------------------------------------- /kanalony-web-server/app/controllers/InvalidMetricsException.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | /** 4 | * Created by elad.benedict on 2/22/2016. 5 | */ 6 | 7 | class InvalidMetricsException(s : String) extends Exception(s) {} 8 | -------------------------------------------------------------------------------- /kanalony-model/src/main/scala/com/kaltura/model/entities/Entry.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.model.entities 2 | 3 | case class Entry( 4 | id:String, 5 | categories:Option[String] = None 6 | ) 7 | -------------------------------------------------------------------------------- /kanalony-web-server/app/controllers/InvalidOrderByException.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | /** 4 | * Created by orlylampert on 11/21/16. 5 | */ 6 | class InvalidOrderByException extends Exception("Order By filed is not part of the result") 7 | 8 | -------------------------------------------------------------------------------- /kanalony-web-server/app/controllers/MetricNotSuppliedException.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | /** 4 | * Created by elad.benedict on 2/23/2016. 5 | */ 6 | 7 | class MetricNotSuppliedException extends Exception("Mandatory field Metric not supplied") 8 | -------------------------------------------------------------------------------- /kanalony-enrichment/src/main/resources/config.properties.template: -------------------------------------------------------------------------------- 1 | kanalony.events_enrichment.application_name=EventsEnhancer 2 | kanalony.events_enrichment.cassandra_host=127.0.0.1 3 | kanalony.core.ip2location.data_filepath=/opt/kaltura/data/geoip/IP-COUNTRY-CITY-ISP.BIN -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/QueryNotSupportedException.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | /** 4 | * Created by elad.benedict on 2/23/2016. 5 | */ 6 | 7 | class QueryNotSupportedException(s : String) extends Exception(s) {} -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/queries/model/PagerDefinition.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries.model 2 | 3 | /** 4 | * Created by orlylampert on 11/21/16. 5 | */ 6 | case class PagerDefinition(size:Int, index:Int = -1) 7 | -------------------------------------------------------------------------------- /kanalony-logstash/src/test/scala/utils/KafkaDriver.scala: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | /** 4 | * Created by elad.benedict on 3/31/2016. 5 | */ 6 | 7 | object KafkaDriver { 8 | def main(args: Array[String]) { 9 | new KafkaConsumer().read(println) 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /kanalony-enrichment/tasks/deploy_driver_on_aws.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | scp /Users/ofirk/projects/kaltura/kanalony/kanalony-enrichment/target/scala-2.11/kanalony-enrichment.jar pa-front-stg1:~/kanalony/.; 3 | ssh pa-front-stg1 '~/kanalony/scripts/deploy-full-enrichment-prod.sh' 4 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/QueryResult.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | /** 4 | * Created by elad.benedict on 2/16/2016. 5 | */ 6 | case class QueryResult(headers : List[String], rows : List[List[String]]) extends IQueryResult 7 | -------------------------------------------------------------------------------- /kanalony-aggregations/tasks/deploy_driver_on_aws.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | scp /Users/ofirk/projects/kaltura/kanalony/kanalony-aggregations/target/scala-2.11/kanalony-aggregations.jar pa-front-stg1:~/kanalony/.; 3 | ssh pa-front-stg1 '~/kanalony/scripts/deploy-full-aggregations-aggr.sh' 4 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/queryGenerator/IRowBreaker.scala: -------------------------------------------------------------------------------- 1 | package queryGenerator 2 | 3 | /** 4 | * Created by elad.benedict on 2/29/2016. 5 | */ 6 | 7 | trait IRowBreaker { 8 | val partitionKeyParamName : String 9 | val implementingTrait : String 10 | } 11 | -------------------------------------------------------------------------------- /kanalony-aggregations/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | logLevel := Level.Warn 2 | 3 | resolvers += Resolver.jcenterRepo 4 | addSbtPlugin("me.lessis" % "bintray-sbt" % "0.3.0") 5 | addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.8.1") 6 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.0") -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/IQueryResult.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | /** 4 | * Created by elad.benedict on 2/16/2016. 5 | */ 6 | trait IQueryResult { 7 | val headers : List[String] 8 | val rows : List[List[String]] 9 | } 10 | -------------------------------------------------------------------------------- /kanalony-web-server/app/controllers/InvalidDimensionException.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | /** 4 | * Created by elad.benedict on 2/22/2016. 5 | */ 6 | 7 | class InvalidDimensionException(val dimensionName : String) extends Exception("Dimension " + dimensionName + " not supported") {} 8 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/queries/model/OrderDefinition.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries.model 2 | 3 | 4 | /** 5 | * Created by orlylampert on 11/20/16. 6 | */ 7 | case class OrderDefinition (header: String, order: OrderDirection.Value) 8 | 9 | -------------------------------------------------------------------------------- /kanalony-receiver/kanalony.js: -------------------------------------------------------------------------------- 1 | exports.EventsRelay = require('./lib/eventsRelay'); 2 | exports.ConfigurationUtil = require('./lib/configurationUtil'); 3 | exports.TimeUtil = require('./lib/timeUֹtil'); 4 | exports.KafkaProducer = require('./lib/kafkaProducer'); 5 | exports.Logger = require('./lib/logger'); -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/queries/model/IDimensionConstraint.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries.model 2 | 3 | /** 4 | * Created by elad.benedict on 2/16/2016. 5 | */ 6 | trait IDimensionConstraint { 7 | val constraint : QueryConstraint.Value 8 | } 9 | -------------------------------------------------------------------------------- /kanalony-model/src/main/scala/com/kaltura/model/entities/EntryMetadata.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.model.entities 2 | 3 | /** 4 | * Created by orlylampert on 12/27/16. 5 | */ 6 | case class EntryMetadata( 7 | id:String, 8 | name:String) 9 | 10 | 11 | -------------------------------------------------------------------------------- /kanalony-web-server/app/controllers/IResponseFormatter.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import model.AnalyticsResponse 4 | 5 | /** 6 | * Created by elad.benedict on 2/24/2016. 7 | */ 8 | trait IResponseFormatter { 9 | def format(analyticsResponse: AnalyticsResponse) : FormattedResponse 10 | } 11 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/queries/model/QueryConstraint.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries.model 2 | 3 | /** 4 | * Created by elad.benedict on 2/16/2016. 5 | */ 6 | object QueryConstraint extends Enumeration { 7 | val Equality, Range, Unconstrained = Value 8 | } 9 | -------------------------------------------------------------------------------- /kanalony-web-server/app/model/AnalyticsRequest.scala: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | /** 4 | * Created by elad.benedict on 2/21/2016. 5 | */ 6 | case class AnalyticsRequest(from: String, to : String, dimensions: List[String], filters : List[EqualityFilter], metrics : List[String], utcOffset : Int = 0, orderBy : String, pager: Pager) 7 | -------------------------------------------------------------------------------- /kanalony-core/src/main/scala/com/kaltura/core/ip2location/Location.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.ip2location 2 | 3 | case class Location(country: String = "N/A", region: String = "N/A", city: String = "N/A") { 4 | 5 | override def toString(): String = { 6 | s"Country: $country, Region: $region, City: $city" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /kanalony-storage-access/src/main/scala/IConnectorFactory.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage 2 | 3 | import com.websudos.phantom.connectors.KeySpaceDef 4 | 5 | /** 6 | * Created by elad.benedict on 2/7/2016. 7 | */ 8 | trait IConnectorFactory { 9 | val connector : KeySpaceDef 10 | val dimConnector: KeySpaceDef 11 | } 12 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/queries/model/OrderDirection.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries.model 2 | 3 | /** 4 | * Created by orlylampert on 11/20/16. 5 | */ 6 | object OrderDirection extends Enumeration { 7 | val ASC = Value(1, "+") 8 | val DESC = Value(2, "-") 9 | } 10 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/queries/model/IQueryDimensionDefinition.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries.model 2 | 3 | /** 4 | * Created by elad.benedict on 3/14/2016. 5 | */ 6 | 7 | trait IQueryDimensionDefinition extends IDimensionDefinition { 8 | val includeInResult : Boolean 9 | } 10 | -------------------------------------------------------------------------------- /kanalony-core/src/main/scala/com/kaltura/core/logging/MetaLog.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.logging 2 | 3 | import org.slf4j.LoggerFactory 4 | 5 | trait MetaLog[BaseLog] { 6 | val logger = LoggerFactory.getLogger(getClass) 7 | } 8 | 9 | trait BaseLog { 10 | def metaLog: MetaLog[BaseLog] 11 | def logger = metaLog.logger 12 | } 13 | 14 | -------------------------------------------------------------------------------- /kanalony-model/src/main/scala/com/kaltura/model/entities/Partner.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.model.entities 2 | 3 | case class Partner ( 4 | id: Int, 5 | secret: Option[String]=None, 6 | crmId: Option[String]=None, 7 | packageId: Option[Int]=None 8 | ) -------------------------------------------------------------------------------- /kanalony-core/src/main/scala/com/kaltura/core/sessions/IPartnerSecretStore.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.sessions 2 | 3 | trait IPartnerSecretStore { 4 | /** 5 | * returns the partner secret corresponding to the partner Id provided 6 | * @param partnerId 7 | * @return 8 | */ 9 | def getPartnerSecret(partnerId: Int): String 10 | } 11 | -------------------------------------------------------------------------------- /kanalony-storage-access/src/main/scala/kanalony/storage/utils/fs.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.utils 2 | 3 | /** 4 | * Created by elad.benedict on 2/8/2016. 5 | */ 6 | object fs { 7 | def printToFile(f: java.io.File)(op: java.io.PrintWriter => Unit) { 8 | val p = new java.io.PrintWriter(f) 9 | try { op(p) } finally { p.close() } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/queries/model/QueryDimensionDefinition.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries.model 2 | 3 | import kanalony.storage.logic.Dimensions 4 | 5 | case class QueryDimensionDefinition(dimension : Dimensions.Value, constraint : IDimensionConstraint, includeInResult : Boolean) extends IQueryDimensionDefinition 6 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/queryGenerator/DayPartitioner.scala: -------------------------------------------------------------------------------- 1 | package queryGenerator 2 | 3 | /** 4 | * Created by elad.benedict on 3/22/2016. 5 | */ 6 | 7 | case class DayPartitioner() extends IRowBreaker { 8 | override val partitionKeyParamName: String = "days" 9 | override val implementingTrait: String = "IDailyPartitionedQueryParams" 10 | } 11 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/queryGenerator/YearPartitioner.scala: -------------------------------------------------------------------------------- 1 | package queryGenerator 2 | 3 | /** 4 | * Created by elad.benedict on 3/22/2016. 5 | */ 6 | 7 | case class YearPartitioner() extends IRowBreaker { 8 | override val partitionKeyParamName: String = "years" 9 | override val implementingTrait: String = "IYearlyPartitionedQueryParams" 10 | } 11 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/queryGenerator/MonthPartitioner.scala: -------------------------------------------------------------------------------- 1 | package queryGenerator 2 | 3 | /** 4 | * Created by elad.benedict on 3/22/2016. 5 | */ 6 | 7 | case class MonthPartitioner() extends IRowBreaker { 8 | override val partitionKeyParamName: String = "months" 9 | override val implementingTrait: String = "IMonthlyPartitionedQueryParams" 10 | } 11 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/queries/model/IDimensionDefinition.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries.model 2 | 3 | import kanalony.storage.logic.Dimensions 4 | 5 | /** 6 | * Created by elad.benedict on 3/14/2016. 7 | */ 8 | 9 | trait IDimensionDefinition { 10 | val dimension : Dimensions.Value 11 | val constraint : IDimensionConstraint 12 | } 13 | -------------------------------------------------------------------------------- /kanalony-web-server/conf/routes: -------------------------------------------------------------------------------- 1 | # Routes 2 | # This file defines all application routes (Higher priority routes first) 3 | # ~~~~ 4 | 5 | # Home page 6 | 7 | POST /query controllers.Application.query 8 | 9 | # Map static resources from the /public folder to the /assets URL path 10 | GET /assets/*file controllers.Assets.versioned(path="/public", file: Asset) 11 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/queries/model/DimensionDefinition.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries.model 2 | 3 | import kanalony.storage.logic.Dimensions 4 | 5 | /** 6 | * Created by elad.benedict on 2/16/2016. 7 | */ 8 | 9 | case class DimensionDefinition(dimension : Dimensions.Value, constraint : IDimensionConstraint) extends IDimensionDefinition 10 | -------------------------------------------------------------------------------- /kanalony-core/src/main/scala/com/kaltura/core/userAgent/UserAgent.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.userAgent 2 | 3 | import com.kaltura.core.userAgent.enums.{OperatingSystem,Browser,Device} 4 | 5 | /** 6 | * Created by ofirk on 20/10/2015. 7 | */ 8 | case class UserAgent (browser:Browser.Value, 9 | operatingSystem: OperatingSystem.Value, 10 | device: Device.Value) 11 | -------------------------------------------------------------------------------- /kanalony-storage-logic/src/main/scala/kanalony/storage/logic/IQueryLocator.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | import com.kaltura.model.entities.Metric 4 | 5 | /** 6 | * Created by elad.benedict on 2/16/2016. 7 | */ 8 | 9 | trait IQueryLocator { 10 | def locate(queryParams: QueryParams, computedDimensions: IComputedDimensions, computedMetrics: IComputedMetrics) : List[(IQuery, List[Metric])] 11 | } 12 | -------------------------------------------------------------------------------- /schema/cassandra/20160303_0953/1_create_enrichment_cache_keyspace.cql: -------------------------------------------------------------------------------- 1 | CREATE KEYSPACE enrichment_cache WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '2'} AND durable_writes = true; 2 | CREATE TABLE enrichment_cache.dim_partners ( 3 | id int PRIMARY KEY, 4 | secret text, 5 | crm_id text 6 | ); 7 | CREATE TABLE enrichment_cache.dim_entries ( 8 | id text PRIMARY KEY, 9 | categories text 10 | ); 11 | -------------------------------------------------------------------------------- /kanalony-core/src/main/scala/com/kaltura/core/utils/ReadableDateUnits.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.utils 2 | 3 | import org.joda.time.DateTime 4 | 5 | /** 6 | * Created by orlylampert on 3/16/16. 7 | */ 8 | object ReadableDateUnits { 9 | implicit class ReadableDateUnits(date: DateTime) { 10 | def getYearMonthDay = getYearMonth * 100 + date.getDayOfMonth 11 | 12 | def getYearMonth = date.getYear * 100 + date.getMonthOfYear 13 | 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /kanalony-core/src/main/scala/com/kaltura/core/utils/ReadableTimeUnits.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.utils 2 | 3 | /** 4 | * Created by ofirk on 16/02/2016. 5 | */ 6 | object ReadableTimeUnits { 7 | implicit class ReadableTimeUnits(unit: Int) { 8 | def days = unit * 60 * 60 * 24 9 | def day = days 10 | 11 | def hours = unit * 60 * 60 12 | def hour = hours 13 | 14 | def minutes = unit * 60 15 | def minute = minutes 16 | } 17 | } 18 | 19 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/IYearlyPartitionedQueryParams.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | import org.joda.time.{DateTimeZone, DateTime} 4 | 5 | /** 6 | * Created by elad.benedict on 2/21/2016. 7 | */ 8 | trait IYearlyPartitionedQueryParams { 9 | val startTime : DateTime 10 | val endTime : DateTime 11 | val years = (startTime.withZone(DateTimeZone.UTC).getYear to endTime.withZone(DateTimeZone.UTC).getYear).toList 12 | } 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Node 6 | pids 7 | *.pid 8 | *.seed 9 | lib-cov 10 | coverage 11 | .grunt 12 | .lock-wscript 13 | build/Release 14 | node_modules 15 | 16 | # Scala 17 | *.class 18 | 19 | # sbt specific 20 | .cache 21 | .history 22 | .lib/ 23 | dist/* 24 | target/ 25 | lib_managed/ 26 | src_managed/ 27 | project/boot/ 28 | project/plugins/project/ 29 | out/* 30 | 31 | # Intellij 32 | .idea/ 33 | *.iml 34 | *.iws 35 | 36 | # Mac 37 | .DS_Store 38 | 39 | -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/IAggregate.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import com.kaltura.model.events.EnrichedPlayerEvent 4 | import org.apache.spark.streaming.dstream.DStream 5 | import org.joda.time.DateTime 6 | 7 | trait IAggregate { 8 | def aggregate(enrichedEvents: DStream[EnrichedPlayerEvent]) : Unit 9 | def ttl: Int 10 | def getAggrTime(eventTime: DateTime): DateTime 11 | def getAggrTimeUnit: String 12 | 13 | } 14 | 15 | 16 | -------------------------------------------------------------------------------- /kanalony-storage-access/src/main/scala/kanalony/storage/generator/ColumnNames.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.generator 2 | 3 | /** 4 | * Created by elad.benedict on 3/2/2016. 5 | */ 6 | object ColumnNames extends Enumeration { 7 | val partner_id, entry_id, metric, year, month, day, country, city, operating_system, 8 | browser, device, domain, referrer, application, custom_var1, custom_var2, custom_var3, 9 | playback_context, value, hour, minute, tensecs, category = Value 10 | 11 | } 12 | -------------------------------------------------------------------------------- /kanalony-web-server/app/controllers/ResponseFormatterFactory.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import play.api.mvc.{AnyContent, Request} 4 | 5 | /** 6 | * Created by elad.benedict on 2/24/2016. 7 | */ 8 | 9 | object ResponseFormatterFactory { 10 | def get(request: Request[AnyContent]) : IResponseFormatter = { 11 | if (request.accepts("text/csv")) 12 | { 13 | new CsvResponseFormatter() 14 | } 15 | else { 16 | new JsonResponseFormatter() 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /kanalony-storage-logic/src/main/scala/kanalony/storage/logic/queries/DailyMaxQuery.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries 2 | 3 | import kanalony.storage.logic.{IQueryLocator, QueryParams} 4 | 5 | /** 6 | * Created by elad.benedict on 3/28/2016. 7 | */ 8 | class DailyMaxQuery(queryParams: QueryParams, queryLocator: IQueryLocator) extends DailyQueryBase(queryParams, queryLocator) { 9 | override def computeGroupAggregatedValue: (List[List[String]]) => Double = _.map(countFieldExtractor(_)).max 10 | } 11 | -------------------------------------------------------------------------------- /kanalony-web-server/app/controllers/CsvResponseFormatter.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import model.AnalyticsResponse 4 | 5 | /** 6 | * Created by elad.benedict on 2/24/2016. 7 | */ 8 | 9 | class CsvResponseFormatter() extends IResponseFormatter { 10 | override def format(analyticsResponse: AnalyticsResponse): FormattedResponse = { 11 | val resData = (analyticsResponse.headers :: analyticsResponse.data).map(_.mkString(",")).mkString("\n") 12 | FormattedResponse(resData, "text/csv") 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /kanalony-model/src/main/scala/com/kaltura/model/events/AccessLogRow.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.model.events 2 | 3 | import org.joda.time.DateTime 4 | 5 | case class AccessLogRow( 6 | host:String, 7 | request:String, 8 | eventTime:DateTime, 9 | remoteAddr:String, 10 | proxyRemoteAddr:String, 11 | userAgent:String, 12 | params:Map[String,String] 13 | ) -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/IAggregateHourly.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import com.kaltura.core.utils.ReadableTimeUnits.ReadableTimeUnits 4 | import org.joda.time.DateTime 5 | 6 | /** 7 | * Created by orlylampert on 2/21/16. 8 | */ 9 | trait IAggregateHourly extends IAggregate { 10 | override def ttl = 65 minutes 11 | override def getAggrTime(eventTime: DateTime): DateTime = eventTime.hourOfDay().roundFloorCopy() 12 | override def getAggrTimeUnit: String = "hour" 13 | 14 | } 15 | -------------------------------------------------------------------------------- /kanalony-web-server/app/controllers/JsonResponseFormatter.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import model.AnalyticsResponse 4 | import argonaut._ 5 | import Argonaut._ 6 | import model.Implicits._ 7 | 8 | /** 9 | * Created by elad.benedict on 2/24/2016. 10 | */ 11 | class JsonResponseFormatter() extends IResponseFormatter { 12 | override def format(analyticsResponse: AnalyticsResponse): FormattedResponse = { 13 | val resData = analyticsResponse.asJson 14 | FormattedResponse(resData.toString, "application/json") 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/IAggregateMinutely.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import com.kaltura.core.utils.ReadableTimeUnits.ReadableTimeUnits 4 | import org.joda.time.DateTime 5 | 6 | /** 7 | * Created by orlylampert on 2/21/16. 8 | */ 9 | trait IAggregateMinutely extends IAggregate { 10 | override def ttl = 5 minutes 11 | override def getAggrTime(eventTime: DateTime): DateTime = eventTime.minuteOfHour().roundFloorCopy() 12 | override def getAggrTimeUnit: String = "minute" 13 | 14 | } 15 | -------------------------------------------------------------------------------- /kanalony-web-server/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | // The Play plugin 2 | addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.4.6") 3 | 4 | // web plugins 5 | 6 | addSbtPlugin("com.typesafe.sbt" % "sbt-coffeescript" % "1.0.0") 7 | 8 | addSbtPlugin("com.typesafe.sbt" % "sbt-less" % "1.0.6") 9 | 10 | addSbtPlugin("com.typesafe.sbt" % "sbt-jshint" % "1.0.3") 11 | 12 | addSbtPlugin("com.typesafe.sbt" % "sbt-rjs" % "1.0.7") 13 | 14 | addSbtPlugin("com.typesafe.sbt" % "sbt-digest" % "1.1.0") 15 | 16 | addSbtPlugin("com.typesafe.sbt" % "sbt-mocha" % "1.1.0") 17 | -------------------------------------------------------------------------------- /kanalony-core/src/main/scala/com/kaltura/core/utils/CollectionOps.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.utils 2 | 3 | object CollectionOps { 4 | /** 5 | * Returns true if arr is null or empty 6 | * @param arr 7 | * @tparam A 8 | * @return 9 | */ 10 | def isEmpty[A](arr: Array[A]): Boolean = arr == null || arr.length == 0 11 | 12 | /** 13 | * Returns true if str is null or empty 14 | * @param str 15 | * @tparam A 16 | * @return 17 | */ 18 | def isEmpty[A](str: String): Boolean = str == null || str.length == 0 19 | 20 | val EMPTY_STRING = "" 21 | } 22 | -------------------------------------------------------------------------------- /kanalony-receiver/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kanalony-receiver", 3 | "version": "1.0.0", 4 | "description": "Receive and process the player events and other event based data", 5 | "main": "server.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "start": "node server.js" 9 | }, 10 | "author": "Ofir Kerker", 11 | "license": "AGPL-3.0", 12 | "dependencies": { 13 | "JSONStream": "^1.0.6", 14 | "kafka-node": "^0.2.28", 15 | "minimist": "^1.2.0", 16 | "mkdirp": "^0.5.1", 17 | "winston": "^1.0.1" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /kanalony-core/src/main/scala/com/kaltura/core/userAgent/enums/Device.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.userAgent.enums 2 | 3 | /** 4 | * Created by ofirk on 01/02/2016. 5 | */ 6 | object Device extends Enumeration { 7 | val COMPUTER = Value(1, "Computer") 8 | val MOBILE = Value(2, "Mobile") 9 | val TABLET = Value(3, "Tablet") 10 | val GAME_CONSOLE = Value(4, "Game console") 11 | val DMR = Value(5, "Digital media receiver") 12 | val WEARABLE = Value(6, "Wearable computer") 13 | val UNKNOWN = Value(7, "Unknown Device") 14 | val INVALID = Value(-1, "Invalid Device") 15 | 16 | } 17 | -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/IAggregateTenSecs.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import org.joda.time.DateTime 4 | import com.kaltura.core.utils.ReadableTimeUnits.ReadableTimeUnits 5 | 6 | /** 7 | * Created by orlylampert on 3/2/16. 8 | */ 9 | trait IAggregateTenSecs extends IAggregate { 10 | override def ttl = 1 minutes 11 | def getAggrTime(eventTime: DateTime): DateTime = eventTime.minuteOfHour().roundFloorCopy().withSecondOfMinute(eventTime.getSecondOfMinute() / 10 * 10) 12 | override def getAggrTimeUnit: String = "tensecs" 13 | 14 | } 15 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/IQuery.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | import com.kaltura.model.entities.{Metric, Metrics} 4 | import kanalony.storage.logic.queries.model._ 5 | 6 | import scala.concurrent.Future 7 | 8 | /** 9 | * Created by elad.benedict on 2/10/2016. 10 | */ 11 | 12 | trait IQuery { 13 | def isMetricSupported(metric: Metric) : Boolean 14 | def supportedWellKnownMetrics : Set[Metric] 15 | def dimensionInformation : List[IDimensionDefinition] 16 | def query(params : QueryParams) : Future[List[IQueryResult]] // IQueryResult per metric 17 | } 18 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/queryGenerator/RowBreakerFactory.scala: -------------------------------------------------------------------------------- 1 | package queryGenerator 2 | 3 | import kanalony.storage.generator.ColumnNames 4 | 5 | /** 6 | * Created by elad.benedict on 3/22/2016. 7 | */ 8 | object RowBreakerFactory { 9 | def getBreaker(columnName : ColumnNames.Value) : IRowBreaker = columnName match { 10 | case ColumnNames.year => YearPartitioner() 11 | case ColumnNames.month => MonthPartitioner() 12 | case ColumnNames.day => DayPartitioner() 13 | case _ => throw new IllegalArgumentException(s"Unsupported column name for partitioning ${columnName.toString}") 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /kanalony-model/src/main/scala/com/kaltura/model/dao/PartnerDAO.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.model.dao 2 | 3 | import com.kaltura.core.logging.{BaseLog, MetaLog} 4 | import com.kaltura.model.entities.Partner 5 | /** 6 | * Created by ofirk on 18/01/2016. 7 | */ 8 | object PartnerDAO extends DAOBase[Partner, Int] with MetaLog[BaseLog] { 9 | 10 | def getById(id: Int): Option[Partner] = { 11 | withPartnerImpersonation(id,"get partner data", Some(Partner(id))) { kalturaAPI => 12 | val partner = kalturaAPI.getPartnerService.get(id) 13 | Some(Partner(id, Option(partner.adminSecret), Option(partner.crmId), Option(partner.partnerPackage))) 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /kanalony-storage-logic/src/main/scala/kanalony/storage/logic/queries/DailyCountQuery.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries 2 | 3 | import com.kaltura.model.entities.Metrics 4 | import kanalony.storage.logic._ 5 | import kanalony.storage.logic.queries.model._ 6 | import org.joda.time.DateTime 7 | import scala.concurrent.ExecutionContext.Implicits.global 8 | 9 | import scala.concurrent.Future 10 | 11 | 12 | 13 | class DailyCountQuery(queryParams: QueryParams, queryLocator: IQueryLocator) extends DailyQueryBase(queryParams, queryLocator) { 14 | override def computeGroupAggregatedValue: (List[List[String]]) => Double = _.foldLeft(0.0)(_ + countFieldExtractor(_)) 15 | } 16 | 17 | -------------------------------------------------------------------------------- /kanalony-storage-access/src/main/scala/kanalony/storage/generator/IColumnDefinition.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.generator 2 | 3 | /** 4 | * Created by elad.benedict on 2/7/2016. 5 | */ 6 | trait IColumnDefinition { 7 | val name : ColumnNames.Value 8 | val typeName : ColumnType.Value 9 | val inPartitionKey : Boolean 10 | val inClusteringKey : Boolean 11 | } 12 | 13 | object ColumnQueryKind extends Enumeration { 14 | val Equality, Range, List = Value 15 | } 16 | 17 | trait IQueryableColumnDefinition extends IColumnDefinition{ 18 | val queryKind : ColumnQueryKind.Value 19 | } 20 | 21 | trait IClusteringColumnDefinition extends IColumnDefinition { 22 | val orderBy : OrderBy.Value 23 | } 24 | -------------------------------------------------------------------------------- /kanalony-storage-logic/src/main/scala/kanalony/storage/logic/queries/EstimatedMinutesWatchedQuery.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries 2 | 3 | import com.kaltura.model.entities.{Metric, Metrics} 4 | import kanalony.storage.logic._ 5 | 6 | /** 7 | * Created by elad.benedict on 3/14/2016. 8 | */ 9 | 10 | class EstimatedMinutesWatchedQuery(queryParams: QueryParams, queryLocator: IQueryLocator) extends ComputedQuery(Metrics.estimatedMinutesWatched, queryParams, queryLocator) { 11 | override val requiredMetrics: List[Metric] = List(Metrics.view) 12 | 13 | override def computeValue(groupMetricsValues: List[SingleMetricValue]): Double = { 14 | groupMetricsValues.head.value / 6 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/queries/model/DimensionConstraintDeclaration.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries.model 2 | 3 | /** 4 | * Created by elad.benedict on 2/16/2016. 5 | */ 6 | class DimensionConstraintDeclaration(val constraint: QueryConstraint.Value) extends IDimensionConstraint { 7 | def canEqual(a: Any) = a.isInstanceOf[DimensionConstraintDeclaration] 8 | 9 | override def equals(that: Any): Boolean = 10 | that match { 11 | case that: DimensionConstraintDeclaration => 12 | that.canEqual(this) && this.constraint == that.constraint 13 | case _ => false 14 | } 15 | 16 | override def hashCode:Int = { 17 | constraint.hashCode 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /kanalony-receiver/lib/configurationUtil.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | /** 5 | * Returns a configuration value by provided key or provided defaultValue if not exists 6 | * @param configKey 7 | * @param defaultValue 8 | * @returns {*|null} 9 | */ 10 | getOrElse: function(configKey, defaultValue) { 11 | return this.get(configKey) || defaultValue; 12 | }, 13 | 14 | /** 15 | * Returns a configuration value by provided key or null if not exists 16 | * @param configKey 17 | * @returns {null} 18 | */ 19 | get: function(configKey){ 20 | // TODO - read configuration from file using an environment variable 21 | return null; 22 | } 23 | }; 24 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/queries/model/DimensionUnconstrained.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries.model 2 | 3 | /** 4 | * Created by elad.benedict on 2/16/2016. 5 | */ 6 | class DimensionUnconstrained extends IDimensionConstraint { 7 | override val constraint: QueryConstraint.Value = QueryConstraint.Unconstrained 8 | 9 | def canEqual(a: Any) = a.isInstanceOf[DimensionUnconstrained] 10 | 11 | override def equals(that: Any): Boolean = 12 | that match { 13 | case that: DimensionUnconstrained => 14 | that.canEqual(this) && this.constraint == that.constraint 15 | case _ => false 16 | } 17 | 18 | override def hashCode:Int = { 19 | constraint.hashCode() 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /kanalony-logstash/build.sbt: -------------------------------------------------------------------------------- 1 | lazy val `kanalony-logstash` = (project in file(".")). 2 | settings( 3 | name := "kanalony-web-server", 4 | version := "1.0", 5 | scalaVersion := "2.11.7", 6 | libraryDependencies ++= Seq( 7 | "joda-time" % "joda-time" % "2.8.1", 8 | "org.scalatest" %% "scalatest" % "2.2.6" % "test", 9 | "io.gatling.highcharts" % "gatling-charts-highcharts" % "2.1.7" % "test", 10 | "org.apache.kafka" % "kafka_2.11" % "0.9.0.1" % "test" 11 | ), 12 | resolvers ++= Seq("scalaz-bintray" at "http://dl.bintray.com/scalaz/releases", 13 | "Artima Maven Repository" at "http://repo.artima.com/releases") 14 | ) -------------------------------------------------------------------------------- /kanalony-core/src/main/scala/com/kaltura/core/cassandra/ClusterManager.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.cassandra 2 | 3 | import java.net.InetAddress 4 | 5 | import com.datastax.driver.core.{Cluster, Session} 6 | import com.kaltura.core.utils.ConfigurationManager 7 | 8 | /** 9 | * Created by ofirk on 26/01/2016. 10 | */ 11 | object ClusterManager { 12 | private val clusterBuilder = Cluster.builder() 13 | ConfigurationManager.getOrElse("kanalony.events_enrichment.cassandra_host","127.0.0.1") 14 | .split(",").foreach(x => clusterBuilder.addContactPoints(x)) 15 | private val cluster = clusterBuilder.build() 16 | 17 | private val session = cluster.connect() 18 | 19 | def getSession = session 20 | 21 | def close() = if (cluster != null) cluster.close() 22 | } 23 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/ComputedQueryFactory.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | import com.kaltura.model.entities.{Metric, Metrics} 4 | 5 | /** 6 | * Created by elad.benedict on 3/7/2016. 7 | */ 8 | 9 | abstract class ComputedQueryFactory[T] { 10 | type QueryCreator = (QueryParams) => List[(IQuery, List[Metric])] 11 | protected val queryCreatorGetter : Map[T,QueryCreator] 12 | def getErrorMessage(value : T) : String 13 | 14 | lazy val values = queryCreatorGetter.keys.toSet 15 | 16 | def getQueryCreator(value: T): QueryCreator = { 17 | if (!values.contains(value)) 18 | { 19 | throw new IllegalArgumentException(getErrorMessage(value)) 20 | } 21 | queryCreatorGetter(value) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/IDailyPartitionedQueryParams.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | import org.joda.time.{DateTimeZone, Days, DateTime, Months} 4 | import com.kaltura.core.utils.ReadableDateUnits.ReadableDateUnits 5 | 6 | /** 7 | * Created by elad.benedict on 2/21/2016. 8 | */ 9 | trait IDailyPartitionedQueryParams { 10 | val startTime : DateTime 11 | val endTime : DateTime 12 | val days = { 13 | val dayCount = Days.daysBetween(startTime.withZone(DateTimeZone.UTC), endTime.withZone(DateTimeZone.UTC)).getDays 14 | (0 to dayCount).map(numOfDaysToAdd => { 15 | val dayDateTime = startTime.withZone(DateTimeZone.UTC).plusDays(numOfDaysToAdd) 16 | dayDateTime.getYearMonthDay 17 | }).toList 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/IUserAccessQuery.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | import com.kaltura.model.entities.Metrics 4 | /** 5 | * Created by elad.benedict on 2/10/2016. 6 | */ 7 | trait IUserActivityQuery extends IQuery{ 8 | val supportedWellKnownMetrics = Set(Metrics.playerImpression,Metrics.playRequested,Metrics.play,Metrics.estimatedMinutesWatched, 9 | Metrics.averageViewDuration,Metrics.playThrough25,Metrics.playThrough50, 10 | Metrics.playThrough75,Metrics.playThrough100,Metrics.averageViewDropOff, 11 | Metrics.segmentsWatched,Metrics.percentageWatched,Metrics.view,Metrics.dvrView,Metrics.peakView, 12 | Metrics.peakDvrView,Metrics.bufferingTime,Metrics.averageActualBitrate, 13 | Metrics.loadToPlayTime, Metrics.tenSecsViewed) 14 | } 15 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/queries/model/DimensionEqualityConstraint.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries.model 2 | 3 | /** 4 | * Created by elad.benedict on 2/16/2016. 5 | */ 6 | class DimensionEqualityConstraint[T](val values : Set[T]) extends IDimensionConstraint { 7 | override val constraint: QueryConstraint.Value = QueryConstraint.Equality 8 | 9 | def canEqual(a: Any) = a.isInstanceOf[DimensionEqualityConstraint[T]] 10 | 11 | override def equals(that: Any): Boolean = 12 | that match { 13 | case that: DimensionEqualityConstraint[T] => 14 | that.canEqual(this) && this.values == that.values && this.constraint == that.constraint 15 | case _ => false 16 | } 17 | 18 | override def hashCode:Int = { 19 | values.hashCode() ^ constraint.hashCode() 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /kanalony-storage-access/src/main/scala/ConnectorFactory.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage 2 | 3 | import com.typesafe.config.ConfigFactory 4 | import com.websudos.phantom.connectors.{ContactPoints, KeySpaceDef} 5 | import collection.JavaConversions._ 6 | 7 | /** 8 | * Created by elad.benedict on 2/7/2016. 9 | */ 10 | 11 | object ConnectorFactory extends IConnectorFactory { 12 | val config = ConfigFactory.load() 13 | val cassandraHosts : Seq[String] = config.getStringList("config.cassandra.hosts") 14 | val cassandraPort = config.getInt("config.cassandra.port") 15 | val keyspace = config.getString("config.cassandra.keyspace") 16 | override val connector : KeySpaceDef = ContactPoints(cassandraHosts, cassandraPort).keySpace(keyspace) 17 | override val dimConnector : KeySpaceDef = ContactPoints(cassandraHosts, cassandraPort).keySpace("enrichment_cache") 18 | } 19 | -------------------------------------------------------------------------------- /kanalony-web-server/app/model/Implicits.scala: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import argonaut.Argonaut._ 4 | 5 | /** 6 | * Created by elad.benedict on 2/21/2016. 7 | */ 8 | object Implicits { 9 | implicit lazy val AnalyticsResponseCodecJson = casecodec2(AnalyticsResponse.apply, AnalyticsResponse.unapply)("headers", "data") 10 | implicit lazy val EqualityFilterCodecJson = casecodec2(EqualityFilter.apply, EqualityFilter.unapply)("dimension", "values") 11 | implicit lazy val PagerCodecJson = casecodec2(Pager.apply, Pager.unapply)("size", "index") 12 | 13 | implicit lazy val AnalyticsRequestCodecJson = casecodec8(AnalyticsRequest.apply, AnalyticsRequest.unapply)("from", "to", "dimensions", "filters", "metrics", "utcOffset", "orderBy", "pager") 14 | implicit lazy val ErrorResponseCodedJson = casecodec2(ErrorResponse.apply, ErrorResponse.unapply)("kind", "data") 15 | } 16 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/queries/model/DimensionRangeConstraint.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries.model 2 | 3 | /** 4 | * Created by elad.benedict on 2/16/2016. 5 | */ 6 | class DimensionRangeConstraint[T](val start : T, val end : T) extends IDimensionConstraint { 7 | override val constraint: QueryConstraint.Value = QueryConstraint.Range 8 | 9 | def canEqual(a: Any) = a.isInstanceOf[DimensionRangeConstraint[T]] 10 | 11 | override def equals(that: Any): Boolean = 12 | that match { 13 | case that: DimensionRangeConstraint[T] => 14 | that.canEqual(this) && this.start == that.start && this.end == that.end && this.constraint == that.constraint 15 | case _ => false 16 | } 17 | 18 | override def hashCode:Int = { 19 | start.hashCode() ^ end.hashCode() ^ constraint.hashCode() 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/queryGenerator/QueryListGenerator.scala: -------------------------------------------------------------------------------- 1 | package queryGenerator 2 | 3 | import kanalony.storage.generator.{TableAccessorGenerator, TablesMetadata} 4 | 5 | /** 6 | * Created by elad.benedict on 3/6/2016. 7 | */ 8 | 9 | object QueryListGenerator { 10 | def generate() = { 11 | val queryNames = TablesMetadata.metadata.map(tm => QueryTypeGenerator.getQueryName(tm)) 12 | QueryTemplates.QueryListTemplate.content.stripMargin 13 | .replace(QueryTemplates.QueryListTemplate.queryObjectsPlaceholder, queryNames.mkString(",\n")) 14 | .replace(QueryTemplates.QueryListTemplate.queryObjectsDefinitionPlaceholder, TablesMetadata.metadata.map(tm => s"object ${QueryTypeGenerator.getQueryName(tm)} extends ${QueryTypeGenerator.getQueryName(tm)}(DbClientFactory.${TableAccessorGenerator.generateClassName(tm)})").mkString("\n"))} 15 | } 16 | -------------------------------------------------------------------------------- /kanalony-core/src/test/scala/com/kaltura/utils/CollectionsHelperSpec.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.utils 2 | 3 | import com.kaltura.core.utils.CollectionOps 4 | import org.scalatest._ 5 | 6 | class CollectionsHelperSpec extends FlatSpec with Matchers with BeforeAndAfterAll { 7 | 8 | "null array" should "be empty" in { 9 | val arr: Array[Int] = null 10 | CollectionOps.isEmpty(arr) shouldBe true 11 | } 12 | 13 | "null string" should "be empty" in { 14 | val str: String = null 15 | CollectionOps.isEmpty(str) shouldBe true 16 | } 17 | 18 | "a string with some characters" should "not be empty" in { 19 | val str: String = "string" 20 | CollectionOps.isEmpty(str) shouldBe false 21 | } 22 | 23 | "an array with some elements" should "not be empty" in { 24 | val arr = Array(1,2,3) 25 | CollectionOps.isEmpty(arr) shouldBe false 26 | } 27 | 28 | 29 | } -------------------------------------------------------------------------------- /kanalony-model/src/main/scala/com/kaltura/model/events/RawPlayerEvent.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.model.events 2 | 3 | import org.joda.time.DateTime 4 | 5 | case class RawPlayerEvent(eventTime: DateTime, 6 | remoteAddr: String = "", 7 | proxyRemoteAddr: String = "", 8 | userAgent: String = "", 9 | params: Map[String,String]) { 10 | lazy val partnerId: Option[Int] = { 11 | try { 12 | Some(params.get("partnerId").get.toInt) 13 | } catch { 14 | case e: Exception => None 15 | } 16 | } 17 | 18 | lazy val eventType: Option[String] = params.get("eventType") 19 | lazy val entryId: Option[String] = params.get("entryId") 20 | 21 | def isValid: Boolean = eventType.isDefined && eventType.get.nonEmpty && partnerId.isDefined && entryId.isDefined && entryId.get.nonEmpty 22 | } 23 | -------------------------------------------------------------------------------- /kanalony-model/src/test/scala-2.10/DAOTests.scala: -------------------------------------------------------------------------------- 1 | import com.kaltura.model.dao.PartnerDAO 2 | import com.kaltura.model.entities.Partner 3 | import org.scalatest.{BeforeAndAfterAll, Matchers, FlatSpec} 4 | import scala.math.pow 5 | /** 6 | * Created by ofirk on 28/01/2016. 7 | */ 8 | class DAOTests extends FlatSpec with Matchers with BeforeAndAfterAll { 9 | 10 | "PartnerDAO.getById" should "not be empty" in { 11 | val partner: Option[Partner] = PartnerDAO.getById(1091) 12 | partner.isEmpty shouldBe false 13 | 14 | time { 15 | Range(1, 50000).foreach { i => 16 | PartnerDAO.getById(i) 17 | } 18 | } 19 | } 20 | 21 | def time[R](block: => R): R = { 22 | val t0 = System.nanoTime() 23 | val result = block // call-by-name 24 | val t1 = System.nanoTime() 25 | println("Elapsed time: " + (t1 - t0)/pow(10,9) + "s") 26 | result 27 | } 28 | 29 | } -------------------------------------------------------------------------------- /kanalony-model/src/main/scala/com/kaltura/model/dao/EntryMetadataDAO.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.model.dao 2 | 3 | import com.kaltura.core.utils.KalturaImpersonationClient 4 | import com.kaltura.model.entities.EntryMetadata 5 | 6 | /** 7 | * Created by orlylampert on 12/27/16. 8 | */ 9 | object EntryMetadataDAO extends DAOBase[EntryMetadata, String] { 10 | def getById(partnerId: Int, entryId:String): Option[EntryMetadata] = { 11 | withPartnerImpersonation(partnerId, s"fetch entry data for entry $entryId", Some(EntryMetadata("-1", "Missing Entry Name"))) { kalturaAPI => 12 | val entryName = getEntryName(kalturaAPI, entryId) 13 | Some(EntryMetadata(entryId, entryName)) 14 | } 15 | } 16 | 17 | def getEntryName(kalturaAPI: KalturaImpersonationClient ,entryId: String): String = { 18 | val kEntry = kalturaAPI.getBaseEntryService.get(entryId) 19 | kEntry.name 20 | } 21 | 22 | } -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/QueryParams.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | import com.kaltura.model.entities.Metric 4 | import kanalony.storage.logic.queries.model._ 5 | import org.joda.time.{DateTimeZone, LocalDateTime} 6 | 7 | /** 8 | * Created by elad.benedict on 2/14/2016. 9 | */ 10 | 11 | case class QueryParams(dimensionDefinitions : List[IQueryDimensionDefinition], metrics : List[Metric], start : LocalDateTime, end : LocalDateTime, timezoneOffset : Int = 0, orderBy: OrderDefinition, pager: PagerDefinition) { 12 | val timezoneOffsetHours = timezoneOffset / 60 13 | val timezoneOffsetMinutes = timezoneOffset % 60 14 | 15 | val startUtc = start.toDateTime(DateTimeZone.forOffsetHoursMinutes(timezoneOffsetHours, timezoneOffsetMinutes)) 16 | val endUtc = end.toDateTime(DateTimeZone.forOffsetHoursMinutes(timezoneOffsetHours, timezoneOffsetMinutes)) 17 | } 18 | -------------------------------------------------------------------------------- /kanalony-core/src/main/java/com/kaltura/core/ip2location/Coordinate.java: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.ip2location; 2 | 3 | public class Coordinate { 4 | 5 | protected String name; 6 | protected float longitude; 7 | protected float latitude; 8 | 9 | public Coordinate(String name, float latitude, float longitude) { 10 | this.name = name; 11 | this.longitude = longitude; 12 | this.latitude = latitude; 13 | } 14 | 15 | public String getName() { 16 | return name; 17 | } 18 | 19 | public void setName(String country) { 20 | this.name = country; 21 | } 22 | 23 | public float getLongitude() { 24 | return longitude; 25 | } 26 | 27 | public void setLongitude(float longitude) { 28 | this.longitude = longitude; 29 | } 30 | 31 | public float getLatitude() { 32 | return latitude; 33 | } 34 | 35 | public void setLatitude(float latitude) { 36 | this.latitude = latitude; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /kanalony-model/src/main/scala/com/kaltura/model/cache/CacheBase.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.model.cache 2 | 3 | import com.datastax.driver.core.{Row, Statement} 4 | import com.datastax.driver.core.querybuilder.QueryBuilder 5 | import com.datastax.driver.core.querybuilder.QueryBuilder._ 6 | import QueryBuilder.{eq => eql} 7 | import com.kaltura.core.cassandra.ClusterManager 8 | 9 | /** 10 | * Created by ofirk on 07/02/2016. 11 | */ 12 | trait CacheBase[T, IDType] { 13 | def cassandraSession = ClusterManager.getSession 14 | val keySpace = "enrichment_cache" 15 | val tableName: String 16 | val idFieldName: String 17 | 18 | def findById(id:IDType): Option[T] = { 19 | val s:Statement = QueryBuilder 20 | .select() 21 | .all() 22 | .from(keySpace, tableName) 23 | .where(eql(idFieldName,id)) 24 | .limit(1) 25 | fromRow(cassandraSession.execute(s).one) 26 | } 27 | 28 | def fromRow(row: Row): Option[T] 29 | } 30 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/Driver.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | import java.io.File 4 | import kanalony.storage.utils.fs 5 | import kanalony.storage.generator.TablesMetadata 6 | import queryGenerator._ 7 | 8 | /** 9 | * Created by elad.benedict on 3/3/2016. 10 | */ 11 | object Driver { 12 | def main(args: Array[String]) { 13 | TablesMetadata.metadata.foreach(tm => { 14 | val paramType = ParamsTypeGenerator().generate(tm.tableName, tm.primaryKey.pk.columns map { ColumnExtendedDefinition.convert(_) }) 15 | val queryType = QueryTypeGenerator(tm).generate() 16 | val result = queryType + "\n\n" + paramType 17 | fs.printToFile(new File(QueryTypeGenerator.getQueryName(tm) + ".scala"))(p=>(p.write(result))) 18 | val queryList = QueryListGenerator.generate() 19 | fs.printToFile(new File("Queries.scala"))(p=>(p.write(queryList))) 20 | }) 21 | 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /kanalony-enrichment/tasks/upload_driver.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | scp /Users/ofirk/projects/kaltura/kanalony/kanalony-enrichment/target/scala-2.11/kanalony-enrichment.jar il-bigdata-5:/opt/kaltura/kanalony/.; 3 | scp /Users/ofirk/projects/kaltura/kanalony/kanalony-enrichment/target/scala-2.11/kanalony-enrichment.jar il-bigdata-6:/opt/kaltura/kanalony/.; 4 | scp /Users/ofirk/projects/kaltura/kanalony/kanalony-enrichment/target/scala-2.11/kanalony-enrichment.jar il-bigdata-7:/opt/kaltura/kanalony/.; 5 | scp /Users/ofirk/projects/kaltura/kanalony/kanalony-enrichment/target/scala-2.11/kanalony-enrichment.jar il-bigdata-8:/opt/kaltura/kanalony/.; 6 | scp /Users/ofirk/projects/kaltura/kanalony/kanalony-enrichment/target/scala-2.11/kanalony-enrichment.jar il-bigdata-9:/opt/kaltura/kanalony/.; 7 | scp /Users/ofirk/projects/kaltura/kanalony/kanalony-enrichment/target/scala-2.11/kanalony-enrichment.jar il-bigdata-10:/opt/kaltura/kanalony/.; 8 | ssh il-bigdata-5 '/opt/kaltura/scripts/start_enr_driver.sh' 9 | -------------------------------------------------------------------------------- /kanalony-storage-logic/src/main/scala/kanalony/storage/logic/queries/AverageTimeViewedQuery.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries 2 | 3 | import com.kaltura.model.entities.{Metric, Metrics} 4 | import kanalony.storage.logic.{IQueryLocator, QueryParams} 5 | 6 | /** 7 | * Created by elad.benedict on 3/15/2016. 8 | */ 9 | 10 | class AverageTimeViewedQuery(queryParams: QueryParams, queryLocator: IQueryLocator) extends ComputedQuery(Metrics.averageViewDuration, queryParams, queryLocator) { 11 | override val requiredMetrics: List[Metric] = List(Metrics.estimatedMinutesWatched, Metrics.play) 12 | 13 | override def computeValue(groupMetricsValues: List[SingleMetricValue]): Double = { 14 | val estimatedMinutesWatched = groupMetricsValues.find(_.metric == Metrics.estimatedMinutesWatched).get.value 15 | val plays = groupMetricsValues.find(_.metric == Metrics.play).get.value 16 | if (plays == 0) { 0 } 17 | else { estimatedMinutesWatched/60/plays } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /kanalony-core/src/test/scala/com/kaltura/ip2location/IP2LocationSpec.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.ip2location 2 | 3 | import com.kaltura.core.ip2location.LocationResolver 4 | import org.scalatest._ 5 | 6 | class IP2LocationSpec extends FlatSpec with Matchers with BeforeAndAfterAll { 7 | 8 | val locationResolver = new LocationResolver() 9 | 10 | "Google's DNS server IP" should "be in Mountain View, USA" in { 11 | val location = locationResolver.parse("8.8.8.8") 12 | location.country should equal ("United States") 13 | location.region should equal ("California") 14 | location.city should equal ("Mountain View") 15 | } 16 | 17 | 18 | "A local IP" should "not be resolved successfully" in { 19 | val location = locationResolver.parse("127.0.0.1") 20 | location.country should equal ("N/A") 21 | location.region should equal ("N/A") 22 | location.city should equal ("N/A") 23 | } 24 | 25 | override def afterAll = locationResolver.close 26 | 27 | } 28 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/IMonthlyPartitionedQueryParams.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | import com.kaltura.core.utils.ReadableDateUnits.ReadableDateUnits 4 | import org.joda.time.{DateTimeZone, Months, DateTime} 5 | 6 | /** 7 | * Created by elad.benedict on 2/21/2016. 8 | */ 9 | trait IMonthlyPartitionedQueryParams { 10 | val startTime : DateTime 11 | val endTime : DateTime 12 | val months = { 13 | val startTimeFirstDayOfMonth = startTime.withZone(DateTimeZone.UTC).withDayOfMonth(1) 14 | val endTimeFirstDayOfNextMonth = endTime.withZone(DateTimeZone.UTC).plusMonths(1).withDayOfMonth(1) 15 | val monthCount = Months.monthsBetween(startTimeFirstDayOfMonth, endTimeFirstDayOfNextMonth).getMonths - 1 16 | (0 to monthCount).map(numOfMonthsToAdd => { 17 | val monthDateTime = startTime.withZone(DateTimeZone.UTC).plusMonths(numOfMonthsToAdd) 18 | monthDateTime.getYearMonth 19 | }).toList 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /kanalony-storage-logic/src/test/scala/kanalony/storage/logic/EstimatedMinutesWatchedQueryTests.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | import com.kaltura.model.entities.{Metric, Metrics} 4 | import kanalony.storage.logic.queries.{EstimatedMinutesWatchedQuery, ComputedQuery, PlayRatioQuery} 5 | 6 | /** 7 | * Created by elad.benedict on 4/10/2016. 8 | */ 9 | class EstimatedMinutesWatchedQueryTests extends ComputedMetricTestsBase { 10 | override protected def createComputedMetricQuery(): (QueryParams, IQueryLocator) => ComputedQuery = (qp, ql) => new EstimatedMinutesWatchedQuery(qp, ql) 11 | 12 | override val dependentMetricsResultSet: Map[Metric, List[IQueryResult]] = { 13 | Map(Metrics.tenSecsViewed -> List(QueryResult(List("partner","entry", "tenSecsViewed"), List(List("1","1","6"),List("1","2","12"))))) 14 | } 15 | override val expectedQueryResult: QueryResult = QueryResult(List("partner","entry","estimatedMinutesWatched"), List(List("1","2","2.0"),List("1","1","1.0"))) 16 | } 17 | -------------------------------------------------------------------------------- /kanalony-storage-access/src/main/scala/kanalony/storage/generator/DbClientFactoryGenerator.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.generator 2 | 3 | import kanalony.storage.generator.GenerationTemplates.dbClientFactoryTemplate 4 | 5 | /** 6 | * Created by elad.benedict on 3/6/2016. 7 | */ 8 | object DbClientFactoryGenerator { 9 | def generate() = { 10 | dbClientFactoryTemplate.content.stripMargin 11 | .replace(dbClientFactoryTemplate.accessorObjectDefinitionsPlaceholder, 12 | TablesMetadata.metadata.map(tm => { 13 | val accessorName = TableAccessorGenerator.generateClassName(tm) 14 | val interfaceName = TableAccessorGenerator.generateInterfaceName(tm) 15 | val privateMemberName = s"${accessorName}Obj" 16 | s"private object ${privateMemberName} extends ${accessorName} with connector.Connector with ${interfaceName}" + "\n" + 17 | s"lazy val ${accessorName} : ${interfaceName} = ${privateMemberName}" 18 | }).mkString("\n")) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /kanalony-aggregations/tasks/upload_driver.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | scp /Users/ofirk/projects/kaltura/kanalony/kanalony-aggregations/target/scala-2.11/kanalony-aggregations.jar il-bigdata-5:/opt/kaltura/kanalony/.; 3 | scp /Users/ofirk/projects/kaltura/kanalony/kanalony-aggregations/target/scala-2.11/kanalony-aggregations.jar il-bigdata-6:/opt/kaltura/kanalony/.; 4 | scp /Users/ofirk/projects/kaltura/kanalony/kanalony-aggregations/target/scala-2.11/kanalony-aggregations.jar il-bigdata-7:/opt/kaltura/kanalony/.; 5 | scp /Users/ofirk/projects/kaltura/kanalony/kanalony-aggregations/target/scala-2.11/kanalony-aggregations.jar il-bigdata-8:/opt/kaltura/kanalony/.; 6 | scp /Users/ofirk/projects/kaltura/kanalony/kanalony-aggregations/target/scala-2.11/kanalony-aggregations.jar il-bigdata-9:/opt/kaltura/kanalony/.; 7 | scp /Users/ofirk/projects/kaltura/kanalony/kanalony-aggregations/target/scala-2.11/kanalony-aggregations.jar il-bigdata-10:/opt/kaltura/kanalony/.; 8 | ssh il-bigdata-5 '/opt/kaltura/scripts/start_agg_driver.sh' 9 | -------------------------------------------------------------------------------- /kanalony-receiver/lib/kafkaProducer.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var kafka = require('kafka-node'), 4 | logger = require('./logger')(module); 5 | 6 | var kafkaProducer = function(zkConnectionString, onReady){ 7 | this.zkConnectionString = zkConnectionString; 8 | this.onReady = onReady || function(){}; 9 | this.connect(); 10 | }; 11 | 12 | kafkaProducer.prototype.connect = function() { 13 | var that = this; 14 | this.client  = new kafka.Client(this.zkConnectionString); 15 | this.producer = new kafka.Producer(this.client, {requireAcks: 1, ackTimeoutMs: 50}); 16 | this.producer.on('ready', function () { 17 | logger.info("Producer is ready!"); 18 | that.onReady(); 19 | }); 20 | 21 | this.producer.on('error', function (err) { 22 | logger.warn("Producer Error:", err); 23 | }); 24 | }; 25 | 26 | kafkaProducer.prototype.send = function(payload, cb) { 27 | this.producer.send(payload, cb); 28 | }; 29 | 30 | module.exports = kafkaProducer; 31 | 32 | -------------------------------------------------------------------------------- /kanalony-core/src/test/scala/com/kaltura/core/urls/UrlParserSpec.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.urls 2 | 3 | import org.scalatest.{Matchers, FlatSpec} 4 | 5 | class UrlParserSpec extends FlatSpec with Matchers { 6 | 7 | "Complex Url" should "be resolved successfully" in { 8 | val url = "https://kaltura.atlassian.net:8080/secure/RapidBoard.jspa?rapidView=181&view=planning&selectedIssue=PLAT-3791" 9 | val urlParts = UrlParser.getUrlParts(url) 10 | urlParts.domain should be ("kaltura.atlassian.net") 11 | urlParts.canonicalUrl should be ("kaltura.atlassian.net:8080/secure/RapidBoard.jspa") 12 | urlParts.originalUrl should be (url) 13 | } 14 | 15 | "Simple Url" should "be resolved successfully" in { 16 | val url = "http://kaltura.atlassian.net" 17 | val urlParts = UrlParser.getUrlParts(url) 18 | urlParts.domain should be ("kaltura.atlassian.net") 19 | urlParts.canonicalUrl should be ("kaltura.atlassian.net") 20 | urlParts.originalUrl should be (url) 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /kanalony-storage-logic/src/main/scala/kanalony/storage/logic/queries/PlayRatioQuery.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries 2 | 3 | import com.kaltura.model.entities.{Metric, Metrics} 4 | import kanalony.storage.logic._ 5 | import kanalony.storage.logic.queries.model._ 6 | import scala.concurrent.ExecutionContext.Implicits.global 7 | import scala.concurrent.Future 8 | 9 | /** 10 | * Created by elad.benedict on 3/13/2016. 11 | */ 12 | 13 | class PlayRatioQuery(queryParams: QueryParams, queryLocator: IQueryLocator) extends ComputedQuery(Metrics.playRatio, queryParams, queryLocator) { 14 | override val requiredMetrics: List[Metric] = List(Metrics.play, Metrics.playerImpression) 15 | 16 | override def computeValue(groupMetricsValues: List[SingleMetricValue]): Double = { 17 | val playValue = groupMetricsValues.find(_.metric == Metrics.play).get.value 18 | val playerImpressionValue = groupMetricsValues.find(_.metric == Metrics.playerImpression).get.value 19 | playValue/playerImpressionValue 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /config/config.properties.template: -------------------------------------------------------------------------------- 1 | # Mandatory 2 | kanalony.events_enhancer.application_name=EventsEnrichment 3 | kanalony.events_enrichment.admin_partner_secret= 4 | kanalony.core.ip2location.data_filepath=/opt/kaltura/conf/geoip/IP-COUNTRY-CITY-ISP.BIN 5 | kanalony.events_aggregation.application_name=EventsAggregation 6 | 7 | # Optional 8 | # kanalony.events_enhancer.master=spark://localhost:7077 9 | # kanalony.events_enhancer.cassandra_host=localhost 10 | # kanalony.events_enhancer.kafka_brokers=localhost:9092 11 | # kanalony.events_enhancer.batch_duration=2 12 | # kanalony.events_enhancer.backpressure=true 13 | # kanalony.checkpoint_root_path=/tmp/checkpoint 14 | 15 | # kanalony.events_aggregations.kafka_brokers=localhost:9092 16 | # kanalony.events_aggregations.master=spark://localhost:7077 17 | # kanalony.events_aggregations.cassandra_host=localhost 18 | # kanalony.events_aggregations.batch_duration=10 19 | # kanalony.events_aggregations.concurrentJobs=10 20 | # kanalony.events_aggregations.backpressure=true 21 | -------------------------------------------------------------------------------- /kanalony-storage-logic/src/test/scala/kanalony/storage/logic/PlayRatioQueryTests.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | import com.kaltura.model.entities.{Metrics, Metric} 4 | import kanalony.storage.logic.queries.{PlayRatioQuery, ComputedQuery} 5 | 6 | /** 7 | * Created by elad.benedict on 4/10/2016. 8 | */ 9 | class PlayRatioQueryTests extends ComputedMetricTestsBase { 10 | override protected def createComputedMetricQuery(): (QueryParams, IQueryLocator) => ComputedQuery = (qp, ql) => new PlayRatioQuery(qp, ql) 11 | 12 | override val dependentMetricsResultSet: Map[Metric, List[IQueryResult]] = { 13 | Map(Metrics.play -> List(QueryResult(List("partner","entry", "play"), List(List("1","1","2"),List("1","2","3")))), 14 | Metrics.playerImpression -> List(QueryResult(List("partner","entry","playImpression"), List(List("1","1","4"),List("1","2","4"))))) 15 | } 16 | override val expectedQueryResult: QueryResult = QueryResult(List("partner","entry","playRatio"), List(List("1","2","0.75"),List("1","1","0.5"))) 17 | } 18 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/queryGenerator/ColumnExtendedDefinition.scala: -------------------------------------------------------------------------------- 1 | package queryGenerator 2 | 3 | import kanalony.storage.generator._ 4 | 5 | /** 6 | * Created by elad.benedict on 2/25/2016. 7 | */ 8 | 9 | trait IColumnExtendedDefinition extends IColumnDefinition { 10 | val inferred : Boolean 11 | } 12 | 13 | class ColumnExtendedDefinition(val name : ColumnNames.Value, val typeName : ColumnType.Value, val inPartitionKey : Boolean, val inClusteringKey : Boolean, override val inferred : Boolean = false) extends IColumnExtendedDefinition 14 | 15 | object ColumnExtendedDefinition { 16 | def isColumnImplicit(name: String): Boolean = name match { 17 | case "year" => true 18 | case "month" => true 19 | case "day" => true 20 | case _ => false 21 | } 22 | 23 | def convert(colDef : IColumnDefinition) : IColumnExtendedDefinition = { 24 | new ColumnExtendedDefinition(colDef.name, colDef.typeName, colDef.inPartitionKey, colDef.inClusteringKey, isColumnImplicit(colDef.name.toString)) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /kanalony-model/src/main/scala/com/kaltura/model/dao/DAOBase.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.model.dao 2 | 3 | import com.kaltura.client.KalturaApiException 4 | import com.kaltura.core.utils.{KalturaImpersonationClient, KalturaAPIFactory} 5 | 6 | trait DAOBase[T, IDType]{ 7 | 8 | def withPartnerImpersonation[A](partnerId:Int, action:String, naVal:A)(execution: (KalturaImpersonationClient) => A): A = { 9 | val kalturaAPI = KalturaAPIFactory.getClient 10 | kalturaAPI.setPartnerId(partnerId) 11 | try { 12 | execution(kalturaAPI) 13 | } catch { 14 | case kae: KalturaApiException => { 15 | if (kae.code == "INVALID_KS") { 16 | kalturaAPI.setKs(KalturaAPIFactory.resetKS) 17 | execution(kalturaAPI) 18 | } 19 | else System.err.println(s"Unable to $action for partner $partnerId):");kae.printStackTrace();naVal 20 | } 21 | case e: Exception => System.err.println(s"Unable to $action for partner $partnerId):");e.printStackTrace();naVal 22 | } finally { 23 | kalturaAPI.removePartnerId() 24 | } 25 | } 26 | 27 | } -------------------------------------------------------------------------------- /kanalony-enrichment/src/main/scala/com/kaltura/enrichment/IEnrich.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.enrichment 2 | 3 | import java.util.concurrent.TimeUnit 4 | 5 | import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache, Cache} 6 | import com.kaltura.model.events.RawPlayerEvent 7 | import org.apache.spark.rdd.RDD 8 | 9 | 10 | /** 11 | * Created by ofirk on 16/02/2016. 12 | */ 13 | trait IEnrich[IDType <: Object,EnrichType <: Object] { 14 | 15 | val maxCacheSize = 100000 16 | val expireAfterWrite: (Long, TimeUnit) = (1L, TimeUnit.HOURS) 17 | val localCache = buildLocalCache 18 | 19 | def buildLocalCache: LoadingCache[IDType, EnrichType] = { 20 | CacheBuilder.newBuilder() 21 | .maximumSize(maxCacheSize) 22 | .expireAfterWrite(expireAfterWrite._1, expireAfterWrite._2) 23 | .build( 24 | new CacheLoader[IDType ,EnrichType]() { 25 | override def load(id: IDType) : EnrichType = loadEntity(id) 26 | } 27 | ) 28 | } 29 | def loadEntity(id: IDType) : EnrichType 30 | def enrich(playerEvents:RawPlayerEvent):RawPlayerEvent 31 | } 32 | 33 | -------------------------------------------------------------------------------- /kanalony-storage-logic-generated/build.sbt: -------------------------------------------------------------------------------- 1 | lazy val `kanalony-model` = RootProject(file("../kanalony-model")) 2 | lazy val `kanalony-storage-access` = RootProject(file("../kanalony-storage-access")) 3 | lazy val `kanalony-storage-logic-core` = RootProject(file("../kanalony-storage-logic-core")) 4 | lazy val `kanalony-storage-logic-generated` = (project in file(".")). 5 | settings( 6 | name := "kanalony-storage-logic-generated", 7 | version := "1.0", 8 | scalaVersion := "2.11.7", 9 | libraryDependencies ++= Seq( 10 | "com.websudos" %% "phantom-dsl" % "1.22.0", 11 | "com.websudos" %% "phantom-testkit" % "1.12.2", 12 | "joda-time" % "joda-time" % "2.8.1", 13 | "com.google.guava" % "guava" % "12.0", 14 | "org.scalamock" %% "scalamock-scalatest-support" % "3.2.2" % "test", 15 | "org.scalatest" %% "scalatest" % "2.2.4" % "test" 16 | ) 17 | ).dependsOn(`kanalony-storage-access`,`kanalony-model`,`kanalony-storage-logic-core`) -------------------------------------------------------------------------------- /kanalony-storage-logic-core/build.sbt: -------------------------------------------------------------------------------- 1 | lazy val `kanalony-model` = RootProject(file("../kanalony-model")) 2 | lazy val `kanalony-storage-access` = RootProject(file("../kanalony-storage-access")) 3 | lazy val `kanalony-storage-access-generated` = RootProject(file("../kanalony-storage-access-generated")) 4 | lazy val `kanalony-storage-logic-core` = (project in file(".")). 5 | settings( 6 | name := "kanalony-storage-logic-core", 7 | version := "1.0", 8 | scalaVersion := "2.11.7", 9 | libraryDependencies ++= Seq( 10 | "com.websudos" %% "phantom-dsl" % "1.22.0", 11 | "com.websudos" %% "phantom-testkit" % "1.12.2", 12 | "joda-time" % "joda-time" % "2.8.1", 13 | "com.google.guava" % "guava" % "12.0", 14 | "org.scalamock" %% "scalamock-scalatest-support" % "3.2.2" % "test", 15 | "org.scalatest" %% "scalatest" % "2.2.4" % "test" 16 | ) 17 | ).dependsOn(`kanalony-storage-access`,`kanalony-storage-access-generated`,`kanalony-model`) -------------------------------------------------------------------------------- /kanalony-storage-logic/src/test/scala/kanalony/storage/logic/AverageTimeViewedQueryTests.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | import com.kaltura.model.entities.{Metric, Metrics} 4 | import kanalony.storage.logic.queries.{AverageTimeViewedQuery, AverageViewDropOffQuery, ComputedQuery} 5 | 6 | /** 7 | * Created by elad.benedict on 4/10/2016. 8 | */ 9 | class AverageTimeViewedQueryTests extends ComputedMetricTestsBase { 10 | override protected def createComputedMetricQuery(): (QueryParams, IQueryLocator) => ComputedQuery = (qp, ql) => new AverageTimeViewedQuery(qp, ql) 11 | 12 | override val dependentMetricsResultSet: Map[Metric, List[IQueryResult]] = { 13 | Map(Metrics.play -> List(QueryResult(List("partner","entry", "play"), List(List("1","2","4"),List("1","1","2")))), 14 | Metrics.estimatedMinutesWatched -> List(QueryResult(List("partner","entry", "estimatedMinutesWatched"), List(List("1","2","240"),List("1","1","120"))))) 15 | 16 | } 17 | override val expectedQueryResult: QueryResult = QueryResult(List("partner","entry","averageViewDuration"), List(List("1","2","1.0"),List("1","1","1.0"))) 18 | } 19 | -------------------------------------------------------------------------------- /kanalony-enrichment/src/main/scala/com/kaltura/enrichment/EnrichByEntry.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.enrichment 2 | 3 | 4 | import com.kaltura.model.cache.{EntryMetadataCache, EntryCache} 5 | import com.kaltura.model.entities.Entry 6 | import com.kaltura.model.events.RawPlayerEvent 7 | 8 | /** 9 | * Created by ofirk on 16/02/2016. 10 | */ 11 | class EnrichByEntry extends IEnrich[(Int,String),Entry] { 12 | 13 | override def loadEntity(entryPartnerId: (Int, String)) : Entry = { 14 | EntryMetadataCache.getById(entryPartnerId._1, entryPartnerId._2) 15 | EntryCache.getById(entryPartnerId._1, entryPartnerId._2) 16 | } 17 | 18 | override def enrich(playerEvent: RawPlayerEvent): RawPlayerEvent = { 19 | if (playerEvent.isValid) { 20 | val partnerId = playerEvent.partnerId.get 21 | if (partnerId > 0) { 22 | val entryId = playerEvent.entryId.get 23 | val categories = localCache.get((partnerId, entryId)).categories.getOrElse("") 24 | playerEvent.copy(params = playerEvent.params + ("categories" -> categories)) 25 | } 26 | else playerEvent 27 | } 28 | else playerEvent 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /kanalony-model/src/main/scala/com/kaltura/model/cache/EntryMetadataCache.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.model.cache 2 | 3 | import com.datastax.driver.core.Row 4 | import com.datastax.driver.core.querybuilder.QueryBuilder 5 | import com.kaltura.model.dao.EntryMetadataDAO 6 | import com.kaltura.model.entities.EntryMetadata 7 | 8 | /** 9 | * Created by orlylampert on 12/27/16. 10 | */ 11 | sealed class EntryMetadataCache extends CacheBase[EntryMetadata,String]{ 12 | override val tableName = "dim_entries_metadata" 13 | override val idFieldName = "id" 14 | override def fromRow(row: Row) = if (row != null) Some(EntryMetadata(row.getString(idFieldName), row.getString("name"))) else None 15 | 16 | def getById(partnerId: Int, entryId: String) : EntryMetadata = { 17 | findById(entryId).getOrElse { 18 | val entry = EntryMetadataDAO.getById(partnerId, entryId).getOrElse(EntryMetadata("-1", "Missing Entry Name")) 19 | cassandraSession.execute(QueryBuilder 20 | .insertInto(keySpace, tableName) 21 | .value("id", entry.id) 22 | .value("name", entry.name) 23 | 24 | ) 25 | entry 26 | 27 | } 28 | } 29 | } 30 | 31 | object EntryMetadataCache extends EntryMetadataCache -------------------------------------------------------------------------------- /kanalony-enrichment/src/main/scala/com/kaltura/enrichment/EnrichByPartner.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.enrichment 2 | 3 | import com.kaltura.core.sessions.KSParserBase 4 | import com.kaltura.model.cache.PartnerCache 5 | import com.kaltura.model.entities.Partner 6 | import com.kaltura.model.events.RawPlayerEvent 7 | 8 | /** 9 | * Created by ofirk on 31/01/2016. 10 | */ 11 | class EnrichByPartner extends IEnrich[Integer,Partner] with KSParserBase { 12 | 13 | override def loadEntity(partnerId: Integer) : Partner = PartnerCache.getById(partnerId) 14 | 15 | def enrich(playerEvent:RawPlayerEvent):RawPlayerEvent = { 16 | if (playerEvent.isValid) { 17 | val partnerId = playerEvent.partnerId.get 18 | if (partnerId > 0) { 19 | val ks = playerEvent.params.getOrElse("ks", "") 20 | val ksData = parse(ks).getOrElse(KSData(partnerId)) 21 | val userId = if (ksData.userId == "0") "Unknown" else ksData.userId 22 | playerEvent.copy(params = playerEvent.params + ("userId" -> userId)) 23 | } 24 | else playerEvent 25 | } 26 | else playerEvent 27 | } 28 | 29 | override def getPartnerSecret(partnerId:Int) = localCache.get(partnerId).secret.getOrElse("") 30 | 31 | } 32 | -------------------------------------------------------------------------------- /kanalony-web-server/conf/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %coloredLevel - %logger - %message%n%xException 8 | 9 | 10 | 11 | 15 | 16 | 17 | 18 | 19 | logs/access.log 20 | 21 | %date{yyyy-MM-dd HH:mm:ss ZZZZ} %message%n 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /kanalony-model/build.sbt: -------------------------------------------------------------------------------- 1 | lazy val sparkVersion = "2.0.0" 2 | lazy val json4sVersion = "3.2.10" 3 | lazy val `kanalony-core` = RootProject(file("../kanalony-core")) 4 | lazy val `kanalony-model` = (project in file(".")). 5 | settings( 6 | name := "kanalony-model", 7 | version := "1.0", 8 | scalaVersion := "2.11.7", 9 | libraryDependencies ++= Seq( 10 | "org.scalatest" %% "scalatest" % "2.2.4" % "test", 11 | "org.apache.spark" %% "spark-core" % sparkVersion, 12 | "org.json4s" %% "json4s-jackson" % json4sVersion, 13 | "org.json4s" %% "json4s-native" % json4sVersion, 14 | "org.json4s" %% "json4s-ext" % json4sVersion, 15 | "joda-time" % "joda-time" % "2.8.1", 16 | "com.datastax.cassandra" % "cassandra-driver-core" % "3.0.2", 17 | "org.scalikejdbc" %% "scalikejdbc" % "2.3.4", 18 | "mysql" % "mysql-connector-java" % "5.1.38", 19 | "com.kaltura" % "kalturaClient" % "3.2.1" 20 | ) 21 | ).dependsOn(`kanalony-core`) -------------------------------------------------------------------------------- /kanalony-web-server/app/Global.scala: -------------------------------------------------------------------------------- 1 | import play.api.libs.concurrent.Execution.Implicits.defaultContext 2 | import play.api.mvc.{Filter, RequestHeader, Result, WithFilters} 3 | 4 | import scala.concurrent.Future 5 | 6 | object Global extends WithFilters(AccessLog) 7 | { 8 | 9 | 10 | } 11 | 12 | object AccessLog extends Filter { 13 | 14 | def apply(nextFilter: RequestHeader => Future[Result]) 15 | (requestHeader: RequestHeader): Future[Result] = { 16 | 17 | val startTime = System.currentTimeMillis 18 | 19 | nextFilter(requestHeader).map { result => 20 | 21 | val msg = s"method=${requestHeader.method} uri=${requestHeader.uri} remote-address=${requestHeader.remoteAddress} " + 22 | s"domain=${requestHeader.domain} query-string=${requestHeader.rawQueryString} " + 23 | s"referrer=${requestHeader.headers.get("referrer").getOrElse("N/A")} " + 24 | s"user-agent=[${requestHeader.headers.get("user-agent").getOrElse("N/A")}]" 25 | val endTime = System.currentTimeMillis 26 | val requestTime = endTime - startTime 27 | 28 | play.Logger.of("accesslog").info(msg) 29 | 30 | result.withHeaders("Request-Time" -> requestTime.toString) 31 | 32 | } 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /kanalony-model/src/main/scala/com/kaltura/model/cache/EntryCache.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.model.cache 2 | 3 | import com.datastax.driver.core.{Statement, Row} 4 | import com.datastax.driver.core.querybuilder.QueryBuilder 5 | import com.kaltura.core.utils.ReadableTimeUnits._ 6 | import com.kaltura.model.dao.EntryDAO 7 | import com.kaltura.model.entities.Entry 8 | 9 | /** 10 | * Created by ofirk on 07/02/2016. 11 | */ 12 | sealed class EntryCache extends CacheBase[Entry,String]{ 13 | override val tableName = "dim_entries" 14 | override val idFieldName = "id" 15 | override def fromRow(row: Row) = if (row != null) Some(Entry(row.getString(idFieldName), Option(row.getString("categories")))) else None 16 | 17 | val ttl = 1 day 18 | 19 | def getById(partnerId: Int, entryId: String) : Entry = { 20 | findById(entryId).getOrElse { 21 | val entry = EntryDAO.getById(partnerId, entryId).getOrElse(Entry(entryId)) 22 | cassandraSession.execute(QueryBuilder 23 | .insertInto(keySpace, tableName) 24 | .value("id", entry.id) 25 | .value("categories", entry.categories.orNull) 26 | .using(QueryBuilder.ttl(ttl)) 27 | ) 28 | entry 29 | } 30 | } 31 | } 32 | 33 | object EntryCache extends EntryCache -------------------------------------------------------------------------------- /kanalony-receiver/server.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var http = require('http'), 4 | kanalony  = require('./kanalony'), 5 | timeUtil = kanalony.TimeUtil, 6 | config = kanalony.ConfigurationUtil, 7 | logger = kanalony.Logger(module), 8 | relay = new kanalony.EventsRelay(config.getOrElse('kanalony.zookeeper.connection_string','127.0.0.1:2181/'), 9 | config.getOrElse('kanalony.receiver.kafka_topic','player-events')), 10 | host = config.getOrElse('kanalony.receiver.server_host','0.0.0.0'), 11 | port = config.getOrElse('kanalony.receiver.server_port','5555'); 12 | 13 | http.createServer(function (req, res) { 14 | if (relay.isRequestValid(req)){ 15 | writeResponse(res, 200, timeUtil.currentDateTimeAsUTCString()); // Client doesn't need to wait for response from Kafka 16 | relay.produceEvent(req); 17 | } 18 | else { 19 | writeResponse(res, 400, 'Bad request'); 20 | } 21 | 22 | }).listen(port, host); 23 | 24 | function writeResponse(res, code, body) { 25 | res.writeHead(code, {'Content-Type': 'text/plain'}); 26 | res.end(body); 27 | } 28 | 29 | logger.info('Server is running at http://' + host + ':' + port + '/'); 30 | 31 | -------------------------------------------------------------------------------- /kanalony-storage-access/src/main/scala/kanalony/storage/generator/Driver.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.generator 2 | import java.io.File 3 | import kanalony.storage.utils.fs 4 | 5 | /** 6 | * Created by elad.benedict on 2/8/2016. 7 | */ 8 | 9 | object Driver { 10 | 11 | def main (args: Array[String]) { 12 | 13 | val tablesMetadata = TablesMetadata.metadata 14 | 15 | println(new File(".").getAbsoluteFile) 16 | 17 | tablesMetadata.foreach(tm => { 18 | val generatedEntity = new EntityClassGenerator(tm).generate().stripMargin 19 | val tableAccessorGenerator = new TableAccessorGenerator(tm) 20 | val generatedTableAcecssor = tableAccessorGenerator.generate().stripMargin 21 | val generatedInterface = tableAccessorGenerator.generateInterface().stripMargin 22 | fs.printToFile(new File(TableAccessorGenerator.generateClassName(tm) + ".scala"))( 23 | p => { 24 | p.write({ 25 | generatedTableAcecssor + "\n\n" + 26 | generatedEntity + "\n\n" + 27 | generatedInterface 28 | }) 29 | }) 30 | }) 31 | val dbClientFactory = DbClientFactoryGenerator.generate() 32 | fs.printToFile(new File("DbClientFactory.scala"))(p=>(p.write(dbClientFactory))) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /kanalony-core/build.sbt: -------------------------------------------------------------------------------- 1 | lazy val sparkVersion = "2.0.0" 2 | lazy val `kanalony-core` = (project in file(".")). 3 | settings( 4 | name := "kanalony-core", 5 | version := "1.0", 6 | scalaVersion := "2.11.7", 7 | libraryDependencies ++= Seq( 8 | "org.scalatest" %% "scalatest" % "2.2.4" % "test", 9 | "org.slf4j" % "slf4j-api" % "1.7.12", 10 | "org.slf4j" % "slf4j-simple" % "1.7.12", 11 | "org.apache.spark" %% "spark-core" % sparkVersion, 12 | "org.apache.spark" %% "spark-streaming" % sparkVersion, 13 | "org.apache.spark" %% "spark-streaming-kafka-0-8" % sparkVersion, 14 | "eu.bitwalker" % "UserAgentUtils" % "1.18", 15 | "com.datastax.cassandra" % "cassandra-driver-core" % "3.0.2", 16 | "com.datastax.spark" %% "spark-cassandra-connector" % "2.0.0-M1", 17 | "commons-validator" % "commons-validator" % "1.5.0", 18 | "com.kaltura" % "kalturaClient" % "3.2.1", 19 | "com.google.guava" % "guava" % "18.0" 20 | ) 21 | ) 22 | 23 | 24 | -------------------------------------------------------------------------------- /kanalony-receiver/lib/timeUֹtil.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | 5 | /** 6 | * Returns Date and time as UTC String, for example: Wed, 30 Sep 2015 08:08:01 GMT 7 | * @returns {string} 8 | */ 9 | currentDateTimeAsUTCString: function() { 10 | return new Date().toUTCString(); 11 | }, 12 | 13 | /** 14 | * Returns Date and time as ISO String, for example: 2015-09-30T08:08:58.208Z 15 | * @returns {string} 16 | */ 17 | currentDateTimeAsISOString: function() { 18 | return new Date().toISOString(); 19 | }, 20 | 21 | /** 22 | * Returns current date and time rounded by minutes 23 | * @returns {number} 24 | */ 25 | currentDateTimeAsMinuteString: function() { 26 | return this.dateTimeAsMinuteString(new Date()); 27 | }, 28 | 29 | /** 30 | * Returns date and time rounded by minutes according to the provided value 31 | * @param d 32 | * @returns {string} 33 | */ 34 | dateTimeAsMinuteString: function(d) { 35 | if (isNaN(d)) { return d; } 36 | var minutesDateTime = d.getFullYear()*100000000 + (d.getMonth()+1)*1000000 + d.getDate()*10000 + d.getHours()*100 + d.getMinutes(); 37 | return minutesDateTime.toString(); 38 | } 39 | }; 40 | -------------------------------------------------------------------------------- /kanalony-model/src/main/scala/com/kaltura/model/dao/EntryDAO.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.model.dao 2 | 3 | import com.kaltura.client.types.KalturaCategoryEntryFilter 4 | import com.kaltura.core.utils.KalturaImpersonationClient 5 | import com.kaltura.model.entities.Entry 6 | 7 | import scala.collection.JavaConversions._ 8 | 9 | /** 10 | * Created by ofirk on 27/01/2016. 11 | */ 12 | object EntryDAO extends DAOBase[Entry, String] { 13 | def getById(partnerId: Int, entryId:String): Option[Entry] = { 14 | withPartnerImpersonation(partnerId, s"fetch entry data for entry $entryId", Some(Entry(entryId))) { kalturaAPI => 15 | val categoriesSet = getEntryCategories(kalturaAPI, entryId) 16 | Some(Entry(entryId, Some(categoriesSet.mkString(",")))) 17 | } 18 | } 19 | 20 | def getEntryCategories(kalturaAPI: KalturaImpersonationClient ,entryId: String): Set[String] = { 21 | val categoriesSet = scala.collection.mutable.Set[String]() 22 | val categoryEntryFilter = new KalturaCategoryEntryFilter() 23 | categoryEntryFilter.entryIdEqual = entryId 24 | val kCategories = kalturaAPI.getCategoryEntryService.list(categoryEntryFilter) 25 | kCategories.objects.foreach { kce => categoriesSet ++= kce.categoryFullIds.split(">") } 26 | categoriesSet.toSet 27 | } 28 | 29 | } -------------------------------------------------------------------------------- /kanalony-core/src/main/java/com/kaltura/core/ip2location/Utils.java: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.ip2location; 2 | import java.io.IOException; 3 | import java.io.RandomAccessFile; 4 | 5 | public class Utils { 6 | 7 | 8 | public static String readString(RandomAccessFile f, long l) throws IOException { 9 | f.seek(l - 1); 10 | int length = (int)f.readUnsignedByte(); 11 | byte[] strArr = new byte[length]; 12 | f.read(strArr); 13 | return new String(strArr); 14 | } 15 | 16 | public static long readUInteger(RandomAccessFile f) throws IOException { 17 | byte b1 = f.readByte(); 18 | byte b2 = f.readByte(); 19 | byte b3 = f.readByte(); 20 | byte b4 = f.readByte(); 21 | int number = ((b4 & 0xff) << 24) 22 | | ((b3 & 0xff) << 16) 23 | | ((b2 & 0xff) << 8) 24 | | ((b1 & 0xff) << 0); 25 | 26 | return (number & 0x00000000ffffffffL); 27 | } 28 | 29 | public static long readUInteger(RandomAccessFile f, long offset) throws IOException { 30 | f.seek(offset - 1); 31 | return readUInteger(f); 32 | 33 | } 34 | 35 | public static float readFloat(RandomAccessFile f, long offset) throws IOException { 36 | return Float.intBitsToFloat((int) readUInteger(f, offset)); 37 | } 38 | 39 | } 40 | 41 | 42 | -------------------------------------------------------------------------------- /kanalony-enrichment/src/main/scala/com/kaltura/enrichment/EventsFilter.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.enrichment 2 | 3 | import java.nio.file.{Files, Paths} 4 | 5 | import com.kaltura.core.utils.ConfigurationManager 6 | import com.kaltura.model.events.RawPlayerEvent 7 | import com.kaltura.model.cache.PartnerCache 8 | import com.kaltura.model.entities.Partner 9 | 10 | /** 11 | * Created by ofirk on 29/05/2016. 12 | */ 13 | class EventsFilter { 14 | var includeAll = true 15 | var enabledPartners: Set[Int] = _ 16 | val enabledPartnersFilePath = ConfigurationManager.get("kanalony.events_enrichment.enabled_partners_file_path") 17 | if(Files.exists(Paths.get(enabledPartnersFilePath))) { 18 | val source = scala.io.Source.fromFile(enabledPartnersFilePath) 19 | enabledPartners = try source.getLines.toList.map(_.toInt).toSet finally source.close() 20 | includeAll = false 21 | } 22 | val enabledPackages = ConfigurationManager.getOrElse("kanalony.events_enrichment.enabled_partners_package_ids", "*").split(",") 23 | 24 | def include(event: RawPlayerEvent): Boolean = includeAll || (event.partnerId.isDefined && enabledPackages.contains(PartnerCache.getById(event.partnerId.get).packageId.getOrElse(-1).toString)) || (event.partnerId.isDefined && enabledPartners.contains(event.partnerId.get)) 25 | 26 | } 27 | 28 | -------------------------------------------------------------------------------- /kanalony-web-server/build.sbt: -------------------------------------------------------------------------------- 1 | import play.sbt.PlayImport._ 2 | 3 | lazy val `kanalony-storage-logic` = RootProject(file("../kanalony-storage-logic")) 4 | lazy val `kanalony-storage-access` = RootProject(file("../kanalony-storage-access")) 5 | lazy val `kanalony-storage-access-generated` = RootProject(file("../kanalony-storage-access-generated")) 6 | lazy val `kanalony-web-server` = (project in file(".")).enablePlugins(PlayScala). 7 | settings( 8 | name := "kanalony-web-server", 9 | version := "1.0", 10 | scalaVersion := "2.11.7", 11 | libraryDependencies ++= Seq( 12 | "joda-time" % "joda-time" % "2.8.1", 13 | cache, 14 | ws, 15 | specs2 % Test, 16 | "io.argonaut" %% "argonaut" % "6.0.4", 17 | "org.scalatest" %% "scalatest" % "2.2.6" % "test", 18 | "io.gatling.highcharts" % "gatling-charts-highcharts" % "2.1.7" % "test" 19 | ), 20 | resolvers ++= Seq("scalaz-bintray" at "http://dl.bintray.com/scalaz/releases", 21 | "Artima Maven Repository" at "http://repo.artima.com/releases"), 22 | routesGenerator := InjectedRoutesGenerator 23 | ).dependsOn(`kanalony-storage-logic`,`kanalony-storage-access`,`kanalony-storage-access-generated`) -------------------------------------------------------------------------------- /kanalony-storage-logic/build.sbt: -------------------------------------------------------------------------------- 1 | lazy val `kanalony-model` = RootProject(file("../kanalony-model")) 2 | lazy val `kanalony-storage-access` = RootProject(file("../kanalony-storage-access")) 3 | lazy val `kanalony-storage-access-generated` = RootProject(file("../kanalony-storage-access-generated")) 4 | lazy val `kanalony-storage-logic-core` = RootProject(file("../kanalony-storage-logic-core")) 5 | lazy val `kanalony-storage-logic-generated` = RootProject(file("../kanalony-storage-logic-generated")) 6 | lazy val `kanalony-storage-logic` = (project in file(".")). 7 | settings( 8 | name := "kanalony-storage-logic", 9 | version := "1.0", 10 | scalaVersion := "2.11.7", 11 | libraryDependencies ++= Seq( 12 | "com.websudos" %% "phantom-dsl" % "1.22.0", 13 | "com.websudos" %% "phantom-testkit" % "1.12.2", 14 | "joda-time" % "joda-time" % "2.8.1", 15 | "com.google.guava" % "guava" % "12.0", 16 | "org.scalamock" %% "scalamock-scalatest-support" % "3.2.2" % "test", 17 | "org.scalatest" %% "scalatest" % "2.2.4" % "test" 18 | ) 19 | ).dependsOn(`kanalony-storage-access`,`kanalony-storage-access-generated`,`kanalony-model`,`kanalony-storage-logic-generated`,`kanalony-storage-logic-core`) -------------------------------------------------------------------------------- /kanalony-storage-logic/src/main/scala/kanalony/storage/logic/queries/AverageViewDropOffQuery.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.queries 2 | 3 | import com.kaltura.model.entities.{Metric, Metrics} 4 | import kanalony.storage.logic.{IQueryLocator, QueryParams} 5 | 6 | /** 7 | * Created by elad.benedict on 3/15/2016. 8 | */ 9 | 10 | class AverageViewDropOffQuery(queryParams: QueryParams, queryLocator: IQueryLocator) extends ComputedQuery(Metrics.averageViewDropOff, queryParams, queryLocator) { 11 | override val requiredMetrics: List[Metric] = List(Metrics.play, Metrics.playThrough25, Metrics.playThrough50, Metrics.playThrough75, Metrics.playThrough100) 12 | 13 | override def computeValue(groupMetricsValues: List[SingleMetricValue]): Double = { 14 | val plays = groupMetricsValues.find(_.metric == Metrics.play).get.value 15 | val playThrough25 = groupMetricsValues.find(_.metric == Metrics.playThrough25).get.value 16 | val playThrough50 = groupMetricsValues.find(_.metric == Metrics.playThrough50).get.value 17 | val playThrough75 = groupMetricsValues.find(_.metric == Metrics.playThrough75).get.value 18 | val playThrough100 = groupMetricsValues.find(_.metric == Metrics.playThrough100).get.value 19 | if (plays == 0) { 0 } 20 | else { (0.25*playThrough25 + 0.25*playThrough50 + 0.25*playThrough75 + 0.25*playThrough100) / plays } 21 | } 22 | } -------------------------------------------------------------------------------- /kanalony-model/src/main/scala/com/kaltura/model/cache/PartnerCache.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.model.cache 2 | 3 | import com.datastax.driver.core.Row 4 | import com.datastax.driver.core.querybuilder.QueryBuilder 5 | import com.kaltura.model.dao.PartnerDAO 6 | import com.kaltura.model.entities.Partner 7 | import com.kaltura.core.utils.ReadableTimeUnits._ 8 | 9 | /** 10 | * Created by ofirk on 07/02/2016. 11 | */ 12 | sealed class PartnerCache extends CacheBase[Partner, Int]{ 13 | 14 | override val tableName = "dim_partners" 15 | override val idFieldName = "id" 16 | override def fromRow(row: Row) = if (row != null) Some(Partner(row.getInt("id"), Option(row.getString("secret")), Option(row.getString("crm_id")), Option(row.getInt("package_id")))) else None 17 | val ttl = 1 day 18 | 19 | def getById(id: Int) : Partner = { 20 | findById(id).getOrElse { 21 | println("Global cache miss") 22 | val partner = PartnerDAO.getById(id).getOrElse(Partner(id)) 23 | cassandraSession.execute(QueryBuilder 24 | .insertInto(keySpace, tableName) 25 | .value("id", partner.id) 26 | .value("secret", partner.secret.orNull) 27 | .value("crm_id", partner.crmId.orNull) 28 | .value("package_id", partner.packageId.orNull) 29 | .using(QueryBuilder.ttl(ttl)) 30 | ) 31 | partner 32 | } 33 | } 34 | } 35 | 36 | object PartnerCache extends PartnerCache 37 | -------------------------------------------------------------------------------- /kanalony-model/src/main/scala/com/kaltura/model/events/EnrichedPlayerEvent.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.model.events 2 | 3 | import com.kaltura.core.ip2location.Location 4 | import com.kaltura.core.urls.UrlParts 5 | import com.kaltura.core.userAgent.UserAgent 6 | import org.joda.time.DateTime 7 | 8 | case class EnrichedPlayerEvent(eventType: String, 9 | eventTime: DateTime, 10 | partnerId: Int, 11 | entryId: String, 12 | flavourId: String = "", 13 | userId: String = "", 14 | location: Location = null, 15 | userAgent: UserAgent = null, 16 | urlParts: UrlParts = null, 17 | kalsig: String = "", 18 | categories: String = "", 19 | application: String = "", 20 | playbackContext: String = "", 21 | playbackType: String = "", 22 | customVar1: String = "", 23 | customVar2: String = "", 24 | customVar3: String = "" 25 | ) 26 | -------------------------------------------------------------------------------- /kanalony-storage-access/src/main/scala/kanalony/storage/generator/EntityClassGenerator.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.generator 2 | 3 | import com.google.common.base.CaseFormat 4 | 5 | /** 6 | * Created by elad.benedict on 2/7/2016. 7 | */ 8 | class EntityClassGenerator(val tableMetadata: TableMetadata) { 9 | 10 | private def generateClassColumns(columns: List[IColumnDefinition]) : String = { 11 | columns.map { col => s"${EntityClassGenerator.getParamName(col.name)}:${col.typeName}" }.mkString(",\n") 12 | } 13 | 14 | def generate() = { 15 | val entityTemplate = GenerationTemplates.entityClassTemplate; 16 | val entityTemplateContent = entityTemplate.content 17 | val columns = tableMetadata.columns 18 | val generatedColumnsDefs = generateClassColumns(columns) 19 | entityTemplateContent 20 | .replace(entityTemplate.classNamePlaceholder, EntityClassGenerator.getEntityName(tableMetadata)) 21 | .replace(entityTemplate.colDefsPlaceholder, generatedColumnsDefs) 22 | } 23 | } 24 | 25 | object EntityClassGenerator { 26 | def getEntityName(tm : TableMetadata): String ={ 27 | CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, tm.tableName) + GenerationTemplates.entityClassTemplate.classNameSuffix 28 | } 29 | 30 | def getParamName(columnName : ColumnNames.Value): String = { 31 | CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.LOWER_CAMEL, columnName.toString) 32 | } 33 | } -------------------------------------------------------------------------------- /kanalony-core/src/main/java/com/kaltura/core/ip2location/DbAttribtues.java: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.ip2location; 2 | import java.io.IOException; 3 | import java.io.RandomAccessFile; 4 | 5 | 6 | public class DbAttribtues { 7 | 8 | protected final int type; 9 | protected final int column; 10 | protected final int year; 11 | protected final int month; 12 | protected final int day; 13 | protected final long count; 14 | protected final long addr; 15 | protected final long ipVersion; 16 | 17 | public DbAttribtues(RandomAccessFile f) throws IOException { 18 | this.type = f.readUnsignedByte(); 19 | this.column = f.readUnsignedByte(); 20 | this.year = f.readUnsignedByte(); 21 | this.month = f.readUnsignedByte(); 22 | this.day = f.readUnsignedByte(); 23 | this.count = Utils.readUInteger(f); 24 | this.addr = Utils.readUInteger(f); 25 | this.ipVersion = Utils.readUInteger(f); 26 | } 27 | 28 | public int getType() { 29 | return type; 30 | } 31 | 32 | public int getColumn() { 33 | return column; 34 | } 35 | 36 | public int getYear() { 37 | return year; 38 | } 39 | 40 | public int getMonth() { 41 | return month; 42 | } 43 | 44 | public int getDay() { 45 | return day; 46 | } 47 | 48 | public long getCount() { 49 | return count; 50 | } 51 | 52 | public long getAddr() { 53 | return addr; 54 | } 55 | 56 | public long getIpVersion() { 57 | return ipVersion; 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /kanalony-storage-access-generated/src/main/scala/kanalony/storage/generated/EntryTableAccessor.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.generated 2 | 3 | import com.websudos.phantom.CassandraTable 4 | import com.websudos.phantom.dsl._ 5 | 6 | import scala.concurrent.Future 7 | 8 | /** 9 | * Created by orlylampert on 12/12/16. 10 | */ 11 | abstract class EntryTableAccessor extends CassandraTable[EntryTableAccessor, EntryRow] with RootConnector { 12 | 13 | object id extends StringColumn(this)with PartitionKey[String] 14 | object name extends StringColumn(this) 15 | object categories extends OptionalStringColumn(this) 16 | 17 | 18 | 19 | override def tableName = "dim_entries_metadata" 20 | 21 | def fromRow(row: Row): EntryRow = { 22 | EntryRow( 23 | id(row), 24 | name(row) 25 | ) 26 | } 27 | 28 | def store(entity: EntryRow): Future[ResultSet] = { 29 | insert.value(_.id, entity.entryId) 30 | .value(_.name, entity.entryName) 31 | .future() 32 | } 33 | 34 | def query(entryId : String) : Future[List[EntryRow]] = { 35 | select.where(_.id eqs entryId) 36 | .fetch()(session, scala.concurrent.ExecutionContext.Implicits.global, space) 37 | } 38 | 39 | def query(entryIdList : List[String]) : Future[List[EntryRow]] = { 40 | select.where(_.id in entryIdList) 41 | .fetch()(session, scala.concurrent.ExecutionContext.Implicits.global, space) 42 | } 43 | 44 | } 45 | 46 | case class EntryRow(entryId:String, 47 | entryName:String) 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /kanalony-storage-logic/src/main/scala/kanalony/storage/logic/ComputedDimensions.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | import com.kaltura.model.entities.{Metric, AggregationKind, Metrics} 4 | import kanalony.storage.logic.queries.{DailyMaxQuery, DailyCountQuery} 5 | 6 | /** 7 | * Created by elad.benedict on 3/7/2016. 8 | */ 9 | 10 | trait IComputedDimensions { 11 | def getQueryCreator(value: Dimensions.Value): (QueryParams) => List[(IQuery, List[Metric])] 12 | def values : Set[Dimensions.Value] 13 | } 14 | 15 | object ComputedDimensions extends ComputedQueryFactory[Dimensions.Value] with IComputedDimensions { 16 | 17 | val queryCreatorGetter = Map((Dimensions.day, dailyQueryCreator)) 18 | 19 | def dailyQueryCreator : (QueryParams) => List[(IQuery, List[Metric])] = { 20 | (qp) => { 21 | var res : List[(IQuery, List[Metric])] = List() 22 | val dailyMaxMetrics = qp.metrics.filter(_.aggregationKind == AggregationKind.Max) 23 | if (dailyMaxMetrics.nonEmpty) 24 | { 25 | res = res :+ (new DailyMaxQuery(qp, QueryLocator), dailyMaxMetrics) 26 | } 27 | 28 | val dailyCountMetrics = qp.metrics.filter(_.aggregationKind == AggregationKind.Sum) 29 | if (dailyCountMetrics.nonEmpty) 30 | { 31 | res = res :+ (new DailyCountQuery(qp, QueryLocator), dailyCountMetrics) 32 | } 33 | 34 | res 35 | } 36 | } 37 | 38 | override def getErrorMessage(value: Dimensions.Value): String = s"Computed dimension ${value} is currently not supported" 39 | 40 | } -------------------------------------------------------------------------------- /kanalony-storage-logic/src/test/scala/kanalony/storage/logic/AverageViewDropOffQueryTests.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | import com.kaltura.model.entities.{Metrics, Metric} 4 | import kanalony.storage.logic.queries.{AverageViewDropOffQuery, ComputedQuery, EstimatedMinutesWatchedQuery} 5 | 6 | /** 7 | * Created by elad.benedict on 4/10/2016. 8 | */ 9 | class AverageViewDropOffQueryTests extends ComputedMetricTestsBase { 10 | override protected def createComputedMetricQuery(): (QueryParams, IQueryLocator) => ComputedQuery = (qp, ql) => new AverageViewDropOffQuery(qp, ql) 11 | 12 | override val dependentMetricsResultSet: Map[Metric, List[IQueryResult]] = { 13 | Map(Metrics.play -> List(QueryResult(List("partner","entry", "play"), List(List("1","2","16"),List("1","1","4")))), 14 | Metrics.playThrough25 -> List(QueryResult(List("partner","entry", "playThrough25"), List(List("1","2","16"),List("1","1","4")))), 15 | Metrics.playThrough50 -> List(QueryResult(List("partner","entry", "playThrough50"), List(List("1","2","12"),List("1","1","3")))), 16 | Metrics.playThrough75 -> List(QueryResult(List("partner","entry", "playThrough75"), List(List("1","2","8"),List("1","1","2")))), 17 | Metrics.playThrough100 -> List(QueryResult(List("partner","entry", "playThrough100"), List(List("1","2","4"),List("1","1","1"))))) 18 | 19 | } 20 | override val expectedQueryResult: QueryResult = QueryResult(List("partner","entry","averageViewDropOff"), List(List("1","2","0.625"),List("1","1","0.625"))) 21 | } 22 | -------------------------------------------------------------------------------- /kanalony-core/src/main/scala/com/kaltura/core/streaming/StreamManager.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.streaming 2 | 3 | import java.util 4 | 5 | import kafka.serializer.StringDecoder 6 | import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, Producer} 7 | import org.apache.spark.streaming.StreamingContext 8 | import org.apache.spark.streaming.dstream.InputDStream 9 | import org.apache.spark.streaming.kafka.KafkaUtils 10 | 11 | object StreamManager { 12 | 13 | def createStream(ssc: StreamingContext, kafkaBrokers: String, topics: Set[String]): InputDStream[(String, String)] = { 14 | val kafkaParams = Map[String, String]("metadata.broker.list" -> kafkaBrokers) 15 | createStream(ssc, topics, kafkaParams) 16 | } 17 | 18 | def createStream(ssc: StreamingContext, topics: Set[String], kafkaParams: Map[String, String]): InputDStream[(String, String)] = { 19 | KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder]( 20 | ssc, kafkaParams, topics) 21 | } 22 | 23 | 24 | def createProducer(kafkaBrokers:String): Producer[String, String] = { 25 | val props = new util.HashMap[String, Object]() 26 | props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBrokers) 27 | props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, 28 | "org.apache.kafka.common.serialization.StringSerializer") 29 | props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, 30 | "org.apache.kafka.common.serialization.StringSerializer") 31 | 32 | new KafkaProducer[String, String](props) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /kanalony-storage-access/build.sbt: -------------------------------------------------------------------------------- 1 | import sbt.classpath.ClasspathUtilities 2 | 3 | lazy val generateStorageAccessTask = taskKey[AnyRef]("Generates storage accessors") 4 | lazy val `kanalony-storage-access` = (project in file(".")). 5 | settings( 6 | name := "kanalony-storage-access", 7 | version := "1.0", 8 | scalaVersion := "2.11.7", 9 | libraryDependencies ++= Seq( 10 | "com.datastax.cassandra" % "cassandra-driver-core" % "3.0.2", 11 | "com.websudos" %% "phantom-dsl" % "1.22.0", 12 | "com.websudos" %% "phantom-testkit" % "1.12.2", 13 | "com.typesafe" % "config" % "1.3.0" 14 | ), 15 | resolvers ++= Seq( 16 | "Typesafe repository snapshots" at "http://repo.typesafe.com/typesafe/snapshots/", 17 | "Typesafe repository releases" at "http://repo.typesafe.com/typesafe/releases/", 18 | "Sonatype repo" at "https://oss.sonatype.org/content/groups/scala-tools/", 19 | "Sonatype releases" at "https://oss.sonatype.org/content/repositories/releases", 20 | "Sonatype snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", 21 | "Sonatype staging" at "http://oss.sonatype.org/content/repositories/staging", 22 | "Java.net Maven2 Repository" at "http://download.java.net/maven/2/", 23 | "Twitter Repository" at "http://maven.twttr.com", 24 | "Websudos releases" at "https://dl.bintray.com/websudos/oss-releases/" 25 | ) 26 | ) -------------------------------------------------------------------------------- /kanalony-core/src/main/java/com/kaltura/core/ip2location/SerializableIP2LocationReader.java: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.ip2location; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.io.Externalizable; 7 | import java.io.IOException; 8 | import java.io.ObjectInput; 9 | import java.io.ObjectOutput; 10 | 11 | 12 | public class SerializableIP2LocationReader implements Externalizable { 13 | 14 | private String fileName; 15 | private Ip2LocationReader reader; 16 | 17 | private static Logger LOG = LoggerFactory.getLogger(SerializableIP2LocationReader.class); 18 | 19 | public SerializableIP2LocationReader() { 20 | fileName = "test"; 21 | } 22 | 23 | public SerializableIP2LocationReader(String fileName) throws IOException { 24 | this.fileName = fileName; 25 | reader = new Ip2LocationReader(); 26 | reader.init(fileName); 27 | } 28 | 29 | public void close() { 30 | try { 31 | reader.close(); 32 | } catch (IOException e) { 33 | // TODO Auto-generated catch block 34 | LOG.error("Failed while closing", e); 35 | } 36 | } 37 | 38 | @Override 39 | public void readExternal(ObjectInput in) throws IOException, 40 | ClassNotFoundException { 41 | fileName = (String) in.readObject(); 42 | reader = new Ip2LocationReader(); 43 | reader.init(fileName); 44 | 45 | } 46 | 47 | @Override 48 | public void writeExternal(ObjectOutput out) throws IOException { 49 | out.writeObject(fileName); 50 | 51 | } 52 | 53 | public Ip2LocationRecord getAll(String ip) throws IOException { 54 | return reader.getAll(ip); 55 | } 56 | 57 | 58 | } 59 | -------------------------------------------------------------------------------- /kanalony-core/src/main/scala/com/kaltura/core/ip2location/LocationResolver.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.ip2location 2 | 3 | import java.io.Serializable 4 | 5 | import com.kaltura.core.logging.{BaseLog, MetaLog} 6 | import com.kaltura.core.utils.ConfigurationManager 7 | import org.apache.commons.validator.Validator 8 | import org.apache.commons.validator.routines.InetAddressValidator 9 | 10 | /** 11 | * Created by ofirk on 06/10/2015. 12 | */ 13 | class LocationResolver extends Serializable with MetaLog[BaseLog] { 14 | val reader = new SerializableIP2LocationReader(ConfigurationManager.get("kanalony.core.ip2location.data_filepath")) 15 | val inetAddressValidator = InetAddressValidator.getInstance() 16 | 17 | def parse(ipAddr: String): Location = { 18 | try { 19 | val ipRecord: Ip2LocationRecord = reader.getAll(ipAddr) 20 | Location(repair(ipRecord.getCountryLong), repair(ipRecord.getRegion), repair(ipRecord.getCity)) 21 | } 22 | catch { 23 | case e: Exception => { 24 | logger.debug("Failed to parse IP '" + ipAddr + "' exp: " + e.printStackTrace()) // maybe add exception description 25 | } 26 | Location() 27 | } 28 | } 29 | 30 | def parseWithProxy(remoteAddr: String, proxyAddr: String): Location = { 31 | if (inetAddressValidator.isValid(proxyAddr)) 32 | parse(proxyAddr) 33 | else if (inetAddressValidator.isValid(remoteAddr)) 34 | parse(remoteAddr) 35 | else 36 | Location() 37 | } 38 | 39 | private def repair(locationValue: String) = if (locationValue == "-") "N/A" else locationValue 40 | 41 | def close() = reader.close() 42 | 43 | } 44 | -------------------------------------------------------------------------------- /kanalony-core/src/main/scala/com/kaltura/core/utils/KalturaAPI.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.utils 2 | 3 | import com.kaltura.client.enums.KalturaSessionType 4 | import com.kaltura.client.{KalturaClient, KalturaConfiguration} 5 | import com.kaltura.core.logging.{BaseLog, MetaLog} 6 | import scala.util.control.NonFatal 7 | 8 | 9 | class KalturaImpersonationClient(config:KalturaConfiguration) extends KalturaClient(config) { 10 | 11 | def removePartnerId() = { 12 | this.requestConfiguration.remove("partnerId") 13 | } 14 | } 15 | 16 | object KalturaAPIFactory extends MetaLog[BaseLog] { 17 | logger.info("Initializing Kaltura API") 18 | val config = new KalturaConfiguration() 19 | config.setEndpoint(ConfigurationManager.getOrElse("kanalony.core.kaltura_api.endpoint","https://www.kaltura.com/")) 20 | config.setTimeout(ConfigurationManager.getOrElse("kanalony.core.kaltura_api.timeout","2000").toInt) 21 | val internalClient = new KalturaClient(config) 22 | var ks: String = _ 23 | resetKS 24 | logger.info("Kaltura API Initialized!") 25 | 26 | def getClient: KalturaImpersonationClient = { 27 | val client = new KalturaImpersonationClient(config) 28 | client.setKs(ks) 29 | client 30 | } 31 | 32 | def resetKS:String = { 33 | synchronized { 34 | try { 35 | ks = internalClient.getSessionService.start(ConfigurationManager.getOrElse("kanalony.events_enrichment.admin_partner_secret", ""), "batchUser", KalturaSessionType.ADMIN, -1, 86400, "disableentitlement") 36 | } catch { 37 | case NonFatal(e) => logger.error("Error creating KS!", e) 38 | } 39 | } 40 | ks 41 | } 42 | 43 | 44 | } -------------------------------------------------------------------------------- /kanalony-logstash/config/logstash_nginx_log_input.conf.local: -------------------------------------------------------------------------------- 1 | ################################################################### 2 | # Logstash configuration file for kAnalony events collection # 3 | # Author: Ofir Kerker # 4 | # Date: December 23, 2015 # 5 | ################################################################### 6 | 7 | input { 8 | file { 9 | path => ["/usr/local/var/log/nginx/access.log*"] 10 | exclude => "*.gz" 11 | start_position => "beginning" 12 | sincedb_path => "/var/log/kanalony/.sincedb_player" 13 | sincedb_write_interval => 1 14 | } 15 | } 16 | 17 | filter { 18 | grok { 19 | match => [ "message" , "%{COMBINEDAPACHELOG} %{NOTSPACE:proxyRemoteAddr}"] 20 | overwrite => [ "message" ] 21 | } 22 | 23 | date { 24 | match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ] 25 | remove_field => [ "timestamp" ] 26 | target => "eventTime" 27 | } 28 | 29 | mutate { 30 | remove_field => [ "message", "path", "type", "ident", "auth", "verb", "httpversion", "response", "bytes", "referrer" ] 31 | rename => { "clientip" => "remoteAddr"} 32 | rename => { "agent" => "userAgent"} 33 | } 34 | 35 | urldecode { 36 | field => "request" 37 | } 38 | 39 | kv { 40 | source => "request" 41 | field_split => "&?" 42 | target => "params" 43 | } 44 | } 45 | 46 | output { 47 | kafka { 48 | bootstrap_servers => "localhost:9092" 49 | topic_id => "player-events" 50 | acks => "1" 51 | batch_size => "2000" 52 | message_key => "%{params[event:entryId]}" 53 | # codec => rubydebug 54 | } 55 | # for debug: 56 | stdout { codec => rubydebug } 57 | } -------------------------------------------------------------------------------- /kanalony-logstash/config/logstash_nginx_log_input.conf.staging: -------------------------------------------------------------------------------- 1 | ################################################################### 2 | # Logstash configuration file for kAnalony events collection # 3 | # Author: Ofir Kerker # 4 | # Date: December 23, 2015 # 5 | ################################################################### 6 | 7 | input { 8 | file { 9 | path => ["/var/log/nginx/access.log*"] 10 | exclude => "*.gz" 11 | start_position => "beginning" 12 | sincedb_path => "/var/log/kanalony/.sincedb_player" 13 | sincedb_write_interval => 1 14 | } 15 | } 16 | 17 | filter { 18 | grok { 19 | match => [ "message" , "%{COMBINEDAPACHELOG}"] 20 | overwrite => [ "message" ] 21 | } 22 | 23 | date { 24 | match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ] 25 | remove_field => [ "timestamp" ] 26 | target => "eventTime" 27 | } 28 | 29 | mutate { 30 | remove_field => [ "message", "path", "type", "ident", "auth", "verb", "httpversion", "response", "bytes", "referrer" ] 31 | rename => { "clientip" => "remoteIp"} 32 | rename => { "agent" => "userAgent"} 33 | } 34 | 35 | urldecode { 36 | field => "request" 37 | } 38 | 39 | kv { 40 | source => "request" 41 | field_split => "&?" 42 | target => "params" 43 | } 44 | } 45 | 46 | output { 47 | kafka { 48 | bootstrap_servers => "il-bigdata-1:9092,il-bigdata-2:9092,il-bigdata-3:9092" 49 | topic_id => "player-events" 50 | acks => "1" 51 | batch_size => "2000" 52 | message_key => "%{params[event:entryId]}" 53 | # codec => rubydebug 54 | } 55 | # for debug: 56 | # stdout { codec => rubydebug } 57 | } -------------------------------------------------------------------------------- /kanalony-core/src/main/scala/com/kaltura/core/userAgent/UserAgentResolver.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.userAgent 2 | 3 | import java.util.concurrent.TimeUnit 4 | 5 | import com.google.common.cache.{CacheLoader, CacheBuilder} 6 | import com.kaltura.core.userAgent.enums.{Device, OperatingSystem, Browser} 7 | 8 | 9 | /** 10 | * Resolves browser and operating system from user agent string 11 | */ 12 | object UserAgentResolver extends Serializable { 13 | 14 | private val maxCacheSize = 100000 15 | private val expireAfterAccess: (Long, TimeUnit) = (1L, TimeUnit.HOURS) 16 | private val cache = buildCache 17 | 18 | /** 19 | * Returns and UserAgent class describing the browser and operating system corresponding to the provided user-agent string 20 | * @param ua 21 | * @return 22 | */ 23 | def resolve(ua:String): UserAgent = { 24 | val userAgent = cache.get(ua) 25 | val operatingSystem = userAgent.getOperatingSystem 26 | 27 | UserAgent(Browser.values.find(_.toString == userAgent.getBrowser.getName).getOrElse(Browser.UNKNOWN), 28 | OperatingSystem.values.find(_.toString == operatingSystem.getName).getOrElse(OperatingSystem.UNKNOWN), 29 | Device.values.find(_.toString == operatingSystem.getDeviceType.getName).getOrElse(Device.UNKNOWN)) 30 | } 31 | 32 | private def buildCache = { 33 | CacheBuilder.newBuilder() 34 | .maximumSize(maxCacheSize) 35 | .expireAfterAccess(expireAfterAccess._1, expireAfterAccess._2) 36 | .build( 37 | new CacheLoader[String ,eu.bitwalker.useragentutils.UserAgent]() { 38 | override def load(ua: String) = eu.bitwalker.useragentutils.UserAgent.parseUserAgentString(ua) 39 | } 40 | ) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /kanalony-storage-access-generated/build.sbt: -------------------------------------------------------------------------------- 1 | import sbt.classpath.ClasspathUtilities 2 | 3 | lazy val generateStorageAccessTask = taskKey[AnyRef]("Generates storage accessors") 4 | lazy val `kanalony-storage-access` = RootProject(file("../kanalony-storage-access")) 5 | lazy val `kanalony-storage-access-generated` = (project in file(".")). 6 | settings( 7 | name := "kanalony-storage-access-generated", 8 | version := "1.0", 9 | scalaVersion := "2.11.7", 10 | libraryDependencies ++= Seq( 11 | "com.datastax.cassandra" % "cassandra-driver-core" % "3.0.2", 12 | "com.websudos" %% "phantom-dsl" % "1.22.0", 13 | "com.websudos" %% "phantom-testkit" % "1.12.2", 14 | "com.typesafe" % "config" % "1.3.0" 15 | ), 16 | resolvers ++= Seq( 17 | "Typesafe repository snapshots" at "http://repo.typesafe.com/typesafe/snapshots/", 18 | "Typesafe repository releases" at "http://repo.typesafe.com/typesafe/releases/", 19 | "Sonatype repo" at "https://oss.sonatype.org/content/groups/scala-tools/", 20 | "Sonatype releases" at "https://oss.sonatype.org/content/repositories/releases", 21 | "Sonatype snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", 22 | "Sonatype staging" at "http://oss.sonatype.org/content/repositories/staging", 23 | "Java.net Maven2 Repository" at "http://download.java.net/maven/2/", 24 | "Twitter Repository" at "http://maven.twttr.com", 25 | "Websudos releases" at "https://dl.bintray.com/websudos/oss-releases/" 26 | ) 27 | ).dependsOn(`kanalony-storage-access`) -------------------------------------------------------------------------------- /kanalony-web-server/conf/application.conf: -------------------------------------------------------------------------------- 1 | # This is the main configuration file for the application. 2 | # ~~~~~ 3 | 4 | # Secret key 5 | # ~~~~~ 6 | # The secret key is used to secure cryptographics functions. 7 | # 8 | # This must be changed for production, but we recommend not changing it in this file. 9 | # 10 | # See http://www.playframework.com/documentation/latest/ApplicationSecret for more details. 11 | play.crypto.secret = "changeme" 12 | 13 | # The application languages 14 | # ~~~~~ 15 | play.i18n.langs = [ "en" ] 16 | 17 | # Cassandra configuration 18 | config.cassandra.hosts = ["192.168.161.48"] 19 | config.cassandra.port = 9042 20 | config.cassandra.keyspace = "kanalony_agg" 21 | 22 | # Router 23 | # ~~~~~ 24 | # Define the Router object to use for this application. 25 | # This router will be looked up first when the application is starting up, 26 | # so make sure this is the entry point. 27 | # Furthermore, it's assumed your route file is named properly. 28 | # So for an application router like `my.application.Router`, 29 | # you may need to define a router file `conf/my.application.routes`. 30 | # Default to Routes in the root package (and conf/routes) 31 | # play.http.router = my.application.Routes 32 | 33 | # Database configuration 34 | # ~~~~~ 35 | # You can declare as many datasources as you want. 36 | # By convention, the default datasource is named `default` 37 | # 38 | # db.default.driver=org.h2.Driver 39 | # db.default.url="jdbc:h2:mem:play" 40 | # db.default.username=sa 41 | # db.default.password="" 42 | 43 | # Evolutions 44 | # ~~~~~ 45 | # You can disable evolutions if needed 46 | # play.evolutions.enabled=false 47 | 48 | # You can disable evolutions for a specific datasource if necessary 49 | # play.evolutions.db.default.enabled=false 50 | -------------------------------------------------------------------------------- /kanalony-enrichment/build.sbt: -------------------------------------------------------------------------------- 1 | lazy val sparkVersion = "2.0.0" 2 | lazy val json4sVersion = "3.2.10" 3 | lazy val `kanalony-model` = RootProject(file("../kanalony-model")) 4 | lazy val `kanalony-core` = RootProject(file("../kanalony-core")) 5 | lazy val `kanalony-enrichment` = (project in file(".")). 6 | settings( 7 | name := "kanalony-enrichment", 8 | version := "1.0", 9 | scalaVersion := "2.11.7", 10 | libraryDependencies ++= Seq( 11 | "org.apache.spark" %% "spark-core" % sparkVersion, 12 | "org.apache.spark" %% "spark-streaming" % sparkVersion, 13 | "org.apache.spark" %% "spark-streaming-kafka-0-8" % sparkVersion, 14 | "org.json4s" %% "json4s-jackson" % json4sVersion, 15 | "org.json4s" %% "json4s-native" % json4sVersion, 16 | "org.json4s" %% "json4s-ext" % json4sVersion, 17 | "io.dropwizard.metrics" % "metrics-core" % "3.1.2", 18 | "joda-time" % "joda-time" % "2.9.1", 19 | "com.google.guava" % "guava" % "18.0", 20 | "org.apache.hadoop" % "hadoop-aws" % "2.7.1", 21 | "com.amazonaws" % "aws-java-sdk" % "1.7.4", 22 | 23 | // Test 24 | "org.scalatest" %% "scalatest" % "2.2.4" % "test" 25 | ) 26 | ).dependsOn(`kanalony-model`, `kanalony-core`) 27 | 28 | assemblyMergeStrategy in assembly := { 29 | case PathList("META-INF", xs @ _*) => MergeStrategy.discard 30 | case x => MergeStrategy.first 31 | } 32 | assemblyJarName := "kanalony-enrichment.jar" 33 | assemblyOutputPath := file("../out/" + assemblyJarName) 34 | 35 | -------------------------------------------------------------------------------- /kanalony-web-server/test/load/DataApiLoadTest.scala: -------------------------------------------------------------------------------- 1 | package kanalony.tests.load 2 | 3 | /** 4 | * Created by elad.benedict on 3/30/2016. 5 | */ 6 | 7 | import io.gatling.core.Predef._ 8 | import io.gatling.http.Predef._ 9 | import scala.concurrent.duration._ 10 | 11 | class DataApiLoadTest extends Simulation { 12 | 13 | val webServerHostname = "il-bigdata-4.dev.kaltura.com" 14 | val queryUrl = s"http://${webServerHostname}:9000/query" 15 | val queryData = """{"from":"1","to":"1487265988000","filters":[{"dimension":"partner","values":["1"]},{"dimension":"entry","values":["1_Entry_1","1_Entry_5"]}],"dimensions":["day","partner","country"],"metrics":["play"]}""" 16 | 17 | val httpConf = http 18 | .contentTypeHeader("application/json") 19 | .doNotTrackHeader("1") 20 | 21 | val scn = scenario("DataApiLoadTest").exec( 22 | http("BasicQuery") 23 | .post(queryUrl) 24 | .body(StringBody(queryData)) 25 | //.check(bodyString.saveAs("responseData")) 26 | ).exec(session => { 27 | //val ResBody = session.get("responseData").asOption[String] 28 | //println(maybeId.getOrElse("ERROR GETTING RESPONSE DATA")) 29 | session 30 | }) 31 | 32 | setUp( 33 | scn.inject( 34 | constantUsersPerSec(100) during (30 seconds), 35 | constantUsersPerSec(200) during (30 seconds), 36 | constantUsersPerSec(300) during (30 seconds), 37 | constantUsersPerSec(400) during (30 seconds), 38 | constantUsersPerSec(500) during (30 seconds), 39 | constantUsersPerSec(600) during (30 seconds), 40 | constantUsersPerSec(700) during (30 seconds), 41 | constantUsersPerSec(800) during (30 seconds), 42 | constantUsersPerSec(900) during (30 seconds), 43 | constantUsersPerSec(1000) during (30 seconds) 44 | ).protocols(httpConf)) 45 | } -------------------------------------------------------------------------------- /kanalony-core/src/main/scala/com/kaltura/core/urls/UrlParser.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.urls 2 | 3 | import scala.collection.mutable.ArrayBuffer 4 | import scala.util.Try 5 | 6 | /** 7 | * Parses URLs 8 | */ 9 | object UrlParser { 10 | 11 | case class QueryStringKeyValuePair(key: String, value: String) 12 | 13 | def parseUrl(url: String) : Array[QueryStringKeyValuePair] = { 14 | url.split("\\?", 2) match { 15 | case Array(_, qs: String) => parseQueryString(decodeUrl(qs)) 16 | case _ => null 17 | } 18 | } 19 | 20 | def parseQueryString(queryString: String) : Array[QueryStringKeyValuePair] = { 21 | val pairs = new ArrayBuffer[QueryStringKeyValuePair] 22 | val pairSplits = queryString.split("&") 23 | for (pairSplit <- pairSplits) { 24 | val keyValue = pairSplit.split("=", 2) 25 | pairs += QueryStringKeyValuePair(decodeUrl(keyValue(0)), if(keyValue.length >1) decodeUrl(keyValue(1)) else null) 26 | } 27 | pairs.toArray 28 | } 29 | 30 | def getUrlParts(url: String) : UrlParts = { 31 | try { 32 | val uri = new java.net.URI(url.replaceAll(" ","%20")) 33 | UrlParts( defaultIfEmpty(uri.getHost), 34 | defaultIfEmpty(uri.getAuthority,uri.getPath), 35 | defaultIfEmpty(url)) 36 | } catch { 37 | case e: Exception => println(s"Error parsing url: $url, error: ${e.toString}"); UrlParts("N/A","N/A","N/A") 38 | } 39 | } 40 | 41 | def decodeUrl(url: String) = Try(java.net.URLDecoder.decode(url, "UTF-8")).getOrElse("") 42 | 43 | def defaultIfEmpty(urlPart:String) = if(urlPart == null || urlPart.isEmpty) "N/A" else urlPart 44 | def defaultIfEmpty(urlPart1:String, urlPart2:String) = if(urlPart1 == null || urlPart1.isEmpty) "N/A" else if (urlPart2 == null) urlPart1 else urlPart1 + urlPart2 45 | } 46 | -------------------------------------------------------------------------------- /kanalony-logstash/src/test/scala/utils/KafkaConsumer.scala: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import java.util.Properties 4 | import java.util.concurrent.atomic.AtomicLong 5 | 6 | import kafka.consumer.{Consumer, ConsumerConfig, Whitelist} 7 | import kafka.serializer.DefaultDecoder 8 | import org.joda.time.DateTime 9 | 10 | /** 11 | * Created by elad.benedict on 3/31/2016. 12 | */ 13 | 14 | class KafkaConsumer { 15 | 16 | val topic = "player-events" 17 | val groupId = "1" 18 | val zookeeperConnect = "il-bigdata-2:2181" 19 | val readFromStartOfStream = true 20 | 21 | val props = new Properties() 22 | props.put("group.id", groupId) 23 | props.put("zookeeper.connect", zookeeperConnect) 24 | props.put("auto.offset.reset", if(readFromStartOfStream) "smallest" else "largest") 25 | props.put("mirror.consumer.numthreads", "10") 26 | 27 | val config = new ConsumerConfig(props) 28 | val connector = Consumer.create(config) 29 | val filterSpec = new Whitelist(topic) 30 | 31 | val messageNum : AtomicLong = new AtomicLong(0) 32 | var lastSecNum : Long = 0 33 | var lastSec = 0 34 | val start = DateTime.now() 35 | 36 | val stream = connector.createMessageStreamsByFilter(filterSpec, 1, new DefaultDecoder(), new DefaultDecoder()).toList.head 37 | 38 | def read(write: (String)=>Unit) = { 39 | for(messageAndTopic <- stream) { 40 | //write("messageAndTopic.offset.toString + " " + new String(messageAndTopic.message, "UTF-8")) 41 | val length = messageNum.incrementAndGet() 42 | val now = DateTime.now() 43 | if (now.getSecondOfMinute != lastSec) 44 | { 45 | synchronized { 46 | println(s"${now.toLocalTime} ::: ${length - lastSecNum}") 47 | lastSecNum = length 48 | lastSec = now.getSecondOfMinute 49 | } 50 | } 51 | } 52 | } 53 | 54 | def close() { 55 | connector.shutdown() 56 | } 57 | } -------------------------------------------------------------------------------- /kanalony-storage-logic/src/main/scala/kanalony/storage/logic/ComputedMetrics.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | import com.kaltura.model.entities.{Metric, Metrics} 4 | import kanalony.storage.logic.queries.{AverageViewDropOffQuery, AverageTimeViewedQuery, EstimatedMinutesWatchedQuery, PlayRatioQuery} 5 | 6 | trait IComputedMetrics { 7 | def getQueryCreator(m : Metric) : (QueryParams) => List[(IQuery, List[Metric])] 8 | def values : Set[Metric] 9 | } 10 | 11 | object ComputedMetrics extends ComputedQueryFactory[Metric] with IComputedMetrics { 12 | 13 | val queryCreatorGetter = Map((Metrics.playRatio, playRatioQueryCreator), 14 | (Metrics.estimatedMinutesWatched, estimatedMinutesWatchedQueryCreator), 15 | (Metrics.averageViewDuration, averageViewDurationQueryCreator), 16 | (Metrics.averageViewDropOff, averageViewDropOffQueryCreator)) 17 | 18 | def playRatioQueryCreator: (QueryParams) => List[(IQuery, List[Metric])] = { 19 | (qp) => List((new PlayRatioQuery(qp, QueryLocator), List(Metrics.playRatio))) 20 | } 21 | 22 | def estimatedMinutesWatchedQueryCreator: (QueryParams) => List[(IQuery, List[Metric])] = { 23 | (qp) => List((new EstimatedMinutesWatchedQuery(qp, QueryLocator), List(Metrics.estimatedMinutesWatched))) 24 | } 25 | 26 | def averageViewDurationQueryCreator: (QueryParams) => List[(IQuery, List[Metric])] = { 27 | (qp) => List((new AverageTimeViewedQuery(qp, QueryLocator), List(Metrics.averageViewDuration))) 28 | } 29 | 30 | def averageViewDropOffQueryCreator: (QueryParams) => List[(IQuery, List[Metric])] = { 31 | (qp) => List((new AverageViewDropOffQuery(qp, QueryLocator), List(Metrics.averageViewDropOff))) 32 | } 33 | 34 | override def getErrorMessage(value: Metric): String = s"Computed metric ${value} is currently not supported" 35 | } 36 | -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/kanalony/storage/logic/Dimensions.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | import com.datastax.spark.connector.ColumnName 4 | import kanalony.storage.generator.ColumnNames 5 | 6 | object Dimensions extends Enumeration { 7 | val partner,entry,knownUserId,device,operatingSystem,browser, 8 | country,city,syndicationDomain,syndicationURL,application, 9 | category,playbackContext,month,day,hour,minute,tenSeconds,streamingProtocol, 10 | expectedQuality,uiConfID,metric,cf1,cf2,cf3,referrer = Value 11 | 12 | implicit def fromColumnName(name : ColumnNames.Value) : Dimensions.Value = name match { 13 | case ColumnNames.tensecs => Dimensions.tenSeconds 14 | case ColumnNames.application => Dimensions.application 15 | case ColumnNames.category => Dimensions.category 16 | case ColumnNames.browser => Dimensions.browser 17 | case ColumnNames.`custom_var1` => Dimensions.cf1 18 | case ColumnNames.`custom_var2` => Dimensions.cf2 19 | case ColumnNames.`custom_var3` => Dimensions.cf3 20 | case ColumnNames.city => Dimensions.city 21 | case ColumnNames.country => Dimensions.country 22 | case ColumnNames.device => Dimensions.device 23 | case ColumnNames.domain => Dimensions.syndicationDomain 24 | case ColumnNames.entry_id => Dimensions.entry 25 | case ColumnNames.hour => Dimensions.hour 26 | case ColumnNames.metric => Dimensions.metric 27 | case ColumnNames.minute => Dimensions.minute 28 | case ColumnNames.month => Dimensions.month 29 | case ColumnNames.`operating_system` => Dimensions.operatingSystem 30 | case ColumnNames.partner_id => Dimensions.partner 31 | case ColumnNames.`playback_context`=> Dimensions.playbackContext 32 | case ColumnNames.referrer => Dimensions.referrer 33 | case other => throw new IllegalArgumentException(s"column ${other} has no corresponding dimension") 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kAnalony Project 2 | 3 | ## Goals 4 | The purpose of the kAnalony project is to create a robust and scalable infrastructure to accommodate the existing and future Analytical needs of Kaltura: 5 | 6 | * Aggregate event-based time-series data using predefined configurable dimensions on predefined time resolutions 7 | * Custom batch processing on aggregated data 8 | * Custom batch processing on raw data 9 | * Expose raw events data for preferred clients 10 | 11 | ## Background 12 | There are currently 3 different Analytics systems in Kaltura: 13 | 14 | 1. Comprehensive metrics for VOD data, based on Pentaho's Kettle open source project and MySQL 15 | 2. Basic metrics for Live entries (including DVR) data, based on Apache Spark and Cassandra 16 | 3. Comprehensive metrics for OTT data, based on Sisense product 17 | 18 | The new kAnalony project will create a scalable and robust infrastructure to facilitate the requirements of all current 3 systems. 19 | It is based on [Apache Spark](https://spark.apache.org/), [Cassandra](http://cassandra.apache.org/) and [Kafka](http://kafka.apache.org/), 20 | written mostly with [Scala](http://www.scala-lang.org/). 21 | 22 | ## Why kAnalony? 23 | The project name is a combination of the words Kaltura and Analytics, 24 | and sound like the word Cannelloni which is a cylindrical type of pasta generally served baked with a filling and covered by a sauce in Italian cuisine (Source: [Wikipedia](http://en.wikipedia.org/wiki/Cannelloni)). 25 | 26 | ## Components 27 | * **kAnalony Receivers** - a group of Node.js servers behind a load balancer, which receive analytics events and push a transformation of them to a Kafka cluster. 28 | 29 | ## Installation 30 | 31 | * [Kafka Cluster Installation](https://kaltura.atlassian.net/wiki/display/KANAL/Kafka+Cluster+Installation) 32 | * [Spark-Cassandra Cluster Installation](https://kaltura.atlassian.net/wiki/display/KANAL/Spark-Cassandra+Cluster+Installation) -------------------------------------------------------------------------------- /kanalony-model/src/main/scala/com/kaltura/model/entities/PlayerEventTypes.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.model.entities 2 | 3 | /** 4 | * Created by ofirk on 29/03/2016. 5 | */ 6 | object PlayerEventTypes extends Enumeration { 7 | val unknown = Value(0,"unknown") 8 | val playerImpression = Value(1,"playerImpression") 9 | val playRequested = Value(2,"playRequested") 10 | val play = Value(3,"play") 11 | val resume = Value(4,"resume") 12 | val playReached25 = Value(11,"playThrough25") 13 | val playReached50 = Value(12,"playThrough50") 14 | val playReached75 = Value(13,"playThrough75") 15 | val playReached100 = Value(14,"playThrough100") 16 | val shareClicked = Value(21,"shareClicked") 17 | val shared = Value(22,"shared") 18 | val downloadClicked = Value(23,"downloadClicked") 19 | val reportClicked = Value(24,"reportClicked") 20 | val reportSubmitted = Value(25,"reportSubmitted") 21 | val enterFullscreen = Value(31,"enterFullscreen") 22 | val exitFullscreen = Value(32,"exitFullscreen") 23 | val pauseClicked = Value(33,"pauseClicked") 24 | val replay = Value(34,"replay") 25 | val seek = Value(35,"seek") 26 | val relatedClicked = Value(36,"relatedClicked") 27 | val relatedSelected = Value(37,"relatedSelected") 28 | val captions = Value(38,"captions") 29 | val sourceSelected = Value(39,"sourceSelected") 30 | val info = Value(40,"info") 31 | val speed = Value(41,"speed") 32 | val view = Value(99,"view") 33 | 34 | def apply(eventType:String): PlayerEventTypes.Value = { 35 | val eventTypeInt = 36 | try { 37 | Some(eventType.toInt) 38 | } catch { 39 | case e: Exception => None 40 | } 41 | PlayerEventTypes.values.find(_.id == eventTypeInt.getOrElse(0)).getOrElse(PlayerEventTypes.unknown) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/AggregationByPartner.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import com.datastax.spark.connector.{SomeColumns, _} 4 | import com.kaltura.aggregations.keys.AggregationKey 5 | import com.kaltura.model.events.EnrichedPlayerEvent 6 | import org.joda.time.DateTime 7 | import com.kaltura.core.utils.ReadableDateUnits.ReadableDateUnits 8 | 9 | /** 10 | * Created by orlylampert on 3/2/16. 11 | */ 12 | abstract class AggregationByPartner extends BaseAggregation[AggregationKey, PartnerRes] with IAggregate with Serializable{ 13 | val columns: List[(String, String)] = List[(String, String)]( 14 | ("partner_id","partner_id"), 15 | ("metric","metric"), 16 | (getAggrTimeUnit,"time"), 17 | ("value","value")) 18 | 19 | 20 | override def aggKey(e: EnrichedPlayerEvent): AggregationKey = AggregationKey(e.partnerId, e.eventType, getAggrTime(e.eventTime)) 21 | override def toRow(pair: (AggregationKey, Long)): PartnerRes = PartnerRes(partnerId = pair._1.partnerId, metric = pair._1.metric, year = pair._1.time.getYear, day = pair._1.time.getYearMonthDay, time = pair._1.time, value = pair._2) 22 | 23 | } 24 | 25 | 26 | object HourlyAggregationByPartner extends AggregationByPartner with IAggregateHourly { 27 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 28 | "hourly_agg" -> toSomeColumns(columns :+ ("year", "year")) 29 | ) 30 | } 31 | 32 | object MinutelyAggregationByPartner extends AggregationByPartner with IAggregateMinutely { 33 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 34 | "minutely_agg" -> toSomeColumns(columns :+ ("day", "day")) 35 | ) 36 | } 37 | 38 | object TenSecsAggregationByPartner extends AggregationByPartner with IAggregateTenSecs { 39 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 40 | "tensecs_agg" -> toSomeColumns(columns :+ ("day", "day")) 41 | ) 42 | } 43 | 44 | case class PartnerRes(partnerId: Int, metric: String, year: Int, day: Int, time: DateTime, value: Long) 45 | -------------------------------------------------------------------------------- /kanalony-model/src/main/scala/com/kaltura/model/events/PlayerEventParser.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.model.events 2 | 3 | import java.text.SimpleDateFormat 4 | 5 | import com.kaltura.core.logging.{BaseLog, MetaLog} 6 | import com.kaltura.core.userAgent.enums.{Browser, Device, OperatingSystem} 7 | import org.json4s.DefaultFormats 8 | import org.json4s.ext.EnumNameSerializer 9 | import org.json4s.jackson.JsonMethods._ 10 | import org.json4s.jackson.Serialization._ 11 | 12 | object PlayerEventParser extends MetaLog[BaseLog] { 13 | implicit val formats = new DefaultFormats { 14 | override def dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'") 15 | } ++ org.json4s.ext.JodaTimeSerializers.all + 16 | new EnumNameSerializer(Browser) + 17 | new EnumNameSerializer(OperatingSystem) + 18 | new EnumNameSerializer(Device) 19 | 20 | def parsePlayerEvent(playerEvent: String): Option[RawPlayerEvent] = { 21 | try { 22 | val row:AccessLogRow = parse(playerEvent).extract[AccessLogRow] 23 | Some(RawPlayerEvent(row.eventTime, 24 | row.remoteAddr, 25 | row.proxyRemoteAddr, 26 | row.userAgent, 27 | row.params)) 28 | } 29 | catch { 30 | case e: Exception => { 31 | logger.warn(s"Unable to parse log row: $playerEvent, error: $e") 32 | None 33 | } 34 | } 35 | } 36 | 37 | def parseEnhancedPlayerEvent(playerEvent: String): Option[EnrichedPlayerEvent] = { 38 | try { 39 | Some(parse(playerEvent).extract[EnrichedPlayerEvent]) 40 | } 41 | catch { 42 | case e: Exception => { 43 | logger.warn(s"Unable to enriched player event: $playerEvent, error: $e") 44 | None 45 | } 46 | } 47 | } 48 | 49 | 50 | def asJson(playerEvent: RawPlayerEvent): String = { 51 | write(playerEvent) 52 | } 53 | 54 | def asJson(enrichedPlayerEvent: EnrichedPlayerEvent) : String = { 55 | write(enrichedPlayerEvent) 56 | } 57 | 58 | } 59 | 60 | 61 | -------------------------------------------------------------------------------- /kanalony-storage-logic/src/test/scala/kanalony/storage/logic/DailyCountQueryTests.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | import com.kaltura.model.entities.Metrics 4 | import kanalony.storage.logic.queries.DailyCountQuery 5 | import kanalony.storage.test.logic.queries.DailyQueryTestsBase 6 | import org.joda.time.{LocalDateTime, DateTime} 7 | 8 | /** 9 | * Created by elad.benedict on 4/6/2016. 10 | */ 11 | 12 | class DailyCountQueryTests extends DailyQueryTestsBase[DailyCountQuery] { 13 | it("Should aggregate data by day")({ 14 | 15 | val metricQueryStub = stub[IQuery] 16 | (metricQueryStub.query(_)). 17 | when(*). 18 | returns(createCompletedFuture(List(QueryResult(List("partner", "entry", "hour", "play"), 19 | List(List("1","1", new DateTime(2016,1,1,1,1).toString(),"3"), 20 | List("1","1", new DateTime(2016,1,1,1,2).toString(),"4"), 21 | List("1","1", new DateTime(2016,1,2,1,2).toString(),"5")))))) 22 | 23 | val params = QueryParams(List(createPartnerDimensionDefintion(Set(1)), createEntryDimensionDefintion(Set("1")), createDayDimensionDefintion()), List(Metrics.play), new LocalDateTime(1), new LocalDateTime(2)) 24 | val expectedParams = QueryParams(List(createPartnerDimensionDefintion(Set(1)), createEntryDimensionDefintion(Set("1")), createHourDimensionDefintion()), List(Metrics.play), new LocalDateTime(1), new LocalDateTime(2)) 25 | configureStub(expectedParams, List((metricQueryStub, List(Metrics.play)))) // The response is meaningless for this test - just use some IQuery 26 | val dailyQuery = dailyQueryCreatorWithStub(params) 27 | whenReady(dailyQuery.query(params)){ 28 | res => assert(res == List(QueryResult(List("partner","entry","day","play"),List(List("1","1",new DateTime(2016,1,2,1,1).toLocalDate.toString(),"5.0"), List("1","1",new DateTime(2016,1,1,1,1).toLocalDate.toString(),"7.0"))))) 29 | } 30 | }) 31 | 32 | override def createDailyQuery(queryParams: QueryParams, queryLocator: IQueryLocator): DailyCountQuery = { 33 | new DailyCountQuery(queryParams, queryLocator) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /kanalony-storage-logic/src/test/scala/kanalony/storage/logic/DailyMaxQueryTests.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic 2 | 3 | import com.kaltura.model.entities.Metrics 4 | import kanalony.storage.logic.queries.DailyMaxQuery 5 | import kanalony.storage.test.logic.queries.DailyQueryTestsBase 6 | import org.joda.time.{LocalDateTime, DateTime} 7 | 8 | /** 9 | * Created by elad.benedict on 4/7/2016. 10 | */ 11 | class DailyMaxQueryTests extends DailyQueryTestsBase[DailyMaxQuery] { 12 | it("Should aggregate data by day")({ 13 | 14 | val metricQueryStub = stub[IQuery] 15 | (metricQueryStub.query(_)). 16 | when(*). 17 | returns(createCompletedFuture(List(QueryResult(List("partner", "entry", "hour", "peakView"), 18 | List(List("1","1", new DateTime(2016,1,1,1,1).toString(),"3"), 19 | List("1","1", new DateTime(2016,1,1,1,2).toString(),"4"), 20 | List("1","1", new DateTime(2016,1,2,1,2).toString(),"5")))))) 21 | 22 | val params = QueryParams(List(createPartnerDimensionDefintion(Set(1)), createEntryDimensionDefintion(Set("1")), createDayDimensionDefintion()), List(Metrics.peakView), new LocalDateTime(1), new LocalDateTime(2)) 23 | val expectedParams = QueryParams(List(createPartnerDimensionDefintion(Set(1)), createEntryDimensionDefintion(Set("1")), createHourDimensionDefintion()), List(Metrics.peakView), new LocalDateTime(1), new LocalDateTime(2)) 24 | configureStub(expectedParams, List((metricQueryStub, List(Metrics.peakView)))) // The response is meaningless for this test - just use some IQuery 25 | val dailyQuery = dailyQueryCreatorWithStub(params) 26 | whenReady(dailyQuery.query(params)){ 27 | res => assert(res == List(QueryResult(List("partner","entry","day","peakView"),List(List("1","1",new DateTime(2016,1,2,1,1).toLocalDate.toString(),"5.0"), List("1","1",new DateTime(2016,1,1,1,1).toLocalDate.toString(),"4.0"))))) 28 | } 29 | }) 30 | 31 | override def createDailyQuery(queryParams: QueryParams, queryLocator: IQueryLocator): DailyMaxQuery = { 32 | new DailyMaxQuery(queryParams, queryLocator) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /kanalony-aggregations/build.sbt: -------------------------------------------------------------------------------- 1 | lazy val sparkVersion = "2.0.0" 2 | lazy val `kanalony-model` = RootProject(file("../kanalony-model")) 3 | lazy val `kanalony-core` = RootProject(file("../kanalony-core")) 4 | lazy val `kanalony-storage-access` = RootProject(file("../kanalony-storage-access")) 5 | lazy val `kanalony-aggregations` = (project in file(".")). 6 | settings( 7 | name := "kanalony-aggregations", 8 | version := "1.0", 9 | scalaVersion := "2.11.7", 10 | resolvers += Resolver.jcenterRepo, 11 | libraryDependencies ++= Seq( 12 | "org.clapper" %% "classutil" % "1.0.6", 13 | "org.apache.spark" %% "spark-core" % sparkVersion, 14 | "org.apache.spark" %% "spark-streaming" % sparkVersion, 15 | "org.apache.spark" %% "spark-streaming-kafka-0-8" % sparkVersion, 16 | "org.apache.spark" %% "spark-sql" % sparkVersion, 17 | "com.datastax.cassandra" % "cassandra-driver-core" % "3.0.2", 18 | "com.datastax.spark" %% "spark-cassandra-connector" % "2.0.0-M1", 19 | "joda-time" % "joda-time" % "2.8.2", 20 | "org.apache.hadoop" % "hadoop-aws" % "2.7.1", 21 | "com.amazonaws" % "aws-java-sdk" % "1.7.4", 22 | "de.javakaffee" % "kryo-serializers" % "0.37", 23 | // Test 24 | "org.scalatest" %% "scalatest" % "2.2.4" % "test" 25 | ) 26 | ).dependsOn(`kanalony-model`, `kanalony-core`, `kanalony-storage-access`) 27 | 28 | // There is a conflict between Guava versions on Cassandra Drive and Hadoop 29 | // Shading Guava Package 30 | assemblyShadeRules in assembly := Seq( 31 | ShadeRule.rename("com.google.**" -> "shadeio.@1").inAll 32 | ) 33 | 34 | assemblyMergeStrategy in assembly := { 35 | case PathList("META-INF", xs @ _*) => MergeStrategy.discard 36 | case x => MergeStrategy.first 37 | } 38 | assemblyJarName := "kanalony-aggregations.jar" 39 | assemblyOutputPath := file("../out/" + assemblyJarName) 40 | 41 | -------------------------------------------------------------------------------- /kanalony-receiver/lib/eventsRelay.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var fs = require('fs'), 4 | url = require('url'), 5 | path = require('path'), 6 | logger = require('./logger')(module), 7 | config = require('./configurationUtil'), 8 | timeUtil = require('./timeUֹtil'), 9 | KafkaProducer = require('./kafkaProducer'); 10 | 11 | var eventsRelay = function(zkConnectionString, topic){ 12 | this.producer = new KafkaProducer(zkConnectionString); 13 | this.topic = topic; 14 | this.fallbackPath = config.getOrElse('kanalony.receiver.fallback.path','/tmp/fallback'); 15 | }; 16 | 17 | eventsRelay.prototype.isRequestValid = function(req) { 18 | // TODO - validate the request params to make sure this is a stats request 19 | return true; 20 | }; 21 | 22 | eventsRelay.prototype.produceEvent = function(req) { 23 | var payload = this.buildPayload(req); 24 | var that = this; 25 | this.producer.send(payload, function(err){ 26 | if(err) { 27 | var filePath = path.join(that.fallbackPath, 'erroneous-events_' + timeUtil.currentDateTimeAsMinuteString() + '.log'); 28 | fs.appendFile(filePath, JSON.stringify(payload), function(err){ 29 | if(err) { 30 | logger.error("Unable to write event to fallback storage " + filePath + ', ' + err + '\n' + JSON.stringify(payload)); 31 | } 32 | }); 33 | } 34 | }); 35 | }; 36 | 37 | eventsRelay.prototype.buildPayload = function(req) { 38 | var payload = [{ topic: this.topic, messages: this.buildMessages(req) }]; 39 | logger.debug("payload:", payload); 40 | return payload; 41 | }; 42 | 43 | eventsRelay.prototype.buildMessages = function(req) { 44 | var params = {}; 45 | var remoteIp = ""; 46 | try { 47 | params = url.parse(req.url, true).query; 48 | remoteIp = req.connection.remoteAddress; 49 | } 50 | catch(err){} 51 | return [JSON.stringify({ eventTime: timeUtil.currentDateTimeAsISOString(), userAgent: req.headers["user-agent"], remoteIp: remoteIp, referrer: req.headers["referer"], params: params })]; 52 | }; 53 | 54 | module.exports = eventsRelay; 55 | -------------------------------------------------------------------------------- /kanalony-logstash/src/test/scala/load/EventParsingLoadTest.scala: -------------------------------------------------------------------------------- 1 | package load 2 | 3 | /** 4 | * Created by elad.benedict on 3/30/2016. 5 | */ 6 | 7 | import io.gatling.core.Predef._ 8 | import io.gatling.http.Predef._ 9 | 10 | import scala.concurrent.duration._ 11 | 12 | class EventParsingLoadTest extends Simulation { 13 | 14 | val webServerHostname = "il-bigdata-4.dev.kaltura.com" 15 | val queryUrl = s"http://${webServerHostname}/api_v3/index.php?service=stats&action=collect&kalsig=1ebb5aea0c5f253fd8c578febe6a752f&clientTag=kdp%3Av3%2E9%2E2&event%3AobjectType=KalturaStatsEvent&event%3AsessionId=C90BFCFC%2D2EBF%2D5893%2D892D%2D2121162F414A&apiVersion=3%2E1%2E5&event%3AisFirstInSession=false&event%3Aduration=194&ignoreNull=1&event%3AeventType=13&event%3Aseek=false&event%3Areferrer=http%253A%2F%2Fabc%2Ego%2Ecom%2Fshows%2Fthe%2Dbachelorette%2Fvideo%2Fmost%2Drecent%2FVDKA0%5Flawz79v7&event%3AentryId=0%5Fn5nbuy6i&ks=djJ8MTA5MXwjFUzIifsgEbq-qlc8DhLFAVhV-xecSlzm_smRiTZabayc9bKahRKZhSjuPhDTFc-mHKD1uUd6PhcowDdW6TtDw1ozzDuqD2GlWH-Y4XiQTQ%3D%3D&event%3AeventTimestamp=1435075182567&event%3AuiconfId=23521211&event%3ApartnerId=1091&event%3AcurrentPoint=18&event%3AclientVer=3%2E0%3Av3%2E9%2E2-13" 16 | 17 | val httpConf = http.doNotTrackHeader("1") 18 | 19 | val scn = scenario("EventParsingLoadTest").exec( 20 | http("BasicQuery") 21 | .get(queryUrl) 22 | //.check(bodyString.saveAs("responseData")) 23 | ).exec(session => { 24 | //val ResBody = session.get("responseData").asOption[String] 25 | //println(maybeId.getOrElse("ERROR GETTING RESPONSE DATA")) 26 | session 27 | }) 28 | 29 | setUp(scn.inject( 30 | constantUsersPerSec(100) during (30 seconds), 31 | constantUsersPerSec(500) during (30 seconds), 32 | constantUsersPerSec(1000) during (30 seconds), 33 | constantUsersPerSec(1500) during (30 seconds), 34 | constantUsersPerSec(2000) during (30 seconds), 35 | constantUsersPerSec(3000) during (30 seconds), 36 | constantUsersPerSec(4000) during (30 seconds), 37 | constantUsersPerSec(4500) during (30 seconds), 38 | constantUsersPerSec(5500) during (30 seconds), 39 | constantUsersPerSec(6500) during (30 seconds) 40 | ).protocols(httpConf)) 41 | } -------------------------------------------------------------------------------- /kanalony-storage-logic-core/src/main/scala/queryGenerator/ParamsTypeGenerator.scala: -------------------------------------------------------------------------------- 1 | package queryGenerator 2 | 3 | import com.google.common.base.CaseFormat 4 | import kanalony.storage.generator._ 5 | import org.joda.time.DateTime 6 | 7 | object ParamsTypeGenerator { 8 | 9 | def apply() = new ParamsTypeGenerator() 10 | def getClassName(tableName : String) = s"${CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, tableName)}QueryParams" 11 | 12 | def getParamName(colDef : IColumnExtendedDefinition) = { 13 | if (colDef.inferred) 14 | { 15 | RowBreakerFactory.getBreaker(colDef.name).partitionKeyParamName 16 | } 17 | else { 18 | QueryMethodsGenerator.getListParamName(colDef) 19 | } 20 | } 21 | } 22 | 23 | case class ImplicitColumnInferrer(cols : List[IColumnExtendedDefinition]) { 24 | def implementation: String = { 25 | if (cols.isEmpty) { 26 | return "" 27 | } 28 | 29 | if (cols.length != 1) 30 | { 31 | throw new IllegalArgumentException("Only a single inferred column is supported") 32 | } 33 | 34 | val implementingTraitName = RowBreakerFactory.getBreaker(cols.head.name).implementingTrait 35 | s"extends ${implementingTraitName}" 36 | } 37 | } 38 | 39 | class ParamsTypeGenerator() { 40 | val paramsClassTemplate = "case class %QUERY_PARAMS_CLASS_NAME%(startTime : DateTime, endTime : DateTime, %EQ_COL_DEFS%) %IMPLICIT_VALUES_INFERRER%" 41 | val colDefsPlaceholder = "%EQ_COL_DEFS%" 42 | val classNamePlaceholder = "%QUERY_PARAMS_CLASS_NAME%" 43 | val implicitValueImplementerPlaceholder = "%IMPLICIT_VALUES_INFERRER%" 44 | 45 | def generate(tableName : String, colDefs : List[IColumnExtendedDefinition]) = { 46 | val explicitColumns = colDefs.filter(c => !(c.inferred)).map( c => new QueryableColumnDefinition(c.name, c.typeName, ColumnQueryKind.List, c.inPartitionKey, c.inClusteringKey)) 47 | val implicitColumns = colDefs.filter(_.inferred) 48 | val generatedColDefs = QueryMethodsGenerator.generateColDefs(explicitColumns) 49 | 50 | paramsClassTemplate 51 | .replace(colDefsPlaceholder, generatedColDefs) 52 | .replace(classNamePlaceholder, ParamsTypeGenerator.getClassName(tableName)) 53 | .replace(implicitValueImplementerPlaceholder, ImplicitColumnInferrer(implicitColumns).implementation) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/AggregationByDomain.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import com.datastax.spark.connector.{SomeColumns, _} 4 | import com.kaltura.aggregations.keys.AggregationDomainKey 5 | import com.kaltura.model.events.EnrichedPlayerEvent 6 | import org.joda.time.DateTime 7 | import com.kaltura.core.utils.ReadableDateUnits.ReadableDateUnits 8 | 9 | abstract class AggregationByDomain extends BaseAggregation[AggregationDomainKey, PartnerDomainRes] with IAggregate with Serializable { 10 | 11 | val columns: List[(String, String)] = List[(String, String)]( 12 | ("partner_id","partner_id"), 13 | ("domain","domain"), 14 | ("metric","metric"), 15 | (getAggrTimeUnit,"time"), 16 | ("value","value")) 17 | 18 | override def aggKey(e: EnrichedPlayerEvent): AggregationDomainKey = AggregationDomainKey(e.partnerId, e.eventType, getAggrTime(e.eventTime), e.urlParts.domain) 19 | override def toRow(pair: (AggregationDomainKey, Long)): PartnerDomainRes = PartnerDomainRes(partnerId = pair._1.partnerId, domain = pair._1.domain, metric = pair._1.metric, year = pair._1.time.getYear, day = pair._1.time.getYearMonthDay, time = pair._1.time, value = pair._2) 20 | } 21 | 22 | object HourlyAggregationByDomain extends AggregationByDomain with IAggregateHourly { 23 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 24 | "hourly_agg_prtn_domain" -> toSomeColumns(columns :+ ("year", "year")), 25 | "hourly_agg_clst_domain" -> toSomeColumns(columns :+ ("year", "year")) 26 | ) 27 | } 28 | 29 | object MinutelyAggregationByDomain extends AggregationByDomain with IAggregateMinutely { 30 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 31 | "minutely_agg_prtn_domain" -> toSomeColumns(columns :+ ("day", "day")), 32 | "minutely_agg_clst_domain" -> toSomeColumns(columns :+ ("day", "day")) 33 | ) 34 | } 35 | 36 | object TenSecsAggregationByDomain extends AggregationByDomain with IAggregateTenSecs { 37 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 38 | "tensecs_agg_prtn_domain" -> toSomeColumns(columns :+ ("day", "day")), 39 | "tensecs_agg_clst_domain" -> toSomeColumns(columns :+ ("day", "day")) 40 | ) 41 | } 42 | case class PartnerDomainRes(partnerId: Int, metric: String, year: Int, day: Int, time: DateTime, domain: String, value: Long) 43 | 44 | -------------------------------------------------------------------------------- /kanalony-receiver/lib/logger.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var winston = require('winston'); 4 | var config = require('./configurationUtil'); 5 | 6 | winston.emitErrs = true; 7 | var path = require('path'); 8 | var mkdirp = require('mkdirp'); 9 | 10 | var logPath = config.getOrElse('kanalony.receiver.log_file_path','kanalony-receiver.log'); 11 | var logFullPath = path.resolve('.', logPath); 12 | 13 | mkdirp.sync(path.dirname(logFullPath)); 14 | 15 | var getLabel = function(callingModule) { 16 | var parts = callingModule.filename.split('/'); 17 | return parts[parts.length - 2] + '/' + parts.pop(); 18 | }; 19 | 20 | var logger = function (callingModule) { 21 | var loggerTimestamp = function() { 22 | return new Date().toUTCString(); 23 | }; 24 | var loggerFormatter = function(options) { 25 | return '['+ options.level.toUpperCase() +'] '+ options.timestamp() +': ' + options.label + ' - ' + (undefined !== options.message ? options.message : '') + 26 | (options.meta && Object.keys(options.meta).length ? '\n\t'+ JSON.stringify(options.meta) : '' ); 27 | }; 28 | return new winston.Logger({ 29 | transports: [ 30 | new (winston.transports.File)({ 31 | name: 'info-file', 32 | filename: logPath, 33 | level: config.get('kanalony.receiver.log_level.file') || config.getOrElse('kanalony.receiver.log_level','info'), 34 | json: false, 35 | colorize: false, 36 | handleExceptions: false, 37 | humanReadableUnhandledException: true, 38 | label: getLabel(callingModule), 39 | timestamp: loggerTimestamp, 40 | formatter: loggerFormatter 41 | }), 42 | new winston.transports.Console({ 43 | level: config.get('kanalony.receiver.log_level.console') || config.getOrElse('kanalony.receiver.log_level','info'), 44 | handleExceptions: false, 45 | json: false, 46 | colorize: true, 47 | label: getLabel(callingModule), 48 | humanReadableUnhandledException: true, 49 | timestamp: loggerTimestamp, 50 | formatter: loggerFormatter 51 | }) 52 | ], 53 | exitOnError: false 54 | }); 55 | }; 56 | module.exports = logger; -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/AggregationByCustomVar3.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import com.datastax.spark.connector.{SomeColumns, _} 4 | import com.kaltura.aggregations.keys.AggregationCustomVarKey 5 | import com.kaltura.model.events.EnrichedPlayerEvent 6 | import org.joda.time.DateTime 7 | import com.kaltura.core.utils.ReadableDateUnits.ReadableDateUnits 8 | 9 | 10 | abstract class AggregationByCustomVar3 extends BaseAggregation[AggregationCustomVarKey, PartnerCustomVar3Res] with IAggregate with Serializable{ 11 | 12 | 13 | val columns: List[(String, String)] = List[(String, String)]( 14 | ("partner_id","partner_id"), 15 | ("custom_var3","cv3"), 16 | ("metric","metric"), 17 | (getAggrTimeUnit,"time"), 18 | ("value","value")) 19 | 20 | override def aggKey(e: EnrichedPlayerEvent): AggregationCustomVarKey = AggregationCustomVarKey(e.partnerId, e.eventType, getAggrTime(e.eventTime), e.customVar3) 21 | override def toRow(pair: (AggregationCustomVarKey, Long)): PartnerCustomVar3Res = PartnerCustomVar3Res(pair._1.partnerId, pair._1.cv, pair._1.metric, pair._1.time.getYear, pair._1.time.getYearMonthDay, pair._1.time, pair._2) 22 | 23 | 24 | } 25 | 26 | object HourlyAggregationByCustomVar3 extends AggregationByCustomVar3 with IAggregateHourly { 27 | 28 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 29 | "hourly_agg_prtn_cv3" -> toSomeColumns(columns :+ ("year", "year")), 30 | "hourly_agg_clst_cv3" -> toSomeColumns(columns :+ ("year", "year")) 31 | ) 32 | } 33 | 34 | object MinutelyAggregationByCustomVar3 extends AggregationByCustomVar3 with IAggregateMinutely { 35 | 36 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 37 | "minutely_agg_prtn_cv3" -> toSomeColumns(columns :+ ("day", "day")), 38 | "minutely_agg_clst_cv3" -> toSomeColumns(columns :+ ("day", "day")) 39 | ) 40 | } 41 | 42 | object TenSecsAggregationByCustomVar3 extends AggregationByCustomVar3 with IAggregateTenSecs { 43 | 44 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 45 | "tensecs_agg_prtn_cv3" -> toSomeColumns(columns :+ ("day", "day")), 46 | "tensecs_agg_clst_cv3" -> toSomeColumns(columns :+ ("day", "day")) 47 | ) 48 | } 49 | 50 | case class PartnerCustomVar3Res(partnerId: Int, cv3: String, metric: String, year: Int, day: Int, time: DateTime, value: Long) 51 | -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/AggregationByDevice.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import com.datastax.spark.connector.{SomeColumns, _} 4 | import com.kaltura.aggregations.keys.AggregationDeviceKey 5 | import com.kaltura.model.events.EnrichedPlayerEvent 6 | import org.joda.time.DateTime 7 | import com.kaltura.core.utils.ReadableDateUnits.ReadableDateUnits 8 | 9 | 10 | abstract class AggregationByDevice extends BaseAggregation[AggregationDeviceKey, PartnerDeviceRes] with IAggregate with Serializable { 11 | 12 | val columns: List[(String, String)] = List[(String, String)]( 13 | ("partner_id","partner_id"), 14 | ("device","device"), 15 | ("metric","metric"), 16 | (getAggrTimeUnit,"time"), 17 | ("value","value")) 18 | 19 | override def aggKey(e: EnrichedPlayerEvent): AggregationDeviceKey = AggregationDeviceKey(e.partnerId, e.eventType, getAggrTime(e.eventTime), e.userAgent.device.id) 20 | override def toRow(pair: (AggregationDeviceKey, Long)): PartnerDeviceRes = PartnerDeviceRes(partnerId = pair._1.partnerId, device = pair._1.device, metric = pair._1.metric, year = pair._1.time.getYear, day = pair._1.time.getYearMonthDay, time = pair._1.time, value = pair._2) 21 | 22 | 23 | } 24 | 25 | object HourlyAggregationByDevice extends AggregationByDevice with IAggregateHourly { 26 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 27 | "hourly_agg_prtn_device" -> toSomeColumns(columns :+ ("year", "year")), 28 | "hourly_agg_clst_device" -> toSomeColumns(columns :+ ("year", "year")) 29 | ) 30 | } 31 | 32 | object MinutelyAggregationByDevice extends AggregationByDevice with IAggregateMinutely { 33 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 34 | "minutely_agg_prtn_device" -> toSomeColumns(columns :+ ("day", "day")), 35 | "minutely_agg_clst_device" -> toSomeColumns(columns :+ ("day", "day")) 36 | ) 37 | } 38 | 39 | object TenSecsAggregationByDevice extends AggregationByDevice with IAggregateTenSecs { 40 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 41 | "tensecs_agg_prtn_device" -> toSomeColumns(columns :+ ("day", "day")), 42 | "tensecs_agg_clst_device" -> toSomeColumns(columns :+ ("day", "day")) 43 | ) 44 | } 45 | case class PartnerDeviceRes(partnerId: Int, metric: String, year: Int, day: Int, time: DateTime, device: Int, value: Long) 46 | 47 | -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/AggregationByCustomVar1.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import com.datastax.spark.connector.{SomeColumns, _} 4 | import com.kaltura.aggregations.keys.AggregationCustomVarKey 5 | import com.kaltura.model.events.EnrichedPlayerEvent 6 | import org.joda.time.DateTime 7 | import com.kaltura.core.utils.ReadableDateUnits.ReadableDateUnits 8 | 9 | 10 | 11 | abstract class AggregationByCustomVar1 extends BaseAggregation[AggregationCustomVarKey, PartnerCustomVar1Res] with IAggregate with Serializable{ 12 | 13 | 14 | val columns: List[(String, String)] = List[(String, String)]( 15 | ("partner_id","partner_id"), 16 | ("custom_var1","cv1"), 17 | ("metric","metric"), 18 | (getAggrTimeUnit,"time"), 19 | ("value","value")) 20 | 21 | override def aggKey(e: EnrichedPlayerEvent): AggregationCustomVarKey = AggregationCustomVarKey(e.partnerId, e.eventType, getAggrTime(e.eventTime), e.customVar1) 22 | override def toRow(pair: (AggregationCustomVarKey, Long)): PartnerCustomVar1Res = PartnerCustomVar1Res(pair._1.partnerId, pair._1.cv, pair._1.metric, pair._1.time.getYear, pair._1.time.getYearMonthDay, pair._1.time, pair._2) 23 | 24 | 25 | } 26 | 27 | object HourlyAggregationByCustomVar1 extends AggregationByCustomVar1 with IAggregateHourly { 28 | 29 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 30 | "hourly_agg_prtn_cv1" -> toSomeColumns(columns :+ ("year", "year")), 31 | "hourly_agg_clst_cv1" -> toSomeColumns(columns :+ ("year", "year")) 32 | ) 33 | } 34 | 35 | object MinutelyAggregationByCustomVar1 extends AggregationByCustomVar1 with IAggregateMinutely { 36 | 37 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 38 | "minutely_agg_prtn_cv1" -> toSomeColumns(columns :+ ("day", "day")), 39 | "minutely_agg_clst_cv1" -> toSomeColumns(columns :+ ("day", "day")) 40 | ) 41 | } 42 | 43 | object TenSecsAggregationByCustomVar1 extends AggregationByCustomVar1 with IAggregateTenSecs { 44 | 45 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 46 | "tensecs_agg_prtn_cv1" -> toSomeColumns(columns :+ ("day", "day")), 47 | "tensecs_agg_clst_cv1" -> toSomeColumns(columns :+ ("day", "day")) 48 | ) 49 | } 50 | 51 | case class PartnerCustomVar1Res(partnerId: Int, cv1: String, metric: String, year: Int, day: Int, time: DateTime, value: Long) 52 | -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/AggregationByCustomVar2.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import com.datastax.spark.connector.{SomeColumns, _} 4 | import com.kaltura.aggregations.keys.AggregationCustomVarKey 5 | import com.kaltura.model.events.EnrichedPlayerEvent 6 | import org.joda.time.DateTime 7 | import com.kaltura.core.utils.ReadableDateUnits.ReadableDateUnits 8 | 9 | 10 | abstract class AggregationByCustomVar2 extends BaseAggregation[AggregationCustomVarKey, PartnerCustomVar2Res] with IAggregate with Serializable{ 11 | 12 | 13 | val columns: List[(String, String)] = List[(String, String)]( 14 | ("partner_id","partner_id"), 15 | ("custom_var2","cv2"), 16 | ("metric","metric"), 17 | (getAggrTimeUnit,"time"), 18 | ("value","value")) 19 | 20 | override def aggKey(e: EnrichedPlayerEvent): AggregationCustomVarKey = AggregationCustomVarKey(e.partnerId, e.eventType, getAggrTime(e.eventTime), e.customVar2) 21 | override def toRow(pair: (AggregationCustomVarKey, Long)): PartnerCustomVar2Res = PartnerCustomVar2Res(pair._1.partnerId, pair._1.cv, pair._1.metric, pair._1.time.getYear, day = pair._1.time.getYearMonthDay, pair._1.time, pair._2) 22 | 23 | 24 | } 25 | 26 | object HourlyAggregationByCustomVar2 extends AggregationByCustomVar2 with IAggregateHourly { 27 | 28 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 29 | "hourly_agg_prtn_cv2" -> toSomeColumns(columns :+ ("year", "year")), 30 | "hourly_agg_clst_cv2" -> toSomeColumns(columns :+ ("year", "year")) 31 | ) 32 | } 33 | 34 | object MinutelyAggregationByCustomVar2 extends AggregationByCustomVar2 with IAggregateMinutely { 35 | 36 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 37 | "minutely_agg_prtn_cv2" -> toSomeColumns(columns :+ ("day", "day")), 38 | "minutely_agg_clst_cv2" -> toSomeColumns(columns :+ ("day", "day")) 39 | ) 40 | } 41 | 42 | object TenSecsAggregationByCustomVar2 extends AggregationByCustomVar2 with IAggregateTenSecs { 43 | 44 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 45 | "tensecs_agg_prtn_cv2" -> toSomeColumns(columns :+ ("day", "day")), 46 | "tensecs_agg_clst_cv2" -> toSomeColumns(columns :+ ("day", "day")) 47 | ) 48 | } 49 | 50 | case class PartnerCustomVar2Res(partnerId: Int, cv2: String, metric: String, year: Int, day: Int, time: DateTime, value: Long) 51 | -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/AggregationByEntry.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import com.datastax.spark.connector.{SomeColumns, _} 4 | import com.kaltura.aggregations.keys.AggregationEntryKey 5 | import com.kaltura.model.events.EnrichedPlayerEvent 6 | import org.joda.time.DateTime 7 | import com.kaltura.core.utils.ReadableDateUnits.ReadableDateUnits 8 | 9 | abstract class AggregationByEntry extends BaseAggregation[AggregationEntryKey, PartnerEntryRes] with IAggregate with Serializable { 10 | 11 | val columns: List[(String, String)] = List[(String, String)]( 12 | ("partner_id","partnerId"), 13 | ("entry_id","entryId"), 14 | ("metric","metric"), 15 | (getAggrTimeUnit,"time"), 16 | ("value","value")) 17 | 18 | // TODO - dropped month column temporarily 19 | override def aggKey(e: EnrichedPlayerEvent): AggregationEntryKey = AggregationEntryKey(e.partnerId, e.entryId, e.eventType, getAggrTime(e.eventTime)) 20 | override def toRow(pair: (AggregationEntryKey, Long)): PartnerEntryRes = PartnerEntryRes(partnerId = pair._1.partnerId, entryId = pair._1.entryId, metric = pair._1.metric, month = pair._1.time getYearMonth ,day = pair._1.time getYearMonthDay, time = pair._1.time, value = pair._2) 21 | } 22 | 23 | object HourlyAggregationByEntry extends AggregationByEntry with IAggregateHourly { 24 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 25 | "hourly_agg_prtn_entry" -> toSomeColumns(columns :+ ("month", "month")), 26 | "hourly_agg_clst_entry" -> toSomeColumns(columns :+ ("month", "month")) 27 | ) 28 | 29 | } 30 | 31 | object MinutelyAggregationByEntry extends AggregationByEntry with IAggregateMinutely { 32 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 33 | "minutely_agg_prtn_entry" -> toSomeColumns(columns :+ ("day", "day")), 34 | "minutely_agg_clst_entry" -> toSomeColumns(columns :+ ("day", "day")) 35 | ) 36 | } 37 | 38 | object TenSecsAggregationByEntry extends AggregationByEntry with IAggregateTenSecs { 39 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 40 | "tensecs_agg_prtn_entry" -> toSomeColumns(columns :+ ("day", "day")), 41 | "tensecs_agg_clst_entry" -> toSomeColumns(columns :+ ("day", "day")) 42 | ) 43 | } 44 | 45 | case class PartnerEntryRes(partnerId: Int, entryId: String, metric: String, month: Int, day: Int, time: DateTime, value: Long) 46 | 47 | -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/AggregationByBrowser.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import com.datastax.spark.connector.{SomeColumns, _} 4 | import com.kaltura.aggregations.keys.AggregationBrowserKey 5 | import com.kaltura.model.events.EnrichedPlayerEvent 6 | import org.joda.time.DateTime 7 | import com.kaltura.core.utils.ReadableDateUnits.ReadableDateUnits 8 | 9 | 10 | 11 | abstract class AggregationByBrowser extends BaseAggregation[AggregationBrowserKey, PartnerBrowserRes] with IAggregate with Serializable { 12 | 13 | val columns: List[(String, String)] = List[(String, String)]( 14 | ("partner_id","partner_id"), 15 | ("browser","browser"), 16 | ("metric","metric"), 17 | (getAggrTimeUnit,"time"), 18 | ("value","value")) 19 | 20 | override def aggKey(e: EnrichedPlayerEvent): AggregationBrowserKey = AggregationBrowserKey(e.partnerId, e.eventType, getAggrTime(e.eventTime), e.userAgent.browser.id) 21 | override def toRow(pair: (AggregationBrowserKey, Long)): PartnerBrowserRes = PartnerBrowserRes(partnerId = pair._1.partnerId, browser = pair._1.browser, metric = pair._1.metric, year = pair._1.time.getYear, day = pair._1.time.getYearMonthDay, time = pair._1.time, value = pair._2) 22 | 23 | 24 | } 25 | 26 | object HourlyAggregationByBrowser extends AggregationByBrowser with IAggregateHourly { 27 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 28 | "hourly_agg_prtn_browser" -> toSomeColumns(columns :+ ("year", "year")), 29 | "hourly_agg_clst_browser" -> toSomeColumns(columns :+ ("year", "year")) 30 | ) 31 | } 32 | 33 | object MinutelyAggregationByBrowser extends AggregationByBrowser with IAggregateMinutely { 34 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 35 | "minutely_agg_prtn_browser" -> toSomeColumns(columns :+ ("day", "day")), 36 | "minutely_agg_clst_browser" -> toSomeColumns(columns :+ ("day", "day")) 37 | ) 38 | } 39 | 40 | object TenSecsAggregationByBrowser extends AggregationByBrowser with IAggregateTenSecs { 41 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 42 | "tensecs_agg_prtn_browser" -> toSomeColumns(columns :+ ("day", "day")), 43 | "tensecs_agg_clst_browser" -> toSomeColumns(columns :+ ("day", "day")) 44 | ) 45 | } 46 | case class PartnerBrowserRes(partnerId: Int, metric: String, year: Int, day: Int, time: DateTime, browser: Int, value: Long) 47 | 48 | -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/AggregationByCountry.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import com.datastax.spark.connector.{SomeColumns, _} 4 | import com.kaltura.aggregations.keys.AggregationCountryKey 5 | import com.kaltura.model.events.EnrichedPlayerEvent 6 | import org.joda.time.DateTime 7 | import com.kaltura.core.utils.ReadableDateUnits.ReadableDateUnits 8 | 9 | 10 | 11 | abstract class AggregationByCountry extends BaseAggregation[AggregationCountryKey, PartnerCountryRes] with IAggregate with Serializable{ 12 | 13 | val columns: List[(String, String)] = List[(String, String)]( 14 | ("partner_id","partner_id"), 15 | ("country","country"), 16 | ("metric","metric"), 17 | (getAggrTimeUnit,"time"), 18 | ("value","value")) 19 | 20 | 21 | override def aggKey(e: EnrichedPlayerEvent): AggregationCountryKey = AggregationCountryKey(e.partnerId, e.eventType, getAggrTime(e.eventTime), e.location.country) 22 | override def toRow(pair: (AggregationCountryKey, Long)): PartnerCountryRes = PartnerCountryRes(partnerId = pair._1.partnerId, country = pair._1.country, metric = pair._1.metric, year = pair._1.time.getYear, day = pair._1.time.getYearMonthDay, time = pair._1.time, value = pair._2) 23 | 24 | } 25 | 26 | object HourlyAggregationByCountry extends AggregationByCountry with IAggregateHourly { 27 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 28 | "hourly_agg_prtn_country" -> toSomeColumns(columns :+ ("year", "year")), 29 | "hourly_agg_clst_country" -> toSomeColumns(columns :+ ("year", "year")) 30 | ) 31 | } 32 | 33 | object MinutelyAggregationByCountry extends AggregationByCountry with IAggregateMinutely { 34 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 35 | "minutely_agg_prtn_country" -> toSomeColumns(columns :+ ("day", "day")), 36 | "minutely_agg_clst_country" -> toSomeColumns(columns :+ ("day", "day")) 37 | ) 38 | } 39 | 40 | object TenSecsAggregationByCountry extends AggregationByCountry with IAggregateTenSecs { 41 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 42 | "tensecs_agg_prtn_country" -> toSomeColumns(columns :+ ("day", "day")), 43 | "tensecs_agg_clst_country" -> toSomeColumns(columns :+ ("day", "day")) 44 | ) 45 | } 46 | 47 | case class PartnerCountryRes(partnerId: Int, metric: String, year: Int, day: Int, time: DateTime, country: String, value: Long) 48 | 49 | -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/AggregationByCountryBrowser.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import com.datastax.spark.connector.{SomeColumns, _} 4 | import com.kaltura.aggregations.keys.AggregationCountryBrowserKey 5 | import com.kaltura.model.events.EnrichedPlayerEvent 6 | import org.joda.time.DateTime 7 | import com.kaltura.core.utils.ReadableDateUnits.ReadableDateUnits 8 | 9 | 10 | abstract class AggregationByCountryBrowser extends BaseAggregation[AggregationCountryBrowserKey, PartnerCountryBrowserRes] with IAggregate with Serializable{ 11 | 12 | val columns: List[(String, String)] = List[(String, String)]( 13 | ("partner_id","partner_id"), 14 | ("country","country"), 15 | ("browser","browser"), 16 | ("metric","metric"), 17 | (getAggrTimeUnit,"time"), 18 | ("value","value")) 19 | 20 | override def aggKey(e: EnrichedPlayerEvent): AggregationCountryBrowserKey = AggregationCountryBrowserKey(e.partnerId, e.eventType, getAggrTime(e.eventTime), e.location.country, e.userAgent.browser.id) 21 | override def toRow(pair: (AggregationCountryBrowserKey, Long)): PartnerCountryBrowserRes = PartnerCountryBrowserRes(partnerId= pair._1.partnerId, country = pair._1.country, metric = pair._1.metric, year = pair._1.time.getYear, day = pair._1.time.getYearMonthDay, time = pair._1.time, browser = pair._1.browser, value = pair._2) 22 | } 23 | 24 | object HourlyAggregationByCountryBrowser extends AggregationByCountryBrowser with IAggregateHourly { 25 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 26 | "hourly_agg_prtn_country_clst_browser" -> toSomeColumns(columns :+ ("year", "year")) 27 | ) 28 | } 29 | 30 | object MinutelyAggregationByCountryBrowser extends AggregationByCountryBrowser with IAggregateMinutely { 31 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 32 | "minutely_agg_prtn_country_clst_browser" -> toSomeColumns(columns :+ ("day", "day")) 33 | ) 34 | } 35 | 36 | object TenSecsAggregationByCountryBrowser extends AggregationByCountryBrowser with IAggregateTenSecs { 37 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 38 | "tensecs_agg_prtn_country_clst_browser" -> toSomeColumns(columns :+ ("day", "day")) 39 | ) 40 | } 41 | 42 | 43 | 44 | case class PartnerCountryBrowserRes(partnerId: Int, metric: String, year: Int, day: Int, time: DateTime, country: String, browser: Int, value: Long) 45 | 46 | -------------------------------------------------------------------------------- /kanalony-core/src/main/java/com/kaltura/core/ip2location/city2Location/CityCoordinatesCreator.java: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.ip2location.city2Location; 2 | 3 | import com.kaltura.core.ip2location.DbAttribtues; 4 | import com.kaltura.core.ip2location.Ip2LocationRecord; 5 | 6 | import java.io.IOException; 7 | import java.io.RandomAccessFile; 8 | import java.util.HashMap; 9 | import java.util.HashSet; 10 | import java.util.Map; 11 | import java.util.Set; 12 | 13 | public class CityCoordinatesCreator { 14 | 15 | private RandomAccessFile file = null; 16 | private DbAttribtues dbAttr = null; 17 | 18 | public void init(String fileName) throws IOException { 19 | file = new RandomAccessFile(fileName, "r"); 20 | this.dbAttr = new DbAttribtues(file); 21 | } 22 | 23 | public void close() throws IOException { 24 | if(file != null) 25 | file.close(); 26 | } 27 | 28 | protected Map> map = new HashMap>(); 29 | 30 | public void writeFile() throws IOException { 31 | initMap(); 32 | } 33 | 34 | private void initMap() throws IOException { 35 | 36 | long limit = this.dbAttr.getCount(); 37 | for(long cur = 0 ; cur < limit ; cur++) { 38 | Ip2LocationRecord record = new Ip2LocationRecord(this.file, this.dbAttr, cur); 39 | String country = record.getCountryLong(); 40 | String city = record.getCity(); 41 | 42 | if((city == null) || (country == null)) { 43 | continue; 44 | } 45 | 46 | if(!map.containsKey(country)) { 47 | map.put(country, new HashSet()); 48 | } 49 | 50 | if(map.get(country).contains(city)) { 51 | continue; 52 | } else { 53 | System.out.println("\"" + country + "\",\"" + city + "\"," + String.format("%.4f", record.getLatitude()) + "," + 54 | String.format("%.4f", record.getLongitude()) ); 55 | map.get(country).add(city); 56 | } 57 | 58 | } 59 | } 60 | 61 | /** 62 | * This scripts takes IP2Location file and prints for each city & country its longitude & latitude 63 | * @param args IP2Location file path 64 | * @throws IOException 65 | */ 66 | public static void main(String[] args) throws IOException { 67 | if(args.length != 1) { 68 | System.out.println("Wrong number of params"); 69 | System.exit(-1); 70 | } 71 | 72 | CityCoordinatesCreator creator = new CityCoordinatesCreator(); 73 | creator.init(args[0]); 74 | creator.writeFile(); 75 | creator.close(); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /kanalony-core/src/main/scala/com/kaltura/core/utils/ConfigurationManager.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.utils 2 | 3 | import java.io.{FileInputStream, InputStream} 4 | import java.util.Properties 5 | 6 | import com.kaltura.core.logging.{MetaLog, BaseLog} 7 | 8 | 9 | import scala.collection.JavaConverters._ 10 | import scala.collection.mutable.Map 11 | import scala.util.Try 12 | 13 | /** 14 | * A utility for reading configuration properties 15 | * Author: @kerkero 16 | */ 17 | object ConfigurationManager extends Serializable with MetaLog[BaseLog]{ 18 | 19 | protected var configData: Map[String,String] = Map() 20 | 21 | loadConfiguration(configFilePath); 22 | 23 | protected def configFilePath: String = { 24 | var confPath: String = "/opt/kaltura/conf" 25 | if (System.getenv.containsKey("KALTURA_CONF_PATH")) { 26 | confPath = System.getenv("KALTURA_CONF_PATH") 27 | } 28 | confPath + "/config.properties" 29 | } 30 | 31 | /* 32 | * Load configuration properties from configuration files 33 | */ 34 | def loadConfiguration(configFilePath: String): Unit = { 35 | val props: Properties = new Properties 36 | try { 37 | //logger.info(s"Loading configuration from path: $configFilePath") 38 | val file: InputStream = new FileInputStream(configFilePath) 39 | props.load(file) 40 | file.close() 41 | } 42 | catch { 43 | case _ : Throwable => throw new Exception(s"Unable to load configuration file [path: $configFilePath]") 44 | } 45 | 46 | loadConfiguration(props.asScala); 47 | } 48 | 49 | /* 50 | * Load configuration properties from a {key, value} map, might override configuration that's already set 51 | */ 52 | def loadConfiguration(configProperties: Map[String,String]): Unit = { 53 | configProperties.foreach((pair: (String, String)) => configData(pair._1) = pair._2) 54 | } 55 | 56 | /* 57 | * Gets a configuration value by key 58 | */ 59 | def get(configKey: String): String = { 60 | try{ 61 | configData(configKey) 62 | } 63 | catch { 64 | case _: Exception => throw new Exception(s"Unable to find a configuration value for [key: $configKey]") 65 | } 66 | } 67 | 68 | /* 69 | * Gets a configuration value by key 70 | */ 71 | def getOrElse(configKey: String, defaultValue: String): String = { 72 | Try(configData(configKey)).getOrElse(defaultValue) 73 | } 74 | 75 | def set(configKey: String, configValue: String): Unit = { 76 | configData(configKey) = configValue 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/AggregationByApplication.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import com.datastax.spark.connector.{SomeColumns, _} 4 | import com.kaltura.aggregations.keys.AggregationApplicationKey 5 | import com.kaltura.model.events.EnrichedPlayerEvent 6 | import org.joda.time.DateTime 7 | import com.kaltura.core.utils.ReadableDateUnits.ReadableDateUnits 8 | 9 | 10 | /** 11 | * Created by orlylampert on 3/2/16. 12 | */ 13 | abstract class AggregationByApplication extends BaseAggregation[AggregationApplicationKey, PartnerApplicationRes] with IAggregate with Serializable{ 14 | val columns: List[(String, String)] = List[(String, String)]( 15 | ("partner_id","partner_id"), 16 | ("application","application"), 17 | ("metric","metric"), 18 | (getAggrTimeUnit,"time"), 19 | ("value","value")) 20 | 21 | 22 | override def aggKey(e: EnrichedPlayerEvent): AggregationApplicationKey = AggregationApplicationKey(e.partnerId, e.eventType, getAggrTime(e.eventTime), e.application) 23 | override def toRow(pair: (AggregationApplicationKey, Long)): PartnerApplicationRes = PartnerApplicationRes(partnerId = pair._1.partnerId, application = pair._1.application, metric = pair._1.metric, year = pair._1.time.getYear, day = pair._1.time.getYearMonthDay, time = pair._1.time, value = pair._2) 24 | 25 | } 26 | 27 | 28 | object HourlyAggregationByApplication extends AggregationByApplication with IAggregateHourly { 29 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 30 | "hourly_agg_prtn_app" -> toSomeColumns(columns :+ ("year", "year")), 31 | "hourly_agg_clst_app" -> toSomeColumns(columns :+ ("year", "year")) 32 | ) 33 | } 34 | 35 | object MinutelyAggregationByApplication extends AggregationByApplication with IAggregateMinutely { 36 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 37 | "minutely_agg_prtn_app" -> toSomeColumns(columns :+ ("day", "day")), 38 | "minutely_agg_clst_app" -> toSomeColumns(columns :+ ("day", "day")) 39 | ) 40 | } 41 | 42 | object TenSecsAggregationByApplication extends AggregationByApplication with IAggregateTenSecs { 43 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 44 | "tensecs_agg_prtn_app" -> toSomeColumns(columns :+ ("day", "day")), 45 | "tensecs_agg_clst_app" -> toSomeColumns(columns :+ ("day", "day")) 46 | ) 47 | } 48 | 49 | case class PartnerApplicationRes(partnerId: Int, metric: String, year: Int, day: Int, time: DateTime, application: String, value: Long) 50 | -------------------------------------------------------------------------------- /kanalony-core/src/main/java/com/kaltura/core/ip2location/Ip2LocationReader.java: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.ip2location; 2 | 3 | import java.io.IOException; 4 | import java.io.RandomAccessFile; 5 | import java.net.UnknownHostException; 6 | 7 | 8 | public class Ip2LocationReader { 9 | 10 | private DbAttribtues dbAttr = null; 11 | private RandomAccessFile file = null; 12 | private boolean isLocked = false; 13 | 14 | public void init(String fileName) throws IOException { 15 | file = new RandomAccessFile(fileName, "r"); 16 | this.dbAttr = new DbAttribtues(file); 17 | lock(); 18 | } 19 | 20 | public void unlock() { 21 | this.isLocked = false; 22 | } 23 | 24 | public boolean isLocked() 25 | { 26 | return isLocked; 27 | } 28 | 29 | public void lock() 30 | { 31 | isLocked = true; 32 | } 33 | 34 | public void close() throws IOException { 35 | if(file != null) 36 | file.close(); 37 | } 38 | 39 | public Ip2LocationRecord getAll(String ip) throws IOException { 40 | return getRecord(ip); 41 | } 42 | 43 | public Ip2LocationRecord find(String ip) throws IOException { 44 | return getRecord(ip); 45 | } 46 | 47 | public Ip2LocationRecord getRecord(String ip) throws IOException { 48 | long baseaddr = this.dbAttr.getAddr(); 49 | long low = 0; 50 | long high = this.dbAttr.getCount(); 51 | int column = this.dbAttr.getColumn(); 52 | 53 | long ipno = ipv4ToNumber(ip); 54 | 55 | while(low <= high) { 56 | long mid = ((low + high) / 2); 57 | 58 | long ipfrom = Utils.readUInteger(file, baseaddr + (mid) * (column * 4)); 59 | long ipto = Utils.readUInteger(file, baseaddr + (mid + 1) * (column * 4)); 60 | 61 | if((ipfrom <= ipno) && (ipno < ipto)) { 62 | return new Ip2LocationRecord(this.file, this.dbAttr, mid); 63 | } else { 64 | if(ipno < ipfrom) 65 | high = mid - 1; 66 | else 67 | low = mid + 1; 68 | } 69 | } 70 | return null; 71 | } 72 | 73 | public static long ipv4ToNumber (String ipaddr) throws UnknownHostException { 74 | 75 | long long_ip = 0; 76 | String[] ipAddressInArray = ipaddr.split("\\."); 77 | 78 | for (int i = 3; i >= 0; i--) { 79 | 80 | long ip = Long.parseLong(ipAddressInArray[3 - i]); 81 | long_ip |= ip << (i * 8); 82 | 83 | } 84 | if (long_ip < 0) { 85 | long_ip += Math.pow(2, 32); 86 | } 87 | return long_ip; 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/AggregationByOperatingSystem.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import com.datastax.spark.connector.SomeColumns 4 | import com.kaltura.aggregations.keys.AggregationOperatingSystemKey 5 | import com.kaltura.core.utils.ReadableDateUnits.ReadableDateUnits 6 | import com.kaltura.model.events.EnrichedPlayerEvent 7 | import org.joda.time.DateTime 8 | 9 | 10 | abstract class AggregationByOperatingSystem extends BaseAggregation[AggregationOperatingSystemKey, PartnerOperatingSystemRes] with IAggregate with Serializable { 11 | 12 | val columns: List[(String, String)] = List[(String, String)]( 13 | ("partner_id","partnerId"), 14 | ("operating_system","operatingSystem"), 15 | ("metric","metric"), 16 | (getAggrTimeUnit,"time"), 17 | ("value","value")) 18 | 19 | override def aggKey(e: EnrichedPlayerEvent): AggregationOperatingSystemKey = AggregationOperatingSystemKey(e.partnerId, e.eventType, getAggrTime(e.eventTime), e.userAgent.operatingSystem.id) 20 | override def toRow(pair: (AggregationOperatingSystemKey, Long)): PartnerOperatingSystemRes = PartnerOperatingSystemRes(partnerId = pair._1.partnerId, operatingSystem = pair._1.operatingSystem, metric = pair._1.metric, year = pair._1.time.getYear, day = pair._1.time.getYearMonthDay, time = pair._1.time, value = pair._2) 21 | 22 | } 23 | 24 | object HourlyAggregationByOperatingSystem extends AggregationByOperatingSystem with IAggregateHourly { 25 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 26 | "hourly_agg_prtn_os" -> toSomeColumns(columns :+ ("year", "year")), 27 | "hourly_agg_clst_os" -> toSomeColumns(columns :+ ("year", "year")) 28 | ) 29 | } 30 | 31 | object MinutelyAggregationByOperatingSystem extends AggregationByOperatingSystem with IAggregateMinutely { 32 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 33 | "minutely_agg_prtn_os" -> toSomeColumns(columns :+ ("day", "day")), 34 | "minutely_agg_clst_os" -> toSomeColumns(columns :+ ("day", "day")) 35 | ) 36 | } 37 | 38 | object TenSecsAggregationByOperatingSystem extends AggregationByOperatingSystem with IAggregateTenSecs { 39 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 40 | "tensecs_agg_prtn_os" -> toSomeColumns(columns :+ ("day", "day")), 41 | "tensecs_agg_clst_os" -> toSomeColumns(columns :+ ("day", "day")) 42 | ) 43 | } 44 | case class PartnerOperatingSystemRes(partnerId: Int, metric: String, year: Int, day: Int, time: DateTime, operatingSystem: Int, value: Long) 45 | 46 | -------------------------------------------------------------------------------- /kanalony-storage-logic-generated/src/main/scala/kanalony/storage/logic/generated/HourlyAggQuery.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.generated 2 | 3 | import kanalony.storage.generated._ 4 | import kanalony.storage.logic._ 5 | import kanalony.storage.logic.queries.model._ 6 | import kanalony.storage.DbClientFactory._ 7 | import org.joda.time.{DateTimeZone, DateTime} 8 | import scala.concurrent.Future 9 | 10 | class HourlyAggQuery(accessor : IHourlyAggTableAccessor) extends QueryBase[HourlyAggQueryParams, HourlyAggRow] with IUserActivityQuery { 11 | private[logic] override def extractParams(params: QueryParams): HourlyAggQueryParams = { 12 | val (partner_id) = QueryParamsValidator.extractEqualityConstraintParams[Int]((Dimensions.partner), params) 13 | HourlyAggQueryParams(params.startUtc, params.endUtc, partner_id, params.metrics.map(_.name)) 14 | } 15 | 16 | override def supportsUserDefinedMetrics = true 17 | 18 | private[logic] override def executeQuery(params: HourlyAggQueryParams): Future[List[HourlyAggRow]] = { 19 | accessor.query(params.partnerIdList,params.metricList,params.years,params.startTime,params.endTime) 20 | } 21 | 22 | override private[logic] def getResultHeaders(): List[String] = { 23 | List(Dimensions.partner.toString,Dimensions.metric.toString,Dimensions.hour.toString,"value") 24 | } 25 | 26 | override protected def getResultRow(row: HourlyAggRow): List[String] = { 27 | List(row.partnerId.toString,row.metric.toString,row.hour.toString,row.value.toString) 28 | } 29 | 30 | override val dimensionInformation: List[DimensionDefinition] = { 31 | List(DimensionDefinition(Dimensions.partner, new DimensionConstraintDeclaration(QueryConstraint.Equality)), 32 | DimensionDefinition(Dimensions.hour, new DimensionConstraintDeclaration(QueryConstraint.Range))) 33 | } 34 | 35 | override def metricValueLocationIndex(): Int = 3 36 | 37 | override private[logic] def extractMetric(row: HourlyAggRow): String = row.metric 38 | 39 | override private[logic] def updateTimezoneOffset(row : HourlyAggRow, timezoneOffsetFromUtc : Int) : HourlyAggRow = { 40 | HourlyAggRow(row.partnerId, row.metric, row.year, row.hour.withZone(DateTimeZone.forOffsetHoursMinutes(timezoneOffsetFromUtc / 60, timezoneOffsetFromUtc % 60)), row.value) 41 | } 42 | 43 | } 44 | 45 | case class HourlyAggQueryParams(startTime : DateTime, endTime : DateTime, partnerIdList : List[Int], metricList : List[String]) extends IYearlyPartitionedQueryParams -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/AggregationByCountryOperatingSystem.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import com.datastax.spark.connector.{SomeColumns, _} 4 | import com.kaltura.aggregations.keys.AggregationCountryOperatingSystemKey 5 | import com.kaltura.model.events.EnrichedPlayerEvent 6 | import org.joda.time.DateTime 7 | import com.kaltura.core.utils.ReadableDateUnits.ReadableDateUnits 8 | 9 | abstract class AggregationByCountryOperatingSystem extends BaseAggregation[AggregationCountryOperatingSystemKey, PartnerCountryOperatingSystemRes] with IAggregate with Serializable{ 10 | 11 | val columns: List[(String, String)] = List[(String, String)]( 12 | ("partner_id","partner_id"), 13 | ("country","country"), 14 | ("operating_system","operatingSystem"), 15 | ("metric","metric"), 16 | (getAggrTimeUnit,"time"), 17 | ("value","value")) 18 | 19 | override def aggKey(e: EnrichedPlayerEvent): AggregationCountryOperatingSystemKey = AggregationCountryOperatingSystemKey(e.partnerId, e.eventType, getAggrTime(e.eventTime), e.location.country, e.userAgent.operatingSystem.id) 20 | override def toRow(pair: (AggregationCountryOperatingSystemKey, Long)): PartnerCountryOperatingSystemRes = PartnerCountryOperatingSystemRes(partnerId= pair._1.partnerId, country = pair._1.country, metric = pair._1.metric, year = pair._1.time.getYear, day = pair._1.time.getYearMonthDay, time = pair._1.time, operatingSystem = pair._1.operatingSystem, value = pair._2) 21 | } 22 | 23 | object HourlyAggregationByCountryOperatingSystem extends AggregationByCountryOperatingSystem with IAggregateHourly { 24 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 25 | "hourly_agg_prtn_country_clst_os" -> toSomeColumns(columns :+ ("year", "year")) 26 | ) 27 | } 28 | 29 | object MinutelyAggregationByCountryOperatingSystem extends AggregationByCountryOperatingSystem with IAggregateMinutely { 30 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 31 | "minutely_agg_prtn_country_clst_os" -> toSomeColumns(columns :+ ("day", "day")) 32 | ) 33 | } 34 | 35 | object TenSecsAggregationByCountryOperatingSystem extends AggregationByCountryOperatingSystem with IAggregateTenSecs { 36 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 37 | "tensecs_agg_prtn_country_clst_os" -> toSomeColumns(columns :+ ("day", "day")) 38 | ) 39 | } 40 | 41 | 42 | case class PartnerCountryOperatingSystemRes(partnerId: Int, metric: String, year: Int, day: Int, time: DateTime, country: String, operatingSystem: Int, value: Long) 43 | 44 | -------------------------------------------------------------------------------- /kanalony-core/src/test/scala/com/kaltura/core/userAgent/UserAgentSpec.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.userAgent 2 | 3 | import org.scalatest.{BeforeAndAfterAll, Matchers, FlatSpec} 4 | import com.kaltura.core.userAgent.enums.{Browser,Device,OperatingSystem} 5 | 6 | /** 7 | * Created by ofirk on 20/10/2015. 8 | */ 9 | class UserAgentSpec extends FlatSpec with Matchers with BeforeAndAfterAll { 10 | 11 | "Chrome 41 on Windwos 7" should "be resolved successfully" in { 12 | val ua:String = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.89 Safari/537.36" 13 | val userAgent = UserAgentResolver.resolve(ua) 14 | userAgent.browser should be (Browser.CHROME41) 15 | userAgent.device should be (Device.COMPUTER) 16 | userAgent.operatingSystem should be (OperatingSystem.WINDOWS_7) 17 | } 18 | 19 | "Firefox 3 on OS X" should "be resolved successfully" in { 20 | val ua:String = "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10.5; en-US; rv:1.9.0.3) Gecko/2008092414 Firefox/3.0.3" 21 | val userAgent = UserAgentResolver.resolve(ua) 22 | userAgent.browser should be (Browser.FIREFOX3) 23 | userAgent.device should be (Device.COMPUTER) 24 | userAgent.operatingSystem should be (OperatingSystem.MAC_OS_X) 25 | } 26 | 27 | "Curl" should "be resolved successfully" in { 28 | val ua:String = "curl/7.19.5" 29 | val userAgent = UserAgentResolver.resolve(ua) 30 | userAgent.browser should be (Browser.DOWNLOAD) 31 | userAgent.device should be (Device.UNKNOWN) 32 | userAgent.operatingSystem should be (OperatingSystem.UNKNOWN) 33 | } 34 | 35 | "Garbage" should "be resolved successfully" in { 36 | val ua:String = "sdiajfoiadsjfoi" 37 | val userAgent = UserAgentResolver.resolve(ua) 38 | userAgent.browser should be (Browser.UNKNOWN) 39 | userAgent.device should be (Device.UNKNOWN) 40 | userAgent.operatingSystem should be (OperatingSystem.UNKNOWN) 41 | } 42 | 43 | "Empty" should "be resolved successfully" in { 44 | val ua:String = "" 45 | val userAgent = UserAgentResolver.resolve(ua) 46 | userAgent.browser should be (Browser.UNKNOWN) 47 | userAgent.device should be (Device.UNKNOWN) 48 | userAgent.operatingSystem should be (OperatingSystem.UNKNOWN) 49 | } 50 | 51 | "Null" should "be resolved successfully" in { 52 | val ua:String = null 53 | val userAgent = UserAgentResolver.resolve(ua) 54 | userAgent.browser should be (Browser.UNKNOWN) 55 | userAgent.device should be (Device.UNKNOWN) 56 | userAgent.operatingSystem should be (OperatingSystem.UNKNOWN) 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /kanalony-storage-logic-generated/src/main/scala/kanalony/storage/logic/generated/TensecsAggQuery.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.generated 2 | 3 | import kanalony.storage.generated._ 4 | import kanalony.storage.logic._ 5 | import kanalony.storage.logic.queries.model._ 6 | import kanalony.storage.DbClientFactory._ 7 | import org.joda.time.{DateTimeZone, DateTime} 8 | import scala.concurrent.Future 9 | 10 | class TensecsAggQuery(accessor : ITensecsAggTableAccessor) extends QueryBase[TensecsAggQueryParams, TensecsAggRow] with IUserActivityQuery { 11 | private[logic] override def extractParams(params: QueryParams): TensecsAggQueryParams = { 12 | val (partner_id) = QueryParamsValidator.extractEqualityConstraintParams[Int]((Dimensions.partner), params) 13 | TensecsAggQueryParams(params.startUtc, params.endUtc, partner_id, params.metrics.map(_.name)) 14 | } 15 | 16 | override def supportsUserDefinedMetrics = true 17 | 18 | private[logic] override def executeQuery(params: TensecsAggQueryParams): Future[List[TensecsAggRow]] = { 19 | accessor.query(params.partnerIdList,params.metricList,params.days,params.startTime,params.endTime) 20 | } 21 | 22 | override private[logic] def getResultHeaders(): List[String] = { 23 | List(Dimensions.partner.toString,Dimensions.metric.toString,Dimensions.tenSeconds.toString,"value") 24 | } 25 | 26 | override protected def getResultRow(row: TensecsAggRow): List[String] = { 27 | List(row.partnerId.toString,row.metric.toString,row.tensecs.toString,row.value.toString) 28 | } 29 | 30 | override val dimensionInformation: List[DimensionDefinition] = { 31 | List(DimensionDefinition(Dimensions.partner, new DimensionConstraintDeclaration(QueryConstraint.Equality)), 32 | DimensionDefinition(Dimensions.tenSeconds, new DimensionConstraintDeclaration(QueryConstraint.Range))) 33 | } 34 | 35 | override def metricValueLocationIndex(): Int = 3 36 | 37 | override private[logic] def extractMetric(row: TensecsAggRow): String = row.metric 38 | 39 | override private[logic] def updateTimezoneOffset(row : TensecsAggRow, timezoneOffsetFromUtc : Int) : TensecsAggRow = { 40 | TensecsAggRow(row.partnerId, row.metric, row.day, row.tensecs.withZone(DateTimeZone.forOffsetHoursMinutes(timezoneOffsetFromUtc / 60, timezoneOffsetFromUtc % 60)), row.value) 41 | } 42 | 43 | } 44 | 45 | case class TensecsAggQueryParams(startTime : DateTime, endTime : DateTime, partnerIdList : List[Int], metricList : List[String]) extends IDailyPartitionedQueryParams -------------------------------------------------------------------------------- /kanalony-storage-logic-generated/src/main/scala/kanalony/storage/logic/generated/MinutelyAggQuery.scala: -------------------------------------------------------------------------------- 1 | package kanalony.storage.logic.generated 2 | 3 | import kanalony.storage.generated._ 4 | import kanalony.storage.logic._ 5 | import kanalony.storage.logic.queries.model._ 6 | import kanalony.storage.DbClientFactory._ 7 | import org.joda.time.{DateTimeZone, DateTime} 8 | import scala.concurrent.Future 9 | 10 | class MinutelyAggQuery(accessor : IMinutelyAggTableAccessor) extends QueryBase[MinutelyAggQueryParams, MinutelyAggRow] with IUserActivityQuery { 11 | private[logic] override def extractParams(params: QueryParams): MinutelyAggQueryParams = { 12 | val (partner_id) = QueryParamsValidator.extractEqualityConstraintParams[Int]((Dimensions.partner), params) 13 | MinutelyAggQueryParams(params.startUtc, params.endUtc, partner_id, params.metrics.map(_.name)) 14 | } 15 | 16 | override def supportsUserDefinedMetrics = true 17 | 18 | private[logic] override def executeQuery(params: MinutelyAggQueryParams): Future[List[MinutelyAggRow]] = { 19 | accessor.query(params.partnerIdList,params.metricList,params.days,params.startTime,params.endTime) 20 | } 21 | 22 | override private[logic] def getResultHeaders(): List[String] = { 23 | List(Dimensions.partner.toString,Dimensions.metric.toString,Dimensions.minute.toString,"value") 24 | } 25 | 26 | override protected def getResultRow(row: MinutelyAggRow): List[String] = { 27 | List(row.partnerId.toString,row.metric.toString,row.minute.toString,row.value.toString) 28 | } 29 | 30 | override val dimensionInformation: List[DimensionDefinition] = { 31 | List(DimensionDefinition(Dimensions.partner, new DimensionConstraintDeclaration(QueryConstraint.Equality)), 32 | DimensionDefinition(Dimensions.minute, new DimensionConstraintDeclaration(QueryConstraint.Range))) 33 | } 34 | 35 | override def metricValueLocationIndex(): Int = 3 36 | 37 | override private[logic] def extractMetric(row: MinutelyAggRow): String = row.metric 38 | 39 | override private[logic] def updateTimezoneOffset(row : MinutelyAggRow, timezoneOffsetFromUtc : Int) : MinutelyAggRow = { 40 | MinutelyAggRow(row.partnerId, row.metric, row.day, row.minute.withZone(DateTimeZone.forOffsetHoursMinutes(timezoneOffsetFromUtc / 60, timezoneOffsetFromUtc % 60)), row.value) 41 | } 42 | 43 | } 44 | 45 | case class MinutelyAggQueryParams(startTime : DateTime, endTime : DateTime, partnerIdList : List[Int], metricList : List[String]) extends IDailyPartitionedQueryParams -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/AggregationByCustomVar1CustomVar2.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import com.datastax.spark.connector.{SomeColumns, _} 4 | import com.kaltura.aggregations.keys.AggregationCustomVar1CustomVar2Key 5 | import com.kaltura.model.events.EnrichedPlayerEvent 6 | import org.joda.time.DateTime 7 | import com.kaltura.core.utils.ReadableDateUnits.ReadableDateUnits 8 | 9 | 10 | 11 | abstract class AggregationByCustomVar1CustomVar2 extends BaseAggregation[AggregationCustomVar1CustomVar2Key, PartnerCustomVar1CustomVar2Res] with IAggregate with Serializable{ 12 | 13 | val columns: List[(String, String)] = List[(String, String)]( 14 | ("partner_id","partner_id"), 15 | ("custom_var1","cv1"), 16 | ("custom_var2","cv2"), 17 | ("metric","metric"), 18 | (getAggrTimeUnit,"time"), 19 | ("value","value")) 20 | 21 | override def aggKey(e: EnrichedPlayerEvent): AggregationCustomVar1CustomVar2Key = AggregationCustomVar1CustomVar2Key(e.partnerId, e.eventType, getAggrTime(e.eventTime), e.customVar1, e.customVar2) 22 | override def toRow(pair: (AggregationCustomVar1CustomVar2Key, Long)): PartnerCustomVar1CustomVar2Res = PartnerCustomVar1CustomVar2Res(pair._1.partnerId, pair._1.cv1, pair._1.cv2, pair._1.metric, pair._1.time.getYear, pair._1.time.getYearMonthDay, pair._1.time, pair._2) 23 | } 24 | 25 | object HourlyAggregationByCustomVar1CustomVar2 extends AggregationByCustomVar1CustomVar2 with IAggregateHourly { 26 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 27 | "hourly_agg_prtn_cv1_cv2" -> toSomeColumns(columns :+ ("year", "year")), 28 | "hourly_agg_prtn_cv1_clst_cv2" -> toSomeColumns(columns :+ ("year", "year")) 29 | ) 30 | } 31 | 32 | object MinutelyAggregationByCustomVar1CustomVar2 extends AggregationByCustomVar1CustomVar2 with IAggregateMinutely { 33 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 34 | "minutely_agg_prtn_cv1_cv2" -> toSomeColumns(columns :+ ("day", "day")), 35 | "minutely_agg_prtn_cv1_clst_cv2" -> toSomeColumns(columns :+ ("day", "day")) 36 | ) 37 | } 38 | 39 | object TenSecsAggregationByCustomVar1CustomVar2 extends AggregationByCustomVar1CustomVar2 with IAggregateTenSecs { 40 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 41 | "tensecs_agg_prtn_cv1_cv2" -> toSomeColumns(columns :+ ("day", "day")), 42 | "tensecs_agg_prtn_cv1_clst_cv2" -> toSomeColumns(columns :+ ("day", "day")) 43 | ) 44 | } 45 | case class PartnerCustomVar1CustomVar2Res(partner_id:Int, cv1:String, cv2:String, metric:String, year:Int, day: Int, time:DateTime, value:Long) 46 | -------------------------------------------------------------------------------- /kanalony-aggregations/src/main/scala/com/kaltura/aggregations/AggregationByPlaybackContext.scala: -------------------------------------------------------------------------------- 1 | package com.kaltura.aggregations 2 | 3 | import com.datastax.spark.connector.{SomeColumns, _} 4 | import com.kaltura.aggregations.keys.AggregationPlaybackContextKey 5 | import com.kaltura.model.events.EnrichedPlayerEvent 6 | import org.joda.time.DateTime 7 | import com.kaltura.core.utils.ReadableDateUnits.ReadableDateUnits 8 | 9 | abstract class AggregationByPlaybackContext extends BaseAggregation[AggregationPlaybackContextKey, PartnerPlaybackContextRes] with IAggregate with Serializable{ 10 | val columns: List[(String, String)] = List[(String, String)]( 11 | ("partner_id","partnerId"), 12 | ("playback_context","playbackContext"), 13 | ("metric","metric"), 14 | (getAggrTimeUnit,"time"), 15 | ("value","value")) 16 | 17 | 18 | override def aggKey(e: EnrichedPlayerEvent): AggregationPlaybackContextKey = AggregationPlaybackContextKey(e.partnerId, e.eventType, getAggrTime(e.eventTime), e.playbackContext) 19 | override def toRow(pair: (AggregationPlaybackContextKey, Long)): PartnerPlaybackContextRes = PartnerPlaybackContextRes(partnerId = pair._1.partnerId, playbackContext = pair._1.playbackContext, metric = pair._1.metric, year = pair._1.time.getYear, day = pair._1.time getYearMonthDay, time = pair._1.time, value = pair._2) 20 | 21 | } 22 | 23 | 24 | object HourlyAggregationByPlaybackContext extends AggregationByPlaybackContext with IAggregateHourly { 25 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 26 | "hourly_agg_prtn_playbackcontext" -> toSomeColumns(columns :+ ("year", "year")), 27 | "hourly_agg_clst_playbackcontext" -> toSomeColumns(columns :+ ("year", "year")) 28 | ) 29 | } 30 | 31 | object MinutelyAggregationByPlaybackContext extends AggregationByPlaybackContext with IAggregateMinutely { 32 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 33 | "minutely_agg_prtn_playbackcontext" -> toSomeColumns(columns :+ ("day", "day")), 34 | "minutely_agg_clst_playbackcontext" -> toSomeColumns(columns :+ ("day", "day")) 35 | ) 36 | } 37 | 38 | object TenSecsAggregationByPlaybackContext extends AggregationByPlaybackContext with IAggregateTenSecs { 39 | override lazy val tableMetadata: Map[String, SomeColumns] = Map( 40 | "tensecs_agg_prtn_playbackcontext" -> toSomeColumns(columns :+ ("day", "day")), 41 | "tensecs_agg_clst_playbackcontext" -> toSomeColumns(columns :+ ("day", "day")) 42 | ) 43 | } 44 | 45 | case class PartnerPlaybackContextRes(partnerId: Int, metric: String, year: Int, day: Int, time: DateTime, playbackContext: String, value: Long) 46 | -------------------------------------------------------------------------------- /kanalony-enrichment/src/test/scala/org/apache/spark/SparkFunSuite.scala: -------------------------------------------------------------------------------- 1 | package org.apache.spark 2 | 3 | /* 4 | * Licensed to the Apache Software Foundation (ASF) under one or more 5 | * contributor license agreements. See the NOTICE file distributed with 6 | * this work for additional information regarding copyright ownership. 7 | * The ASF licenses this file to You under the Apache License, Version 2.0 8 | * (the "License"); you may not use this file except in compliance with 9 | * the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, 15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * See the License for the specific language governing permissions and 17 | * limitations under the License. 18 | */ 19 | 20 | // scalastyle:off 21 | 22 | import com.kaltura.enrichment.EventsEnrichment._ 23 | import org.apache.log4j.{Level, Logger} 24 | import org.scalatest.{FunSuite, Outcome} 25 | 26 | /** 27 | * Base abstract class for all unit tests in Spark for handling common functionality. 28 | */ 29 | abstract class SparkFunSuite extends FunSuite with Logging { 30 | // scalastyle:on 31 | 32 | /** 33 | * Log the suite name and the test name before and after each test. 34 | * 35 | * Subclasses should never override this method. If they wish to run 36 | * custom code before and after each test, they should mix in the 37 | * {{org.scalatest.BeforeAndAfter}} trait instead. 38 | */ 39 | final protected override def withFixture(test: NoArgTest): Outcome = { 40 | val testName = test.text 41 | val suiteName = this.getClass.getName 42 | val shortSuiteName = suiteName.replaceAll("org.apache.spark", "o.a.s") 43 | setStreamingLogLevels 44 | try { 45 | logInfo(s"\n\n===== TEST OUTPUT FOR $shortSuiteName: '$testName' =====\n") 46 | test() 47 | } finally { 48 | logInfo(s"\n\n===== FINISHED $shortSuiteName: '$testName' =====\n") 49 | } 50 | } 51 | 52 | def setStreamingLogLevels { 53 | val log4jInitialized = Logger.getRootLogger.getAllAppenders.hasMoreElements 54 | if (!log4jInitialized) { 55 | // We first log something to initialize Spark's default logging, then we override the 56 | // logging level. 57 | logInfo("Setting log level to [WARN] for streaming example." + 58 | " To override add a custom log4j.properties to the classpath.") 59 | } 60 | Logger.getRootLogger.setLevel(Level.WARN) 61 | } 62 | 63 | } -------------------------------------------------------------------------------- /kanalony-core/src/main/java/com/kaltura/core/ip2location/country2location/CountryLocator.java: -------------------------------------------------------------------------------- 1 | package com.kaltura.core.ip2location.country2location; 2 | 3 | import com.kaltura.core.ip2location.Coordinate; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | import java.io.*; 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | import java.util.regex.Matcher; 11 | import java.util.regex.Pattern; 12 | 13 | public class CountryLocator { 14 | 15 | private static Logger LOG = LoggerFactory.getLogger(CountryLocator.class); 16 | private static final String COUNTRIES_FILE_NAME = "/countriesCoordinates.txt"; 17 | 18 | private final String countryPattern = "[\\w, \\-\\.\\(\\)\\']+"; 19 | private Pattern cityFormat = Pattern.compile("\"(" + countryPattern + ")\",(\\-?[\\d\\.]+),(\\-?[\\d\\.]+)"); 20 | 21 | 22 | /** 23 | * There aren't many countries around the world, so loading it to memory shouldn't be a problem. 24 | */ 25 | private Map countryToCoord = new HashMap(); 26 | 27 | public CountryLocator() { 28 | InputStream is = null; 29 | BufferedReader br = null; 30 | try { 31 | is = getClass().getResourceAsStream(COUNTRIES_FILE_NAME); 32 | br = new BufferedReader(new InputStreamReader(is)); 33 | String line; 34 | while ((line = br.readLine()) != null) { 35 | 36 | Matcher m = cityFormat.matcher(line); 37 | if (!m.matches()) 38 | throw new RuntimeException("Failed to parse input file " + line); 39 | 40 | String country = m.group(1); 41 | Coordinate cr = new Coordinate(country, Float.parseFloat(m.group(2)), Float.parseFloat(m.group(3))); 42 | countryToCoord.put(country, cr); 43 | } 44 | br.close(); 45 | } catch (IOException e) { 46 | LOG.error("Failed to init from file."); 47 | } finally { 48 | try { 49 | if(is != null) 50 | is.close(); 51 | 52 | if (br != null) 53 | br.close(); 54 | } catch (IOException e) { 55 | LOG.error("Failed to close file."); 56 | } 57 | 58 | } 59 | } 60 | 61 | public Coordinate getCountryCoordinates(String country) { 62 | if(countryToCoord.containsKey(country)) { 63 | return countryToCoord.get(country); 64 | } 65 | 66 | return new Coordinate(country, 0, 0); 67 | } 68 | 69 | 70 | public static void main(String[] args) throws FileNotFoundException { 71 | CountryLocator reader = new CountryLocator(); 72 | Coordinate ilCoord = reader.getCountryCoordinates("ISRAEL"); 73 | System.out.println(ilCoord.getName()); 74 | System.out.println(ilCoord.getLatitude()); 75 | System.out.println(ilCoord.getLongitude()); 76 | } 77 | } 78 | --------------------------------------------------------------------------------