├── .gitignore ├── .gitmodules ├── .jvmopts ├── .mailmap ├── CLA.md ├── LICENSE.txt ├── README.md ├── accounts ├── build.sbt ├── configs │ ├── dev │ │ ├── accounts-v1.conf │ │ └── accounts-v1.logging.xml │ ├── reset.eml.html.mustache │ ├── reset.eml.txt.mustache │ └── reset.subj.mustache └── src │ ├── main │ └── scala │ │ └── com │ │ └── precog │ │ └── accounts │ │ ├── AccountManager.scala │ │ ├── AccountService.scala │ │ ├── AccountServiceHandlers.scala │ │ ├── MongoAccountManager.scala │ │ ├── MongoAccountsServer.scala │ │ └── ResetToken.scala │ └── test │ ├── resources │ ├── reset.eml.html.mustache │ ├── reset.eml.txt.mustache │ └── reset.subj.mustache │ └── scala │ └── com │ └── precog │ └── accounts │ ├── AccountServiceSpec.scala │ ├── InMemoryAccountManager.scala │ └── MongoAccountsManagerSpec.scala ├── auth ├── build.sbt ├── configs │ └── dev │ │ ├── dev-auth-v1.conf │ │ └── dev-auth-v1.logging.xml └── src │ ├── main │ └── scala │ │ └── com │ │ └── precog │ │ └── auth │ │ ├── MongoAPIKeyManager.scala │ │ ├── MongoAPIKeyServer.scala │ │ ├── SecurityService.scala │ │ ├── SecurityServiceHandlers.scala │ │ └── WebAPIKeyManager.scala │ └── test │ └── scala │ └── com │ └── precog │ └── auth │ ├── MongoAPIKeyManagerSpec.scala │ ├── SecurityServiceSpec.scala │ └── WebAPIKeyFinderSpec.scala ├── bifrost ├── build.sbt ├── configs │ ├── VERSION │ └── dev │ │ ├── bifrost-v2.conf │ │ └── bifrost-v2.logging.xml ├── src │ ├── main │ │ ├── resources │ │ │ └── rebel.xml │ │ └── scala │ │ │ └── com │ │ │ └── precog │ │ │ ├── bifrost │ │ │ ├── JobQueryState.scala │ │ │ ├── ManagedPlatform.scala │ │ │ ├── ManagedQueryModule.scala │ │ │ ├── PerAccountThreadPoolModule.scala │ │ │ ├── QueryT.scala │ │ │ ├── RecoverJson.scala │ │ │ ├── ShardQueryExecutor.scala │ │ │ ├── ShardService.scala │ │ │ ├── TerminateJson.scala │ │ │ ├── UserQuery.scala │ │ │ ├── nihdb │ │ │ │ ├── NIHDBQueryExecutor.scala │ │ │ │ └── NIHDBShardServer.scala │ │ │ ├── service │ │ │ │ ├── AsyncQueryResultServiceHandler.scala │ │ │ │ ├── BrowseServiceHandler.scala │ │ │ │ ├── DataServiceHandler.scala │ │ │ │ ├── QueryDeleteHandler.scala │ │ │ │ ├── QueryServiceHandler.scala │ │ │ │ ├── ScheduleServiceHandlers.scala │ │ │ │ └── ShardServiceCombinators.scala │ │ │ └── util │ │ │ │ └── QueryBlast.scala │ │ │ └── shard.scala │ └── test │ │ ├── resources │ │ └── queries │ │ │ ├── clicks-times.qrl │ │ │ ├── enormous-crosses.qrl.pending │ │ │ ├── histogram1.qrl │ │ │ ├── isolated-clicks.qrl │ │ │ ├── sliding-window.qrl │ │ │ └── solve-with-reduction.qrl │ │ └── scala │ │ └── com │ │ └── precog │ │ └── bifrost │ │ ├── ManagedQueryExecutorSpec.scala │ │ ├── ManagedQueryModuleSpec.scala │ │ ├── RecoverJsonSpecs.scala │ │ ├── StorageMetadataClientSpecs.scala │ │ ├── TerminateJsonSpecs.scala │ │ ├── Ticker.scala │ │ └── service │ │ └── ShardServiceSpec.scala └── test.sh ├── build-test.sh ├── build.sbt ├── bytecode ├── build.sbt └── src │ ├── main │ └── scala │ │ └── com │ │ └── precog │ │ └── bytecode │ │ ├── Instructions.scala │ │ ├── JType.scala │ │ ├── Library.scala │ │ └── Version.scala │ └── test │ └── scala │ └── com │ └── precog │ └── bytecode │ ├── RandomLibrary.scala │ ├── StaticLibrary.scala │ └── UtilGenerators.scala ├── common ├── build.sbt └── src │ ├── main │ └── scala │ │ └── com │ │ └── precog │ │ ├── common.scala │ │ └── common │ │ ├── CPath.scala │ │ ├── CValue.scala │ │ ├── Codec.scala │ │ ├── ColumnRef.scala │ │ ├── Config.scala │ │ ├── JValueByteChunkTranscoder.scala │ │ ├── Metadata.scala │ │ ├── NetUtils.scala │ │ ├── Path.scala │ │ ├── SystemCoordination.scala │ │ ├── ZookeeperSystemCoordination.scala │ │ ├── accounts.scala │ │ ├── accounts │ │ ├── Account.scala │ │ ├── AccountDetails.scala │ │ ├── AccountFinder.scala │ │ ├── AccountRequiredService.scala │ │ ├── CachingAccountFinder.scala │ │ ├── StaticAccountFinder.scala │ │ └── WebAccountFinder.scala │ │ ├── backport │ │ └── lang │ │ │ ├── Boolean.scala │ │ │ └── Long.scala │ │ ├── client.scala │ │ ├── client │ │ └── BaseClient.scala │ │ ├── concurrency │ │ └── AuditThreadPool.scala │ │ ├── ingest │ │ ├── FileContent.scala │ │ ├── Ingest.scala │ │ └── IngestMessage.scala │ │ ├── jobs.scala │ │ ├── jobs │ │ ├── FileJobManager.scala │ │ ├── FileStorage.scala │ │ ├── InMemoryFileStorage.scala │ │ ├── InMemoryJobManager.scala │ │ ├── Job.scala │ │ ├── JobManager.scala │ │ ├── JobResult.scala │ │ ├── JobState.scala │ │ └── WebJobManager.scala │ │ ├── kafka │ │ └── KafkaCodecs.scala │ │ ├── security.scala │ │ ├── security │ │ ├── APIKey.scala │ │ ├── APIKeyFinder.scala │ │ ├── APIKeyManager.scala │ │ ├── AccessControl.scala │ │ ├── Authorities.scala │ │ ├── CachingAPIKeyFinder.scala │ │ ├── CachingAPIKeyManager.scala │ │ ├── Grant.scala │ │ ├── InMemoryAPIKeyManager.scala │ │ ├── Permission.scala │ │ ├── PermissionsFinder.scala │ │ ├── StaticAPIKeyFinder.scala │ │ └── service │ │ │ ├── APIKeyRequiredService.scala │ │ │ ├── WebAPIKeyFinder.scala │ │ │ └── v1.scala │ │ ├── serialization.scala │ │ └── services │ │ ├── CORSHeaderHandler.scala │ │ ├── EitherServiceCombinators.scala │ │ ├── PathServiceCombinators.scala │ │ ├── ServiceHandlerUtil.scala │ │ └── ServiceLocation.scala │ └── test │ ├── resources │ └── log4j.properties │ └── scala │ └── com │ └── precog │ └── common │ ├── CPathSpec.scala │ ├── CodecSpec.scala │ ├── MetadataSpec.scala │ ├── PathSpec.scala │ ├── ZookeeperSystemCoordinationSpec.scala │ ├── accounts │ └── TestAccountFinder.scala │ ├── ingest │ ├── ArchiveSpecs.scala │ ├── EventIdSpecs.scala │ ├── EventMessageSerializationSpec.scala │ └── EventSpec.scala │ ├── jobs │ └── FileStorageSpec.scala │ ├── security │ ├── APIKeyFinderSpec.scala │ ├── APIKeyManagerSpec.scala │ ├── AccessControlSpec.scala │ └── SerializationSpecs.scala │ └── util │ ├── ArbitraryEventMessage.scala │ ├── ArbitraryJValue.scala │ └── SampleSet.scala ├── direct-ingest.sh ├── dvergr ├── build.sbt ├── configs │ └── dev │ │ ├── jobs-v1.conf │ │ └── jobs-v1.logging.xml └── src │ ├── main │ └── scala │ │ └── com │ │ └── precog │ │ └── dvergr │ │ ├── AuthService.scala │ │ ├── GridFSFileStorage.scala │ │ ├── JobService.scala │ │ ├── JobServiceHandlers.scala │ │ ├── MongoJobManager.scala │ │ └── MongoJobServer.scala │ └── test │ └── scala │ └── com │ └── precog │ └── dvergr │ ├── GridFSFileStorageSpec.scala │ ├── JobManagerSpec.scala │ └── TestJobService.scala ├── gjallerhorn ├── build.sbt ├── scripts │ ├── SITA1.sh │ ├── keenful1.sh │ ├── keenful2.sh │ ├── keenful3.sh │ ├── precog1beta.sh │ ├── precog2beta.sh │ ├── snapengage1.sh │ └── snapengage2.sh └── src │ └── main │ ├── resources │ └── logback.xml │ └── scala │ └── com │ └── precog │ └── gjallerhorn │ ├── Account.scala │ ├── AccountsTask.scala │ ├── AnalyticsTask.scala │ ├── ApiResult.scala │ ├── IngestStress.scala │ ├── IngestTask.scala │ ├── MetadataTask.scala │ ├── RunAll.scala │ ├── Runner.scala │ ├── ScenariosTask.scala │ ├── SecurityTask.scala │ ├── Settings.scala │ └── Task.scala ├── headers ├── scala-header.txt └── sh-header.txt ├── ingest ├── README ├── build.sbt ├── configs │ ├── VERSION │ └── dev │ │ ├── ingest-v2.conf │ │ └── ingest-v2.logging.xml └── src │ ├── main │ └── scala │ │ └── com │ │ └── precog │ │ └── ingest │ │ ├── EventIdSequence.scala │ │ ├── EventService.scala │ │ ├── EventStore.scala │ │ ├── kafka │ │ ├── KafkaEventServer.scala │ │ ├── KafkaEventStore.scala │ │ └── KafkaRelayAgent.scala │ │ ├── service.scala │ │ ├── service │ │ ├── ArchiveServiceHandler.scala │ │ ├── CSVIngestProcessing.scala │ │ ├── FileStoreHandler.scala │ │ ├── IngestProcessor.scala │ │ ├── IngestServiceHandler.scala │ │ ├── IngestSupport.scala │ │ └── JSONIngestProcessing.scala │ │ └── util │ │ ├── CsvIngester.scala │ │ └── FutureUtils.scala │ └── test │ └── scala │ └── com │ └── precog │ └── ingest │ ├── kafka │ └── SystemEventIdSequenceSpec.scala │ ├── service │ ├── EventServiceSpec.scala │ └── TestEventService.scala │ └── util │ ├── DirectIngestBenchmark.scala │ └── WebappIngestBenchmark.scala ├── jprofiler ├── build.sbt ├── setup-jnilib.py └── src │ └── main │ ├── resources │ └── jprofile.xml │ └── scala │ └── com │ └── precog │ └── jprofiler │ └── Run.scala ├── logging └── src │ └── test │ └── resources │ ├── log4j.properties │ └── logback-test.xml ├── miklagard ├── desktop │ ├── .gitignore │ ├── README.rst │ ├── add-labcoat-nopg.sh │ ├── build-dist-jar.sh │ ├── build.sbt │ ├── desktop.install4j │ ├── installer_files │ │ ├── README.html │ │ ├── README.txt │ │ ├── Validations.class │ │ ├── Validations.java │ │ ├── icons │ │ │ ├── LargeIcon.png │ │ │ ├── P.icns │ │ │ ├── P.png │ │ │ ├── large_icon.ico │ │ │ ├── lc_icon.16.png │ │ │ ├── lc_icon.32.png │ │ │ ├── lc_icon.48.png │ │ │ ├── lc_icon48.icns │ │ │ └── logo-precog.png │ │ ├── mac-conf │ │ │ ├── shard-v2.conf │ │ │ └── shard-v2.logging.xml │ │ ├── shard-v2.conf │ │ ├── shard-v2.logging.xml │ │ └── unix-conf │ │ │ ├── shard-v2.conf │ │ │ └── shard-v2.logging.xml │ ├── lib │ │ ├── kafka-core-assembly-0.7.5.jar │ │ └── zookeeper-3.4.4.jar │ ├── proguard.conf │ ├── src │ │ └── main │ │ │ ├── resources │ │ │ ├── LargeIcon.png │ │ │ └── web │ │ │ │ └── version.html │ │ │ └── scala │ │ │ └── com │ │ │ └── precog │ │ │ └── desktop │ │ │ ├── DesktopIngestShardServer.scala │ │ │ └── DesktopShardServer.scala │ └── start-desktop-standalone.sh ├── jdbc │ ├── .gitignore │ ├── CHANGELOG.md │ ├── README.md │ ├── build-zip.sh │ ├── build.sbt │ ├── config.cfg │ ├── precog.bat │ ├── precog.sh │ ├── proguard.conf │ └── src │ │ ├── main │ │ ├── resources │ │ │ ├── logback.xml │ │ │ └── web │ │ │ │ ├── favicon.ico │ │ │ │ └── index.html │ │ └── scala │ │ │ └── com │ │ │ └── precog │ │ │ ├── shard │ │ │ └── jdbc │ │ │ │ ├── JDBCQueryExecutor.scala │ │ │ │ ├── JDBCShardServer.scala │ │ │ │ └── JDBCStorageMetadata.scala │ │ │ └── yggdrasil │ │ │ └── jdbc │ │ │ └── JDBCColumnarTableModule.scala │ │ └── test │ │ └── scala │ │ └── com │ │ └── precog │ │ └── yggdrasil │ │ └── jdbc │ │ └── JDBCPlatformSpecs.scala └── mongo │ ├── CHANGELOG.md │ ├── README.md │ ├── build-zip.sh │ ├── build.sbt │ ├── config.cfg │ ├── precog.bat │ ├── precog.sh │ ├── proguard.conf │ └── src │ ├── main │ ├── resources │ │ ├── logback.xml │ │ └── web │ │ │ ├── favicon.ico │ │ │ └── index.html │ └── scala │ │ └── com │ │ └── precog │ │ ├── shard │ │ ├── MongoShardServer.scala │ │ └── mongo │ │ │ ├── MongoQueryExecutor.scala │ │ │ └── MongoStorageMetadata.scala │ │ └── yggdrasil │ │ └── mongo │ │ └── MongoColumnarTableModule.scala │ └── test │ └── scala │ └── com │ └── precog │ └── yggdrasil │ └── mongo │ └── MongoPlatformSpecs.scala ├── mimir ├── build.sbt └── src │ ├── main │ └── scala │ │ └── com │ │ └── precog │ │ └── mimir │ │ ├── ArrayLib.scala │ │ ├── AssignClusters.scala │ │ ├── BigDecimalOperations.scala │ │ ├── Clustering.scala │ │ ├── CondRewriter.scala │ │ ├── CrossOrdering.scala │ │ ├── DAG.scala │ │ ├── DAGTransform.scala │ │ ├── Evaluator.scala │ │ ├── EvaluatorMethods.scala │ │ ├── FSLib.scala │ │ ├── InfixLib.scala │ │ ├── JoinOptimizer.scala │ │ ├── LinearRegression.scala │ │ ├── LogisticRegression.scala │ │ ├── MathLib.scala │ │ ├── Memoizer.scala │ │ ├── MemoryDatasetConsumer.scala │ │ ├── ModelLib.scala │ │ ├── Normalization.scala │ │ ├── OpFinder.scala │ │ ├── PrecogLib.scala │ │ ├── PredicatePullups.scala │ │ ├── PredictionLib.scala │ │ ├── QueryLogger.scala │ │ ├── RandomForest.scala │ │ ├── RandomLib.scala │ │ ├── RangeUtil.scala │ │ ├── ReductionFinder.scala │ │ ├── ReductionLib.scala │ │ ├── RegressionSupport.scala │ │ ├── StaticInliner.scala │ │ ├── StatsLib.scala │ │ ├── StdLib.scala │ │ ├── StdLibEvaluatorStack.scala │ │ ├── StringLib.scala │ │ ├── Summary.scala │ │ ├── TimeLib.scala │ │ ├── TransSpecable.scala │ │ ├── TreeHugger.scala │ │ ├── TypeInferencer.scala │ │ ├── TypeLib.scala │ │ └── UnaryLib.scala │ └── test │ ├── resources │ ├── clicks.json │ ├── clicks2.json │ ├── election │ │ └── tweets.json │ ├── het │ │ ├── array.json │ │ ├── arrays.json │ │ ├── heightWeight.json │ │ ├── heightWeightAcrossSlices.json │ │ ├── het-arrays.json │ │ ├── het-pairs.json │ │ ├── iso8601.json │ │ ├── iso8601AcrossSlices.json │ │ ├── millisSinceEpoch.json │ │ ├── millisTimeZone.json │ │ ├── numbers.json │ │ ├── numbers10.json │ │ ├── numbers2.json │ │ ├── numbers4.json │ │ ├── numbers6.json │ │ ├── numbers7.json │ │ ├── numbers8.json │ │ ├── numbers9.json │ │ ├── numbersAcrossSlices.json │ │ ├── numbersAcrossSlices2.json │ │ ├── numbersDoubleLong.json │ │ ├── pairs.json │ │ ├── random.json │ │ ├── stringNums.json │ │ ├── strings.json │ │ ├── strings2.json │ │ └── timeString.json │ ├── hom │ │ ├── arrays.json │ │ ├── clusteringData.json │ │ ├── clusteringModel.json │ │ ├── decimals.json │ │ ├── heightWeight.json │ │ ├── heightWeightAcrossSlices.json │ │ ├── heightWeight_neg.json │ │ ├── iso8601.json │ │ ├── iso8601AcrossSlices.json │ │ ├── millisSinceEpoch.json │ │ ├── millisTimeZone.json │ │ ├── model1.json │ │ ├── model1data.json │ │ ├── model2.json │ │ ├── model2data.json │ │ ├── number.json │ │ ├── numbers.json │ │ ├── numbers2.json │ │ ├── numbers3.json │ │ ├── numbers4.json │ │ ├── numbers5.json │ │ ├── numbers6.json │ │ ├── numbers7.json │ │ ├── numbersAcrossSlices.json │ │ ├── numbersHet.json │ │ ├── numbersmod.json │ │ ├── pairs.json │ │ ├── smallnumbers.json │ │ ├── strings.json │ │ ├── stuff │ │ │ └── numbersdiff.json │ │ ├── timeString.json │ │ ├── timerange.json │ │ └── users.json │ └── uncorrelated.json │ └── scala │ └── com │ └── precog │ └── mimir │ ├── ArrayLibSpecs.scala │ ├── ClusteringSpecs.scala │ ├── ClusteringTestSupport.scala │ ├── CrossOrderingSpecs.scala │ ├── DAGRewriterSpecs.scala │ ├── DAGSpecs.scala │ ├── EvaluatorSpecs.scala │ ├── FSLibSpecs.scala │ ├── FullStdLibModule.scala │ ├── JobQueryLoggerSpec.scala │ ├── JoinOptimizerSpecs.scala │ ├── LinearRegressionSpecs.scala │ ├── LogisticRegressionSpecs.scala │ ├── MathLibSpecs.scala │ ├── MemoizerSpecs.scala │ ├── NormalizationSpecs.scala │ ├── PrecogLibSpecs.scala │ ├── PredicatePullupSpecs.scala │ ├── RandomLibSpecs.scala │ ├── ReductionFinderSpecs.scala │ ├── ReductionLibSpecs.scala │ ├── RegressionTestSupport.scala │ ├── StaticInlinerSpecs.scala │ ├── StatsLibSpecs.scala │ ├── StringLibSpecs.scala │ ├── TimeComparisonSpecs.scala │ ├── TimeDifferenceSpecs.scala │ ├── TimeExtractionSpecs.scala │ ├── TimeMillisSpecs.scala │ ├── TimeParsingSpecs.scala │ ├── TimePeriodSpecs.scala │ ├── TimePlusSpecs.scala │ ├── TimeTruncationSpecs.scala │ ├── TimeZoneSpecs.scala │ ├── TypeInferencerSpecs.scala │ └── TypeLibSpecs.scala ├── mirror ├── build.sbt └── src │ ├── main │ └── scala │ │ └── com │ │ └── precog │ │ └── mirror │ │ ├── CLI.scala │ │ ├── Evaluator.scala │ │ └── Library.scala │ └── test │ └── scala │ └── com │ └── precog │ └── mirror │ └── EvaluatorSpecs.scala ├── muspelheim ├── build.sbt └── src │ ├── main │ └── scala │ │ └── com │ │ └── precog │ │ └── muspelheim │ │ └── ParseEvalStack.scala │ └── test │ ├── python │ └── newlinejson.py │ ├── resources │ └── test_data │ │ ├── auto-mpg.json │ │ ├── bar.json │ │ ├── byAccount.json │ │ ├── campaigns.json │ │ ├── clicks.json │ │ ├── clicks2.json │ │ ├── clicks20k.json │ │ ├── conversions.json │ │ ├── customers.json │ │ ├── devices.json │ │ ├── election │ │ ├── tweets.json │ │ └── tweets2.json │ │ ├── fastspring_mixed_type.json │ │ ├── fastspring_nulls.json │ │ ├── foo.json │ │ ├── iris.json │ │ ├── movie_ratings.json │ │ ├── obnoxious.json │ │ ├── orders.json │ │ ├── organizations.json │ │ ├── pageViews.json │ │ ├── payments.json │ │ ├── small.json │ │ ├── subscription.json │ │ ├── summer_games │ │ ├── athletes.json │ │ ├── historic_medals.json │ │ └── london_medals.json │ │ ├── test │ │ ├── empty_array.json │ │ ├── empty_object.json │ │ └── null.json │ │ ├── transaction.json │ │ ├── tutorial │ │ └── transactions.json │ │ ├── users.json │ │ └── views.json │ ├── ruby │ └── gen-assertions.rb │ └── scala │ └── com │ └── precog │ └── muspelheim │ ├── BasicValidationSpecs.scala │ ├── ClusteringSpecs.scala │ ├── EnrichmentSpecs.scala │ ├── EvalStackSpecs.scala │ ├── HelloQuirrelSpecs.scala │ ├── LinearRegressionSpecs.scala │ ├── LogisticRegressionSpecs.scala │ ├── MiscStackSpecs.scala │ ├── NonObjectStackSpecs.scala │ ├── NormalizationSpecs.scala │ ├── ParseEvalStackSpecs.scala │ ├── ProjectionMetadataSpecs.scala │ ├── RandomForestSpecs.scala │ ├── RandomStackSpecs.scala │ ├── RankSpecs.scala │ ├── RawJsonStorageModule.scala │ ├── RenderStackSpecs.scala │ └── UndefinedLiteralSpecs.scala ├── niflheim ├── build.sbt └── src │ ├── main │ └── scala │ │ └── com │ │ └── precog │ │ └── niflheim │ │ ├── CTypeFlags.scala │ │ ├── Chef.scala │ │ ├── Chunker.scala │ │ ├── CookStateLog.scala │ │ ├── CookStateLogSpecs.scala │ │ ├── CookedBlock.scala │ │ ├── CookedBlockFormat.scala │ │ ├── CookedReader.scala │ │ ├── NIHDBActor.scala │ │ ├── NIHDBSnapshot.scala │ │ ├── RawHandler.scala │ │ ├── RawLoader.scala │ │ ├── Reduction.scala │ │ ├── Segment.scala │ │ ├── SegmentFormat.scala │ │ ├── Segments.scala │ │ ├── StorageReader.scala │ │ ├── V1SegmentFormat.scala │ │ ├── VersionedSegmentFormat.scala │ │ └── Versioning.scala │ └── test │ └── scala │ └── com │ └── precog │ └── niflheim │ ├── CookedBlockSpecs.scala │ ├── RawHandlerSpecs.scala │ ├── SegmentFormatSpec.scala │ └── SegmentFormatSupport.scala ├── performance ├── build.sbt └── src │ ├── main │ └── scala │ │ └── com │ │ └── precog │ │ └── performance │ │ ├── Baseline.scala │ │ ├── Performance.scala │ │ └── PerformanceUtil.scala │ └── test │ └── scala │ └── com │ └── precog │ └── performance │ ├── PerformanceSpec.scala │ ├── PerformanceSuite.scala │ ├── RoutingPerformanceSpec.scala │ └── YggdrasilPerformanceSpec.scala ├── project ├── Build.scala ├── build.properties └── plugins.sbt ├── quirrel ├── build.sbt └── src │ ├── main │ └── scala │ │ └── com │ │ └── precog │ │ ├── quirrel.scala │ │ └── quirrel │ │ ├── Compiler.scala │ │ ├── Phases.scala │ │ ├── QuirrelCache.scala │ │ ├── QuirrelConsole.scala │ │ ├── Solver.scala │ │ ├── Tracer.scala │ │ ├── Utils.scala │ │ ├── emitter │ │ ├── Emitter.scala │ │ ├── GroupFinder.scala │ │ ├── GroupSolver.scala │ │ └── Solutions.scala │ │ ├── errors.scala │ │ ├── parser │ │ ├── AST.scala │ │ └── Parser.scala │ │ └── typer │ │ ├── Binder.scala │ │ ├── ProvenanceChecker.scala │ │ └── TreeShaker.scala │ └── test │ └── scala │ └── com │ └── precog │ └── quirrel │ ├── CompilerUtils.scala │ ├── LineErrorsSpecs.scala │ ├── PhasesSpecs.scala │ ├── QuirrelCacheSpecs.scala │ ├── RandomLibrarySpec.scala │ ├── SolverSpecs.scala │ ├── StaticLibrarySpec.scala │ ├── StubPhases.scala │ ├── emitter │ ├── EmitterSpecs.scala │ └── GroupSolverSpecs.scala │ ├── parser │ └── ParserSpecs.scala │ └── typer │ ├── BinderSpecs.scala │ ├── InfinityCheckerSpecs.scala │ ├── NullProvenanceSpecs.scala │ ├── ProvenanceCheckingSpecs.scala │ ├── ProvenanceComputationSpecs.scala │ ├── RelationSpecs.scala │ └── TreeShakerSpecs.scala ├── ragnarok ├── disjunction-notes-cross.txt ├── disjunction-notes-no-cross.txt ├── medals-notes.txt ├── scripts │ ├── disjunction.sh │ └── medals.sh └── src │ ├── main │ ├── resources │ │ └── logback.xml │ └── scala │ │ └── com │ │ └── precog │ │ └── ragnarok │ │ ├── BaselineComparisons.scala │ │ ├── EvaluatingPerfTestRunner.scala │ │ ├── IngestTest.scala │ │ ├── JsonPerfTestRunner.scala │ │ ├── MetricSpace.scala │ │ ├── NIHDBIngestSupport.scala │ │ ├── NIHDBPerfTestRunner.scala │ │ ├── NihdbStressTest.scala │ │ ├── PerfTest.scala │ │ ├── PerfTestPrettyPrinters.scala │ │ ├── PerfTestRunner.scala │ │ ├── PerfTestSuite.scala │ │ ├── PerfTestUtil.scala │ │ ├── PlatformPerfTests.scala │ │ ├── RunConfig.scala │ │ ├── Statistics.scala │ │ ├── Timer.scala │ │ └── test │ │ ├── ArrayObjectSuite.scala │ │ ├── AthletesTestSuite.scala │ │ ├── ClicksLikePerfTestSuite.scala │ │ ├── ClicksTestSuite.scala │ │ ├── ConversionsTestSuite.scala │ │ ├── DisjunctionTestSuite.scala │ │ ├── GroupingTestSuite.scala │ │ ├── ImpressionsTestSuite.scala │ │ ├── KeenfulTestSuite1.scala │ │ ├── KeenfulTestSuite2.scala │ │ ├── KeenfulTestSuite3.scala │ │ ├── LinearRegressionTestSuite.scala │ │ ├── MedalsTestSuite.scala │ │ ├── MiscStackSpecsSuite.scala │ │ ├── NoDataQuery.scala │ │ ├── Platform868.scala │ │ ├── Platform967.scala │ │ ├── SitaAssign.scala │ │ ├── SitaClustering.scala │ │ ├── SitaTestSuite.scala │ │ └── SnapTestSuite.scala │ └── test │ └── scala │ └── com │ └── precog │ └── ragnarok │ ├── PerfTestRunnerSpec.scala │ ├── PerfTestSuiteSpec.scala │ └── StatisticsSpec.scala ├── ratatoskr ├── build.sbt └── src │ └── main │ └── scala │ └── com │ └── precog │ └── ratatoskr │ └── Ratatoskr.scala ├── run.sh ├── scripts ├── delta.py ├── fixjdbm.scala ├── fixjdbm.sh ├── newActivity.py ├── parse-kafka.rb └── shardtool ├── standalone └── src │ └── main │ └── scala │ └── com │ └── precog │ └── standalone │ ├── StandaloneIngestServer.scala │ ├── StandaloneQueryExecutor.scala │ └── StandaloneShardServer.scala ├── start-shard.sh ├── surtr ├── build.sbt ├── dist │ ├── README │ └── bin │ │ └── repl ├── dist_private │ ├── bin │ │ ├── init │ │ └── insert │ └── conf │ │ ├── init.properties │ │ └── insert.properties └── src │ ├── main │ ├── resources │ └── scala │ │ └── com │ │ └── precog │ │ └── surtr │ │ ├── Color.scala │ │ ├── REPL.scala │ │ └── SBTConsole.scala │ └── test │ └── scala │ └── com │ └── precog │ └── surtr │ ├── NIHDBFileStoreSpec.scala │ └── NIHDBPlatformSpecs.scala ├── util ├── build.sbt └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── precog │ │ │ └── util │ │ │ └── BitSet.java │ └── scala │ │ └── com │ │ └── precog │ │ ├── util.scala │ │ └── util │ │ ├── Bijection.scala │ │ ├── BitSetUtil.scala │ │ ├── ByteBufferPool.scala │ │ ├── Close.scala │ │ ├── DateTimeUtil.scala │ │ ├── FileLock.scala │ │ ├── FileOps.scala │ │ ├── HttpClient.scala │ │ ├── IOUtils.scala │ │ ├── IdGen.scala │ │ ├── Identifier.scala │ │ ├── IntList.scala │ │ ├── Interval.scala │ │ ├── JBigDecimalOrdering.scala │ │ ├── JsonUtil.scala │ │ ├── KMap.scala │ │ ├── Levenshtein.scala │ │ ├── Loop.scala │ │ ├── MapUtils.scala │ │ ├── NumericComparisons.scala │ │ ├── PrecogUnit.scala │ │ ├── RawBitSet.scala │ │ ├── RingDeque.scala │ │ ├── Timing.scala │ │ ├── VectorCase.scala │ │ ├── VectorClock.scala │ │ ├── XLightWebHttpClient.scala │ │ ├── atoms.scala │ │ ├── cache │ │ └── Cache.scala │ │ ├── email │ │ └── TemplateEmailer.scala │ │ └── scalaz_extensions.scala │ └── test │ └── scala │ └── com │ └── precog │ └── util │ ├── AtomSpecs.scala │ ├── EchoHttpClient.scala │ ├── IOUtilsSpecs.scala │ ├── LevenshteinSpecs.scala │ ├── MapUtilsSpecs.scala │ ├── RingDequeSpecs.scala │ ├── VectorClockSpec.scala │ └── XLightWebHttpClientSpecs.scala ├── yggdrasil ├── build.sbt ├── conf │ └── dev │ │ └── ygg_shard.conf ├── lib │ └── jdbm-3.0-SNAPSHOT.jar └── src │ ├── main │ └── scala │ │ └── com │ │ ├── precog.scala │ │ └── precog │ │ └── yggdrasil │ │ ├── EnormousCartesianException.scala │ │ ├── FNModule.scala │ │ ├── IdSource.scala │ │ ├── PersistentJValue.scala │ │ ├── ProjectionModule.scala │ │ ├── SValue.scala │ │ ├── Schema.scala │ │ ├── StorageFormat.scala │ │ ├── TableModule.scala │ │ ├── TransSpecModule.scala │ │ ├── YggConfig.scala │ │ ├── actor.scala │ │ ├── actor │ │ ├── BatchHandler.scala │ │ ├── KafkaShardIngestActor.scala │ │ ├── ProjectionsActor.scala │ │ ├── ShardSystemActor.scala │ │ └── ShardSystemActorConfigs.scala │ │ ├── execution │ │ ├── EvaluationContext.scala │ │ ├── Platform.scala │ │ └── QueryExecutor.scala │ │ ├── jdbm3 │ │ ├── ByteArraySerializer.scala │ │ ├── CValueSerializer.scala │ │ ├── IdentitiesComparator.scala │ │ ├── JDBMProjection.scala │ │ ├── JDBMRawSortProjection.scala │ │ ├── JDBMSlice.scala │ │ ├── RowFormat.scala │ │ ├── SortingSerialization.scala │ │ └── StdCodecs.scala │ │ ├── metadata.scala │ │ ├── metadata │ │ └── StorageMetadata.scala │ │ ├── nihdb │ │ ├── NIHDBProjection.scala │ │ ├── SegmentsWrapper.scala │ │ └── StressTest.scala │ │ ├── scheduling │ │ ├── InMemoryScheduleStorage.scala │ │ ├── MongoScheduleStorage.scala │ │ ├── ScheduleStorage.scala │ │ ├── ScheduledRunReport.scala │ │ ├── ScheduledTask.scala │ │ ├── Scheduler.scala │ │ └── SchedulingActor.scala │ │ ├── serialization │ │ ├── StreamSerialization.scala │ │ └── bijections.scala │ │ ├── table │ │ ├── ArrayColumn.scala │ │ ├── BlockStoreColumnarTableModule.scala │ │ ├── CFN.scala │ │ ├── CPathTraversal.scala │ │ ├── Column.scala │ │ ├── ColumnSupport.scala │ │ ├── ColumnarTableModule.scala │ │ ├── DerefSlice.scala │ │ ├── HashedSlice.scala │ │ ├── IndicesModule.scala │ │ ├── MemoColumn.scala │ │ ├── RowComparator.scala │ │ ├── SamplableTableModule.scala │ │ ├── Slice.scala │ │ ├── SliceColumnarTableModule.scala │ │ ├── SliceTransform.scala │ │ ├── VFSColumnarTableModule.scala │ │ └── cf │ │ │ ├── Math.scala │ │ │ ├── Std.scala │ │ │ └── Util.scala │ │ ├── util │ │ ├── CPathComparator.scala │ │ ├── CPathUtils.scala │ │ ├── HetOrder.scala │ │ └── IdSourceScannerModule.scala │ │ ├── vfs.scala │ │ └── vfs │ │ ├── ActorVFS.scala │ │ ├── BlobMetadata.scala │ │ ├── FileSystem.scala │ │ ├── InMemoryVFS.scala │ │ ├── ResourceError.scala │ │ ├── SecureVFS.scala │ │ ├── VFS.scala │ │ ├── VFSPathUtils.scala │ │ └── VersionLog.scala │ └── test │ ├── resources │ └── z10k_nl.json │ └── scala │ └── com │ └── precog │ └── yggdrasil │ ├── ArbitraryProjectionDescriptor.scala │ ├── ArbitrarySValue.scala │ ├── CanonicalizeSpecs.scala │ ├── CogroupSpec.scala │ ├── ConcatSpecs.scala │ ├── CrossSpec.scala │ ├── PrecogJValueOrdering.scala │ ├── SValueSpec.scala │ ├── SampleData.scala │ ├── SchemaSpecs.scala │ ├── StubStorageModule.scala │ ├── TableModuleTestSupport.scala │ ├── TakeRangeSpecs.scala │ ├── ToArraySpecs.scala │ ├── TransSpecModuleSpec.scala │ ├── TransformSpec.scala │ ├── actor │ └── RoutingTableSpec.scala │ ├── jdbm3 │ ├── CValueGenerators.scala │ └── RowFormatSpec.scala │ ├── kafka │ └── ProjectionActorSpec.scala │ ├── nihdb │ └── NIHDBProjectionSpecs.scala │ ├── packageSpec.scala │ ├── table │ ├── ArbitrarySlice.scala │ ├── BlockAlignSpec.scala │ ├── BlockLoadSpec.scala │ ├── BlockSortSpec.scala │ ├── BlockStoreColumnarTableModuleSpec.scala │ ├── BlockStoreTestSupport.scala │ ├── ColumnarTableModuleSpec.scala │ ├── ColumnarTableModuleTestSupport.scala │ ├── CompactSpec.scala │ ├── DistinctSpec.scala │ ├── FNSpec.scala │ ├── GrouperSpec.scala │ ├── IndicesSpec.scala │ ├── MergeSpec.scala │ ├── PartitionMergeSpec.scala │ ├── RowIteratorSpec.scala │ ├── SampleSpec.scala │ ├── SchemasSpec.scala │ ├── SliceSpec.scala │ ├── StubColumnarTableModule.scala │ ├── StubIdSourceScannerModule.scala │ └── TableSpec.scala │ ├── test │ └── YId.scala │ ├── util │ ├── CPathTraversalSpec.scala │ └── IdSourceScannerModuleSpec.scala │ └── vfs │ └── StubVFSMetadata.scala └── yggdrasilProf └── src └── main ├── resources └── jprofile.xml └── scala └── com └── precog └── yggdrasil └── test └── Run.scala /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | .*.marks 3 | .classpath 4 | .project 5 | .settings 6 | .history 7 | ingest.*.log 8 | tags 9 | blueeyes 10 | client-libraries 11 | scalaz 12 | .idea*/ 13 | .ensime 14 | quirrel/examples 15 | hs_err_pid*log 16 | *~ 17 | *.hprof 18 | *.swp 19 | *.out 20 | .DS_Store 21 | tools/lib/* 22 | yggdrasil/data 23 | /TAGS 24 | /standaloneArtifacts 25 | *.jps 26 | jprofiler.db 27 | jprofiler.jnilib 28 | .cache 29 | /mongo/proguard.log 30 | /mongo/precog*.zip 31 | /mongo/precog/ 32 | /jdbc/proguard.log 33 | /jdbc/precog*.zip 34 | /jdbc/precog/ 35 | .tag 36 | /personal 37 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "mongo/quirrelide"] 2 | path = mongo/quirrelide 3 | url = git@github.com:precog/quirrelide.git 4 | -------------------------------------------------------------------------------- /.jvmopts: -------------------------------------------------------------------------------- 1 | -Xms4096m 2 | -Xmx4096m 3 | -XX:MaxPermSize=1024m 4 | -XX:ReservedCodeCacheSize=512m 5 | -Xloggc:target/gc.log 6 | -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- 1 | Alissa Pajer 2 | Alissa Pajer 3 | Brian McKenna 4 | Brian McKenna 5 | Daniel Spiewak 6 | Daniel Spiewak 7 | Derek Chen-Becker 8 | Derek Chen-Becker 9 | Erik Osheim 10 | John A. De Goes 11 | Kris Nuttycombe 12 | Kris Nuttycombe 13 | Kris Nuttycombe 14 | Miles Sabin 15 | Miles Sabin 16 | Nathan Lubchenco 17 | Nick Matheson 18 | Tom Bowles 19 | Tom Switzer 20 | Tom Switzer -------------------------------------------------------------------------------- /accounts/build.sbt: -------------------------------------------------------------------------------- 1 | // 2 | // ____ ____ _____ ____ ___ ____ 3 | // | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | // | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | // | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | // |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | // 8 | // This program is free software: you can redistribute it and/or modify it under the terms of the 9 | // GNU Affero General Public License as published by the Free Software Foundation, either version 10 | // 3 of the License, or (at your option) any later version. 11 | // 12 | // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | // the GNU Affero General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Affero General Public License along with this 17 | // program. If not, see . 18 | // 19 | // 20 | 21 | name := "accounts" 22 | 23 | libraryDependencies ++= Seq( 24 | "ch.qos.logback" % "logback-classic" % "1.0.0", 25 | "org.jvnet.mock-javamail" % "mock-javamail" % "1.9" % "test" 26 | ) 27 | 28 | mainClass := Some("com.precog.accounts.MongoAccountServer") 29 | -------------------------------------------------------------------------------- /accounts/configs/dev/accounts-v1.conf: -------------------------------------------------------------------------------- 1 | server { 2 | port = 30064 3 | sslPort = 30065 4 | sslEnable = false 5 | 6 | log { 7 | level = "info" 8 | console = true 9 | filename = "/var/log/precog/accounts-v1.server.log" 10 | roll = "daily" 11 | use_parents = false 12 | } 13 | } 14 | 15 | services { 16 | accounts { 17 | v1 { 18 | log { 19 | level = "debug" 20 | console = true 21 | filename = "/var/log/precog/accounts-v1.log" 22 | roll = "daily" 23 | use_parents = false 24 | } 25 | 26 | requestLog { 27 | enabled = true 28 | file = "/var/log/precog/accounts-v1.request.log" 29 | roll = "daily" 30 | fields = "date time c-ip cs-method cs-uri-stem cs-uri-query sc-status cs-content" 31 | formatter = "w3cs" 32 | excludePaths = ["/accounts/v1/health"] 33 | } 34 | 35 | zookeeper { 36 | hosts = localhost:2181 37 | accountId { 38 | path = "/accounts/beta/v1/accountId" 39 | } 40 | } 41 | 42 | email { 43 | params { 44 | servicehost = "devapi.precog.com" 45 | } 46 | template_dir = "/etc/precog/templates" 47 | } 48 | 49 | mongo { 50 | servers = ["localhost"] 51 | database = "dev_accounts_v1" 52 | collection = "accounts" 53 | deletedCollection = "deleted_accounts" 54 | } 55 | 56 | accounts { 57 | rootAccountId = 0000000001 58 | } 59 | 60 | security { 61 | service { 62 | protocol = "http" 63 | host = "localhost" 64 | port = 30062 65 | path = "/security/v1/" 66 | } 67 | cached = true 68 | rootKey = "CE1DE42A-D9EA-4494-8240-680230067C7C" 69 | } 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /accounts/configs/dev/accounts-v1.logging.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | ${LOG_DIR}/${SERVICE}.server.log 7 | 8 | 9 | ${LOG_DIR}/${SERVICE}.server-%d{yyyyMMdd}.log 10 | 11 | 12 | 30 13 | 14 | 15 | %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /accounts/configs/reset.eml.html.mustache: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |

5 | Hello 6 |

7 | 8 |

9 | We have received a request for a password reset token from {{requestor}} 10 | at {{time}}. If you did not request this, please reply to this email and let 11 | us know. 12 |

13 | 14 |

15 | Please click this link to reset your password. 16 |

17 | 18 |

19 | If you have any questions or concerns, please reply to this email. 20 |

21 | 22 |

23 | Thank you, 24 |

25 | 26 |

27 | Precog Support
28 | support@precog.com 29 |

30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /accounts/configs/reset.eml.txt.mustache: -------------------------------------------------------------------------------- 1 | Hello, 2 | We have received a request for a password reset token from {{requestor}} 3 | at {{time}}. If you did not request this, please reply to this email and let 4 | us know. Visit the link below to reset your password 5 | 6 | https://www.precog.com/reset-password-complete?service={{servicehost}}&accountId={{accountId}}&token={{token}} 7 | 8 | If you have any questions or concerns, please reply to this email. 9 | 10 | Thank you, 11 | 12 | Precog Support 13 | support@precog.com 14 | -------------------------------------------------------------------------------- /accounts/configs/reset.subj.mustache: -------------------------------------------------------------------------------- 1 | Your Precog password reset request -------------------------------------------------------------------------------- /accounts/src/test/resources/reset.eml.html.mustache: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |

5 | Hello 6 |

7 | 8 |

9 | We have received a request for a password reset token from {{requestor}} 10 | at {{time}}. If you did not request this, please reply to this email and let 11 | us know. 12 |

13 | 14 |

15 | Please click this link to reset your password. 16 |

17 | 18 |

19 | If you have any questions or concerns, please reply to this email. 20 |

21 | 22 |

23 | Thank you, 24 |

25 | 26 |

27 | Precog Support
28 | support@precog.com 29 |

30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /accounts/src/test/resources/reset.eml.txt.mustache: -------------------------------------------------------------------------------- 1 | Hello, 2 | We have received a request for a password reset token from {{requestor}} 3 | at {{time}}. If you did not request this, please reply to this email and let 4 | us know. Visit the link below to reset your password 5 | 6 | https://www.precog.com/reset-password-complete?service={{servicehost}}&accountId={{accountId}}&token={{token}} 7 | 8 | If you have any questions or concerns, please reply to this email. 9 | 10 | Thank you, 11 | 12 | Precog Support 13 | support@precog.com 14 | -------------------------------------------------------------------------------- /accounts/src/test/resources/reset.subj.mustache: -------------------------------------------------------------------------------- 1 | {{token}} -------------------------------------------------------------------------------- /auth/build.sbt: -------------------------------------------------------------------------------- 1 | // 2 | // ____ ____ _____ ____ ___ ____ 3 | // | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | // | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | // | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | // |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | // 8 | // This program is free software: you can redistribute it and/or modify it under the terms of the 9 | // GNU Affero General Public License as published by the Free Software Foundation, either version 10 | // 3 of the License, or (at your option) any later version. 11 | // 12 | // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | // the GNU Affero General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Affero General Public License along with this 17 | // program. If not, see . 18 | // 19 | // 20 | 21 | name := "auth" 22 | 23 | libraryDependencies ++= Seq( 24 | "ch.qos.logback" % "logback-classic" % "1.0.0" 25 | ) 26 | 27 | mainClass := Some("com.precog.auth.MongoAPIKeyServer") 28 | -------------------------------------------------------------------------------- /auth/configs/dev/dev-auth-v1.conf: -------------------------------------------------------------------------------- 1 | server { 2 | port = 30062 3 | sslPort = 30063 4 | sslEnable = false 5 | 6 | log { 7 | level = "info" 8 | console = true 9 | filename = "/var/log/precog/auth-v1.server.log" 10 | roll = "daily" 11 | use_parents = false 12 | } 13 | } 14 | 15 | services { 16 | security { 17 | v1 { 18 | log { 19 | level = "debug" 20 | console = true 21 | filename = "/var/log/precog/auth-v1.log" 22 | roll = "daily" 23 | use_parents = false 24 | } 25 | 26 | requestLog { 27 | enabled = true 28 | file = "/var/log/precog/auth-v1.request.log" 29 | roll = "daily" 30 | fields = "date time c-ip cs-method cs-uri-stem cs-uri-query sc-status cs-content" 31 | formatter = "w3cs" 32 | excludePaths = ["/security/v1/health"] 33 | } 34 | 35 | security { 36 | mongo { 37 | servers = ["localhost"] 38 | database = "dev_auth_v1" 39 | collection = "tokens" 40 | } 41 | cached = true 42 | rootKey = "CE1DE42A-D9EA-4494-8240-680230067C7C" 43 | } 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /auth/configs/dev/dev-auth-v1.logging.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | ${LOG_DIR}/${SERVICE}.server.log 7 | 8 | 9 | ${LOG_DIR}/${SERVICE}.server-%d{yyyyMMdd}.log 10 | 11 | 12 | 30 13 | 14 | 15 | %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /auth/src/main/scala/com/precog/auth/MongoAPIKeyServer.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog 21 | package auth 22 | 23 | import common.security._ 24 | 25 | import akka.dispatch.Future 26 | 27 | import blueeyes.bkka._ 28 | import blueeyes.BlueEyesServer 29 | 30 | import org.streum.configrity.Configuration 31 | 32 | import scalaz._ 33 | 34 | object MongoAPIKeyServer extends BlueEyesServer with SecurityService with AkkaDefaults { 35 | implicit val executionContext = defaultFutureDispatch 36 | implicit val M: Monad[Future] = new FutureMonad(executionContext) 37 | def APIKeyManager(config: Configuration): (APIKeyManager[Future], Stoppable) = MongoAPIKeyManager(config) 38 | val clock = blueeyes.util.Clock.System 39 | } 40 | -------------------------------------------------------------------------------- /bifrost/build.sbt: -------------------------------------------------------------------------------- 1 | // 2 | // ____ ____ _____ ____ ___ ____ 3 | // | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | // | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | // | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | // |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | // 8 | // This program is free software: you can redistribute it and/or modify it under the terms of the 9 | // GNU Affero General Public License as published by the Free Software Foundation, either version 10 | // 3 of the License, or (at your option) any later version. 11 | // 12 | // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | // the GNU Affero General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Affero General Public License along with this 17 | // program. If not, see . 18 | // 19 | // 20 | 21 | name := "bifrost" 22 | 23 | libraryDependencies ++= Seq( 24 | "ch.qos.logback" % "logback-classic" % "1.0.0" 25 | ) 26 | 27 | mainClass := Some("com.precog.bifrost.nihdb.NIHDBShardServer") 28 | -------------------------------------------------------------------------------- /bifrost/configs/VERSION: -------------------------------------------------------------------------------- 1 | V2 2 | -------------------------------------------------------------------------------- /bifrost/configs/dev/bifrost-v2.logging.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | ${LOG_DIR}/${SERVICE}.server.log 7 | 8 | 9 | ${LOG_DIR}/${SERVICE}.server-%d{yyyyMMdd}.log 10 | 11 | 12 | 30 13 | 14 | 15 | %d{ISO8601} [%.10thread] %.-1level %logger{20} {%X} - %msg%n 16 | 17 | 18 | 19 | 20 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /bifrost/src/main/resources/rebel.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /bifrost/src/main/scala/com/precog/shard.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog 21 | 22 | import com.precog.mimir._ 23 | import com.precog.common.security._ 24 | import com.precog.common.jobs._ 25 | import com.precog.yggdrasil.execution._ 26 | import com.precog.yggdrasil.table.Slice 27 | 28 | import akka.dispatch.Future 29 | import scalaz.StreamT 30 | import blueeyes.json.JValue 31 | import java.nio.CharBuffer 32 | 33 | package object bifrost { 34 | type QueryResult = Either[JValue, StreamT[Future, CharBuffer]] 35 | 36 | type JobQueryT[M[+_], +A] = QueryT[JobQueryState, M, A] 37 | type JobQueryTF[+A] = JobQueryT[Future, A] 38 | } 39 | -------------------------------------------------------------------------------- /bifrost/src/test/resources/queries/clicks-times.qrl: -------------------------------------------------------------------------------- 1 | clicks := ./clicks 2 | 3 | solve 'time = clicks.time 4 | belowTime := max(clicks.time where clicks.time < 'time) 5 | aboveTime := min(clicks.time where clicks.time > 'time) 6 | 7 | { 8 | time: 'time, 9 | below: belowTime, 10 | above: aboveTime 11 | } 12 | -------------------------------------------------------------------------------- /bifrost/src/test/resources/queries/enormous-crosses.qrl.pending: -------------------------------------------------------------------------------- 1 | tweets := ./tweets 2 | tweets' := new tweets 3 | 4 | tweets ~ tweets' 5 | [tweets, tweets'] 6 | -------------------------------------------------------------------------------- /bifrost/src/test/resources/queries/histogram1.qrl: -------------------------------------------------------------------------------- 1 | campaigns := ./campaigns 2 | organizations := ./organizations 3 | 4 | solve 'revenue = organizations.revenue & 'campaign = organizations.campaign 5 | campaigns' := campaigns where campaigns.campaign = 'campaign 6 | { revenue: 'revenue, num: count(campaigns') } 7 | -------------------------------------------------------------------------------- /bifrost/src/test/resources/queries/isolated-clicks.qrl: -------------------------------------------------------------------------------- 1 | clicks := ./clicks 2 | 3 | spacings := solve 'time 4 | click := clicks where clicks.time = 'time 5 | belowTime := max(clicks.time where clicks.time < 'time) 6 | aboveTime := min(clicks.time where clicks.time > 'time) 7 | 8 | { 9 | click: click, 10 | below: click.time - belowTime, 11 | above: aboveTime - click.time 12 | } 13 | 14 | meanAbove := mean(spacings.above) 15 | meanBelow := mean(spacings.below) 16 | 17 | spacings.click where spacings.below > meanBelow & spacings.above > meanAbove 18 | -------------------------------------------------------------------------------- /bifrost/src/test/resources/queries/sliding-window.qrl: -------------------------------------------------------------------------------- 1 | campaigns := ./campaigns 2 | nums := distinct(campaigns.cpm where campaigns.cpm < 10) 3 | solve 'n 4 | m := max(nums where nums < 'n) 5 | (nums where nums = 'n) + m 6 | -------------------------------------------------------------------------------- /bifrost/src/test/resources/queries/solve-with-reduction.qrl: -------------------------------------------------------------------------------- 1 | clicks := ./clicks 2 | 3 | countsForTimezone := solve 'timeZone 4 | clicksForZone := clicks where clicks.timeZone = 'timeZone 5 | {timeZone: 'timeZone, clickCount: count(clicksForZone)} 6 | 7 | mostClicks := max(countsForTimezone.clickCount) 8 | 9 | countsForTimezone where countsForTimezone.clickCount = mostClicks 10 | -------------------------------------------------------------------------------- /bifrost/test.sh: -------------------------------------------------------------------------------- 1 | ## 2 | ## ____ ____ _____ ____ ___ ____ 3 | ## | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | ## | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | ## | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | ## |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | ## 8 | ## This program is free software: you can redistribute it and/or modify it under the terms of the 9 | ## GNU Affero General Public License as published by the Free Software Foundation, either version 10 | ## 3 of the License, or (at your option) any later version. 11 | ## 12 | ## This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | ## without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | ## the GNU Affero General Public License for more details. 15 | ## 16 | ## You should have received a copy of the GNU Affero General Public License along with this 17 | ## program. If not, see . 18 | ## 19 | ## 20 | #!/bin/bash 21 | cd $(dirname $0)/.. 22 | exec ./run.sh $* -q bifrost/src/test/resources/queries/ muspelheim/src/test/resources/test_data/{campaigns,clicks,organizations,election/tweets}.json 23 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | // 2 | // ____ ____ _____ ____ ___ ____ 3 | // | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | // | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | // | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | // |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | // 8 | // This program is free software: you can redistribute it and/or modify it under the terms of the 9 | // GNU Affero General Public License as published by the Free Software Foundation, either version 10 | // 3 of the License, or (at your option) any later version. 11 | // 12 | // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | // the GNU Affero General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Affero General Public License along with this 17 | // program. If not, see . 18 | // 19 | // 20 | 21 | import sbt._ 22 | import Keys._ 23 | 24 | parallelExecution := false 25 | 26 | (test in Test) <<= (test in Test, test in LocalProject("surtr")) map { (_, _) => () } 27 | -------------------------------------------------------------------------------- /bytecode/build.sbt: -------------------------------------------------------------------------------- 1 | // 2 | // ____ ____ _____ ____ ___ ____ 3 | // | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | // | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | // | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | // |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | // 8 | // This program is free software: you can redistribute it and/or modify it under the terms of the 9 | // GNU Affero General Public License as published by the Free Software Foundation, either version 10 | // 3 of the License, or (at your option) any later version. 11 | // 12 | // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | // the GNU Affero General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Affero General Public License along with this 17 | // program. If not, see . 18 | // 19 | // 20 | 21 | name := "bytecode" 22 | 23 | logBuffered := false 24 | 25 | publishArtifact in packageDoc := false 26 | 27 | initialCommands in console := """ 28 | | import com.precog.bytecode._ 29 | | import java.nio.ByteBuffer 30 | | 31 | | val cake = new BytecodeReader with BytecodeWriter with DAG with util.DAGPrinter 32 | | 33 | | def printBuffer(buffer: ByteBuffer) { 34 | | try { 35 | | val b = buffer.get() 36 | | printf("%02X ", b) 37 | | } catch { 38 | | case _ => return 39 | | } 40 | | printBuffer(buffer) 41 | | }""".stripMargin 42 | -------------------------------------------------------------------------------- /bytecode/src/main/scala/com/precog/bytecode/Version.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.bytecode 21 | 22 | trait Version { 23 | val Major = 0 24 | val Minor = 1 25 | val Release = 0 26 | } 27 | -------------------------------------------------------------------------------- /bytecode/src/test/scala/com/precog/bytecode/UtilGenerators.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.bytecode 21 | 22 | import org.scalacheck._ 23 | 24 | trait UtilGenerators { 25 | import Arbitrary.arbitrary 26 | import Gen._ 27 | 28 | implicit def arbVector[A : Arbitrary]: Arbitrary[Vector[A]] = 29 | Arbitrary(genVector[A]) 30 | 31 | private def genVector[A : Arbitrary]: Gen[Vector[A]] = 32 | listOf(implicitly[Arbitrary[A]].arbitrary) map { xs => Vector(xs: _*) } 33 | } 34 | -------------------------------------------------------------------------------- /common/src/main/scala/com/precog/common.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog 21 | 22 | package object common { 23 | type ProducerId = Int 24 | type SequenceId = Int 25 | 26 | final class StringExtensions(s: String) { 27 | def cpath = CPath(s) 28 | } 29 | 30 | implicit def stringExtensions(s: String) = new StringExtensions(s) 31 | } 32 | 33 | 34 | // vim: set ts=4 sw=4 et: 35 | -------------------------------------------------------------------------------- /common/src/main/scala/com/precog/common/ColumnRef.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.common 21 | 22 | import scalaz.syntax.semigroup._ 23 | import scalaz.syntax.order._ 24 | 25 | case class ColumnRef(selector: CPath, ctype: CType) 26 | 27 | object ColumnRef { 28 | def identity(ctype: CType) = ColumnRef(CPath.Identity, ctype) 29 | 30 | implicit object order extends scalaz.Order[ColumnRef] { 31 | def order(r1: ColumnRef, r2: ColumnRef): scalaz.Ordering = { 32 | (r1.selector ?|? r2.selector) |+| (r1.ctype ?|? r2.ctype) 33 | } 34 | } 35 | 36 | implicit val ordering: scala.math.Ordering[ColumnRef] = order.toScalaOrdering 37 | } 38 | -------------------------------------------------------------------------------- /common/src/main/scala/com/precog/common/Config.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.common 21 | 22 | import org.streum.configrity.Configuration 23 | 24 | trait Config { 25 | def config: Configuration 26 | } 27 | -------------------------------------------------------------------------------- /common/src/main/scala/com/precog/common/NetUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.common 21 | 22 | import blueeyes.core.http.HttpRequest 23 | 24 | object NetUtils { 25 | def remoteIpFrom(req: HttpRequest[_]) = req.remoteHost.map(_.getHostAddress).getOrElse("Unknown") 26 | } 27 | -------------------------------------------------------------------------------- /common/src/main/scala/com/precog/common/accounts.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.common 21 | 22 | package object accounts { 23 | type AccountId = String 24 | type ResetTokenId = String 25 | } 26 | -------------------------------------------------------------------------------- /common/src/main/scala/com/precog/common/backport/lang/Boolean.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.common.backport.lang 21 | 22 | object Boolean { 23 | /** Backport of Java 7's java.lang.Boolean's static compare method */ 24 | @inline 25 | def compare(x: Boolean, y: Boolean): Int = { 26 | if (x == y) 0 27 | else if (!x && y) -1 28 | else 1 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /common/src/main/scala/com/precog/common/backport/lang/Long.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.common.backport.lang 21 | 22 | object Long { 23 | /** Backport of Java 7's java.lang.Long's static compare method */ 24 | @inline 25 | def compare(x: Long, y: Long): Int = { 26 | if (x == y) 0 27 | else if (x < y) -1 28 | else 1 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /common/src/main/scala/com/precog/common/jobs.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.common 21 | 22 | package object jobs { 23 | type ChannelId = String 24 | type JobId = String 25 | type MessageId = Long 26 | type StatusId = Long 27 | 28 | trait IdExtractor { 29 | val NonNegInt = """(\d+)""".r 30 | 31 | def unapply(str: String): Option[Long] = str match { 32 | case NonNegInt(str) => Some(str.toLong) 33 | case _ => None 34 | } 35 | } 36 | 37 | object StatusId extends IdExtractor 38 | object MessageId extends IdExtractor 39 | } 40 | 41 | -------------------------------------------------------------------------------- /common/src/main/scala/com/precog/common/jobs/FileStorage.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.common 21 | package jobs 22 | 23 | import blueeyes.core.http.{ MimeType, MimeTypes } 24 | 25 | import scalaz.StreamT 26 | 27 | case class FileData[M[+_]](mimeType: Option[MimeType], data: StreamT[M, Array[Byte]]) 28 | 29 | /** 30 | * An abstraction for storing/manipulating/retrieving files. 31 | */ 32 | trait FileStorage[M[+_]] { 33 | def exists(file: String): M[Boolean] 34 | def save(file: String, data: FileData[M]): M[Unit] 35 | def load(file: String): M[Option[FileData[M]]] 36 | def remove(file: String): M[Unit] 37 | } 38 | -------------------------------------------------------------------------------- /common/src/main/scala/com/precog/common/security/AccessControl.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.common 21 | package security 22 | 23 | import org.joda.time.DateTime 24 | 25 | import com.weiglewilczek.slf4s.Logging 26 | 27 | import scalaz._ 28 | import scalaz.std.option._ 29 | import scalaz.syntax.monad._ 30 | 31 | trait AccessControl[M[+_]] { 32 | def hasCapability(apiKey: APIKey, perms: Set[Permission], at: Option[DateTime]): M[Boolean] 33 | } 34 | 35 | class UnrestrictedAccessControl[M[+_]: Applicative] extends AccessControl[M] { 36 | def hasCapability(apiKey: APIKey, perms: Set[Permission], at: Option[DateTime]): M[Boolean] = true.point[M] 37 | } 38 | -------------------------------------------------------------------------------- /common/src/main/scala/com/precog/common/services/EitherServiceCombinators.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.common 21 | package services 22 | 23 | import blueeyes.core.http._ 24 | import blueeyes.core.service._ 25 | 26 | trait EitherServiceCombinators { 27 | def left[A, B, C](h: HttpService[Either[A, B], C]): HttpService[A, C] = h.contramap(Left(_)) 28 | 29 | def right[A, B, C](h: HttpService[Either[A, B], C]): HttpService[B, C] = h.contramap(Right(_)) 30 | } 31 | 32 | // vim: set ts=4 sw=4 et: 33 | -------------------------------------------------------------------------------- /common/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=OFF, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L - %m%n 9 | -------------------------------------------------------------------------------- /common/src/test/scala/com/precog/common/ingest/EventIdSpecs.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.common.ingest 21 | 22 | import org.scalacheck.Gen 23 | 24 | import org.specs2.ScalaCheck 25 | import org.specs2.mutable.Specification 26 | 27 | object EventIdSpecs extends Specification with ScalaCheck { 28 | implicit val idRange = Gen.chooseNum[Int](0, Int.MaxValue) 29 | 30 | "EventId" should { 31 | "support round-trip encap/decap of producer/sequence ids" in check { (prod: Int, seq: Int) => 32 | val uid = EventId(prod, seq).uid 33 | 34 | EventId.producerId(uid) mustEqual prod 35 | EventId.sequenceId(uid) mustEqual seq 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /dvergr/build.sbt: -------------------------------------------------------------------------------- 1 | // 2 | // ____ ____ _____ ____ ___ ____ 3 | // | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | // | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | // | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | // |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | // 8 | // This program is free software: you can redistribute it and/or modify it under the terms of the 9 | // GNU Affero General Public License as published by the Free Software Foundation, either version 10 | // 3 of the License, or (at your option) any later version. 11 | // 12 | // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | // the GNU Affero General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Affero General Public License along with this 17 | // program. If not, see . 18 | // 19 | // 20 | 21 | name := "dvergr" 22 | 23 | libraryDependencies += "org.specs2" %% "specs2" % "1.12.3-SNAPSHOT" % "test" 24 | 25 | mainClass := Some("com.precog.dvergr.MongoJobServer") 26 | -------------------------------------------------------------------------------- /dvergr/configs/dev/jobs-v1.conf: -------------------------------------------------------------------------------- 1 | server { 2 | port = 30066 3 | sslPort = 30067 4 | sslEnable = false 5 | 6 | log { 7 | level = "info" 8 | console = true 9 | filename = "/var/log/precog/jobs-v1.server.log" 10 | roll = "daily" 11 | use_parents = false 12 | } 13 | } 14 | 15 | services { 16 | jobs { 17 | v1 { 18 | log { 19 | level = "debug" 20 | console = true 21 | filename = "/var/log/precog/jobs-v1.log" 22 | roll = "daily" 23 | use_parents = false 24 | } 25 | 26 | requestLog { 27 | enabled = false 28 | file = "/var/log/precog/jobs-v1.request.log" 29 | roll = "daily" 30 | fields = "date time c-ip cs-method cs-uri-stem cs-uri-query sc-status cs-content" 31 | formatter = "w3cs" 32 | excludePaths = ["/jobs/v1/health"] 33 | } 34 | 35 | zookeeper { 36 | hosts = "localhost:2181" 37 | jobId { 38 | path = "/jobs/beta/v1/jobId" 39 | } 40 | } 41 | 42 | mongo { 43 | servers = ["localhost"] 44 | database = "dev_jobs_v1" 45 | jobsCollection = "jobs" 46 | messagesCollection = "job_messages" 47 | } 48 | 49 | auth { 50 | service { 51 | protocol = "http" 52 | host = "localhost" 53 | port = 30062 54 | path = "/security/v1/" 55 | } 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /dvergr/configs/dev/jobs-v1.logging.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | ${LOG_DIR}/${SERVICE}.server.log 7 | 8 | 9 | ${LOG_DIR}/${SERVICE}.server-%d{yyyyMMdd}.log 10 | 11 | 12 | 30 13 | 14 | 15 | %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /dvergr/src/test/scala/com/precog/dvergr/GridFSFileStorageSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.dvergr 21 | 22 | import com.precog.common._ 23 | import com.precog.common.jobs._ 24 | 25 | import org.specs2.mutable._ 26 | 27 | import blueeyes.persistence.mongo._ 28 | 29 | import scalaz._ 30 | import scalaz.syntax.monad._ 31 | 32 | class GridFSFileStorageSpec extends Specification with RealMongoSpecSupport { 33 | include(new FileStorageSpec[Need] { 34 | val M: Monad[Need] with Comonad[Need] = Need.need 35 | lazy val fs = GridFSFileStorage(realMongo.getDB("gridFsFileStorageSpec"))(M) 36 | }) 37 | } 38 | -------------------------------------------------------------------------------- /gjallerhorn/build.sbt: -------------------------------------------------------------------------------- 1 | // 2 | // ____ ____ _____ ____ ___ ____ 3 | // | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | // | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | // | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | // |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | // 8 | // This program is free software: you can redistribute it and/or modify it under the terms of the 9 | // GNU Affero General Public License as published by the Free Software Foundation, either version 10 | // 3 of the License, or (at your option) any later version. 11 | // 12 | // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | // the GNU Affero General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Affero General Public License along with this 17 | // program. If not, see . 18 | // 19 | // 20 | 21 | import AssemblyKeys._ 22 | 23 | name := "gjallerhorn" 24 | 25 | fork := false 26 | 27 | parallelExecution := false 28 | 29 | libraryDependencies ++= Seq( 30 | "org.scalacheck" %% "scalacheck" % "1.10.0", 31 | "org.specs2" %% "specs2" % "1.12.3", 32 | "net.databinder.dispatch" %% "dispatch-core" % "0.9.5" 33 | ) 34 | 35 | mainClass := Some("com.precog.gjallerhorn.RunAll") 36 | -------------------------------------------------------------------------------- /gjallerhorn/scripts/SITA1.sh: -------------------------------------------------------------------------------- 1 | ## 2 | ## ____ ____ _____ ____ ___ ____ 3 | ## | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | ## | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | ## | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | ## |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | ## 8 | ## This program is free software: you can redistribute it and/or modify it under the terms of the 9 | ## GNU Affero General Public License as published by the Free Software Foundation, either version 10 | ## 3 of the License, or (at your option) any later version. 11 | ## 12 | ## This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | ## without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | ## the GNU Affero General Public License for more details. 15 | ## 16 | ## You should have received a copy of the GNU Affero General Public License along with this 17 | ## program. If not, see . 18 | ## 19 | ## 20 | curl "http://staging.precog.com/analytics/v1/fs/0000000068?apiKey=9120C30A-DDDD-4E93-BA08-97C942645E7C&q=import%20std%3A%3Astats%3A%3A*%20locations%20%3A%3D%20%2F%2Fdevicelocations%2F2012%2F11%2F01%20points%20%3A%3D%20%7B%20x%3A%20locations.x%2C%20y%3A%20locations.y%20%7D%20model%20%3A%3D%20kMedians(points%2C%205)%20locations%20with%20%7Bclusters%3A%20assignClusters(points%2C%20model)%7D" 21 | 22 | #SITA create clusters and assign clusters 400k rows ran in 25 seconds on 3-22-13 -------------------------------------------------------------------------------- /gjallerhorn/scripts/keenful1.sh: -------------------------------------------------------------------------------- 1 | ## 2 | ## ____ ____ _____ ____ ___ ____ 3 | ## | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | ## | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | ## | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | ## |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | ## 8 | ## This program is free software: you can redistribute it and/or modify it under the terms of the 9 | ## GNU Affero General Public License as published by the Free Software Foundation, either version 10 | ## 3 of the License, or (at your option) any later version. 11 | ## 12 | ## This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | ## without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | ## the GNU Affero General Public License for more details. 15 | ## 16 | ## You should have received a copy of the GNU Affero General Public License along with this 17 | ## program. If not, see . 18 | ## 19 | ## 20 | curl "http://staging.precog.com/analytics/v1/fs/0000000071?apiKey=BD491DD0-B3C3-4D3B-A5BC-ECDED9798CA8&q=import%20std%3A%3Atime%3A%3A*%20data%20%3A%3D%20%2F%2Fprod%2F510d6f801c8430000967f555%2Factions%20data'%20%3A%3D%20data%20where%20getMillis%20(data.action.created_date)%20%3E%20getMillis(%222013-03-03%22)%20%26%20getMillis%20(data.action.created_date)%20%3C%20getMillis(%222013-03-10%22)%20count(distinct(data'.visitor.id))" 21 | 22 | #Keenful Query, ran in 9 seconds on 3-22-13 -------------------------------------------------------------------------------- /gjallerhorn/scripts/precog2beta.sh: -------------------------------------------------------------------------------- 1 | ## 2 | ## ____ ____ _____ ____ ___ ____ 3 | ## | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | ## | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | ## | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | ## |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | ## 8 | ## This program is free software: you can redistribute it and/or modify it under the terms of the 9 | ## GNU Affero General Public License as published by the Free Software Foundation, either version 10 | ## 3 of the License, or (at your option) any later version. 11 | ## 12 | ## This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | ## without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | ## the GNU Affero General Public License for more details. 15 | ## 16 | ## You should have received a copy of the GNU Affero General Public License along with this 17 | ## program. If not, see . 18 | ## 19 | ## 20 | curl "https://beta.precog.com/analytics/v1/fs/0000000024/?apiKey=A1C62105-691B-4D77-9372-36A693E5D905&q=count(%2F%2Fnathan%2Fpoliticalsentiment%2Ftwitter%2F*%2Frawtweets%20)" 21 | 22 | #Precog Query: Election 2012 Twitter -- just counting 1.8 million rows ran in 44 seconds on 3-22-13 -------------------------------------------------------------------------------- /gjallerhorn/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /gjallerhorn/src/main/scala/com/precog/gjallerhorn/Account.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.gjallerhorn 21 | 22 | case class Account(user: String, password: String, accountId: String, apiKey: String, rootPath: String) { 23 | def bareRootPath = rootPath.substring(1, rootPath.length - 1) 24 | } 25 | -------------------------------------------------------------------------------- /gjallerhorn/src/main/scala/com/precog/gjallerhorn/RunAll.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.gjallerhorn 21 | 22 | object RunAll extends Runner { 23 | def tasks(settings: Settings) = List( 24 | new AccountsTask(settings), 25 | new SecurityTask(settings), 26 | new MetadataTask(settings), 27 | new AnalyticsTask(settings), 28 | new IngestTask(settings), 29 | new ScenariosTask(settings) 30 | ) 31 | } 32 | -------------------------------------------------------------------------------- /gjallerhorn/src/main/scala/com/precog/gjallerhorn/Runner.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.gjallerhorn 21 | 22 | import dispatch._ 23 | import specs2._ 24 | 25 | trait Runner { 26 | def tasks(settings: Settings): List[Task] 27 | 28 | def main(args: Array[String]) { 29 | try { 30 | run(tasks(Settings.fromFile(new java.io.File("bifrost.out"))):_*) 31 | } finally { 32 | Http.shutdown() 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /headers/scala-header.txt: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | -------------------------------------------------------------------------------- /headers/sh-header.txt: -------------------------------------------------------------------------------- 1 | ## 2 | ## ____ ____ _____ ____ ___ ____ 3 | ## | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | ## | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | ## | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | ## |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | ## 8 | ## This program is free software: you can redistribute it and/or modify it under the terms of the 9 | ## GNU Affero General Public License as published by the Free Software Foundation, either version 10 | ## 3 of the License, or (at your option) any later version. 11 | ## 12 | ## This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | ## without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | ## the GNU Affero General Public License for more details. 15 | ## 16 | ## You should have received a copy of the GNU Affero General Public License along with this 17 | ## program. If not, see . 18 | ## 19 | ## 20 | -------------------------------------------------------------------------------- /ingest/build.sbt: -------------------------------------------------------------------------------- 1 | // 2 | // ____ ____ _____ ____ ___ ____ 3 | // | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | // | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | // | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | // |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | // 8 | // This program is free software: you can redistribute it and/or modify it under the terms of the 9 | // GNU Affero General Public License as published by the Free Software Foundation, either version 10 | // 3 of the License, or (at your option) any later version. 11 | // 12 | // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | // the GNU Affero General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Affero General Public License along with this 17 | // program. If not, see . 18 | // 19 | // 20 | 21 | name := "ingest" 22 | 23 | parallelExecution in test := false 24 | 25 | libraryDependencies ++= Seq( 26 | "ch.qos.logback" % "logback-classic" % "1.0.0" 27 | ) 28 | 29 | mainClass := Some("com.precog.ingest.kafka.KafkaEventServer") 30 | -------------------------------------------------------------------------------- /ingest/configs/VERSION: -------------------------------------------------------------------------------- 1 | V2 2 | -------------------------------------------------------------------------------- /ingest/configs/dev/ingest-v2.logging.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | ${LOG_DIR}/${SERVICE}.server.log 7 | 8 | 9 | ${LOG_DIR}/${SERVICE}.server-%d{yyyyMMdd}.log 10 | 11 | 12 | 30 13 | 14 | 15 | %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /ingest/src/main/scala/com/precog/ingest/EventStore.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.ingest 21 | 22 | import com.precog.common.ingest._ 23 | import com.precog.util.PrecogUnit 24 | 25 | import akka.dispatch.Future 26 | import akka.util.Timeout 27 | 28 | import scalaz._ 29 | 30 | case class StoreFailure(message: String) 31 | 32 | trait EventStore[M[+_]] { 33 | def save(action: Event, timeout: Timeout): M[StoreFailure \/ PrecogUnit] 34 | } 35 | -------------------------------------------------------------------------------- /ingest/src/main/scala/com/precog/ingest/service.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.ingest 21 | 22 | import _root_.blueeyes.json._ 23 | 24 | package object service { 25 | def cleanPath(string: String): String = "/" + string.split("/").map(_.trim).filter(_.length > 0).mkString("/") 26 | 27 | implicit def jpath2rich(jpath: JPath): RichJPath = new RichJPath(jpath) 28 | class RichJPath(jpath: JPath) { 29 | def endsInInfiniteValueSpace = jpath.nodes.exists { 30 | case JPathField(name) => name startsWith "~" 31 | case _ => false 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /ingest/src/main/scala/com/precog/ingest/util/FutureUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.ingest.util 21 | 22 | import akka.dispatch.{Future, ExecutionContext} 23 | 24 | object FutureUtils { 25 | def singleSuccess[T](futures: Seq[Future[T]])(implicit executor: ExecutionContext): Future[Option[T]] = Future.find(futures){ _ => true } 26 | } 27 | -------------------------------------------------------------------------------- /jprofiler/setup-jnilib.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from os import getcwd 4 | from os.path import realpath, join, exists 5 | import platform 6 | from sys import exit 7 | from urllib2 import urlopen 8 | 9 | def die(msg): print msg; exit(1) 10 | 11 | subproject = join(realpath(getcwd()), "jprofiler") 12 | jnilib_path = join(subproject, "jprofiler.jnilib") 13 | 14 | if not exists(subproject): die("couldn't find 'jprofiler' directory") 15 | 16 | if exists(jnilib_path): exit(0) 17 | 18 | archs = {'Darwin': 'osx.jnilib', 'Linux': 'linux.jnilib'} 19 | name = archs.get(platform.system()) 20 | if name is None: die("unknown arch: %s" % arch) 21 | 22 | url = "http://s3.amazonaws.com/ops.reportgrid.com/jprofiler/%s" % name 23 | print "getting jnilib from %s" % url 24 | 25 | u = urlopen(url) 26 | open(jnilib_path, 'wb').write(u.read()) 27 | -------------------------------------------------------------------------------- /logging/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=OFF, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L - %m%n 9 | -------------------------------------------------------------------------------- /logging/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | target/testlog.log 4 | 5 | 7 | 8 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /miklagard/desktop/.gitignore: -------------------------------------------------------------------------------- 1 | precog/ -------------------------------------------------------------------------------- /miklagard/desktop/add-labcoat-nopg.sh: -------------------------------------------------------------------------------- 1 | ## 2 | ## ____ ____ _____ ____ ___ ____ 3 | ## | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | ## | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | ## | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | ## |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | ## 8 | ## This program is free software: you can redistribute it and/or modify it under the terms of the 9 | ## GNU Affero General Public License as published by the Free Software Foundation, either version 10 | ## 3 of the License, or (at your option) any later version. 11 | ## 12 | ## This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | ## without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | ## the GNU Affero General Public License for more details. 15 | ## 16 | ## You should have received a copy of the GNU Affero General Public License along with this 17 | ## program. If not, see . 18 | ## 19 | ## 20 | #!/bin/bash 21 | 22 | cd `dirname $0` 23 | 24 | if [[ `ls target/*.jar | wc -l` != 1 ]]; then 25 | echo "Missing/too many jars!" 26 | exit 1 27 | fi 28 | 29 | TMPDIR=precog 30 | 31 | mkdir -p $TMPDIR 32 | rm -rf $TMPDIR/* 33 | 34 | cp target/desktop-assembly*.jar $TMPDIR/precog-desktop.jar 35 | mkdir web 36 | cp -R ../../quirrelide/build/* web/ 37 | rm -rf web/php web/*.php 38 | zip -ru $TMPDIR/precog-desktop.jar web 39 | rm -rf web 40 | 41 | -------------------------------------------------------------------------------- /miklagard/desktop/build-dist-jar.sh: -------------------------------------------------------------------------------- 1 | ## 2 | ## ____ ____ _____ ____ ___ ____ 3 | ## | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | ## | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | ## | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | ## |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | ## 8 | ## This program is free software: you can redistribute it and/or modify it under the terms of the 9 | ## GNU Affero General Public License as published by the Free Software Foundation, either version 10 | ## 3 of the License, or (at your option) any later version. 11 | ## 12 | ## This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | ## without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | ## the GNU Affero General Public License for more details. 15 | ## 16 | ## You should have received a copy of the GNU Affero General Public License along with this 17 | ## program. If not, see . 18 | ## 19 | ## 20 | #!/bin/bash 21 | 22 | set -e 23 | 24 | VERSION=`git describe` 25 | pushd .. 26 | sbt desktop/assembly 27 | popd 28 | 29 | TMPDIR=precog 30 | mkdir -p $TMPDIR 31 | rm -rf $TMPDIR/* 32 | java -Xmx2048m -jar ../tools/lib/proguard.jar @proguard.conf -injars target/desktop-assembly-$VERSION.jar -outjars $TMPDIR/precog-desktop.jar 2>&1 | tee proguard.log 33 | test -d web || mkdir web 34 | cp -R ../../quirrelide/build/* web/ 35 | rm -rf web/php web/*.php 36 | zip -ru $TMPDIR/precog-desktop.jar web 37 | rm -rf web 38 | -------------------------------------------------------------------------------- /miklagard/desktop/installer_files/Validations.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/precog/platform/9655622c4969a025ac198674d06b287fa5bbb8d1/miklagard/desktop/installer_files/Validations.class -------------------------------------------------------------------------------- /miklagard/desktop/installer_files/Validations.java: -------------------------------------------------------------------------------- 1 | import java.net.InetAddress; 2 | import java.net.Socket; 3 | 4 | class Validations{ 5 | public static boolean isPortUsed(int port){ 6 | try { 7 | Socket conn = new Socket(InetAddress.getLocalHost(), port); 8 | conn.close(); 9 | return true; 10 | } catch (Exception e) { 11 | return false; 12 | } 13 | } 14 | 15 | public static boolean isPortNumberValid(String portNumber){ 16 | int port = Integer.parseInt(portNumber); 17 | if (0< port && port<65535){ 18 | return true; 19 | } else { 20 | return false; 21 | } 22 | } 23 | } -------------------------------------------------------------------------------- /miklagard/desktop/installer_files/icons/LargeIcon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/precog/platform/9655622c4969a025ac198674d06b287fa5bbb8d1/miklagard/desktop/installer_files/icons/LargeIcon.png -------------------------------------------------------------------------------- /miklagard/desktop/installer_files/icons/P.icns: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/precog/platform/9655622c4969a025ac198674d06b287fa5bbb8d1/miklagard/desktop/installer_files/icons/P.icns -------------------------------------------------------------------------------- /miklagard/desktop/installer_files/icons/P.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/precog/platform/9655622c4969a025ac198674d06b287fa5bbb8d1/miklagard/desktop/installer_files/icons/P.png -------------------------------------------------------------------------------- /miklagard/desktop/installer_files/icons/large_icon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/precog/platform/9655622c4969a025ac198674d06b287fa5bbb8d1/miklagard/desktop/installer_files/icons/large_icon.ico -------------------------------------------------------------------------------- /miklagard/desktop/installer_files/icons/lc_icon.16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/precog/platform/9655622c4969a025ac198674d06b287fa5bbb8d1/miklagard/desktop/installer_files/icons/lc_icon.16.png -------------------------------------------------------------------------------- /miklagard/desktop/installer_files/icons/lc_icon.32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/precog/platform/9655622c4969a025ac198674d06b287fa5bbb8d1/miklagard/desktop/installer_files/icons/lc_icon.32.png -------------------------------------------------------------------------------- /miklagard/desktop/installer_files/icons/lc_icon.48.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/precog/platform/9655622c4969a025ac198674d06b287fa5bbb8d1/miklagard/desktop/installer_files/icons/lc_icon.48.png -------------------------------------------------------------------------------- /miklagard/desktop/installer_files/icons/lc_icon48.icns: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/precog/platform/9655622c4969a025ac198674d06b287fa5bbb8d1/miklagard/desktop/installer_files/icons/lc_icon48.icns -------------------------------------------------------------------------------- /miklagard/desktop/installer_files/icons/logo-precog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/precog/platform/9655622c4969a025ac198674d06b287fa5bbb8d1/miklagard/desktop/installer_files/icons/logo-precog.png -------------------------------------------------------------------------------- /miklagard/desktop/installer_files/mac-conf/shard-v2.logging.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | ${LOG_DIR}/${SERVICE}.server.log 7 | 8 | 9 | ${LOG_DIR}/${SERVICE}.server-%d{yyyyMMdd}.log 10 | 11 | 12 | 30 13 | 14 | 15 | %d{ISO8601} [%.10thread] %.-1level %logger{20} {%X} - %msg%n 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /miklagard/desktop/installer_files/shard-v2.logging.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | ${LOG_DIR}/${SERVICE}.server.log 7 | 8 | 9 | ${LOG_DIR}/${SERVICE}.server-%d{yyyyMMdd}.log 10 | 11 | 12 | 30 13 | 14 | 15 | %d{ISO8601} [%.10thread] %.-1level %logger{20} {%X} - %msg%n 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /miklagard/desktop/installer_files/unix-conf/shard-v2.logging.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | ${LOG_DIR}/${SERVICE}.server.log 7 | 8 | 9 | ${LOG_DIR}/${SERVICE}.server-%d{yyyyMMdd}.log 10 | 11 | 12 | 30 13 | 14 | 15 | %d{ISO8601} [%.10thread] %.-1level %logger{20} {%X} - %msg%n 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /miklagard/desktop/lib/kafka-core-assembly-0.7.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/precog/platform/9655622c4969a025ac198674d06b287fa5bbb8d1/miklagard/desktop/lib/kafka-core-assembly-0.7.5.jar -------------------------------------------------------------------------------- /miklagard/desktop/lib/zookeeper-3.4.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/precog/platform/9655622c4969a025ac198674d06b287fa5bbb8d1/miklagard/desktop/lib/zookeeper-3.4.4.jar -------------------------------------------------------------------------------- /miklagard/desktop/src/main/resources/LargeIcon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/precog/platform/9655622c4969a025ac198674d06b287fa5bbb8d1/miklagard/desktop/src/main/resources/LargeIcon.png -------------------------------------------------------------------------------- /miklagard/desktop/src/main/resources/web/version.html: -------------------------------------------------------------------------------- 1 | 1.0.0 2 | -------------------------------------------------------------------------------- /miklagard/jdbc/.gitignore: -------------------------------------------------------------------------------- 1 | *.html 2 | -------------------------------------------------------------------------------- /miklagard/jdbc/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Precog for PostgreSQL ChangeLog 2 | =============================== 3 | 4 | Version 1.2.0 5 | ------------- 6 | * Numerous bug fixes and performance improvements 7 | * A small change to the configuration file format: 8 | 9 | The lines 10 | 11 | query { 12 | v1 { 13 | 14 | need to change to 15 | 16 | analytics { 17 | v2 { 18 | 19 | Version 1.1.0 20 | ------------- 21 | * Added support for hstore and json columns 22 | * Numerous bug fixes 23 | 24 | *Note:* While json and hstore columns are supported for read, 25 | PostgreSQL doesn't currently support key/property level queries 26 | against them. Because of that, any queries that utilize hstore or 27 | json columns will read all contained column data before filtering 28 | down to only the required columns. 29 | 30 | Version 1.0.0 31 | ------------- 32 | * Initial release 33 | -------------------------------------------------------------------------------- /miklagard/jdbc/build.sbt: -------------------------------------------------------------------------------- 1 | // 2 | // ____ ____ _____ ____ ___ ____ 3 | // | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | // | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | // | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | // |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | // 8 | // This program is free software: you can redistribute it and/or modify it under the terms of the 9 | // GNU Affero General Public License as published by the Free Software Foundation, either version 10 | // 3 of the License, or (at your option) any later version. 11 | // 12 | // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | // the GNU Affero General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Affero General Public License along with this 17 | // program. If not, see . 18 | // 19 | // 20 | 21 | name := "jdbc" 22 | 23 | libraryDependencies ++= Seq( 24 | "com.h2database" % "h2" % "1.3.170" % "test", 25 | "postgresql" % "postgresql" % "9.1-901.jdbc4" 26 | ) 27 | 28 | parallelExecution in Test := true 29 | 30 | mainClass := Some("com.precog.bifrost.jdbc.JDBCShardServer") 31 | -------------------------------------------------------------------------------- /miklagard/jdbc/config.cfg: -------------------------------------------------------------------------------- 1 | server { 2 | port = 8888 3 | sslPort = 8889 4 | sslEnable = false 5 | } 6 | 7 | services { 8 | analytics { 9 | v2 { 10 | queryExecutor { 11 | databases { 12 | # Database config lines are of the form 13 | # test = "jdbc:postgresql://localhost/test" 14 | } 15 | } 16 | 17 | security { 18 | masterAccount { 19 | apiKey = "12345678-1234-1234-1234-123456789abc" 20 | } 21 | } 22 | 23 | labcoat { 24 | port = 8000 25 | } 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /miklagard/jdbc/precog.bat: -------------------------------------------------------------------------------- 1 | java.exe -jar precog.jar --configFile config.cfg -------------------------------------------------------------------------------- /miklagard/jdbc/precog.sh: -------------------------------------------------------------------------------- 1 | ## 2 | ## ____ ____ _____ ____ ___ ____ 3 | ## | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | ## | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | ## | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | ## |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | ## 8 | ## This program is free software: you can redistribute it and/or modify it under the terms of the 9 | ## GNU Affero General Public License as published by the Free Software Foundation, either version 10 | ## 3 of the License, or (at your option) any later version. 11 | ## 12 | ## This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | ## without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | ## the GNU Affero General Public License for more details. 15 | ## 16 | ## You should have received a copy of the GNU Affero General Public License along with this 17 | ## program. If not, see . 18 | ## 19 | ## 20 | #!/bin/bash 21 | 22 | java -jar precog.jar --configFile config.cfg 23 | -------------------------------------------------------------------------------- /miklagard/jdbc/proguard.conf: -------------------------------------------------------------------------------- 1 | -keep public class com.precog.bifrost.jdbc.JDBCShardServer { 2 | public static void main(java.lang.String[]); 3 | } 4 | -keep class !com.precog.** { *; } 5 | -libraryjars /lib/rt.jar 6 | -verbose 7 | -dontshrink 8 | -dontoptimize 9 | -dontpreverify 10 | -ignorewarnings -------------------------------------------------------------------------------- /miklagard/jdbc/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /miklagard/jdbc/src/main/resources/web/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/precog/platform/9655622c4969a025ac198674d06b287fa5bbb8d1/miklagard/jdbc/src/main/resources/web/favicon.ico -------------------------------------------------------------------------------- /miklagard/jdbc/src/main/resources/web/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Labcoat 6 | 7 | 8 | 9 | 10 | 11 | 12 | 26 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /miklagard/mongo/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Precog for MongoDB Changelog 2 | ============================ 3 | 4 | Version 1.3.0 5 | ------------- 6 | * Numerous bug fixes and performance improvements 7 | * A small change to the configuration file format: 8 | 9 | The lines 10 | 11 | query { 12 | v1 { 13 | 14 | need to change to 15 | 16 | analytics { 17 | v2 { 18 | 19 | 20 | Version 1.2.0 21 | ------------- 22 | * Add the "include_ids" configuration parameter to include Mongo's 23 | "\_id" field to results. To enable, add the parameter to the queryExecutor block. For example: 24 | 25 | queryExecutor { 26 | include_ids = true 27 | mongo { 28 | server = "mongodb://localhost:27017" 29 | } 30 | } 31 | 32 | Version 1.1.2 33 | ------------- 34 | * Fix a bug resulting in concurrent usage of Mongo cursors 35 | * Miscellaneous evaluator bugfixes 36 | 37 | Version 1.1.1 38 | ------------- 39 | 40 | * Fix a bug in table loads that caused join queries to return 41 | empty sets 42 | 43 | Version 1.1 44 | ----------- 45 | * Add support for database authentication for MongoDB (see 46 | README.md for configuration details) 47 | 48 | * Improve performance of field filtering. For example, running a 49 | `count(load("/foo/bar").test)` will generally perform faster than 50 | `count(//foo/bar)`, since mongo will only load the `test` property 51 | instead of whole documents. 52 | 53 | Version 1.0.4 54 | ------------- 55 | * Fix a bug in the quirrel solver that could cause hangs or 56 | invalid results 57 | -------------------------------------------------------------------------------- /miklagard/mongo/build.sbt: -------------------------------------------------------------------------------- 1 | // 2 | // ____ ____ _____ ____ ___ ____ 3 | // | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | // | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | // | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | // |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | // 8 | // This program is free software: you can redistribute it and/or modify it under the terms of the 9 | // GNU Affero General Public License as published by the Free Software Foundation, either version 10 | // 3 of the License, or (at your option) any later version. 11 | // 12 | // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | // the GNU Affero General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Affero General Public License along with this 17 | // program. If not, see . 18 | // 19 | // 20 | 21 | name := "mongo" 22 | 23 | parallelExecution in Test := true 24 | 25 | fork in ScctTest := true 26 | 27 | javaOptions in ScctTest := Seq("-Xmx2G", "-XX:MaxPermSize=512m") // required if using fork in Test 28 | 29 | mainClass := Some("com.precog.bifrost.mongo.MongoShardServer") 30 | -------------------------------------------------------------------------------- /miklagard/mongo/config.cfg: -------------------------------------------------------------------------------- 1 | server { 2 | port = 8888 3 | sslPort = 8889 4 | sslEnable = false 5 | } 6 | 7 | services { 8 | analytics { 9 | v2 { 10 | queryExecutor { 11 | mongo { 12 | server = "mongodb://localhost:27017" 13 | } 14 | } 15 | 16 | security { 17 | masterAccount { 18 | apiKey = "12345678-1234-1234-1234-123456789abc" 19 | } 20 | } 21 | 22 | labcoat { 23 | port = 8000 24 | } 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /miklagard/mongo/precog.bat: -------------------------------------------------------------------------------- 1 | java.exe -jar precog.jar --configFile config.cfg -------------------------------------------------------------------------------- /miklagard/mongo/precog.sh: -------------------------------------------------------------------------------- 1 | ## 2 | ## ____ ____ _____ ____ ___ ____ 3 | ## | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | ## | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | ## | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | ## |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | ## 8 | ## This program is free software: you can redistribute it and/or modify it under the terms of the 9 | ## GNU Affero General Public License as published by the Free Software Foundation, either version 10 | ## 3 of the License, or (at your option) any later version. 11 | ## 12 | ## This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | ## without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | ## the GNU Affero General Public License for more details. 15 | ## 16 | ## You should have received a copy of the GNU Affero General Public License along with this 17 | ## program. If not, see . 18 | ## 19 | ## 20 | #!/bin/bash 21 | 22 | java -jar precog.jar --configFile config.cfg 23 | -------------------------------------------------------------------------------- /miklagard/mongo/proguard.conf: -------------------------------------------------------------------------------- 1 | -keep public class com.precog.bifrost.mongo.MongoShardServer { 2 | public static void main(java.lang.String[]); 3 | } 4 | -keep class !com.precog.** { *; } 5 | -libraryjars /lib/rt.jar 6 | -verbose 7 | -dontshrink 8 | -dontoptimize 9 | -dontpreverify 10 | -ignorewarnings -------------------------------------------------------------------------------- /miklagard/mongo/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /miklagard/mongo/src/main/resources/web/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/precog/platform/9655622c4969a025ac198674d06b287fa5bbb8d1/miklagard/mongo/src/main/resources/web/favicon.ico -------------------------------------------------------------------------------- /miklagard/mongo/src/main/resources/web/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Labcoat 6 | 7 | 8 | 9 | 10 | 11 | 12 | 26 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /mimir/build.sbt: -------------------------------------------------------------------------------- 1 | // 2 | // ____ ____ _____ ____ ___ ____ 3 | // | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | // | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | // | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | // |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | // 8 | // This program is free software: you can redistribute it and/or modify it under the terms of the 9 | // GNU Affero General Public License as published by the Free Software Foundation, either version 10 | // 3 of the License, or (at your option) any later version. 11 | // 12 | // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | // the GNU Affero General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Affero General Public License along with this 17 | // program. If not, see . 18 | // 19 | // 20 | 21 | name := "mimir" 22 | 23 | scalacOptions += "-Ydependent-method-types" 24 | 25 | libraryDependencies ++= Seq( 26 | "com.eed3si9n" % "treehugger_2.9.1" % "0.1.2", 27 | "gov.nist.math" % "jama" % "1.0.2", 28 | "org.apache.commons" % "commons-math3" % "3.1.1" 29 | ) 30 | 31 | logBuffered := false // gives us incremental output from Specs2 32 | 33 | parallelExecution in Test := false 34 | -------------------------------------------------------------------------------- /mimir/src/main/scala/com/precog/mimir/CondRewriter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog 21 | package mimir 22 | 23 | trait CondRewriter extends DAG { 24 | import instructions._ 25 | import dag._ 26 | 27 | def rewriteConditionals(node: DepGraph): DepGraph = { 28 | node mapDown { recurse => { 29 | case peer @ IUI(true, Filter(leftJoin, left, pred1), Filter(rightJoin, right, Operate(Comp, pred2))) if pred1 == pred2 => 30 | Cond(recurse(pred1), recurse(left), leftJoin, recurse(right), rightJoin)(peer.loc) 31 | }} 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/array.json: -------------------------------------------------------------------------------- 1 | [9, 10, 11] 2 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/arrays.json: -------------------------------------------------------------------------------- 1 | true 2 | null 3 | {"baz": null} 4 | "daniel" 5 | [-9, -42, 42, 87, 4] 6 | {"foo":"bar"} 7 | [7, 6, 12, 0] 8 | [1024, 57, 77] 9 | [[-9, -42, 42, 87, 4]] 10 | [46.2, -100, 1, 19, 22, 11, 104, -27, 6] 11 | [-2790111, 244, 13, 11] 12 | "spiewak" 13 | "twitter" 14 | 47 15 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/heightWeight.json: -------------------------------------------------------------------------------- 1 | {"userId":1,"height":89,"weight":129} 2 | {"userId":2,"height":47,"weight":100} 3 | {"userId":3,"height":30,"weight":88} 4 | {"userId":4,"height":62,"weight":110} 5 | {"userId":5,"height":93,"weight":131} 6 | {"userId":6,"height":null,"weight":"129"} 7 | {"userId":7,"height":[15],"weight":true} 8 | {"userId":8,"height":34.3,"weight":{"foo":12}} 9 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/heightWeightAcrossSlices.json: -------------------------------------------------------------------------------- 1 | {"userId":1,"height":89,"weight":129} 2 | {"userId":2,"height":47,"weight":100} 3 | {"userId":3,"height":30,"weight":88} 4 | {"userId":4,"height":62,"weight":110} 5 | {"userId":5,"height":93,"weight":131} 6 | {"userId":6,"height":null,"weight":"129"} 7 | {"userId":7,"height":[15],"weight":true} 8 | {"userId":8,"height":34.3,"weight":{"foo":12}} 9 | {"userId":5,"height":102,"weight":191} 10 | {"userId":5,"height":true,"weight":131} 11 | {"userId":5,"height":93,"weight":[]} 12 | {"userId":5,"height":{},"weight":161} 13 | {"userId":5,"height":110,"weight":261} 14 | {"userId":5,"height":false,"weight":1.111} 15 | {"userId":5,"height":83.5,"weight":131} 16 | {"userId":5,"height":93,"weight":145.7} 17 | {"userId":5,"height":[{"bar":88}],"weight":131} 18 | {"userId":5,"height":93.3,"weight":131.3} 19 | {"userId":5,"height":44,"weight":102} 20 | {"userId":5,"height":"tall","weight":"heavy"} 21 | {"userId":5,"height":"precog","weight":131} 22 | {"userId":5,"height":88,"weight":164} 23 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/het-arrays.json: -------------------------------------------------------------------------------- 1 | [-9, -42, 42, 87, 4] 2 | [7, 6, true, 0] 3 | [1024, 57, "daniel"] 4 | [46.2, -100, 1, 19, 22, 11, 104, -27, 6] 5 | [-2790111, 244, null, 11] 6 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/het-pairs.json: -------------------------------------------------------------------------------- 1 | {"first":42,"second":6} 2 | {"first":true,"second":0} 3 | {"first":"daniel","second":-38} 4 | {"first":1,"second":null} 5 | {"first":null,"second":4} 6 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/iso8601.json: -------------------------------------------------------------------------------- 1 | "2010-04-29T09:37:52.599+08:00" 2 | 13 3 | "2011-02-21T20:09:59.165+09:00" 4 | true 5 | "apolune" 6 | "2011-09-06T06:44:52.848-10:00" 7 | "2012-zeugma" 8 | [3, 8] 9 | "2012-02-11T09:11:33.394-07:00" 10 | {"test": "foobar"} 11 | "2012-12-28T22:38:19.430+06:00" 12 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/iso8601AcrossSlices.json: -------------------------------------------------------------------------------- 1 | "2012-05-05T08:58:10.171+10:00" 2 | null 3 | [9, 10, 11] 4 | "2009-08-17T05:54:08.513+02:00" 5 | "2010-10-25T01:51:16.248+04:00" 6 | 12 7 | "2011-06-25T00:18:50.873-11:00" 8 | "2007-07-14T03:49:30.311-07:00" 9 | false 10 | "2008-10-24T11:44:19.844+03:00" 11 | {"first":42,"second":null} 12 | "2011-10-13T15:47:40.629+08:00" 13 | "2008-05-27T16:27:24.858Z" 14 | [{"c": null}] 15 | {"b": [9]} 16 | "2010-11-21T23:50:10.932+06:00" 17 | true 18 | "2008-07-02T18:53:43.506-04:00" 19 | "supernova" 20 | 0 21 | [] 22 | {} -------------------------------------------------------------------------------- /mimir/src/test/resources/het/millisSinceEpoch.json: -------------------------------------------------------------------------------- 1 | 1330192887710 2 | "supernova" 3 | 1329583492854 4 | [4,5,6] 5 | {"foo": 12} 6 | 1329848922774 7 | 1329583492780 8 | 1330447492420 9 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/millisTimeZone.json: -------------------------------------------------------------------------------- 1 | -36000000 2 | "quasar" 3 | [1,2,3] 4 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/numbers.json: -------------------------------------------------------------------------------- 1 | true 2 | 42 3 | false 4 | "daniel" 5 | 12 6 | {"test":"fubar"} 7 | 77 8 | 1 9 | 13 10 | [] 11 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/numbers10.json: -------------------------------------------------------------------------------- 1 | [] 2 | {} 3 | [9,10] 4 | {"foo": null} 5 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/numbers2.json: -------------------------------------------------------------------------------- 1 | true 2 | 42 3 | false 4 | "daniel" 5 | 12 6 | {"test":"fubar"} 7 | 77 8 | 1 9 | 1 10 | 13 11 | [] 12 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/numbers4.json: -------------------------------------------------------------------------------- 1 | true 2 | 0 3 | false 4 | "daniel" 5 | -1 6 | {"test":"fubar"} 7 | 42 8 | 1 9 | -23 10 | 1 11 | [] 12 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/numbers6.json: -------------------------------------------------------------------------------- 1 | -10 2 | [9,10,11] 3 | "alissa" 4 | 34 5 | 0 6 | null 7 | -10 8 | 11 9 | true 10 | {"foo":"bar"} 11 | [] 12 | 12 13 | {} 14 | 11 15 | 11 16 | false 17 | 5 18 | 34 19 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/numbers7.json: -------------------------------------------------------------------------------- 1 | 0 2 | {"foo": 0} 3 | {"foo": {"bar": 0}} 4 | [0] 5 | [0, "a"] 6 | ["a", 0] 7 | "a" 8 | true 9 | null 10 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/numbers8.json: -------------------------------------------------------------------------------- 1 | 0 2 | {"foo": 0} 3 | {"foo": {"bar": 0}} 4 | [0] 5 | ["a", 0] 6 | "a" 7 | true 8 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/numbers9.json: -------------------------------------------------------------------------------- 1 | 99 2 | {} 3 | [] 4 | {"foo": "baz"} 5 | [3,4,"fubar"] 6 | null 7 | true 8 | false 9 | -1 10 | 0 11 | 1 12 | "fubar" 13 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/numbersAcrossSlices.json: -------------------------------------------------------------------------------- 1 | 5 2 | 0 3 | [] 4 | 1 5 | {} 6 | "Quinn is a kitty!" 7 | -1 8 | 1 9 | 12 10 | true 11 | null 12 | {"a": 1} 13 | [6, -6] 14 | 0 15 | {"b": [9]} 16 | [{"c": null}] 17 | false 18 | null 19 | 2 20 | "" 21 | false 22 | -3 23 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/numbersAcrossSlices2.json: -------------------------------------------------------------------------------- 1 | 5 2 | 0 3 | [] 4 | 1 5 | {} 6 | "Quinn is a kitty!" 7 | -1 8 | 1.2 9 | 12 10 | true 11 | null 12 | {"a": 1} 13 | [6.2, -6] 14 | 0 15 | {"b": [9]} 16 | [{"c": null}] 17 | false 18 | null 19 | 2 20 | false 21 | -3.5 22 | true 23 | [1] 24 | [null] 25 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/numbersDoubleLong.json: -------------------------------------------------------------------------------- 1 | -30.2 2 | -30.2 3 | -2 4 | -2.00000 5 | 0 6 | 10.1 7 | 12.6 8 | 15 9 | 15.0 10 | 15 11 | 15.0 12 | 30 13 | 40.3 14 | 40.3 15 | 50 16 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/pairs.json: -------------------------------------------------------------------------------- 1 | true 2 | {"first":42,"second":null} 3 | {"twelf":42,"coq":6,"sasylf":false} 4 | null 5 | {"first":12,"second":0} 6 | {"first":null,"second":-38} 7 | 12 8 | 46 9 | {"first":1,"second":166} 10 | {"first":13,"second":1} 11 | [7,"daniel",false,"spiewak"] 12 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/random.json: -------------------------------------------------------------------------------- 1 | 4 2 | false 3 | 3 4 | null 5 | 4 6 | true 7 | [4,5,6] 8 | "a" 9 | 4 10 | "a" 11 | "b" 12 | true 13 | 3 14 | "a" 15 | {"a":1} 16 | false 17 | "a" 18 | true 19 | 4 20 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/stringNums.json: -------------------------------------------------------------------------------- 1 | "42" 2 | "42.0" 3 | "42.123" 4 | "-666" 5 | "2e3" 6 | "0e9" 7 | "2.23532235235235353252352343636953295923" 8 | "1.2e3" 9 | "" 10 | "xyz" 11 | null 12 | "0x123" 13 | ".999" 14 | "99." 15 | "0123" 16 | "1.2.3" 17 | "1e2.3" 18 | "--123" 19 | "+inf" 20 | "nan" 21 | "NaN" 22 | "a12351" 23 | "2423412b" 24 | "123 123" 25 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/strings.json: -------------------------------------------------------------------------------- 1 | "" 2 | true 3 | false 4 | "quirky" 5 | 42 6 | null 7 | "solstice + 7" 8 | [1,2,3] 9 | "Monkey: [Brains]" 10 | 0 11 | "(\"alpha\", \"beta\", \"gamma\")" 12 | " Whitespace is awesome !!!1!! " 13 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/strings2.json: -------------------------------------------------------------------------------- 1 | "this,is,delimited" 2 | "this is a string" 3 | null 4 | 12324.123215 5 | 999 6 | [] 7 | {} 8 | ["this", "is", "an", "array"] 9 | {"objects": "are great"} 10 | "" 11 | "also,delmited" 12 | ",starts,with,comma" 13 | "ends,with,comma," 14 | "lots,,,,of,,,,commas" 15 | ",,,,,,," 16 | "," 17 | ",,crazy,," 18 | "something,basically,reasonable" 19 | -------------------------------------------------------------------------------- /mimir/src/test/resources/het/timeString.json: -------------------------------------------------------------------------------- 1 | "Jun 03 2010 04:12:33.323" 2 | "" 3 | "Dec 09 string" 4 | "Aug 12 2011 22:42:33.310" 5 | [1,2,3] 6 | {"a": 1, "b": 2} 7 | "Jun 04 2010 13:31:49.002" 8 | 234223 9 | true 10 | "Oct 09 2010 09:27:31.953" 11 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/arrays.json: -------------------------------------------------------------------------------- 1 | [-9, -42, 42, 87, 4] 2 | [7, 6, 12, 0] 3 | [1024, 57, 77] 4 | [46.2, -100, 1, 19, 22, 11, 104, -27, 6] 5 | [-2790111, 244, 13, 11] 6 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/clusteringData.json: -------------------------------------------------------------------------------- 1 | {"foo": 4.8, "bar": 9} 2 | [5, 3, 4] 3 | [0, 3, 5, 2] 4 | {"foo":3, "baz": 34.3} 5 | null 6 | {"foo":true, "bar": 33.3} 7 | {"baz": 30} 8 | [2, 3.9, 4] 9 | {"foo": 4, "bar": 9, "ack": [5, 3, 4]} 10 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/clusteringModel.json: -------------------------------------------------------------------------------- 1 | { "model1": { "cluster1": {"foo":3.0, "bar": 2.0}, "cluster2": {"foo":4.4, "bar":9}}, "model2": {"cluster1": {"baz": 4.0 } } } 2 | { "model1": {"cluster1": [2.1,3.3,4], "cluster2": [6,3,2], "cluster3": [0,3.2,5.1], "cluster4": [0,33,2]}} 3 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/decimals.json: -------------------------------------------------------------------------------- 1 | 1.23919271772881 2 | 123.191293912 3 | 99.9999999999 4 | 0.001111111 5 | 0.500000001 6 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/heightWeight.json: -------------------------------------------------------------------------------- 1 | {"userId":1,"height":89,"weight":129} 2 | {"userId":2,"height":47,"weight":100} 3 | {"userId":3,"height":30,"weight":88} 4 | {"userId":4,"height":62,"weight":110} 5 | {"userId":5,"height":93,"weight":131} 6 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/heightWeightAcrossSlices.json: -------------------------------------------------------------------------------- 1 | {"userId":1,"height":89,"weight":129} 2 | {"userId":2,"height":47,"weight":100} 3 | {"userId":3,"height":30,"weight":88} 4 | {"userId":4,"height":62,"weight":110} 5 | {"userId":5,"height":93,"weight":131} 6 | {"userId":6,"height":99,"weight":191} 7 | {"userId":7,"height":33,"weight":120} 8 | {"userId":8,"height":100,"weight":149} 9 | {"userId":9,"height":34,"weight":159} 10 | {"userId":10,"height":101,"weight":153} 11 | {"userId":11,"height":77,"weight":138} 12 | {"userId":12,"height":99,"weight":191} 13 | {"userId":13,"height":82,"weight":231} 14 | {"userId":14,"height":78,"weight":176} 15 | {"userId":15,"height":104,"weight":29} 16 | {"userId":16,"height":59,"weight":88} 17 | {"userId":17,"height":48,"weight":80} 18 | {"userId":18,"height":93,"weight":122} 19 | {"userId":19,"height":82,"weight":139} 20 | {"userId":20,"height":93,"weight":131} 21 | {"userId":21,"height":77,"weight":177} 22 | {"userId":22,"height":82,"weight":204} 23 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/heightWeight_neg.json: -------------------------------------------------------------------------------- 1 | {"userId":1,"height":89,"weight":129} 2 | {"userId":2,"height":47,"weight":100} 3 | {"userId":2,"height":-33,"weight":23} 4 | {"userId":3,"height":30,"weight":88} 5 | {"userId":4,"height":62,"weight":110} 6 | {"userId":5,"height":93,"weight":131} 7 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/iso8601.json: -------------------------------------------------------------------------------- 1 | "2010-04-29T09:37:52.599+08:00" 2 | "2012-02-11T09:11:33.394-07:00" 3 | "2011-09-06T06:44:52.848-10:00" 4 | "2012-12-28T22:38:19.430+06:00" 5 | "2011-02-21T20:09:59.165+09:00" 6 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/iso8601AcrossSlices.json: -------------------------------------------------------------------------------- 1 | "2012-12-28T22:38:19.430+06:00" 2 | "2008-01-10T18:36:48.745-03:00" 3 | "2009-08-04T04:52:17.443Z" 4 | "2012-03-04T12:19:00.040Z" 5 | "2008-03-06T21:02:28.910-11:00" 6 | "2009-10-29T02:43:41.657+04:00" 7 | "2008-05-23T17:31:37.488-00:00" 8 | "2009-07-17T10:30:16.115+07:00" 9 | "2011-03-06T13:56:56.877-02:00" 10 | "2010-02-09T02:20:17.040-05:00" 11 | "2012-08-15T21:05:04.684Z" 12 | "2007-02-04T10:58:14.041-01:00" 13 | "2012-10-11T00:36:31.692-02:00" 14 | "2012-07-30T13:18:40.252-03:00" 15 | "2009-05-18T11:33:38.358+11:00" 16 | "2011-10-27T01:11:04.423-04:00" 17 | "2011-08-11T19:29:55.119+05:00" 18 | "2009-05-02T01:14:41.555-10:00" 19 | "2011-02-15T13:49:53.937+07:00" 20 | "2007-03-24T04:49:22.259-09:00" 21 | "2011-02-10T14:53:34.278-01:00" 22 | "2012-03-14T03:48:21.874Z" -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/millisSinceEpoch.json: -------------------------------------------------------------------------------- 1 | 1330192887710 2 | 1329583492854 3 | 1329848922774 4 | 1329583492780 5 | 1330447492420 6 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/millisTimeZone.json: -------------------------------------------------------------------------------- 1 | -36000000 2 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/model1.json: -------------------------------------------------------------------------------- 1 | { "model1": {"coefficients": [{"foo":{"estimate":3.0}, "bar":{"estimate":2.0}}, {"estimate":5}]}, "model2": {"coefficients": [{"foo":{"estimate":3.0}}, {"estimate":5}]} } 2 | { "model3": {"coefficients": [{"bar":{"estimate": 2.5}}, {"estimate":2}]} } 3 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/model1data.json: -------------------------------------------------------------------------------- 1 | {"foo": 12.5, "bar": 3} 2 | {"foo": 12.0} 3 | {"foo": 1, "bar": 2, "baz": 3} 4 | {"foo": 4, "bar": -5.2} 5 | {"bar": 7.1} 6 | [3, true, null, 3.2] 7 | {"foo": 7, "bar": -1, "baz": 4.4} 8 | {"foo": 6, "bar": 6, "ack": null} 9 | {"foo": 8, "bar": 5} 10 | "foobar" 11 | {"foo": -1, "bar": -1} 12 | {"foo": true, "bar": 9} 13 | {"foo": -7, "bar": -1} 14 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/model2.json: -------------------------------------------------------------------------------- 1 | { "model1": {"coefficients": [{"foo":{"estimate":3.0}, "bar":{"estimate":2.0}}, {"estimate":5}]}, "model1": {"coefficients": [{"foo":{"estimate":3.0}}, {"estimate":5}]}, "model3": {"coefficients": [{"baz": {"estimate":4.0}}, {"estimate":6}]} } 2 | { "model2": {"coefficients": [{"foo":{"estimate":3.0}, "baz":{"estimate":2.0}}, {"estimate":5}]}, "model1": {"coefficients": [{"foo":{"estimate":3.0}}, {"estimate":6}]} } 3 | { "model2": {"coefficients": [[{"estimate":2}, {"estimate":7}], {"estimate":2}]}, "model3": {"coefficients": [{"bar": {"estimate":-1}}, {"estimate":2}]} } 4 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/model2data.json: -------------------------------------------------------------------------------- 1 | {"foo": 1, "bar": 2, "baz": 3} 2 | {"foo": 4, "bar": -5.2} 3 | [1] 4 | {"bar": 7.1} 5 | [3, 4] 6 | {"foo": 6, "bar": 6, "ack": null} 7 | [6, 9, 2] 8 | {"foo": -1, "bar": true} 9 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/number.json: -------------------------------------------------------------------------------- 1 | 28932847237483 2 | -2327988239478983 3 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/numbers.json: -------------------------------------------------------------------------------- 1 | 42 2 | 12 3 | 77 4 | 1 5 | 13 6 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/numbers2.json: -------------------------------------------------------------------------------- 1 | 42 2 | 12 3 | 77 4 | 1 5 | 13 6 | 1 7 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/numbers3.json: -------------------------------------------------------------------------------- 1 | 77 2 | -1 3 | 14 4 | 42 5 | 0 6 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/numbers4.json: -------------------------------------------------------------------------------- 1 | 0 2 | -1 3 | 1 4 | 42 5 | 1 6 | -23 7 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/numbers5.json: -------------------------------------------------------------------------------- 1 | 4 2 | 7 3 | 12 4 | -13 5 | -34 6 | 0 7 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/numbers6.json: -------------------------------------------------------------------------------- 1 | -10 2 | 34 3 | 0 4 | -10 5 | 11 6 | 12 7 | 11 8 | 11 9 | 5 10 | 34 11 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/numbers7.json: -------------------------------------------------------------------------------- 1 | 0 2 | 1 3 | 13 4 | -1 5 | 10 6 | 77 7 | -77 8 | 12 9 | 3 10 | 8 11 | 11 12 | 0 13 | 2 14 | -9 15 | 3 16 | 5 17 | 1 18 | 2 19 | 88 20 | 90 21 | 34 22 | -36 23 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/numbersAcrossSlices.json: -------------------------------------------------------------------------------- 1 | -7 2 | 15 3 | -13 4 | 11 5 | 7 6 | 11 7 | -7 8 | 0 9 | 14 10 | -3 11 | 6 12 | -12 13 | 10 14 | -9 15 | 15 16 | -5 17 | -13 18 | -14 19 | 11 20 | -5 21 | -5 22 | 13 23 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/numbersHet.json: -------------------------------------------------------------------------------- 1 | 9 2 | 0 3 | -3423.23425 4 | 34.1 5 | 9999 6 | 12.2 7 | 8.293 8 | 122 9 | 13 10 | -1.1 11 | -500000.0 12 | 1 13 | -1 14 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/numbersmod.json: -------------------------------------------------------------------------------- 1 | 2 2 | 3 3 | 12 4 | 5 5 | 4 6 | 24 7 | 8 8 | 27 9 | 19 10 | 20 11 | 21 12 | 22 13 | 23 14 | 6 15 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/pairs.json: -------------------------------------------------------------------------------- 1 | {"first":42,"second":6} 2 | {"first":12,"second":0} 3 | {"first":77,"second":-38} 4 | {"first":1,"second":166} 5 | {"first":13,"second":1} 6 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/smallnumbers.json: -------------------------------------------------------------------------------- 1 | 3 2 | 4 3 | 5 4 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/strings.json: -------------------------------------------------------------------------------- 1 | "" 2 | "quirky" 3 | "solstice + 7" 4 | "Monkey: [Brains]" 5 | "(\"alpha\", \"beta\", \"gamma\")" 6 | " Whitespace is awesome !!!1!! " 7 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/stuff/numbersdiff.json: -------------------------------------------------------------------------------- 1 | 42 2 | 12 3 | 77 4 | 1 5 | 13 6 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/timeString.json: -------------------------------------------------------------------------------- 1 | "Jun 03 2010 04:12:33.323" 2 | "Aug 12 2011 22:42:33.310" 3 | "Jun 04 2010 13:31:49.002" 4 | "Oct 09 2010 09:27:31.953" 5 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/timerange.json: -------------------------------------------------------------------------------- 1 | {"start": "1991-06-14T07:03:07.037Z", "end": "1991-06-14T08:03:07.037Z", "step": "PT01H"} 2 | {"start": "1991-06-14T07:03:07.037Z", "end": "1991-12-14T07:03:07.037Z", "step": "P01Y"} 3 | {"start": "1991-06-14T07:03:07.037Z", "end": "1991-06-14T07:03:15.037Z", "step": "PT05S"} 4 | {"start": "1991-06-14T07:03:07.037Z", "end": "1991-10-14T07:03:07.037Z", "step": "P02M"} 5 | -------------------------------------------------------------------------------- /mimir/src/test/resources/hom/users.json: -------------------------------------------------------------------------------- 1 | {"userId":1,"name":"Miles"} 2 | {"userId":2,"name":"Kris"} 3 | {"userId":3,"name":"Daniel"} 4 | {"userId":4,"name":"Alissa"} 5 | {"userId":5,"name":"Derek"} 6 | -------------------------------------------------------------------------------- /mimir/src/test/resources/uncorrelated.json: -------------------------------------------------------------------------------- 1 | [0, 8] 2 | [1, 4] 3 | [2, 2] 4 | [3, 1] 5 | [4, 2] 6 | [5, 4] 7 | [6, 8] 8 | -------------------------------------------------------------------------------- /mimir/src/test/scala/com/precog/mimir/FullStdLibModule.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.mimir 21 | 22 | import scalaz._ 23 | 24 | trait FullStdLibModule[M[+_]] extends StdLibOpFinderModule[M] 25 | with ReductionFinderModule[M] 26 | with EvaluatorModule[M] { 27 | trait Lib extends StdLibOpFinder with StdLib 28 | object library extends Lib 29 | } 30 | -------------------------------------------------------------------------------- /mirror/build.sbt: -------------------------------------------------------------------------------- 1 | // 2 | // ____ ____ _____ ____ ___ ____ 3 | // | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | // | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | // | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | // |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | // 8 | // This program is free software: you can redistribute it and/or modify it under the terms of the 9 | // GNU Affero General Public License as published by the Free Software Foundation, either version 10 | // 3 of the License, or (at your option) any later version. 11 | // 12 | // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | // the GNU Affero General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Affero General Public License along with this 17 | // program. If not, see . 18 | // 19 | // 20 | 21 | name := "mirror" 22 | 23 | logBuffered := false // gives us incremental output from Specs2 24 | -------------------------------------------------------------------------------- /muspelheim/build.sbt: -------------------------------------------------------------------------------- 1 | // 2 | // ____ ____ _____ ____ ___ ____ 3 | // | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | // | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | // | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | // |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | // 8 | // This program is free software: you can redistribute it and/or modify it under the terms of the 9 | // GNU Affero General Public License as published by the Free Software Foundation, either version 10 | // 3 of the License, or (at your option) any later version. 11 | // 12 | // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | // the GNU Affero General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Affero General Public License along with this 17 | // program. If not, see . 18 | // 19 | // 20 | 21 | name := "muspelheim" 22 | 23 | libraryDependencies ++= Seq( 24 | "org.reflections" % "reflections" % "0.9.5" % "test", 25 | "com.typesafe.akka" % "akka-testkit" % "2.0" % "test" 26 | ) 27 | 28 | logBuffered := false // gives us incremental output from Specs2 29 | -------------------------------------------------------------------------------- /muspelheim/src/main/scala/com/precog/muspelheim/ParseEvalStack.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog 21 | package muspelheim 22 | 23 | import yggdrasil._ 24 | import yggdrasil.table.cf 25 | import mimir._ 26 | 27 | import quirrel._ 28 | import quirrel.emitter._ 29 | import quirrel.parser._ 30 | import quirrel.typer._ 31 | 32 | import scalaz._ 33 | 34 | trait ParseEvalStack[M[+_]] extends Compiler 35 | with LineErrors 36 | with Emitter 37 | with StdLibEvaluatorStack[M] 38 | 39 | -------------------------------------------------------------------------------- /muspelheim/src/test/python/newlinejson.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import sys 4 | 5 | def newline_json(in_file, out_file): 6 | for line in json.load(in_file): 7 | json.dump(line, out_file) 8 | out_file.write('\n') 9 | 10 | if __name__ == '__main__': 11 | if len(sys.argv) < 2: 12 | print "Usage: python newlinejson.py [path to ordinary json file]" 13 | sys.exit(1) 14 | 15 | f = open(sys.argv[1], 'r') 16 | try: 17 | newline_json(f, sys.stdout) 18 | finally: 19 | f.close() 20 | -------------------------------------------------------------------------------- /muspelheim/src/test/resources/test_data/bar.json: -------------------------------------------------------------------------------- 1 | {"a": -1, "c": 8, "b": -1} 2 | {"a": 1, "c": 9, "b": -1} 3 | {"a": -1, "c": 10, "b": 6} 4 | {"a": 3, "c": 11, "b": 7} 5 | {"a": 0, "c": 12, "b": -1} 6 | {"a": 0, "c": 13, "b": -1} 7 | -------------------------------------------------------------------------------- /muspelheim/src/test/resources/test_data/fastspring_mixed_type.json: -------------------------------------------------------------------------------- 1 | {"customer": {"firstName": "John", "country": "CA", "zipcode": "11111", "lastName": "Smith", "organization": "", "email": "john@fastspring.com"}, "product": {"name": "Subscription 1"}, "endDate": "null", "timestamp": {"seconds": 0, "year": 112, "month": 2, "hours": 0, "time": 1331078400000, "date": 7, "minutes": 0, "day": 3, "timezoneOffset": 0}, "currency": "USD", "event": "activated", "regularPriceUsd": 10, "quantity": 1} 2 | {"customer": {"firstName": "Ryan", "country": "US", "zipcode": "93101", "lastName": "Dewell", "organization": "", "email": "ryan@fastspring.com"}, "product": {"name": "ABC Subscription"}, "endDate": {"seconds": 0, "year": 112, "month": 2, "hours": 0, "time": 1331078400000, "date": 7, "minutes": 0, "day": 3, "timezoneOffset": 0}, "timestamp": {"seconds": 0, "year": 112, "month": 2, "hours": 0, "time": 1331078400000, "date": 7, "minutes": 0, "day": 3, "timezoneOffset": 0}, "currency": "USD", "reason": "canceled", "event": "deactivated", "regularPriceUsd": 9, "quantity": 1} 3 | -------------------------------------------------------------------------------- /muspelheim/src/test/resources/test_data/fastspring_nulls.json: -------------------------------------------------------------------------------- 1 | {"customer": {"firstName": "John", "country": "CA", "zipcode": "11111", "lastName": "Smith", "organization": "", "email": "john@fastspring.com"}, "product": {"name": "Subscription 1"}, "endDate": null, "timestamp": {"seconds": 0, "year": 112, "month": 2, "hours": 0, "time": 1331078400000, "date": 7, "minutes": 0, "day": 3, "timezoneOffset": 0}, "currency": "USD", "event": "activated", "regularPriceUsd": 10, "quantity": 1} 2 | {"customer": {"firstName": "Ryan", "country": "US", "zipcode": "93101", "lastName": "Dewell", "organization": "", "email": "ryan@fastspring.com"}, "product": {"name": "ABC Subscription"}, "endDate": {"seconds": 0, "year": 112, "month": 2, "hours": 0, "time": 1331078400000, "date": 7, "minutes": 0, "day": 3, "timezoneOffset": 0}, "timestamp": {"seconds": 0, "year": 112, "month": 2, "hours": 0, "time": 1331078400000, "date": 7, "minutes": 0, "day": 3, "timezoneOffset": 0}, "currency": "USD", "reason": "canceled", "event": "deactivated", "regularPriceUsd": 9, "quantity": 1} 3 | -------------------------------------------------------------------------------- /muspelheim/src/test/resources/test_data/foo.json: -------------------------------------------------------------------------------- 1 | {"a": 0, "b": 4} 2 | {"a": 1, "b": 5} 3 | {"a": 2, "b": 6} 4 | {"a": 3, "b": 7} 5 | -------------------------------------------------------------------------------- /muspelheim/src/test/resources/test_data/test/empty_array.json: -------------------------------------------------------------------------------- 1 | [] 2 | {"foo": []} 3 | -------------------------------------------------------------------------------- /muspelheim/src/test/resources/test_data/test/empty_object.json: -------------------------------------------------------------------------------- 1 | {} 2 | {"foo": {}} 3 | -------------------------------------------------------------------------------- /muspelheim/src/test/resources/test_data/test/null.json: -------------------------------------------------------------------------------- 1 | null 2 | {"foo": null} 3 | -------------------------------------------------------------------------------- /muspelheim/src/test/scala/com/precog/muspelheim/EvalStackSpecs.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog 21 | package muspelheim 22 | 23 | import yggdrasil._ 24 | import blueeyes.json._ 25 | import com.precog.common._ 26 | import org.specs2.mutable._ 27 | 28 | trait EvalStackSpecs extends Specification { 29 | type TestStack <: EvalStackLike 30 | val stack: TestStack 31 | } 32 | 33 | trait EvalStackLike { 34 | type IdType 35 | 36 | def eval(str: String, debug: Boolean = false): Set[SValue] 37 | def evalE(str: String, debug: Boolean = false): Set[(Vector[IdType], SValue)] 38 | } 39 | -------------------------------------------------------------------------------- /niflheim/src/main/scala/com/precog/niflheim/CookedBlock.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.niflheim 21 | 22 | import java.io.File 23 | 24 | final class CookedBlock(segments: Map[SegmentId, File]) { 25 | } 26 | 27 | object CookedBlock { 28 | def fromFiles(files: Seq[File]): CookedBlock = { 29 | sys.error("...") 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /niflheim/src/main/scala/com/precog/niflheim/StorageReader.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.niflheim 21 | 22 | import com.precog.common._ 23 | import com.precog.util._ 24 | 25 | trait StorageReader { 26 | def snapshot(pathConstraints: Option[Set[CPath]]): Block 27 | def snapshotRef(refConstraints: Option[Set[ColumnRef]]): Block 28 | def structure: Iterable[ColumnRef] 29 | 30 | def isStable: Boolean 31 | 32 | def id: Long 33 | 34 | /** 35 | * Returns the total length of the block. 36 | */ 37 | def length: Int 38 | 39 | override def toString = "StorageReader: id = %d, length = %d, structure = %s".format(id, length, structure) 40 | } 41 | -------------------------------------------------------------------------------- /performance/src/test/scala/com/precog/performance/PerformanceSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.performance 21 | 22 | import org.specs2.mutable.Specification 23 | 24 | import java.io.File 25 | import java.nio.ByteBuffer 26 | 27 | class PerformanceSuite 28 | with RoutingPerformanceSpec 29 | with YggdrasilPerformanceSpec 30 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.12.2 2 | -------------------------------------------------------------------------------- /quirrel/build.sbt: -------------------------------------------------------------------------------- 1 | // 2 | // ____ ____ _____ ____ ___ ____ 3 | // | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | // | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | // | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | // |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | // 8 | // This program is free software: you can redistribute it and/or modify it under the terms of the 9 | // GNU Affero General Public License as published by the Free Software Foundation, either version 10 | // 3 of the License, or (at your option) any later version. 11 | // 12 | // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | // the GNU Affero General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Affero General Public License along with this 17 | // program. If not, see . 18 | // 19 | // 20 | 21 | name := "quirrel" 22 | 23 | libraryDependencies ++= Seq( 24 | "com.codecommit" %% "gll-combinators" % "2.2-SNAPSHOT" 25 | ) 26 | 27 | initialCommands in console := """ 28 | | import com.codecommit.gll.LineStream 29 | | 30 | | import com.precog.quirrel._ 31 | | import emitter._ 32 | | import parser._ 33 | | import typer._ 34 | | import QuirrelConsole._ 35 | """.stripMargin 36 | 37 | logBuffered := false // gives us incremental output from Specs2 38 | -------------------------------------------------------------------------------- /quirrel/src/main/scala/com/precog/quirrel.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog 21 | 22 | package object quirrel { 23 | type TicId = String 24 | 25 | case class Identifier(namespace: Vector[String], id: String) { 26 | override def toString = (namespace map { _ + "::" } mkString) + id 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /quirrel/src/main/scala/com/precog/quirrel/Utils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.quirrel 21 | 22 | object Utils { 23 | def merge[A, B](first: Map[A, Set[B]], second: Map[A, Set[B]]): Map[A, Set[B]] = { 24 | second.foldLeft(first) { 25 | case (acc, (key, set)) if acc contains key => acc.updated(key, acc(key) ++ set) 26 | case (acc, pair) => acc + pair 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /quirrel/src/main/scala/com/precog/quirrel/emitter/Solutions.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.quirrel 21 | package emitter 22 | 23 | trait Solutions extends parser.AST { 24 | sealed trait Solution 25 | 26 | case class Conjunction(left: Solution, right: Solution) extends Solution 27 | case class Disjunction(left: Solution, right: Solution) extends Solution 28 | case class Definition(expr: Expr) extends Solution 29 | } 30 | -------------------------------------------------------------------------------- /quirrel/src/test/scala/com/precog/quirrel/CompilerUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.quirrel 21 | 22 | import org.specs2.mutable._ 23 | 24 | import com.codecommit.gll._ 25 | 26 | trait CompilerUtils extends Specification with Compiler with Errors { 27 | def compileSingle(str: LineStream): Expr = { 28 | val forest = compile(str) 29 | val validForest = forest filter { tree => 30 | tree.errors forall isWarning 31 | } 32 | 33 | if (validForest.size == 1) { 34 | validForest.head 35 | } else { 36 | forest must haveSize(1) 37 | forest.head 38 | } 39 | } 40 | 41 | def compileSingle(str: String): Expr = compileSingle(LineStream(str)) 42 | } 43 | -------------------------------------------------------------------------------- /quirrel/src/test/scala/com/precog/quirrel/RandomLibrarySpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog 21 | package quirrel 22 | 23 | import bytecode.{Instructions, RandomLibrary} 24 | 25 | trait RandomLibrarySpec extends Instructions { 26 | type Lib = RandomLibrary 27 | val library = new RandomLibrary{} 28 | } 29 | -------------------------------------------------------------------------------- /quirrel/src/test/scala/com/precog/quirrel/StaticLibrarySpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog 21 | package quirrel 22 | 23 | import bytecode.{Instructions, StaticLibrary} 24 | 25 | trait StaticLibrarySpec extends Instructions { 26 | type Lib = StaticLibrary 27 | val library = new StaticLibrary{} 28 | } 29 | -------------------------------------------------------------------------------- /ragnarok/scripts/disjunction.sh: -------------------------------------------------------------------------------- 1 | ## 2 | ## ____ ____ _____ ____ ___ ____ 3 | ## | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | ## | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | ## | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | ## |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | ## 8 | ## This program is free software: you can redistribute it and/or modify it under the terms of the 9 | ## GNU Affero General Public License as published by the Free Software Foundation, either version 10 | ## 3 of the License, or (at your option) any later version. 11 | ## 12 | ## This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | ## without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | ## the GNU Affero General Public License for more details. 15 | ## 16 | ## You should have received a copy of the GNU Affero General Public License along with this 17 | ## program. If not, see . 18 | ## 19 | ## 20 | #!/bin/bash 21 | 22 | java \ 23 | -cp ragnarok/target/ragnarok-assembly-*.jar \ 24 | com.precog.ragnarok.test.DisjunctionTestSuite \ 25 | --root-dir /home/miles/projects/eclipse/precog-2.9.2/workspace/platform/jprofiler/jprofiler.db --runs 1 --dry-runs 0 --timeout 28800 --json 26 | 27 | -------------------------------------------------------------------------------- /ragnarok/scripts/medals.sh: -------------------------------------------------------------------------------- 1 | ## 2 | ## ____ ____ _____ ____ ___ ____ 3 | ## | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | ## | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | ## | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | ## |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | ## 8 | ## This program is free software: you can redistribute it and/or modify it under the terms of the 9 | ## GNU Affero General Public License as published by the Free Software Foundation, either version 10 | ## 3 of the License, or (at your option) any later version. 11 | ## 12 | ## This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | ## without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | ## the GNU Affero General Public License for more details. 15 | ## 16 | ## You should have received a copy of the GNU Affero General Public License along with this 17 | ## program. If not, see . 18 | ## 19 | ## 20 | #!/bin/bash 21 | 22 | java \ 23 | -cp ragnarok/target/ragnarok-assembly-*.jar \ 24 | com.precog.ragnarok.test.MedalsTestSuite \ 25 | --root-dir /home/miles/projects/eclipse/precog-2.9.2/workspace/platform/jprofiler/jprofiler.db --runs 2 --dry-runs 1 --timeout 28800 --json 26 | 27 | -------------------------------------------------------------------------------- /ragnarok/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /ragnarok/src/main/scala/com/precog/ragnarok/NihdbStressTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.ragnarok 21 | 22 | import com.precog.yggdrasil.nihdb._ 23 | 24 | object NihdbStressTest { 25 | def main(args: Array[String]) { 26 | new StressTest().main(args) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /ragnarok/src/main/scala/com/precog/ragnarok/PerfTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.ragnarok 21 | 22 | import scalaz._ 23 | 24 | sealed trait PerfTest 25 | case class RunQuery(query: String) extends PerfTest 26 | case object RunSequential extends PerfTest 27 | case object RunConcurrent extends PerfTest 28 | case class Group(name: String) extends PerfTest 29 | 30 | -------------------------------------------------------------------------------- /ragnarok/src/main/scala/com/precog/ragnarok/test/ClicksTestSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | //package com.precog 21 | //package ragnarok 22 | //package test 23 | // 24 | //object ClicksTestSuite extends ClicksLikePerfTestSuite { 25 | // val data = "//clicks" 26 | // 27 | // "simple" := simpleQueries() 28 | // "grouping" := groupingQueries() 29 | // "advanced grouping" := advancedGroupingQueries() 30 | //} 31 | // 32 | // 33 | -------------------------------------------------------------------------------- /ragnarok/src/main/scala/com/precog/ragnarok/test/ConversionsTestSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | //package com.precog 21 | //package ragnarok 22 | //package test 23 | // 24 | // 25 | //object ConversionsTestSuite extends ClicksLikePerfTestSuite { 26 | // val data = "//conversions" 27 | // 28 | // "simple" := simpleQueries() 29 | // "grouping" := groupingQueries() 30 | // "advanced grouping" := advancedGroupingQueries() 31 | //} 32 | -------------------------------------------------------------------------------- /ragnarok/src/main/scala/com/precog/ragnarok/test/GroupingTestSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | //package com.precog 21 | //package ragnarok 22 | //package test 23 | // 24 | //object BugTestSuite extends PerfTestSuite { 25 | // query( 26 | // """ 27 | // | conversions := //conversions 28 | // | 29 | // | result := solve 'customerId 30 | // | conversions' := conversions where conversions.customer.ID = 'customerId 31 | // | 32 | // | {count: count(conversions'), customerId: 'customerId} 33 | // | 34 | // | result 35 | // | """.stripMargin) 36 | //} 37 | -------------------------------------------------------------------------------- /ragnarok/src/main/scala/com/precog/ragnarok/test/ImpressionsTestSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | //package com.precog 21 | //package ragnarok 22 | //package test 23 | // 24 | //object ImpressionsTestSuite extends ClicksLikePerfTestSuite { 25 | // val data = "//impressions" 26 | // 27 | // "simple" := simpleQueries() 28 | // "grouping" := groupingQueries() 29 | // "advanced grouping" := advancedGroupingQueries() 30 | //} 31 | // 32 | // 33 | -------------------------------------------------------------------------------- /ragnarok/src/main/scala/com/precog/ragnarok/test/KeenfulTestSuite1.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog 21 | package ragnarok 22 | package test 23 | 24 | object KeenfulTestSuite1 extends PerfTestSuite { 25 | query( 26 | """ 27 | import std::time::* 28 | 29 | data := //keenful 30 | 31 | data' := data where 32 | getMillis (data.action.created_date) > getMillis("2013-03-03") & 33 | getMillis (data.action.created_date) < getMillis("2013-03-10") 34 | 35 | count(distinct(data'.visitor.id)) 36 | --26136 ms 37 | --2971 ms (1/1) 38 | --3113 ms (6/3) 39 | """) 40 | } 41 | -------------------------------------------------------------------------------- /ragnarok/src/main/scala/com/precog/ragnarok/test/KeenfulTestSuite3.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog 21 | package ragnarok 22 | package test 23 | 24 | object KeenfulTestSuite3 extends PerfTestSuite { 25 | query( 26 | """ 27 | import std::stats::rank 28 | data := //keenful 29 | 30 | data' := data where data.action.verb = "view" 31 | 32 | byItem := solve 'item_id, 'item_url 33 | data'' := data' where data'.item.id = 'item_id & data'.item.url = 'item_url 34 | { 35 | item_id: 'item_id, 36 | count: count(data''), 37 | item_url : 'item_url 38 | } 39 | 40 | byItem' := byItem with {rank : rank(byItem.count)} 41 | 42 | byItem' where byItem'.rank > max(byItem'.rank - 5) 43 | """) 44 | } 45 | -------------------------------------------------------------------------------- /ragnarok/src/main/scala/com/precog/ragnarok/test/LinearRegressionTestSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog 21 | package ragnarok 22 | package test 23 | 24 | object LinearRegressionTestSuite extends PerfTestSuite { 25 | query( 26 | """ 27 | locations := //sita 28 | 29 | std::stats::linearRegression(locations.acc, { y: locations.y , x: locations.x, id: locations.id }) 30 | """) 31 | } 32 | -------------------------------------------------------------------------------- /ragnarok/src/main/scala/com/precog/ragnarok/test/Platform868.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog 21 | package ragnarok 22 | package test 23 | 24 | object Platform868 extends PerfTestSuite { 25 | query( 26 | """ 27 | billing := //billing 28 | conversions := //conversions 29 | 30 | solve 'customerId 31 | billing' := billing where billing.customer.ID = 'customerId 32 | conversions' := conversions where conversions.customer.ID = 'customerId 33 | 34 | billing' ~ conversions' 35 | {customerId: 'customerId, lastDate: billing'.date} 36 | """) 37 | } 38 | -------------------------------------------------------------------------------- /ragnarok/src/main/scala/com/precog/ragnarok/test/Platform967.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/precog/platform/9655622c4969a025ac198674d06b287fa5bbb8d1/ragnarok/src/main/scala/com/precog/ragnarok/test/Platform967.scala -------------------------------------------------------------------------------- /ragnarok/src/main/scala/com/precog/ragnarok/test/SitaAssign.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog 21 | package ragnarok 22 | package test 23 | 24 | object SitaAssign extends PerfTestSuite { 25 | query( 26 | """ 27 | locations := //sita 28 | --locations := //sita100k 29 | import std::stats::* 30 | 31 | model := kMedians({x: locations.x, y: locations.y}, 5) 32 | 33 | -- sloooow 34 | assignClusters( locations, model) 35 | """) 36 | } 37 | -------------------------------------------------------------------------------- /ragnarok/src/main/scala/com/precog/ragnarok/test/SitaClustering.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog 21 | package ragnarok 22 | package test 23 | 24 | object SitaClustering extends PerfTestSuite { 25 | query( 26 | """ 27 | import std::stats::* 28 | 29 | locations := //sita1k 30 | 31 | points := { x: locations.x, y: locations.y } 32 | kMedians(points, 5) 33 | """ 34 | ) 35 | } 36 | -------------------------------------------------------------------------------- /scripts/delta.py: -------------------------------------------------------------------------------- 1 | import json 2 | import sys 3 | 4 | 5 | def main(): 6 | if len(sys.argv) > 1: 7 | f = open(sys.argv[1], "r") 8 | else: 9 | f = sys.stdin 10 | run = json.load(f) 11 | for test in run: 12 | if test[u'delta'] != 'insignificant' and u'query' in test: 13 | print u"--------------------------------------------------------------" 14 | print test[u'query'] 15 | print u"Was: %s" % test[u'baseline'][u'mean'] 16 | print u"Now: %s" % test[u'stats'][u'mean'] 17 | print u"Change: %s" % test[u'delta'] 18 | print 19 | print 20 | 21 | if __name__ == "__main__": 22 | main() 23 | -------------------------------------------------------------------------------- /scripts/fixjdbm.sh: -------------------------------------------------------------------------------- 1 | ## 2 | ## ____ ____ _____ ____ ___ ____ 3 | ## | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | ## | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | ## | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | ## |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | ## 8 | ## This program is free software: you can redistribute it and/or modify it under the terms of the 9 | ## GNU Affero General Public License as published by the Free Software Foundation, either version 10 | ## 3 of the License, or (at your option) any later version. 11 | ## 12 | ## This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | ## without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | ## the GNU Affero General Public License for more details. 15 | ## 16 | ## You should have received a copy of the GNU Affero General Public License along with this 17 | ## program. If not, see . 18 | ## 19 | ## 20 | #!/bin/bash 21 | 22 | [ "$#" -eq 1 ] || { 23 | echo "Usage: fixjdbm.sh " 24 | exit 1 25 | } 26 | 27 | [ -f "byIdentity.d.0" ] || { 28 | echo "This script needs to be run from a jdbm/ directory for a projection!" 29 | exit 1 30 | } 31 | 32 | mkdir recovery 33 | monit unmonitor $1 34 | stop $1 35 | cp byIdentity* recovery/ 36 | cd recovery 37 | ~ubuntu/scala-2.9.2/bin/scala -cp /usr/share/java/$1.jar ~ubuntu/fixjdbm.scala 38 | cp fixed/* ../ 39 | start $1 40 | monit monitor $1 41 | -------------------------------------------------------------------------------- /scripts/parse-kafka.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby 2 | 3 | require 'rubygems' 4 | require 'json' 5 | 6 | Event = Struct.new :num, :id, :path, :token, :data 7 | 8 | EVENT_REGEXP = /^Event-([^\s]+) Id: ([^\s]+) Path: ([^\s]+) Token: ([^\s]+)/ 9 | 10 | def usage 11 | puts "usage: parse-kafka.rb " 12 | exit 100 13 | end 14 | 15 | usage if ARGV.size != 2 16 | 17 | input = ARGV[0] 18 | dir = ARGV[1] 19 | 20 | def rec_mkdir(dir) 21 | parent = File.dirname dir 22 | rec_mkdir parent unless File.exists? parent 23 | Dir.mkdir dir 24 | end 25 | 26 | File.open input, 'r' do |file| 27 | line = file.gets 28 | 29 | while line 30 | md = EVENT_REGEXP.match line 31 | 32 | if md 33 | num = md[1].to_i 34 | id = md[2] 35 | path = md[3] 36 | token = md[4] 37 | 38 | to_parse = '' 39 | line = file.gets 40 | while line && !(EVENT_REGEXP.match line) 41 | to_parse += line 42 | line = file.gets 43 | end 44 | 45 | begin 46 | data = JSON.parse to_parse.strip 47 | rescue Exception => e 48 | puts e 49 | puts 'WARNING: failed to process!' 50 | puts 'Event num: ' + num.to_s 51 | puts to_parse.strip 52 | end 53 | 54 | if data 55 | event = Event.new(num, id, path, token, data) 56 | out_path = (dir + event.path).gsub /\/$/, '.jsonb' 57 | 58 | parent_dir = File.dirname out_path 59 | rec_mkdir parent_dir unless File.exists? parent_dir 60 | 61 | File.open out_path, 'a+' do |out_file| 62 | out_file.puts(JSON.generate event.data) 63 | end 64 | end 65 | else 66 | puts 'Malformed line: ' + line 67 | exit -1 68 | end 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /surtr/dist/README: -------------------------------------------------------------------------------- 1 | COMMAND LINE 2 | 3 | To run the interactive command line: 4 | 5 | bin/repl 6 | 7 | EXAMPLE QUERIES 8 | 9 | quirrel> count(dataset(//campaigns)) 10 | | 11 | 12 | count(dataset(//campaigns)) 13 | 14 | 15 | quirrel> tests := dataset(//campaigns) 16 | | count(tests where tests.gender = "male") 17 | | 18 | 19 | tests := dataset(//campaigns) 20 | count(tests where tests.gender = "male") 21 | 22 | 23 | quirrel> tests := dataset(//campaigns) 24 | | histogram('platform) := 25 | | { platform: 'platform, num: count(tests where tests.platform = 'platform) } 26 | | histogram 27 | | 28 | 29 | tests := dataset(//campaigns) 30 | histogram('platform) := 31 | { platform: 'platform, num: count(tests where tests.platform = 'platform) } 32 | histogram 33 | -------------------------------------------------------------------------------- /surtr/dist/bin/repl: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | java -Xmx1G -jar lib/pandora-assembly-0.0.1-SNAPSHOT.jar $@ 4 | -------------------------------------------------------------------------------- /surtr/dist_private/bin/init: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | java -cp lib/pandora-assembly-0.0.1-SNAPSHOT.jar com.precog.ingest.util.WebappIngestProducer conf/init.properties 4 | -------------------------------------------------------------------------------- /surtr/dist_private/bin/insert: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | java -cp lib/pandora-assembly-0.0.1-SNAPSHOT.jar com.precog.ingest.util.WebappIngestProducer conf/insert.properties 4 | -------------------------------------------------------------------------------- /surtr/dist_private/conf/init.properties: -------------------------------------------------------------------------------- 1 | messages=100 2 | delay=-1 3 | threads=1 4 | repeats=1 5 | apiKey=C18ED787-BF07-4097-B819-0415C759C8D5 6 | serviceUrl=http://localhost:8181/track/ 7 | -------------------------------------------------------------------------------- /surtr/dist_private/conf/insert.properties: -------------------------------------------------------------------------------- 1 | messages=100 2 | delay=-1 3 | threads=1 4 | repeats=1 5 | apiKey=C18ED787-BF07-4097-B819-0415C759C8D5 6 | serviceUrl=http://localhost:8181/track/insert/ 7 | -------------------------------------------------------------------------------- /surtr/src/main/resources: -------------------------------------------------------------------------------- 1 | ../../../muspelheim/src/test/resources/test_data/ -------------------------------------------------------------------------------- /util/build.sbt: -------------------------------------------------------------------------------- 1 | // 2 | // ____ ____ _____ ____ ___ ____ 3 | // | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | // | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | // | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | // |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | // 8 | // This program is free software: you can redistribute it and/or modify it under the terms of the 9 | // GNU Affero General Public License as published by the Free Software Foundation, either version 10 | // 3 of the License, or (at your option) any later version. 11 | // 12 | // This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | // the GNU Affero General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Affero General Public License along with this 17 | // program. If not, see . 18 | // 19 | // 20 | 21 | name := "util" 22 | 23 | libraryDependencies ++= Seq( 24 | "commons-io" % "commons-io" % "2.3", 25 | "javax.mail" % "mail" % "1.4", 26 | "org.fusesource.scalate" % "scalate-core_2.9" % "1.6.1" 27 | ) 28 | 29 | logBuffered := false // gives us incremental output from Specs2 30 | -------------------------------------------------------------------------------- /util/src/main/scala/com/precog/util/Close.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.util 21 | 22 | trait Close[-A] { 23 | def close(a: A): Unit 24 | } 25 | 26 | object Close { 27 | import java.io._ 28 | 29 | implicit object InputStreamClose extends Close[InputStream] { 30 | def close(in: InputStream) = { 31 | in.close 32 | } 33 | } 34 | 35 | implicit object OutputStreamClose extends Close[OutputStream] { 36 | def close(a: OutputStream) = { 37 | a.flush 38 | a.close 39 | } 40 | } 41 | } 42 | 43 | 44 | // vim: set ts=4 sw=4 et: 45 | -------------------------------------------------------------------------------- /util/src/main/scala/com/precog/util/IdGen.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.util 21 | 22 | import java.util.concurrent.atomic.AtomicInteger 23 | 24 | class IdGen { 25 | private[this] val currentId = new AtomicInteger(0) 26 | 27 | def nextInt(): Int = currentId.getAndIncrement() 28 | } 29 | 30 | // Shared Int could easily overflow: Unshare? Extend to a Long? Different approach? 31 | object IdGen extends IdGen 32 | -------------------------------------------------------------------------------- /util/src/main/scala/com/precog/util/Identifier.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.util 21 | 22 | /** 23 | * Opaque symbolic identifier (like Int, but better!). 24 | */ 25 | final class Identifier extends AnyRef 26 | -------------------------------------------------------------------------------- /util/src/main/scala/com/precog/util/JBigDecimalOrdering.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.common 21 | 22 | import scala.math.Ordering 23 | 24 | import java.math._ 25 | 26 | object JBigDecimalOrdering extends Ordering[BigDecimal] { 27 | def compare(a : BigDecimal, b : BigDecimal) = a.compareTo(b) 28 | } 29 | -------------------------------------------------------------------------------- /util/src/main/scala/com/precog/util/PrecogUnit.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.util 21 | 22 | /** 23 | * This class exists as a replacement for Unit in Unit-returning functions. 24 | * The main issue with unit is that the coercion of any return value to 25 | * unit means that we were sometimes masking mis-returns of functions. In 26 | * particular, functions returning IO[Unit] would happily coerce IO => Unit, 27 | * which essentially discarded the inner IO work. 28 | */ 29 | sealed trait PrecogUnit 30 | 31 | object PrecogUnit extends PrecogUnit { 32 | implicit def liftUnit(unit: Unit): PrecogUnit = this 33 | } 34 | -------------------------------------------------------------------------------- /util/src/main/scala/com/precog/util/scalaz_extensions.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.util 21 | 22 | import scalaz._ 23 | import scalaz.StreamT._ 24 | 25 | package object scalaz_extensions { 26 | /* 27 | def mapM[M[_], A, B](stream: StreamT[M, A])(f: A => M[B])(implicit m: Monad[M]): StreamT[M, B] = stepBind(stream) { 28 | _( yieldd = (a, s) => m.map(f(a)) { Yield(_, mapM(s)(f)) } 29 | , skip = s => m.point(Skip(mapM(s)(f))) 30 | , done = m.point(Done) 31 | ) 32 | } 33 | 34 | private def stepBind[M[_], A, B](stream: StreamT[M, A])(f: Step[A, StreamT[M, A]] => M[Step[B, StreamT[M, B]]])(implicit M: Monad[M]): StreamT[M, B] = 35 | StreamT(M.bind(stream.step)(f)) 36 | */ 37 | } 38 | -------------------------------------------------------------------------------- /yggdrasil/conf/dev/ygg_shard.conf: -------------------------------------------------------------------------------- 1 | precog { 2 | storage { 3 | root = /tmp/precog/data/ 4 | sortBufferSize = 100000 5 | } 6 | kafka { 7 | enabled = true 8 | topic { 9 | events = test-topic-4 10 | } 11 | consumer { 12 | zk { 13 | connect = 127.0.0.1:2181 14 | connecttimeout { 15 | ms = 1000000 16 | } 17 | } 18 | groupid = test_group_1 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /yggdrasil/lib/jdbm-3.0-SNAPSHOT.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/precog/platform/9655622c4969a025ac198674d06b287fa5bbb8d1/yggdrasil/lib/jdbm-3.0-SNAPSHOT.jar -------------------------------------------------------------------------------- /yggdrasil/src/main/scala/com/precog/yggdrasil/EnormousCartesianException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.yggdrasil 21 | 22 | case class EnormousCartesianException(left: TableSize, right: TableSize) extends RuntimeException { 23 | override def getMessage = 24 | "cannot evaluate cartesian of sets with size %s and %s".format(left, right) 25 | } 26 | -------------------------------------------------------------------------------- /yggdrasil/src/main/scala/com/precog/yggdrasil/IdSource.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.yggdrasil 21 | 22 | trait IdSource { 23 | def nextIdBlock(n: Int): Long 24 | def nextId(): Long 25 | } 26 | 27 | final class FreshAtomicIdSource extends IdSource { 28 | private val source = new java.util.concurrent.atomic.AtomicLong 29 | def nextId() = source.getAndIncrement 30 | def nextIdBlock(n: Int): Long = { 31 | var nextId = source.get() 32 | while (!source.compareAndSet(nextId, nextId + n)) { 33 | nextId = source.get() 34 | } 35 | nextId 36 | } 37 | } 38 | 39 | // vim: set ts=4 sw=4 et: 40 | -------------------------------------------------------------------------------- /yggdrasil/src/main/scala/com/precog/yggdrasil/StorageFormat.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.yggdrasil 21 | 22 | sealed trait StorageFormat { 23 | def min(i: Int): Int 24 | val isFixed: Boolean 25 | } 26 | 27 | case object LengthEncoded extends StorageFormat { 28 | def min(i: Int) = i 29 | final val isFixed = false 30 | } 31 | 32 | case class FixedWidth(width: Int) extends StorageFormat { 33 | def min(i: Int) = width min i 34 | final val isFixed = true 35 | } 36 | 37 | 38 | // vim: set ts=4 sw=4 et: 39 | -------------------------------------------------------------------------------- /yggdrasil/src/main/scala/com/precog/yggdrasil/actor.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.yggdrasil 21 | 22 | package actor { 23 | // A case object used as a request for status information across actor types 24 | case object Status 25 | } 26 | 27 | 28 | // vim: set ts=4 sw=4 et: 29 | -------------------------------------------------------------------------------- /yggdrasil/src/main/scala/com/precog/yggdrasil/execution/Platform.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.yggdrasil 21 | package execution 22 | 23 | import com.precog.common._ 24 | import com.precog.common.security._ 25 | import com.precog.yggdrasil.vfs._ 26 | 27 | import blueeyes.json._ 28 | import scalaz._ 29 | 30 | trait Execution[M[+_], +A] { 31 | def executorFor(apiKey: APIKey): EitherT[M, String, QueryExecutor[M, A]] 32 | } 33 | 34 | trait Platform[M[+_], Block, +A] extends Execution[M, A] with SecureVFSModule[M, Block] { 35 | def vfs: SecureVFS 36 | } 37 | 38 | -------------------------------------------------------------------------------- /yggdrasil/src/main/scala/com/precog/yggdrasil/jdbm3/JDBMProjection.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.yggdrasil 21 | package jdbm3 22 | 23 | object JDBMProjection { 24 | final val MAX_SPINS = 20 // FIXME: This is related to the JDBM ConcurrentMod exception, and should be removed when that's cleaned up 25 | } 26 | 27 | // FIXME: Again, related to JDBM concurent mod exception 28 | class VicciniException(message: String) extends java.io.IOException("Inconceivable! " + message) 29 | 30 | -------------------------------------------------------------------------------- /yggdrasil/src/main/scala/com/precog/yggdrasil/serialization/StreamSerialization.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.yggdrasil.serialization 21 | 22 | import java.io._ 23 | import java.util.zip._ 24 | 25 | trait StreamSerialization { 26 | def iStream(file: File): DataInputStream 27 | def oStream(file: File): DataOutputStream 28 | } 29 | 30 | trait ZippedStreamSerialization extends StreamSerialization { 31 | def iStream(file: File) = new DataInputStream(new GZIPInputStream(new FileInputStream(file))) 32 | def oStream(file: File) = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(file))) 33 | } 34 | 35 | 36 | // vim: set ts=4 sw=4 et: 37 | -------------------------------------------------------------------------------- /yggdrasil/src/main/scala/com/precog/yggdrasil/vfs.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.yggdrasil 21 | 22 | package object vfs { 23 | type VersionId = java.util.UUID 24 | } 25 | 26 | -------------------------------------------------------------------------------- /yggdrasil/src/test/scala/com/precog/yggdrasil/actor/RoutingTableSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.yggdrasil 21 | package actor 22 | 23 | import com.precog.common._ 24 | import com.precog.common.ingest._ 25 | 26 | 27 | import blueeyes.json._ 28 | 29 | import org.specs2.mutable._ 30 | import org.specs2.matcher.{Matcher, MatchResult, Expectable} 31 | 32 | import scala.collection.immutable.ListMap 33 | import scala.math.BigDecimal 34 | 35 | class RoutingTableSpec extends Specification { 36 | "SinglePathProjectionRoutingTable" should { 37 | "test something" in todo 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /yggdrasil/src/test/scala/com/precog/yggdrasil/packageSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.yggdrasil 21 | 22 | import org.specs2.mutable._ 23 | 24 | object packageSpec extends Specification { 25 | "identity order" should { 26 | "return non-equal results for non-equal length identities" in todo 27 | } 28 | } 29 | 30 | 31 | // vim: set ts=4 sw=4 et: 32 | -------------------------------------------------------------------------------- /yggdrasil/src/test/scala/com/precog/yggdrasil/table/StubIdSourceScannerModule.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * ____ ____ _____ ____ ___ ____ 3 | * | _ \ | _ \ | ____| / ___| / _/ / ___| Precog (R) 4 | * | |_) | | |_) | | _| | | | | /| | | _ Advanced Analytics Engine for NoSQL Data 5 | * | __/ | _ < | |___ | |___ |/ _| | | |_| | Copyright (C) 2010 - 2013 SlamData, Inc. 6 | * |_| |_| \_\ |_____| \____| /__/ \____| All Rights Reserved. 7 | * 8 | * This program is free software: you can redistribute it and/or modify it under the terms of the 9 | * GNU Affero General Public License as published by the Free Software Foundation, either version 10 | * 3 of the License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 13 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 14 | * the GNU Affero General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Affero General Public License along with this 17 | * program. If not, see . 18 | * 19 | */ 20 | package com.precog.yggdrasil 21 | package table 22 | 23 | import com.precog.yggdrasil.util._ 24 | 25 | trait StubIdSourceScannerModule { 26 | type YggConfig = IdSourceConfig 27 | val yggConfig = new IdSourceConfig { 28 | val idSource = new FreshAtomicIdSource 29 | } 30 | } 31 | --------------------------------------------------------------------------------