├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── build_all ├── clean_all ├── deploy_all ├── geospatial ├── LICENSE ├── bin │ ├── post_install │ └── pre_remove ├── build.properties ├── build.xml ├── conf │ ├── log4cxx.properties │ └── mr4c-ld.conf ├── conf_docs │ ├── Doxyfile │ ├── doxygen_footer.html │ └── doxygen_main.tmpl ├── input │ └── thumb.png ├── makefile ├── src │ └── cpp │ │ ├── api │ │ ├── coord │ │ │ ├── BoundingBox.h │ │ │ ├── EastNorthCoord.h │ │ │ ├── EastNorthTrans.h │ │ │ ├── ImageBox.h │ │ │ ├── LatLonCoord.h │ │ │ ├── NormMercCoord.h │ │ │ ├── SimpleEastNorthTrans.h │ │ │ └── coord_api.h │ │ ├── gdal │ │ │ ├── GDALCoordTrans.h │ │ │ ├── GDALFile.h │ │ │ ├── GDALFileBase.h │ │ │ ├── GDALLocalFile.h │ │ │ ├── GDALMemoryFile.h │ │ │ ├── GDALUtils.h │ │ │ └── gdal_api.h │ │ ├── mbtiles │ │ │ ├── MBTilesDataset.h │ │ │ ├── MBTilesUtils.h │ │ │ ├── TileExtractor.h │ │ │ ├── TileKey.h │ │ │ └── mbtiles_api.h │ │ └── mr4c_geo_api.h │ │ └── impl │ │ ├── coord │ │ ├── BoundingBox.cpp │ │ ├── EastNorthCoord.cpp │ │ ├── ImageBox.cpp │ │ ├── LatLonCoord.cpp │ │ ├── NormMercCoord.cpp │ │ └── SimpleEastNorthTrans.cpp │ │ ├── gdal │ │ ├── GDALCoordTrans.cpp │ │ ├── GDALFileBase.cpp │ │ ├── GDALLocalFile.cpp │ │ ├── GDALMemoryFile.cpp │ │ └── GDALUtils.cpp │ │ └── mbtiles │ │ ├── MBTiler.cpp │ │ ├── MBTilesDataset.cpp │ │ ├── MBTilesUtils.cpp │ │ ├── TileExtractor.cpp │ │ └── TileKey.cpp ├── test │ └── cpp │ │ ├── api │ │ └── MR4CGeoTests.h │ │ ├── suites │ │ ├── Cover.cpp │ │ ├── Local.cpp │ │ └── Suite.cpp │ │ └── tests │ │ ├── MR4CGeoTests.cpp │ │ ├── coord │ │ ├── CoordTests.cpp │ │ ├── TestBoundingBox.cpp │ │ ├── TestEastNorthCoord.cpp │ │ ├── TestImageBox.cpp │ │ ├── TestLatLonCoord.cpp │ │ ├── TestNormMercCoord.cpp │ │ └── TestSimpleEastNorthTrans.cpp │ │ ├── gdal │ │ ├── GDALTests.cpp │ │ ├── TestGDALCoordTrans.cpp │ │ ├── TestGDALLocalFile.cpp │ │ ├── TestGDALMemoryFile.cpp │ │ └── TestGDALUtils.cpp │ │ └── mbtiles │ │ ├── MBTilesTests.cpp │ │ ├── TestMBTilesDataset.cpp │ │ ├── TestMBTilesUtils.cpp │ │ ├── TestTileExtractor.cpp │ │ └── TestTileKey.cpp └── tools │ ├── prep_env │ ├── run_coverage │ ├── run_tests │ └── test_local ├── java ├── LICENSE ├── bin │ ├── mr4c │ ├── mr4c_debug │ ├── mr4c_diff │ ├── mr4c_hadoop │ ├── mr4c_hadoop_remote │ ├── mr4c_test │ ├── post_install │ └── pre_remove ├── build.properties ├── build.xml ├── conf │ ├── log4cxx.properties │ ├── log4j.properties │ ├── mr4c-hadoop-site.xml │ └── site.json ├── conf_ivy │ ├── ivy_common.xml │ ├── ivy_mrv1.xml │ ├── ivy_yarn.xml │ └── ivysettings.xml ├── input │ ├── conftest │ │ ├── file1.properties │ │ ├── file2.properties │ │ ├── file3.properties │ │ ├── file4.properties │ │ ├── file5.properties │ │ └── file6.properties │ └── data │ │ ├── dataset │ │ ├── logsbuilder │ │ │ ├── log1.log │ │ │ ├── log2.log │ │ │ └── somepath │ │ │ │ └── log3.log │ │ ├── logsrc │ │ │ ├── notask │ │ │ │ └── logs │ │ │ │ │ ├── log1.log │ │ │ │ │ ├── log2.log │ │ │ │ │ └── log3.log │ │ │ └── withtask │ │ │ │ └── logs │ │ │ │ └── taskid1 │ │ │ │ ├── log1.log │ │ │ │ ├── log2.log │ │ │ │ └── log3.log │ │ ├── simple │ │ │ ├── not_simple.json │ │ │ └── simple.json │ │ ├── test1 │ │ │ ├── input_data │ │ │ │ ├── ss01_c1_2455874.21556848_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556848_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556871_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556871_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556894_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556894_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556917_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556917_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556941_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556941_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556964_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556964_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556987_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556987_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557010_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557010_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557033_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557033_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557056_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557056_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557079_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557079_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557103_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557103_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557126_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557126_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557149_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557149_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557172_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557172_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557195_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557195_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557218_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557218_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557242_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557242_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557265_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557265_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557288_MS_1.5bps.jpc │ │ │ │ └── ss01_c1_2455874.21557288_PAN_1.5bps.jpc │ │ │ ├── input_data_self │ │ │ │ ├── directory.json │ │ │ │ ├── ss01_c1_2455874.21556848_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556848_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556871_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556871_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556894_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556894_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556917_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556917_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556941_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556941_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556964_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556964_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556987_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21556987_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557010_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557010_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557033_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557033_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557056_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557056_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557079_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557079_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557103_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557103_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557126_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557126_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557149_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557149_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557172_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557172_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557195_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557195_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557218_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557218_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557242_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557242_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557265_MS_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557265_PAN_1.5bps.jpc │ │ │ │ ├── ss01_c1_2455874.21557288_MS_1.5bps.jpc │ │ │ │ └── ss01_c1_2455874.21557288_PAN_1.5bps.jpc │ │ │ ├── source.json │ │ │ └── source_self.json │ │ └── test2 │ │ │ └── map.json │ │ └── images │ │ ├── README │ │ ├── directory.json │ │ ├── ss01_c1_2455874.21556848_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21556848_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21556871_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21556871_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21556894_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21556894_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21556917_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21556917_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21556941_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21556941_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21556964_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21556964_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21556987_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21556987_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557010_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557010_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557033_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557033_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557056_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557056_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557079_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557079_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557103_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557103_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557126_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557126_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557149_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557149_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557172_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557172_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557195_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557195_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557218_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557218_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557242_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557242_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557265_MS_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557265_PAN_1.5bps.jpc │ │ ├── ss01_c1_2455874.21557288_MS_1.5bps.jpc │ │ └── ss01_c1_2455874.21557288_PAN_1.5bps.jpc ├── ivy.xml ├── src │ ├── java │ │ └── com │ │ │ └── google │ │ │ └── mr4c │ │ │ ├── AlgoRunner.java │ │ │ ├── AlgoTestRunner.java │ │ │ ├── DiffRunner.java │ │ │ ├── algorithm │ │ │ ├── Algorithm.java │ │ │ ├── AlgorithmBase.java │ │ │ ├── AlgorithmContext.java │ │ │ ├── AlgorithmData.java │ │ │ ├── AlgorithmEnvironment.java │ │ │ ├── AlgorithmSchema.java │ │ │ ├── AlgorithmType.java │ │ │ ├── Algorithms.java │ │ │ ├── EnvironmentSet.java │ │ │ ├── JavaAlgorithm.java │ │ │ └── LogLevel.java │ │ │ ├── config │ │ │ ├── ConfigDescriptor.java │ │ │ ├── ConfigLoader.java │ │ │ ├── ConfigUtils.java │ │ │ ├── Document.java │ │ │ ├── MR4CRunnerConfig.java │ │ │ ├── algorithm │ │ │ │ ├── AlgorithmConfig.java │ │ │ │ ├── DimensionConfig.java │ │ │ │ └── algo.json │ │ │ ├── category │ │ │ │ ├── Category.java │ │ │ │ ├── CategoryBuilder.java │ │ │ │ ├── CategoryConfig.java │ │ │ │ ├── CategoryInfo.java │ │ │ │ ├── CategoryParser.java │ │ │ │ ├── CategorySystemData.java │ │ │ │ ├── CoreConfig.java │ │ │ │ ├── CustomConfig.java │ │ │ │ ├── HadoopConfig.java │ │ │ │ ├── MR4CConfig.java │ │ │ │ ├── MR4CConfigBuilder.java │ │ │ │ ├── S3Config.java │ │ │ │ └── StatsConfig.java │ │ │ ├── diff │ │ │ │ ├── DiffConfig.java │ │ │ │ └── diff.json │ │ │ ├── execution │ │ │ │ ├── DatasetConfig.java │ │ │ │ ├── DimensionConfig.java │ │ │ │ ├── DirectoryConfig.java │ │ │ │ ├── ExecutionConfig.java │ │ │ │ ├── LocationsConfig.java │ │ │ │ ├── MapConfig.java │ │ │ │ ├── PatternMapperConfig.java │ │ │ │ ├── ValueConfig.java │ │ │ │ ├── dataset.json │ │ │ │ ├── directory.json │ │ │ │ ├── exe.json │ │ │ │ └── map.json │ │ │ ├── resources │ │ │ │ ├── LimitSource.java │ │ │ │ ├── Resource.java │ │ │ │ ├── ResourceConfig.java │ │ │ │ ├── ResourceInfo.java │ │ │ │ ├── ResourceLimit.java │ │ │ │ └── ResourceRequest.java │ │ │ ├── site │ │ │ │ ├── ClusterConfig.java │ │ │ │ ├── MR4CSite.java │ │ │ │ └── SiteConfig.java │ │ │ └── test │ │ │ │ ├── AlgoTestConfig.java │ │ │ │ └── test.json │ │ │ ├── content │ │ │ ├── AbstractContentFactory.java │ │ │ ├── ContentFactories.java │ │ │ ├── ContentFactory.java │ │ │ ├── ContentTypes.java │ │ │ ├── HDFSContentFactory.java │ │ │ ├── LocalContentFactory.java │ │ │ ├── RelativeContentFactory.java │ │ │ ├── S3ContentFactory.java │ │ │ └── S3Credentials.java │ │ │ ├── dataset │ │ │ ├── DataFile.java │ │ │ ├── DataFileFinder.java │ │ │ ├── Dataset.java │ │ │ ├── DatasetContext.java │ │ │ ├── DatasetDiff.java │ │ │ ├── DatasetTransformer.java │ │ │ └── LogsDatasetBuilder.java │ │ │ ├── hadoop │ │ │ ├── Cluster.java │ │ │ ├── Clusters.java │ │ │ ├── DataKeyList.java │ │ │ ├── DataLocalizer.java │ │ │ ├── HadoopAlgoRunner.java │ │ │ ├── HadoopBinding.java │ │ │ ├── HadoopContext.java │ │ │ ├── HadoopInputFormat.java │ │ │ ├── HadoopMapper.java │ │ │ ├── HadoopOutputFormat.java │ │ │ ├── HadoopReducer.java │ │ │ ├── HadoopUtils.java │ │ │ ├── LocalHadoopAlgoRunner.java │ │ │ ├── MR4CArgumentParser.java │ │ │ ├── MR4CGenericOptions.java │ │ │ ├── MR4CGenericOptionsParser.java │ │ │ ├── MR4CInputFormat.java │ │ │ ├── MR4CInputSplit.java │ │ │ ├── MR4CMRJob.java │ │ │ ├── MR4CMapper.java │ │ │ ├── MR4COutputFormat.java │ │ │ ├── MR4CRecordReader.java │ │ │ ├── MR4CRecordWriter.java │ │ │ ├── MR4CReducer.java │ │ │ ├── RemoteAlgoRunner.java │ │ │ ├── StaticHadoopBinder.java.mrv1 │ │ │ ├── StaticHadoopBinder.java.yarn │ │ │ ├── mrv1 │ │ │ │ └── MRv1Binding.java │ │ │ └── yarn │ │ │ │ └── YarnBinding.java │ │ │ ├── keys │ │ │ ├── BasicDataKeyFilter.java │ │ │ ├── BasicElementFilter.java │ │ │ ├── CompoundDataKey.java │ │ │ ├── DataKey.java │ │ │ ├── DataKeyComparator.java │ │ │ ├── DataKeyDimension.java │ │ │ ├── DataKeyElement.java │ │ │ ├── DataKeyFactory.java │ │ │ ├── DataKeyFilter.java │ │ │ ├── DataKeyUtils.java │ │ │ ├── DimensionBasedKeyFilter.java │ │ │ ├── DimensionPartition.java │ │ │ ├── ElementFilter.java │ │ │ ├── ElementTransformer.java │ │ │ ├── HasDimensionFilter.java │ │ │ ├── IdentityDataKeyFilter.java │ │ │ ├── KeyDimensionPartitioner.java │ │ │ ├── KeyTransformer.java │ │ │ ├── Keyspace.java │ │ │ ├── KeyspaceDimension.java │ │ │ ├── KeyspacePartition.java │ │ │ ├── KeyspacePartitioner.java │ │ │ ├── NarrowingFilter.java │ │ │ └── SimpleDataKey.java │ │ │ ├── mbtiles │ │ │ ├── MBTilesFile.java │ │ │ ├── Tile.java │ │ │ ├── TileFormat.java │ │ │ └── TileKey.java │ │ │ ├── message │ │ │ ├── DefaultMessageHandler.java │ │ │ ├── HttpMessageHandler.java │ │ │ ├── Message.java │ │ │ ├── MessageHandler.java │ │ │ └── Messages.java │ │ │ ├── metadata │ │ │ ├── DefaultMetadataVisitor.java │ │ │ ├── MetadataArray.java │ │ │ ├── MetadataElement.java │ │ │ ├── MetadataElementType.java │ │ │ ├── MetadataField.java │ │ │ ├── MetadataKey.java │ │ │ ├── MetadataKeyExtractor.java │ │ │ ├── MetadataList.java │ │ │ ├── MetadataMap.java │ │ │ ├── MetadataVisitor.java │ │ │ ├── PrimitiveFactory.java │ │ │ └── PrimitiveType.java │ │ │ ├── nativec │ │ │ ├── ExternalAlgorithm.java │ │ │ ├── ExternalAlgorithmData.java │ │ │ ├── ExternalAlgorithmDataSerializer.java │ │ │ ├── ExternalAlgorithmSerializer.java │ │ │ ├── ExternalContext.java │ │ │ ├── ExternalDataFile.java │ │ │ ├── ExternalDataFileSink.java │ │ │ ├── ExternalDataFileSource.java │ │ │ ├── ExternalDataset.java │ │ │ ├── ExternalDatasetSerializer.java │ │ │ ├── ExternalEntry.java │ │ │ ├── ExternalFactory.java │ │ │ ├── ExternalRandomAccessFileSink.java │ │ │ ├── ExternalRandomAccessFileSource.java │ │ │ ├── NativeAlgorithm.java │ │ │ └── jna │ │ │ │ ├── JnaDataFileSink.java │ │ │ │ ├── JnaDataFileSource.java │ │ │ │ ├── JnaExternalAlgorithm.java │ │ │ │ ├── JnaExternalAlgorithmData.java │ │ │ │ ├── JnaExternalContext.java │ │ │ │ ├── JnaExternalDataFile.java │ │ │ │ ├── JnaExternalDataset.java │ │ │ │ ├── JnaExternalEntry.java │ │ │ │ ├── JnaExternalFactory.java │ │ │ │ ├── JnaNativeAlgorithm.java │ │ │ │ ├── JnaRandomAccessFile.java │ │ │ │ ├── JnaRandomAccessFileSink.java │ │ │ │ ├── JnaRandomAccessFileSource.java │ │ │ │ └── JnaUtils.java │ │ │ ├── serialize │ │ │ ├── AlgorithmSerializer.java │ │ │ ├── ConfigSerializer.java │ │ │ ├── DatasetSerializer.java │ │ │ ├── KeyspaceSerializer.java │ │ │ ├── PropertiesSerializer.java │ │ │ ├── Serializer.java │ │ │ ├── SerializerFactories.java │ │ │ ├── SerializerFactory.java │ │ │ ├── SerializerType.java │ │ │ ├── bean │ │ │ │ ├── AlgorithmBeanSerializer.java │ │ │ │ ├── BeanBasedAlgorithmSerializer.java │ │ │ │ ├── BeanBasedDatasetSerializer.java │ │ │ │ ├── BeanBasedKeyspaceSerializer.java │ │ │ │ ├── DatasetBeanSerializer.java │ │ │ │ ├── KeyspaceBeanSerializer.java │ │ │ │ ├── algorithm │ │ │ │ │ └── AlgorithmSchemaBean.java │ │ │ │ ├── dataset │ │ │ │ │ ├── DataFileBean.java │ │ │ │ │ ├── DatasetBean.java │ │ │ │ │ ├── DatasetFileBean.java │ │ │ │ │ └── DatasetMetadataBean.java │ │ │ │ ├── keys │ │ │ │ │ ├── DataKeyBean.java │ │ │ │ │ ├── DataKeyElementBean.java │ │ │ │ │ ├── KeyspaceBean.java │ │ │ │ │ └── KeyspaceDimensionBean.java │ │ │ │ └── metadata │ │ │ │ │ ├── MetadataArrayBean.java │ │ │ │ │ ├── MetadataBeans.java │ │ │ │ │ ├── MetadataElementBean.java │ │ │ │ │ ├── MetadataEntryBean.java │ │ │ │ │ ├── MetadataFieldBean.java │ │ │ │ │ ├── MetadataKeyBean.java │ │ │ │ │ ├── MetadataListBean.java │ │ │ │ │ └── MetadataMapBean.java │ │ │ ├── json │ │ │ │ ├── DocumentSerializer.java │ │ │ │ ├── JsonAlgorithmBeanSerializer.java │ │ │ │ ├── JsonConfigSerializer.java │ │ │ │ ├── JsonDatasetBeanSerializer.java │ │ │ │ ├── JsonKeyspaceBeanSerializer.java │ │ │ │ ├── JsonPropertiesSerializer.java │ │ │ │ └── MetadataEntryBeanDeserializer.java │ │ │ └── param │ │ │ │ └── ParameterizedConfigSerializer.java │ │ │ ├── sources │ │ │ ├── AbstractDataFileSink.java │ │ │ ├── AbstractDataFileSource.java │ │ │ ├── AbstractDatasetSource.java │ │ │ ├── AbstractFilesDatasetSource.java │ │ │ ├── AggregateFileSource.java │ │ │ ├── AlgoTestSource.java │ │ │ ├── ArchiveDatasetSource.java │ │ │ ├── ArchiveSource.java │ │ │ ├── BinaryDatasetSource.java │ │ │ ├── BytesDataFileSink.java │ │ │ ├── BytesDataFileSource.java │ │ │ ├── CompositeKeyFileMapper.java │ │ │ ├── ConfiguredAlgoTestSource.java │ │ │ ├── ConfiguredDiffSource.java │ │ │ ├── ConfiguredExecutionSource.java │ │ │ ├── CustomAlgoTestSource.java │ │ │ ├── CustomDiffSource.java │ │ │ ├── CustomExecutionSource.java │ │ │ ├── DataFileSink.java │ │ │ ├── DataFileSource.java │ │ │ ├── DataKeyFileMapper.java │ │ │ ├── DatasetSource.java │ │ │ ├── DatasetSources.java │ │ │ ├── DiffSource.java │ │ │ ├── DiskFileSource.java │ │ │ ├── ExecutionSource.java │ │ │ ├── FileSource.java │ │ │ ├── FileSources.java │ │ │ ├── FilesDatasetSource.java │ │ │ ├── FilesDatasetSourceConfig.java │ │ │ ├── HDFSFileSource.java │ │ │ ├── HadoopFileSource.java │ │ │ ├── HeterogenousFileSource.java │ │ │ ├── InMemoryArchiveSource.java │ │ │ ├── InMemoryFileSource.java │ │ │ ├── LogsDatasetSource.java │ │ │ ├── MBTilesDatasetSource.java │ │ │ ├── MapFileSource.java │ │ │ ├── MetafilesDatasetSource.java │ │ │ ├── NullDataFileSink.java │ │ │ ├── NullDatasetSource.java │ │ │ ├── PatternKeyFileMapper.java │ │ │ ├── RandomAccessFileSink.java │ │ │ ├── RandomAccessFileSource.java │ │ │ ├── RandomAccessible.java │ │ │ ├── S3FileSource.java │ │ │ ├── SimpleDatasetSource.java │ │ │ ├── SourceLocationsConfig.java │ │ │ ├── SourceUtils.java │ │ │ ├── StagedDatasetSource.java │ │ │ ├── TransformedDatasetSource.java │ │ │ ├── TransformedDatasetSourceConfig.java │ │ │ ├── URIDataFileSink.java │ │ │ └── URIDataFileSource.java │ │ │ ├── stats │ │ │ ├── MR4CStats.java │ │ │ ├── NoOpStatsClient.java │ │ │ ├── StatsClient.java │ │ │ └── StatsTimer.java │ │ │ └── util │ │ │ ├── CollectionUtils.java │ │ │ ├── CombinatoricUtils.java │ │ │ ├── CustomFormat.java │ │ │ ├── FileFinder.java │ │ │ ├── FinalMonitor.java │ │ │ ├── MR4CLogging.java │ │ │ ├── NamespacedProperties.java │ │ │ ├── Partitioner.java │ │ │ ├── PathUtils.java │ │ │ └── SetAnalysis.java │ └── resources │ │ └── mr4c-hadoop-default.xml ├── test │ ├── java │ │ └── com │ │ │ └── google │ │ │ └── mr4c │ │ │ ├── AlgoRunnerTest.java │ │ │ ├── algorithm │ │ │ ├── AlgorithmDataTest.java │ │ │ ├── AlgorithmDataTestUtils.java │ │ │ └── AlgorithmSchemaTest.java │ │ │ ├── config │ │ │ ├── ConfigDescriptorTest.java │ │ │ ├── ConfigUtilsTest.java │ │ │ ├── algorithm │ │ │ │ ├── AlgoConfigTestUtils.java │ │ │ │ ├── AlgorithmConfigTest.java │ │ │ │ └── DimensionConfigTest.java │ │ │ ├── category │ │ │ │ ├── CategoryBuilderTest.java │ │ │ │ ├── CategoryConfigTest.java │ │ │ │ ├── CategoryParserTest.java │ │ │ │ ├── MR4CConfigBuilderTest.java │ │ │ │ ├── MR4CConfigTest.java │ │ │ │ ├── TestCategoryInfo.java │ │ │ │ └── TestCategorySystemData.java │ │ │ ├── diff │ │ │ │ ├── DiffConfigTest.java │ │ │ │ └── DiffConfigTestUtils.java │ │ │ ├── execution │ │ │ │ ├── ConfigTestUtils.java │ │ │ │ ├── DatasetConfigTest.java │ │ │ │ ├── DirectoryConfigTest.java │ │ │ │ ├── ExecutionConfigTest.java │ │ │ │ ├── LocationsConfigTest.java │ │ │ │ ├── MapConfigTest.java │ │ │ │ └── PatternMapperConfigTest.java │ │ │ ├── resources │ │ │ │ ├── LimitSourceTest.java │ │ │ │ ├── ResourceConfigTest.java │ │ │ │ ├── ResourceLimitTest.java │ │ │ │ ├── ResourceRequestTest.java │ │ │ │ └── TestResourceInfo.java │ │ │ ├── site │ │ │ │ ├── ClusterConfigTest.java │ │ │ │ ├── SiteConfigTest.java │ │ │ │ └── SiteConfigTestUtils.java │ │ │ └── test │ │ │ │ ├── AlgoTestConfigTest.java │ │ │ │ └── AlgoTestConfigTestUtils.java │ │ │ ├── content │ │ │ ├── RelativeContentFactoryTest.java │ │ │ └── S3CredentialsTest.java │ │ │ ├── dataset │ │ │ ├── DataFileTest.java │ │ │ ├── DatasetDiffTest.java │ │ │ ├── DatasetTest.java │ │ │ ├── DatasetTestUtils.java │ │ │ ├── DatasetTransformerTest.java │ │ │ └── LogsDatasetBuilderTest.java │ │ │ ├── hadoop │ │ │ ├── ClusterTest.java │ │ │ ├── DataKeyListTest.java │ │ │ ├── DataLocalizerTest.java │ │ │ ├── HadoopTestBinding.java │ │ │ ├── HadoopTestUtils.java │ │ │ ├── MR4CArgumentParserTest.java │ │ │ ├── MR4CGenericOptionsParserTest.java │ │ │ ├── MR4CGenericOptionsTest.java │ │ │ ├── MR4CInputFormatTest.java │ │ │ ├── MR4CInputSplitTest.java │ │ │ ├── MR4CMRJobTest.java │ │ │ ├── MR4CMapperTest.java │ │ │ ├── MR4CRecordWriterTest.java │ │ │ ├── MR4CReducerTest.java │ │ │ ├── StaticHadoopTestBinder.java.mrv1 │ │ │ ├── StaticHadoopTestBinder.java.yarn │ │ │ ├── mrv1 │ │ │ │ └── MRv1TestBinding.java │ │ │ └── yarn │ │ │ │ └── YarnTestBinding.java │ │ │ ├── keys │ │ │ ├── BasicDataKeyFilterTest.java │ │ │ ├── BasicElementFilterTest.java │ │ │ ├── CompoundDataKeyTest.java │ │ │ ├── DataKeyComparatorTest.java │ │ │ ├── DataKeyDimensionTest.java │ │ │ ├── DataKeyElementTest.java │ │ │ ├── DataKeyUtilsTest.java │ │ │ ├── DimensionBasedKeyFilterTest.java │ │ │ ├── ElementTransformerTest.java │ │ │ ├── KeyDimensionPartitionerTest.java │ │ │ ├── KeyTransformerTest.java │ │ │ ├── KeyspaceDimensionTest.java │ │ │ ├── KeyspacePartitionerTest.java │ │ │ ├── KeyspaceTest.java │ │ │ └── SimpleDataKeyTest.java │ │ │ ├── mbtiles │ │ │ ├── MBTilesFileTest.java │ │ │ ├── MBTilesTestUtil.java │ │ │ ├── TileFormatTest.java │ │ │ ├── TileKeyTest.java │ │ │ └── TileTest.java │ │ │ ├── message │ │ │ └── MessageTest.java │ │ │ ├── metadata │ │ │ ├── MetadataArrayTest.java │ │ │ ├── MetadataFieldTest.java │ │ │ ├── MetadataKeyExtractorTest.java │ │ │ ├── MetadataKeyTest.java │ │ │ ├── MetadataListTest.java │ │ │ └── MetadataMapTest.java │ │ │ ├── nativec │ │ │ ├── ExternalAlgorithmDataSerializerTest.java │ │ │ ├── ExternalAlgorithmSerializerTest.java │ │ │ ├── ExternalDatasetSerializerTest.java │ │ │ └── jna │ │ │ │ └── JnaExternalEntryTest.java │ │ │ ├── serialize │ │ │ ├── bean │ │ │ │ ├── BeanBasedAlgorithmSerializerTest.java │ │ │ │ ├── BeanBasedDatasetSerializerTest.java │ │ │ │ ├── BeanBasedKeyspaceSerializerTest.java │ │ │ │ ├── algorithm │ │ │ │ │ └── AlgorithmSchemaBeanTest.java │ │ │ │ ├── dataset │ │ │ │ │ ├── DataFileBeanTest.java │ │ │ │ │ └── DatasetBeanTest.java │ │ │ │ ├── keys │ │ │ │ │ ├── DataKeyBeanTest.java │ │ │ │ │ └── DataKeyElementBeanTest.java │ │ │ │ └── metadata │ │ │ │ │ ├── MetadataArrayBeanTest.java │ │ │ │ │ ├── MetadataFieldBeanTest.java │ │ │ │ │ ├── MetadataKeyBeanTest.java │ │ │ │ │ ├── MetadataListBeanTest.java │ │ │ │ │ └── MetadataMapBeanTest.java │ │ │ └── json │ │ │ │ ├── JsonAlgorithmBeanSerializerTest.java │ │ │ │ ├── JsonConfigSerializerTest.java │ │ │ │ ├── JsonDatasetBeanSerializerTest.java │ │ │ │ ├── JsonKeyspaceBeanSerializerTest.java │ │ │ │ └── JsonPropertiesSerializerTest.java │ │ │ ├── sources │ │ │ ├── ArchiveDatasetSourceTest.java │ │ │ ├── ArchiveSourceTester.java │ │ │ ├── BinaryDatasetSourceTest.java │ │ │ ├── CompositeKeyFileMapperTest.java │ │ │ ├── DiskFileSourceTest.java │ │ │ ├── FileSourceTester.java │ │ │ ├── FilesDatasetSourceTest.java │ │ │ ├── HDFSFileSourceTest.java │ │ │ ├── HeterogenousFileSourceTest.java │ │ │ ├── InMemoryArchiveSourceTest.java │ │ │ ├── InMemoryFileSourceTest.java │ │ │ ├── LogsDatasetSourceTest.java │ │ │ ├── MBTilesDatasetSourceTest.java │ │ │ ├── MapFileSourceDFSTest.java │ │ │ ├── MapFileSourceLocalTest.java │ │ │ ├── MetafilesDatasetSourceTest.java │ │ │ ├── PatternKeyFileMapperTest.java │ │ │ ├── RandomAccessFileSinkTest.java │ │ │ ├── RandomAccessFileSourceTest.java │ │ │ ├── SimpleDatasetSourceTest.java │ │ │ ├── SourceTestUtils.java │ │ │ ├── SourceUtilsTest.java │ │ │ ├── StagedDatasetSourceTest.java │ │ │ └── TransformedDatasetSourceTest.java │ │ │ ├── testing │ │ │ └── TestDataManager.java │ │ │ ├── testingv2 │ │ │ └── AlgorithmTestBase.java │ │ │ └── util │ │ │ ├── ByteBufferInputStreamTest.java │ │ │ ├── CollectionUtilsTest.java │ │ │ ├── CombinatoricUtilsTest.java │ │ │ ├── CustomFormatTest.java │ │ │ ├── MR4CLoggingTest.java │ │ │ ├── NamespacedPropertiesTest.java │ │ │ ├── PartitionerTest.java │ │ │ ├── PathUtilsTest.java │ │ │ └── SetAnalysisTest.java │ └── resources │ │ └── empty.jar ├── third_party │ ├── etsy │ │ ├── LICENSE │ │ ├── README.google │ │ └── src │ │ │ └── java │ │ │ └── com │ │ │ └── google │ │ │ └── mr4c │ │ │ └── stats │ │ │ └── StatsdClient.java │ ├── hadoop │ │ ├── LICENSE │ │ ├── README.google │ │ └── test │ │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── hadoop │ │ │ └── net │ │ │ └── StaticMapping.java │ └── stack_overflow │ │ ├── LICENSE │ │ ├── README.google │ │ └── src │ │ └── java │ │ └── com │ │ └── google │ │ └── mr4c │ │ └── util │ │ └── ByteBufferInputStream.java └── tools │ ├── build_mrv1 │ └── build_yarn ├── native ├── LICENSE ├── bin │ ├── post_install │ └── pre_remove ├── build.properties ├── build.xml ├── conf │ ├── log4cxx.properties │ └── mr4c-ld.conf ├── conf_docs │ ├── Doxyfile │ ├── doxygen_footer.html │ └── doxygen_main.tmpl ├── makefile ├── src │ └── cpp │ │ ├── api │ │ ├── algo_dev_api.h │ │ ├── algorithm │ │ │ ├── Algorithm.h │ │ │ ├── AlgorithmAutoRegister.h │ │ │ ├── AlgorithmConfig.h │ │ │ ├── AlgorithmData.h │ │ │ ├── AlgorithmRegistry.h │ │ │ ├── AlgorithmRunner.h │ │ │ └── algorithm_api.h │ │ ├── catalog │ │ │ ├── DimensionCatalog.h │ │ │ ├── ImageTypes.h │ │ │ └── catalog_api.h │ │ ├── context │ │ │ ├── AlgorithmContext.h │ │ │ ├── Logger.h │ │ │ ├── Message.h │ │ │ ├── MessageConsumer.h │ │ │ ├── ProgressReporter.h │ │ │ └── context_api.h │ │ ├── dataset │ │ │ ├── DataFile.h │ │ │ ├── DataFileSink.h │ │ │ ├── DataFileSource.h │ │ │ ├── Dataset.h │ │ │ ├── DatasetContext.h │ │ │ ├── LocalDataFileSink.h │ │ │ ├── LocalDataFileSource.h │ │ │ ├── LocalTempFile.h │ │ │ ├── RandomAccessFile.h │ │ │ ├── SimpleDataFileSource.h │ │ │ ├── WritableRandomAccessFile.h │ │ │ └── dataset_api.h │ │ ├── error │ │ │ ├── Error.h │ │ │ ├── ErrorReporter.h │ │ │ ├── JsonErrorSerializer.h │ │ │ └── error_api.h │ │ ├── external │ │ │ ├── CExternalAlgorithm.h │ │ │ ├── CExternalAlgorithmData.h │ │ │ ├── CExternalContext.h │ │ │ ├── CExternalDataFile.h │ │ │ ├── CExternalDataFileSink.h │ │ │ ├── CExternalDataFileSource.h │ │ │ ├── CExternalDataset.h │ │ │ ├── CExternalEntry.h │ │ │ ├── CExternalRandomAccessFileSink.h │ │ │ ├── CExternalRandomAccessFileSource.h │ │ │ ├── ExternalAlgorithm.h │ │ │ ├── ExternalAlgorithmData.h │ │ │ ├── ExternalAlgorithmDataSerializer.h │ │ │ ├── ExternalAlgorithmSerializer.h │ │ │ ├── ExternalContext.h │ │ │ ├── ExternalDataFile.h │ │ │ ├── ExternalDataFileSink.h │ │ │ ├── ExternalDataFileSource.h │ │ │ ├── ExternalDataset.h │ │ │ ├── ExternalDatasetContext.h │ │ │ ├── ExternalDatasetSerializer.h │ │ │ ├── ExternalEntry.h │ │ │ ├── ExternalRandomAccessFile.h │ │ │ ├── ExternalRandomAccessFileSink.h │ │ │ ├── ExternalRandomAccessFileSource.h │ │ │ ├── capi │ │ │ │ ├── CExternalAlgorithmAPI.h │ │ │ │ ├── CExternalAlgorithmDataAPI.h │ │ │ │ ├── CExternalContextAPI.h │ │ │ │ ├── CExternalDataFileAPI.h │ │ │ │ ├── CExternalDataFileSinkAPI.h │ │ │ │ ├── CExternalDataFileSourceAPI.h │ │ │ │ ├── CExternalDatasetAPI.h │ │ │ │ ├── CExternalEntryAPI.h │ │ │ │ ├── CExternalRandomAccessFileAPI.h │ │ │ │ └── external_capi.h │ │ │ └── external_api.h │ │ ├── keys │ │ │ ├── DataKey.h │ │ │ ├── DataKeyBuilder.h │ │ │ ├── DataKeyDimension.h │ │ │ ├── DataKeyElement.h │ │ │ ├── Keyspace.h │ │ │ ├── KeyspaceBuilder.h │ │ │ ├── KeyspaceDimension.h │ │ │ └── keys_api.h │ │ ├── metadata │ │ │ ├── MetadataArray.h │ │ │ ├── MetadataElement.h │ │ │ ├── MetadataField.h │ │ │ ├── MetadataKey.h │ │ │ ├── MetadataList.h │ │ │ ├── MetadataMap.h │ │ │ ├── Primitive.h │ │ │ └── metadata_api.h │ │ ├── mr4c_api.h │ │ ├── serialize │ │ │ ├── AlgorithmConfigSerializer.h │ │ │ ├── AlgorithmSerializer.h │ │ │ ├── DatasetSerializer.h │ │ │ ├── KeyspaceSerializer.h │ │ │ ├── PropertiesSerializer.h │ │ │ ├── Serializer.h │ │ │ ├── SerializerFactory.h │ │ │ ├── SerializerRegistry.h │ │ │ ├── json │ │ │ │ ├── JanssonUtil.h │ │ │ │ ├── JsonAlgorithmConfigSerializer.h │ │ │ │ ├── JsonAlgorithmSerializer.h │ │ │ │ ├── JsonCommonSerializer.h │ │ │ │ ├── JsonDatasetSerializer.h │ │ │ │ ├── JsonKeyspaceSerializer.h │ │ │ │ ├── JsonPropertiesSerializer.h │ │ │ │ ├── JsonSerializerFactory.h │ │ │ │ └── json_api.h │ │ │ └── serialize_api.h │ │ └── util │ │ │ ├── ArrayUtil.h │ │ │ ├── Helpers.h │ │ │ ├── IOUtil.h │ │ │ ├── MR4CEnvironment.h │ │ │ ├── MR4CLogging.h │ │ │ ├── MR4CTempFiles.h │ │ │ ├── Properties.h │ │ │ ├── StackUtil.h │ │ │ └── util_api.h │ │ └── impl │ │ ├── algorithm │ │ ├── Algorithm.cpp │ │ ├── AlgorithmAutoRegister.cpp │ │ ├── AlgorithmConfig.cpp │ │ ├── AlgorithmData.cpp │ │ ├── AlgorithmRegistry.cpp │ │ └── AlgorithmRunner.cpp │ │ ├── catalog │ │ ├── DimensionCatalog.cpp │ │ └── ImageTypes.cpp │ │ ├── context │ │ ├── AlgorithmContext.cpp │ │ ├── Logger.cpp │ │ └── Message.cpp │ │ ├── dataset │ │ ├── DataFile.cpp │ │ ├── DataFileSource.cpp │ │ ├── Dataset.cpp │ │ ├── DatasetContext.cpp │ │ ├── LocalDataFileSink.cpp │ │ ├── LocalDataFileSource.cpp │ │ ├── LocalTempFile.cpp │ │ └── SimpleDataFileSource.cpp │ │ ├── error │ │ ├── Error.cpp │ │ ├── ErrorReporter.cpp │ │ └── JsonErrorSerializer.cpp │ │ ├── external │ │ ├── CExternalAlgorithm.cpp │ │ ├── CExternalAlgorithmData.cpp │ │ ├── CExternalContext.cpp │ │ ├── CExternalDataFile.cpp │ │ ├── CExternalDataFileSink.cpp │ │ ├── CExternalDataFileSource.cpp │ │ ├── CExternalDataset.cpp │ │ ├── CExternalEntry.cpp │ │ ├── CExternalRandomAccessFileSink.cpp │ │ ├── CExternalRandomAccessFileSource.cpp │ │ ├── ExternalAlgorithm.cpp │ │ ├── ExternalAlgorithmData.cpp │ │ ├── ExternalAlgorithmDataSerializer.cpp │ │ ├── ExternalAlgorithmSerializer.cpp │ │ ├── ExternalContext.cpp │ │ ├── ExternalDataFile.cpp │ │ ├── ExternalDataFileSink.cpp │ │ ├── ExternalDataFileSource.cpp │ │ ├── ExternalDataset.cpp │ │ ├── ExternalDatasetContext.cpp │ │ ├── ExternalDatasetSerializer.cpp │ │ ├── ExternalEntry.cpp │ │ ├── ExternalRandomAccessFile.cpp │ │ ├── ExternalRandomAccessFileSink.cpp │ │ └── ExternalRandomAccessFileSource.cpp │ │ ├── keys │ │ ├── DataKey.cpp │ │ ├── DataKeyBuilder.cpp │ │ ├── DataKeyDimension.cpp │ │ ├── DataKeyElement.cpp │ │ ├── Keyspace.cpp │ │ ├── KeyspaceBuilder.cpp │ │ └── KeyspaceDimension.cpp │ │ ├── metadata │ │ ├── MetadataArray.cpp │ │ ├── MetadataElement.cpp │ │ ├── MetadataField.cpp │ │ ├── MetadataKey.cpp │ │ ├── MetadataList.cpp │ │ ├── MetadataMap.cpp │ │ └── Primitive.cpp │ │ ├── serialize │ │ ├── SerializerRegistry.cpp │ │ └── json │ │ │ ├── JanssonUtil.cpp │ │ │ ├── JsonAlgorithmConfigSerializer.cpp │ │ │ ├── JsonAlgorithmSerializer.cpp │ │ │ ├── JsonCommonSerializer.cpp │ │ │ ├── JsonDatasetSerializer.cpp │ │ │ ├── JsonKeyspaceSerializer.cpp │ │ │ ├── JsonPropertiesSerializer.cpp │ │ │ └── JsonSerializerFactory.cpp │ │ └── util │ │ ├── IOUtil.cpp │ │ ├── MR4CEnvironment.cpp │ │ ├── MR4CLogging.cpp │ │ ├── MR4CTempFiles.cpp │ │ ├── Properties.cpp │ │ └── StackUtil.cpp ├── test │ └── cpp │ │ ├── api │ │ ├── MR4CTests.h │ │ ├── algorithm │ │ │ └── AlgorithmDataTestUtil.h │ │ ├── dataset │ │ │ └── DatasetTestUtil.h │ │ ├── keys │ │ │ └── KeyspaceTestUtil.h │ │ └── multithread │ │ │ └── SimultaneousThreadRunner.h │ │ ├── suites │ │ ├── Cover.cpp │ │ ├── Local.cpp │ │ └── Suite.cpp │ │ └── tests │ │ ├── MR4CTests.cpp │ │ ├── algorithm │ │ ├── AlgorithmDataTestUtil.cpp │ │ ├── AlgorithmTests.cpp │ │ ├── TestAlgorithmConfig.cpp │ │ └── TestAlgorithmData.cpp │ │ ├── context │ │ ├── ContextTests.cpp │ │ └── TestMessage.cpp │ │ ├── dataset │ │ ├── DatasetTestUtil.cpp │ │ ├── DatasetTests.cpp │ │ ├── TestDataFile.cpp │ │ ├── TestDataset.cpp │ │ ├── TestLocalDataFileSink.cpp │ │ ├── TestLocalDataFileSource.cpp │ │ ├── TestLocalTempFile.cpp │ │ └── TestSimpleDataFileSource.cpp │ │ ├── error │ │ ├── ErrorTests.cpp │ │ ├── TestError.cpp │ │ └── TestJsonErrorSerializer.cpp │ │ ├── external │ │ ├── ExternalTests.cpp │ │ ├── TestExternalAlgorithmDataSerializer.cpp │ │ ├── TestExternalDatasetSerializer.cpp │ │ └── TestExternalEntry.cpp │ │ ├── keys │ │ ├── KeysTests.cpp │ │ ├── KeyspaceTestUtil.cpp │ │ ├── TestDataKey.cpp │ │ ├── TestDataKeyBuilder.cpp │ │ ├── TestDataKeyDimension.cpp │ │ ├── TestDataKeyElement.cpp │ │ ├── TestKeyspace.cpp │ │ ├── TestKeyspaceBuilder.cpp │ │ └── TestKeyspaceDimension.cpp │ │ ├── metadata │ │ ├── MetadataTests.cpp │ │ ├── TestMetadataArray.cpp │ │ ├── TestMetadataField.cpp │ │ ├── TestMetadataKey.cpp │ │ ├── TestMetadataList.cpp │ │ ├── TestMetadataMap.cpp │ │ └── TestPrimitive.cpp │ │ ├── multithread │ │ ├── MultithreadTests.cpp │ │ ├── SimultaneousThreadRunner.cpp │ │ ├── TestDataFileMultithread.cpp │ │ ├── TestDatasetMultithread.cpp │ │ ├── TestExternalRandomAccessFileMultithread.cpp │ │ └── TestExternalRandomAccessFileSinkMultithread.cpp │ │ ├── serialize │ │ └── json │ │ │ ├── JsonTests.cpp │ │ │ ├── TestJsonAlgorithmConfigSerializer.cpp │ │ │ ├── TestJsonAlgorithmSerializer.cpp │ │ │ ├── TestJsonDatasetSerializer.cpp │ │ │ ├── TestJsonKeyspaceSerializer.cpp │ │ │ └── TestJsonPropertiesSerializer.cpp │ │ └── util │ │ ├── TestArrayUtil.cpp │ │ ├── TestIOUtil.cpp │ │ ├── TestMR4CEnvironment.cpp │ │ ├── TestMR4CLogging.cpp │ │ ├── TestMR4CTempFiles.cpp │ │ ├── TestProperties.cpp │ │ └── UtilTests.cpp └── tools │ ├── run_coverage │ ├── run_tests │ └── test_local ├── remove_all ├── test ├── README.md ├── bin │ ├── check_mr4cref_output │ ├── configure_mr4cref_input │ ├── run_mr4cref │ ├── test_hadoop │ └── viewlog ├── conf │ ├── algo_config.json │ ├── reduce_algo_config.json │ └── site.json ├── input │ ├── 4.j2k │ ├── 8-0-4-50-6-75.j2k │ ├── 912-white-720-others.j2k │ └── back-75-14-white.j2k ├── makefile ├── openjpeg_build.sh ├── ref │ └── avg_pixels.csv ├── src │ └── cpp │ │ ├── J2kDecoder.cpp │ │ ├── J2kDecoder.h │ │ └── hello_histo.cpp ├── test │ └── cpp │ │ ├── suites │ │ ├── Local.cpp │ │ └── Suite.cpp │ │ └── tests │ │ ├── TestHisto.cpp │ │ ├── algo.json │ │ ├── algo_config.json │ │ ├── input │ │ ├── 4.j2k │ │ ├── 8-0-4-50-6-75.j2k │ │ ├── 912-white-720-others.j2k │ │ └── back-75-14-white.j2k │ │ ├── int_conf.json │ │ ├── reduce_algo.json │ │ └── reduce_algo_config.json └── test_mr4c.sh └── tutorial ├── README.md ├── UserGuide └── README.md ├── clean_all ├── example1_HelloWorld ├── README.md ├── helloworld.json ├── helloworld.sh ├── makefile └── src │ └── helloworld.cpp ├── example2_IO ├── README.md ├── changeimage.json ├── changeimage.sh ├── input │ └── test.bin ├── makefile ├── output │ └── testOut.bin └── src │ └── changeimage.cpp ├── example3_Dimensions ├── README.md ├── dimensions.json ├── dimensions.sh ├── input │ ├── test_1.bin │ └── test_2.bin ├── makefile └── src │ └── dimensions.cpp ├── example4_ExternalLib ├── README.md ├── imageinfo.json ├── imageinfo.sh ├── input │ └── sample.tif ├── makefile └── src │ └── imageinfo.cpp ├── example5_json ├── README.md ├── input │ └── s01_20140120T161125Z_metadata.txt ├── json.json ├── json.sh ├── makefile └── src │ └── json.cpp ├── example6_ImageAlgo ├── README.md ├── content.json ├── content.sh ├── input │ ├── s00001_20140209_145912_mul_d2_0005.tif │ ├── s00001_20140213_071323_mul_d2_0005.tif │ └── s01_20140214T071847Z_ms_d2_0005.tif ├── makefile ├── output │ ├── s00001_20140209_145912_mul_d2_0005_class.tif │ ├── s00001_20140213_071323_mul_d2_0005_class.tif │ └── s01_20140214T071847Z_ms_d2_0005_class.tif └── src │ └── content.cpp ├── example7_yarn ├── README.md ├── input │ ├── s01_20140409T084855Z_ms_d1_0001.tif │ ├── s01_20140409T084855Z_ms_d1_0002.tif │ ├── s01_20140409T084855Z_ms_d1_0003.tif │ ├── s01_20140409T084855Z_ms_d1_0004.tif │ └── s01_20140409T084855Z_ms_d1_0005.tif ├── makefile ├── map.json ├── mapReduce.sh ├── reduce.json ├── site.json └── src │ ├── map.cpp │ └── reduce.cpp ├── example8_mbtiles ├── README.md ├── conf │ ├── algo.properties │ ├── exe.json │ └── runtime.properties ├── input.tif └── run └── example9_RandomAccess ├── README.md ├── RandomAccess.json ├── RandomAccess.sh ├── input ├── s01_20140409T084855Z_ms_d1_0001.tif ├── s01_20140409T084855Z_ms_d1_0002.tif ├── s01_20140409T084855Z_ms_d1_0003.tif ├── s01_20140409T084855Z_ms_d1_0004.tif └── s01_20140409T084855Z_ms_d1_0005.tif ├── makefile ├── site.json └── src └── RandomAccess.cpp /build_all: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | cd native 6 | make 7 | cd .. 8 | 9 | cd geospatial 10 | make 11 | cd .. 12 | 13 | cd java 14 | tools/build_yarn 15 | cd .. 16 | 17 | -------------------------------------------------------------------------------- /clean_all: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | cd native 6 | make clean 7 | cd .. 8 | 9 | cd geospatial 10 | make clean 11 | cd .. 12 | 13 | cd java 14 | ant clean 15 | cd .. 16 | 17 | -------------------------------------------------------------------------------- /deploy_all: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Deploys MR4C to /usr/local/mr4c 4 | 5 | set -e 6 | 7 | export MR4C_HOME=/usr/local/mr4c 8 | 9 | cd native 10 | sudo make deploy 11 | cd .. 12 | sudo /usr/local/mr4c/native/bin/post_install 13 | 14 | cd geospatial 15 | sudo make deploy 16 | cd .. 17 | sudo /usr/local/mr4c/geospatial/bin/post_install 18 | 19 | cd java 20 | sudo ant deploy 21 | cd .. 22 | sudo /usr/local/mr4c/java/bin/post_install 23 | 24 | 25 | -------------------------------------------------------------------------------- /geospatial/bin/post_install: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | THIS=`readlink -f $0` 4 | BIN=`dirname $THIS` 5 | MR4C_GEO=`dirname $BIN` 6 | 7 | ln -s -f $MR4C_GEO/dist/libmr4cgeo.so /usr/local/lib/libmr4cgeo.so 8 | ln -s -f $MR4C_GEO/include /usr/include/mr4c_geospatial 9 | ln -s -f $MR4C_GEO/conf/mr4c-ld.conf /etc/ld.so.conf.d/mr4c-geo-ld.conf 10 | 11 | /sbin/ldconfig 12 | -------------------------------------------------------------------------------- /geospatial/bin/pre_remove: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | rm -f /usr/local/lib/libmr4cgeo.so 4 | rm -f /usr/include/mr4c_geospatial 5 | rm -f /etc/ld.so.conf.d/mr4c-geo-ld.conf 6 | -------------------------------------------------------------------------------- /geospatial/build.properties: -------------------------------------------------------------------------------- 1 | tools.dir=tools 2 | 3 | # override in build.local.properties 4 | mr4c.native.root=${basedir}/../native 5 | 6 | mr4c.native.lib.dir=${mr4c.native.root}/dist 7 | 8 | 9 | -------------------------------------------------------------------------------- /geospatial/conf/mr4c-ld.conf: -------------------------------------------------------------------------------- 1 | /usr/local/mr4c/geospatial/dist 2 | -------------------------------------------------------------------------------- /geospatial/conf_docs/doxygen_footer.html: -------------------------------------------------------------------------------- 1 | 2 |
3 | $datetime   4 | doxygen 5 | $doxygenversion 6 |
7 | 8 |

Copyright 2014 Google Inc. All rights reserved.

9 | 10 | 11 | -------------------------------------------------------------------------------- /geospatial/conf_docs/doxygen_main.tmpl: -------------------------------------------------------------------------------- 1 | /** \mainpage MR4C Geospatial Library Documentation 2 | * 3 | * 4 | * \section intro_sec Introduction 5 | * 6 | */ 7 | -------------------------------------------------------------------------------- /geospatial/input/thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/geospatial/input/thumb.png -------------------------------------------------------------------------------- /geospatial/src/cpp/api/coord/coord_api.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_GEO_COORD_API_H__ 18 | #define __MR4C_GEO_COORD_API_H__ 19 | 20 | #include "BoundingBox.h" 21 | #include "EastNorthCoord.h" 22 | #include "EastNorthTrans.h" 23 | #include "ImageBox.h" 24 | #include "LatLonCoord.h" 25 | #include "NormMercCoord.h" 26 | #include "SimpleEastNorthTrans.h" 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /geospatial/src/cpp/api/gdal/gdal_api.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_GEO_GDAL_API_H__ 18 | #define __MR4C_GEO_GDAL_API_H__ 19 | 20 | #include "GDALCoordTrans.h" 21 | #include "GDALFile.h" 22 | #include "GDALFileBase.h" 23 | #include "GDALLocalFile.h" 24 | #include "GDALMemoryFile.h" 25 | #include "GDALUtils.h" 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /geospatial/src/cpp/api/mbtiles/mbtiles_api.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_GEO_MBTILES_API_H__ 18 | #define __MR4C_GEO_MBTILES_API_H__ 19 | 20 | #include "MBTilesDataset.h" 21 | #include "MBTilesUtils.h" 22 | #include "TileExtractor.h" 23 | #include "TileKey.h" 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /geospatial/src/cpp/api/mr4c_geo_api.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_GEO_API_H__ 18 | #define __MR4C_GEO_API_H__ 19 | 20 | #include "coord/coord_api.h" 21 | #include "gdal/gdal_api.h" 22 | #include "mbtiles/mbtiles_api.h" 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /geospatial/test/cpp/api/MR4CGeoTests.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_GEO_TESTS_H__ 18 | #define __MR4C_GEO_TESTS_H__ 19 | 20 | #include 21 | 22 | namespace MR4C { 23 | 24 | std::string extractTestName(int argc, char* argv[]); 25 | } 26 | 27 | #endif 28 | 29 | -------------------------------------------------------------------------------- /geospatial/test/cpp/suites/Cover.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | #include "MR4CGeoTests.h" 20 | 21 | // local test runner that just dumps to the console 22 | int main( int argc, char* argv[]) 23 | { 24 | std::string name = MR4C::extractTestName(argc, argv); 25 | CPPUNIT_NS::TextTestRunner testrunner; 26 | testrunner.addTest( CPPUNIT_NS::TestFactoryRegistry::getRegistry(name).makeTest()); 27 | testrunner.run(); 28 | } 29 | 30 | 31 | -------------------------------------------------------------------------------- /geospatial/test/cpp/suites/Local.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | #include "MR4CGeoTests.h" 20 | 21 | // local test runner that just dumps to the console 22 | int main( int argc, char* argv[]) 23 | { 24 | std::string name = MR4C::extractTestName(argc, argv); 25 | CPPUNIT_NS::TextTestRunner testrunner; 26 | testrunner.addTest( CPPUNIT_NS::TestFactoryRegistry::getRegistry(name).makeTest()); 27 | testrunner.run(); 28 | 29 | } 30 | 31 | 32 | -------------------------------------------------------------------------------- /geospatial/test/cpp/tests/MR4CGeoTests.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | 20 | CPPUNIT_REGISTRY_ADD("CoordTests", "MR4CGeoTests"); 21 | CPPUNIT_REGISTRY_ADD("GDALTests", "MR4CGeoTests"); 22 | CPPUNIT_REGISTRY_ADD("MBTilesTests", "MR4CGeoTests"); 23 | 24 | namespace MR4C { 25 | 26 | std::string extractTestName(int argc, char* argv[]) { 27 | if ( argc==1 ) { 28 | return "MR4CGeoTests"; 29 | } else { 30 | return argv[1]; 31 | } 32 | } 33 | } 34 | 35 | 36 | -------------------------------------------------------------------------------- /geospatial/test/cpp/tests/coord/CoordTests.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | 20 | CPPUNIT_REGISTRY_ADD("TestBoundingBox", "CoordTests"); 21 | CPPUNIT_REGISTRY_ADD("TestEastNorthCoord", "CoordTests"); 22 | CPPUNIT_REGISTRY_ADD("TestImageBox", "CoordTests"); 23 | CPPUNIT_REGISTRY_ADD("TestLatLonCoord", "CoordTests"); 24 | CPPUNIT_REGISTRY_ADD("TestNormMercCoord", "CoordTests"); 25 | 26 | -------------------------------------------------------------------------------- /geospatial/test/cpp/tests/gdal/GDALTests.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | 20 | CPPUNIT_REGISTRY_ADD("TestGDALCoordTrans", "GDALTests"); 21 | CPPUNIT_REGISTRY_ADD("TestGDALLocalFile", "GDALTests"); 22 | CPPUNIT_REGISTRY_ADD("TestGDALMemoryFile", "GDALTests"); 23 | CPPUNIT_REGISTRY_ADD("TestGDALUtils", "GDALTests"); 24 | 25 | -------------------------------------------------------------------------------- /geospatial/test/cpp/tests/mbtiles/MBTilesTests.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | 20 | CPPUNIT_REGISTRY_ADD("TestTileKey", "MBTilesTests"); 21 | CPPUNIT_REGISTRY_ADD("TestMBTilesDataset", "MBTilesTests"); 22 | CPPUNIT_REGISTRY_ADD("TestMBTilesUtils", "MBTilesTests"); 23 | CPPUNIT_REGISTRY_ADD("TestTileExtractor", "MBTilesTests"); 24 | 25 | -------------------------------------------------------------------------------- /geospatial/tools/prep_env: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export LD_LIBRARY_PATH=${DIR}/../../native/dist:$LD_LIBRARY_PATH 4 | if [ -f "env.local" ] 5 | then 6 | source env.local 7 | fi 8 | 9 | [ -z "$MR4C_LOG4CXX_CONFIG" ] && MR4C_LOG4CXX_CONFIG=${DIR}/../conf/log4cxx.properties 10 | export MR4C_LOG4CXX_CONFIG 11 | -------------------------------------------------------------------------------- /geospatial/tools/run_coverage: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | DIR=`dirname $0` 6 | 7 | source ${DIR}/prep_env 8 | 9 | $DIR/../exe/run_coverage $* 10 | -------------------------------------------------------------------------------- /geospatial/tools/run_tests: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | DIR=`dirname $0` 6 | 7 | source ${DIR}/prep_env 8 | 9 | echo $LD_LIBRARY_PATH 10 | 11 | $DIR/../exe/run_tests $* 12 | -------------------------------------------------------------------------------- /geospatial/tools/test_local: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | DIR=`dirname $0` 6 | 7 | source ${DIR}/prep_env 8 | 9 | $DIR/../exe/test_local $* 10 | -------------------------------------------------------------------------------- /java/bin/mr4c: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # args: 4 | # $1 = URI to execution config file 5 | 6 | 7 | THIS=`readlink -f $0` 8 | BIN=`dirname $THIS` 9 | MR4C_JAVA=`dirname $BIN` 10 | 11 | MR4C_JAR_WITH_LIBS=$MR4C_JAVA/dist/mr4c-with-libs.jar 12 | MR4C_CLASSPATH=$MR4C_JAR_WITH_LIBS:$MR4C_JAVA/lib/'*':$MR4C_ALGORITHM_CLASSPATH 13 | MR4C_LIBPATH=$LD_LIBRARY_PATH:.:/usr/local/lib 14 | 15 | if [ $# -lt 1 ] 16 | then 17 | echo "Usage: mr4c execution_config_uri [name=val ...]" 18 | exit -1 19 | fi 20 | 21 | java -cp "$MR4C_CLASSPATH" -Djna.library.path=$MR4C_LIBPATH -Dmr4c.algorithm.classpath=$MR4C_ALGORITHM_CLASSPATH -Dmr4c.log4j=$MR4C_LOG4J_CONFIG -Dmr4c.site=$MR4C_SITE $MR4C_JAVA_OPTS com.google.mr4c.AlgoRunner $* 22 | -------------------------------------------------------------------------------- /java/bin/mr4c_debug: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # args: 4 | # $1 = URI to execution config file 5 | 6 | THIS=`readlink -f $0` 7 | BIN=`dirname $THIS` 8 | MR4C_JAVA=`dirname $BIN` 9 | 10 | MR4C_JAR_WITH_LIBS=$MR4C_JAVA/dist/mr4c-with-libs.jar 11 | MR4C_CLASSPATH=$MR4C_JAR_WITH_LIBS:$MR4C_JAVA/lib/'*':$MR4C_ALGORITHM_CLASSPATH 12 | MR4C_LIBPATH=$LD_LIBRARY_PATH:.:/usr/local/lib 13 | 14 | if [ $# -lt 1 ] 15 | then 16 | echo "Usage: mr4c execution_config_uri [name=val ...]" 17 | exit -1 18 | fi 19 | 20 | gdb --args java -cp "$MR4C_CLASSPATH" -Djna.library.path=$MR4C_LIBPATH -Dmr4c.log4j=$MR4C_LOG4J_CONFIG -Dmr4c.site=$MR4C_SITE $MR4C_JAVA_OPTS com.google.mr4c.AlgoRunner $* 21 | -------------------------------------------------------------------------------- /java/bin/mr4c_diff: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # args: 4 | # $1 = URI to diff config file 5 | 6 | THIS=`readlink -f $0` 7 | BIN=`dirname $THIS` 8 | MR4C_JAVA=`dirname $BIN` 9 | 10 | MR4C_JAR_WITH_LIBS=$MR4C_JAVA/dist/mr4c-with-libs.jar 11 | MR4C_CLASSPATH=$MR4C_JAR_WITH_LIBS:$MR4C_JAVA/lib/'*' 12 | MR4C_LIBPATH=$LD_LIBRARY_PATH:.:/usr/local/lib 13 | 14 | if [ $# -lt 1 ] 15 | then 16 | echo "Usage: mr4c_diff diff_config_uri [name=val ...]" 17 | exit -1 18 | fi 19 | 20 | java -cp "$MR4C_CLASSPATH" -Dmr4c.log4j=$MR4C_LOG4J_CONFIG $MR4C_JAVA_OPTS com.google.mr4c.DiffRunner $* 21 | -------------------------------------------------------------------------------- /java/bin/mr4c_hadoop: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # args: 4 | # $1 = URI to execution config file 5 | # $2 = number of map tasks 6 | 7 | if [ $# -lt 1 ] 8 | then 9 | echo "Usage: mr4c_hadoop execution_config_uri num_tasks [name=val ...]" 10 | exit -1 11 | fi 12 | 13 | THIS=`readlink -f $0` 14 | BIN=`dirname $THIS` 15 | MR4C_JAVA=`dirname $BIN` 16 | 17 | MR4C_JAR_WITH_LIBS=$MR4C_JAVA/dist/mr4c-with-libs.jar 18 | MR4C_CONF=$MR4C_JAVA/conf 19 | MR4C_CLASSPATH=$MR4C_CONF:$MR4C_JAR_WITH_LIBS:$MR4C_JAVA/lib/'*':$MR4C_ALGORITHM_CLASSPATH 20 | MR4C_LIBPATH=$LD_LIBRARY_PATH:.:/usr/local/lib 21 | 22 | java -cp "$MR4C_CLASSPATH" -Djna.library.path=$MR4C_LIBPATH -Dmr4c.hadoop.algorithm.classpath=$MR4C_ALGORITHM_CLASSPATH -Dmr4c.log4j=$MR4C_LOG4J_CONFIG -Dmr4c.site=$MR4C_SITE $MR4C_JAVA_OPTS com.google.mr4c.hadoop.LocalHadoopAlgoRunner $MR4C_JAR_WITH_LIBS $* 23 | 24 | -------------------------------------------------------------------------------- /java/bin/mr4c_hadoop_remote: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # args: 4 | # $1 = URI to execution config file 5 | # $2 = name of cluster 6 | # $3 = number of map tasks 7 | 8 | if [ $# -lt 1 ] 9 | then 10 | echo "Usage: mr4c_hadoop_remote execution_config_uri [cluster_name] [num_tasks] [name=val ...]" 11 | exit -1 12 | fi 13 | 14 | THIS=`readlink -f $0` 15 | BIN=`dirname $THIS` 16 | MR4C_JAVA=`dirname $BIN` 17 | 18 | MR4C_JAR_WITH_LIBS=$MR4C_JAVA/dist/mr4c-with-libs.jar 19 | MR4C_CONF=$MR4C_JAVA/conf 20 | MR4C_CLASSPATH=$MR4C_CONF:$MR4C_JAR_WITH_LIBS:$MR4C_JAVA/lib/'*':$MR4C_ALGORITHM_CLASSPATH 21 | MR4C_LIBPATH=$LD_LIBRARY_PATH:.:/usr/local/lib 22 | 23 | java -cp "$MR4C_CLASSPATH" -Djna.library.path=$MR4C_LIBPATH -Dmr4c.hadoop.algorithm.classpath=$MR4C_ALGORITHM_CLASSPATH -Dmr4c.log4j=$MR4C_LOG4J_CONFIG -Dmr4c.site=$MR4C_SITE $MR4C_JAVA_OPTS com.google.mr4c.hadoop.RemoteAlgoRunner $MR4C_JAR_WITH_LIBS $* 24 | 25 | -------------------------------------------------------------------------------- /java/bin/mr4c_test: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # args: 4 | # $1 = URI to test config file 5 | 6 | THIS=`readlink -f $0` 7 | BIN=`dirname $THIS` 8 | MR4C_JAVA=`dirname $BIN` 9 | 10 | MR4C_JAR_WITH_LIBS=$MR4C_JAVA/dist/mr4c-with-libs.jar 11 | MR4C_CLASSPATH=$MR4C_JAR_WITH_LIBS:$MR4C_JAVA/lib/'*':$MR4C_ALGORITHM_CLASSPATH 12 | MR4C_LIBPATH=$LD_LIBRARY_PATH:.:/usr/local/lib 13 | 14 | if [ $# -lt 1 ] 15 | then 16 | echo "Usage: mr4c_test test_config_uri [name=val ...]" 17 | exit -1 18 | fi 19 | 20 | java -cp "$MR4C_CLASSPATH" -Dmr4c.log4j=$MR4C_LOG4J_CONFIG $MR4C_JAVA_OPTS com.google.mr4c.AlgoTestRunner $* 21 | -------------------------------------------------------------------------------- /java/bin/post_install: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | THIS=`readlink -f $0` 4 | BIN=`dirname $THIS` 5 | MR4C_JAVA=`dirname $BIN` 6 | 7 | ln -s -f $MR4C_JAVA/bin/mr4c /usr/local/bin/mr4c 8 | ln -s -f $MR4C_JAVA/bin/mr4c_debug /usr/local/bin/mr4c_debug 9 | ln -s -f $MR4C_JAVA/bin/mr4c_hadoop /usr/local/bin/mr4c_hadoop 10 | ln -s -f $MR4C_JAVA/bin/mr4c_hadoop_remote /usr/local/bin/mr4c_hadoop_remote 11 | ln -s -f $MR4C_JAVA/bin/mr4c_diff /usr/local/bin/mr4c_diff 12 | ln -s -f $MR4C_JAVA/bin/mr4c_test /usr/local/bin/mr4c_test 13 | 14 | mkdir -p /etc/mr4c 15 | ln -s -f $MR4C_JAVA/conf/log4j.properties /etc/mr4c/log4j.properties 16 | ln -s -f $MR4C_JAVA/conf/site.json /etc/mr4c/site.json 17 | ln -s -f $MR4C_JAVA/conf/mr4c-hadoop-site.xml /etc/mr4c/mr4c-hadoop-site.xml 18 | -------------------------------------------------------------------------------- /java/bin/pre_remove: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -f /usr/local/bin/mr4c 3 | rm -f /usr/local/bin/mr4c_debug 4 | rm -f /usr/local/bin/mr4c_hadoop 5 | rm -f /usr/local/bin/mr4c_hadoop_remote 6 | rm -f /usr/local/bin/mr4c_diff 7 | rm -f /usr/local/bin/mr4c_test 8 | rm -f /etc/mr4c/log4j.properties 9 | rm -f /etc/mr4c/site.json 10 | rm -f /etc/mr4c/mr4c-hadoop-site.xml 11 | -------------------------------------------------------------------------------- /java/conf/mr4c-hadoop-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /java/conf/site.json: -------------------------------------------------------------------------------- 1 | { 2 | "clusters" : {} 3 | } 4 | -------------------------------------------------------------------------------- /java/conf_ivy/ivysettings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /java/input/conftest/file1.properties: -------------------------------------------------------------------------------- 1 | name2=somethingelse 2 | name3=val3 3 | -------------------------------------------------------------------------------- /java/input/conftest/file2.properties: -------------------------------------------------------------------------------- 1 | name3=blahblah 2 | name4=val4 3 | -------------------------------------------------------------------------------- /java/input/conftest/file3.properties: -------------------------------------------------------------------------------- 1 | name4=oops 2 | name5=val5 3 | -------------------------------------------------------------------------------- /java/input/conftest/file4.properties: -------------------------------------------------------------------------------- 1 | name5=yoyoyoyo 2 | name6=val6 3 | -------------------------------------------------------------------------------- /java/input/conftest/file5.properties: -------------------------------------------------------------------------------- 1 | name6=totally wrong 2 | name7=val7 3 | -------------------------------------------------------------------------------- /java/input/conftest/file6.properties: -------------------------------------------------------------------------------- 1 | name7=notit 2 | name8=val8 3 | -------------------------------------------------------------------------------- /java/input/data/dataset/logsbuilder/log1.log: -------------------------------------------------------------------------------- 1 | Some log content 2 | -------------------------------------------------------------------------------- /java/input/data/dataset/logsbuilder/log2.log: -------------------------------------------------------------------------------- 1 | Some more log content 2 | -------------------------------------------------------------------------------- /java/input/data/dataset/logsbuilder/somepath/log3.log: -------------------------------------------------------------------------------- 1 | And even more log content 2 | -------------------------------------------------------------------------------- /java/input/data/dataset/logsrc/notask/logs/log1.log: -------------------------------------------------------------------------------- 1 | Some log content 2 | -------------------------------------------------------------------------------- /java/input/data/dataset/logsrc/notask/logs/log2.log: -------------------------------------------------------------------------------- 1 | Some more log content 2 | -------------------------------------------------------------------------------- /java/input/data/dataset/logsrc/notask/logs/log3.log: -------------------------------------------------------------------------------- 1 | And even more log content 2 | -------------------------------------------------------------------------------- /java/input/data/dataset/logsrc/withtask/logs/taskid1/log1.log: -------------------------------------------------------------------------------- 1 | Some log content 2 | -------------------------------------------------------------------------------- /java/input/data/dataset/logsrc/withtask/logs/taskid1/log2.log: -------------------------------------------------------------------------------- 1 | Some more log content 2 | -------------------------------------------------------------------------------- /java/input/data/dataset/logsrc/withtask/logs/taskid1/log3.log: -------------------------------------------------------------------------------- 1 | And even more log content 2 | -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556848_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556848_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556848_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556848_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556871_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556871_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556871_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556871_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556894_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556894_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556894_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556894_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556917_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556917_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556917_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556917_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556941_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556941_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556941_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556941_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556964_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556964_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556964_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556964_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556987_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556987_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556987_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21556987_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557010_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557010_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557010_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557010_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557033_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557033_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557033_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557033_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557056_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557056_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557056_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557056_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557079_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557079_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557079_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557079_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557103_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557103_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557103_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557103_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557126_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557126_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557126_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557126_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557149_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557149_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557149_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557149_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557172_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557172_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557172_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557172_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557195_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557195_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557195_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557195_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557218_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557218_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557218_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557218_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557242_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557242_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557242_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557242_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557265_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557265_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557265_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557265_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557288_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557288_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557288_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data/ss01_c1_2455874.21557288_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/directory.json: -------------------------------------------------------------------------------- 1 | { 2 | "mapper" : { 3 | "pattern" : "ss01_c${sensor}_${frame}_${type}_1.5bps.jpc", 4 | "dimensions" : [ "frame" , "sensor", "type" ] 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556848_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556848_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556848_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556848_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556871_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556871_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556871_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556871_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556894_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556894_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556894_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556894_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556917_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556917_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556917_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556917_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556941_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556941_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556941_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556941_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556964_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556964_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556964_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556964_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556987_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556987_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556987_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21556987_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557010_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557010_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557010_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557010_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557033_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557033_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557033_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557033_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557056_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557056_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557056_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557056_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557079_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557079_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557079_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557079_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557103_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557103_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557103_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557103_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557126_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557126_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557126_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557126_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557149_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557149_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557149_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557149_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557172_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557172_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557172_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557172_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557195_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557195_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557195_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557195_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557218_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557218_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557218_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557218_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557242_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557242_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557242_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557242_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557265_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557265_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557265_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557265_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557288_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557288_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557288_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/dataset/test1/input_data_self/ss01_c1_2455874.21557288_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/dataset/test1/source.json: -------------------------------------------------------------------------------- 1 | { 2 | "mapper" : { 3 | "pattern" : "ss01_c${sensor}_${frame}_${type}_1.5bps.jpc", 4 | "dimensions" : [ "frame" , "sensor", "type" ] 5 | }, 6 | "selfConfig" : false 7 | } 8 | -------------------------------------------------------------------------------- /java/input/data/dataset/test1/source_self.json: -------------------------------------------------------------------------------- 1 | { 2 | "mapper" : { 3 | "pattern" : "ss01_c${sensor}_${frame}_${type}_1.5bps.jpc", 4 | "dimensions" : [ "frame" , "sensor", "type" ] 5 | }, 6 | "selfConfig" : true 7 | } 8 | -------------------------------------------------------------------------------- /java/input/data/dataset/test2/map.json: -------------------------------------------------------------------------------- 1 | { 2 | "dimensions" : [ 3 | { 4 | "name" : "frame", 5 | "mapTo" : "FRAME" 6 | }, 7 | { 8 | "name" : "sensor", 9 | "mapTo" : "SENSOR" 10 | }, 11 | { 12 | "name" : "type", 13 | "mapTo" : "IMAGE_TYPE", 14 | "values" : [ 15 | { 16 | "name" : "MS", 17 | "mapTo" : "MULTISPECTRAL" 18 | }, 19 | { 20 | "name" : "PAN", 21 | "mapTo" : "PANCHROMATIC" 22 | } 23 | ] 24 | } 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /java/input/data/images/README: -------------------------------------------------------------------------------- 1 | Extra file to check we can skip over it 2 | -------------------------------------------------------------------------------- /java/input/data/images/directory.json: -------------------------------------------------------------------------------- 1 | { 2 | "mapper" : { 3 | "pattern" : "ss01_c${SENSOR}_${FRAME}_${IMAGE_TYPE}_${ID}.jpc", 4 | "dimensions" : [ "FRAME","SENSOR","IMAGE_TYPE" ] 5 | }, 6 | "ignore" : true 7 | } 8 | -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21556848_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21556848_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21556848_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21556848_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21556871_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21556871_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21556871_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21556871_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21556894_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21556894_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21556894_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21556894_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21556917_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21556917_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21556917_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21556917_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21556941_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21556941_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21556941_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21556941_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21556964_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21556964_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21556964_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21556964_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21556987_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21556987_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21556987_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21556987_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557010_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557010_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557010_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557010_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557033_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557033_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557033_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557033_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557056_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557056_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557056_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557056_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557079_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557079_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557079_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557079_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557103_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557103_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557103_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557103_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557126_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557126_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557126_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557126_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557149_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557149_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557149_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557149_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557172_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557172_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557172_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557172_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557195_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557195_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557195_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557195_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557218_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557218_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557218_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557218_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557242_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557242_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557242_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557242_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557265_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557265_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557265_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557265_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557288_MS_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557288_MS_1.5bps.jpc -------------------------------------------------------------------------------- /java/input/data/images/ss01_c1_2455874.21557288_PAN_1.5bps.jpc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/input/data/images/ss01_c1_2455874.21557288_PAN_1.5bps.jpc -------------------------------------------------------------------------------- /java/ivy.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 13 | 14 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/algorithm/AlgorithmContext.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.algorithm; 18 | 19 | import com.google.mr4c.message.Message; 20 | 21 | public interface AlgorithmContext { 22 | 23 | void log(LogLevel level, String msg); 24 | 25 | void progress(float percentDone, String msg); 26 | 27 | void failure(String msg); 28 | 29 | boolean isFailed(); 30 | 31 | String getEnvironmentDescription(); 32 | 33 | void sendMessage(Message msg); 34 | 35 | } 36 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/algorithm/AlgorithmType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.algorithm; 18 | 19 | public enum AlgorithmType { 20 | NATIVEC, 21 | JAVA 22 | } 23 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/algorithm/EnvironmentSet.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.algorithm; 18 | 19 | public enum EnvironmentSet { 20 | RUNTIME, 21 | JAVA, 22 | CUSTOM, 23 | RAW 24 | } 25 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/algorithm/LogLevel.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.algorithm; 18 | 19 | public enum LogLevel { 20 | INFO, 21 | ERROR, 22 | DEBUG, 23 | WARN 24 | } 25 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/config/algorithm/algo.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "registered name", 3 | "type" : "NATIVEC, JAVA" 4 | "artifact" : "lib or class name", 5 | "inputs" : [ "dataset1"], 6 | "outputs" : [ "dataset1", "dataset2" ] 7 | "optionalInputs" : [ "dataset2" ] 8 | "extras" : [ "libname1", "libname2" ] 9 | "dimensions" : [ 10 | { 11 | "name" : "dim1", 12 | "canSplit" : true or false, defaults to true 13 | "overlapBefore" : optional int, defaults to zero 14 | "overlapAfter" : optional int, defaults to zero 15 | "maxSize" : optional int 16 | "dependent" : true or false, defaults to false 17 | } 18 | ] 19 | 20 | } 21 | 22 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/config/category/CategorySystemData.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.config.category; 18 | 19 | import java.util.Properties; 20 | 21 | /** 22 | * Interface for retrieving category data from system properties in various ways 23 | */ 24 | public interface CategorySystemData { 25 | 26 | Properties getPropertiesFromDeprecatedNames(); 27 | 28 | Properties getPropertiesFromSystemProperties(); 29 | 30 | Properties getPropertiesFromExternalNames(); 31 | 32 | } 33 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/config/category/S3Config.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.config.category; 18 | 19 | public class S3Config extends CategoryConfig { 20 | 21 | public static final String PROP_ID = "id"; 22 | public static final String PROP_SECRET = "secret"; 23 | 24 | public S3Config() { 25 | super(Category.S3); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/config/diff/diff.json: -------------------------------------------------------------------------------- 1 | { 2 | "expected" : { 3 | content of dataset.json 4 | }, 5 | "actual" : { 6 | content of dataset.json 7 | }, 8 | "diff" : { 9 | content of dataset.json 10 | }, 11 | "diffParam" : "name of param in dff" 12 | } 13 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/config/execution/dataset.json: -------------------------------------------------------------------------------- 1 | { 2 | "scheme" : "directory, collection, etc.", 3 | "location" : "URI understood by the scheme", 4 | "srcConfig" : { 5 | "name" : "named config", 6 | "file" : "URI to a config file" 7 | }, 8 | "mapConfig" : { 9 | "name" : "named config", 10 | "file" : "URI to a config file" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/config/execution/directory.json: -------------------------------------------------------------------------------- 1 | { 2 | "mapper" : { 3 | "pattern" : "file pattern", 4 | "dimensions" : [ "dim1", "dim2" ] 5 | } 6 | "mappers" : [ // list of mapper 7 | ] 8 | "ignore" : "boolean to ignore files that don't match" 9 | } 10 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/config/execution/exe.json: -------------------------------------------------------------------------------- 1 | { 2 | "algoConfig" : { 3 | "name" : "named config", 4 | "file" : "URI to a config file" 5 | }, 6 | 7 | "inputs" : { 8 | "dataset1" : { 9 | content of dataset.json 10 | } 11 | 12 | "outputs" : { 13 | "dataset1" : { 14 | content of dataset.json 15 | } 16 | 17 | "params" : { 18 | "name1" : "val1", 19 | "name2" : "val2" 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/config/execution/map.json: -------------------------------------------------------------------------------- 1 | { 2 | "dimensions" : [ 3 | { 4 | "name" : "dsname1", 5 | "mapTo" : "algoname1", // optional 6 | }, 7 | { 8 | "name" : "dsname2", 9 | "values" : [ 10 | { 11 | "name" : "dsname1" 12 | "mapTo" : "algoname1" 13 | } 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/config/resources/ResourceInfo.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.config.resources; 18 | 19 | import com.google.mr4c.config.category.Category; 20 | 21 | public interface ResourceInfo { 22 | 23 | String getResourceName(); 24 | 25 | Category getConfigCategory(); 26 | 27 | String getMinConfigName(); 28 | 29 | String getMaxConfigName(); 30 | 31 | String getHadoopName(); 32 | 33 | String getMaxHadoopName(); 34 | 35 | } 36 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/config/test/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "exeConfig" : { 3 | "inline" : "inlined config", 4 | "file" : "URI to a config file" 5 | }, 6 | "outputs" : { 7 | "dataset1" : { 8 | content of diff.json 9 | } 10 | 11 | } 12 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/dataset/DataFileFinder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.dataset; 18 | 19 | import com.google.mr4c.keys.DataKey; 20 | 21 | import java.io.IOException; 22 | 23 | public interface DataFileFinder { 24 | 25 | DataFile findDataFile(DataKey key) throws IOException; 26 | 27 | } 28 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/hadoop/MR4COutputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.hadoop; 18 | 19 | import com.google.mr4c.sources.ExecutionSource; 20 | 21 | import java.io.IOException; 22 | 23 | import org.apache.hadoop.mapred.RecordWriter; 24 | import org.apache.hadoop.io.Text; 25 | 26 | public class MR4COutputFormat { 27 | 28 | public static RecordWriter getRecordWriter(ExecutionSource exeSrc) throws IOException { 29 | return new MR4CRecordWriter(exeSrc); 30 | 31 | } 32 | 33 | } 34 | 35 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/hadoop/StaticHadoopBinder.java.mrv1: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.hadoop; 18 | 19 | import com.google.mr4c.hadoop.mrv1.MRv1Binding; 20 | 21 | abstract class StaticHadoopBinder { 22 | 23 | static HadoopBinding createBinding() { 24 | return new MRv1Binding(); 25 | } 26 | 27 | } 28 | 29 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/hadoop/StaticHadoopBinder.java.yarn: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.hadoop; 18 | 19 | import com.google.mr4c.hadoop.yarn.YarnBinding; 20 | 21 | abstract class StaticHadoopBinder { 22 | 23 | static HadoopBinding createBinding() { 24 | return new YarnBinding(); 25 | } 26 | 27 | } 28 | 29 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/keys/DataKey.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.keys; 18 | 19 | import java.util.Set; 20 | 21 | public interface DataKey extends Comparable { 22 | 23 | /** 24 | returns all the dimensions included in this key 25 | */ 26 | Set getDimensions(); 27 | 28 | /** 29 | returns all the elements comprising this key 30 | */ 31 | Set getElements(); 32 | 33 | DataKeyElement getElement(DataKeyDimension dim); 34 | 35 | boolean hasDimension(DataKeyDimension dim); 36 | 37 | } 38 | 39 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/keys/DataKeyFilter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.keys; 18 | 19 | public interface DataKeyFilter { 20 | 21 | boolean filter(DataKey key); 22 | 23 | } 24 | 25 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/keys/ElementFilter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.keys; 18 | 19 | public interface ElementFilter { 20 | 21 | // should throw exception if wrong dimension 22 | boolean filter(DataKeyElement element); 23 | 24 | DataKeyDimension getDimension(); 25 | 26 | } 27 | 28 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/keys/HasDimensionFilter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.keys; 18 | 19 | /** 20 | * Passes any key that has the specified dimension 21 | */ 22 | public class HasDimensionFilter implements DataKeyFilter { 23 | 24 | private DataKeyDimension m_dim; 25 | 26 | public HasDimensionFilter(DataKeyDimension dim) { 27 | m_dim = dim; 28 | } 29 | 30 | public boolean filter(DataKey key) { 31 | return key.hasDimension(m_dim); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/keys/IdentityDataKeyFilter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.keys; 18 | 19 | public class IdentityDataKeyFilter implements DataKeyFilter { 20 | 21 | public static final IdentityDataKeyFilter INSTANCE = new IdentityDataKeyFilter(); 22 | 23 | private IdentityDataKeyFilter(){} 24 | 25 | public boolean filter(DataKey key) { 26 | return true; 27 | } 28 | 29 | } 30 | 31 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/keys/NarrowingFilter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.keys; 18 | 19 | /** 20 | * Passes any key that is a "narrowing" of the source key. 21 | */ 22 | public class NarrowingFilter implements DataKeyFilter { 23 | 24 | private DataKey m_src; 25 | 26 | public NarrowingFilter(DataKey src) { 27 | m_src = src; 28 | } 29 | 30 | public boolean filter(DataKey key) { 31 | return DataKeyUtils.narrows(m_src, key); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/message/MessageHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.message; 18 | 19 | import java.io.IOException; 20 | import java.net.URI; 21 | 22 | public interface MessageHandler { 23 | 24 | void setURI(URI uri); 25 | 26 | void handleMessage(Message msg) throws IOException; 27 | 28 | } 29 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/metadata/MetadataElement.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.metadata; 18 | 19 | public interface MetadataElement { 20 | 21 | MetadataElementType getMetadataElementType(); 22 | 23 | void accept(MetadataVisitor visitor); 24 | } 25 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/metadata/MetadataElementType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.metadata; 18 | 19 | public enum MetadataElementType { 20 | 21 | FIELD, 22 | ARRAY, 23 | LIST, 24 | MAP, 25 | KEY; 26 | 27 | } 28 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/metadata/MetadataVisitor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.metadata; 18 | 19 | public interface MetadataVisitor { 20 | 21 | void visitMapPreIteration(MetadataMap map); 22 | 23 | void visitMapPostIteration(MetadataMap map); 24 | 25 | void visitListPreIteration(MetadataList list); 26 | 27 | void visitListPostIteration(MetadataList list); 28 | 29 | void visitField(MetadataField field); 30 | 31 | void visitArray(MetadataArray array); 32 | 33 | void visitKey(MetadataKey key); 34 | 35 | } 36 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/metadata/PrimitiveType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.metadata; 18 | 19 | public enum PrimitiveType { 20 | 21 | BOOLEAN, 22 | BYTE, 23 | INTEGER, 24 | FLOAT, 25 | DOUBLE, 26 | STRING, 27 | SIZE_T, 28 | LONG_DOUBLE; 29 | 30 | } 31 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/nativec/ExternalAlgorithm.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.nativec; 18 | 19 | public interface ExternalAlgorithm { 20 | 21 | String getName(); 22 | 23 | String getSerializedAlgorithm(); 24 | 25 | void setSerializedAlgorithm(String serializedAlgo); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/nativec/ExternalContext.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.nativec; 18 | 19 | import com.google.mr4c.algorithm.AlgorithmContext; 20 | 21 | public interface ExternalContext { 22 | 23 | AlgorithmContext getContext(); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/nativec/ExternalDataFile.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.nativec; 18 | 19 | public interface ExternalDataFile { 20 | 21 | String getSerializedKey(); 22 | 23 | String getFileName(); 24 | 25 | String getSerializedFile(); 26 | 27 | void setSerializedFile(String serFile); 28 | 29 | ExternalDataFileSource getDataFileSource(); 30 | 31 | void setDataFileSource(ExternalDataFileSource src); 32 | 33 | ExternalDataFileSink getDataFileSink(); 34 | 35 | void setDataFileSink(ExternalDataFileSink sink); 36 | 37 | } 38 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/nativec/ExternalDataFileSink.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.nativec; 18 | 19 | import com.google.mr4c.sources.DataFileSink; 20 | 21 | public interface ExternalDataFileSink { 22 | 23 | DataFileSink getSink(); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/nativec/ExternalDataFileSource.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.nativec; 18 | 19 | import com.google.mr4c.sources.DataFileSource; 20 | 21 | public interface ExternalDataFileSource { 22 | 23 | DataFileSource getSource(); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/nativec/ExternalDataset.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.nativec; 18 | 19 | public interface ExternalDataset { 20 | 21 | String getName(); 22 | 23 | String getSerializedDataset(); 24 | 25 | void setSerializedDataset(String serDataset); 26 | 27 | void addDataFile(ExternalDataFile file); 28 | 29 | ExternalDataFile getDataFile(int index); 30 | 31 | int getFileCount(); 32 | 33 | } 34 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/nativec/ExternalRandomAccessFileSink.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.nativec; 18 | 19 | import com.google.mr4c.sources.RandomAccessFileSink; 20 | 21 | public interface ExternalRandomAccessFileSink { 22 | 23 | RandomAccessFileSink getSink(); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/nativec/ExternalRandomAccessFileSource.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.nativec; 18 | 19 | import com.google.mr4c.sources.RandomAccessFileSource; 20 | 21 | public interface ExternalRandomAccessFileSource { 22 | 23 | RandomAccessFileSource getSource(); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/serialize/AlgorithmSerializer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.serialize; 18 | 19 | import com.google.mr4c.algorithm.AlgorithmSchema; 20 | 21 | import java.io.IOException; 22 | import java.io.Reader; 23 | import java.io.Writer; 24 | 25 | public interface AlgorithmSerializer extends Serializer { 26 | 27 | void serializeAlgorithmSchema(AlgorithmSchema algoSchema, Writer writer) throws IOException; 28 | 29 | AlgorithmSchema deserializeAlgorithmSchema(Reader reader) throws IOException; 30 | 31 | } 32 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/serialize/KeyspaceSerializer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.serialize; 18 | 19 | import com.google.mr4c.keys.Keyspace; 20 | 21 | import java.io.IOException; 22 | import java.io.Reader; 23 | import java.io.Writer; 24 | 25 | public interface KeyspaceSerializer extends Serializer { 26 | 27 | void serializeKeyspace(Keyspace keyspace, Writer writer) throws IOException; 28 | 29 | Keyspace deserializeKeyspace(Reader reader) throws IOException; 30 | 31 | } 32 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/serialize/PropertiesSerializer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.serialize; 18 | 19 | import java.io.IOException; 20 | import java.io.Reader; 21 | import java.io.Writer; 22 | import java.util.Properties; 23 | 24 | public interface PropertiesSerializer extends Serializer { 25 | 26 | void serializeProperties(Properties props, Writer writer) throws IOException; 27 | 28 | Properties deserializeProperties(Reader reader) throws IOException; 29 | 30 | } 31 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/serialize/Serializer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.serialize; 18 | 19 | import com.google.mr4c.keys.Keyspace; 20 | 21 | import java.io.IOException; 22 | import java.io.Reader; 23 | import java.io.Writer; 24 | 25 | public interface Serializer { 26 | 27 | /** 28 | * Returns the content type generated during serialization, and expected during deserialization 29 | */ 30 | String getContentType(); 31 | 32 | } 33 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/serialize/bean/metadata/MetadataElementBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.serialize.bean.metadata; 18 | 19 | import com.google.mr4c.metadata.MetadataElement; 20 | import com.google.mr4c.metadata.MetadataElementType; 21 | 22 | public interface MetadataElementBean { 23 | 24 | MetadataElementType getMetadataElementType(); 25 | 26 | MetadataElement toMetadataElement(); 27 | 28 | } 29 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/sources/AbstractDataFileSink.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.sources; 18 | 19 | import java.io.File; 20 | 21 | /** 22 | * Base DataFileSink with defaults for optional methods 23 | */ 24 | public abstract class AbstractDataFileSink implements DataFileSink { 25 | 26 | public String getFileName() { 27 | return null; 28 | } 29 | 30 | public File getLocalFile() { 31 | return null; 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/sources/DataKeyFileMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.sources; 18 | 19 | import com.google.mr4c.keys.DataKey; 20 | 21 | public interface DataKeyFileMapper { 22 | 23 | String getFileName(DataKey key); 24 | 25 | DataKey getKey(String name); 26 | 27 | boolean canMapName(String name); 28 | 29 | boolean canMapKey(DataKey key); 30 | 31 | // NOTE: Adding getDescription() might be helpful for logging. 32 | // Descriptions could include things like pattern to match, etc. 33 | 34 | } 35 | 36 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/sources/DiffSource.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.sources; 18 | 19 | import java.io.IOException; 20 | 21 | public interface DiffSource { 22 | 23 | public enum DiffOutput { SAME, EXPECTED_ONLY, ACTUAL_ONLY, DIFF_EXPECTED, DIFF_ACTUAL } 24 | 25 | DatasetSource getExpectedDatasetSource() throws IOException; 26 | 27 | DatasetSource getActualDatasetSource() throws IOException; 28 | 29 | DatasetSource getOutputDatasetSource(DiffOutput output) throws IOException; 30 | } 31 | -------------------------------------------------------------------------------- /java/src/java/com/google/mr4c/sources/RandomAccessible.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.sources; 18 | 19 | import java.io.IOException; 20 | import java.io.RandomAccessFile; 21 | 22 | public interface RandomAccessible { 23 | 24 | RandomAccessFile getRandomAccess() throws IOException; 25 | 26 | void close() throws IOException; 27 | 28 | boolean isStaged(); 29 | 30 | boolean isWritable(); 31 | 32 | String getDescription(); 33 | 34 | } 35 | -------------------------------------------------------------------------------- /java/src/resources/mr4c-hadoop-default.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /java/test/java/com/google/mr4c/hadoop/StaticHadoopTestBinder.java.mrv1: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.hadoop; 18 | 19 | import com.google.mr4c.hadoop.mrv1.MRv1TestBinding; 20 | 21 | abstract class StaticHadoopTestBinder { 22 | 23 | static HadoopTestBinding createBinding() { 24 | return new MRv1TestBinding(); 25 | } 26 | 27 | } 28 | 29 | -------------------------------------------------------------------------------- /java/test/java/com/google/mr4c/hadoop/StaticHadoopTestBinder.java.yarn: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.mr4c.hadoop; 18 | 19 | import com.google.mr4c.hadoop.yarn.YarnTestBinding; 20 | 21 | abstract class StaticHadoopTestBinder { 22 | 23 | static HadoopTestBinding createBinding() { 24 | return new YarnTestBinding(); 25 | } 26 | 27 | } 28 | 29 | -------------------------------------------------------------------------------- /java/test/resources/empty.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/java/test/resources/empty.jar -------------------------------------------------------------------------------- /java/third_party/etsy/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010-2014 Etsy 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /java/third_party/etsy/README.google: -------------------------------------------------------------------------------- 1 | URL: https://github.com/etsy/statsd/blob/v0.7.2/examples/StatsdClient.java 2 | Version: v0.7.2 3 | License: MIT 4 | License File: LICENSE 5 | 6 | Description: 7 | Client for emitting stats in the Statsd format 8 | 9 | Local Modifications: 10 | Specify a Java package 11 | Use MR4C Logging 12 | -------------------------------------------------------------------------------- /java/third_party/hadoop/README.google: -------------------------------------------------------------------------------- 1 | URL: https://github.com/apache/hadoop/blob/release-2.5.0/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java 2 | Version: release-2.5.0 3 | License: Apache 4 | License File: LICENSE 5 | 6 | Description: 7 | StaticMapping.java is missing from the Cloudera distribution. 8 | Get NoClassDefFoundError when starting MiniDFSCluster 9 | Following suggestion at http://bigchunk.me/2012/09/07/hadoop-smoke-testing-mr/ 10 | 11 | Local Modifications: 12 | Comment out "Override" annotation so we can compile under YARN and MRv1 13 | -------------------------------------------------------------------------------- /java/third_party/stack_overflow/README.google: -------------------------------------------------------------------------------- 1 | URL: http://stackoverflow.com/questions/4332264/wrapping-a-bytebuffer-with-an-inputstream 2 | Version: N/A 3 | License: CC BY-SA (http://creativecommons.org/licenses/by-sa/3.0/legalcode.txt) 4 | License File: LICENSE 5 | 6 | Description: 7 | Implementation of a ByteBufferInputStream 8 | 9 | Local Modifications: 10 | Add Java package and justification 11 | -------------------------------------------------------------------------------- /java/third_party/stack_overflow/src/java/com/google/mr4c/util/ByteBufferInputStream.java: -------------------------------------------------------------------------------- 1 | package com.google.mr4c.util; 2 | 3 | import java.io.InputStream; 4 | import java.io.IOException; 5 | import java.nio.ByteBuffer; 6 | 7 | /** 8 | * Wraps an InputStream around a ByteBuffer, since the JDK can't be bothered. 9 | * Copied from: http://stackoverflow.com/questions/4332264/wrapping-a-bytebuffer-with-an-inputstream. 10 | * The Avro project has an implementation, org.apache.avro.util.ByteBufferInputStream, which has issues, such as throwing EOFException when you reach the end of the buffer 11 | */ 12 | public class ByteBufferInputStream extends InputStream { 13 | 14 | ByteBuffer buf; 15 | 16 | public ByteBufferInputStream(ByteBuffer buf) { 17 | this.buf = buf; 18 | } 19 | 20 | public int read() throws IOException { 21 | if (!buf.hasRemaining()) { 22 | return -1; 23 | } 24 | return buf.get() & 0xFF; 25 | } 26 | 27 | public int read(byte[] bytes, int off, int len) 28 | throws IOException { 29 | if (!buf.hasRemaining()) { 30 | return -1; 31 | } 32 | 33 | len = Math.min(len, buf.remaining()); 34 | buf.get(bytes, off, len); 35 | return len; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /java/tools/build_mrv1: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # execute any ant command line with MRv1 binding 4 | 5 | set -e 6 | 7 | ant -Dhadoop.binding=mrv1 $* 8 | -------------------------------------------------------------------------------- /java/tools/build_yarn: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # execute any ant command line with YARN binding 4 | 5 | set -e 6 | 7 | ant -Dhadoop.binding=yarn $* 8 | -------------------------------------------------------------------------------- /native/bin/post_install: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | THIS=`readlink -f $0` 4 | BIN=`dirname $THIS` 5 | MR4C_NATIVE=`dirname $BIN` 6 | 7 | ln -s -f $MR4C_NATIVE/dist/libmr4c.so /usr/local/lib/libmr4c.so 8 | ln -s -f $MR4C_NATIVE/include /usr/include/mr4c 9 | ln -s -f $MR4C_NATIVE/conf/mr4c-ld.conf /etc/ld.so.conf.d/mr4c-ld.conf 10 | mkdir -p /etc/mr4c 11 | ln -s -f $MR4C_NATIVE/conf/log4cxx.properties /etc/mr4c/log4cxx.properties 12 | 13 | /sbin/ldconfig 14 | -------------------------------------------------------------------------------- /native/bin/pre_remove: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | rm -f /usr/local/lib/libmr4c.so 4 | rm -f /usr/include/mr4c 5 | rm -f /etc/ld.so.conf.d/mr4c-ld.conf 6 | rm -f /etc/mr4c/log4cxx.properties 7 | -------------------------------------------------------------------------------- /native/build.properties: -------------------------------------------------------------------------------- 1 | tools.dir=tools 2 | -------------------------------------------------------------------------------- /native/conf/mr4c-ld.conf: -------------------------------------------------------------------------------- 1 | /usr/local/mr4c/native/dist 2 | -------------------------------------------------------------------------------- /native/conf_docs/doxygen_footer.html: -------------------------------------------------------------------------------- 1 | 2 |
3 | $datetime   4 | doxygen 5 | $doxygenversion 6 |
7 | 8 |

Copyright 2014 Google Inc. All rights reserved.

9 | 10 | 11 | -------------------------------------------------------------------------------- /native/src/cpp/api/algorithm/algorithm_api.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_ALGORITHM_API_H__ 18 | #define __MR4C_ALGORITHM_API_H__ 19 | 20 | #include "Algorithm.h" 21 | #include "AlgorithmAutoRegister.h" 22 | #include "AlgorithmConfig.h" 23 | #include "AlgorithmData.h" 24 | #include "AlgorithmRegistry.h" 25 | #include "AlgorithmRunner.h" 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /native/src/cpp/api/catalog/catalog_api.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_CATALOG_API_H__ 18 | #define __MR4C_CATALOG_API_H__ 19 | 20 | #include "DimensionCatalog.h" 21 | #include "ImageTypes.h" 22 | 23 | #endif 24 | 25 | -------------------------------------------------------------------------------- /native/src/cpp/api/context/MessageConsumer.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_MESSAGE_CONSUMER_H__ 18 | #define __MR4C_MESSAGE_CONSUMER_H__ 19 | 20 | #include 21 | #include "Message.h" 22 | 23 | namespace MR4C { 24 | 25 | /** 26 | * Class to extend to receive messages sent to topics 27 | */ 28 | class MessageConsumer { 29 | 30 | public: 31 | 32 | virtual void receiveMessage(const Message& msg) =0; 33 | 34 | 35 | }; 36 | 37 | } 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /native/src/cpp/api/context/ProgressReporter.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_PROGRESS_REPORTER_H__ 18 | #define __MR4C_PROGRESS_REPORTER_H__ 19 | 20 | #include 21 | 22 | namespace MR4C { 23 | 24 | /** 25 | * Class to extend to receive progress messages 26 | */ 27 | class ProgressReporter { 28 | 29 | public: 30 | 31 | virtual void progress(float percentDone, const std::string& msg) =0; 32 | 33 | }; 34 | 35 | } 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /native/src/cpp/api/context/context_api.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_CONTEXT_API_H__ 18 | #define __MR4C_CONTEXT_API_H__ 19 | 20 | #include "AlgorithmContext.h" 21 | #include "Logger.h" 22 | #include "Message.h" 23 | #include "MessageConsumer.h" 24 | #include "ProgressReporter.h" 25 | 26 | #endif 27 | -------------------------------------------------------------------------------- /native/src/cpp/api/dataset/DataFileSink.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_DATA_FILE_SINK_H__ 18 | #define __MR4C_DATA_FILE_SINK_H__ 19 | 20 | namespace MR4C { 21 | 22 | 23 | class DataFileSink { 24 | 25 | public: 26 | 27 | /** 28 | * Write num bytes from buf 29 | */ 30 | virtual void write(char* buf, size_t num) =0; 31 | 32 | virtual void close() =0; 33 | 34 | virtual bool isClosed() const =0; 35 | 36 | }; 37 | 38 | } 39 | #endif 40 | 41 | 42 | -------------------------------------------------------------------------------- /native/src/cpp/api/dataset/dataset_api.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_DATASET_API_H__ 18 | #define __MR4C_DATASET_API_H__ 19 | 20 | #include "DataFile.h" 21 | #include "DataFileSink.h" 22 | #include "DataFileSource.h" 23 | #include "Dataset.h" 24 | #include "DatasetContext.h" 25 | #include "LocalDataFileSink.h" 26 | #include "LocalDataFileSource.h" 27 | #include "LocalTempFile.h" 28 | #include "RandomAccessFile.h" 29 | #include "SimpleDataFileSource.h" 30 | #include "WritableRandomAccessFile.h" 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /native/src/cpp/api/error/error_api.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_ERROR_API_H__ 18 | #define __MR4C_ERROR_API_H__ 19 | 20 | #include "Error.h" 21 | #include "ErrorReporter.h" 22 | #include "JsonErrorSerializer.h" 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /native/src/cpp/api/external/CExternalAlgorithm.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_C_EXTERNAL_ALGORITHM_H__ 18 | #define __MR4C_C_EXTERNAL_ALGORITHM_H__ 19 | 20 | #include "ExternalAlgorithm.h" 21 | #include "external/capi/external_capi.h" 22 | 23 | struct CExternalAlgorithmStruct { 24 | MR4C::ExternalAlgorithm* algo; 25 | }; 26 | typedef struct CExternalAlgorithmStruct CExternalAlgorithm; 27 | 28 | CExternalAlgorithmPtr wrapExternalAlgorithm(MR4C::ExternalAlgorithm* algo); 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /native/src/cpp/api/external/CExternalAlgorithmData.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_C_EXTERNAL_ALGORITHM_DATA_H__ 18 | #define __MR4C_C_EXTERNAL_ALGORITHM_DATA_H__ 19 | 20 | #include "ExternalAlgorithmData.h" 21 | #include "external/capi/external_capi.h" 22 | 23 | struct CExternalAlgorithmDataStruct { 24 | MR4C::ExternalAlgorithmData* algoData; 25 | }; 26 | typedef struct CExternalAlgorithmDataStruct CExternalAlgorithmData; 27 | 28 | CExternalAlgorithmDataPtr wrapExternalAlgorithmData(MR4C::ExternalAlgorithmData* algoData); 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /native/src/cpp/api/external/CExternalContext.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_C_EXTERNAL_CONTEXT_H__ 18 | #define __MR4C_C_EXTERNAL_CONTEXT_H__ 19 | 20 | #include "ExternalContext.h" 21 | #include "external/capi/external_capi.h" 22 | 23 | struct CExternalContextStruct { 24 | MR4C::ExternalContext* context; 25 | }; 26 | typedef struct CExternalContextStruct CExternalContext; 27 | 28 | CExternalContextPtr wrapExternalContext(MR4C::ExternalContext* context); 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /native/src/cpp/api/external/CExternalDataFile.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_C_EXTERNAL_DATA_FILE_H__ 18 | #define __MR4C_C_EXTERNAL_DATA_FILE_H__ 19 | 20 | #include "ExternalDataFile.h" 21 | #include "external/capi/external_capi.h" 22 | 23 | struct CExternalDataFileStruct { 24 | MR4C::ExternalDataFile* file; 25 | }; 26 | typedef struct CExternalDataFileStruct CExternalDataFile; 27 | 28 | CExternalDataFilePtr wrapExternalDataFile(MR4C::ExternalDataFile* file); 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /native/src/cpp/api/external/CExternalDataFileSink.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_C_EXTERNAL_DATA_FILE_SINK_H__ 18 | #define __MR4C_C_EXTERNAL_DATA_FILE_SINK_H__ 19 | 20 | #include "ExternalDataFileSink.h" 21 | #include "external/capi/external_capi.h" 22 | 23 | struct CExternalDataFileSinkStruct { 24 | MR4C::ExternalDataFileSink* sink; 25 | }; 26 | typedef struct CExternalDataFileSinkStruct CExternalDataFileSink; 27 | 28 | CExternalDataFileSinkPtr wrapExternalDataFileSink(MR4C::ExternalDataFileSink* src); 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /native/src/cpp/api/external/CExternalDataFileSource.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_C_EXTERNAL_DATA_FILE_SOURCE_H__ 18 | #define __MR4C_C_EXTERNAL_DATA_FILE_SOURCE_H__ 19 | 20 | #include "ExternalDataFileSource.h" 21 | #include "external/capi/external_capi.h" 22 | 23 | struct CExternalDataFileSourceStruct { 24 | MR4C::ExternalDataFileSource* src; 25 | }; 26 | typedef struct CExternalDataFileSourceStruct CExternalDataFileSource; 27 | 28 | CExternalDataFileSourcePtr wrapExternalDataFileSource(MR4C::ExternalDataFileSource* src); 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /native/src/cpp/api/external/CExternalDataset.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_C_EXTERNAL_DATASET_H__ 18 | #define __MR4C_C_EXTERNAL_DATASET_H__ 19 | 20 | #include "ExternalDataset.h" 21 | #include "external/capi/external_capi.h" 22 | 23 | struct CExternalDatasetStruct { 24 | MR4C::ExternalDataset* dataset; 25 | }; 26 | typedef struct CExternalDatasetStruct CExternalDataset; 27 | 28 | CExternalDatasetPtr wrapExternalDataset(MR4C::ExternalDataset* dataset); 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /native/src/cpp/api/external/CExternalEntry.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_C_EXTERNAL_ENTRY_H__ 18 | #define __MR4C_C_EXTERNAL_ENTRY_H__ 19 | 20 | #include "external/capi/external_capi.h" 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /native/src/cpp/api/external/capi/external_capi.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_EXTERNAL_CAPI_H__ 18 | #define __MR4C_EXTERNAL_CAPI_H__ 19 | 20 | #include "CExternalAlgorithmAPI.h" 21 | #include "CExternalAlgorithmDataAPI.h" 22 | #include "CExternalContextAPI.h" 23 | #include "CExternalDataFileAPI.h" 24 | #include "CExternalDataFileSinkAPI.h" 25 | #include "CExternalDataFileSourceAPI.h" 26 | #include "CExternalDatasetAPI.h" 27 | #include "CExternalEntryAPI.h" 28 | #include "CExternalRandomAccessFileAPI.h" 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /native/src/cpp/api/keys/keys_api.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_KEYS_API_H__ 18 | #define __MR4C_KEYS_API_H__ 19 | 20 | #include "DataKeyBuilder.h" 21 | #include "DataKeyDimension.h" 22 | #include "DataKeyElement.h" 23 | #include "DataKey.h" 24 | #include "Keyspace.h" 25 | #include "KeyspaceBuilder.h" 26 | #include "KeyspaceDimension.h" 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /native/src/cpp/api/metadata/metadata_api.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_METADATA_API_H__ 18 | #define __MR4C_METADATA_API_H__ 19 | 20 | #include "MetadataArray.h" 21 | #include "MetadataElement.h" 22 | #include "MetadataField.h" 23 | #include "MetadataKey.h" 24 | #include "MetadataList.h" 25 | #include "MetadataMap.h" 26 | #include "Primitive.h" 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /native/src/cpp/api/serialize/AlgorithmSerializer.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_ALGORITHM_SERIALIZER_H__ 18 | #define __MR4C_ALGORITHM_SERIALIZER_H__ 19 | 20 | #include "algorithm/algorithm_api.h" 21 | #include "Serializer.h" 22 | 23 | namespace MR4C { 24 | 25 | class AlgorithmSerializer : public Serializer { 26 | 27 | public : 28 | 29 | virtual std::string serializeAlgorithm(const Algorithm& algo) const =0; 30 | 31 | virtual void deserializeAlgorithm(const std::string& serializedAlgo, Algorithm& algo) const =0; 32 | 33 | }; 34 | 35 | } 36 | #endif 37 | 38 | -------------------------------------------------------------------------------- /native/src/cpp/api/serialize/KeyspaceSerializer.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_KEYSPACE_SERIALIZER_H__ 18 | #define __MR4C_KEYSPACE_SERIALIZER_H__ 19 | 20 | #include 21 | #include "keys/keys_api.h" 22 | #include "Serializer.h" 23 | 24 | namespace MR4C { 25 | 26 | class KeyspaceSerializer : public Serializer { 27 | 28 | public : 29 | 30 | virtual std::string serializeKeyspace(const Keyspace& keyspace) const =0; 31 | 32 | virtual Keyspace deserializeKeyspace(const std::string& data) const =0; 33 | 34 | }; 35 | 36 | } 37 | #endif 38 | 39 | -------------------------------------------------------------------------------- /native/src/cpp/api/serialize/Serializer.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_SERIALIZER_H__ 18 | #define __MR4C_SERIALIZER_H__ 19 | 20 | #include 21 | 22 | namespace MR4C { 23 | 24 | class Serializer { 25 | 26 | public : 27 | 28 | /** 29 | * Returns the content type generated during serialization, 30 | * and expected during deserialization 31 | */ 32 | virtual std::string getContentType() const =0; 33 | 34 | }; 35 | 36 | } 37 | #endif 38 | 39 | -------------------------------------------------------------------------------- /native/src/cpp/api/serialize/json/json_api.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_SERIALIZE_JSON_API_H__ 18 | #define __MR4C_SERIALIZE_JSON_API_H__ 19 | 20 | #include "JanssonUtil.h" 21 | #include "JsonAlgorithmSerializer.h" 22 | #include "JsonAlgorithmConfigSerializer.h" 23 | #include "JsonCommonSerializer.h" 24 | #include "JsonDatasetSerializer.h" 25 | #include "JsonKeyspaceSerializer.h" 26 | #include "JsonPropertiesSerializer.h" 27 | #include "JsonSerializerFactory.h" 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /native/src/cpp/api/serialize/serialize_api.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_SERIALIZE_API_H__ 18 | #define __MR4C_SERIALIZE_API_H__ 19 | 20 | #include "AlgorithmSerializer.h" 21 | #include "AlgorithmConfigSerializer.h" 22 | #include "DatasetSerializer.h" 23 | #include "KeyspaceSerializer.h" 24 | #include "PropertiesSerializer.h" 25 | #include "Serializer.h" 26 | #include "SerializerFactory.h" 27 | #include "SerializerRegistry.h" 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /native/src/cpp/api/util/StackUtil.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_STACK_UTIL_H__ 18 | #define __MR4C_STACK_UTIL_H__ 19 | 20 | namespace MR4C { 21 | 22 | class StackUtil { 23 | 24 | public : 25 | 26 | static std::string generateBackTrace(); 27 | 28 | static std::string generateBackTrace(int maxFrames); 29 | 30 | static std::string formatLocation(const char* file, const char* function, int line); 31 | 32 | }; 33 | 34 | } 35 | 36 | #endif 37 | -------------------------------------------------------------------------------- /native/src/cpp/api/util/util_api.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_UTIL_API_H__ 18 | #define __MR4C_UTIL_API_H__ 19 | 20 | #include "ArrayUtil.h" 21 | #include "MR4CEnvironment.h" 22 | #include "MR4CLogging.h" 23 | #include "MR4CTempFiles.h" 24 | #include "Helpers.h" 25 | #include "IOUtil.h" 26 | #include "Properties.h" 27 | #include "StackUtil.h" 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /native/src/cpp/impl/algorithm/AlgorithmAutoRegister.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | 20 | #include "algorithm/algorithm_api.h" 21 | 22 | namespace MR4C { 23 | 24 | AlgorithmAutoRegister::AlgorithmAutoRegister(const std::string& name, Algorithm* algorithm) { 25 | AlgorithmRegistry::instance().registerAlgorithm(name, algorithm); 26 | } 27 | 28 | AlgorithmAutoRegister::~AlgorithmAutoRegister(){} 29 | 30 | } 31 | -------------------------------------------------------------------------------- /native/test/cpp/api/MR4CTests.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef __MR4C_TESTS_H__ 18 | #define __MR4C_TESTS_H__ 19 | 20 | #include 21 | 22 | namespace MR4C { 23 | 24 | std::string extractTestName(int argc, char* argv[]); 25 | } 26 | 27 | #endif 28 | 29 | -------------------------------------------------------------------------------- /native/test/cpp/suites/Cover.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | #include "MR4CTests.h" 20 | 21 | // local test runner that just dumps to the console 22 | int main( int argc, char* argv[]) 23 | { 24 | std::string name = MR4C::extractTestName(argc, argv); 25 | CPPUNIT_NS::TextTestRunner testrunner; 26 | testrunner.addTest( CPPUNIT_NS::TestFactoryRegistry::getRegistry(name).makeTest()); 27 | testrunner.run(); 28 | } 29 | 30 | 31 | -------------------------------------------------------------------------------- /native/test/cpp/suites/Local.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | #include "MR4CTests.h" 20 | 21 | // local test runner that just dumps to the console 22 | int main( int argc, char* argv[]) 23 | { 24 | std::string name = MR4C::extractTestName(argc, argv); 25 | CPPUNIT_NS::TextTestRunner testrunner; 26 | testrunner.addTest( CPPUNIT_NS::TestFactoryRegistry::getRegistry(name).makeTest()); 27 | testrunner.run(); 28 | 29 | } 30 | 31 | 32 | -------------------------------------------------------------------------------- /native/test/cpp/tests/algorithm/AlgorithmTests.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | 20 | CPPUNIT_REGISTRY_ADD("TestAlgorithmConfig", "AlgorithmTests"); 21 | CPPUNIT_REGISTRY_ADD("TestAlgorithmData", "AlgorithmTests"); 22 | 23 | -------------------------------------------------------------------------------- /native/test/cpp/tests/context/ContextTests.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | 20 | CPPUNIT_REGISTRY_ADD("TestMessage", "ContextTests"); 21 | 22 | -------------------------------------------------------------------------------- /native/test/cpp/tests/dataset/DatasetTests.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | 20 | CPPUNIT_REGISTRY_ADD("TestDataFile", "DatasetTests"); 21 | CPPUNIT_REGISTRY_ADD("TestDataset", "DatasetTests"); 22 | CPPUNIT_REGISTRY_ADD("TestLocalDataFileSink", "DatasetTests"); 23 | CPPUNIT_REGISTRY_ADD("TestLocalDataFileSource", "DatasetTests"); 24 | CPPUNIT_REGISTRY_ADD("TestLocalTempFile", "DatasetTests"); 25 | CPPUNIT_REGISTRY_ADD("TestSimpleDataFileSource", "DatasetTests"); 26 | 27 | -------------------------------------------------------------------------------- /native/test/cpp/tests/error/ErrorTests.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | 20 | CPPUNIT_REGISTRY_ADD("TestError", "ErrorTests"); 21 | CPPUNIT_REGISTRY_ADD("TestJsonErrorSerializer", "ErrorTests"); 22 | 23 | -------------------------------------------------------------------------------- /native/test/cpp/tests/external/ExternalTests.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | 20 | CPPUNIT_REGISTRY_ADD("TestExternalAlgorithmDataSerializer", "ExternalTests"); 21 | CPPUNIT_REGISTRY_ADD("TestExternalDatasetSerializer", "ExternalTests"); 22 | CPPUNIT_REGISTRY_ADD("TestExternalEntry", "ExternalTests"); 23 | 24 | -------------------------------------------------------------------------------- /native/test/cpp/tests/metadata/MetadataTests.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | 20 | CPPUNIT_REGISTRY_ADD("TestMetadataArray", "MetadataTests"); 21 | CPPUNIT_REGISTRY_ADD("TestMetadataField", "MetadataTests"); 22 | CPPUNIT_REGISTRY_ADD("TestMetadataKey", "MetadataTests"); 23 | CPPUNIT_REGISTRY_ADD("TestMetadataList", "MetadataTests"); 24 | CPPUNIT_REGISTRY_ADD("TestMetadataMap", "MetadataTests"); 25 | CPPUNIT_REGISTRY_ADD("TestPrimitive", "MetadataTests"); 26 | 27 | -------------------------------------------------------------------------------- /native/test/cpp/tests/multithread/MultithreadTests.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | 20 | CPPUNIT_REGISTRY_ADD("TestDataFileMultithread", "MultithreadTests"); 21 | CPPUNIT_REGISTRY_ADD("TestDatasetMultithread", "MultithreadTests"); 22 | CPPUNIT_REGISTRY_ADD("TestExternalRandomAccessFileMultithread", "MultithreadTests"); 23 | CPPUNIT_REGISTRY_ADD("TestExternalRandomAccessFileSinkMultithread", "MultithreadTests"); 24 | -------------------------------------------------------------------------------- /native/test/cpp/tests/serialize/json/JsonTests.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | 20 | CPPUNIT_REGISTRY_ADD("TestJsonAlgorithmSerializer", "JsonTests"); 21 | CPPUNIT_REGISTRY_ADD("TestJsonAlgorithmConfigSerializer", "JsonTests"); 22 | CPPUNIT_REGISTRY_ADD("TestJsonDatasetSerializer", "JsonTests"); 23 | CPPUNIT_REGISTRY_ADD("TestJsonKeyspaceSerializer", "JsonTests"); 24 | CPPUNIT_REGISTRY_ADD("TestJsonPropertiesSerializer", "JsonTests"); 25 | 26 | -------------------------------------------------------------------------------- /native/test/cpp/tests/util/UtilTests.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | 20 | CPPUNIT_REGISTRY_ADD("TestArrayUtil", "UtilTests"); 21 | CPPUNIT_REGISTRY_ADD("TestMR4CEnvironment", "UtilTests"); 22 | CPPUNIT_REGISTRY_ADD("TestMR4CLogging", "UtilTests"); 23 | CPPUNIT_REGISTRY_ADD("TestMR4CTempFiles", "UtilTests"); 24 | CPPUNIT_REGISTRY_ADD("TestIOUtil", "UtilTests"); 25 | CPPUNIT_REGISTRY_ADD("TestProperties", "UtilTests"); 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /native/tools/run_coverage: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR=`dirname $0` 4 | [ -z "$MR4C_LOG4CXX_CONFIG" ] && MR4C_LOG4CXX_CONFIG=${DIR}/../conf/log4cxx.properties 5 | export MR4C_LOG4CXX_CONFIG 6 | $DIR/../exe/run_coverage $* 7 | -------------------------------------------------------------------------------- /native/tools/run_tests: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR=`dirname $0` 4 | [ -z "$MR4C_LOG4CXX_CONFIG" ] && MR4C_LOG4CXX_CONFIG=${DIR}/../conf/log4cxx.properties 5 | export MR4C_LOG4CXX_CONFIG 6 | $DIR/../exe/run_tests $* 7 | -------------------------------------------------------------------------------- /native/tools/test_local: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR=`dirname $0` 4 | [ -z "$MR4C_LOG4CXX_CONFIG" ] && MR4C_LOG4CXX_CONFIG=${DIR}/../conf/log4cxx.properties 5 | export MR4C_LOG4CXX_CONFIG 6 | $DIR/../exe/test_local $* 7 | -------------------------------------------------------------------------------- /remove_all: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Removes MR4C from /usr/local/mr4c 4 | 5 | set -e 6 | 7 | export MR4C_HOME=/usr/local/mr4c 8 | 9 | NATIVE_PR=/usr/local/mr4c/native/bin/pre_remove 10 | [ -f $NATIVE_PR ] && sudo $NATIVE_PR 11 | cd native 12 | sudo make remove 13 | cd .. 14 | 15 | GEO_PR=/usr/local/mr4c/geospatial/bin/pre_remove 16 | [ -f $GEO_PR ] && sudo $GEO_PR 17 | cd geospatial 18 | sudo make remove 19 | cd .. 20 | 21 | JAVA_PR=/usr/local/mr4c/java/bin/pre_remove 22 | [ -f $JAVA_PR ] && sudo $JAVA_PR 23 | cd java 24 | sudo ant remove 25 | cd .. 26 | 27 | 28 | -------------------------------------------------------------------------------- /test/bin/check_mr4cref_output: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | NN="hdfs://localhost:8020" 4 | sourcedir=$NN/user/$USER/mr4c_avgoutput 5 | destdir=/tmp/mr4c_avgoutput 6 | testfile="$destdir/avg_pixels.csv" 7 | reffile="ref/avg_pixels.csv" 8 | 9 | # check ref file 10 | if [ ! -f $reffile ]; then 11 | echo "ERROR: reference file $reffile not found. Please check your installation" 12 | exit 1 13 | fi 14 | 15 | # remove previous run 16 | if [ -d "$destdir" ]; then 17 | echo "removing $destdir" 18 | rm -rf $destdir 19 | fi 20 | 21 | # check for outputdir in HDFS 22 | hdfs dfs -test -d $sourcedir 23 | if [ $? -ne 0 ]; then 24 | echo "$sourcedir not found. Did you execute the test yet?" 25 | exit 1 26 | fi 27 | 28 | hdfs dfs -get $sourcedir $destdir 29 | if [ -d "$destdir" ]; then 30 | echo "Running diff against reference data" 31 | diff -q $reffile $testfile 32 | if [ $? -eq 0 ]; then 33 | echo "SUCCESS: MR4CRef output successfully validated" 34 | else 35 | echo "ERROR: MR4CRef not validated: $reffile $testfile" 36 | exit 1 37 | fi 38 | else 39 | echo "ERROR: $destdir not found" 40 | exit 1 41 | fi 42 | -------------------------------------------------------------------------------- /test/bin/configure_mr4cref_input: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | NN=hdfs://localhost:8020 3 | sourcedir="input" 4 | destdir=$NN/user/$USER/mr4c_input 5 | 6 | if [ -d "$sourcedir" ]; then 7 | hdfs dfs -rm -r $destdir 8 | hdfs dfs -put $sourcedir $destdir 9 | hdfs dfs -ls $destdir 10 | if [ $? -eq 0 ]; then 11 | echo "SUCCESS: MR4CRef input successfully loaded into HDFS" 12 | fi 13 | else 14 | echo "ERROR: $sourcedir not found" 15 | exit 1 16 | fi 17 | 18 | exit 0 -------------------------------------------------------------------------------- /test/bin/test_hadoop: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SERVICES=`ls /etc/init.d/hadoop-* |grep -v proxy` 4 | 5 | for s in $SERVICES 6 | do 7 | sudo service `basename $s` status 8 | if [ $? -ne 0 ]; then 9 | echo "ERROR: $s is not running. Starting:" 10 | sudo service $s start 11 | fi 12 | done 13 | -------------------------------------------------------------------------------- /test/bin/viewlog: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | job=$1 3 | d="/var/lib/hadoop-hdfs/cache/mapred/mapred/local/userlogs/${job}" 4 | 5 | for d in `sudo find $d -type d` 6 | do 7 | sudo cat $d/syslog 8 | done -------------------------------------------------------------------------------- /test/conf/reduce_algo_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "algoConfig" : { 3 | "inline" : { 4 | "name" : "avg_pixel_reduce", 5 | "type" : "NATIVEC", 6 | "artifact" : "histogram", 7 | "extras" : [ "openjpeg", "libopenjpeg.so.1" ], 8 | "dimensions" : [ 9 | { 10 | "name" : "NAME", 11 | "canSplit" : false 12 | } 13 | ] 14 | 15 | 16 | } 17 | }, 18 | "inputs" : { 19 | "intermediate_avg_pixels" : { 20 | "scheme" : "directory", 21 | "location" : "${avginputdir}" 22 | } 23 | }, 24 | 25 | "outputs" : { 26 | "avg_pixels" : { 27 | "scheme" : "binary", 28 | "location" : "${avgoutputdir}/avg_pixels.csv" 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /test/conf/site.json: -------------------------------------------------------------------------------- 1 | { 2 | "clusters" : { 3 | "local" : { 4 | "jobTracker" : "localhost:8032" 5 | ,"nameNode" : "hdfs://localhost:8020" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /test/input/4.j2k: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/test/input/4.j2k -------------------------------------------------------------------------------- /test/input/8-0-4-50-6-75.j2k: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/test/input/8-0-4-50-6-75.j2k -------------------------------------------------------------------------------- /test/input/912-white-720-others.j2k: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/test/input/912-white-720-others.j2k -------------------------------------------------------------------------------- /test/input/back-75-14-white.j2k: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/test/input/back-75-14-white.j2k -------------------------------------------------------------------------------- /test/openjpeg_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | VERSION="openjpeg_v1_4_sources_r697" 4 | FILE="${VERSION}.tgz" 5 | DIR="openjpeg" 6 | URL="https://openjpeg.googlecode.com/files/${FILE}" 7 | 8 | wget --no-check-certificate ${URL} 9 | 10 | if [ -f "$FILE" ]; then 11 | tar -xzf ${FILE} 12 | mv ${VERSION} ${DIR} 13 | cd ${DIR} 14 | ./configure --prefix=/usr/local 15 | make 16 | sudo make install 17 | cd - 18 | rm ${FILE} 19 | else 20 | echo "File ${FILE} was not downloaded from ${URL}" 21 | exit 1 22 | fi 23 | -------------------------------------------------------------------------------- /test/src/cpp/J2kDecoder.cpp: -------------------------------------------------------------------------------- 1 | #include "J2kDecoder.h" 2 | 3 | 4 | 5 | J2kImage::J2kImage( opj_image_t * image ) : m_image( image ) { 6 | } 7 | 8 | 9 | J2kImage::~J2kImage() { 10 | opj_image_destroy(m_image); 11 | } 12 | 13 | J2kImage::DimType J2kImage::getDimensions() const { 14 | opj_image_comp_t comp = m_image->comps[ 0 ]; 15 | return DimType( comp.w, comp.h ); 16 | } 17 | 18 | int * J2kImage::getPixels() const { 19 | return m_image->comps[ 0 ].data; 20 | } 21 | 22 | J2kImageDecoder::J2kImageDecoder() { 23 | opj_set_default_decoder_parameters(¶meters); 24 | 25 | dinfo = opj_create_decompress(CODEC_J2K); 26 | 27 | opj_setup_decoder(dinfo, ¶meters); 28 | } 29 | 30 | J2kImage * J2kImageDecoder::getImage( char * data, size_t len ) { 31 | opj_image_t *image; 32 | 33 | opj_cio_t *cio = opj_cio_open((opj_common_ptr)dinfo, 34 | reinterpret_cast(data), len); 35 | 36 | image = opj_decode(dinfo, cio); 37 | opj_cio_close(cio); 38 | return new J2kImage( image ); 39 | } 40 | 41 | J2kImageDecoder::~J2kImageDecoder() { 42 | opj_destroy_decompress(dinfo); 43 | } 44 | 45 | 46 | -------------------------------------------------------------------------------- /test/src/cpp/J2kDecoder.h: -------------------------------------------------------------------------------- 1 | #ifndef __MR4CREF_J2KDECODER_H__ 2 | #define __MR4CREF_J2KDECODER_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | class J2kImageDecoder; 9 | 10 | class J2kImage { 11 | opj_image_t * m_image; 12 | friend class J2kImageDecoder; 13 | 14 | protected: 15 | J2kImage( opj_image_t * image ); 16 | 17 | public: 18 | typedef std::pair DimType; 19 | 20 | ~J2kImage(); 21 | 22 | DimType getDimensions() const; 23 | 24 | int * getPixels() const; 25 | }; 26 | 27 | class J2kImageDecoder { 28 | 29 | opj_dparameters_t parameters; 30 | opj_dinfo_t * dinfo; 31 | 32 | public: 33 | J2kImageDecoder(); 34 | 35 | J2kImage * getImage( char * data, size_t len ); 36 | 37 | ~J2kImageDecoder(); 38 | 39 | }; 40 | 41 | 42 | 43 | #endif // __MR4CREF_J2KDECODER_H__ 44 | -------------------------------------------------------------------------------- /test/test/cpp/suites/Local.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | // local test runner that just dumps to the console 5 | int main( int argc, char* argv[]) 6 | { 7 | CPPUNIT_NS::TextTestRunner testrunner; 8 | testrunner.addTest( CPPUNIT_NS::TestFactoryRegistry::getRegistry().makeTest()); 9 | testrunner.run(); 10 | 11 | } 12 | 13 | 14 | -------------------------------------------------------------------------------- /test/test/cpp/tests/algo.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "histogram", 3 | "type" : "NATIVEC", 4 | "artifact" : "histogram" 5 | } 6 | -------------------------------------------------------------------------------- /test/test/cpp/tests/algo_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "algoConfig" : { 3 | "file" : "algo.json" 4 | }, 5 | "inputs" : { 6 | "images" : { 7 | "scheme" : "directory", 8 | "location" : "input", 9 | "srcConfig" : { 10 | "inline" : { 11 | "mapper" : { 12 | "pattern" : "${NAME}.j2k", 13 | "dimensions" : ["NAME"] 14 | }, 15 | "ignore" : true, 16 | "selfConfig" : false 17 | } 18 | } 19 | } 20 | }, 21 | 22 | "outputs" : { 23 | "histograms" : { 24 | "scheme" : "directory", 25 | "location" : "output", 26 | "srcConfig" : { 27 | "file" : "int_conf.json" 28 | } 29 | }, 30 | "intermediate_avg_pixels" : { 31 | "scheme" : "directory", 32 | "location" : "avg_output", 33 | "srcConfig" : { 34 | "file" : "int_conf.json" 35 | } 36 | } 37 | }, 38 | "params" : { 39 | "num_buckets" : "20" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /test/test/cpp/tests/input/4.j2k: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/test/test/cpp/tests/input/4.j2k -------------------------------------------------------------------------------- /test/test/cpp/tests/input/8-0-4-50-6-75.j2k: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/test/test/cpp/tests/input/8-0-4-50-6-75.j2k -------------------------------------------------------------------------------- /test/test/cpp/tests/input/912-white-720-others.j2k: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/test/test/cpp/tests/input/912-white-720-others.j2k -------------------------------------------------------------------------------- /test/test/cpp/tests/input/back-75-14-white.j2k: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/test/test/cpp/tests/input/back-75-14-white.j2k -------------------------------------------------------------------------------- /test/test/cpp/tests/int_conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "mapper" : { 3 | "pattern" : "${NAME}.csv", 4 | "dimensions" : ["NAME"] 5 | }, 6 | "ignore" : true 7 | } 8 | 9 | -------------------------------------------------------------------------------- /test/test/cpp/tests/reduce_algo.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "avg_pixel_reduce", 3 | "type" : "NATIVEC", 4 | "artifact" : "histogram", 5 | "dimensions" : [ 6 | { 7 | "name" : "NAME", 8 | "canSplit" : false 9 | } 10 | ] 11 | 12 | 13 | } 14 | -------------------------------------------------------------------------------- /test/test/cpp/tests/reduce_algo_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "algoConfig" : { 3 | "file" : "reduce_algo.json" 4 | }, 5 | "inputs" : { 6 | "intermediate_avg_pixels" : { 7 | "scheme" : "directory", 8 | "location" : "avg_output" 9 | } 10 | }, 11 | 12 | "outputs" : { 13 | "avg_pixels" : { 14 | "scheme" : "binary", 15 | "location" : "output/avg_pixels.csv" 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /test/test_mr4c.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | echo "***********************" 6 | echo "Running MR4C Test" 7 | echo "***********************" 8 | 9 | #add dist to LD_LIBRARY_PATH 10 | export LD_LIBRARY_PATH=./dist:$LD_LIBRARY_PATH 11 | 12 | bin/configure_mr4cref_input 13 | bin/run_mr4cref 14 | bin/check_mr4cref_output 15 | -------------------------------------------------------------------------------- /tutorial/clean_all: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | pushd example1_HelloWorld 6 | make clean 7 | popd 8 | 9 | pushd example2_IO 10 | make clean 11 | popd 12 | 13 | pushd example3_Dimensions 14 | make clean 15 | popd 16 | 17 | pushd example4_ExternalLib 18 | make clean 19 | popd 20 | 21 | pushd example5_json 22 | make clean 23 | popd 24 | 25 | pushd example6_ImageAlgo 26 | make clean 27 | popd 28 | 29 | pushd example7_yarn 30 | make clean 31 | popd 32 | 33 | pushd example9_RandomAccess 34 | make clean 35 | popd 36 | 37 | -------------------------------------------------------------------------------- /tutorial/example1_HelloWorld/helloworld.json: -------------------------------------------------------------------------------- 1 | { 2 | "algoConfig" : { 3 | "inline" : { 4 | "name" : "HelloWorld", 5 | "type" : "NATIVEC", 6 | "artifact" : "HelloWorld" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /tutorial/example1_HelloWorld/helloworld.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/lib 4 | mr4c helloworld.json 5 | -------------------------------------------------------------------------------- /tutorial/example1_HelloWorld/makefile: -------------------------------------------------------------------------------- 1 | CC=g++ 2 | SRC_DIR =./src 3 | LIB_DIR = ./lib 4 | OBJ_DIR = ./objs 5 | 6 | SRCS = $(shell find $(SRC_DIR) -name *.cpp) 7 | OBJS = $(patsubst $(SRC_DIR)%.cpp, $(OBJ_DIR)%.o, $(SRCS) ) 8 | 9 | MR4C_HOME ?= /usr/local/mr4c 10 | MR4CINC = $(MR4C_HOME)/native/include 11 | MR4CLIB = $(MR4C_HOME)/native/lib 12 | 13 | #default target 14 | all: clean init compile lib 15 | 16 | init: 17 | mkdir -p $(OBJ_DIR) 18 | mkdir -p $(LIB_DIR) 19 | 20 | compile: ./src/helloworld.cpp 21 | $(CC) -c -I$(MR4CINC) -o ./objs/helloworld.o -fPIC -Wall -std=c++0x ./src/helloworld.cpp 22 | 23 | lib: ./objs/helloworld.o 24 | $(CC) -L$(MR4CLIB) -lmr4c -rdynamic -shared -fPIC -std=c++0x -o ./lib/libHelloWorld.so ./objs/helloworld.o 25 | 26 | clean: 27 | rm -rf $(OBJ_DIR) 28 | rm -rf $(LIB_DIR) 29 | 30 | 31 | .PHONY: clean, init, compile, lib, all 32 | -------------------------------------------------------------------------------- /tutorial/example2_IO/README.md: -------------------------------------------------------------------------------- 1 | #MR4C Change Image Example 2 | 3 | 4 | ##Description 5 | 6 | This example application illustrates simple input/output using MR4C. 7 | 8 | Contact mr4c@googlegroups.com with any question or comments. 9 | 10 | ##Build 11 | Navigate to the example2_IO folder and run the following commands: 12 | 13 | make 14 | 15 | ##Running bbChangeImage 16 | 17 | To run bbChangeImage execute the following command from the cloned folder: 18 | 19 | ./changeimage.sh 20 | 21 | This will input the file from the input folder, execute the algorithm, 22 | and output the changed values into a new file in the output folder. 23 | The input and output characters are also printed to stdout in the section of output 24 | between the **NATIVE_OUTPUT** header/footer. 25 | -------------------------------------------------------------------------------- /tutorial/example2_IO/changeimage.json: -------------------------------------------------------------------------------- 1 | { "algoConfig" : { "inline" : { "artifact" : "ChangeImage", 2 | "name" : "changeimage", 3 | "type" : "NATIVEC" 4 | } }, 5 | "inputs" : { "imageIn" : { "location" : "input/test.bin", 6 | "scheme" : "binary" 7 | } }, 8 | "outputs" : { "imageOut" : { "location" : "output/testOut.bin", 9 | "scheme" : "binary" 10 | } } 11 | } 12 | -------------------------------------------------------------------------------- /tutorial/example2_IO/changeimage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/lib 4 | mr4c ./changeimage.json 5 | -------------------------------------------------------------------------------- /tutorial/example2_IO/input/test.bin: -------------------------------------------------------------------------------- 1 | 0123456789abcdefghijklmnopqrstuvxyz 2 | -------------------------------------------------------------------------------- /tutorial/example2_IO/makefile: -------------------------------------------------------------------------------- 1 | CC=g++ 2 | SRC_DIR =./src 3 | LIB = ./lib 4 | OBJ_DIR = ./objs 5 | 6 | SRCS = $(shell find $(SRC_DIR) -name *.cpp) 7 | OBJS = $(patsubst $(SRC_DIR)%.cpp, $(OBJ_DIR)%.o, $(SRCS) ) 8 | 9 | MR4C_HOME ?= /usr/local/mr4c 10 | MR4CINC = $(MR4C_HOME)/native/include 11 | MR4CLIB = $(MR4C_HOME)/native/lib 12 | 13 | #default target 14 | all: clean init compile lib 15 | 16 | clean: 17 | rm -rf $(OBJ_DIR)/* 18 | rm -rf $(LIB)/* 19 | rm -rf $(OBJ_DIR) 20 | rm -rf $(LIB) 21 | 22 | init: 23 | mkdir -p $(OBJ_DIR) 24 | mkdir -p $(LIB) 25 | 26 | compile: ./src/changeimage.cpp 27 | $(CC) -c -I$(MR4CINC) -o ./objs/changeimage.o -fPIC -Wall -std=c++0x ./src/changeimage.cpp 28 | 29 | lib: ./objs/changeimage.o 30 | $(CC) -L$(MR4CLIB) -lmr4c -rdynamic -shared -fPIC -std=c++0x -o ./lib/libChangeImage.so ./objs/changeimage.o 31 | 32 | 33 | .PHONY: clean, lib, init, all 34 | -------------------------------------------------------------------------------- /tutorial/example2_IO/output/testOut.bin: -------------------------------------------------------------------------------- 1 | 123456789:bcdefghijklmnopqrstuvwyz{ 2 | -------------------------------------------------------------------------------- /tutorial/example3_Dimensions/README.md: -------------------------------------------------------------------------------- 1 | #Dimensions 2 | 3 | This example builds on the IO example but instead of specifying files for input and output this algorithm reads/writes all files to/from a folder/URI using dimensions. This makes the algrithm very flexible and scale to very large datasets with a lot of elements. 4 | 5 | ###Configuration: 6 | We use the mappers in dimension.json to configure our dataset dimensions: 7 | 8 | "mapper" : { 9 | "pattern" : "test_${NAME}.bin", 10 | "dimensions" : ["NAME"] 11 | }, 12 | 13 | This construction allows for any files in the input or output folder that fit the pattern to be interpreted as elements in the NAME dimension. In out algorithm we can use these dimensions as follows: 14 | 15 | //get keyspace elements (files in directory by dimension) 16 | Keyspace keyspace = data.getKeyspace(); 17 | std::vector names = keyspace.getKeyspaceDimension(DataKeyDimension("NAME")).getElements(); 18 | 19 | //iterate through the elements in the NAMES dimension 20 | for ( size_t i=0; i < names.size() ; i++ ) { 21 | 22 | //get key 23 | DataKey myKey = names[i]; 24 | 25 | //open file for each key 26 | DataFile* myFile = input->getDataFile(myKey); 27 | } 28 | 29 | -------------------------------------------------------------------------------- /tutorial/example3_Dimensions/dimensions.json: -------------------------------------------------------------------------------- 1 | {"algoConfig" : { 2 | "inline" : { 3 | "artifact" : "Dimensions", 4 | "name" : "dimensions", 5 | "type" : "NATIVEC", 6 | "dimensions" : [ 7 | { 8 | "name" : "NAME", 9 | "canSplit" : true 10 | } 11 | ] 12 | } 13 | }, 14 | "inputs" : { 15 | "imagesIn" : { 16 | "location" : "input/", 17 | "scheme" : "directory", 18 | "srcConfig" : { 19 | "inline" : { 20 | "mapper" : { 21 | "pattern" : "test_${NAME}.bin", 22 | "dimensions" : ["NAME"] 23 | }, 24 | "selfConfig" : false 25 | } 26 | } 27 | } 28 | }} 29 | 30 | -------------------------------------------------------------------------------- /tutorial/example3_Dimensions/dimensions.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/lib 4 | mr4c ./dimensions.json 5 | -------------------------------------------------------------------------------- /tutorial/example3_Dimensions/input/test_1.bin: -------------------------------------------------------------------------------- 1 | 0123456789abcdefghijklmnopqrstuvwxyz 2 | 3 | -------------------------------------------------------------------------------- /tutorial/example3_Dimensions/input/test_2.bin: -------------------------------------------------------------------------------- 1 | zyxwvutsrqponmlkjihgfedcba9876543210 2 | 3 | -------------------------------------------------------------------------------- /tutorial/example3_Dimensions/makefile: -------------------------------------------------------------------------------- 1 | CC=g++ 2 | SRC_DIR =./src 3 | LIB = ./lib 4 | OBJ_DIR = ./objs 5 | 6 | SRCS = $(shell find $(SRC_DIR) -name *.cpp) 7 | OBJS = $(patsubst $(SRC_DIR)%.cpp, $(OBJ_DIR)%.o, $(SRCS) ) 8 | 9 | MR4C_HOME ?= /usr/local/mr4c 10 | MR4CINC = $(MR4C_HOME)/native/include 11 | MR4CLIB = $(MR4C_HOME)/native/lib 12 | 13 | #default target 14 | all: clean init compile lib 15 | 16 | clean: 17 | rm -rf $(OBJ_DIR)/* 18 | rm -rf $(LIB)/* 19 | rm -rf $(OBJ_DIR) 20 | rm -rf $(LIB) 21 | 22 | init: 23 | mkdir -p $(OBJ_DIR) 24 | mkdir -p $(LIB) 25 | 26 | compile: ./src/dimensions.cpp 27 | $(CC) -c -I$(MR4CINC) -o ./objs/dimensions.o -fPIC -Wall -std=c++0x ./src/dimensions.cpp 28 | 29 | lib: ./objs/dimensions.o 30 | $(CC) -L$(MR4CLIB) -lmr4c -rdynamic -shared -fPIC -std=c++0x -o ./lib/libDimensions.so ./objs/dimensions.o 31 | 32 | 33 | .PHONY: clean, lib, init, all 34 | -------------------------------------------------------------------------------- /tutorial/example4_ExternalLib/README.md: -------------------------------------------------------------------------------- 1 | ##Introduction 2 | The ExternalLib example is our first example utilizing a third party library in addition to our own algorithm. 3 | Additionally, we are using GDAL to read a geotiff and report some of the important metadata. 4 | This method can be used to convert a MR4C::Dataset into a GDALDataset 5 | allowing the algorithm to have access to all of the GDAL library classes. 6 | You will notice that this requires an extra step of creating a gdal virtual file, 7 | which is a buffer in memory that can be used in the place of any path references required in third party libraries. MR4C keeps track of datasets elements as raw char pointers without traditional file system paths. 8 | 9 | -------------------------------------------------------------------------------- /tutorial/example4_ExternalLib/imageinfo.json: -------------------------------------------------------------------------------- 1 | { "algoConfig" : { "inline" : { "artifact" : "ImageInfo", 2 | "name" : "imageinfo", 3 | "type" : "NATIVEC", 4 | "extras" : ["gdal"] 5 | } }, 6 | "inputs" : { "imageIn" : { "location" : "input/sample.tif", 7 | "scheme" : "binary" 8 | } } 9 | } 10 | -------------------------------------------------------------------------------- /tutorial/example4_ExternalLib/imageinfo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/lib 4 | mr4c ./imageinfo.json 5 | -------------------------------------------------------------------------------- /tutorial/example4_ExternalLib/input/sample.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/tutorial/example4_ExternalLib/input/sample.tif -------------------------------------------------------------------------------- /tutorial/example4_ExternalLib/makefile: -------------------------------------------------------------------------------- 1 | CC=g++ 2 | SRC_DIR =./src 3 | LIB = ./lib 4 | OBJ_DIR = ./objs 5 | 6 | SRCS = $(shell find $(SRC_DIR) -name *.cpp) 7 | OBJS = $(patsubst $(SRC_DIR)%.cpp, $(OBJ_DIR)%.o, $(SRCS) ) 8 | 9 | MR4C_HOME ?= /usr/local/mr4c 10 | MR4CINC = $(MR4C_HOME)/native/include 11 | MR4CLIB = $(MR4C_HOME)/native/lib 12 | 13 | GDALINC = /usr/local/include 14 | GDALLIB = /usr/local/lib 15 | 16 | #default target 17 | all: clean init compile lib 18 | 19 | clean: 20 | rm -rf $(OBJ_DIR)/* 21 | rm -rf $(LIB)/* 22 | rm -rf $(OBJ_DIR) 23 | rm -rf $(LIB) 24 | 25 | init: 26 | mkdir -p $(OBJ_DIR) 27 | mkdir -p $(LIB) 28 | 29 | compile: ./src/imageinfo.cpp 30 | $(CC) -c -I$(MR4CINC) -I$(GDALINC) -o ./objs/imageinfo.o -fPIC -Wall -std=c++0x ./src/imageinfo.cpp 31 | 32 | lib: ./objs/imageinfo.o 33 | $(CC) -L$(MR4CLIB) -lmr4c -L$(GDALLIB) -lgdal -rdynamic -shared -fPIC -std=c++0x -o ./lib/libImageInfo.so ./objs/imageinfo.o 34 | 35 | 36 | .PHONY: clean, lib, init, all 37 | -------------------------------------------------------------------------------- /tutorial/example5_json/json.json: -------------------------------------------------------------------------------- 1 | { 2 | "algoConfig": { 3 | "inline": { 4 | "artifact": "JSON", 5 | "dimensions": [ 6 | { 7 | "canSplit": true, 8 | "name": "NAME" 9 | } 10 | ], 11 | "extras": [ 12 | "jansson" 13 | ], 14 | "name": "json", 15 | "type": "NATIVEC" 16 | } 17 | }, 18 | "inputs": { 19 | "metadataIn": { 20 | "location": "input/", 21 | "scheme": "directory", 22 | "srcConfig": { 23 | "inline": { 24 | "mapper": { 25 | "dimensions": [ 26 | "NAME" 27 | ], 28 | "pattern": "${NAME}_metadata.txt" 29 | }, 30 | "selfConfig": false 31 | } 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /tutorial/example5_json/json.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/lib 4 | mr4c ./json.json 5 | -------------------------------------------------------------------------------- /tutorial/example5_json/makefile: -------------------------------------------------------------------------------- 1 | CC=g++ 2 | SRC_DIR =./src 3 | LIB = ./lib 4 | OBJ_DIR = ./objs 5 | 6 | SRCS = $(shell find $(SRC_DIR) -name *.cpp) 7 | OBJS = $(patsubst $(SRC_DIR)%.cpp, $(OBJ_DIR)%.o, $(SRCS) ) 8 | 9 | MR4C_HOME ?= /usr/local/mr4c 10 | MR4CINC = $(MR4C_HOME)/native/include 11 | MR4CLIB = $(MR4C_HOME)/native/lib 12 | JANSSONINC = /usr/include 13 | JANSSONLIB = /usr/lib 14 | #default target 15 | all: clean init compile lib 16 | 17 | clean: 18 | rm -rf $(OBJ_DIR)/* 19 | rm -rf $(LIB)/* 20 | rm -rf $(OBJ_DIR) 21 | rm -rf $(LIB) 22 | 23 | init: 24 | mkdir -p $(OBJ_DIR) 25 | mkdir -p $(LIB) 26 | 27 | compile: ./src/json.cpp 28 | $(CC) -c -I$(MR4CINC) -o ./objs/json.o -fPIC -Wall -std=c++0x ./src/json.cpp 29 | 30 | lib: ./objs/json.o 31 | $(CC) -I$JANSSONINC -L$(JANSSONINC) -ljansson -L$(MR4CLIB) -lmr4c -rdynamic -shared -fPIC -std=c++0x -o ./lib/libJSON.so ./objs/json.o 32 | 33 | 34 | .PHONY: clean, lib, init, all 35 | -------------------------------------------------------------------------------- /tutorial/example6_ImageAlgo/README.md: -------------------------------------------------------------------------------- 1 | ##Introduction 2 | This example builds on everything we learned in the previous examples: 3 | 4 | 1. Read in several files from a directory 5 | 2. Organize them by dimension 6 | 3. Use gdal to read the geotiff format files into virtual files in memory 7 | 4. Run a basic image content algorithm to classify each pixel in the buffer as ground, vegetation, or water 8 | 5. write out a gdal geotiff as a virtual file 9 | 6. Use MR4C dimensions to write output files from memory 10 | 11 | 12 | ##References 13 | 14 | - Please refer to the [GDAL documentation](http://www.gdal.org/cpl__vsi_8h.html) for more info on how to use virtual files. 15 | - Please download and build [GDAL from source](http://trac.osgeo.org/gdal/wiki/BuildHints). 16 | -------------------------------------------------------------------------------- /tutorial/example6_ImageAlgo/content.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/lib 4 | mr4c ./content.json 5 | -------------------------------------------------------------------------------- /tutorial/example6_ImageAlgo/input/s00001_20140209_145912_mul_d2_0005.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/tutorial/example6_ImageAlgo/input/s00001_20140209_145912_mul_d2_0005.tif -------------------------------------------------------------------------------- /tutorial/example6_ImageAlgo/input/s00001_20140213_071323_mul_d2_0005.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/tutorial/example6_ImageAlgo/input/s00001_20140213_071323_mul_d2_0005.tif -------------------------------------------------------------------------------- /tutorial/example6_ImageAlgo/input/s01_20140214T071847Z_ms_d2_0005.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/tutorial/example6_ImageAlgo/input/s01_20140214T071847Z_ms_d2_0005.tif -------------------------------------------------------------------------------- /tutorial/example6_ImageAlgo/makefile: -------------------------------------------------------------------------------- 1 | CC=g++ 2 | SRC_DIR =./src 3 | LIB=./lib 4 | OBJ_DIR=./objs 5 | 6 | 7 | MR4C_HOME ?= /usr/local/mr4c 8 | MR4CINC = $(MR4C_HOME)/native/include 9 | MR4CGEOINC = $(MR4C_HOME)/geospatial/include 10 | MR4CLIB = $(MR4C_HOME)/native/lib 11 | MR4CGEOLIB = $(MR4C_HOME)/geospatial/lib 12 | 13 | GDALINC = /usr/local/include 14 | GDALLIB = /usr/local/lib 15 | 16 | #default target 17 | all: clean init compile lib 18 | 19 | clean: 20 | rm -rf $(LIB) 21 | rm -rf $(OBJ_DIR) 22 | 23 | init: 24 | mkdir -p $(LIB) 25 | mkdir -p $(OBJ_DIR) 26 | 27 | compile: ./src/content.cpp 28 | $(CC) -c -I$(MR4CINC) -I$(MR4CGEOINC) -I$(GDALINC) -fPIC -Wall -std=c++0x -o $(OBJ_DIR)/content.o $(SRC_DIR)/content.cpp 29 | 30 | lib: $(OBJ_DIR)/content.o 31 | $(CC) -L$(MR4CLIB) -L$(MR4CGEOLIB) -L$(GDALLIB) -rdynamic -shared -fPIC -o $(LIB)/libContent.so $(OBJ_DIR)/*.o -lmr4c -lmr4cgeo -lgdal 32 | 33 | 34 | .PHONY: clean, lib, init, all 35 | -------------------------------------------------------------------------------- /tutorial/example6_ImageAlgo/output/s00001_20140209_145912_mul_d2_0005_class.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/tutorial/example6_ImageAlgo/output/s00001_20140209_145912_mul_d2_0005_class.tif -------------------------------------------------------------------------------- /tutorial/example6_ImageAlgo/output/s00001_20140213_071323_mul_d2_0005_class.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/tutorial/example6_ImageAlgo/output/s00001_20140213_071323_mul_d2_0005_class.tif -------------------------------------------------------------------------------- /tutorial/example6_ImageAlgo/output/s01_20140214T071847Z_ms_d2_0005_class.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/tutorial/example6_ImageAlgo/output/s01_20140214T071847Z_ms_d2_0005_class.tif -------------------------------------------------------------------------------- /tutorial/example7_yarn/input/s01_20140409T084855Z_ms_d1_0001.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/tutorial/example7_yarn/input/s01_20140409T084855Z_ms_d1_0001.tif -------------------------------------------------------------------------------- /tutorial/example7_yarn/input/s01_20140409T084855Z_ms_d1_0002.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/tutorial/example7_yarn/input/s01_20140409T084855Z_ms_d1_0002.tif -------------------------------------------------------------------------------- /tutorial/example7_yarn/input/s01_20140409T084855Z_ms_d1_0003.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/tutorial/example7_yarn/input/s01_20140409T084855Z_ms_d1_0003.tif -------------------------------------------------------------------------------- /tutorial/example7_yarn/input/s01_20140409T084855Z_ms_d1_0004.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/tutorial/example7_yarn/input/s01_20140409T084855Z_ms_d1_0004.tif -------------------------------------------------------------------------------- /tutorial/example7_yarn/input/s01_20140409T084855Z_ms_d1_0005.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/tutorial/example7_yarn/input/s01_20140409T084855Z_ms_d1_0005.tif -------------------------------------------------------------------------------- /tutorial/example7_yarn/makefile: -------------------------------------------------------------------------------- 1 | CC=g++ 2 | SRC_DIR =./src 3 | LIB=./lib 4 | OBJ_DIR=./objs 5 | 6 | 7 | MR4C_HOME ?= /usr/local/mr4c 8 | MR4CINC = $(MR4C_HOME)/native/include 9 | MR4CGEOINC = $(MR4C_HOME)/geospatial/include 10 | MR4CLIB = $(MR4C_HOME)/native/lib 11 | 12 | GDALINC = /usr/local/include 13 | GDALLIB = /usr/local/lib 14 | 15 | #default target 16 | all: clean init compile libMap libReduce 17 | 18 | clean: 19 | rm -rf $(LIB) 20 | rm -rf $(OBJ_DIR) 21 | 22 | init: 23 | mkdir -p $(LIB) 24 | mkdir -p $(OBJ_DIR) 25 | 26 | compile: $(SRC_DIR)/map.cpp $(SRC_DIR)/reduce.cpp 27 | $(CC) -c -I$(MR4CINC) -I$(MR4CGEOINC) -I$(GDALINC) -I$(SRC_DIR) -fPIC -Wall -std=c++0x -o $(OBJ_DIR)/map.o $(SRC_DIR)/map.cpp 28 | $(CC) -c -I$(MR4CINC) -I$(MR4CGEOINC) -I$(SRC_DIR) -fPIC -Wall -std=c++0x -o $(OBJ_DIR)/reduce.o $(SRC_DIR)/reduce.cpp 29 | 30 | libMap: $(OBJ_DIR)/map.o 31 | $(CC) -L$(MR4CLIB) -L$(MR4CGEOLIB) -L$(GDALLIB) -rdynamic -shared -fPIC $(OBJ_DIR)/map.o -o $(LIB)/libMap.so -lmr4c -lmr4cgeo -lgdal 32 | 33 | libReduce: $(OBJ_DIR)/reduce.o 34 | $(CC) -L$(MR4CLIB) -rdynamic -shared -fPIC -o $(LIB)/libReduce.so $(OBJ_DIR)/reduce.o -lmr4c 35 | 36 | .PHONY: clean, libMap, libReduce, init, all 37 | -------------------------------------------------------------------------------- /tutorial/example7_yarn/mapReduce.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export MR4C_SITE=./site.json; 4 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/lib:/usr/local/lib 5 | 6 | set -e 7 | 8 | #stage data 9 | 10 | if ! hadoop fs -test -d skysatIn 11 | then 12 | hadoop fs -mkdir skysatIn 13 | hadoop fs -mkdir skysatReduce 14 | hadoop fs -put ./input/* skysatIn 15 | fi 16 | 17 | #run mr4c 18 | #mapper 19 | mr4c_hadoop ./map.json -Htasks=2 -Hcores.min=1 -Hcores.max=2 -Hmemory.min=1024 -Hmemory.max=2048 20 | 21 | #pause between map and reduce 22 | sleep 1 23 | 24 | #reducer 25 | mr4c_hadoop ./reduce.json -Htasks=1 -Hcores.min=1 -Hcores.max=2 -Hmemory.min=1024 -Hmemory.max=2048 26 | -------------------------------------------------------------------------------- /tutorial/example7_yarn/reduce.json: -------------------------------------------------------------------------------- 1 | { 2 | "algoConfig" : 3 | { 4 | "inline" : 5 | { 6 | "artifact" : "Reduce", 7 | "name" : "reduce", 8 | "type" : "NATIVEC", 9 | "dimensions" : [ 10 | { 11 | "name" : "NAME", 12 | "canSplit" : false 13 | } 14 | ] 15 | } 16 | }, 17 | 18 | "inputs" : 19 | { 20 | "hist" : 21 | { 22 | "location" : "hdfs://localhost:8020/user/mr4c/skysatReduce", 23 | "scheme" : "directory", 24 | "srcConfig" : 25 | { 26 | "inline" : 27 | { 28 | "ignore" : true, 29 | "selfConfig" : true 30 | } 31 | } 32 | } 33 | }, 34 | 35 | "outputs" : 36 | { 37 | "summaryOut" : 38 | { 39 | "location" : "hdfs://localhost:8020/user/mr4c/summary.csv", 40 | "scheme" : "binary" 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /tutorial/example7_yarn/site.json: -------------------------------------------------------------------------------- 1 | { 2 | "clusters" : { 3 | "local" : { 4 | "jobTracker" : "localhost:8032" 5 | ,"nameNode" : "hdfs://localhost:8020" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /tutorial/example8_mbtiles/conf/algo.properties: -------------------------------------------------------------------------------- 1 | name=TilerExample 2 | description=Description 3 | version=1 4 | #minZoom=10 5 | #maxZoom=15 6 | -------------------------------------------------------------------------------- /tutorial/example8_mbtiles/conf/exe.json: -------------------------------------------------------------------------------- 1 | { 2 | "algoConfig" : { 3 | "inline" : { 4 | "name" : "mbtiler", 5 | "type" : "NATIVEC", 6 | "artifact" : "mr4cgeo" 7 | } 8 | }, 9 | 10 | "inputs" : { 11 | "input" : { 12 | "scheme" : "binary", 13 | "location" : "${input.loc}" 14 | } 15 | }, 16 | 17 | "outputs" : { 18 | "output" : { 19 | "scheme" : "mbtiles", 20 | "location" : "${output.loc}" 21 | } 22 | } 23 | 24 | } 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /tutorial/example8_mbtiles/conf/runtime.properties: -------------------------------------------------------------------------------- 1 | input.loc=hdfs://localhost:8020/user/mr4c/example8/input.tif 2 | output.loc=hdfs://localhost:8020/user/mr4c/example8/output.mbtiles 3 | -------------------------------------------------------------------------------- /tutorial/example8_mbtiles/input.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/tutorial/example8_mbtiles/input.tif -------------------------------------------------------------------------------- /tutorial/example8_mbtiles/run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ ! `hdfs dfs -test -d example8` ]; then hdfs dfs -mkdir /user/mr4c/example8 ; fi 4 | if [ ! `hdfs dfs -test -e example8/input.tif` ]; then hdfs dfs -put ./input.tif example8 ; fi 5 | 6 | mr4c conf/exe.json -R=conf/runtime.properties -A=conf/algo.properties $* 7 | 8 | if [ ! `hdfs dfs -test -e /user/mr4c/example8/output.mbtiles` ]; 9 | then 10 | hdfs dfs -get /user/mr4c/example8/output.mbtiles ./ 11 | rm -rf ./mr4ctemp* 12 | hdfs dfs -rm -r /user/mr4c/example8 13 | fi 14 | 15 | -------------------------------------------------------------------------------- /tutorial/example9_RandomAccess/RandomAccess.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export MR4C_SITE=./site.json; 4 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/lib:/usr/local/lib 5 | 6 | #stage data 7 | if [ ! `hadoop fs -test -d skysatIn` ] ; 8 | then 9 | hadoop fs -mkdir skysatIn 10 | hadoop fs -mkdir skysatOut 11 | hadoop fs -put ./input/* skysatIn 12 | fi 13 | #run mr4c 14 | mr4c_hadoop ./RandomAccess.json -Htasks=2 -Hcores.min=1 -Hcores.max=2 -Hmemory.min=1024 -Hmemory.max=2048 15 | 16 | -------------------------------------------------------------------------------- /tutorial/example9_RandomAccess/input/s01_20140409T084855Z_ms_d1_0001.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/tutorial/example9_RandomAccess/input/s01_20140409T084855Z_ms_d1_0001.tif -------------------------------------------------------------------------------- /tutorial/example9_RandomAccess/input/s01_20140409T084855Z_ms_d1_0002.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/tutorial/example9_RandomAccess/input/s01_20140409T084855Z_ms_d1_0002.tif -------------------------------------------------------------------------------- /tutorial/example9_RandomAccess/input/s01_20140409T084855Z_ms_d1_0003.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/tutorial/example9_RandomAccess/input/s01_20140409T084855Z_ms_d1_0003.tif -------------------------------------------------------------------------------- /tutorial/example9_RandomAccess/input/s01_20140409T084855Z_ms_d1_0004.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/tutorial/example9_RandomAccess/input/s01_20140409T084855Z_ms_d1_0004.tif -------------------------------------------------------------------------------- /tutorial/example9_RandomAccess/input/s01_20140409T084855Z_ms_d1_0005.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/mr4c/fe367fac1d634fe5cea477e14ba39d0df630255f/tutorial/example9_RandomAccess/input/s01_20140409T084855Z_ms_d1_0005.tif -------------------------------------------------------------------------------- /tutorial/example9_RandomAccess/makefile: -------------------------------------------------------------------------------- 1 | CC=g++ 2 | SRC_DIR=./src 3 | LIB=./lib 4 | OBJ_DIR=./objs 5 | TARGET=RandomAccess 6 | 7 | MR4C_HOME ?= /usr/local/mr4c 8 | MR4CINC = $(MR4C_HOME)/native/include 9 | MR4CLIB = $(MR4C_HOME)/native/lib 10 | 11 | #default target 12 | all: clean init compile lib 13 | 14 | clean: 15 | rm -rf $(LIB) 16 | rm -rf $(OBJ_DIR) 17 | 18 | init: 19 | mkdir -p $(LIB) 20 | mkdir -p $(OBJ_DIR) 21 | 22 | compile: 23 | $(CC) -c -I$(MR4CINC) -I$(MR4CGEOINC) -I$(SRC_DIR) -fPIC -Wall -std=c++0x -o $(OBJ_DIR)/$(TARGET).o $(SRC_DIR)/$(TARGET).cpp 24 | 25 | lib: $(OBJ_DIR)/$(TARGET).o 26 | $(CC) -L$(MR4CLIB) -L$(MR4CGEOLIB) -L$(GDALLIB) -rdynamic -shared -fPIC $(OBJ_DIR)/$(TARGET).o -o $(LIB)/lib$(TARGET).so -lmr4c 27 | 28 | .PHONY: clean, compile,lib, init, all 29 | -------------------------------------------------------------------------------- /tutorial/example9_RandomAccess/site.json: -------------------------------------------------------------------------------- 1 | { 2 | "clusters" : { 3 | "local" : { 4 | "jobTracker" : "localhost:8032" 5 | ,"nameNode" : "hdfs://localhost:8020" 6 | } 7 | } 8 | } 9 | --------------------------------------------------------------------------------