├── .DS_Store ├── .buildkite ├── branches.sh ├── dra-workflow.trigger.sh ├── dra-workflow.yml ├── dra.sh ├── hooks │ └── pre-command ├── pipeline.py ├── pull-requests.json └── tests.trigger.sh ├── .ci └── java-versions.properties ├── .github ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── docs-build.yml │ └── docs-cleanup.yml ├── .gitignore ├── LICENSE.txt ├── NOTICE.txt ├── README.md ├── build.gradle ├── buildSrc ├── build.gradle ├── esh-version.properties ├── esh-version.properties~ └── src │ └── main │ ├── groovy │ └── org │ │ └── elasticsearch │ │ └── hadoop │ │ └── gradle │ │ ├── BaseBuildPlugin.groovy │ │ ├── BuildPlugin.groovy │ │ ├── EshVersionProperties.groovy │ │ ├── IntegrationBuildPlugin.groovy │ │ ├── RootBuildPlugin.groovy │ │ ├── buildtools │ │ ├── AntFixture.groovy │ │ ├── AntFixtureStop.groovy │ │ └── AntTask.groovy │ │ ├── fixture │ │ ├── ElasticsearchFixturePlugin.groovy │ │ └── hadoop │ │ │ ├── ConfigFormats.groovy │ │ │ ├── HadoopClusterFormationTasks.groovy │ │ │ ├── HadoopFixturePlugin.groovy │ │ │ ├── InstanceInfo.groovy │ │ │ ├── RoleDescriptor.groovy │ │ │ ├── ServiceDescriptor.groovy │ │ │ ├── SetupTaskFactory.groovy │ │ │ ├── conf │ │ │ ├── EndProcessConfiguration.groovy │ │ │ ├── HadoopClusterConfiguration.groovy │ │ │ ├── InstanceConfiguration.groovy │ │ │ ├── ProcessConfiguration.groovy │ │ │ ├── RoleConfiguration.groovy │ │ │ ├── ServiceConfiguration.groovy │ │ │ └── SettingsContainer.groovy │ │ │ ├── services │ │ │ ├── HadoopServiceDescriptor.groovy │ │ │ ├── HiveServiceDescriptor.groovy │ │ │ └── SparkYarnServiceDescriptor.groovy │ │ │ └── tasks │ │ │ ├── AbstractClusterTask.groovy │ │ │ ├── DfsCopy.groovy │ │ │ ├── HadoopMRJob.groovy │ │ │ ├── HiveBeeline.groovy │ │ │ └── SparkApp.groovy │ │ └── util │ │ ├── CiUtils.groovy │ │ └── ObjectUtil.groovy │ ├── java │ └── org │ │ └── elasticsearch │ │ └── hadoop │ │ └── gradle │ │ ├── buildtools │ │ ├── ConcatFilesTask.java │ │ ├── DependenciesInfoPlugin.java │ │ ├── DependenciesInfoTask.java │ │ ├── DependencyLicensesTask.java │ │ ├── Fixture.java │ │ ├── LazyFileOutputStream.java │ │ ├── LicenseAnalyzer.java │ │ ├── LicenseHeadersTask.java │ │ ├── LoggedExec.java │ │ ├── UpdateShasTask.java │ │ ├── Util.java │ │ ├── info │ │ │ ├── BuildParams.java │ │ │ ├── GlobalBuildInfoPlugin.java │ │ │ └── JavaHome.java │ │ └── package-info.java │ │ ├── scala │ │ └── SparkVariantPlugin.java │ │ └── util │ │ ├── ConfigureUtil.java │ │ ├── Resources.java │ │ └── WaitForURL.java │ └── resources │ ├── META-INF │ └── gradle-plugins │ │ ├── es.hadoop.build.base.properties │ │ ├── es.hadoop.build.integration.properties │ │ ├── es.hadoop.build.properties │ │ ├── es.hadoop.build.root.properties │ │ ├── es.hadoop.cluster.properties │ │ └── spark.variants.properties │ ├── minimumCompilerVersion │ ├── minimumGradleVersion │ └── minimumRuntimeVersion ├── catalog-info.yaml ├── dist ├── build.gradle └── licenses │ ├── commons-codec-1.4.jar.sha1 │ ├── commons-codec-LICENSE.txt │ ├── commons-codec-NOTICE.txt │ ├── commons-httpclient-3.0.1.jar.sha1 │ ├── commons-httpclient-LICENSE.txt │ ├── commons-httpclient-NOTICE.txt │ ├── commons-logging-1.1.1.jar.sha1 │ ├── commons-logging-LICENSE.txt │ ├── commons-logging-NOTICE.txt │ ├── hadoop-LICENSE.txt │ ├── hadoop-NOTICE.txt │ ├── hadoop-client-3.1.2.jar.sha1 │ ├── hadoop-common-3.1.2.jar.sha1 │ ├── hadoop-mapreduce-client-core-3.1.2.jar.sha1 │ ├── hive-LICENSE.txt │ ├── hive-NOTICE.txt │ ├── hive-exec-3.1.3.jar.sha1 │ ├── hive-metastore-3.1.3.jar.sha1 │ ├── hive-service-3.1.3.jar.sha1 │ ├── jackson-LICENSE.txt │ ├── jackson-NOTICE.txt │ ├── jackson-core-asl-1.8.8.jar.sha1 │ ├── jackson-mapper-asl-1.8.8.jar.sha1 │ ├── jaxb-api-2.3.1.jar.sha1 │ ├── jaxb-api-LICENSE.txt │ ├── jaxb-api-NOTICE.txt │ ├── scala-LICENSE.txt │ ├── scala-NOTICE.txt │ ├── scala-library-2.13.13.jar.sha1 │ ├── scala-reflect-2.13.13.jar.sha1 │ ├── spark-LICENSE.txt │ ├── spark-NOTICE.txt │ ├── spark-core_2.13-3.4.3.jar.sha1 │ ├── spark-sql_2.13-3.4.3.jar.sha1 │ ├── spark-streaming_2.13-3.4.3.jar.sha1 │ └── spark-yarn_2.13-3.4.3.jar.sha1 ├── docs ├── docset.yml ├── reference │ ├── apache-hive-integration.md │ ├── apache-spark-support.md │ ├── architecture.md │ ├── cloudrestricted-environments.md │ ├── configuration.md │ ├── error-handlers.md │ ├── hadoop-metrics.md │ ├── index.md │ ├── installation.md │ ├── kerberos.md │ ├── key-features.md │ ├── license.md │ ├── logging.md │ ├── mapping-types.md │ ├── mapreduce-integration.md │ ├── performance-considerations.md │ ├── reference.md │ ├── requirements.md │ ├── resources.md │ ├── runtime-options.md │ ├── security.md │ ├── setup-requirements.md │ └── toc.yml └── release-notes │ ├── breaking-changes.md │ ├── deprecations.md │ ├── index.md │ ├── known-issues.md │ └── toc.yml ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── hive ├── build.gradle ├── licenses │ ├── commons-logging-1.1.1.jar.sha1 │ ├── commons-logging-LICENSE.txt │ ├── commons-logging-NOTICE.txt │ ├── hive-LICENSE.txt │ ├── hive-NOTICE.txt │ ├── hive-exec-3.1.3.jar.sha1 │ ├── hive-metastore-3.1.3.jar.sha1 │ ├── hive-service-3.1.3.jar.sha1 │ ├── jaxb-api-2.3.1.jar.sha1 │ ├── jaxb-api-LICENSE.txt │ └── jaxb-api-NOTICE.txt └── src │ ├── itest │ ├── java │ │ └── org │ │ │ └── elasticsearch │ │ │ └── hadoop │ │ │ └── integration │ │ │ └── hive │ │ │ ├── AbstractHiveExtraTests.java │ │ │ ├── AbstractHiveReadJsonTest.java │ │ │ ├── AbstractHiveSaveJsonTest.java │ │ │ ├── AbstractHiveSaveTest.java │ │ │ ├── AbstractHiveSearchJsonTest.java │ │ │ ├── AbstractHiveSearchTest.java │ │ │ ├── HiveEmbeddedServer2.java │ │ │ ├── HiveInstance.java │ │ │ ├── HiveJdbc.java │ │ │ └── HiveSuite.java │ └── resources │ │ ├── cars-bulk.txt │ │ ├── hive-compound.dat │ │ ├── hive-compound.json │ │ ├── hive-site.xml │ │ ├── log4j2.properties │ │ └── org │ │ └── elasticsearch │ │ └── hadoop │ │ └── hive │ │ ├── hive-date-mapping.json │ │ ├── hive-date-mappingresponse.json │ │ ├── hive-date-source.json │ │ └── hive-date-typeless-mapping.json │ ├── main │ └── java │ │ └── org │ │ └── elasticsearch │ │ └── hadoop │ │ └── hive │ │ ├── EsHiveInputFormat.java │ │ ├── EsHiveOutputFormat.java │ │ ├── EsSerDe.java │ │ ├── EsStorageHandler.java │ │ ├── HiveBytesArrayWritable.java │ │ ├── HiveBytesConverter.java │ │ ├── HiveConstants.java │ │ ├── HiveFieldExtractor.java │ │ ├── HiveType.java │ │ ├── HiveUtils.java │ │ ├── HiveValueReader.java │ │ ├── HiveValueWriter.java │ │ ├── HiveWritableValueWriter.java │ │ └── package-info.java │ └── test │ ├── java │ └── org │ │ └── elasticsearch │ │ └── hadoop │ │ ├── hive │ │ ├── FieldAliasTest.java │ │ └── HiveValueReaderTest.java │ │ └── serialization │ │ ├── HiveFieldExtractorTests.java │ │ ├── HiveTypeToJsonTest.java │ │ └── handler │ │ └── write │ │ └── impl │ │ └── HiveSerializationEventConverterTest.java │ └── resources │ └── org │ └── elasticsearch │ └── hadoop │ └── hive │ ├── hive-date-mapping.json │ ├── hive-date-mappingresponse.json │ ├── hive-date-source.json │ └── hive-date-typeless-mapping.json ├── licenses ├── commons-codec-1.4.jar.sha1 ├── commons-codec-LICENSE.txt ├── commons-codec-NOTICE.txt ├── commons-logging-1.1.1.jar.sha1 ├── commons-logging-LICENSE.txt ├── commons-logging-NOTICE.txt ├── hadoop-LICENSE.txt ├── hadoop-NOTICE.txt ├── hadoop-client-3.1.2.jar.sha1 ├── hadoop-common-3.1.2.jar.sha1 ├── hadoop-mapreduce-client-core-3.1.2.jar.sha1 ├── hive-LICENSE.txt ├── hive-NOTICE.txt ├── hive-exec-3.1.3.jar.sha1 ├── hive-metastore-3.1.3.jar.sha1 ├── hive-service-3.1.3.jar.sha1 ├── jaxb-api-2.3.1.jar.sha1 ├── jaxb-api-LICENSE.txt ├── jaxb-api-NOTICE.txt ├── scala-LICENSE.txt ├── scala-NOTICE.txt ├── scala-library-2.13.13.jar.sha1 ├── scala-reflect-2.13.13.jar.sha1 ├── slf4j-api-2.0.6.jar.sha1 ├── slf4j-api-LICENSE.txt ├── slf4j-api-NOTICE.txt ├── spark-LICENSE.txt ├── spark-NOTICE.txt ├── spark-catalyst_2.13-3.4.3.jar.sha1 ├── spark-core_2.13-3.4.3.jar.sha1 ├── spark-sql_2.13-3.4.3.jar.sha1 ├── spark-streaming_2.13-3.4.3.jar.sha1 └── spark-yarn_2.13-3.4.3.jar.sha1 ├── mr ├── build.gradle ├── licenses │ ├── commons-codec-1.4.jar.sha1 │ ├── commons-codec-LICENSE.txt │ ├── commons-codec-NOTICE.txt │ ├── commons-logging-1.1.1.jar.sha1 │ ├── commons-logging-LICENSE.txt │ ├── commons-logging-NOTICE.txt │ ├── hadoop-LICENSE.txt │ ├── hadoop-NOTICE.txt │ ├── hadoop-client-3.1.2.jar.sha1 │ ├── hadoop-common-3.1.2.jar.sha1 │ ├── hadoop-mapreduce-client-core-3.1.2.jar.sha1 │ ├── jaxb-api-2.3.1.jar.sha1 │ ├── jaxb-api-LICENSE.txt │ └── jaxb-api-NOTICE.txt └── src │ ├── itest │ ├── java │ │ └── org │ │ │ └── elasticsearch │ │ │ └── hadoop │ │ │ ├── ProvisionerTest.java │ │ │ ├── VersionTest.java │ │ │ ├── integration │ │ │ ├── mr │ │ │ │ ├── AbstractExtraMRTests.java │ │ │ │ ├── AbstractMRNewApiSaveTest.java │ │ │ │ ├── AbstractMRNewApiSearchTest.java │ │ │ │ ├── AbstractMROldApiSaveTest.java │ │ │ │ ├── AbstractMROldApiSearchTest.java │ │ │ │ └── MRSuite.java │ │ │ └── rest │ │ │ │ ├── AbstractRestQueryTest.java │ │ │ │ ├── AbstractRestSaveTest.java │ │ │ │ ├── RestSuite.java │ │ │ │ └── ssl │ │ │ │ └── SSLTests.java │ │ │ └── rest │ │ │ ├── commonshttp │ │ │ └── auth │ │ │ │ └── spnego │ │ │ │ ├── AbstractHadoopUserKerberosTest.java │ │ │ │ ├── AbstractSpnegoAuthSchemeTest.java │ │ │ │ ├── AbstractSpnegoNegotiatorTest.java │ │ │ │ └── KerberosSuite.java │ │ │ └── pooling │ │ │ ├── AbstractTransportPoolTest.java │ │ │ └── PoolingSuite.java │ └── resources │ │ ├── org │ │ └── elasticsearch │ │ │ └── hadoop │ │ │ └── integration │ │ │ ├── mr-child.json │ │ │ ├── mr-multi-value-string.json │ │ │ ├── mr-nested.json │ │ │ └── mr-parent.json │ │ └── ssl │ │ ├── client.jks │ │ ├── server.cer │ │ ├── server.jks │ │ ├── server.key │ │ └── server.pem │ ├── main │ ├── java │ │ └── org │ │ │ └── elasticsearch │ │ │ └── hadoop │ │ │ ├── EsHadoopException.java │ │ │ ├── EsHadoopIllegalArgumentException.java │ │ │ ├── EsHadoopIllegalStateException.java │ │ │ ├── EsHadoopUnsupportedOperationException.java │ │ │ ├── cfg │ │ │ ├── CompositeSettings.java │ │ │ ├── ConfigurationOptions.java │ │ │ ├── FieldPresenceValidation.java │ │ │ ├── FilteredSettings.java │ │ │ ├── HadoopSettings.java │ │ │ ├── HadoopSettingsManager.java │ │ │ ├── InternalConfigurationOptions.java │ │ │ ├── PropertiesSettings.java │ │ │ ├── Settings.java │ │ │ ├── SettingsManager.java │ │ │ ├── SettingsView.java │ │ │ └── package-info.java │ │ │ ├── cli │ │ │ ├── ConsolePrompt.java │ │ │ ├── Keytool.java │ │ │ └── Prompt.java │ │ │ ├── handler │ │ │ ├── ErrorCollector.java │ │ │ ├── ErrorHandler.java │ │ │ ├── EsHadoopAbortHandlerException.java │ │ │ ├── Exceptional.java │ │ │ ├── HandlerLoader.java │ │ │ ├── HandlerResult.java │ │ │ └── impl │ │ │ │ ├── AbortOnFailure.java │ │ │ │ ├── AbstractHandlerLoader.java │ │ │ │ ├── BaseExceptional.java │ │ │ │ ├── DropAndLog.java │ │ │ │ ├── LogRenderer.java │ │ │ │ ├── PreloadedHandlerLoader.java │ │ │ │ └── elasticsearch │ │ │ │ ├── BaseEventConverter.java │ │ │ │ ├── ElasticsearchHandler.java │ │ │ │ └── EventConverter.java │ │ │ ├── mr │ │ │ ├── Counter.java │ │ │ ├── EsInputFormat.java │ │ │ ├── EsMapReduceUtil.java │ │ │ ├── EsOutputFormat.java │ │ │ ├── HadoopCfgUtils.java │ │ │ ├── HadoopIOUtils.java │ │ │ ├── HeartBeat.java │ │ │ ├── LinkedMapWritable.java │ │ │ ├── MultiOutputFormat.java │ │ │ ├── ReportingUtils.java │ │ │ ├── SafeWritableConverter.java │ │ │ ├── WritableArrayWritable.java │ │ │ ├── WritableBytesConverter.java │ │ │ ├── WritableCompatUtil.java │ │ │ ├── WritableValueReader.java │ │ │ ├── WritableValueWriter.java │ │ │ ├── compat │ │ │ │ ├── CompatHandler.java │ │ │ │ ├── CompatProxy.java │ │ │ │ ├── Counter.java │ │ │ │ ├── CounterGroup.java │ │ │ │ ├── JobContext.java │ │ │ │ ├── MapContext.java │ │ │ │ ├── ReduceContext.java │ │ │ │ ├── ReflectiveInvoker.java │ │ │ │ ├── TaskAttemptContext.java │ │ │ │ └── TaskInputOutputContext.java │ │ │ ├── package-info.java │ │ │ └── security │ │ │ │ ├── EsTokenIdentifier.java │ │ │ │ ├── HadoopRealUserProvider.java │ │ │ │ ├── HadoopUser.java │ │ │ │ ├── HadoopUserProvider.java │ │ │ │ └── TokenUtil.java │ │ │ ├── package-info.java │ │ │ ├── rest │ │ │ ├── DelegatingInputStream.java │ │ │ ├── ErrorExtractor.java │ │ │ ├── ErrorUtils.java │ │ │ ├── EsHadoopInvalidRequest.java │ │ │ ├── EsHadoopNoNodesLeftException.java │ │ │ ├── EsHadoopParsingException.java │ │ │ ├── EsHadoopRemoteException.java │ │ │ ├── EsHadoopTransportException.java │ │ │ ├── HeaderProcessor.java │ │ │ ├── HttpRetryPolicy.java │ │ │ ├── HttpStatus.java │ │ │ ├── InitializationUtils.java │ │ │ ├── NetworkClient.java │ │ │ ├── NetworkUtils.java │ │ │ ├── NoHttpRetryPolicy.java │ │ │ ├── PartitionDefinition.java │ │ │ ├── Request.java │ │ │ ├── Resource.java │ │ │ ├── Response.java │ │ │ ├── RestClient.java │ │ │ ├── RestRepository.java │ │ │ ├── RestService.java │ │ │ ├── Retry.java │ │ │ ├── ReusableInputStream.java │ │ │ ├── ScrollQuery.java │ │ │ ├── SearchRequestBuilder.java │ │ │ ├── SimpleHttpRetryPolicy.java │ │ │ ├── SimpleRequest.java │ │ │ ├── SimpleResponse.java │ │ │ ├── Transport.java │ │ │ ├── TransportFactory.java │ │ │ ├── bulk │ │ │ │ ├── BulkProcessor.java │ │ │ │ ├── BulkResponse.java │ │ │ │ └── handler │ │ │ │ │ ├── BulkWriteErrorCollector.java │ │ │ │ │ ├── BulkWriteErrorHandler.java │ │ │ │ │ ├── BulkWriteFailure.java │ │ │ │ │ ├── DelayableErrorCollector.java │ │ │ │ │ ├── IBulkWriteErrorHandler.java │ │ │ │ │ └── impl │ │ │ │ │ ├── BulkErrorEventConverter.java │ │ │ │ │ ├── BulkLogRenderer.java │ │ │ │ │ ├── BulkWriteHandlerLoader.java │ │ │ │ │ ├── DelegatingErrorHandler.java │ │ │ │ │ └── HttpRetryHandler.java │ │ │ ├── commonshttp │ │ │ │ ├── BytesArrayRequestEntity.java │ │ │ │ ├── CommonsHttpTransport.java │ │ │ │ ├── CommonsHttpTransportFactory.java │ │ │ │ ├── DelegatedProtocol.java │ │ │ │ ├── DeleteMethodWithBody.java │ │ │ │ ├── GetMethodWithBody.java │ │ │ │ ├── ProtocolAwareHostConfiguration.java │ │ │ │ ├── SSLSocketFactory.java │ │ │ │ ├── SocksSocketFactory.java │ │ │ │ └── auth │ │ │ │ │ ├── EsHadoopAuthPolicies.java │ │ │ │ │ ├── bearer │ │ │ │ │ ├── EsApiKeyAuthScheme.java │ │ │ │ │ └── EsApiKeyCredentials.java │ │ │ │ │ └── spnego │ │ │ │ │ ├── SpnegoAuthScheme.java │ │ │ │ │ ├── SpnegoCredentials.java │ │ │ │ │ └── SpnegoNegotiator.java │ │ │ ├── pooling │ │ │ │ ├── PooledHttpTransportFactory.java │ │ │ │ ├── PooledTransportManager.java │ │ │ │ └── TransportPool.java │ │ │ ├── query │ │ │ │ ├── BoolQueryBuilder.java │ │ │ │ ├── ConstantScoreQueryBuilder.java │ │ │ │ ├── FilteredQueryBuilder.java │ │ │ │ ├── MatchAllQueryBuilder.java │ │ │ │ ├── QueryBuilder.java │ │ │ │ ├── QueryStringQueryBuilder.java │ │ │ │ ├── QueryUtils.java │ │ │ │ ├── RawQueryBuilder.java │ │ │ │ ├── SimpleQueryParser.java │ │ │ │ └── TermQueryBuilder.java │ │ │ ├── request │ │ │ │ ├── GetAliasesRequestBuilder.java │ │ │ │ └── RequestBuilder.java │ │ │ └── stats │ │ │ │ ├── Stats.java │ │ │ │ └── StatsAware.java │ │ │ ├── security │ │ │ ├── AuthenticationMethod.java │ │ │ ├── EsHadoopSecurityException.java │ │ │ ├── EsToken.java │ │ │ ├── JdkUser.java │ │ │ ├── JdkUserProvider.java │ │ │ ├── KeystoreWrapper.java │ │ │ ├── LoginUtil.java │ │ │ ├── SecureSettings.java │ │ │ ├── User.java │ │ │ └── UserProvider.java │ │ │ ├── serialization │ │ │ ├── BytesConverter.java │ │ │ ├── EsHadoopSerializationException.java │ │ │ ├── FieldType.java │ │ │ ├── Generator.java │ │ │ ├── JdkBytesConverter.java │ │ │ ├── MapFieldExtractor.java │ │ │ ├── Parser.java │ │ │ ├── ParsingUtils.java │ │ │ ├── ScrollReader.java │ │ │ ├── ScrollReaderConfigBuilder.java │ │ │ ├── SettingsAware.java │ │ │ ├── builder │ │ │ │ ├── AbstractValueReader.java │ │ │ │ ├── ContentBuilder.java │ │ │ │ ├── FilteringValueWriter.java │ │ │ │ ├── JdkValueReader.java │ │ │ │ ├── JdkValueWriter.java │ │ │ │ ├── NoOpValueWriter.java │ │ │ │ ├── ValueParsingCallback.java │ │ │ │ ├── ValueReader.java │ │ │ │ └── ValueWriter.java │ │ │ ├── bulk │ │ │ │ ├── AbstractBulkFactory.java │ │ │ │ ├── BulkCommand.java │ │ │ │ ├── BulkCommands.java │ │ │ │ ├── BulkEntryWriter.java │ │ │ │ ├── BulkFactory.java │ │ │ │ ├── CreateBulkFactory.java │ │ │ │ ├── DeleteBulkFactory.java │ │ │ │ ├── DeleteTemplatedBulk.java │ │ │ │ ├── IndexBulkFactory.java │ │ │ │ ├── JsonScriptTemplateBulk.java │ │ │ │ ├── JsonTemplatedBulk.java │ │ │ │ ├── MetadataExtractor.java │ │ │ │ ├── PerEntityPoolingMetadataExtractor.java │ │ │ │ ├── RawJson.java │ │ │ │ ├── ScriptTemplateBulk.java │ │ │ │ ├── TemplatedBulk.java │ │ │ │ └── UpdateBulkFactory.java │ │ │ ├── dto │ │ │ │ ├── IndicesAliases.java │ │ │ │ ├── NodeInfo.java │ │ │ │ ├── ShardInfo.java │ │ │ │ └── mapping │ │ │ │ │ ├── Field.java │ │ │ │ │ ├── FieldParser.java │ │ │ │ │ ├── GeoField.java │ │ │ │ │ ├── GeoPointType.java │ │ │ │ │ ├── GeoShapeType.java │ │ │ │ │ ├── Mapping.java │ │ │ │ │ ├── MappingSet.java │ │ │ │ │ └── MappingUtils.java │ │ │ ├── field │ │ │ │ ├── AbstractDefaultParamsExtractor.java │ │ │ │ ├── AbstractIndexExtractor.java │ │ │ │ ├── ChainedFieldExtractor.java │ │ │ │ ├── ConstantFieldExtractor.java │ │ │ │ ├── DateIndexFormatter.java │ │ │ │ ├── DefaultIndexExtractor.java │ │ │ │ ├── DefaultParamsExtractor.java │ │ │ │ ├── ExtractorUtils.java │ │ │ │ ├── FieldExplainer.java │ │ │ │ ├── FieldExtractor.java │ │ │ │ ├── FieldFilter.java │ │ │ │ ├── IndexExtractor.java │ │ │ │ ├── IndexFormatter.java │ │ │ │ ├── JsonFieldExtractors.java │ │ │ │ ├── MapWritableFieldExtractor.java │ │ │ │ └── NoOpFieldExtractor.java │ │ │ ├── handler │ │ │ │ ├── SerdeErrorCollector.java │ │ │ │ ├── read │ │ │ │ │ ├── DeserializationErrorHandler.java │ │ │ │ │ ├── DeserializationFailure.java │ │ │ │ │ ├── IDeserializationErrorHandler.java │ │ │ │ │ └── impl │ │ │ │ │ │ ├── AbortOnlyHandlerLoader.java │ │ │ │ │ │ ├── DelegatingErrorHandler.java │ │ │ │ │ │ ├── DeserializationEventConverter.java │ │ │ │ │ │ ├── DeserializationHandlerLoader.java │ │ │ │ │ │ └── DeserializationLogRenderer.java │ │ │ │ └── write │ │ │ │ │ ├── ISerializationErrorHandler.java │ │ │ │ │ ├── SerializationErrorHandler.java │ │ │ │ │ ├── SerializationFailure.java │ │ │ │ │ └── impl │ │ │ │ │ ├── DelegatingErrorHandler.java │ │ │ │ │ ├── SerializationEventConverter.java │ │ │ │ │ ├── SerializationHandlerLoader.java │ │ │ │ │ └── SerializationLogRenderer.java │ │ │ └── json │ │ │ │ ├── BackportedJacksonMappingIterator.java │ │ │ │ ├── BackportedJsonStringEncoder.java │ │ │ │ ├── BackportedObjectReader.java │ │ │ │ ├── BlockAwareJsonParser.java │ │ │ │ ├── JacksonJsonGenerator.java │ │ │ │ ├── JacksonJsonParser.java │ │ │ │ ├── JsonFactory.java │ │ │ │ └── ObjectReader.java │ │ │ └── util │ │ │ ├── ArrayUtils.java │ │ │ ├── Assert.java │ │ │ ├── ByteSequence.java │ │ │ ├── BytesArray.java │ │ │ ├── BytesArrayPool.java │ │ │ ├── BytesRef.java │ │ │ ├── ClusterInfo.java │ │ │ ├── ClusterName.java │ │ │ ├── Constants.java │ │ │ ├── DateUtils.java │ │ │ ├── EsMajorVersion.java │ │ │ ├── FastByteArrayInputStream.java │ │ │ ├── FastByteArrayOutputStream.java │ │ │ ├── FieldAlias.java │ │ │ ├── IOUtils.java │ │ │ ├── ObjectUtils.java │ │ │ ├── OsUtil.java │ │ │ ├── ReflectionUtils.java │ │ │ ├── SettingsUtils.java │ │ │ ├── StringUtils.java │ │ │ ├── TrackingBytesArray.java │ │ │ ├── UnicodeUtil.java │ │ │ ├── Version.java │ │ │ ├── WritableUtils.java │ │ │ ├── ecs │ │ │ ├── ElasticCommonSchema.java │ │ │ ├── FieldNames.java │ │ │ ├── HostData.java │ │ │ └── MessageTemplate.java │ │ │ ├── encoding │ │ │ └── HttpEncodingTools.java │ │ │ ├── regex │ │ │ └── Regex.java │ │ │ └── unit │ │ │ ├── Booleans.java │ │ │ ├── ByteSizeUnit.java │ │ │ ├── ByteSizeValue.java │ │ │ ├── Strings.java │ │ │ └── TimeValue.java │ └── resources │ │ ├── META-INF │ │ └── services │ │ │ ├── org.apache.hadoop.security.token.TokenIdentifier │ │ │ └── org.apache.hadoop.security.token.TokenRenewer │ │ └── org │ │ └── elasticsearch │ │ └── hadoop │ │ └── mr │ │ └── Counter.properties │ └── test │ ├── java │ └── org │ │ └── elasticsearch │ │ └── hadoop │ │ ├── cfg │ │ ├── CompositeSettingsTest.java │ │ ├── FilteredSettingsTest.java │ │ ├── SettingsTest.java │ │ └── SettingsViewTest.java │ │ ├── cli │ │ ├── KeytoolTest.java │ │ └── TestPrompt.java │ │ ├── mr │ │ ├── LinkedMapWritableTest.java │ │ ├── WritableBytesConverterTests.java │ │ └── security │ │ │ ├── EsTokenIdentifierTest.java │ │ │ └── HadoopUserTest.java │ │ ├── rest │ │ ├── ApplyAliasMetaDataTest.java │ │ ├── ErrorExtractorTest.java │ │ ├── ErrorUtilsTest.java │ │ ├── FindPartitionsTest.java │ │ ├── HeaderProcessorTest.java │ │ ├── InitializationUtilsTest.java │ │ ├── NetworkClientTest.java │ │ ├── PartitionDefinitionTest.java │ │ ├── QueryBuilderTestUtils.java │ │ ├── QueryTest.java │ │ ├── ResourceTest.java │ │ ├── RestClientTest.java │ │ ├── RestServiceTest.java │ │ ├── ScrollQueryTest.java │ │ ├── SearchRequestBuilderTest.java │ │ ├── bulk │ │ │ ├── BulkOutputGenerator.java │ │ │ ├── BulkProcessorTest.java │ │ │ ├── bwc │ │ │ │ ├── BulkOutputGeneratorBase.java │ │ │ │ ├── BulkOutputGeneratorV1.java │ │ │ │ ├── BulkOutputGeneratorV2.java │ │ │ │ └── BulkOutputGeneratorV5.java │ │ │ └── handler │ │ │ │ ├── BulkWriteErrorCollectorTest.java │ │ │ │ └── impl │ │ │ │ └── BulkErrorEventConverterTest.java │ │ ├── commonshttp │ │ │ └── CommonsHttpTransportTests.java │ │ └── query │ │ │ ├── QueryUtilsTest.java │ │ │ ├── RandomQueryBuilders.java │ │ │ └── RawQueryBuilderTest.java │ │ ├── security │ │ ├── JdkUserTest.java │ │ └── KeystoreWrapperTest.java │ │ ├── serialization │ │ ├── CommandTest.java │ │ ├── FieldExtractorTests.java │ │ ├── JdkTypeToJsonTest.java │ │ ├── JdkValueReaderTest.java │ │ ├── JsonIdTest.java │ │ ├── JsonPathTest.java │ │ ├── JsonValuePathTest.java │ │ ├── ScrollReaderJsonTest.java │ │ ├── ScrollReaderTest.java │ │ ├── ValueReaderTest.java │ │ ├── WritableTypeToJsonTest.java │ │ ├── WritableValueReaderTest.java │ │ ├── builder │ │ │ ├── JdkExtendedBooleanValueReaderReaderTest.java │ │ │ ├── JdkValueReaderTest.java │ │ │ ├── JdkValueWriterTest.java │ │ │ └── NoOpValueWriterTest.java │ │ ├── bulk │ │ │ └── BulkEntryWriterTest.java │ │ ├── dto │ │ │ ├── NodeInfoTest.java │ │ │ └── mapping │ │ │ │ └── MappingTest.java │ │ ├── field │ │ │ ├── DateIndexFormatterTest.java │ │ │ ├── DefaultIndexExtractorTest.java │ │ │ ├── FieldFilterTest.java │ │ │ └── JsonFieldExtractorsTest.java │ │ ├── handler │ │ │ ├── read │ │ │ │ └── impl │ │ │ │ │ └── DeserializationEventConverterTest.java │ │ │ └── write │ │ │ │ └── impl │ │ │ │ └── SerializationEventConverterTest.java │ │ └── json │ │ │ ├── BlockAwareJsonParserTest.java │ │ │ └── JacksonJsonGeneratorTest.java │ │ └── util │ │ ├── ArrayUtilsTest.java │ │ ├── BytesArrayPoolTest.java │ │ ├── BytesArrayTest.java │ │ ├── ClusterInfoTest.java │ │ ├── DateUtilsTest.java │ │ ├── EsMajorVersionTest.java │ │ ├── FastByteArrayInputStreamTest.java │ │ ├── IOUtilsTest.java │ │ ├── LevenshteinTests.java │ │ ├── SettingsUtilsTest.java │ │ ├── StringUtilsTest.java │ │ ├── TrackingBytesArrayTest.java │ │ ├── WritableUtilsTest.java │ │ ├── ecs │ │ └── ElasticCommonSchemaTest.java │ │ ├── encoding │ │ └── HttpEncodingToolsTest.java │ │ └── unit │ │ ├── BooleansTest.java │ │ ├── ByteSizeUnitTests.java │ │ ├── ByteSizeValueTests.java │ │ └── TimeValueTests.java │ └── resources │ ├── core-site.xml │ ├── log4j2.properties │ ├── log4jnull.properties │ └── org │ └── elasticsearch │ └── hadoop │ ├── rest │ ├── bulk-error.json │ ├── bulk-retry-input-template.json │ ├── get-aliases-empty-response.json │ ├── get-aliases-response.json │ ├── node-list.json │ ├── search-shards-response.json │ └── shard-list.json │ ├── serialization │ ├── dto │ │ ├── client-nodes-v1.json │ │ ├── client-nodes-v2.json │ │ ├── client-nodes-v5.json │ │ └── mapping │ │ │ ├── typed │ │ │ ├── attachment.json │ │ │ ├── basic.json │ │ │ ├── completion.json │ │ │ ├── dynamic-template.json │ │ │ ├── geo.json │ │ │ ├── ip.json │ │ │ ├── join-type.json │ │ │ ├── mapping_with_fields_named_properties_and_type.json │ │ │ ├── multi_level_field_with_same_name.json │ │ │ ├── multiple-indices-multiple-conflicting-types.json │ │ │ ├── multiple-indices-multiple-types.json │ │ │ ├── multiple-indices-multiple-upcastable-types.json │ │ │ ├── multiple-types.json │ │ │ ├── nested-mapping.json │ │ │ ├── nested_arrays_mapping.json │ │ │ └── primitives.json │ │ │ └── typeless │ │ │ ├── attachment.json │ │ │ ├── basic.json │ │ │ ├── completion.json │ │ │ ├── dynamic-template.json │ │ │ ├── geo.json │ │ │ ├── ip.json │ │ │ ├── join-type.json │ │ │ ├── mapping_with_fields_named_properties_and_type.json │ │ │ ├── multi_level_field_with_same_name.json │ │ │ ├── multiple-indices-multiple-conflicting-types.json │ │ │ ├── multiple-indices-multiple-types.json │ │ │ ├── multiple-indices-multiple-upcastable-types.json │ │ │ ├── nested-mapping.json │ │ │ ├── nested_arrays_mapping.json │ │ │ └── primitives.json │ ├── nested-fields.json │ ├── parser-test-nested.json │ ├── parser-test.json │ ├── scroll-test.json │ ├── scrollReaderTestData │ │ ├── empty-list │ │ │ ├── mapping.json │ │ │ └── scroll.json │ │ ├── empty-source │ │ │ └── scroll.json │ │ ├── fields │ │ │ └── scroll.json │ │ ├── join │ │ │ ├── mapping.json │ │ │ └── scroll.json │ │ ├── list │ │ │ └── scroll.json │ │ ├── matched-queries │ │ │ └── scroll.json │ │ ├── multi-type │ │ │ ├── mapping.json │ │ │ └── scroll.json │ │ ├── nested-data │ │ │ ├── mapping.json │ │ │ └── scroll.json │ │ ├── nested-list │ │ │ ├── mapping.json │ │ │ └── scroll.json │ │ ├── no-scroll-id │ │ │ └── scroll.json │ │ ├── numbers-as-strings │ │ │ ├── mapping.json │ │ │ └── scroll.json │ │ ├── object-fields │ │ │ ├── mapping.json │ │ │ └── scroll.json │ │ └── source │ │ │ ├── mapping.json │ │ │ └── scroll.json │ └── tweet.json │ └── util │ └── textdata.txt ├── qa ├── .DS_Store ├── build.gradle └── kerberos │ ├── .DS_Store │ ├── build.gradle │ └── src │ ├── itest │ ├── java │ │ └── org │ │ │ └── elasticsearch │ │ │ └── hadoop │ │ │ └── qa │ │ │ └── kerberos │ │ │ ├── AbstractClusterVerificationTests.java │ │ │ ├── AbstractKerberosClientTest.java │ │ │ ├── KerberosQaSuite.java │ │ │ └── LocalKdc.java │ └── resources │ │ └── artists.dat │ └── main │ ├── java │ └── org │ │ └── elasticsearch │ │ └── hadoop │ │ └── qa │ │ └── kerberos │ │ ├── ExtendedClient.java │ │ ├── dfs │ │ └── SecureFsShell.java │ │ ├── hive │ │ └── SecureBeeline.java │ │ ├── mr │ │ ├── ConfTest.java │ │ ├── LoadToES.java │ │ └── ReadFromES.java │ │ ├── security │ │ └── KeytabLogin.java │ │ └── setup │ │ └── SetupKerberosUsers.java │ ├── resources │ ├── hive │ │ ├── load_to_es.sql │ │ ├── patches │ │ │ ├── 1.2.2 │ │ │ │ └── beeline.sh │ │ │ ├── 3.1.2 │ │ │ │ └── beeline.sh │ │ │ └── 3.1.3 │ │ │ │ └── beeline.sh │ │ ├── read_from_es.sql │ │ └── top_25_words.sql │ ├── ssl │ │ └── server.jks │ └── test.properties │ └── scala │ └── org │ └── elasticsearch │ └── hadoop │ └── qa │ └── kerberos │ └── spark │ ├── LoadToES.scala │ └── ReadFromES.scala ├── repository-hdfs └── README.md ├── settings.gradle ├── spark ├── .DS_Store ├── core │ ├── build.gradle │ ├── licenses │ │ ├── commons-logging-1.1.1.jar.sha1 │ │ ├── commons-logging-LICENSE.txt │ │ ├── commons-logging-NOTICE.txt │ │ ├── scala-LICENSE.txt │ │ ├── scala-NOTICE.txt │ │ ├── scala-library-2.13.13.jar.sha1 │ │ ├── scala-reflect-2.13.13.jar.sha1 │ │ ├── spark-LICENSE.txt │ │ ├── spark-NOTICE.txt │ │ └── spark-core_2.13-3.4.3.jar.sha1 │ └── src │ │ ├── itest │ │ ├── java │ │ │ └── org │ │ │ │ └── elasticsearch │ │ │ │ └── spark │ │ │ │ └── integration │ │ │ │ ├── AbstractHadoopBasicSparkTest.java │ │ │ │ ├── AbstractJavaEsSparkTest.java │ │ │ │ ├── ScalaUtils.java │ │ │ │ ├── SparkSuite.java │ │ │ │ └── SparkUtils.java │ │ ├── resources │ │ │ ├── basic.json │ │ │ └── simple.json │ │ └── scala │ │ │ └── org │ │ │ └── elasticsearch │ │ │ └── spark │ │ │ ├── integration │ │ │ ├── AbstractScalaEsSpark.scala │ │ │ └── SparkScalaSuite.java │ │ │ └── serialization │ │ │ └── Bean.java │ │ ├── main │ │ └── scala │ │ │ └── org │ │ │ └── elasticsearch │ │ │ └── spark │ │ │ ├── cfg │ │ │ ├── SparkSettings.java │ │ │ ├── SparkSettingsManager.java │ │ │ └── package-info.java │ │ │ ├── package.scala │ │ │ ├── rdd │ │ │ ├── AbstractEsRDD.scala │ │ │ ├── AbstractEsRDDIterator.scala │ │ │ ├── CompatUtils.java │ │ │ ├── CompatibilityLevel.scala │ │ │ ├── EsRDDWriter.scala │ │ │ ├── EsSpark.scala │ │ │ ├── JDKCollectionConvertersCompat.scala │ │ │ ├── JavaEsRDD.scala │ │ │ ├── Metadata.java │ │ │ ├── ScalaEsRDD.scala │ │ │ └── api │ │ │ │ └── java │ │ │ │ ├── JavaEsSpark.scala │ │ │ │ └── package-info.java │ │ │ └── serialization │ │ │ ├── ReflectionUtils.scala │ │ │ ├── ScalaMapFieldExtractor.scala │ │ │ ├── ScalaMetadataExtractor.scala │ │ │ ├── ScalaValueReader.scala │ │ │ └── ScalaValueWriter.scala │ │ └── test │ │ └── scala │ │ └── org │ │ └── elasticsearch │ │ └── spark │ │ ├── ScalaExtendedBooleanValueReaderTest.scala │ │ ├── ScalaValueReaderTest.scala │ │ ├── cfg │ │ └── SparkConfigTest.scala │ │ └── serialization │ │ ├── Bean.java │ │ ├── ScalaReflectionUtilsTest.scala │ │ ├── ScalaValueReaderTest.scala │ │ ├── ScalaValueWriterTest.scala │ │ ├── handler │ │ └── write │ │ │ └── imple │ │ │ └── ScalaSerializationEventConverterTest.scala │ │ └── testbeans │ │ ├── Contact.java │ │ └── ContactBook.java └── sql-30 │ ├── build.gradle │ ├── licenses │ ├── commons-logging-1.1.1.jar.sha1 │ ├── commons-logging-LICENSE.txt │ ├── commons-logging-NOTICE.txt │ ├── jaxb-api-2.3.1.jar.sha1 │ ├── jaxb-api-LICENSE.txt │ ├── jaxb-api-NOTICE.txt │ ├── scala-LICENSE.txt │ ├── scala-NOTICE.txt │ ├── scala-library-2.13.13.jar.sha1 │ ├── scala-reflect-2.13.13.jar.sha1 │ ├── slf4j-api-2.0.6.jar.sha1 │ ├── slf4j-api-LICENSE.txt │ ├── slf4j-api-NOTICE.txt │ ├── spark-LICENSE.txt │ ├── spark-NOTICE.txt │ ├── spark-catalyst_2.13-3.4.3.jar.sha1 │ ├── spark-core_2.13-3.4.3.jar.sha1 │ ├── spark-sql_2.13-3.4.3.jar.sha1 │ ├── spark-streaming_2.13-3.4.3.jar.sha1 │ └── spark-yarn_2.13-3.4.3.jar.sha1 │ └── src │ ├── itest │ ├── java │ │ └── org │ │ │ └── elasticsearch │ │ │ └── spark │ │ │ ├── integration │ │ │ ├── AbstractJavaEsSparkSQLTest.java │ │ │ ├── AbstractJavaEsSparkStreamingTest.java │ │ │ ├── ScalaUtils.java │ │ │ ├── SparkSQLSuite.java │ │ │ ├── SparkStreamingSuite.java │ │ │ └── SparkUtils.java │ │ │ └── sql │ │ │ └── UtilsTest.java │ ├── resources │ │ ├── basic.json │ │ ├── multi-level-doc.json │ │ ├── simple.json │ │ └── small-sample.json │ └── scala │ │ └── org │ │ └── elasticsearch │ │ └── spark │ │ ├── integration │ │ ├── AbstractJavaEsSparkStructuredStreamingTest.java │ │ ├── AbstractScalaEsScalaSparkStreaming.scala │ │ ├── AbstractScalaEsSparkSQL.scala │ │ ├── AbstractScalaEsSparkStructuredStreaming.scala │ │ ├── SparkSQLScalaSuite.java │ │ ├── SparkStreamingScalaSuite.java │ │ ├── SparkStructuredStreamingScalaSuite.java │ │ └── SparkStructuredStreamingSuite.java │ │ ├── serialization │ │ ├── Bean.java │ │ ├── Garbage.scala │ │ ├── ModuleCaseClass.scala │ │ └── Trip.scala │ │ └── sql │ │ ├── SchemaUtilsTestable.scala │ │ ├── ServiceLoadingTest.scala │ │ └── streaming │ │ ├── NetcatTestServer.java │ │ ├── StreamingQueryTestHarness.scala │ │ └── api │ │ └── java │ │ └── JavaStreamingQueryTestHarness.java │ ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ ├── org.apache.spark.security.HadoopDelegationTokenProvider │ │ │ └── org.apache.spark.sql.sources.DataSourceRegister │ └── scala │ │ └── org │ │ └── elasticsearch │ │ └── spark │ │ ├── deploy │ │ └── yarn │ │ │ └── security │ │ │ └── EsServiceCredentialProvider.scala │ │ ├── sql │ │ ├── DataFrameFieldExtractor.scala │ │ ├── DataFrameValueWriter.scala │ │ ├── DefaultSource.scala │ │ ├── DefaultSource15.scala │ │ ├── EsDataFrameWriter.scala │ │ ├── EsSparkSQL.scala │ │ ├── RowValueReader.scala │ │ ├── ScalaEsRow.scala │ │ ├── ScalaEsRowRDD.scala │ │ ├── ScalaEsRowValueReader.scala │ │ ├── SchemaUtils.scala │ │ ├── SparkSQLCompatibilityLevel.scala │ │ ├── Utils.java │ │ ├── api │ │ │ └── java │ │ │ │ ├── JavaEsSparkSQL.scala │ │ │ │ └── package-info.java │ │ ├── package.scala │ │ └── streaming │ │ │ ├── EsCommitProtocol.scala │ │ │ ├── EsSinkMetadataLog.scala │ │ │ ├── EsSinkStatus.scala │ │ │ ├── EsSparkSqlStreamingSink.scala │ │ │ ├── EsStreamQueryWriter.scala │ │ │ ├── NullMetadataLog.scala │ │ │ ├── SparkSqlStreamingConfigs.scala │ │ │ └── StructuredStreamingVersionLock.scala │ │ └── streaming │ │ ├── EsSparkStreaming.scala │ │ ├── api │ │ └── java │ │ │ ├── JavaEsSparkStreaming.scala │ │ │ └── package-info.java │ │ └── package.scala │ └── test │ └── scala │ └── org │ └── elasticsearch │ └── spark │ ├── rdd │ └── CompatibilityCheckTest.scala │ └── sql │ ├── DataFrameFieldExtractorTest.scala │ ├── DataFrameValueWriterTest.scala │ ├── DefaultSourceTest.scala │ ├── RowSerializationEventConverterTest.scala │ ├── SchemaUtilsTest.scala │ └── streaming │ └── StructuredStreamingVersionLockTest.scala ├── test ├── build.gradle ├── fixtures │ ├── build.gradle │ └── minikdc │ │ ├── build.gradle │ │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── elasticsearch │ │ └── hadoop │ │ └── test │ │ └── fixture │ │ └── minikdc │ │ └── MiniKdcFixture.java └── shared │ ├── build.gradle │ └── src │ ├── main │ ├── java │ │ └── org │ │ │ ├── elasticsearch │ │ │ └── hadoop │ │ │ │ ├── EsAssume.java │ │ │ │ ├── HdpBootstrap.java │ │ │ │ ├── NullPrintStream.java │ │ │ │ ├── Provisioner.java │ │ │ │ ├── QueryTestParams.java │ │ │ │ ├── Stream.java │ │ │ │ ├── TestData.java │ │ │ │ ├── fixtures │ │ │ │ ├── EsEmbeddedCluster.java │ │ │ │ ├── KDCFixture.java │ │ │ │ └── LocalEs.java │ │ │ │ ├── fs │ │ │ │ ├── HdfsUtils.java │ │ │ │ └── NTFSLocalFileSystem.java │ │ │ │ ├── mr │ │ │ │ └── PrintStreamOutputFormat.java │ │ │ │ ├── rest │ │ │ │ ├── RestUtils.java │ │ │ │ └── ssl │ │ │ │ │ └── BasicSSLServer.java │ │ │ │ ├── security │ │ │ │ └── UgiUtil.java │ │ │ │ ├── serialization │ │ │ │ ├── AbstractValueReaderTest.java │ │ │ │ ├── BaseValueReaderTest.java │ │ │ │ ├── JsonUtils.java │ │ │ │ └── builder │ │ │ │ │ └── AbstractExtendedBooleanValueReaderTest.java │ │ │ │ └── util │ │ │ │ ├── BytesUtils.java │ │ │ │ ├── TestSettings.java │ │ │ │ └── TestUtils.java │ │ │ └── junit │ │ │ └── rules │ │ │ ├── ChainedExternalResource.java │ │ │ └── LazyTempFolder.java │ └── resources │ │ ├── artists.dat │ │ ├── artists.json │ │ ├── data │ │ └── join │ │ │ ├── data │ │ │ ├── all.dat │ │ │ ├── children.dat │ │ │ └── parents.dat │ │ │ ├── json │ │ │ ├── all.json │ │ │ ├── children.json │ │ │ └── parents.json │ │ │ └── mapping │ │ │ ├── typed.json │ │ │ └── typeless.json │ │ ├── gibberish.dat │ │ ├── gibberish.json │ │ ├── org │ │ └── elasticsearch │ │ │ └── hadoop │ │ │ └── integration │ │ │ ├── query.dsl │ │ │ └── query.uri │ │ └── test.properties │ └── test │ ├── java │ └── org │ │ └── elasticsearch │ │ └── hadoop │ │ └── util │ │ └── BytesUtilsTest.java │ └── resources │ └── org │ └── elasticsearch │ └── hadoop │ └── util │ ├── escape-size.txt │ └── escaped-chars.txt └── thirdparty ├── build.gradle └── licenses ├── commons-httpclient-3.0.1.jar.sha1 ├── commons-httpclient-LICENSE.txt ├── commons-httpclient-NOTICE.txt ├── jackson-LICENSE.txt ├── jackson-NOTICE.txt ├── jackson-core-asl-1.8.8.jar.sha1 └── jackson-mapper-asl-1.8.8.jar.sha1 /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/elasticsearch-hadoop/aae4528ee8a31e7106e0ee4b671162fb70cf15f1/.DS_Store -------------------------------------------------------------------------------- /.buildkite/branches.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This determines which branches will have pipelines triggered periodically, for tests and dra workflows. 4 | BRANCHES=( $(curl -s https://raw.githubusercontent.com/elastic/elasticsearch/main/branches.json | jq -r '.branches[].branch') ) 5 | -------------------------------------------------------------------------------- /.buildkite/dra-workflow.trigger.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | echo "steps:" 6 | 7 | source .buildkite/branches.sh 8 | 9 | for BRANCH in "${BRANCHES[@]}"; do 10 | cat < 15 | 16 | Licensed under the Apache License, Version 2.0 (the "License"); you may not 17 | use this file except in compliance with the License. You may obtain a copy of 18 | the License at 19 | 20 | http://www.apache.org/licenses/LICENSE-2.0 21 | 22 | Unless required by applicable law or agreed to in writing, software 23 | distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 24 | WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 25 | License for the specific language governing permissions and limitations under 26 | the License. 27 | ``` 28 | 29 | -------------------------------------------------------------------------------- /docs/reference/resources.md: -------------------------------------------------------------------------------- 1 | --- 2 | mapped_pages: 3 | - https://www.elastic.co/guide/en/elasticsearch/hadoop/current/resources.html 4 | navigation_title: Resources 5 | --- 6 | # {{esh-full}} resources 7 | 8 | Project home page 9 | : [GitHub](http://github.com/elasticsearch/elasticsearch-hadoop) 10 | 11 | Source repository 12 | : [GitHub](http://github.com/elasticsearch/elasticsearch-hadoop) 13 | 14 | Issue tracker 15 | : [GitHub](http://github.com/elasticsearch/elasticsearch-hadoop/issues) 16 | 17 | Mailing list / forum 18 | : [Google Groups](https://groups.google.com/forum/?fromgroups#!forum/elasticsearch) - please add `[Hadoop]` prefix to the topic message 19 | 20 | Twitter 21 | : [Elasticsearch](http://twitter.com/elasticsearch), [Costin Leau](http://twitter.com/costinl) 22 | 23 | {{es}} community page 24 | : [community](http://www.elastic.co/community/) 25 | 26 | {{es}} resources page 27 | : [documentation](http://www.elastic.co/learn) 28 | 29 | -------------------------------------------------------------------------------- /docs/reference/setup-requirements.md: -------------------------------------------------------------------------------- 1 | --- 2 | navigation_title: Setup and requirements 3 | mapped_pages: 4 | - https://www.elastic.co/guide/en/elasticsearch/hadoop/current/doc-sections.html#_setup_requirements 5 | --- 6 | # {{esh-full}} setup and requirements 7 | 8 | This section provides an overview of the project, its requirements (and supported environment and libraries) plus information on how to easily install elasticsearch-hadoop in your environment. 9 | -------------------------------------------------------------------------------- /docs/reference/toc.yml: -------------------------------------------------------------------------------- 1 | project: 'Elasticsearch for Apache Hadoop reference' 2 | toc: 3 | - file: index.md 4 | - file: setup-requirements.md 5 | children: 6 | - file: key-features.md 7 | - file: requirements.md 8 | - file: installation.md 9 | - file: reference.md 10 | children: 11 | - file: architecture.md 12 | - file: configuration.md 13 | - file: runtime-options.md 14 | - file: security.md 15 | - file: logging.md 16 | - file: mapreduce-integration.md 17 | - file: apache-hive-integration.md 18 | - file: apache-spark-support.md 19 | - file: mapping-types.md 20 | - file: error-handlers.md 21 | - file: kerberos.md 22 | - file: hadoop-metrics.md 23 | - file: performance-considerations.md 24 | - file: cloudrestricted-environments.md 25 | - file: resources.md 26 | - file: license.md 27 | -------------------------------------------------------------------------------- /docs/release-notes/deprecations.md: -------------------------------------------------------------------------------- 1 | --- 2 | navigation_title: "Deprecations" 3 | --- 4 | 5 | # Elasticsearch for Apache Hadoop deprecations [elasticsearch-apache-hadoop-deprecations] 6 | Over time, certain Elastic functionality becomes outdated and is replaced or removed. To help with the transition, Elastic deprecates functionality for a period before removal, giving you time to update your applications. 7 | 8 | Review the deprecated functionality for Elasticsearch for Apache Hadoop. While deprecations have no immediate impact, we strongly encourage you update your implementation after you upgrade. [Upgrade](docs-content://deploy-manage/upgrade.md) 9 | 10 | % ## Next version 11 | 12 | % ::::{dropdown} Deprecation title 13 | % Description of the deprecation. 14 | % For more information, check [PR #](PR link). 15 | % **Impact**
Impact of deprecation. 16 | % **Action**
Steps for mitigating deprecation impact. 17 | % :::: 18 | 19 | % ## 9.0.0 [elasticsearch-apache-hadoop-900-deprecations] 20 | 21 | % ::::{dropdown} Deprecation title 22 | % Description of the deprecation. 23 | % For more information, check [PR #](PR link). 24 | % **Impact**
Impact of deprecation. 25 | % **Action**
Steps for mitigating deprecation impact. 26 | % :::: -------------------------------------------------------------------------------- /docs/release-notes/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | navigation_title: "Elasticsearch for Apache Hadoop" 3 | mapped_pages: 4 | - https://www.elastic.co/guide/en/elasticsearch/hadoop/current/release-notes.html 5 | --- 6 | 7 | # Elasticsearch for Apache Hadoop release notes [elasticsearch-apache-hadoop-release-notes] 8 | 9 | Review the changes, fixes, and more in each version of Elasticsearch for Apache Hadoop. 10 | 11 | To check for security updates, go to [Security announcements for the Elastic stack](https://discuss.elastic.co/c/announcements/security-announcements/31). 12 | 13 | % Release notes include only features, enhancements, and fixes. Add breaking changes, deprecations, and known issues to the applicable release notes sections. 14 | 15 | % ## version.next [felasticsearch-apache-hadoop-next-release-notes] 16 | 17 | % ### Features and enhancements [elasticsearch-apache-hadoop-next-features-enhancements] 18 | % * 19 | 20 | % ### Fixes [elasticsearch-apache-hadoop-next-fixes] 21 | % * 22 | 23 | ## 9.0.0 [elasticsearch-apache-hadoop-900-release-notes] 24 | 25 | ### Features and enhancements [elasticsearch-apache-hadoop-900-features-enhancements] 26 | 27 | ### Fixes [elasticsearch-apache-hadoop-900-fixes] -------------------------------------------------------------------------------- /docs/release-notes/known-issues.md: -------------------------------------------------------------------------------- 1 | --- 2 | navigation_title: "Known issues" 3 | 4 | --- 5 | 6 | # Elasticsearch Apache Hadoop known issues [elasticsearch-apache-hadoop-known-issues] 7 | 8 | % Use the following template to add entries to this page. 9 | 10 | % :::{dropdown} Title of known issue 11 | % **Details** 12 | % On [Month/Day/Year], a known issue was discovered that [description of known issue]. 13 | 14 | % **Workaround** 15 | % Workaround description. 16 | 17 | % **Resolved** 18 | % On [Month/Day/Year], this issue was resolved. 19 | 20 | ::: -------------------------------------------------------------------------------- /docs/release-notes/toc.yml: -------------------------------------------------------------------------------- 1 | toc: 2 | - file: index.md 3 | - file: known-issues.md 4 | - file: breaking-changes.md 5 | - file: deprecations.md -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | # Enforce the build to fail on deprecated gradle api usage 2 | systemProp.org.gradle.warning.mode=fail 3 | 4 | ## Settings 5 | org.gradle.daemon=false 6 | org.gradle.jvmargs=-Xmx4g 7 | 8 | # java homes resolved by environment variables 9 | org.gradle.java.installations.auto-detect=false 10 | org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME 11 | 12 | ## Dependecies Version 13 | # Logging 14 | slf4jVersion = 2.0.6 15 | log4jVersion = 2.17.1 16 | 17 | # Hadoop versions 18 | hadoop3Version = 3.1.2 19 | hadoop2Version = 2.7.6 20 | hadoop22Version = 2.2.0 21 | 22 | # Common libraries 23 | hiveVersion = 3.1.3 24 | # note the versions here are tied to the ones in Hadoop distro - 1.8.8 25 | jacksonVersion = 1.8.8 26 | 27 | # Spark 28 | spark13Version = 1.6.2 29 | spark20Version = 2.3.0 30 | spark22Version = 2.2.3 31 | spark24Version = 2.4.4 32 | spark30Version = 3.4.3 33 | 34 | # same as Spark's 35 | scala210Version = 2.10.7 36 | scala210MajorVersion = 2.10 37 | scala211Version = 2.11.12 38 | scala211MajorVersion = 2.11 39 | scala212Version = 2.12.19 40 | scala212MajorVersion = 2.12 41 | scala213Version = 2.13.13 42 | scala213MajorVersion = 2.13 43 | 44 | # Testing 45 | junitVersion = 4.11 46 | mockitoVersion = 1.8.5 47 | hamcrestVersion = 1.3 48 | minikdcVersion = 3.4.0 49 | antlrVersion = 3.4 50 | thriftVersion = 0.5.0 51 | 52 | groovyVersion = 2.4.4 53 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/elasticsearch-hadoop/aae4528ee8a31e7106e0ee4b671162fb70cf15f1/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionSha256Sum=fba8464465835e74f7270bbf43d6d8a8d7709ab0a43ce1aa3323f73e9aa0c612 4 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-all.zip 5 | networkTimeout=10000 6 | validateDistributionUrl=true 7 | zipStoreBase=GRADLE_USER_HOME 8 | zipStorePath=wrapper/dists 9 | -------------------------------------------------------------------------------- /hive/licenses/commons-logging-1.1.1.jar.sha1: -------------------------------------------------------------------------------- 1 | 5043bfebc3db072ed80fbd362e7caf00e885d8ae -------------------------------------------------------------------------------- /hive/licenses/commons-logging-NOTICE.txt: -------------------------------------------------------------------------------- 1 | Apache Commons Logging 2 | Copyright 2003-2007 The Apache Software Foundation 3 | 4 | This product includes software developed by 5 | The Apache Software Foundation (http://www.apache.org/). 6 | -------------------------------------------------------------------------------- /hive/licenses/hive-NOTICE.txt: -------------------------------------------------------------------------------- 1 | Apache Hive 2 | Copyright 2008-2014 The Apache Software Foundation 3 | 4 | This product includes software developed by The Apache Software 5 | Foundation (http://www.apache.org/). 6 | 7 | This product includes Jersey (https://jersey.java.net/) 8 | Copyright (c) 2010-2014 Oracle and/or its affiliates. 9 | 10 | This project includes software copyrighted by Microsoft Corporation and 11 | licensed under the Apache License, Version 2.0. -------------------------------------------------------------------------------- /hive/licenses/hive-exec-3.1.3.jar.sha1: -------------------------------------------------------------------------------- 1 | 6cf7f97e1438f94f196b7fb480d5d985a04336a9 -------------------------------------------------------------------------------- /hive/licenses/hive-metastore-3.1.3.jar.sha1: -------------------------------------------------------------------------------- 1 | 99a1348743f3550dd4524408725efab8eb319960 -------------------------------------------------------------------------------- /hive/licenses/hive-service-3.1.3.jar.sha1: -------------------------------------------------------------------------------- 1 | 49fd7ad46801a895d2b185706637a8cb302fc044 -------------------------------------------------------------------------------- /hive/licenses/jaxb-api-2.3.1.jar.sha1: -------------------------------------------------------------------------------- 1 | 8531ad5ac454cc2deb9d4d32c40c4d7451939b5d -------------------------------------------------------------------------------- /hive/licenses/jaxb-api-NOTICE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/elasticsearch-hadoop/aae4528ee8a31e7106e0ee4b671162fb70cf15f1/hive/licenses/jaxb-api-NOTICE.txt -------------------------------------------------------------------------------- /hive/src/itest/java/org/elasticsearch/hadoop/integration/hive/HiveInstance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.integration.hive; 20 | 21 | import java.util.List; 22 | 23 | interface HiveInstance { 24 | 25 | void start() throws Exception; 26 | 27 | List execute(String cleanDdl) throws Exception; 28 | 29 | void stop() throws Exception; 30 | 31 | } 32 | -------------------------------------------------------------------------------- /hive/src/itest/resources/cars-bulk.txt: -------------------------------------------------------------------------------- 1 | { "index": {}} 2 | { "price" : 30000, "color" : "green", "make" : "ford", "sold" : "2014-05-18", "&c" : "foobar" } 3 | { "index": {}} 4 | { "price" : 15000, "color" : "blue", "make" : "toyota", "sold" : "2014-07-02", "&c" : "foobar" } 5 | { "index": {}} 6 | { "price" : 12000, "color" : "green", "make" : "toyota", "sold" : "2014-08-19", "&c" : "foobar" } 7 | { "index": {}} 8 | { "price" : 20000, "color" : "red", "make" : "honda", "sold" : "2014-11-05", "&c" : "foobar" } 9 | { "index": {}} 10 | { "price" : 80000, "color" : "red", "make" : "bmw", "sold" : "2014-01-01", "&c" : "foobar" } 11 | { "index": {}} 12 | { "price" : 25000, "color" : "blue", "make" : "ford", "sold" : "2014-02-12", "&c" : "foobar" } 13 | -------------------------------------------------------------------------------- /hive/src/itest/resources/org/elasticsearch/hadoop/hive/hive-date-mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "data" : { 3 | "properties" : { 4 | "&t" : { 5 | "type" : "date" 6 | } 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /hive/src/itest/resources/org/elasticsearch/hadoop/hive/hive-date-mappingresponse.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "properties" : { 5 | "&t" : { 6 | "type" : "date" 7 | } 8 | } 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /hive/src/itest/resources/org/elasticsearch/hadoop/hive/hive-date-source.json: -------------------------------------------------------------------------------- 1 | { 2 | "_scroll_id": "c2NhbjswOzE7dG90YWxfaGl0czoxOw==", 3 | "took": 18, 4 | "timed_out": false, 5 | "_shards": { 6 | "total": 1, 7 | "successful": 1, 8 | "failed": 0 9 | }, 10 | "hits": { 11 | "total": 1, 12 | "max_score": 0.0, 13 | "hits": [ 14 | { 15 | "_index": "hive", 16 | "_type": "date-as-long", 17 | "_id": "1", 18 | "_score": 0.0, 19 | "_source": { 20 | "type": 1, 21 | "&t": 1407239910771 22 | } 23 | } 24 | ] 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /hive/src/itest/resources/org/elasticsearch/hadoop/hive/hive-date-typeless-mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "properties" : { 3 | "&t" : { 4 | "type" : "date" 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /hive/src/main/java/org/elasticsearch/hadoop/hive/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | /** 20 | * Integration for Apache Hive. 21 | *

22 | * Exposes Elasticsearch as tables which can be natively updated or queried through HiveQL. 23 | */ 24 | package org.elasticsearch.hadoop.hive; 25 | -------------------------------------------------------------------------------- /hive/src/test/resources/org/elasticsearch/hadoop/hive/hive-date-mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "data" : { 3 | "properties" : { 4 | "&t" : { 5 | "type" : "date" 6 | } 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /hive/src/test/resources/org/elasticsearch/hadoop/hive/hive-date-mappingresponse.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "properties" : { 5 | "&t" : { 6 | "type" : "date" 7 | } 8 | } 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /hive/src/test/resources/org/elasticsearch/hadoop/hive/hive-date-source.json: -------------------------------------------------------------------------------- 1 | { 2 | "_scroll_id": "c2NhbjswOzE7dG90YWxfaGl0czoxOw==", 3 | "took": 18, 4 | "timed_out": false, 5 | "_shards": { 6 | "total": 1, 7 | "successful": 1, 8 | "failed": 0 9 | }, 10 | "hits": { 11 | "total": 1, 12 | "max_score": 0.0, 13 | "hits": [ 14 | { 15 | "_index": "hive", 16 | "_type": "date-as-long", 17 | "_id": "1", 18 | "_score": 0.0, 19 | "_source": { 20 | "type": 1, 21 | "&t": 1407239910771 22 | } 23 | } 24 | ] 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /hive/src/test/resources/org/elasticsearch/hadoop/hive/hive-date-typeless-mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "properties" : { 3 | "&t" : { 4 | "type" : "date" 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /licenses/commons-codec-1.4.jar.sha1: -------------------------------------------------------------------------------- 1 | 4216af16d38465bbab0f3dff8efa14204f7a399a -------------------------------------------------------------------------------- /licenses/commons-codec-NOTICE.txt: -------------------------------------------------------------------------------- 1 | Apache Commons Codec 2 | Copyright 2002-2009 The Apache Software Foundation 3 | 4 | This product includes software developed by 5 | The Apache Software Foundation (http://www.apache.org/). 6 | 7 | -------------------------------------------------------------------------------- 8 | src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java contains 9 | test data from http://aspell.sourceforge.net/test/batch0.tab. 10 | 11 | Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org). Verbatim copying 12 | and distribution of this entire article is permitted in any medium, 13 | provided this notice is preserved. 14 | -------------------------------------------------------------------------------- -------------------------------------------------------------------------------- /licenses/commons-logging-1.1.1.jar.sha1: -------------------------------------------------------------------------------- 1 | 5043bfebc3db072ed80fbd362e7caf00e885d8ae -------------------------------------------------------------------------------- /licenses/commons-logging-NOTICE.txt: -------------------------------------------------------------------------------- 1 | Apache Commons Logging 2 | Copyright 2003-2007 The Apache Software Foundation 3 | 4 | This product includes software developed by 5 | The Apache Software Foundation (http://www.apache.org/). 6 | -------------------------------------------------------------------------------- /licenses/hadoop-NOTICE.txt: -------------------------------------------------------------------------------- 1 | This product includes software developed by The Apache Software 2 | Foundation (http://www.apache.org/). -------------------------------------------------------------------------------- /licenses/hadoop-client-3.1.2.jar.sha1: -------------------------------------------------------------------------------- 1 | 6a815ff6298839c4fb1b96859c583c69f0f7c80c -------------------------------------------------------------------------------- /licenses/hadoop-common-3.1.2.jar.sha1: -------------------------------------------------------------------------------- 1 | 4742708e260bbdee3dda478fd440966c0b0967c0 -------------------------------------------------------------------------------- /licenses/hadoop-mapreduce-client-core-3.1.2.jar.sha1: -------------------------------------------------------------------------------- 1 | 4e7b3fe8875c01c9264f2e31043b04503e3f7022 -------------------------------------------------------------------------------- /licenses/hive-NOTICE.txt: -------------------------------------------------------------------------------- 1 | Apache Hive 2 | Copyright 2008-2014 The Apache Software Foundation 3 | 4 | This product includes software developed by The Apache Software 5 | Foundation (http://www.apache.org/). 6 | 7 | This product includes Jersey (https://jersey.java.net/) 8 | Copyright (c) 2010-2014 Oracle and/or its affiliates. 9 | 10 | This project includes software copyrighted by Microsoft Corporation and 11 | licensed under the Apache License, Version 2.0. -------------------------------------------------------------------------------- /licenses/hive-exec-3.1.3.jar.sha1: -------------------------------------------------------------------------------- 1 | 6cf7f97e1438f94f196b7fb480d5d985a04336a9 -------------------------------------------------------------------------------- /licenses/hive-metastore-3.1.3.jar.sha1: -------------------------------------------------------------------------------- 1 | 99a1348743f3550dd4524408725efab8eb319960 -------------------------------------------------------------------------------- /licenses/hive-service-3.1.3.jar.sha1: -------------------------------------------------------------------------------- 1 | 49fd7ad46801a895d2b185706637a8cb302fc044 -------------------------------------------------------------------------------- /licenses/jaxb-api-2.3.1.jar.sha1: -------------------------------------------------------------------------------- 1 | 8531ad5ac454cc2deb9d4d32c40c4d7451939b5d -------------------------------------------------------------------------------- /licenses/jaxb-api-NOTICE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/elasticsearch-hadoop/aae4528ee8a31e7106e0ee4b671162fb70cf15f1/licenses/jaxb-api-NOTICE.txt -------------------------------------------------------------------------------- /licenses/scala-library-2.13.13.jar.sha1: -------------------------------------------------------------------------------- 1 | 3b1434403ddb147b8030f98f246153e4ce622c78 -------------------------------------------------------------------------------- /licenses/scala-reflect-2.13.13.jar.sha1: -------------------------------------------------------------------------------- 1 | 46866f0e2cf6168322dc11f0f721c2db78d52ac7 -------------------------------------------------------------------------------- /licenses/slf4j-api-2.0.6.jar.sha1: -------------------------------------------------------------------------------- 1 | 88c40d8b4f33326f19a7d3c0aaf2c7e8721d4953 -------------------------------------------------------------------------------- /licenses/slf4j-api-LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2004-2017 QOS.ch 2 | All rights reserved. 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining 5 | a copy of this software and associated documentation files (the 6 | "Software"), to deal in the Software without restriction, including 7 | without limitation the rights to use, copy, modify, merge, publish, 8 | distribute, sublicense, and/or sell copies of the Software, and to 9 | permit persons to whom the Software is furnished to do so, subject to 10 | the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /licenses/slf4j-api-NOTICE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/elasticsearch-hadoop/aae4528ee8a31e7106e0ee4b671162fb70cf15f1/licenses/slf4j-api-NOTICE.txt -------------------------------------------------------------------------------- /licenses/spark-catalyst_2.13-3.4.3.jar.sha1: -------------------------------------------------------------------------------- 1 | 6a9e9c5e149bd019dd5d781888e1e81ca4d32875 -------------------------------------------------------------------------------- /licenses/spark-core_2.13-3.4.3.jar.sha1: -------------------------------------------------------------------------------- 1 | a8055d61a13e9af6871e05ecdca141134d102f1f -------------------------------------------------------------------------------- /licenses/spark-sql_2.13-3.4.3.jar.sha1: -------------------------------------------------------------------------------- 1 | b1744080f5e2f81b9a1a088eb375d33b85132b6d -------------------------------------------------------------------------------- /licenses/spark-streaming_2.13-3.4.3.jar.sha1: -------------------------------------------------------------------------------- 1 | 64c4e38f3a89428926383f4d9f8a8636a81d499b -------------------------------------------------------------------------------- /licenses/spark-yarn_2.13-3.4.3.jar.sha1: -------------------------------------------------------------------------------- 1 | d8fd5251bb23d3bed9d9ef1f19c4402211c78fe4 -------------------------------------------------------------------------------- /mr/licenses/commons-codec-1.4.jar.sha1: -------------------------------------------------------------------------------- 1 | 4216af16d38465bbab0f3dff8efa14204f7a399a -------------------------------------------------------------------------------- /mr/licenses/commons-codec-NOTICE.txt: -------------------------------------------------------------------------------- 1 | Apache Commons Codec 2 | Copyright 2002-2009 The Apache Software Foundation 3 | 4 | This product includes software developed by 5 | The Apache Software Foundation (http://www.apache.org/). 6 | 7 | -------------------------------------------------------------------------------- 8 | src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java contains 9 | test data from http://aspell.sourceforge.net/test/batch0.tab. 10 | 11 | Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org). Verbatim copying 12 | and distribution of this entire article is permitted in any medium, 13 | provided this notice is preserved. 14 | -------------------------------------------------------------------------------- -------------------------------------------------------------------------------- /mr/licenses/commons-logging-1.1.1.jar.sha1: -------------------------------------------------------------------------------- 1 | 5043bfebc3db072ed80fbd362e7caf00e885d8ae -------------------------------------------------------------------------------- /mr/licenses/commons-logging-NOTICE.txt: -------------------------------------------------------------------------------- 1 | Apache Commons Logging 2 | Copyright 2003-2007 The Apache Software Foundation 3 | 4 | This product includes software developed by 5 | The Apache Software Foundation (http://www.apache.org/). 6 | -------------------------------------------------------------------------------- /mr/licenses/hadoop-NOTICE.txt: -------------------------------------------------------------------------------- 1 | This product includes software developed by The Apache Software 2 | Foundation (http://www.apache.org/). -------------------------------------------------------------------------------- /mr/licenses/hadoop-client-3.1.2.jar.sha1: -------------------------------------------------------------------------------- 1 | 6a815ff6298839c4fb1b96859c583c69f0f7c80c -------------------------------------------------------------------------------- /mr/licenses/hadoop-common-3.1.2.jar.sha1: -------------------------------------------------------------------------------- 1 | 4742708e260bbdee3dda478fd440966c0b0967c0 -------------------------------------------------------------------------------- /mr/licenses/hadoop-mapreduce-client-core-3.1.2.jar.sha1: -------------------------------------------------------------------------------- 1 | 4e7b3fe8875c01c9264f2e31043b04503e3f7022 -------------------------------------------------------------------------------- /mr/licenses/jaxb-api-2.3.1.jar.sha1: -------------------------------------------------------------------------------- 1 | 8531ad5ac454cc2deb9d4d32c40c4d7451939b5d -------------------------------------------------------------------------------- /mr/licenses/jaxb-api-NOTICE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/elasticsearch-hadoop/aae4528ee8a31e7106e0ee4b671162fb70cf15f1/mr/licenses/jaxb-api-NOTICE.txt -------------------------------------------------------------------------------- /mr/src/itest/java/org/elasticsearch/hadoop/ProvisionerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop; 20 | 21 | import org.junit.Test; 22 | 23 | public class ProvisionerTest { 24 | 25 | @Test 26 | public void testBasicProvisier() { 27 | Provisioner.provision(HdpBootstrap.hadoopConfig()); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /mr/src/itest/java/org/elasticsearch/hadoop/integration/rest/RestSuite.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.integration.rest; 20 | 21 | import org.elasticsearch.hadoop.fixtures.LocalEs; 22 | import org.junit.ClassRule; 23 | import org.junit.Ignore; 24 | import org.junit.rules.ExternalResource; 25 | import org.junit.runner.RunWith; 26 | import org.junit.runners.Suite; 27 | 28 | @Ignore 29 | @RunWith(Suite.class) 30 | @Suite.SuiteClasses({ AbstractRestSaveTest.class, AbstractRestQueryTest.class }) 31 | public class RestSuite { 32 | 33 | @ClassRule 34 | public static ExternalResource resource = new LocalEs(); 35 | } 36 | -------------------------------------------------------------------------------- /mr/src/itest/java/org/elasticsearch/hadoop/rest/pooling/PoolingSuite.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.hadoop.rest.pooling; 21 | 22 | import org.elasticsearch.hadoop.fixtures.LocalEs; 23 | import org.junit.ClassRule; 24 | import org.junit.rules.ExternalResource; 25 | import org.junit.runner.RunWith; 26 | import org.junit.runners.Suite; 27 | 28 | @RunWith(Suite.class) 29 | @Suite.SuiteClasses({ AbstractTransportPoolTest.class }) 30 | public class PoolingSuite { 31 | 32 | @ClassRule 33 | public static ExternalResource resource = new LocalEs(); 34 | 35 | } 36 | -------------------------------------------------------------------------------- /mr/src/itest/resources/org/elasticsearch/hadoop/integration/mr-child.json: -------------------------------------------------------------------------------- 1 | { 2 | "child" : { 3 | "_parent" : { 4 | "type" : "parent" 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /mr/src/itest/resources/org/elasticsearch/hadoop/integration/mr-multi-value-string.json: -------------------------------------------------------------------------------- 1 | { 2 | "my_array" : ["1.a", "1.b"] 3 | } -------------------------------------------------------------------------------- /mr/src/itest/resources/org/elasticsearch/hadoop/integration/mr-nested.json: -------------------------------------------------------------------------------- 1 | { 2 | "data" : { 3 | "properties" : { 4 | "url" : { 5 | "type" : "nested" 6 | } 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /mr/src/itest/resources/org/elasticsearch/hadoop/integration/mr-parent.json: -------------------------------------------------------------------------------- 1 | { 2 | "parent": {} 3 | } -------------------------------------------------------------------------------- /mr/src/itest/resources/ssl/client.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/elasticsearch-hadoop/aae4528ee8a31e7106e0ee4b671162fb70cf15f1/mr/src/itest/resources/ssl/client.jks -------------------------------------------------------------------------------- /mr/src/itest/resources/ssl/server.cer: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/elasticsearch-hadoop/aae4528ee8a31e7106e0ee4b671162fb70cf15f1/mr/src/itest/resources/ssl/server.cer -------------------------------------------------------------------------------- /mr/src/itest/resources/ssl/server.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/elasticsearch-hadoop/aae4528ee8a31e7106e0ee4b671162fb70cf15f1/mr/src/itest/resources/ssl/server.jks -------------------------------------------------------------------------------- /mr/src/itest/resources/ssl/server.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDrTCCApWgAwIBAgIEK/njaDANBgkqhkiG9w0BAQsFADCBhTELMAkGA1UEBhMC 3 | Uk8xEjAQBgNVBAgTCUJ1Y2hhcmVzdDESMBAGA1UEBxMJQnVjaGFyZXN0MRYwFAYD 4 | VQQKEw1FbGFzdGljc2VhcmNoMR0wGwYDVQQLExRFbGFzdGljc2VhcmNoIEhhZG9v 5 | cDEXMBUGA1UEAxMOZXMtaGFkb29wIHRlc3QwIBcNMTQxMDI4MTY0NjE2WhgPMjI4 6 | ODA4MTExNjQ2MTZaMIGFMQswCQYDVQQGEwJSTzESMBAGA1UECBMJQnVjaGFyZXN0 7 | MRIwEAYDVQQHEwlCdWNoYXJlc3QxFjAUBgNVBAoTDUVsYXN0aWNzZWFyY2gxHTAb 8 | BgNVBAsTFEVsYXN0aWNzZWFyY2ggSGFkb29wMRcwFQYDVQQDEw5lcy1oYWRvb3Ag 9 | dGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALyAOIn6OxGmjM48 10 | 24gRpVO+Ncs6G8ZyqjLxm5WSAbeFOBuFFOrCGsRpv2h5U5RYElqIZ9Szj8QM7/f4 11 | nlmc4zWXcq4t+wpSvGa5helEIk3+aYsvsiBCnuwsfX0WfmT9CI64wNI2gUpLM+ex 12 | 3dhTA9EzxyZytc2E/VP9wBlwMR6ufbhxLOSIWYHW5CH2rtxj2qeKUkKTcn9ZRxwg 13 | JvheXyKQzMzhwNvyvOBqOn2CYdFUWS99pvxUp0lNG+faeMtxqoQLy1Bcp/O1+Svq 14 | hVTBoeUwSKmaJueY7zTDgtA/Q3ijmpd7CoefL9Nw/djxoETeP2xwmLqUsvLsT7fq 15 | zb8aoBcCAwEAAaMhMB8wHQYDVR0OBBYEFJrjXHPRzzYghdo7kBZ00ozizJaTMA0G 16 | CSqGSIb3DQEBCwUAA4IBAQCyWTtuyFMrurux+mi17jIgxakk0gLdm96tjUJ4ZIBm 17 | qgU82FvPGTkP7+Bbf/TNl7Llq3EWBYhhVSEKyQRawobfIX8Pqez3UnQeP2gisIdF 18 | v+6pE0UJowR0tJEiHaBFS1zVci5/t4+MTCIKMlhLopcZHp7LU7CgpF29sKiFVlJ0 19 | z9LK+EVeeFgUUMMk1114h9ycyo1GrhbYFH8svDoVmqIaMhPy1NjoJNfUmxYlkWga 20 | qFkb1jLCncKC43YVK9Xm/Si3mqWD5gbzUavJunxdKtMlStu/HBp++cNIZoMQB/k4 21 | UdFJEUIx0126G1qYObBcAC8AEEiqcWkfOp50HuBmnfct 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/EsHadoopException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop; 20 | 21 | /** 22 | * Base class for Elasticsearch Hadoop exceptions. 23 | */ 24 | public class EsHadoopException extends RuntimeException { 25 | 26 | public EsHadoopException() { 27 | super(); 28 | } 29 | 30 | public EsHadoopException(String message, Throwable cause) { 31 | super(message, cause); 32 | } 33 | 34 | public EsHadoopException(String message) { 35 | super(message); 36 | } 37 | 38 | public EsHadoopException(Throwable cause) { 39 | super(cause); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/EsHadoopIllegalArgumentException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop; 20 | 21 | 22 | public class EsHadoopIllegalArgumentException extends EsHadoopException { 23 | 24 | public EsHadoopIllegalArgumentException() { 25 | super(); 26 | } 27 | 28 | public EsHadoopIllegalArgumentException(String message, Throwable cause) { 29 | super(message, cause); 30 | } 31 | 32 | public EsHadoopIllegalArgumentException(String message) { 33 | super(message); 34 | } 35 | 36 | public EsHadoopIllegalArgumentException(Throwable cause) { 37 | super(cause); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/EsHadoopIllegalStateException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop; 20 | 21 | public class EsHadoopIllegalStateException extends EsHadoopException { 22 | 23 | public EsHadoopIllegalStateException() { 24 | super(); 25 | } 26 | 27 | public EsHadoopIllegalStateException(String message, Throwable cause) { 28 | super(message, cause); 29 | } 30 | 31 | public EsHadoopIllegalStateException(String message) { 32 | super(message); 33 | } 34 | 35 | public EsHadoopIllegalStateException(Throwable cause) { 36 | super(cause); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/cfg/FieldPresenceValidation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.cfg; 20 | 21 | public enum FieldPresenceValidation { 22 | IGNORE, 23 | WARN, 24 | STRICT; 25 | 26 | public boolean isRequired() { 27 | return !(IGNORE == this); 28 | } 29 | } -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/cfg/SettingsManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.cfg; 20 | 21 | public interface SettingsManager { 22 | 23 | Settings load(T configuration); 24 | } 25 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/cfg/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | /** 21 | * Configuration package that handles loading across different environments. 22 | */ 23 | package org.elasticsearch.hadoop.cfg; 24 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/cli/Prompt.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.hadoop.cli; 21 | 22 | public interface Prompt { 23 | 24 | void println(); 25 | 26 | void println(String s); 27 | 28 | void printf(String format, Object... args); 29 | 30 | String readLine(); 31 | 32 | String readLine(String format, Object... args); 33 | 34 | char[] readPassword(String prompt, Object... args); 35 | } 36 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/handler/HandlerLoader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.hadoop.handler; 21 | 22 | import java.util.List; 23 | 24 | /** 25 | * Any object that can produce a list of error handlers of a given type. 26 | * @param ErrorHandler type. 27 | */ 28 | public interface HandlerLoader { 29 | List loadHandlers(); 30 | } 31 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/mr/compat/CompatProxy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.mr.compat; 20 | 21 | /** 22 | * Marker interface for compatible proxy. 23 | */ 24 | public interface CompatProxy { 25 | 26 | } 27 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/mr/compat/Counter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.mr.compat; 20 | 21 | public interface Counter { 22 | 23 | void increment(long incr); 24 | } 25 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/mr/compat/CounterGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.mr.compat; 20 | 21 | public interface CounterGroup { 22 | 23 | } 24 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/mr/compat/JobContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.mr.compat; 20 | 21 | import org.apache.hadoop.conf.Configuration; 22 | 23 | 24 | public interface JobContext { 25 | 26 | Configuration getConfiguration(); 27 | } 28 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/mr/compat/MapContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.mr.compat; 20 | 21 | public interface MapContext { 22 | 23 | } 24 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/mr/compat/ReduceContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.mr.compat; 20 | 21 | public interface ReduceContext { 22 | 23 | } 24 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/mr/compat/TaskAttemptContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.mr.compat; 20 | 21 | import org.apache.hadoop.conf.Configuration; 22 | import org.apache.hadoop.util.Progressable; 23 | 24 | public interface TaskAttemptContext extends Progressable { 25 | 26 | void setStatus(String string); 27 | 28 | Configuration getConfiguration(); 29 | } 30 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/mr/compat/TaskInputOutputContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.mr.compat; 20 | 21 | public interface TaskInputOutputContext { 22 | 23 | org.apache.hadoop.mapreduce.Counter getCounter(Enum counter); 24 | } 25 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/mr/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | /** 20 | * Package providing integration between Elasticsearch and 21 | * Map/Reduce package ('org.apache.hadoop.mapred' and 'org.apache.hadoop.mapreduce'). 22 | * 23 | *

24 | * 25 | * Defaults and settings can be configured through Hadoop configuration object. See {@link org.elasticsearch.hadoop.cfg.ConfigurationOptions} for available properties. 26 | */ 27 | package org.elasticsearch.hadoop.mr; 28 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | /** 20 | * Base package providing common exceptions used by the Elasticsearch Hadoop library. 21 | */ 22 | package org.elasticsearch.hadoop; 23 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/rest/HttpRetryPolicy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.rest; 20 | 21 | 22 | public interface HttpRetryPolicy { 23 | 24 | Retry init(); 25 | 26 | } 27 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/rest/NoHttpRetryPolicy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.rest; 20 | 21 | public class NoHttpRetryPolicy implements HttpRetryPolicy { 22 | 23 | private static class NoRetry implements Retry { 24 | @Override 25 | public boolean retry(int httpStatus) { 26 | return false; 27 | } 28 | } 29 | @Override 30 | public Retry init() { 31 | return new NoRetry(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/rest/Request.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.rest; 20 | 21 | import org.elasticsearch.hadoop.util.ByteSequence; 22 | 23 | 24 | public interface Request { 25 | 26 | enum Method { 27 | DELETE, GET, HEAD, POST, PUT 28 | }; 29 | 30 | Method method(); 31 | 32 | CharSequence uri(); 33 | 34 | CharSequence path(); 35 | 36 | CharSequence params(); 37 | 38 | ByteSequence body(); 39 | } 40 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/rest/Retry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.rest; 20 | 21 | 22 | public interface Retry { 23 | boolean retry(int httpStatus); 24 | } 25 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/rest/ReusableInputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.rest; 20 | 21 | import java.io.InputStream; 22 | 23 | /** 24 | * Marker interface indicating whether the underlying inputstream can be reused/replayed. 25 | */ 26 | public interface ReusableInputStream { 27 | 28 | InputStream copy(); 29 | 30 | } 31 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/rest/Transport.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.rest; 20 | 21 | import java.io.Closeable; 22 | import java.io.IOException; 23 | 24 | import org.elasticsearch.hadoop.rest.stats.StatsAware; 25 | 26 | public interface Transport extends Closeable, StatsAware { 27 | 28 | Response execute(Request request) throws IOException; 29 | 30 | void close(); 31 | } 32 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/rest/bulk/handler/IBulkWriteErrorHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.hadoop.rest.bulk.handler; 21 | 22 | import org.elasticsearch.hadoop.handler.ErrorHandler; 23 | 24 | public interface IBulkWriteErrorHandler extends ErrorHandler> { 25 | 26 | } 27 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/rest/commonshttp/DeleteMethodWithBody.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.rest.commonshttp; 20 | 21 | import org.elasticsearch.hadoop.thirdparty.apache.commons.httpclient.methods.EntityEnclosingMethod; 22 | 23 | public class DeleteMethodWithBody extends EntityEnclosingMethod { 24 | 25 | public DeleteMethodWithBody() {} 26 | 27 | public DeleteMethodWithBody(String uri) { 28 | super(uri); 29 | } 30 | 31 | public String getName() { 32 | return "DELETE"; 33 | } 34 | } 35 | 36 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/rest/request/RequestBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.rest.request; 20 | 21 | import org.elasticsearch.hadoop.rest.RestClient; 22 | 23 | public abstract class RequestBuilder { 24 | protected final RestClient client; 25 | 26 | public RequestBuilder(RestClient client) { 27 | this.client = client; 28 | } 29 | 30 | public abstract T execute(); 31 | 32 | interface Response {} 33 | } -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/rest/stats/StatsAware.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.rest.stats; 20 | 21 | /** 22 | * Interface exposing stats. 23 | */ 24 | public interface StatsAware { 25 | 26 | /** 27 | * Returns the stats of the given component. Potentially expensive call (since it implies doing internal aggregations) so call it only when necessary. 28 | * 29 | * @return component stats. 30 | */ 31 | Stats stats(); 32 | } 33 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/serialization/BytesConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.serialization; 20 | 21 | import org.elasticsearch.hadoop.util.BytesArray; 22 | 23 | public interface BytesConverter { 24 | 25 | /** 26 | * Writes the given object to the given {@link BytesArray}. 27 | * 28 | * @param from object to write 29 | * @param to storage 30 | */ 31 | void convert(Object from, BytesArray to); 32 | } 33 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/serialization/SettingsAware.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.serialization; 20 | 21 | import org.elasticsearch.hadoop.cfg.Settings; 22 | 23 | /** 24 | * Whether a ValueReader/Writer need extra configuration properties. 25 | */ 26 | public interface SettingsAware { 27 | 28 | void setSettings(Settings settings); 29 | } 30 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/serialization/bulk/BulkFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.serialization.bulk; 20 | 21 | 22 | interface BulkFactory { 23 | 24 | BulkCommand createBulk(); 25 | } 26 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/serialization/dto/mapping/GeoField.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.serialization.dto.mapping; 20 | 21 | import org.elasticsearch.hadoop.serialization.FieldType; 22 | 23 | public interface GeoField { 24 | 25 | public enum GeoType { 26 | GEO_POINT, 27 | GEO_SHAPE; 28 | } 29 | 30 | public FieldType rawType(); 31 | public int arrayDepth(); 32 | } 33 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/serialization/field/DefaultIndexExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.serialization.field; 20 | 21 | import org.elasticsearch.hadoop.util.ObjectUtils; 22 | 23 | public class DefaultIndexExtractor extends AbstractIndexExtractor { 24 | 25 | protected FieldExtractor createFieldExtractor(String fieldName) { 26 | settings.setProperty(ConstantFieldExtractor.PROPERTY, fieldName); 27 | return ObjectUtils.instantiate(settings.getMappingDefaultClassExtractor(), settings); 28 | } 29 | } -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/serialization/field/DefaultParamsExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.serialization.field; 20 | 21 | import org.elasticsearch.hadoop.util.ObjectUtils; 22 | 23 | public class DefaultParamsExtractor extends AbstractDefaultParamsExtractor { 24 | 25 | @Override 26 | protected FieldExtractor createFieldExtractor(String fieldName) { 27 | settings.setProperty(ConstantFieldExtractor.PROPERTY, fieldName); 28 | return ObjectUtils. instantiate(settings.getMappingDefaultClassExtractor(), settings); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/serialization/field/FieldExplainer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.serialization.field; 20 | 21 | public interface FieldExplainer { 22 | 23 | String toString(Object field); 24 | } 25 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/serialization/field/IndexExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.serialization.field; 20 | 21 | 22 | 23 | /** 24 | * Produces an index name based on the given pattern and arguments (field values). 25 | */ 26 | public interface IndexExtractor extends FieldExtractor { 27 | 28 | void compile(String pattern); 29 | 30 | /** 31 | * Indicates whether the given string is a pattern or not. 32 | * 33 | * @return true for a pattern, false otherwise 34 | */ 35 | boolean hasPattern(); 36 | } 37 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/serialization/field/IndexFormatter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.serialization.field; 20 | 21 | public interface IndexFormatter { 22 | 23 | void configure(String format); 24 | 25 | String format(String value); 26 | } 27 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/serialization/field/NoOpFieldExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.serialization.field; 20 | 21 | import org.elasticsearch.hadoop.EsHadoopIllegalStateException; 22 | 23 | public class NoOpFieldExtractor implements FieldExtractor { 24 | 25 | @Override 26 | public String field(Object target) { 27 | throw new EsHadoopIllegalStateException("Should not be called - indicates a configuration issue or a bug"); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/serialization/handler/read/IDeserializationErrorHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.hadoop.serialization.handler.read; 21 | 22 | import org.elasticsearch.hadoop.handler.ErrorCollector; 23 | import org.elasticsearch.hadoop.handler.ErrorHandler; 24 | 25 | public interface IDeserializationErrorHandler extends ErrorHandler> { 26 | } 27 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/serialization/handler/write/ISerializationErrorHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.hadoop.serialization.handler.write; 21 | 22 | import org.elasticsearch.hadoop.handler.ErrorCollector; 23 | import org.elasticsearch.hadoop.handler.ErrorHandler; 24 | 25 | public interface ISerializationErrorHandler extends ErrorHandler> { 26 | } 27 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/serialization/json/ObjectReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.serialization.json; 20 | 21 | import org.elasticsearch.hadoop.thirdparty.codehaus.jackson.JsonParser; 22 | 23 | import java.io.IOException; 24 | import java.util.Iterator; 25 | 26 | public interface ObjectReader { 27 | 28 | Iterator readValues(JsonParser parser) throws IOException; 29 | } 30 | -------------------------------------------------------------------------------- /mr/src/main/java/org/elasticsearch/hadoop/util/ByteSequence.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.util; 20 | 21 | import java.io.IOException; 22 | import java.io.OutputStream; 23 | 24 | public interface ByteSequence { 25 | 26 | int length(); 27 | 28 | void writeTo(OutputStream out) throws IOException; 29 | } 30 | -------------------------------------------------------------------------------- /mr/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier: -------------------------------------------------------------------------------- 1 | org.elasticsearch.hadoop.mr.security.EsTokenIdentifier -------------------------------------------------------------------------------- /mr/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer: -------------------------------------------------------------------------------- 1 | org.elasticsearch.hadoop.mr.security.EsTokenIdentifier$Renewer -------------------------------------------------------------------------------- /mr/src/main/resources/org/elasticsearch/hadoop/mr/Counter.properties: -------------------------------------------------------------------------------- 1 | CounterGroupName=Elasticsearch Hadoop Counters 2 | BYTES_RECEIVED.name=Bytes Received 3 | BYTES_ACCEPTED.name=Bytes Accepted 4 | BYTES_RETRIED.name=Bytes Retried 5 | BYTES_SENT.name=Bytes Sent 6 | 7 | BULK_RETRIES.name=Bulk Retries 8 | BULK_RETRIES_TOTAL_TIME_MS.name=Bulk Retries Total Time(ms) 9 | BULK_TOTAL.name=Bulk Total 10 | BULK_TOTAL_TIME_MS.name=Bulk Total Time(ms) 11 | 12 | DOCS_RECEIVED.name=Documents Received 13 | DOCS_ACCEPTED.name=Documents Accepted 14 | DOCS_RETRIED.name=Documents Retried 15 | DOCS_SENT.name=Documents Sent 16 | 17 | NODE_RETRIES.name=Node Retries 18 | NET_RETRIES.name=Network Retries 19 | NET_TOTAL_TIME_MS.name=Network Total Time(ms) 20 | 21 | SCROLL_TOTAL.name=Scroll Total 22 | SCROLL_TOTAL_TIME_MS.name=Scroll Total Time(ms) 23 | -------------------------------------------------------------------------------- /mr/src/test/java/org/elasticsearch/hadoop/serialization/JdkValueReaderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.hadoop.serialization; 21 | 22 | public class JdkValueReaderTest extends BaseValueReaderTest {} 23 | -------------------------------------------------------------------------------- /mr/src/test/java/org/elasticsearch/hadoop/serialization/builder/NoOpValueWriterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.hadoop.serialization.builder; 20 | 21 | import org.elasticsearch.hadoop.EsHadoopIllegalStateException; 22 | import org.junit.Test; 23 | 24 | 25 | public class NoOpValueWriterTest { 26 | 27 | @Test(expected = EsHadoopIllegalStateException.class) 28 | public void testWrite() { 29 | ValueWriter writer = new NoOpValueWriter(); 30 | writer.write(null, null); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /mr/src/test/resources/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /mr/src/test/resources/log4jnull.properties: -------------------------------------------------------------------------------- 1 | # logging configuration used for coverage 2 | status=error 3 | 4 | rootLogger.level=info 5 | rootLogger.appenderRef.null.ref=null 6 | 7 | appender.null.type=NullAppender 8 | appender.null.name=null 9 | appender.null.layout.type=PatternLayout 10 | appender.null.layout.pattern=%d{ABSOLUTE} %5p %t %c{2} - %m%n 11 | 12 | # trim Hive logging 13 | logger.Hive-DataNucleus.name=DataNucleus 14 | logger.Hive-DataNucleus.level=error 15 | logger.Hive-SessionState.name=SessionState 16 | logger.Hive-SessionState.level=fatal 17 | logger.Hive-ExecMapper.name=ExecMapper 18 | logger.Hive-ExecMapper.level=warn 19 | logger.Hive-HiveQL-Package.name=org.apache.hadoop.hive.ql 20 | logger.Hive-HiveQL-Package.level=warn 21 | logger.Hive-HiveQL-Driver.name=org.apache.hadoop.hive.ql.Driver 22 | logger.Hive-HiveQL-Driver.level=warn 23 | logger.Hive-HiveQL-RetryHandler.name=org.apache.hadoop.hive.metastore.RetryingHMSHandler 24 | logger.Hive-HiveQL-RetryHandler.level=fatal 25 | logger.Hive-HiveQL-Exec-Package.name=org.apache.hadoop.hive.ql.exec 26 | logger.Hive-HiveQL-Exec-Package.level=warn 27 | logger.Hive-Metastore.name=org.apache.hadoop.hive.metastore.HiveMetaStore 28 | logger.Hive-Metastore.level=warn 29 | 30 | # ES-Hadoop logging 31 | logger.ESH-Package.name=org.elasticsearch.hadoop 32 | logger.ESH-Package.level=trace 33 | -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/rest/bulk-retry-input-template.json: -------------------------------------------------------------------------------- 1 | {"index":{}} 2 | {"element":"w"} 3 | -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/rest/get-aliases-empty-response.json: -------------------------------------------------------------------------------- 1 | { 2 | "index1": { 3 | "aliases": { 4 | } 5 | } 6 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/rest/get-aliases-response.json: -------------------------------------------------------------------------------- 1 | { 2 | "index1": { 3 | "aliases": { 4 | "alias1": { 5 | "filter": { 6 | "term": { 7 | "system": "hadoop" 8 | } 9 | }, 10 | "index_routing": "2", 11 | "search_routing": "1,2" 12 | }, 13 | "alias2": { 14 | "filter": { 15 | "term": { 16 | "system": "spark" 17 | } 18 | }, 19 | "index_routing": "2", 20 | "search_routing": "1,2" 21 | } 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/client-nodes-v1.json: -------------------------------------------------------------------------------- 1 | { 2 | "ok" : true, 3 | "cluster_name" : "elasticsearch", 4 | "nodes" : { 5 | "fiR5azTbTDiX59m78yzOTw" : { 6 | "name" : "Darkhawk", 7 | "transport_address" : "inet[/192.168.1.50:9300]", 8 | "host" : "cerberus", 9 | "version" : "1.7.3", 10 | "http_address" : "inet[/192.168.1.50:9200]", 11 | "attributes" : { 12 | "data" : "false", 13 | "master" : "false" 14 | }, 15 | "http" : { 16 | "bound_address" : "inet[/0:0:0:0:0:0:0:0:9200]", 17 | "publish_address" : "inet[/192.168.1.50:9200]", 18 | "max_content_length" : "100mb", 19 | "max_content_length_in_bytes" : 104857600 20 | } 21 | }, 22 | "L9DM79IvStq2RStogXR8Sg" : { 23 | "name" : "Unseen", 24 | "transport_address" : "inet[/192.168.1.50:9301]", 25 | "host" : "cerberus", 26 | "version" : "1.7.3", 27 | "http_address" : "inet[/192.168.1.50:9201]", 28 | "http" : { 29 | "bound_address" : "inet[/0:0:0:0:0:0:0:0%0:9201]", 30 | "publish_address" : "inet[/192.168.1.50:9201]", 31 | "max_content_length" : "100mb", 32 | "max_content_length_in_bytes" : 104857600 33 | } 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/client-nodes-v2.json: -------------------------------------------------------------------------------- 1 | { 2 | "ok" : true, 3 | "cluster_name" : "elasticsearch", 4 | "nodes" : { 5 | "fiR5azTbTDiX59m78yzOTw" : { 6 | "name" : "Darkhawk", 7 | "transport_address" : "inet[/192.168.1.50:9300]", 8 | "host" : "cerberus", 9 | "version" : "2.3.3", 10 | "attributes" : { 11 | "data" : "false", 12 | "master" : "false" 13 | }, 14 | "http" : { 15 | "bound_address" : "inet[/0:0:0:0:0:0:0:0:9200]", 16 | "publish_address" : "inet[/192.168.1.50:9200]", 17 | "max_content_length" : "100mb", 18 | "max_content_length_in_bytes" : 104857600 19 | } 20 | }, 21 | "L9DM79IvStq2RStogXR8Sg" : { 22 | "name" : "Unseen", 23 | "transport_address" : "inet[/192.168.1.50:9301]", 24 | "host" : "cerberus", 25 | "version" : "2.3.3", 26 | "http_address" : "inet[/192.168.1.50:9201]", 27 | "http" : { 28 | "bound_address" : "inet[/0:0:0:0:0:0:0:0%0:9201]", 29 | "publish_address" : "inet[/192.168.1.50:9201]", 30 | "max_content_length" : "100mb", 31 | "max_content_length_in_bytes" : 104857600 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/client-nodes-v5.json: -------------------------------------------------------------------------------- 1 | { 2 | "ok" : true, 3 | "cluster_name" : "elasticsearch", 4 | "nodes" : { 5 | "fiR5azTbTDiX59m78yzOTw" : { 6 | "name" : "Darkhawk", 7 | "transport_address" : "192.168.1.50:9300", 8 | "host" : "cerberus", 9 | "version" : "5.0.0-alpha5", 10 | "roles" : [], 11 | "http" : { 12 | "bound_address" : "0:0:0:0:0:0:0:0:9200", 13 | "publish_address" : "192.168.1.50:9200", 14 | "max_content_length" : "100mb", 15 | "max_content_length_in_bytes" : 104857600 16 | } 17 | }, 18 | "L9DM79IvStq2RStogXR8Sg" : { 19 | "name" : "Unseen", 20 | "transport_address" : "192.168.1.50:9301", 21 | "host" : "cerberus", 22 | "version" : "5.0.0-alpha5", 23 | "roles" : [ 24 | "master", 25 | "data", 26 | "ingest" 27 | ], 28 | "http" : { 29 | "bound_address" : "0:0:0:0:0:0:0:0%0:9201", 30 | "publish_address" : "192.168.1.50:9201", 31 | "max_content_length" : "100mb", 32 | "max_content_length_in_bytes" : 104857600 33 | } 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typed/attachment.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "person" : { 5 | "properties" : { 6 | "my_attachment" : { "type" : "attachment" } 7 | } 8 | } 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typed/basic.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "basic" : { 5 | "properties" : { 6 | "name" : { 7 | "type" : "string" 8 | }, 9 | "org.apache.pig.builtin.currenttime_21" : { 10 | "type" : "date", 11 | "format" : "dateOptionalTime" 12 | }, 13 | "url" : { 14 | "type" : "string" 15 | } 16 | } 17 | } 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typed/completion.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "song" : { 5 | "properties" : { 6 | "name" : { "type" : "string" }, 7 | "suggest" : { 8 | "type" : "completion", 9 | "index_analyzer" : "simple", 10 | "search_analyzer" : "simple", 11 | "payloads" : true 12 | } 13 | } 14 | } 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typed/dynamic-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "friend": { 5 | "dynamic_templates": [ 6 | { 7 | "long_to_int": { 8 | "match_mapping_type": "long", 9 | "mapping": { 10 | "type": "integer" 11 | } 12 | } 13 | } 14 | ], 15 | "properties": { 16 | "hobbies": { 17 | "type": "text" 18 | }, 19 | "job": { 20 | "type": "text" 21 | }, 22 | "name": { 23 | "type": "text" 24 | } 25 | } 26 | } 27 | } 28 | } 29 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typed/geo.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "restaurant" : { 5 | "properties" : { 6 | "location" : { 7 | "type" : "geo_point", 8 | "normalize" : true, 9 | "fielddata" : { 10 | "format" : "compressed", 11 | "precision" : "1cm" 12 | } 13 | } 14 | } 15 | } 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typed/ip.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "client" : { 5 | "properties" : { 6 | "location" : { 7 | "type" : "ip", 8 | "ip" : "192.168.0.1" 9 | } 10 | } 11 | } 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typed/join-type.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "join":{ 5 | "properties": { 6 | "id": {"type": "keyword"}, 7 | "company": {"type": "text"}, 8 | "name": {"type": "text"}, 9 | "joiner": { 10 | "type": "join", 11 | "relations": { 12 | "company": "employee" 13 | } 14 | } 15 | } 16 | } 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typed/mapping_with_fields_named_properties_and_type.json: -------------------------------------------------------------------------------- 1 | { 2 | "index" : { 3 | "mappings" : { 4 | "properties" : { 5 | "properties" : { 6 | "field1" : { 7 | "type" : "string" 8 | }, 9 | "properties" : { 10 | "properties" : { 11 | "subfield1" : { 12 | "type" : "string" 13 | }, 14 | "subfield2" : { 15 | "type" : "string" 16 | } 17 | } 18 | }, 19 | "field2" : { 20 | "properties" : { 21 | "subfield3" : { 22 | "type" : "string" 23 | }, 24 | "properties" : { 25 | "type" : "string" 26 | }, 27 | "type" : { 28 | "properties" : { 29 | "properties" : { 30 | "properties" : { 31 | "properties" : { 32 | "type" : "string" 33 | }, 34 | "subfield4" : { 35 | "type" : "string" 36 | } 37 | } 38 | }, 39 | "subfield5" : { 40 | "type" : "string" 41 | } 42 | } 43 | } 44 | } 45 | } 46 | } 47 | } 48 | } 49 | } 50 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typed/multi_level_field_with_same_name.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "artiststimestamp" : { 5 | "properties" : { 6 | "date" : { 7 | "type" : "date", 8 | "format" : "dateOptionalTime" 9 | }, 10 | "links" : { 11 | "properties" : { 12 | "url" : { 13 | "type" : "string" 14 | }, 15 | "image": { 16 | "properties": { 17 | "kind": { 18 | "type": "string" 19 | }, 20 | "url": { 21 | "type": "string" 22 | } 23 | } 24 | } 25 | } 26 | }, 27 | "name" : { 28 | "type" : "string" 29 | } 30 | } 31 | } 32 | } 33 | } 34 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typed/multiple-indices-multiple-conflicting-types.json: -------------------------------------------------------------------------------- 1 | { 2 | "index1": { 3 | "mappings": { 4 | "type1": { 5 | "properties": { 6 | "field1": { "type": "string" }, 7 | "field2": { "type": "keyword" }, 8 | "field3": { "type": "float" } 9 | } 10 | } 11 | } 12 | }, 13 | "index2": { 14 | "mappings": { 15 | "type2": { 16 | "properties": { 17 | "field1": { "type": "keyword" }, 18 | "field2": { "type": "null" }, 19 | "field4": { "type": "integer" } 20 | } 21 | } 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typed/multiple-indices-multiple-types.json: -------------------------------------------------------------------------------- 1 | { 2 | "index1": { 3 | "mappings": { 4 | "type1": { 5 | "properties": { 6 | "field1": { "type": "keyword" }, 7 | "field2": { "type": "float" } 8 | } 9 | } 10 | } 11 | }, 12 | "index2": { 13 | "mappings": { 14 | "type2": { 15 | "properties": { 16 | "field1": { "type": "keyword" }, 17 | "field3": { "type": "integer" } 18 | } 19 | } 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typed/multiple-indices-multiple-upcastable-types.json: -------------------------------------------------------------------------------- 1 | { 2 | "index1": { 3 | "mappings": { 4 | "type1": { 5 | "properties": { 6 | "field1_keyword": { "type": "keyword" }, 7 | "field2_keyword": { "type": "short" }, 8 | "field3_keyword": { "type": "keyword" }, 9 | "field4_integer": { "type": "short" }, 10 | "field5_keyword": { "type": "float" }, 11 | "field6_float": { "type": "half_float" }, 12 | "field7_keyword": { "type": "date" }, 13 | "field8_float": { "type": "float" } 14 | } 15 | } 16 | } 17 | }, 18 | "index2": { 19 | "mappings": { 20 | "type2": { 21 | "properties": { 22 | "field1_keyword": { "type": "keyword" }, 23 | "field2_keyword": { "type": "keyword" }, 24 | "field3_keyword": { "type": "integer" }, 25 | "field4_integer": { "type": "integer" }, 26 | "field5_keyword": { "type": "long" }, 27 | "field6_float": { "type": "float" }, 28 | "field7_keyword": { "type": "text" }, 29 | "field9_integer": { "type": "integer" } 30 | } 31 | } 32 | } 33 | } 34 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typed/multiple-types.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "type1": { 5 | "properties": { 6 | "field1": { "type": "keyword" }, 7 | "field2": { "type": "float" } 8 | } 9 | }, 10 | "type2": { 11 | "properties": { 12 | "field1": { "type": "keyword" }, 13 | "field3": { "type": "integer" } 14 | } 15 | } 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typed/nested-mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "company": { 5 | "properties": { 6 | "name": { "type": "string"}, 7 | "description": {"type": "string"}, 8 | "employees": { 9 | "type": "nested", 10 | "properties": { 11 | "name": {"type": "string"}, 12 | "salary": {"type": "long"} 13 | } 14 | } 15 | } 16 | } 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typed/nested_arrays_mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "nested-array-exclude" : { 5 | "properties" : { 6 | "foo" : { 7 | "type" : "long" 8 | }, 9 | "nested" : { 10 | "properties" : { 11 | "bar" : { 12 | "properties" : { 13 | "date" : { 14 | "type" : "date", 15 | "format" : "strict_date_optional_time||epoch_millis" 16 | }, 17 | "scores" : { 18 | "type" : "long" 19 | } 20 | } 21 | }, 22 | "what" : { 23 | "type" : "string" 24 | } 25 | } 26 | } 27 | } 28 | } 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typeless/attachment.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "properties" : { 5 | "my_attachment" : { "type" : "attachment" } 6 | } 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typeless/basic.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "properties" : { 5 | "name" : { 6 | "type" : "string" 7 | }, 8 | "org.apache.pig.builtin.currenttime_21" : { 9 | "type" : "date", 10 | "format" : "dateOptionalTime" 11 | }, 12 | "url" : { 13 | "type" : "string" 14 | } 15 | } 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typeless/completion.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "properties" : { 5 | "name" : { "type" : "string" }, 6 | "suggest" : { 7 | "type" : "completion", 8 | "index_analyzer" : "simple", 9 | "search_analyzer" : "simple", 10 | "payloads" : true 11 | } 12 | } 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typeless/dynamic-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "dynamic_templates": [ 5 | { 6 | "long_to_int": { 7 | "match_mapping_type": "long", 8 | "mapping": { 9 | "type": "integer" 10 | } 11 | } 12 | } 13 | ], 14 | "properties": { 15 | "hobbies": { 16 | "type": "text" 17 | }, 18 | "job": { 19 | "type": "text" 20 | }, 21 | "name": { 22 | "type": "text" 23 | } 24 | } 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typeless/geo.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "properties" : { 5 | "location" : { 6 | "type" : "geo_point", 7 | "normalize" : true, 8 | "fielddata" : { 9 | "format" : "compressed", 10 | "precision" : "1cm" 11 | } 12 | } 13 | } 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typeless/ip.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "properties" : { 5 | "location" : { 6 | "type" : "ip", 7 | "ip" : "192.168.0.1" 8 | } 9 | } 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typeless/join-type.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "properties": { 5 | "id": {"type": "keyword"}, 6 | "company": {"type": "text"}, 7 | "name": {"type": "text"}, 8 | "joiner": { 9 | "type": "join", 10 | "relations": { 11 | "company": "employee" 12 | } 13 | } 14 | } 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typeless/mapping_with_fields_named_properties_and_type.json: -------------------------------------------------------------------------------- 1 | { 2 | "index" : { 3 | "mappings" : { 4 | "properties" : { 5 | "field1" : { 6 | "type" : "string" 7 | }, 8 | "properties" : { 9 | "properties" : { 10 | "subfield1" : { 11 | "type" : "string" 12 | }, 13 | "subfield2" : { 14 | "type" : "string" 15 | } 16 | } 17 | }, 18 | "field2" : { 19 | "properties" : { 20 | "subfield3" : { 21 | "type" : "string" 22 | }, 23 | "properties" : { 24 | "type" : "string" 25 | }, 26 | "type" : { 27 | "properties" : { 28 | "properties" : { 29 | "properties" : { 30 | "properties" : { 31 | "type" : "string" 32 | }, 33 | "subfield4" : { 34 | "type" : "string" 35 | } 36 | } 37 | }, 38 | "subfield5" : { 39 | "type" : "string" 40 | } 41 | } 42 | } 43 | } 44 | } 45 | } 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typeless/multi_level_field_with_same_name.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "properties" : { 5 | "date" : { 6 | "type" : "date", 7 | "format" : "dateOptionalTime" 8 | }, 9 | "links" : { 10 | "properties" : { 11 | "url" : { 12 | "type" : "string" 13 | }, 14 | "image": { 15 | "properties": { 16 | "kind": { 17 | "type": "string" 18 | }, 19 | "url": { 20 | "type": "string" 21 | } 22 | } 23 | } 24 | } 25 | }, 26 | "name" : { 27 | "type" : "string" 28 | } 29 | } 30 | } 31 | } 32 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typeless/multiple-indices-multiple-conflicting-types.json: -------------------------------------------------------------------------------- 1 | { 2 | "index1": { 3 | "mappings": { 4 | "properties": { 5 | "field1": { "type": "string" }, 6 | "field2": { "type": "keyword" }, 7 | "field3": { "type": "float" } 8 | } 9 | } 10 | }, 11 | "index2": { 12 | "mappings": { 13 | "properties": { 14 | "field1": { "type": "keyword" }, 15 | "field2": { "type": "null" }, 16 | "field4": { "type": "integer" } 17 | } 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typeless/multiple-indices-multiple-types.json: -------------------------------------------------------------------------------- 1 | { 2 | "index1": { 3 | "mappings": { 4 | "properties": { 5 | "field1": { "type": "keyword" }, 6 | "field2": { "type": "float" } 7 | } 8 | } 9 | }, 10 | "index2": { 11 | "mappings": { 12 | "properties": { 13 | "field1": { "type": "keyword" }, 14 | "field3": { "type": "integer" } 15 | } 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typeless/multiple-indices-multiple-upcastable-types.json: -------------------------------------------------------------------------------- 1 | { 2 | "index1": { 3 | "mappings": { 4 | "properties": { 5 | "field1_keyword": { "type": "keyword" }, 6 | "field2_keyword": { "type": "short" }, 7 | "field3_keyword": { "type": "keyword" }, 8 | "field4_integer": { "type": "short" }, 9 | "field5_keyword": { "type": "float" }, 10 | "field6_float": { "type": "half_float" }, 11 | "field7_keyword": { "type": "date" }, 12 | "field8_float": { "type": "float" } 13 | } 14 | } 15 | }, 16 | "index2": { 17 | "mappings": { 18 | "properties": { 19 | "field1_keyword": { "type": "keyword" }, 20 | "field2_keyword": { "type": "keyword" }, 21 | "field3_keyword": { "type": "integer" }, 22 | "field4_integer": { "type": "integer" }, 23 | "field5_keyword": { "type": "long" }, 24 | "field6_float": { "type": "float" }, 25 | "field7_keyword": { "type": "text" }, 26 | "field9_integer": { "type": "integer" } 27 | } 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typeless/nested-mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "properties": { 5 | "name": { "type": "string"}, 6 | "description": {"type": "string"}, 7 | "employees": { 8 | "type": "nested", 9 | "properties": { 10 | "name": {"type": "string"}, 11 | "salary": {"type": "long"} 12 | } 13 | } 14 | } 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typeless/nested_arrays_mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "properties" : { 5 | "foo" : { 6 | "type" : "long" 7 | }, 8 | "nested" : { 9 | "properties" : { 10 | "bar" : { 11 | "properties" : { 12 | "date" : { 13 | "type" : "date", 14 | "format" : "strict_date_optional_time||epoch_millis" 15 | }, 16 | "scores" : { 17 | "type" : "long" 18 | } 19 | } 20 | }, 21 | "what" : { 22 | "type" : "string" 23 | } 24 | } 25 | } 26 | } 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/dto/mapping/typeless/primitives.json: -------------------------------------------------------------------------------- 1 | { 2 | "index" : { 3 | "mappings" : { 4 | "properties" : { 5 | "field01" : { 6 | "type" : "boolean" 7 | }, 8 | "field02" : { 9 | "type" : "byte" 10 | }, 11 | "field03" : { 12 | "type" : "short" 13 | }, 14 | "field04" : { 15 | "type" : "integer" 16 | }, 17 | "field05" : { 18 | "type" : "long" 19 | }, 20 | "field06" : { 21 | "type" : "float" 22 | }, 23 | "field07" : { 24 | "type" : "double" 25 | }, 26 | "field08" : { 27 | "type" : "string" 28 | }, 29 | "field09" : { 30 | "type" : "date" 31 | }, 32 | "field10" : { 33 | "type" : "binary" 34 | }, 35 | "field11" : { 36 | "type" : "text" 37 | }, 38 | "field12" : { 39 | "type" : "keyword" 40 | }, 41 | "field13" : { 42 | "type" : "half_float" 43 | }, 44 | "field14" : { 45 | "type" : "scaled_float", 46 | "scaling_factor" : 100.0 47 | }, 48 | "field15" : { 49 | "type" : "date_nanos" 50 | }, 51 | "field16" : { 52 | "type" : "wildcard" 53 | } 54 | } 55 | } 56 | } 57 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/nested-fields.json: -------------------------------------------------------------------------------- 1 | {"nested" : { "field" : "value", "foo" : 1 }, "nested2" : { "van" : "halen", "paradise" : "lost"} } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/parser-test-nested.json: -------------------------------------------------------------------------------- 1 | { 2 | "firstName": "John", 3 | "lastName": "Smith", 4 | "age": 25, 5 | "trap": { 6 | "bogus":true, 7 | "state":"bogus", 8 | "building": { 9 | "height":"bogus", 10 | "floors": 0, 11 | "flats": 0 12 | } 13 | }, 14 | "address": { 15 | "streetAddress": "21 2nd Street", 16 | "city": "New York", 17 | "state": "NY", 18 | "postalCode": 10021, 19 | "building": { 20 | "height":"tall", 21 | "floors": 10, 22 | "flats": 40 23 | }, 24 | "decor": { 25 | "walls": "white", 26 | "floors": "parquet" 27 | }, 28 | "firstName":"should-not-be-picked" 29 | }, 30 | "phoneNumbers": [ 31 | { 32 | "type": "home", 33 | "number": "212 555-1234" 34 | }, 35 | { 36 | "type": "fax", 37 | "number": "646 555-4567", 38 | "age":-1 39 | } 40 | ], 41 | "state" : "CA", 42 | "small-object": { 43 | "foo": "bar", 44 | "nested": { 45 | "foo":"nested-bar", 46 | "foo1":"bar1" 47 | } 48 | }, 49 | "small-array": [ 50 | "foo", "bar" 51 | ], 52 | "zzz": "end" 53 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/parser-test.json: -------------------------------------------------------------------------------- 1 | { 2 | "firstName": "John", 3 | "lastName": "Smith", 4 | "age": 25, 5 | "address": { 6 | "streetAddress": "21 2nd Street", 7 | "city": "New York", 8 | "state": "NY", 9 | "postalCode": 10021 10 | }, 11 | "phoneNumbers": [ 12 | { 13 | "type": "home", 14 | "number": "212 555-1234" 15 | }, 16 | { 17 | "type": "fax", 18 | "number": "646 555-4567" 19 | } 20 | ] 21 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/scrollReaderTestData/empty-list/mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "artists" : { 3 | "mappings" : { 4 | "properties" : { 5 | "name" : { 6 | "type" : "text", 7 | "fields" : { 8 | "keyword" : { 9 | "type" : "keyword", 10 | "ignore_above" : 256 11 | } 12 | } 13 | }, 14 | "status_code" : { 15 | "type" : "long", 16 | "fields" : { 17 | "keyword" : { 18 | "type" : "keyword", 19 | "ignore_above" : 256 20 | } 21 | } 22 | }, 23 | "ts" : { 24 | "type" : "long" 25 | } 26 | } 27 | } 28 | } 29 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/scrollReaderTestData/empty-list/scroll.json: -------------------------------------------------------------------------------- 1 | { 2 | "_scroll_id" : "FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFmJUOHBDZ0pjU1dPUm9CbThxUWp1M3cAAAAAAABtCxZzRFItZkxtWFRjZWxXZ3pZamhZZWZ3", 3 | "took" : 0, 4 | "timed_out" : false, 5 | "_shards" : { 6 | "total" : 1, 7 | "successful" : 1, 8 | "skipped" : 0, 9 | "failed" : 0 10 | }, 11 | "hits" : { 12 | "total" : { 13 | "value" : 3, 14 | "relation" : "eq" 15 | }, 16 | "max_score" : 1.0, 17 | "hits" : [ 18 | { 19 | "_index" : "empty_list", 20 | "_type" : "_doc", 21 | "_id" : "1", 22 | "_score" : 1.0, 23 | "_source" : { 24 | "status_code" : null, 25 | "ts" : 12345678910, 26 | "name" : "john" 27 | } 28 | }, 29 | { 30 | "_index" : "empty_list", 31 | "_type" : "_doc", 32 | "_id" : "2", 33 | "_score" : 1.0, 34 | "_source" : { 35 | "status_code" : [ 36 | 123 37 | ], 38 | "ts" : null, 39 | "name" : "johnny" 40 | } 41 | }, 42 | { 43 | "_index" : "empty_list", 44 | "_type" : "_doc", 45 | "_id" : "3", 46 | "_score" : 1.0, 47 | "_source" : { 48 | "status_code" : [ ], 49 | "ts" : 12345678910, 50 | "name" : null 51 | } 52 | } 53 | ] 54 | } 55 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/scrollReaderTestData/empty-source/scroll.json: -------------------------------------------------------------------------------- 1 | { 2 | "_scroll_id" : "c2NhbjsxOzE6MjBLOXo0S1RTcktPNmtmSU44YjJZUTsxO3RvdGFsX2hpdHM6MTk2Ow==", 3 | "took" : 30, 4 | "timed_out" : false, 5 | "_shards" : { 6 | "total" : 1, 7 | "successful" : 1, 8 | "failed" : 0 9 | }, 10 | "hits" : { 11 | "total" : 196, 12 | "max_score" : 0.0, 13 | "hits" : [{ 14 | "_index" : "pig", 15 | "_type" : "tupleartists", 16 | "_id" : "23hrGo7VRCyao8lB9Uu5Kw", 17 | "_score" : 0.0 18 | }, { 19 | "_index" : "pig", 20 | "_type" : "tupleartists", 21 | "_id" : "PTi2NxdDRxmXhv6S8DgIeQ", 22 | "_score" : 0.0 23 | }] 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/scrollReaderTestData/fields/scroll.json: -------------------------------------------------------------------------------- 1 | { 2 | "_scroll_id" : "c2NhbjsxOzE6MjBLOXo0S1RTcktPNmtmSU44YjJZUTsxO3RvdGFsX2hpdHM6MTk2Ow==", 3 | "took" : 30, 4 | "timed_out" : false, 5 | "_shards" : { 6 | "total" : 1, 7 | "successful" : 1, 8 | "failed" : 0 9 | }, 10 | "hits" : { 11 | "total" : 196, 12 | "max_score" : 0.0, 13 | "hits" : [{ 14 | "_index" : "pig", 15 | "_type" : "tupleartists", 16 | "_id" : "23hrGo7VRCyao8lB9Uu5Kw", 17 | "_score" : 0.0, 18 | "fields" : { 19 | "special" : "لَيسَ كُل مَ آكتُبه حِكَآيةة عَنْ وَٱقِعِي إنَمَآ هِيآ كَلِمآت رَاقَت لِي", 20 | "location" : "Würzburg", 21 | "fields" : "fields" 22 | } 23 | }, { 24 | "_index" : "pig", 25 | "_type" : "tupleartists", 26 | "_id" : "PTi2NxdDRxmXhv6S8DgIeQ", 27 | "_score" : 0.0, 28 | "_source" : { 29 | "links":{"url":"http://www.last.fm/music/DIR+EN+GREY"} 30 | } 31 | }, { 32 | "_index" : "pig", 33 | "_type" : "tupleartists", 34 | "_id" : "H2opSUi0RAm_WrHUkSSjIA", 35 | "_score" : 0.0, 36 | "_source" : { 37 | "links":{"url":"http://www.last.fm/music/Limbonic+Art"} 38 | } 39 | } 40 | ] 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/scrollReaderTestData/join/mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "properties": { 5 | "id": {"type": "keyword"}, 6 | "company": {"type": "text"}, 7 | "name": {"type": "text"}, 8 | "joiner": { 9 | "type": "join", 10 | "relations": { 11 | "company": "employee" 12 | } 13 | } 14 | } 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/scrollReaderTestData/list/scroll.json: -------------------------------------------------------------------------------- 1 | { 2 | "_scroll_id" : "c2NhbjsxOzQ6UDZLb2VvRjJUVmFPRG1MQzQ1U2lBdzsxO3RvdGFsX2hpdHM6MjE2Ow==", 3 | "took" : 7, 4 | "timed_out" : false, 5 | "_shards" : { 6 | "total" : 1, 7 | "successful" : 1, 8 | "failed" : 0 9 | }, 10 | "hits" : { 11 | "total" : 216, 12 | "max_score" : 0.0, 13 | "hits" : [{ 14 | "_index" : "pig", 15 | "_type" : "tupleartists", 16 | "_id" : "aqOqDwYnTA29J1gfy3m8_Q", 17 | "_score" : 0.0, 18 | "_source" : { 19 | "special" : "لَيسَ كُل مَ آكتُبه حِكَآيةة عَنْ وَٱقِعِي إنَمَآ هِيآ كَلِمآت رَاقَت لِي", 20 | "location" : "Würzburg", 21 | "name" : "www.DjFahad.com", 22 | "links" : ["http://www.last.fm/music/www.DjFahad.com", null] 23 | } 24 | } 25 | ] 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/scrollReaderTestData/multi-type/mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "mappings": { 4 | "type1": { 5 | "properties": { 6 | "field1": { "type": "keyword" }, 7 | "field2": { "type": "keyword" } 8 | } 9 | }, 10 | "type2": { 11 | "properties": { 12 | "field3": { "type": "keyword" } 13 | } 14 | } 15 | } 16 | }, 17 | "index2": { 18 | "mappings": { 19 | "type3": { 20 | "properties": { 21 | "field4": { "type": "keyword" } 22 | } 23 | } 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/scrollReaderTestData/multi-type/scroll.json: -------------------------------------------------------------------------------- 1 | { 2 | "_scroll_id" : "c2NhbjsxOzE6MjBLOXo0S1RTcktPNmtmSU44YjJZUTsxO3RvdGFsX2hpdHM6MTk2Ow==", 3 | "took" : 30, 4 | "timed_out" : false, 5 | "_shards" : { 6 | "total" : 1, 7 | "successful" : 1, 8 | "failed" : 0 9 | }, 10 | "hits" : { 11 | "total" : 196, 12 | "max_score" : 0.0, 13 | "hits" : [{ 14 | "_index" : "index", 15 | "_type" : "type1", 16 | "_id" : "23hrGo7VRCyao8lB9Uu5Kw", 17 | "_score" : 0.0, 18 | "_source" : { 19 | "field1": "value1", 20 | "field2": "value2" 21 | } 22 | }, { 23 | "_index" : "index", 24 | "_type" : "type2", 25 | "_id" : "PTi2NxdDRxmXhv6S8DgIeQ", 26 | "_score" : 0.0, 27 | "_source" : { 28 | "field3": "value3" 29 | } 30 | }, { 31 | "_index" : "index2", 32 | "_type" : "type3", 33 | "_id" : "H2opSUi0RAm_WrHUkSSjIA", 34 | "_score" : 0.0, 35 | "_source" : { 36 | "field4": "value4" 37 | } 38 | } 39 | ] 40 | } 41 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/scrollReaderTestData/nested-data/mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "nested": { 3 | "mappings": { 4 | "properties": { 5 | "a": { 6 | "properties": { 7 | "b": { 8 | "type": "nested", 9 | "properties": { 10 | "c": { 11 | "type": "text" 12 | }, 13 | "d": { 14 | "type": "text" 15 | }, 16 | "e": { 17 | "type": "date" 18 | }, 19 | "f": { 20 | "type": "long" 21 | } 22 | } 23 | } 24 | } 25 | } 26 | } 27 | } 28 | } 29 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/scrollReaderTestData/nested-list/mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "nested-list": { 3 | "mappings": { 4 | "properties": { 5 | "a": { 6 | "type": "long" 7 | } 8 | } 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/scrollReaderTestData/no-scroll-id/scroll.json: -------------------------------------------------------------------------------- 1 | { 2 | "took":0, 3 | "timed_out":false, 4 | "_shards":{ 5 | "total":0, 6 | "successful":0, 7 | "skipped":0, 8 | "failed":0 9 | }, 10 | "hits":{ 11 | "total":{ 12 | "value":0, 13 | "relation":"eq" 14 | }, 15 | "max_score":0.0, 16 | "hits":[ 17 | 18 | ] 19 | } 20 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/scrollReaderTestData/numbers-as-strings/mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "pig" : { 3 | "mappings" : { 4 | "properties" : { 5 | "number" : { 6 | "type" : "long" 7 | } 8 | } 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/scrollReaderTestData/numbers-as-strings/scroll.json: -------------------------------------------------------------------------------- 1 | { 2 | "_scroll_id" : "c2NhbjsxOzE6MjBLOXo0S1RTcktPNmtmSU44YjJZUTsxO3RvdGFsX2hpdHM6MTk2Ow==", 3 | "took" : 30, 4 | "timed_out" : false, 5 | "_shards" : { 6 | "total" : 1, 7 | "successful" : 1, 8 | "failed" : 0 9 | }, 10 | "hits" : { 11 | "total" : 196, 12 | "max_score" : 0.0, 13 | "hits" : [{ 14 | "_index" : "pig", 15 | "_type" : "tupleartists", 16 | "_id" : "23hrGo7VRCyao8lB9Uu5Kw", 17 | "_score" : 0.0, 18 | "_source" : { 19 | "number" : "abcd" 20 | } 21 | }] 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/scrollReaderTestData/object-fields/mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "object": { 3 | "mappings": { 4 | "properties": { 5 | "a": { 6 | "properties": { 7 | "b": { 8 | "properties": { 9 | "c": { 10 | "type": "text" 11 | }, 12 | "d": { 13 | "type": "text" 14 | }, 15 | "e": { 16 | "type": "date" 17 | }, 18 | "f": { 19 | "type": "long" 20 | } 21 | } 22 | } 23 | } 24 | } 25 | } 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/scrollReaderTestData/source/mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "pig" : { 3 | "mappings" : { 4 | "properties" : { 5 | "_source" : { 6 | "type" : "string" 7 | }, 8 | "links" : { 9 | "properties" : { 10 | "picture" : { 11 | "type" : "string" 12 | }, 13 | "url" : { 14 | "type" : "string" 15 | }, 16 | "number" : { 17 | "type" : "short" 18 | } 19 | } 20 | }, 21 | "name" : { 22 | "type" : "string" 23 | } 24 | } 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/serialization/scrollReaderTestData/source/scroll.json: -------------------------------------------------------------------------------- 1 | { 2 | "_scroll_id" : "c2NhbjsxOzE6MjBLOXo0S1RTcktPNmtmSU44YjJZUTsxO3RvdGFsX2hpdHM6MTk2Ow==", 3 | "took" : 30, 4 | "timed_out" : false, 5 | "_shards" : { 6 | "total" : 1, 7 | "successful" : 1, 8 | "failed" : 0 9 | }, 10 | "hits" : { 11 | "total" : 196, 12 | "max_score" : 0.0, 13 | "hits" : [{ 14 | "_index" : "pig", 15 | "_type" : "tupleartists", 16 | "_id" : "23hrGo7VRCyao8lB9Uu5Kw", 17 | "_score" : 0.0, 18 | "_source" : { 19 | "special" : "لَيسَ كُل مَ آكتُبه حِكَآيةة عَنْ وَٱقِعِي إنَمَآ هِيآ كَلِمآت رَاقَت لِي", 20 | "location" : "Würzburg", 21 | "source" : "source" 22 | } 23 | }, { 24 | "_index" : "pig", 25 | "_type" : "tupleartists", 26 | "_id" : "PTi2NxdDRxmXhv6S8DgIeQ", 27 | "_score" : 0.0, 28 | "_source" : { 29 | "links":{"url":"http://www.last.fm/music/DIR+EN+GREY"} 30 | } 31 | }, { 32 | "_index" : "pig", 33 | "_type" : "tupleartists", 34 | "_id" : "H2opSUi0RAm_WrHUkSSjIA", 35 | "_score" : 0.0, 36 | "_source" : { 37 | "links":{"number":"125"} 38 | } 39 | } 40 | ] 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /mr/src/test/resources/org/elasticsearch/hadoop/util/textdata.txt: -------------------------------------------------------------------------------- 1 | Hello World. This is used by IOUtilsTest. -------------------------------------------------------------------------------- /qa/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/elasticsearch-hadoop/aae4528ee8a31e7106e0ee4b671162fb70cf15f1/qa/.DS_Store -------------------------------------------------------------------------------- /qa/kerberos/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/elasticsearch-hadoop/aae4528ee8a31e7106e0ee4b671162fb70cf15f1/qa/kerberos/.DS_Store -------------------------------------------------------------------------------- /qa/kerberos/src/main/resources/hive/load_to_es.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS artist_data; 2 | 3 | CREATE EXTERNAL TABLE IF NOT EXISTS artist_data ( 4 | num STRING, 5 | name STRING, 6 | url STRING, 7 | picture STRING, 8 | ts STRING, 9 | tag STRING) 10 | ROW FORMAT DELIMITED 11 | FIELDS TERMINATED BY '\t' 12 | STORED AS TEXTFILE 13 | LOCATION '/data/artists'; 14 | 15 | SELECT ad.num, ad.name, ad.url, ad.picture, from_unixtime(unix_timestamp()), ad.tag FROM artist_data ad LIMIT 10; 16 | 17 | DROP TABLE IF EXISTS es_artist_data; 18 | 19 | CREATE EXTERNAL TABLE IF NOT EXISTS es_artist_data ( 20 | num STRING, 21 | name STRING, 22 | url STRING, 23 | picture STRING, 24 | ts TIMESTAMP, 25 | tag STRING) 26 | STORED BY 'org.elasticsearch.hadoop.hive.EsStorageHandler' 27 | TBLPROPERTIES( 28 | 'es.resource' = 'qa_kerberos_hive_data', 29 | 'es.security.authentication' = 'kerberos', 30 | 'es.net.spnego.auth.elasticsearch.principal' = 'HTTP/build.elastic.co@BUILD.ELASTIC.CO' 31 | ); 32 | 33 | -- Create random timestamps up front since Hive's timestamp format differs from ISO8601 34 | INSERT OVERWRITE TABLE es_artist_data 35 | SELECT ad.num, ad.name, ad.url, ad.picture, from_unixtime(unix_timestamp()), ad.tag FROM artist_data ad; 36 | -------------------------------------------------------------------------------- /qa/kerberos/src/main/resources/hive/read_from_es.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS es_artist_data; 2 | 3 | CREATE EXTERNAL TABLE IF NOT EXISTS es_artist_data ( 4 | num STRING, 5 | name STRING, 6 | url STRING, 7 | picture STRING, 8 | ts TIMESTAMP, 9 | tag STRING) 10 | STORED BY 'org.elasticsearch.hadoop.hive.EsStorageHandler' 11 | TBLPROPERTIES('es.resource' = 'qa_kerberos_hive_data'); 12 | 13 | DROP TABLE IF EXISTS artist_data; 14 | 15 | -- For some reason, creating and writing to an external table causes Hive to plan a reduce phase. 16 | -- We can't use reduce phases with Kerberos security enabled because they require native libraries. 17 | -- See Hadoop's SecureIOUtils static initialization for more information. 18 | INSERT OVERWRITE DIRECTORY '/data/output/hive' 19 | ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' 20 | SELECT * FROM es_artist_data; 21 | -------------------------------------------------------------------------------- /qa/kerberos/src/main/resources/hive/top_25_words.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE IF NOT EXISTS filecounts ( 2 | word STRING, 3 | num INT) 4 | ROW FORMAT DELIMITED 5 | FIELDS TERMINATED BY ',' 6 | STORED AS TEXTFILE 7 | LOCATION '/output/pride'; 8 | 9 | CREATE TABLE IF NOT EXISTS orccounts ( 10 | word STRING, 11 | num INT) 12 | ROW FORMAT DELIMITED 13 | FIELDS TERMINATED BY ',' 14 | STORED AS ORC; 15 | 16 | INSERT OVERWRITE TABLE orccounts SELECT * FROM filecounts; 17 | 18 | SELECT * FROM orccounts 19 | ORDER BY num DESC 20 | LIMIT 25; 21 | -------------------------------------------------------------------------------- /qa/kerberos/src/main/resources/ssl/server.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/elasticsearch-hadoop/aae4528ee8a31e7106e0ee4b671162fb70cf15f1/qa/kerberos/src/main/resources/ssl/server.jks -------------------------------------------------------------------------------- /qa/kerberos/src/main/resources/test.properties: -------------------------------------------------------------------------------- 1 | # default settings for Hadoop test 2 | 3 | #test.disable.local.es=true 4 | # ES 5 | es.batch.size.bytes=1kb 6 | 7 | # Minimal Security 8 | es.net.http.auth.user=elastic-admin 9 | es.net.http.auth.pass=elastic-password 10 | 11 | # put pressure on the bulk API 12 | es.batch.size.entries=3 13 | es.batch.write.retry.wait=1s 14 | 15 | mapred.max.split.size=134217728 16 | 17 | # M&R 18 | fs.default.name=file:/// 19 | 20 | # Spark props 21 | spark.ui.enabled=false 22 | spark.sql.warehouse.dir=/tmp/spark 23 | spark.serializer=org.apache.spark.serializer.KryoSerializer 24 | spark.executor.extraJavaOptions=-XX:MaxPermSize=256m -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | pluginManagement { 2 | plugins { 3 | id 'com.github.johnrengelman.shadow' version "8.1.1" 4 | } 5 | } 6 | 7 | plugins { 8 | id "com.gradle.develocity" version "3.18.1" 9 | } 10 | 11 | rootProject.name = "elasticsearch-hadoop" 12 | 13 | include 'thirdparty' 14 | 15 | include 'mr' 16 | project(":mr").name = "elasticsearch-hadoop-mr" 17 | include 'hive' 18 | project(":hive").name = "elasticsearch-hadoop-hive" 19 | 20 | include 'spark-core' 21 | project(":spark-core").projectDir = new File(settingsDir, "spark/core") 22 | project(":spark-core").name = "elasticsearch-spark" 23 | 24 | include 'sql-30' 25 | project(":sql-30").projectDir = new File(settingsDir, "spark/sql-30") 26 | project(":sql-30").name = "elasticsearch-spark-30" 27 | 28 | include 'dist' 29 | 30 | include 'test' 31 | include 'test:shared' 32 | include 'test:fixtures' 33 | include 'test:fixtures:minikdc' 34 | 35 | include 'qa' 36 | include 'qa:kerberos' 37 | -------------------------------------------------------------------------------- /spark/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/elasticsearch-hadoop/aae4528ee8a31e7106e0ee4b671162fb70cf15f1/spark/.DS_Store -------------------------------------------------------------------------------- /spark/core/licenses/commons-logging-1.1.1.jar.sha1: -------------------------------------------------------------------------------- 1 | 5043bfebc3db072ed80fbd362e7caf00e885d8ae -------------------------------------------------------------------------------- /spark/core/licenses/commons-logging-NOTICE.txt: -------------------------------------------------------------------------------- 1 | Apache Commons Logging 2 | Copyright 2003-2007 The Apache Software Foundation 3 | 4 | This product includes software developed by 5 | The Apache Software Foundation (http://www.apache.org/). 6 | -------------------------------------------------------------------------------- /spark/core/licenses/scala-library-2.13.13.jar.sha1: -------------------------------------------------------------------------------- 1 | 3b1434403ddb147b8030f98f246153e4ce622c78 -------------------------------------------------------------------------------- /spark/core/licenses/scala-reflect-2.13.13.jar.sha1: -------------------------------------------------------------------------------- 1 | 46866f0e2cf6168322dc11f0f721c2db78d52ac7 -------------------------------------------------------------------------------- /spark/core/licenses/spark-core_2.13-3.4.3.jar.sha1: -------------------------------------------------------------------------------- 1 | a8055d61a13e9af6871e05ecdca141134d102f1f -------------------------------------------------------------------------------- /spark/core/src/itest/java/org/elasticsearch/spark/integration/ScalaUtils.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.spark.integration; 2 | 3 | import scala.collection.mutable.Map; 4 | import scala.collection.mutable.HashMap; 5 | 6 | import java.util.Properties; 7 | 8 | public class ScalaUtils { 9 | /* 10 | * Scala removed scala.collection.JavaConversions.propertiesAsScalaMap() in 2.13, replacing it with an implicit 11 | * jdk.CollectionConverters.asScala. This is an attempt to get a method that works in 2.10-2.13. It can be removed and replaced with 12 | * jdk.CollectionConverters once we no longer support scala older than 2.13. 13 | */ 14 | public static scala.collection.Map propertiesAsScalaMap(Properties props) { 15 | Map scalaMap = new HashMap(); 16 | for (java.util.Map.Entry entry : props.entrySet()) { 17 | scalaMap.put(entry.getKey(), entry.getValue()); 18 | } 19 | return scalaMap; 20 | } 21 | } 22 | 23 | -------------------------------------------------------------------------------- /spark/core/src/itest/java/org/elasticsearch/spark/integration/SparkSuite.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.spark.integration; 20 | 21 | import org.elasticsearch.hadoop.fixtures.LocalEs; 22 | import org.junit.ClassRule; 23 | import org.junit.rules.ExternalResource; 24 | import org.junit.runner.RunWith; 25 | import org.junit.runners.Suite; 26 | 27 | @RunWith(Suite.class) 28 | @Suite.SuiteClasses({ AbstractJavaEsSparkTest.class, AbstractHadoopBasicSparkTest.class }) 29 | //@Suite.SuiteClasses({ AbstractJavaEsSparkTest.class }) 30 | public class SparkSuite { 31 | 32 | @ClassRule 33 | public static ExternalResource resource = new LocalEs(); 34 | 35 | } 36 | -------------------------------------------------------------------------------- /spark/core/src/itest/resources/basic.json: -------------------------------------------------------------------------------- 1 | { "firstName": "John", "isAlive": true, "age": 25, "children": ["Alex", "Joe"], "address": { "streetAddress": "21 2nd Street" } } 2 | -------------------------------------------------------------------------------- /spark/core/src/itest/resources/simple.json: -------------------------------------------------------------------------------- 1 | {"key":"value"} -------------------------------------------------------------------------------- /spark/core/src/main/scala/org/elasticsearch/spark/cfg/SparkSettingsManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.spark.cfg; 20 | 21 | import org.apache.spark.SparkConf; 22 | import org.elasticsearch.hadoop.cfg.SettingsManager; 23 | 24 | public class SparkSettingsManager implements SettingsManager { 25 | 26 | @Override 27 | public SparkSettings load(SparkConf configuration) { 28 | return new SparkSettings(configuration); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /spark/core/src/main/scala/org/elasticsearch/spark/cfg/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | /** 20 | * Configuration package, specific for Apache Spark. 21 | */ 22 | package org.elasticsearch.spark.cfg; 23 | -------------------------------------------------------------------------------- /spark/core/src/main/scala/org/elasticsearch/spark/rdd/CompatibilityLevel.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.spark.rdd 20 | 21 | trait CompatibilityLevel { 22 | def versionId: String 23 | def versionDescription: String 24 | } 25 | -------------------------------------------------------------------------------- /spark/core/src/main/scala/org/elasticsearch/spark/rdd/Metadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.spark.rdd; 20 | 21 | public enum Metadata { 22 | ID, 23 | PARENT, 24 | ROUTING, 25 | /** 26 | * @deprecated TTL is not allowed on index or update requests in Elasticsearch 6.x 27 | */ 28 | @Deprecated 29 | TTL, 30 | /** 31 | * @deprecated Timestamp is not allowed on index or update requests in Elasticsearch 6.x 32 | */ 33 | @Deprecated 34 | TIMESTAMP, 35 | VERSION, 36 | VERSION_TYPE; 37 | } 38 | -------------------------------------------------------------------------------- /spark/core/src/main/scala/org/elasticsearch/spark/rdd/api/java/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | /** 20 | * Java API for Elasticsearch Spark RDD. 21 | */ 22 | package org.elasticsearch.spark.rdd.api.java; 23 | -------------------------------------------------------------------------------- /spark/sql-30/licenses/commons-logging-1.1.1.jar.sha1: -------------------------------------------------------------------------------- 1 | 5043bfebc3db072ed80fbd362e7caf00e885d8ae -------------------------------------------------------------------------------- /spark/sql-30/licenses/commons-logging-NOTICE.txt: -------------------------------------------------------------------------------- 1 | Apache Commons Logging 2 | Copyright 2003-2007 The Apache Software Foundation 3 | 4 | This product includes software developed by 5 | The Apache Software Foundation (http://www.apache.org/). 6 | -------------------------------------------------------------------------------- /spark/sql-30/licenses/jaxb-api-2.3.1.jar.sha1: -------------------------------------------------------------------------------- 1 | 8531ad5ac454cc2deb9d4d32c40c4d7451939b5d -------------------------------------------------------------------------------- /spark/sql-30/licenses/jaxb-api-NOTICE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/elasticsearch-hadoop/aae4528ee8a31e7106e0ee4b671162fb70cf15f1/spark/sql-30/licenses/jaxb-api-NOTICE.txt -------------------------------------------------------------------------------- /spark/sql-30/licenses/scala-library-2.13.13.jar.sha1: -------------------------------------------------------------------------------- 1 | 3b1434403ddb147b8030f98f246153e4ce622c78 -------------------------------------------------------------------------------- /spark/sql-30/licenses/scala-reflect-2.13.13.jar.sha1: -------------------------------------------------------------------------------- 1 | 46866f0e2cf6168322dc11f0f721c2db78d52ac7 -------------------------------------------------------------------------------- /spark/sql-30/licenses/slf4j-api-2.0.6.jar.sha1: -------------------------------------------------------------------------------- 1 | 88c40d8b4f33326f19a7d3c0aaf2c7e8721d4953 -------------------------------------------------------------------------------- /spark/sql-30/licenses/slf4j-api-LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2004-2017 QOS.ch 2 | All rights reserved. 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining 5 | a copy of this software and associated documentation files (the 6 | "Software"), to deal in the Software without restriction, including 7 | without limitation the rights to use, copy, modify, merge, publish, 8 | distribute, sublicense, and/or sell copies of the Software, and to 9 | permit persons to whom the Software is furnished to do so, subject to 10 | the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /spark/sql-30/licenses/slf4j-api-NOTICE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/elasticsearch-hadoop/aae4528ee8a31e7106e0ee4b671162fb70cf15f1/spark/sql-30/licenses/slf4j-api-NOTICE.txt -------------------------------------------------------------------------------- /spark/sql-30/licenses/spark-catalyst_2.13-3.4.3.jar.sha1: -------------------------------------------------------------------------------- 1 | 6a9e9c5e149bd019dd5d781888e1e81ca4d32875 -------------------------------------------------------------------------------- /spark/sql-30/licenses/spark-core_2.13-3.4.3.jar.sha1: -------------------------------------------------------------------------------- 1 | a8055d61a13e9af6871e05ecdca141134d102f1f -------------------------------------------------------------------------------- /spark/sql-30/licenses/spark-sql_2.13-3.4.3.jar.sha1: -------------------------------------------------------------------------------- 1 | b1744080f5e2f81b9a1a088eb375d33b85132b6d -------------------------------------------------------------------------------- /spark/sql-30/licenses/spark-streaming_2.13-3.4.3.jar.sha1: -------------------------------------------------------------------------------- 1 | 64c4e38f3a89428926383f4d9f8a8636a81d499b -------------------------------------------------------------------------------- /spark/sql-30/licenses/spark-yarn_2.13-3.4.3.jar.sha1: -------------------------------------------------------------------------------- 1 | d8fd5251bb23d3bed9d9ef1f19c4402211c78fe4 -------------------------------------------------------------------------------- /spark/sql-30/src/itest/java/org/elasticsearch/spark/integration/ScalaUtils.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.spark.integration; 2 | 3 | import scala.collection.mutable.Map; 4 | import scala.collection.mutable.HashMap; 5 | 6 | import java.util.Properties; 7 | 8 | public class ScalaUtils { 9 | /* 10 | * Scala removed scala.collection.JavaConversions.propertiesAsScalaMap() in 2.13, replacing it with an implicit 11 | * jdk.CollectionConverters.asScala. This is an attempt to get a method that works in 2.10-2.13. It can be removed and replaced with 12 | * jdk.CollectionConverters once we no longer support scala older than 2.13. 13 | */ 14 | public static scala.collection.Map propertiesAsScalaMap(Properties props) { 15 | Map scalaMap = new HashMap(); 16 | for (java.util.Map.Entry entry : props.entrySet()) { 17 | scalaMap.put(entry.getKey(), entry.getValue()); 18 | } 19 | return scalaMap; 20 | } 21 | } -------------------------------------------------------------------------------- /spark/sql-30/src/itest/java/org/elasticsearch/spark/integration/SparkSQLSuite.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.spark.integration; 20 | 21 | import org.elasticsearch.hadoop.fixtures.LocalEs; 22 | import org.junit.ClassRule; 23 | import org.junit.rules.ExternalResource; 24 | import org.junit.runner.RunWith; 25 | import org.junit.runners.Suite; 26 | 27 | @RunWith(Suite.class) 28 | @Suite.SuiteClasses({ AbstractJavaEsSparkSQLTest.class }) 29 | public class SparkSQLSuite { 30 | 31 | @ClassRule 32 | public static ExternalResource resource = new LocalEs(); 33 | 34 | } 35 | -------------------------------------------------------------------------------- /spark/sql-30/src/itest/java/org/elasticsearch/spark/integration/SparkStreamingSuite.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.spark.integration; 21 | 22 | import org.elasticsearch.hadoop.fixtures.LocalEs; 23 | import org.junit.ClassRule; 24 | import org.junit.rules.ExternalResource; 25 | import org.junit.runner.RunWith; 26 | import org.junit.runners.Suite; 27 | 28 | @RunWith(Suite.class) 29 | @Suite.SuiteClasses({ AbstractJavaEsSparkStreamingTest.class }) 30 | public class SparkStreamingSuite { 31 | 32 | @ClassRule 33 | public static ExternalResource resource = new LocalEs(); 34 | 35 | } 36 | -------------------------------------------------------------------------------- /spark/sql-30/src/itest/resources/basic.json: -------------------------------------------------------------------------------- 1 | { "firstName": "John", "isAlive": true, "age": 25, "children": ["Alex", "Joe"], "address": { "streetAddress": "21 2nd Street" } } 2 | -------------------------------------------------------------------------------- /spark/sql-30/src/itest/resources/multi-level-doc.json: -------------------------------------------------------------------------------- 1 | {"EventType":"MessageExportRequested","EventTime":1387710245000,"SessionId":"gsk*****","Trigger":"_null_","ScopedCompanyId":148,"ScopedArchiveId":"anArchive","EventId":"dbnbudzu4wge","ActorEntity":{"CompanyId":148,"IpAddress":"127.0.0.1","EntityId":"602","EntityName":"first-1 last-1","EntityType":"CompanyUser"},"AffectedEntity1":{"EntityId":"5678","EntityName":"5678","EntityType":"MessageExport","ExportPurpose":"FinraAudit","CaseName":"R v Sargisson","NumberMessages":534,"CaseId":"Sarg598","PriceCurrency":"CAD","DeliveryOptions":"DirectDownload","NewestMessageDate":1419112760000,"SpecialRequest":"_null_","ExportName":"Some Export","ExportFormat":"EML","SizeMessagesInBytes":1234789,"ExportDescription":"If the NSA can do it then so can I","ExportOption":"IncludeHiddenRecipientData","Price":500.12,"OldestMessageDate":1387576760000}} -------------------------------------------------------------------------------- /spark/sql-30/src/itest/resources/simple.json: -------------------------------------------------------------------------------- 1 | {"key":"value"} -------------------------------------------------------------------------------- /spark/sql-30/src/itest/resources/small-sample.json: -------------------------------------------------------------------------------- 1 | {"number":"1","name":"MALICE MIZER","url":"http://www.last.fm/music/MALICE+MIZER","picture":"http://userserve-ak.last.fm/serve/252/10808.jpg","@timestamp":"2000-10-06T19:20:25.000Z"} 2 | {"number":"2","name":"Diary of Dreams","url":"http://www.last.fm/music/Diary+of+Dreams","picture":"http://userserve-ak.last.fm/serve/252/3052066.jpg","@timestamp":"2001-10-06T19:20:25.000Z"} 3 | {"number":"3","name":"Carpathian Forest","url":"http://www.last.fm/music/Carpathian+Forest","picture":"http://userserve-ak.last.fm/serve/252/40222717.jpg","@timestamp":"2002-10-06T19:20:25.000Z"} 4 | -------------------------------------------------------------------------------- /spark/sql-30/src/itest/scala/org/elasticsearch/spark/integration/SparkSQLScalaSuite.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.spark.integration; 20 | 21 | import org.elasticsearch.hadoop.fixtures.LocalEs; 22 | import org.junit.ClassRule; 23 | import org.junit.rules.ExternalResource; 24 | import org.junit.runner.RunWith; 25 | import org.junit.runners.Suite; 26 | 27 | @RunWith(Suite.class) 28 | @Suite.SuiteClasses({ AbstractScalaEsScalaSparkSQL.class }) 29 | public class SparkSQLScalaSuite { 30 | 31 | @ClassRule 32 | public static ExternalResource resource = new LocalEs(); 33 | } -------------------------------------------------------------------------------- /spark/sql-30/src/itest/scala/org/elasticsearch/spark/integration/SparkStreamingScalaSuite.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.spark.integration; 20 | 21 | import org.elasticsearch.hadoop.fixtures.LocalEs; 22 | import org.junit.ClassRule; 23 | import org.junit.rules.ExternalResource; 24 | import org.junit.runner.RunWith; 25 | import org.junit.runners.Suite; 26 | 27 | @RunWith(Suite.class) 28 | @Suite.SuiteClasses({ AbstractScalaEsScalaSparkStreaming.class }) 29 | public class SparkStreamingScalaSuite { 30 | 31 | @ClassRule 32 | public static ExternalResource resource = new LocalEs(); 33 | } -------------------------------------------------------------------------------- /spark/sql-30/src/itest/scala/org/elasticsearch/spark/integration/SparkStructuredStreamingScalaSuite.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.spark.integration; 21 | 22 | import org.elasticsearch.hadoop.fixtures.LocalEs; 23 | import org.junit.ClassRule; 24 | import org.junit.rules.ExternalResource; 25 | import org.junit.runner.RunWith; 26 | import org.junit.runners.Suite; 27 | 28 | @RunWith(Suite.class) 29 | @Suite.SuiteClasses({ AbstractScalaEsSparkStructuredStreaming.class }) 30 | public class SparkStructuredStreamingScalaSuite { 31 | 32 | @ClassRule 33 | public static ExternalResource resource = new LocalEs(); 34 | } 35 | -------------------------------------------------------------------------------- /spark/sql-30/src/itest/scala/org/elasticsearch/spark/integration/SparkStructuredStreamingSuite.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.spark.integration; 21 | 22 | import org.elasticsearch.hadoop.fixtures.LocalEs; 23 | import org.junit.ClassRule; 24 | import org.junit.rules.ExternalResource; 25 | import org.junit.runner.RunWith; 26 | import org.junit.runners.Suite; 27 | 28 | @RunWith(Suite.class) 29 | @Suite.SuiteClasses({ AbstractJavaEsSparkStructuredStreamingTest.class }) 30 | public class SparkStructuredStreamingSuite { 31 | 32 | @ClassRule 33 | public static ExternalResource resource = new LocalEs(); 34 | } 35 | -------------------------------------------------------------------------------- /spark/sql-30/src/itest/scala/org/elasticsearch/spark/serialization/Garbage.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.spark.serialization 21 | 22 | class Garbage(i: Int) { 23 | def doNothing(): Unit = () 24 | } 25 | -------------------------------------------------------------------------------- /spark/sql-30/src/itest/scala/org/elasticsearch/spark/serialization/ModuleCaseClass.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.spark.serialization 21 | 22 | case class ModuleCaseClass(id: Integer, departure: String, var arrival: String) { 23 | var l = math.Pi 24 | } 25 | -------------------------------------------------------------------------------- /spark/sql-30/src/itest/scala/org/elasticsearch/spark/serialization/Trip.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.spark.serialization 21 | 22 | case class Trip(departure: String, arrival: String) { 23 | var extra = math.Pi 24 | } 25 | -------------------------------------------------------------------------------- /spark/sql-30/src/itest/scala/org/elasticsearch/spark/sql/SchemaUtilsTestable.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.spark.sql 20 | 21 | import org.elasticsearch.hadoop.cfg.Settings 22 | 23 | object SchemaUtilsTestable { 24 | 25 | def discoverMapping(cfg: Settings) = SchemaUtils.discoverMapping(cfg) 26 | 27 | def rowInfo(cfg: Settings) = { 28 | val schema = SchemaUtils.discoverMapping(cfg) 29 | SchemaUtils.setRowInfo(cfg, schema.struct) 30 | SchemaUtils.getRowInfo(cfg) 31 | } 32 | } -------------------------------------------------------------------------------- /spark/sql-30/src/main/resources/META-INF/services/org.apache.spark.security.HadoopDelegationTokenProvider: -------------------------------------------------------------------------------- 1 | org.elasticsearch.spark.deploy.yarn.security.EsServiceCredentialProvider -------------------------------------------------------------------------------- /spark/sql-30/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister: -------------------------------------------------------------------------------- 1 | org.elasticsearch.spark.sql.DefaultSource15 -------------------------------------------------------------------------------- /spark/sql-30/src/main/scala/org/elasticsearch/spark/sql/DefaultSource15.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.spark.sql 20 | 21 | import org.apache.spark.sql.sources.DataSourceRegister 22 | 23 | // extension of DefaultSource (which is Spark 1.3 and 1.4 compatible) 24 | // for Spark 1.5 compatibility 25 | // since the class is loaded through META-INF/services we can decouple the two 26 | // to have Spark 1.5 byte-code loaded lazily 27 | class DefaultSource15 extends DefaultSource with DataSourceRegister { 28 | 29 | override def shortName(): String = "es" 30 | } -------------------------------------------------------------------------------- /spark/sql-30/src/main/scala/org/elasticsearch/spark/sql/SparkSQLCompatibilityLevel.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.spark.sql 20 | 21 | import org.elasticsearch.spark.rdd.CompatibilityLevel 22 | 23 | /** 24 | * For determining Spark Version Compatibility 25 | */ 26 | class SparkSQLCompatibilityLevel extends CompatibilityLevel { 27 | val versionId = "20" 28 | val versionDescription = "2.0" 29 | } 30 | -------------------------------------------------------------------------------- /spark/sql-30/src/main/scala/org/elasticsearch/spark/sql/api/java/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | /** 20 | * Java API for Elasticsearch Spark SQL. 21 | */ 22 | package org.elasticsearch.spark.sql.api.java; 23 | -------------------------------------------------------------------------------- /spark/sql-30/src/main/scala/org/elasticsearch/spark/sql/streaming/EsSinkStatus.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.spark.sql.streaming 21 | 22 | /** 23 | * An envelope of metadata that is stored in the commit log. Mostly contains statistics about 24 | * what was written in the given batch. 25 | * 26 | * @param taskId of the task that wrote this data 27 | * @param execTimeMillis when the data was written 28 | * @param resource which resource the data was written to, patterns included 29 | * @param records number of records written to resource 30 | */ 31 | case class EsSinkStatus(taskId: Int, execTimeMillis: Long, resource: String, records: Long) 32 | -------------------------------------------------------------------------------- /spark/sql-30/src/main/scala/org/elasticsearch/spark/streaming/api/java/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | /** 21 | * Java API for Elasticsearch Spark Streaming. 22 | */ 23 | package org.elasticsearch.spark.streaming.api.java; -------------------------------------------------------------------------------- /spark/sql-30/src/test/scala/org/elasticsearch/spark/rdd/CompatibilityCheckTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.spark.rdd 20 | 21 | import org.junit.Test 22 | 23 | /** 24 | * Created by james.baiera on 8/17/16. 25 | */ 26 | class CompatibilityCheckTest { 27 | 28 | @Test 29 | def checkCompatibility: Unit = { 30 | CompatUtils.checkSparkLibraryCompatibility(true) 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /test/build.gradle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/elasticsearch-hadoop/aae4528ee8a31e7106e0ee4b671162fb70cf15f1/test/build.gradle -------------------------------------------------------------------------------- /test/fixtures/build.gradle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/elasticsearch-hadoop/aae4528ee8a31e7106e0ee4b671162fb70cf15f1/test/fixtures/build.gradle -------------------------------------------------------------------------------- /test/shared/src/main/resources/data/join/data/all.dat: -------------------------------------------------------------------------------- 1 | 1 "Elastic" "company" 2 | 10 "kimchy" "employee" "1" 3 | 2 "Fringe Cafe" null "company" 4 | 20 "April Ryan" "employee" "2" 5 | 21 "Charlie" "employee" "2" 6 | 3 "WATIcorp" "company" 7 | 30 "Alvin Peats" "employee" "3" -------------------------------------------------------------------------------- /test/shared/src/main/resources/data/join/data/children.dat: -------------------------------------------------------------------------------- 1 | 10 "kimchy" "employee" "1" 2 | 20 "April Ryan" "employee" "2" 3 | 21 "Charlie" "employee" "2" 4 | 30 "Alvin Peats" "employee" "3" -------------------------------------------------------------------------------- /test/shared/src/main/resources/data/join/data/parents.dat: -------------------------------------------------------------------------------- 1 | 1 "Elastic" "company" 2 | 2 "Fringe Cafe" "company" 3 | 3 "WATIcorp" "company" -------------------------------------------------------------------------------- /test/shared/src/main/resources/data/join/json/all.json: -------------------------------------------------------------------------------- 1 | {"id":"1","company":"Elastic","joiner":"company"} 2 | {"id":"10","name":"kimchy","joiner":{"name":"employee","parent":"1"}} 3 | {"id":"2","company":"Fringe Cafe","joiner":"company"} 4 | {"id":"20","name":"April Ryan","joiner":{"name":"employee","parent":"2"}} 5 | {"id":"21","name":"Charlie","joiner":{"name":"employee","parent":"2"}} 6 | {"id":"3","company":"WATIcorp","joiner":"company"} 7 | {"id":"30","name":"Alvin Peats","joiner":{"name":"employee","parent":"3"}} -------------------------------------------------------------------------------- /test/shared/src/main/resources/data/join/json/children.json: -------------------------------------------------------------------------------- 1 | {"id":"10","name":"kimchy","joiner":{"name":"employee","parent":"1"}} 2 | {"id":"20","name":"April Ryan","joiner":{"name":"employee","parent":"2"}} 3 | {"id":"21","name":"Charlie","joiner":{"name":"employee","parent":"2"}} 4 | {"id":"30","name":"Alvin Peats","joiner":{"name":"employee","parent":"3"}} -------------------------------------------------------------------------------- /test/shared/src/main/resources/data/join/json/parents.json: -------------------------------------------------------------------------------- 1 | {"id":"1","company":"Elastic","joiner":"company"} 2 | {"id":"2","company":"Fringe Cafe","joiner":"company"} 3 | {"id":"3","company":"WATIcorp","joiner":"company"} -------------------------------------------------------------------------------- /test/shared/src/main/resources/data/join/mapping/typed.json: -------------------------------------------------------------------------------- 1 | { 2 | "join":{ 3 | "properties": { 4 | "id": {"type": "keyword"}, 5 | "company": {"type": "text"}, 6 | "name": {"type": "text"}, 7 | "joiner": { 8 | "type": "join", 9 | "relations": { 10 | "company": "employee" 11 | } 12 | } 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /test/shared/src/main/resources/data/join/mapping/typeless.json: -------------------------------------------------------------------------------- 1 | { 2 | "properties": { 3 | "id": {"type": "keyword"}, 4 | "company": {"type": "text"}, 5 | "name": {"type": "text"}, 6 | "joiner": { 7 | "type": "join", 8 | "relations": { 9 | "company": "employee" 10 | } 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /test/shared/src/main/resources/gibberish.dat: -------------------------------------------------------------------------------- 1 | 2014-10-02-client_(select(0)from(select(sleep(4)))v)/*'+(select(0)from(select(sleep(4)))v)+'"+(select(0)from(select(sleep(4)))v)+"*/-d6c81da88d482fa636e2cc5b19abb3a6 client_(select(0)from(select(sleep(4)))v)/*'+(select(0)from(select(sleep(4)))v)+'"+(select(0)from(select(sleep(4)))v)+"*/ 2014-10-02 1 2 | 2014-10-02-client_(select(0)from(select(sleep(9)))v)/*'+(select(0)from(select(sleep(9)))v)+'"+(select(0)from(select(sleep(9)))v)+"*/-caba33866b9b6c36c0791fd8f622802a client_(select(0)from(select(sleep(9)))v)/*'+(select(0)from(select(sleep(9)))v)+'"+(select(0)from(select(sleep(9)))v)+"*/ 2014-10-02 1 3 | 2014-10-02-client_(select(0)from(select(sleep(15)))v)/*'+(select(0)from(select(sleep(15)))v)+'"+(select(0)from(select(sleep(15)))v)+"*/-a1ae7275a600d2b9b77c1a35cf2a2203 client_(select(0)from(select(sleep(15)))v)/*'+(select(0)from(select(sleep(15)))v)+'"+(select(0)from(select(sleep(15)))v)+"*/ 2014-10-02 1 4 | -------------------------------------------------------------------------------- /test/shared/src/main/resources/gibberish.json: -------------------------------------------------------------------------------- 1 | { "@id": "2014-10-02-client_(select(0)from(select(sleep(4)))v)/*\u0027+(select(0)from(select(sleep(4)))v)+\u0027\"+(select(0)from(select(sleep(4)))v)+\"*/-d6c81da88d482fa636e2cc5b19abb3a6", "@key": "client_(select(0)from(select(sleep(4)))v)/*\u0027+(select(0)from(select(sleep(4)))v)+\u0027\"+(select(0)from(select(sleep(4)))v)+\"*/", "@timestamp": "2014-10-02", "@value": 1 } 2 | { "@id": "2014-10-02-client_(select(0)from(select(sleep(9)))v)/*\u0027+(select(0)from(select(sleep(9)))v)+\u0027\"+(select(0)from(select(sleep(9)))v)+\"*/-caba33866b9b6c36c0791fd8f622802a","@key": "client_(select(0)from(select(sleep(9)))v)/*\u0027+(select(0)from(select(sleep(9)))v)+\u0027\"+(select(0)from(select(sleep(9)))v)+\"*/", "@timestamp": "2014-10-02", "@value": 1 } 3 | { "@id": "2014-10-02-client_(select(0)from(select(sleep(15)))v)/*\u0027+(select(0)from(select(sleep(15)))v)+\u0027\"+(select(0)from(select(sleep(15)))v)+\"*/-a1ae7275a600d2b9b77c1a35cf2a2203", "@key": "client_(select(0)from(select(sleep(15)))v)/*\u0027+(select(0)from(select(sleep(15)))v)+\u0027\"+(select(0)from(select(sleep(15)))v)+\"*/", "@timestamp": "2014-10-02", "@value": 1 } -------------------------------------------------------------------------------- /test/shared/src/main/resources/org/elasticsearch/hadoop/integration/query.dsl: -------------------------------------------------------------------------------- 1 | { 2 | "query" : { 3 | "query_string" : { 4 | "query" : "m*" 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /test/shared/src/main/resources/org/elasticsearch/hadoop/integration/query.uri: -------------------------------------------------------------------------------- 1 | ?q=m* -------------------------------------------------------------------------------- /test/shared/src/test/resources/org/elasticsearch/hadoop/util/escape-size.txt: -------------------------------------------------------------------------------- 1 | üüüüü -------------------------------------------------------------------------------- /test/shared/src/test/resources/org/elasticsearch/hadoop/util/escaped-chars.txt: -------------------------------------------------------------------------------- 1 | Würzburg 2 | لَيسَ كُل مَ آكتُبه حِكَآيةة عَنْ وَٱقِعِي إنَمَآ هِيآ كَلِمآت رَاقَت لِي 3 | Würzburg 4 | -------------------------------------------------------------------------------- /thirdparty/build.gradle: -------------------------------------------------------------------------------- 1 | import org.elasticsearch.hadoop.gradle.BuildPlugin 2 | 3 | 4 | 5 | buildscript { 6 | repositories { 7 | maven { 8 | url = 'https://jitpack.io' 9 | } 10 | mavenCentral() 11 | } 12 | dependencies { 13 | classpath 'com.github.breskeby:shadow:3b035f2' 14 | } 15 | } 16 | 17 | plugins { 18 | id 'es.hadoop.build' 19 | } 20 | 21 | apply plugin: 'com.github.johnrengelman.shadow' 22 | 23 | 24 | description = "Elasticsearch Hadoop Shaded Third-Party Libraries" 25 | 26 | configurations { 27 | shaded { 28 | transitive = false 29 | canBeResolved = true 30 | } 31 | implementation { 32 | extendsFrom shaded 33 | } 34 | } 35 | 36 | BuildPlugin.disableTransitiveDependencies(project, configurations.shaded) 37 | 38 | dependencies { 39 | shaded("commons-httpclient:commons-httpclient:3.0.1") 40 | shaded("org.codehaus.jackson:jackson-mapper-asl:${project.ext.jacksonVersion}") 41 | shaded("org.codehaus.jackson:jackson-core-asl:${project.ext.jacksonVersion}") 42 | } 43 | 44 | shadowJar { 45 | configurations = [project.configurations.shaded] 46 | relocate 'org.apache.commons.httpclient', 'org.elasticsearch.hadoop.thirdparty.apache.commons.httpclient' 47 | relocate 'org.codehaus.jackson', 'org.elasticsearch.hadoop.thirdparty.codehaus.jackson' 48 | } 49 | -------------------------------------------------------------------------------- /thirdparty/licenses/commons-httpclient-3.0.1.jar.sha1: -------------------------------------------------------------------------------- 1 | d6364bcc1b2b2aa69d008602d36a700453648560 -------------------------------------------------------------------------------- /thirdparty/licenses/commons-httpclient-NOTICE.txt: -------------------------------------------------------------------------------- 1 | Apache HttpComponents Commons HttpClient 2 | Copyright 1999-2008 The Apache Software Foundation 3 | 4 | This product includes software developed by 5 | The Apache Software Foundation (http://www.apache.org/). -------------------------------------------------------------------------------- /thirdparty/licenses/jackson-NOTICE.txt: -------------------------------------------------------------------------------- 1 | # Jackson JSON processor 2 | 3 | Jackson is a high-performance, Free/Open Source JSON processing library. 4 | It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has 5 | been in development since 2007. 6 | It is currently developed by a community of developers, as well as supported 7 | commercially by FasterXML.com. 8 | 9 | ## Licensing 10 | 11 | Jackson core and extension components may licensed under different licenses. 12 | To find the details that apply to this artifact see the accompanying LICENSE file. 13 | For more information, including possible other licensing options, contact 14 | FasterXML.com (http://fasterxml.com). 15 | 16 | ## Credits 17 | 18 | A list of contributors may be found from CREDITS file, which is included 19 | in some artifacts (usually source distributions); but is always available 20 | from the source code management (SCM) system project uses. 21 | -------------------------------------------------------------------------------- /thirdparty/licenses/jackson-core-asl-1.8.8.jar.sha1: -------------------------------------------------------------------------------- 1 | dd2e90bb710ea3bc4610e24299d6a4c8dac5049b -------------------------------------------------------------------------------- /thirdparty/licenses/jackson-mapper-asl-1.8.8.jar.sha1: -------------------------------------------------------------------------------- 1 | 01bb32d9d2527c083a56e234acca49a48d3c65c8 --------------------------------------------------------------------------------