├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── example_files ├── CRM_profiles.csv └── CRM_profiles.json ├── examples ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── adobe │ │ │ └── platform │ │ │ └── ecosystem │ │ │ └── examples │ │ │ ├── authentication │ │ │ ├── AccessTokenProcessor.java │ │ │ └── ETLAuthConnection.java │ │ │ ├── catalog │ │ │ ├── api │ │ │ │ └── CatalogService.java │ │ │ ├── function │ │ │ │ └── DataTypeFunction.java │ │ │ ├── impl │ │ │ │ ├── CatalogAPIStrategy.java │ │ │ │ ├── CatalogFactory.java │ │ │ │ └── CatalogServiceImpl.java │ │ │ └── model │ │ │ │ ├── BaseModel.java │ │ │ │ ├── Batch.java │ │ │ │ ├── Connection.java │ │ │ │ ├── DataSet.java │ │ │ │ ├── DataSetFile.java │ │ │ │ ├── DataSetView.java │ │ │ │ ├── DataType.java │ │ │ │ ├── Dule.java │ │ │ │ ├── EntityBuilder.java │ │ │ │ ├── FileDescription.java │ │ │ │ ├── SDKField.java │ │ │ │ ├── Schema.java │ │ │ │ ├── SchemaField.java │ │ │ │ └── SchemaRef.java │ │ │ ├── constants │ │ │ └── SDKConstants.java │ │ │ ├── data │ │ │ ├── FileFormat.java │ │ │ ├── Pair.java │ │ │ ├── access │ │ │ │ ├── api │ │ │ │ │ └── DataAccessService.java │ │ │ │ ├── factory │ │ │ │ │ └── DataAccessServiceFactory.java │ │ │ │ ├── impl │ │ │ │ │ └── DataAccessServiceImpl.java │ │ │ │ └── model │ │ │ │ │ ├── DataAccessFileEntity.java │ │ │ │ │ └── DataSetFileProcessingEntity.java │ │ │ ├── functions │ │ │ │ └── FieldConverterFunction.java │ │ │ ├── ingestion │ │ │ │ ├── api │ │ │ │ │ └── DataIngestionService.java │ │ │ │ ├── factory │ │ │ │ │ └── DataIngestionServiceFactory.java │ │ │ │ └── impl │ │ │ │ │ └── DataIngestionServiceImpl.java │ │ │ ├── read │ │ │ │ ├── DataReaderFactory.java │ │ │ │ ├── PlatformDataReaderFactory.java │ │ │ │ ├── ReadAttributes.java │ │ │ │ ├── Reader.java │ │ │ │ └── reader │ │ │ │ │ ├── DataAccessAPIReader.java │ │ │ │ │ └── processor │ │ │ │ │ ├── api │ │ │ │ │ └── ReaderProcessor.java │ │ │ │ │ └── impl │ │ │ │ │ ├── CSVReaderProcessor.java │ │ │ │ │ ├── JSONReaderProcessor.java │ │ │ │ │ └── ParquetReaderProcessor.java │ │ │ ├── validation │ │ │ │ ├── api │ │ │ │ │ ├── Rule.java │ │ │ │ │ ├── ValidationRegistry.java │ │ │ │ │ └── ValidationRegistryFactory.java │ │ │ │ ├── exception │ │ │ │ │ ├── ValidationException.java │ │ │ │ │ └── ValidationExceptionBuilder.java │ │ │ │ └── impl │ │ │ │ │ ├── CatalogValidationRegistry.java │ │ │ │ │ ├── CatalogValidationRegistryFactory.java │ │ │ │ │ ├── TraversablePath.java │ │ │ │ │ └── rules │ │ │ │ │ ├── IntegerValidationRule.java │ │ │ │ │ ├── LongValidationRule.java │ │ │ │ │ ├── SchemaValidationRule.java │ │ │ │ │ └── StringValidationRule.java │ │ │ ├── wiring │ │ │ │ ├── DataWiring.java │ │ │ │ ├── DataWiringParam.java │ │ │ │ └── Wiring.java │ │ │ └── write │ │ │ │ ├── DataFormatterFactory.java │ │ │ │ ├── DataWriterFactory.java │ │ │ │ ├── FlushHandler.java │ │ │ │ ├── Formatter.java │ │ │ │ ├── PlatformDataFormatterFactory.java │ │ │ │ ├── PlatformDataWriterFactory.java │ │ │ │ ├── WriteAttributes.java │ │ │ │ ├── Writer.java │ │ │ │ ├── converter │ │ │ │ └── PipelineToJSONConverter.java │ │ │ │ ├── field │ │ │ │ └── converter │ │ │ │ │ └── parquet │ │ │ │ │ ├── JSONParquetFieldConverter.java │ │ │ │ │ ├── ParquetFieldConverter.java │ │ │ │ │ └── catalog │ │ │ │ │ ├── CachedCatalogSchemaParquetFieldConverter.java │ │ │ │ │ └── CatalogSchemaParquetFieldConverter.java │ │ │ │ ├── mapper │ │ │ │ └── MapperUtil.java │ │ │ │ └── writer │ │ │ │ ├── DataIngestionAPIWriter.java │ │ │ │ ├── extractor │ │ │ │ ├── Extractor.java │ │ │ │ └── JsonObjectsExtractor.java │ │ │ │ └── formatter │ │ │ │ ├── CSVDataFormatter.java │ │ │ │ ├── JSONDataFormatter.java │ │ │ │ └── ParquetDataFormatter.java │ │ │ ├── filter │ │ │ ├── Filter.java │ │ │ ├── FilterFactory.java │ │ │ └── SchemaFieldFilter.java │ │ │ ├── schemaregistry │ │ │ ├── SchemaRegistryFactory.java │ │ │ ├── SchemaRegistryServiceImpl.java │ │ │ └── api │ │ │ │ └── SchemaRegistryService.java │ │ │ └── util │ │ │ ├── ConnectorSDKException.java │ │ │ ├── ConnectorSDKUtil.java │ │ │ ├── HttpClientUtil.java │ │ │ ├── JsonUtil.java │ │ │ ├── ResourceName.java │ │ │ ├── ResourceReader.java │ │ │ └── SDKDataTypeJsonUtil.java │ └── resources │ │ ├── config.properties │ │ ├── dev │ │ └── config.properties │ │ ├── log4j2.properties │ │ ├── pre-prod │ │ └── config.properties │ │ └── prod │ │ └── config.properties │ └── test │ ├── java │ └── com │ │ └── adobe │ │ └── platform │ │ └── ecosystem │ │ ├── examples │ │ ├── authentication │ │ │ ├── AuthenticationUtilTest.java │ │ │ ├── ETLAuthConnectionTest.java │ │ │ └── TokenProcessorTest.java │ │ ├── catalog │ │ │ ├── impl │ │ │ │ ├── CatalogFactoryTest.java │ │ │ │ └── CatalogServiceTest.java │ │ │ └── model │ │ │ │ ├── ConnectionTest.java │ │ │ │ ├── DataSetTest.java │ │ │ │ └── DataSetViewTest.java │ │ ├── data │ │ │ ├── access │ │ │ │ ├── factory │ │ │ │ │ └── DataAccessServiceFactoryTest.java │ │ │ │ ├── impl │ │ │ │ │ └── DataAccessServiceTest.java │ │ │ │ └── model │ │ │ │ │ ├── DataAccessFileEntityTest.java │ │ │ │ │ └── DataSetFileProcessingEntityTest.java │ │ │ ├── functions │ │ │ │ └── FieldConverterFunctionTest.java │ │ │ ├── ingestion │ │ │ │ └── impl │ │ │ │ │ └── DataIngestionServiceTest.java │ │ │ ├── read │ │ │ │ ├── PlatformDataReaderFactoryTest.java │ │ │ │ └── reader │ │ │ │ │ └── DataAccessAPIReaderTest.java │ │ │ ├── validation │ │ │ │ └── impl │ │ │ │ │ ├── CatalogValidationRegistryFactoryTest.java │ │ │ │ │ ├── CatalogValidationRegistryTest.java │ │ │ │ │ └── rules │ │ │ │ │ ├── IntegerValidationRuleTest.java │ │ │ │ │ ├── LongValidationRuleTest.java │ │ │ │ │ └── StringValidationRuleTest.java │ │ │ ├── wiring │ │ │ │ └── DataWiringTest.java │ │ │ └── write │ │ │ │ ├── PlatformDataFormatterFactoryTest.java │ │ │ │ ├── PlatformDataWriterFactoryTest.java │ │ │ │ ├── converter │ │ │ │ └── PipelineToJSONConverterTest.java │ │ │ │ ├── field │ │ │ │ └── converter │ │ │ │ │ └── parquet │ │ │ │ │ ├── JSONParquetFieldConverterTest.java │ │ │ │ │ └── catalog │ │ │ │ │ ├── CachedCatalogSchemaParquetFieldConverterTest.java │ │ │ │ │ └── CatalogSchemaParquetFieldConverterTest.java │ │ │ │ ├── mapper │ │ │ │ └── MapperUtilTest.java │ │ │ │ └── writer │ │ │ │ ├── DataIngestionAPIWriterTest.java │ │ │ │ └── formatter │ │ │ │ └── ParquetDataFormatterTest.java │ │ ├── schemaregistry │ │ │ ├── SchemaRegistryFactoryTest.java │ │ │ └── SchemaRegistryServiceTest.java │ │ └── util │ │ │ ├── ConnectorSDKUtilTest.java │ │ │ ├── HTTPClientUtilTest.java │ │ │ ├── JsonUtilTest.java │ │ │ └── ResourceReaderTest.java │ │ └── ut │ │ └── BaseTest.java │ └── resources │ ├── objectWithRootNode.json │ ├── objectXDM.json │ ├── pdfTest.parquet │ ├── primitive.json │ ├── primitiveXdmArray.json │ ├── sample.parquet │ ├── secret.key │ ├── secret_invalid.key │ └── structXdmArray.json ├── parquetio ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── adobe │ │ └── platform │ │ └── ecosystem │ │ └── examples │ │ ├── hierarchy │ │ └── util │ │ │ └── HierarchicalBuilderUtil.java │ │ └── parquet │ │ ├── entity │ │ └── Node.java │ │ ├── exception │ │ ├── ErrorCode.java │ │ ├── ParquetIOErrorCode.java │ │ └── ParquetIOException.java │ │ ├── model │ │ ├── ParquetIODataType.java │ │ ├── ParquetIOField.java │ │ └── ParquetIORepetitionType.java │ │ ├── read │ │ ├── ParquetIOReader.java │ │ ├── ParquetIOReaderImpl.java │ │ ├── ParquetIOReaderIterator.java │ │ ├── ReaderUtil.java │ │ └── configuration │ │ │ └── ParquetReaderConfiguration.java │ │ ├── utility │ │ └── ParquetIOUtil.java │ │ ├── wiring │ │ ├── api │ │ │ └── ParquetIO.java │ │ └── impl │ │ │ └── ParquetIOImpl.java │ │ └── write │ │ ├── ParquetIOWriter.java │ │ └── ParquetIOWriterImpl.java │ └── test │ ├── java │ └── com │ │ └── adobe │ │ └── platform │ │ └── ecosystem │ │ └── examples │ │ └── parquet │ │ ├── read │ │ ├── ParquetIOReaderImplTest.java │ │ └── ParquetIOReaderIteratorTest.java │ │ ├── ut │ │ └── BaseTest.java │ │ ├── wiring │ │ └── ParquetIOImplTest.java │ │ └── write │ │ └── ParquetIOWriterImplTest.java │ └── resources │ ├── allDataTypes.parquet │ ├── int96file.parquet │ └── test_snappy.parquet └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | *.idea 2 | *.iml 3 | */target/* 4 | target/* 5 | *.log 6 | .DS_Store 7 | /target/ 8 | bin/ 9 | .project 10 | .classpath 11 | .settings 12 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Adobe Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at Grp-opensourceoffice@adobe.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at [http://contributor-covenant.org/version/1/4][version] 72 | 73 | [homepage]: http://contributor-covenant.org 74 | [version]: http://contributor-covenant.org/version/1/4/ 75 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Thanks for choosing to contribute! 4 | 5 | The following are a set of guidelines to follow when contributing to this project. 6 | 7 | ## Code Of Conduct 8 | 9 | This project adheres to the Adobe [code of conduct](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to [Grp-opensourceoffice@adobe.com](mailto:Grp-opensourceoffice@adobe.com). 10 | 11 | ## Contributor License Agreement 12 | 13 | All third-party contributions to this project must be accompanied by a signed contributor license agreement. This gives Adobe permission to redistribute your contributions as part of the project. [Sign our CLA](http://opensource.adobe.com/cla.html). You only need to submit an Adobe CLA one time, so if you have submitted one previously, you are good to go! 14 | 15 | ## Code Reviews 16 | 17 | All submissions should come in the form of pull requests and need to be reviewed by project committers. Read [GitHub's pull request documentation](https://help.github.com/articles/about-pull-requests/) for more information on sending pull requests. 18 | 19 | Lastly, please follow the [pull request template](PULL_REQUEST_TEMPLATE.md) when submitting a pull request! 20 | -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/authentication/ETLAuthConnection.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.authentication; 18 | 19 | import com.adobe.platform.ecosystem.examples.catalog.api.CatalogService; 20 | import com.adobe.platform.ecosystem.examples.catalog.impl.CatalogAPIStrategy; 21 | import com.adobe.platform.ecosystem.examples.catalog.impl.CatalogFactory; 22 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 23 | 24 | import java.util.HashMap; 25 | 26 | /** 27 | * Created by nidhi on 22/9/17. 28 | */ 29 | public class ETLAuthConnection { 30 | 31 | /** 32 | * This API checks of the connection is successful with the connection parameters 33 | * provided by the client during ETL connection initialization 34 | * @param accessToken 35 | * @param imsOrg 36 | * @throws ConnectorSDKException 37 | */ 38 | public void validateConnection(String accessToken, String imsOrg) 39 | throws ConnectorSDKException{ 40 | //fetch batches from Catalog to test the validitity of connection credentials 41 | try { 42 | CatalogService catalogService = CatalogFactory.getCatalogService(); 43 | catalogService.getBatches(imsOrg, accessToken, new HashMap<>(), CatalogAPIStrategy.ONCE); 44 | }catch (Exception ex){ 45 | throw new ConnectorSDKException("Please check the connection parameters imsOrg, catalog end point, access token : " + ex.getMessage(), ex); 46 | } 47 | } 48 | 49 | 50 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/catalog/function/DataTypeFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Adobe. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.adobe.platform.ecosystem.examples.catalog.function; 17 | 18 | import com.adobe.platform.ecosystem.examples.catalog.model.DataType; 19 | 20 | import java.util.function.Function; 21 | 22 | /** 23 | * Util class to provide function 24 | * for mapping Catalog {@code String} 25 | * types to SDK's {@code DataType}. 26 | * 27 | * @author vedhera 01/07/2019. 28 | */ 29 | public class DataTypeFunction { 30 | 31 | public static Function primitiveFunction() { 32 | return (type -> { 33 | DataType dataType = null; 34 | switch (type) { 35 | case "string": 36 | dataType = DataType.StringType; 37 | break; 38 | case "long": 39 | dataType = DataType.LongType; 40 | break; 41 | case "date": 42 | // format, we will start parsing date in same format 43 | dataType = DataType.StringType; 44 | break; 45 | case "date-time": 46 | dataType = DataType.DateTimeType; 47 | break; 48 | case "integer": 49 | case "int": 50 | dataType = DataType.IntegerType; 51 | break; 52 | case "byte": 53 | dataType = DataType.ByteType; 54 | break; 55 | case "short": 56 | dataType = DataType.ShortType; 57 | break; 58 | case "number": 59 | dataType = DataType.DoubleType; 60 | break; 61 | case "float": 62 | dataType = DataType.FloatType; 63 | break; 64 | case "double": 65 | dataType = DataType.DoubleType; 66 | break; 67 | case "boolean": 68 | dataType = DataType.BooleanType; 69 | break; 70 | case "binary": 71 | dataType = DataType.BinaryType; 72 | } 73 | return dataType; 74 | }); 75 | } 76 | } 77 | 78 | -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/catalog/impl/CatalogAPIStrategy.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.catalog.impl; 18 | 19 | /** 20 | * Enum to indicate different API strategy 21 | * for calling Catalog API's. 22 | * 23 | * @author vedhera on 1/23/2018. 24 | */ 25 | public enum CatalogAPIStrategy { 26 | ONCE("API call to be executed once."), 27 | REPEATED("API call to be executed till the required entity exhaust."); 28 | 29 | private final String description; 30 | 31 | CatalogAPIStrategy(String description) { 32 | this.description = description; 33 | } 34 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/catalog/impl/CatalogFactory.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.catalog.impl; 18 | 19 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKUtil; 20 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 21 | import com.adobe.platform.ecosystem.examples.util.ResourceName; 22 | import org.apache.http.client.HttpClient; 23 | 24 | import com.adobe.platform.ecosystem.examples.catalog.api.CatalogService; 25 | 26 | /** 27 | * Created by vedhera on 8/25/2017. 28 | */ 29 | 30 | /** 31 | * Factory to provide Catalog 32 | * service implementation. 33 | */ 34 | public class CatalogFactory { 35 | 36 | public static CatalogService getCatalogService(HttpClient httpClient) throws ConnectorSDKException { 37 | ConnectorSDKUtil adobeResourceUtil = ConnectorSDKUtil.getInstance(); 38 | String catalogEndpoint = adobeResourceUtil.getEndPoint(ResourceName.CATALOG); 39 | return new CatalogServiceImpl(catalogEndpoint, httpClient); 40 | } 41 | 42 | 43 | public static CatalogService getCatalogService() throws ConnectorSDKException{ 44 | return getCatalogService(null); 45 | } 46 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/catalog/model/Batch.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.catalog.model; 18 | 19 | /** 20 | * Created by vedhera on 8/25/2017. 21 | */ 22 | 23 | import com.adobe.platform.ecosystem.examples.constants.SDKConstants; 24 | import org.json.simple.JSONArray; 25 | import org.json.simple.JSONObject; 26 | 27 | import java.lang.reflect.Field; 28 | import java.util.HashMap; 29 | import java.util.Map; 30 | 31 | /** 32 | * 33 | */ 34 | public class Batch extends BaseModel { 35 | private static final Map _keys; 36 | 37 | private String status; 38 | private JSONArray relatedObjects; 39 | 40 | static { 41 | _keys = new HashMap() {{ 42 | put(SDKConstants.CATALOG_STATUS, DataType.StringType); 43 | put(SDKConstants.CATALOG_BATCH_RELATEDOBJECTS, DataType.JsonArrayType); 44 | }}; 45 | } 46 | 47 | public Batch() { 48 | super(); 49 | } 50 | 51 | public Batch(JSONObject jsonObject) { 52 | super(jsonObject); 53 | Field[] fields = Batch.class.getDeclaredFields(); 54 | super.populateFields(fields,this._keys,this,jsonObject); 55 | } 56 | 57 | public String getStatus() { 58 | return status; 59 | } 60 | 61 | public JSONArray getRelatedObjects() { 62 | return relatedObjects; 63 | } 64 | 65 | /** 66 | * {@inheritDoc} 67 | */ 68 | @Override 69 | public Batch build(JSONObject obj) { 70 | return new Batch(obj); 71 | } 72 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/catalog/model/Connection.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.catalog.model; 18 | 19 | /** 20 | * Created by prigarg on 9/22/2017. 21 | */ 22 | 23 | import com.adobe.platform.ecosystem.examples.constants.SDKConstants; 24 | import org.json.simple.JSONObject; 25 | 26 | import java.lang.reflect.Field; 27 | import java.util.HashMap; 28 | import java.util.Map; 29 | 30 | public class Connection extends BaseModel { 31 | private static final Map _keys; 32 | 33 | private JSONObject dule; 34 | 35 | private Dule duleObj; 36 | 37 | static { 38 | _keys = new HashMap() {{ 39 | put(SDKConstants.CATALOG_DULE, DataType.Field_ObjectType); 40 | }}; 41 | } 42 | 43 | public Connection() { 44 | super(); 45 | } 46 | 47 | public Connection(JSONObject jsonObject) { 48 | super(jsonObject); 49 | Field[] fields = Connection.class.getDeclaredFields(); 50 | super.populateFields(fields,this._keys,this,jsonObject); 51 | } 52 | 53 | public Dule getDule() { 54 | if(duleObj == null){ 55 | duleObj = new Dule(dule); 56 | } 57 | 58 | return duleObj; 59 | } 60 | 61 | public Boolean getIsDuleEnabled() { return getDule().isEnabled();} 62 | 63 | /** 64 | * {@inheritDoc} 65 | */ 66 | @Override 67 | public Connection build(JSONObject obj) { 68 | return new Connection(obj); 69 | } 70 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/catalog/model/DataSetFile.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.catalog.model; 18 | 19 | /** 20 | * Created by vedhera on 8/25/2017. 21 | */ 22 | 23 | import com.adobe.platform.ecosystem.examples.constants.SDKConstants; 24 | import org.json.simple.JSONObject; 25 | 26 | import java.lang.reflect.Field; 27 | import java.util.HashMap; 28 | import java.util.Map; 29 | 30 | public class DataSetFile extends BaseModel { 31 | private static final Map _keys; 32 | 33 | private String folderName; 34 | 35 | static { 36 | _keys = new HashMap() {{ 37 | put(SDKConstants.CATALOG_FOLDERNAME, DataType.StringType); 38 | }}; 39 | } 40 | 41 | public DataSetFile() { 42 | super(); 43 | } 44 | 45 | public DataSetFile(JSONObject jsonObject) { 46 | super(jsonObject); 47 | Field[] fields = DataSetFile.class.getDeclaredFields(); 48 | super.populateFields(fields,this._keys,this,jsonObject); 49 | } 50 | 51 | public String getFolderName() { 52 | return folderName; 53 | } 54 | 55 | /** 56 | * {@inheritDoc} 57 | */ 58 | @Override 59 | public DataSetFile build(JSONObject obj) { 60 | return new DataSetFile(obj); 61 | } 62 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/catalog/model/DataSetView.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.catalog.model; 18 | 19 | /** 20 | * Created by vedhera on 8/25/2017. 21 | */ 22 | 23 | import com.adobe.platform.ecosystem.examples.constants.SDKConstants; 24 | import org.json.simple.JSONObject; 25 | 26 | import java.lang.reflect.Field; 27 | import java.util.HashMap; 28 | import java.util.List; 29 | import java.util.Map; 30 | 31 | public class DataSetView extends DataSet { 32 | private static final Map _keys; 33 | 34 | private Boolean isLookup; 35 | 36 | static { 37 | _keys = new HashMap() {{ 38 | put(SDKConstants.CATALOG_ISLOOKUP, DataType.BooleanType); 39 | }}; 40 | } 41 | 42 | public DataSetView() { 43 | super(); 44 | } 45 | 46 | public DataSetView(JSONObject jsonObject) { 47 | super(jsonObject); 48 | Field[] fields = DataSetView.class.getDeclaredFields(); 49 | super.populateFields(fields,this._keys,this,jsonObject); 50 | } 51 | 52 | public Boolean getIsLookup() { 53 | return isLookup; 54 | } 55 | 56 | public Boolean getIsDuleEnabled(){ 57 | Boolean _isDuleEnabled = false; 58 | List fields = getFields(false); 59 | 60 | for(int i=0; i< fields.size(); i++) { 61 | _isDuleEnabled = _isDuleEnabled || fields.get(i).getDule().isEnabled(); 62 | } 63 | 64 | for(int i=0; i _keys; 43 | 44 | static { 45 | _keys = new HashMap() {{ 46 | put(SDKConstants.DULE_CONTRACTS, DataType.JsonArrayType); 47 | put(SDKConstants.DULE_IDENTIFIABILITY, DataType.JsonArrayType); 48 | put(SDKConstants.DULE_LOGIN_STATE, DataType.JsonArrayType); 49 | put(SDKConstants.DULE_SPECIAL_TYPES, DataType.JsonArrayType); 50 | put(SDKConstants.DULE_OTHER, DataType.StringType); 51 | }}; 52 | } 53 | 54 | public Dule() { 55 | super(); 56 | } 57 | 58 | public Dule(JSONObject jsonObject) { 59 | super(jsonObject); 60 | Field[] fields = Dule.class.getDeclaredFields(); 61 | super.populateFields(fields,this._keys,this,jsonObject); 62 | } 63 | 64 | public Boolean isEnabled() { 65 | return !contracts.isEmpty() || !identifiability.isEmpty() || !loginState.isEmpty() || 66 | !specialTypes.isEmpty() || !other.isEmpty(); 67 | } 68 | 69 | /** 70 | * {@inheritDoc} 71 | */ 72 | @Override 73 | public Dule build(JSONObject obj) { 74 | return new Dule(obj); 75 | } 76 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/catalog/model/EntityBuilder.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.catalog.model; 18 | 19 | /** 20 | * @author vedhera on 1/23/2018 21 | */ 22 | 23 | import org.json.simple.JSONObject; 24 | 25 | /** 26 | * Base interface for defining 27 | * builder to instantiate objects of 28 | * type E 29 | * @param type on which the constructors will operate. 30 | * @param entity for which builder needs to be invoked. 31 | */ 32 | public interface EntityBuilder { 33 | /** 34 | * Method for defining on how 35 | * to create new objects of 36 | * type E from 37 | * objects of type T 38 | * @param obj 39 | * @return object of type E 40 | */ 41 | E build(T obj); 42 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/catalog/model/FileDescription.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.catalog.model; 18 | 19 | 20 | import static com.adobe.platform.ecosystem.examples.constants.SDKConstants.CATALOG_FILE_DESCRIPTION_DELIMITERS_KEY; 21 | import static com.adobe.platform.ecosystem.examples.constants.SDKConstants.CATALOG_FILE_DESCRIPTION_PERSISTED_KEY; 22 | import static com.adobe.platform.ecosystem.examples.constants.SDKConstants.CATALOG_FORMAT; 23 | 24 | import java.lang.reflect.Field; 25 | import java.util.ArrayList; 26 | import java.util.HashMap; 27 | import java.util.Iterator; 28 | import java.util.List; 29 | import java.util.Map; 30 | import java.util.Set; 31 | import java.util.logging.Level; 32 | import java.util.logging.Logger; 33 | 34 | import org.json.simple.JSONArray; 35 | import org.json.simple.JSONObject; 36 | 37 | import com.adobe.platform.ecosystem.examples.catalog.model.DataSet.FieldsFrom; 38 | import com.adobe.platform.ecosystem.examples.constants.SDKConstants; 39 | import com.adobe.platform.ecosystem.examples.data.FileFormat; 40 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 41 | import com.adobe.platform.ecosystem.examples.util.JsonUtil; 42 | 43 | public class FileDescription { 44 | private boolean persisted; 45 | 46 | private FileFormat format; 47 | 48 | private char delimiter; 49 | 50 | private static final char DEFAULT_DELIMITER = ','; 51 | 52 | public FileDescription(JSONObject fileDescription) throws ConnectorSDKException { 53 | this.persisted = JsonUtil.getBoolean(fileDescription, 54 | CATALOG_FILE_DESCRIPTION_PERSISTED_KEY); 55 | JSONArray delimArray = JsonUtil.getJsonArray(fileDescription, 56 | CATALOG_FILE_DESCRIPTION_DELIMITERS_KEY); 57 | if (delimArray.size() > 0) { 58 | this.delimiter = ((String) delimArray.get(0)).charAt(0); 59 | } else { 60 | this.delimiter = DEFAULT_DELIMITER; 61 | } 62 | String fileFormat = JsonUtil.getString(fileDescription, 63 | CATALOG_FORMAT); 64 | switch (fileFormat) { 65 | case "csv": 66 | this.format = FileFormat.CSV; 67 | break; 68 | case "parquet": 69 | this.format = FileFormat.PARQUET; 70 | break; 71 | case "json": 72 | this.format = FileFormat.JSON; 73 | break; 74 | default: 75 | throw new ConnectorSDKException("File format of type " + fileFormat + " is not supported!."); 76 | } 77 | } 78 | 79 | public boolean isPersisted() { 80 | return persisted; 81 | } 82 | 83 | public FileFormat getFormat() { 84 | return format; 85 | } 86 | 87 | public char getDelimiter() { 88 | return delimiter; 89 | } 90 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/catalog/model/SDKField.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.catalog.model; 18 | 19 | public class SDKField { 20 | 21 | private String name; 22 | private String type; 23 | 24 | public SDKField(String name, String type) { 25 | this.name = name; 26 | this.type = type; 27 | } 28 | 29 | public String getName() { 30 | return name; 31 | } 32 | 33 | public String getType() { 34 | return type; 35 | } 36 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/catalog/model/Schema.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.catalog.model; 18 | 19 | import java.util.ArrayList; 20 | import java.util.HashMap; 21 | import java.util.List; 22 | import java.util.logging.Logger; 23 | 24 | import org.json.simple.JSONObject; 25 | 26 | import com.adobe.platform.ecosystem.examples.catalog.model.DataSet.FieldsFrom; 27 | import com.adobe.platform.ecosystem.examples.constants.SDKConstants; 28 | 29 | /** 30 | * Represent Schema entity in XDM Registry. 31 | */ 32 | public class Schema extends BaseModel { 33 | 34 | private List fieldsList = null; 35 | 36 | private static final Logger logger = Logger.getLogger(Schema.class 37 | .getName()); 38 | 39 | private JSONObject rawSchema = null; 40 | 41 | public Schema() { 42 | super(); 43 | } 44 | 45 | public Schema(JSONObject schemaObj) { 46 | super(schemaObj); 47 | this.rawSchema = schemaObj; 48 | } 49 | 50 | public List getSchemaFields(boolean useFlatNamesForLeafNodes) { 51 | if(fieldsList == null && rawSchema != null) { 52 | HashMap props = (HashMap)rawSchema.get(SDKConstants.PROPERTIES); 53 | if(props == null) { 54 | logger.severe("Improper Schema Object " + rawSchema); 55 | return null; 56 | } 57 | fieldsList = new ArrayList<>(); 58 | props.forEach((key, value) -> { 59 | fieldsList.add(new SchemaField((String)key, (JSONObject) value, useFlatNamesForLeafNodes, DataSet.FieldsFrom.OBSERVABLE_SCHEMA)); 60 | }); 61 | } 62 | return fieldsList; 63 | } 64 | 65 | @Override 66 | public BaseModel build(JSONObject schemaObj) { 67 | return new Schema(schemaObj); 68 | } 69 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/catalog/model/SchemaRef.java: -------------------------------------------------------------------------------- 1 | /* 2 | * ADOBE CONFIDENTIAL 3 | * __________________ 4 | * Copyright 2019 Adobe Systems Incorporated 5 | * All Rights Reserved. 6 | * 7 | * NOTICE: All information contained herein is, and remains 8 | * the property of Adobe Systems Incorporated and its suppliers, 9 | * if any. The intellectual and technical concepts contained 10 | * herein are proprietary to Adobe Systems Incorporated and its 11 | * suppliers and are protected by all applicable intellectual property laws, 12 | * including trade secret and copyright laws. 13 | * 14 | * Dissemination of this information or reproduction of this material 15 | * is strictly forbidden unless prior written permission is obtained 16 | * from Adobe Systems Incorporated. 17 | * 18 | * 19 | */ 20 | package com.adobe.platform.ecosystem.examples.catalog.model; 21 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 22 | import com.adobe.platform.ecosystem.examples.util.JsonUtil; 23 | import org.json.simple.JSONObject; 24 | 25 | import static com.adobe.platform.ecosystem.examples.constants.SDKConstants.CATALOG_SCHEMA_REF_CONTENT_TYPE; 26 | import static com.adobe.platform.ecosystem.examples.constants.SDKConstants.CATALOG_SCHEMA_REF_ID; 27 | 28 | /** 29 | * Pojo for storing schemaRef details in dataset 30 | * 31 | * @author shesriva on 01/14/2019. 32 | * */ 33 | public class SchemaRef { 34 | private String id; 35 | 36 | private String contentType; 37 | 38 | public SchemaRef(JSONObject schemaRef) { 39 | this.id = JsonUtil.getString(schemaRef, CATALOG_SCHEMA_REF_ID); 40 | this.contentType = JsonUtil.getString(schemaRef, CATALOG_SCHEMA_REF_CONTENT_TYPE); 41 | } 42 | 43 | public SchemaRef(String id, String contentType){ 44 | this.id = id; 45 | this.contentType = contentType; 46 | } 47 | 48 | public String getId() { 49 | return id; 50 | } 51 | 52 | public SchemaRef setId(String id) { 53 | this.id = id; 54 | return this; 55 | } 56 | 57 | public String getContentType() { 58 | return contentType; 59 | } 60 | 61 | public void setContentType(String contentType) { 62 | this.contentType = contentType; 63 | } 64 | 65 | @Override 66 | public String toString() { 67 | return "SchemaRef{" + 68 | "id='" + id + '\'' + 69 | ", contentType='" + contentType + '\'' + 70 | '}'; 71 | } 72 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/FileFormat.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data; 18 | 19 | /** 20 | * Created by vedhera on 10/09/2017. 21 | */ 22 | public enum FileFormat { 23 | PARQUET ("parquet"), 24 | CSV ("csv"), 25 | // TODO Intentionally left out, once siphon data tracker start considering JSON as supported format in Pre-flight, will change it to json. 26 | JSON("csv"); 27 | 28 | private final String name; 29 | 30 | FileFormat(String name) { 31 | this.name = name; 32 | } 33 | 34 | public String getExtension() { 35 | return this.name; 36 | } 37 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/Pair.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data; 18 | 19 | /** 20 | * POJO to hold pair of data. 21 | * @param first data type 22 | * @param second data type 23 | * 24 | * @author vedhera on 2/9/2018. 25 | */ 26 | public class Pair { 27 | private F first; //first member of pair 28 | private S second; //second member of pair 29 | 30 | public Pair() {} 31 | 32 | public Pair(F first, S second) { 33 | this.first = first; 34 | this.second = second; 35 | } 36 | 37 | public void setFirst(F first) { 38 | this.first = first; 39 | } 40 | 41 | public void setSecond(S second) { 42 | this.second = second; 43 | } 44 | 45 | public F getFirst() { 46 | return first; 47 | } 48 | 49 | public S getSecond() { 50 | return second; 51 | } 52 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/access/api/DataAccessService.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.access.api; 18 | 19 | /** 20 | * Created by vedhera on 10/09/2017. 21 | */ 22 | 23 | import com.adobe.platform.ecosystem.examples.data.access.model.DataAccessFileEntity; 24 | import com.adobe.platform.ecosystem.examples.data.access.model.DataSetFileProcessingEntity; 25 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 26 | 27 | import java.util.List; 28 | 29 | /** 30 | * Interface to talk to data 31 | * access API's. 32 | */ 33 | public interface DataAccessService { 34 | /** 35 | * Lists all file entities present 36 | * in a datasetfile path(folderName). 37 | * @param imsOrg ImsOrg 38 | * @param accessToken Access Token 39 | * @param dataSetFileId DataSetFile id in Catalog metastore. 40 | * @return List<{@link DataSetFileProcessingEntity}> List of files present under one dataSetFileId. 41 | * @throws ConnectorSDKException 42 | */ 43 | List getDataSetFileEntries(String imsOrg, String accessToken, String dataSetFileId) throws ConnectorSDKException; 44 | 45 | /** 46 | * Lists all dataSetFiles referenced through catalog batch - batchId 47 | * @param imsOrg ImsOrg 48 | * @param accessToken Access Token 49 | * @param batchId batchId id in Catalog metastore. 50 | * * @return List<{@link DataAccessFileEntity}> List of files linked with batchId 51 | * @throws ConnectorSDKException 52 | */ 53 | List getDataSetFilesFromBatchId(String imsOrg, String accessToken, String batchId) throws ConnectorSDKException; 54 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/access/factory/DataAccessServiceFactory.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.access.factory; 18 | 19 | /** 20 | * Created by vedhera on 10/09/2017. 21 | */ 22 | 23 | import com.adobe.platform.ecosystem.examples.data.access.api.DataAccessService; 24 | import com.adobe.platform.ecosystem.examples.data.access.impl.DataAccessServiceImpl; 25 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKUtil; 26 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 27 | import com.adobe.platform.ecosystem.examples.util.ResourceName; 28 | import org.apache.http.client.HttpClient; 29 | 30 | /** 31 | * Factory to provide DataAccessService 32 | * impl. 33 | */ 34 | public class DataAccessServiceFactory { 35 | public static DataAccessService getDataAccessService(HttpClient httpClient) throws ConnectorSDKException{ 36 | String endpoint = ConnectorSDKUtil.getInstance().getEndPoint(ResourceName.DATA_ACCESS); 37 | return new DataAccessServiceImpl(endpoint, httpClient); 38 | } 39 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/access/model/DataAccessFileEntity.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.access.model; 18 | 19 | import com.adobe.platform.ecosystem.examples.constants.SDKConstants; 20 | import org.json.simple.JSONObject; 21 | 22 | /** 23 | * POJO to capture metadata provided 24 | * by DataAccessService while querying for 25 | * datasetfiles through Batch API. 26 | * 27 | * @author vedhera 28 | */ 29 | public class DataAccessFileEntity { 30 | private String dataSetFileId; 31 | 32 | private String dataSetViewId; 33 | 34 | private String dataAccessServiceHref; 35 | 36 | public DataAccessFileEntity(JSONObject jobj) { 37 | if(jobj.containsKey(SDKConstants.DATA_ACCESS_DATASETFILEID_KEY)) { 38 | this.dataSetFileId = (String) jobj.get(SDKConstants.DATA_ACCESS_DATASETFILEID_KEY); 39 | } 40 | 41 | if(jobj.containsKey(SDKConstants.DATA_ACCESS_DATASETVIEWID_KEY)) { 42 | this.dataSetViewId = (String) jobj.get(SDKConstants.DATA_ACCESS_DATASETVIEWID_KEY); 43 | } 44 | 45 | if(jobj.containsKey(SDKConstants.DATA_ACCESS_LINKS_KEY) 46 | && ((JSONObject)jobj.get(SDKConstants.DATA_ACCESS_LINKS_KEY)).containsKey(SDKConstants.DATA_ACCESS_SELF_KEY) 47 | && ((JSONObject)((JSONObject)jobj.get(SDKConstants.DATA_ACCESS_LINKS_KEY)).get(SDKConstants.DATA_ACCESS_SELF_KEY)).containsKey(SDKConstants.DATA_ACCESS_HREF_KEY)) { 48 | this.dataAccessServiceHref = (String) ((JSONObject)((JSONObject)jobj 49 | .get(SDKConstants.DATA_ACCESS_LINKS_KEY)) 50 | .get(SDKConstants.DATA_ACCESS_SELF_KEY)) 51 | .get(SDKConstants.DATA_ACCESS_HREF_KEY); 52 | } 53 | } 54 | 55 | public String getDataSetFileId() { 56 | return dataSetFileId; 57 | } 58 | 59 | public String getDataSetViewId() { 60 | return dataSetViewId; 61 | } 62 | 63 | public String getDataAccessServiceHref() { 64 | return dataAccessServiceHref; 65 | } 66 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/access/model/DataSetFileProcessingEntity.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.access.model; 18 | 19 | /** 20 | * Created by vedhera on 10/09/2017. 21 | */ 22 | 23 | import org.json.simple.JSONObject; 24 | 25 | /** 26 | * Java POJO to encapsulate 27 | * processing links of files per DataSetFile 28 | * entry. 29 | */ 30 | public class DataSetFileProcessingEntity { 31 | private String name; 32 | 33 | private long length; 34 | 35 | private String href; 36 | 37 | public DataSetFileProcessingEntity(JSONObject jobj) { 38 | if(jobj.containsKey("name")) { 39 | this.name = (String) jobj.get("name"); 40 | } 41 | 42 | if(jobj.containsKey("length")) { 43 | this.length = Long.parseLong((String) jobj.get("length")); 44 | } 45 | 46 | if(jobj.containsKey("_links") 47 | && ((JSONObject)jobj.get("_links")).containsKey("self") 48 | && ((JSONObject)((JSONObject)jobj.get("_links")).get("self")).containsKey("href")) { 49 | this.href = (String) ((JSONObject)((JSONObject)jobj.get("_links")).get("self")).get("href"); 50 | } 51 | } 52 | 53 | public String getName() { 54 | return name; 55 | } 56 | 57 | public long getLength() { 58 | return length; 59 | } 60 | 61 | public String getHref() { 62 | return href; 63 | } 64 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/functions/FieldConverterFunction.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.functions; 18 | 19 | import com.adobe.platform.ecosystem.examples.parquet.model.ParquetIODataType; 20 | import com.adobe.platform.ecosystem.examples.catalog.model.DataType; 21 | 22 | import java.util.function.Function; 23 | 24 | /** 25 | * Util class to provide function for 26 | * various conversion operations. 27 | * 28 | * @author vedhera on 2/9/2018 29 | */ 30 | public class FieldConverterFunction { 31 | 32 | public static Function catalogToParquetFieldFunction = ( catalogDataType -> { 33 | if(catalogDataType == DataType.StringType) { 34 | return ParquetIODataType.STRING; 35 | } else if(catalogDataType == DataType.BooleanType) { 36 | return ParquetIODataType.BOOLEAN; 37 | } else if(catalogDataType == DataType.ByteType) { 38 | return ParquetIODataType.BYTE; 39 | } else if(catalogDataType == DataType.ShortType) { 40 | return ParquetIODataType.SHORT; 41 | } else if(catalogDataType == DataType.IntegerType) { 42 | return ParquetIODataType.INTEGER; 43 | } else if(catalogDataType == DataType.LongType) { 44 | return ParquetIODataType.LONG; 45 | } else if(catalogDataType == DataType.FloatType) { 46 | return ParquetIODataType.FLOAT; 47 | } else if(catalogDataType == DataType.DoubleType) { 48 | return ParquetIODataType.DOUBLE; 49 | } else if(catalogDataType == DataType.DateType) { 50 | return ParquetIODataType.DATE; 51 | } else if(catalogDataType == DataType.DateTimeType) { 52 | return ParquetIODataType.TIMESTAMP; 53 | } else if(catalogDataType == DataType.BinaryType) { 54 | return ParquetIODataType.BINARY; 55 | } else if(catalogDataType == DataType.Field_ObjectType) { 56 | return ParquetIODataType.GROUP; 57 | } else if (catalogDataType == DataType.Field_ArrayType) { 58 | return ParquetIODataType.LIST; 59 | } else if (catalogDataType == DataType.Field_MapType) { 60 | return ParquetIODataType.Map; 61 | } 62 | return null; 63 | }); 64 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/ingestion/api/DataIngestionService.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.ingestion.api; 18 | 19 | import com.adobe.platform.ecosystem.examples.data.FileFormat; 20 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 21 | import org.json.simple.JSONObject; 22 | 23 | 24 | /** 25 | * Created by vardgupt on 10/09/2017. 26 | */ 27 | 28 | 29 | /** 30 | * Interface to talk to data ingestion API's 31 | */ 32 | public interface DataIngestionService { 33 | /** 34 | * This method is used to get batchID which is 35 | * required as param for uploading nay file in 36 | * Data Ingestion Service API. 37 | * 38 | * @param imsOrg IMS Org in context. 39 | * @param accessToken accessToken 40 | * @param payload payload for object. 41 | * @return returns the created batch Id. 42 | */ 43 | 44 | String getBatchId(String imsOrg, String accessToken, JSONObject payload) throws ConnectorSDKException; 45 | 46 | /** 47 | * This method is used to upload file via 48 | * Data Ingestion Service API. 49 | * 50 | * @param batchId 51 | * @param dataSetId 52 | * @param imsOrg 53 | * @param accessToken 54 | * @param fileFormat 55 | * @param buffer 56 | * @return 57 | */ 58 | 59 | int writeToBatch(String batchId, String dataSetId, String imsOrg, String accessToken, FileFormat fileFormat, byte[] buffer) throws ConnectorSDKException; 60 | 61 | /** 62 | * This method is used to signal completion 63 | * of batch uploads. 64 | * @param batchId 65 | * @param imsOrg 66 | * @param accessToken 67 | * @return 68 | */ 69 | int signalBatchCompletion(String batchId, String imsOrg, String accessToken) throws ConnectorSDKException; 70 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/ingestion/factory/DataIngestionServiceFactory.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.ingestion.factory; 18 | 19 | /** 20 | * Created by vardgupt on 10/09/2017. 21 | */ 22 | 23 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKUtil; 24 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 25 | import com.adobe.platform.ecosystem.examples.util.ResourceName; 26 | import org.apache.http.client.HttpClient; 27 | 28 | import com.adobe.platform.ecosystem.examples.data.ingestion.api.DataIngestionService; 29 | import com.adobe.platform.ecosystem.examples.data.ingestion.impl.DataIngestionServiceImpl; 30 | 31 | /** 32 | * Factory to provide DataIngestionService 33 | * impl. 34 | */ 35 | public class DataIngestionServiceFactory { 36 | 37 | public static DataIngestionService getDataIngestionService(HttpClient httpClient) throws ConnectorSDKException{ 38 | String endpoint = ConnectorSDKUtil.getInstance().getEndPoint(ResourceName.DATA_INGESTION); 39 | return new DataIngestionServiceImpl(endpoint, httpClient); 40 | } 41 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/read/DataReaderFactory.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.read; 18 | 19 | /** 20 | * Created by vedhera on 8/25/2017. 21 | */ 22 | 23 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 24 | 25 | import java.util.Map; 26 | 27 | /** 28 | * Factory to be implemented to provide 29 | * concrete implementations for data reader. 30 | */ 31 | public interface DataReaderFactory { 32 | /** 33 | * Inteface to provider reader 34 | * which will read data from multiple 35 | * data sources. 36 | * @return Concrete reader implementing {@code Reader} 37 | * @throws ConnectorSDKException 38 | */ 39 | Reader getReader() throws ConnectorSDKException; 40 | 41 | /** 42 | * Inteface to provider reader 43 | * which will read data from multiple 44 | * data sources. 45 | * @param readerAttributes Read attributes. 46 | * @return Concrete reader implementing {@code Reader} 47 | * @throws ConnectorSDKException 48 | */ 49 | Reader getReader(Map readerAttributes) throws ConnectorSDKException; 50 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/read/ReadAttributes.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.read; 18 | /** 19 | * Created by vedhera on 11/17/2017. 20 | */ 21 | 22 | /** 23 | * Java POJO to capture read attributes 24 | */ 25 | public class ReadAttributes { 26 | private String epochStartDate; 27 | 28 | private String duationInMillis; 29 | 30 | public ReadAttributes(String epochStartDate, String duationInMillis) { 31 | this.epochStartDate = epochStartDate; 32 | this.duationInMillis = duationInMillis; 33 | } 34 | 35 | public String getEpochStartDate() { 36 | return epochStartDate; 37 | } 38 | 39 | public String getDuationInMillis() { 40 | return duationInMillis; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/read/Reader.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.read; 18 | /** 19 | * Created by vedhera on 8/25/2017. 20 | */ 21 | 22 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 23 | import org.json.simple.JSONArray; 24 | 25 | /** 26 | * Interface for providing basic 27 | * operations for data source read. 28 | */ 29 | public interface Reader { 30 | /** 31 | * API returning if 32 | * platform has more data 33 | * for given dataSource. 34 | * @return Boolean 35 | */ 36 | Boolean hasMoreData() throws ConnectorSDKException; 37 | 38 | /** 39 | * Reads given number of rows 40 | * from datasource. 41 | * @param rows 42 | * @return 43 | * @throws ConnectorSDKException 44 | */ 45 | JSONArray read(int rows) throws ConnectorSDKException; 46 | 47 | /** 48 | * This API returns the count of 49 | * error rows which were not in 50 | * a valid format 51 | * @return 52 | */ 53 | Integer getErrorRowCount(); 54 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/validation/api/Rule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * ADOBE CONFIDENTIAL 3 | * __________________ 4 | * Copyright 2018 Adobe Systems Incorporated 5 | * All Rights Reserved. 6 | * 7 | * NOTICE: All information contained herein is, and remains 8 | * the property of Adobe Systems Incorporated and its suppliers, 9 | * if any. The intellectual and technical concepts contained 10 | * herein are proprietary to Adobe Systems Incorporated and its 11 | * suppliers and are protected by all applicable intellectual property laws, 12 | * including trade secret and copyright laws. 13 | * 14 | * Dissemination of this information or reproduction of this material 15 | * is strictly forbidden unless prior written permission is obtained 16 | * from Adobe Systems Incorporated. 17 | * 18 | * 19 | */ 20 | package com.adobe.platform.ecosystem.examples.data.validation.api; 21 | 22 | import com.adobe.platform.ecosystem.examples.data.validation.exception.ValidationException; 23 | 24 | /** 25 | * Base interface to define any 26 | * rule. 27 | * 28 | * @param Type parameter 29 | * @author vedhera 11/18/2018. 30 | */ 31 | public interface Rule { 32 | 33 | /** 34 | * Interface to apply the rule 35 | * for a give input {@code input}. 36 | * Throws an exception of type 37 | * {@link ValidationException} to signal 38 | * errors during validation. 39 | * 40 | * @param input input data 41 | * @throws ValidationException 42 | */ 43 | void apply(T input) throws ValidationException; 44 | } 45 | -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/validation/api/ValidationRegistry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * ADOBE CONFIDENTIAL 3 | * __________________ 4 | * Copyright 2018 Adobe Systems Incorporated 5 | * All Rights Reserved. 6 | * 7 | * NOTICE: All information contained herein is, and remains 8 | * the property of Adobe Systems Incorporated and its suppliers, 9 | * if any. The intellectual and technical concepts contained 10 | * herein are proprietary to Adobe Systems Incorporated and its 11 | * suppliers and are protected by all applicable intellectual property laws, 12 | * including trade secret and copyright laws. 13 | * 14 | * Dissemination of this information or reproduction of this material 15 | * is strictly forbidden unless prior written permission is obtained 16 | * from Adobe Systems Incorporated. 17 | * 18 | * 19 | */ 20 | package com.adobe.platform.ecosystem.examples.data.validation.api; 21 | 22 | import com.adobe.platform.ecosystem.examples.data.validation.impl.TraversablePath; 23 | 24 | import java.util.List; 25 | 26 | /** 27 | * Registry to obtain validation rules 28 | * for input schema field path. 29 | * For getting rule for schema {@code firstName} 30 | * below, a {@code List} of (person, firstname) 31 | * should be passed as the input. 32 | * 33 | * 
34 |  *     "person" : {
35 |  *         "type" : "object",
36 |  *         "properties : {
37 |  *             "firstName" : {
38 |  *                 "type" : "string"
39 |  *             }
40 |  *         }
41 |  *     }
42 |  *
43 | * 44 | * @author vedhera on 11/12/2018. 45 | */ 46 | public interface ValidationRegistry { 47 | 48 | /** 49 | * Given a list of traversable schema 50 | * path for a field, returns a {@code List>} 51 | * rules. 52 | * 53 | * @param path root to leaf path. 54 | * @return rules for type {@link java.lang.String} 55 | */ 56 | List> getStringValidationRule(TraversablePath path); 57 | 58 | /** 59 | * Given a list of traversable schema 60 | * path for a field, returns a {@code List>} 61 | * rules. 62 | * 63 | * @param path root to leaf path. 64 | * @return rules for type {@link java.lang.Integer} 65 | */ 66 | List> getIntegerValidationRule(TraversablePath path); 67 | 68 | /** 69 | * Given a list of traversable schema 70 | * path for a field, returns a {@code List>} 71 | * rules. 72 | * 73 | * @param path root to leaf path. 74 | * @return rules for type {@link java.lang.Long} 75 | */ 76 | List> getLongValidationRule(TraversablePath path); 77 | } 78 | -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/validation/api/ValidationRegistryFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * ADOBE CONFIDENTIAL 3 | * __________________ 4 | * Copyright 2018 Adobe Systems Incorporated 5 | * All Rights Reserved. 6 | * 7 | * NOTICE: All information contained herein is, and remains 8 | * the property of Adobe Systems Incorporated and its suppliers, 9 | * if any. The intellectual and technical concepts contained 10 | * herein are proprietary to Adobe Systems Incorporated and its 11 | * suppliers and are protected by all applicable intellectual property laws, 12 | * including trade secret and copyright laws. 13 | * 14 | * Dissemination of this information or reproduction of this material 15 | * is strictly forbidden unless prior written permission is obtained 16 | * from Adobe Systems Incorporated. 17 | * 18 | * 19 | */ 20 | package com.adobe.platform.ecosystem.examples.data.validation.api; 21 | 22 | 23 | import com.adobe.platform.ecosystem.examples.catalog.model.SchemaField; 24 | 25 | /** 26 | * Interface for constructing 27 | * a {@link ValidationRegistry}. 28 | * 29 | * @author vedhera on 11/12/2018. 30 | */ 31 | public interface ValidationRegistryFactory { 32 | 33 | /** 34 | * API to construct the {@link com.adobe.platform.ecosystem.examples.data.validation.impl.rules.SchemaValidationRule} 35 | * from a root of schema field. 36 | * 37 | * @param field The root schema field 38 | * @return The constructed validation registry. 39 | */ 40 | ValidationRegistry get(SchemaField field); 41 | } 42 | -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/validation/exception/ValidationException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * ADOBE CONFIDENTIAL 3 | * __________________ 4 | * Copyright 2018 Adobe Systems Incorporated 5 | * All Rights Reserved. 6 | * 7 | * NOTICE: All information contained herein is, and remains 8 | * the property of Adobe Systems Incorporated and its suppliers, 9 | * if any. The intellectual and technical concepts contained 10 | * herein are proprietary to Adobe Systems Incorporated and its 11 | * suppliers and are protected by all applicable intellectual property laws, 12 | * including trade secret and copyright laws. 13 | * 14 | * Dissemination of this information or reproduction of this material 15 | * is strictly forbidden unless prior written permission is obtained 16 | * from Adobe Systems Incorporated. 17 | * 18 | * 19 | */ 20 | package com.adobe.platform.ecosystem.examples.data.validation.exception; 21 | 22 | /** 23 | * Exception class to signal validation errors occuring 24 | * from {@link com.adobe.platform.ecosystem.examples.data.validation.api.Rule#apply(Object)} 25 | * method. 26 | * 27 | * See {@link ValidationExceptionBuilder} 28 | * for building validation exceptions. 29 | * 30 | * @author vedhera 31 | */ 32 | public class ValidationException extends Exception { 33 | 34 | public ValidationException(String message) { 35 | super(message); 36 | } 37 | 38 | public ValidationException(String message, Throwable cause) { 39 | super(message, cause); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/validation/impl/CatalogValidationRegistryFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * ADOBE CONFIDENTIAL 3 | * __________________ 4 | * Copyright 2018 Adobe Systems Incorporated 5 | * All Rights Reserved. 6 | * 7 | * NOTICE: All information contained herein is, and remains 8 | * the property of Adobe Systems Incorporated and its suppliers, 9 | * if any. The intellectual and technical concepts contained 10 | * herein are proprietary to Adobe Systems Incorporated and its 11 | * suppliers and are protected by all applicable intellectual property laws, 12 | * including trade secret and copyright laws. 13 | * 14 | * Dissemination of this information or reproduction of this material 15 | * is strictly forbidden unless prior written permission is obtained 16 | * from Adobe Systems Incorporated. 17 | * 18 | * 19 | */ 20 | package com.adobe.platform.ecosystem.examples.data.validation.impl; 21 | 22 | import com.adobe.platform.ecosystem.examples.catalog.model.SchemaField; 23 | import com.adobe.platform.ecosystem.examples.data.validation.api.ValidationRegistry; 24 | import com.adobe.platform.ecosystem.examples.data.validation.api.ValidationRegistryFactory; 25 | 26 | /** 27 | * Factory to provide a schema 28 | * based {@link ValidationRegistry}. 29 | * 30 | * @author vedhera on 11/12/2018. 31 | */ 32 | public class CatalogValidationRegistryFactory implements ValidationRegistryFactory { 33 | 34 | /** 35 | * {@inheritDoc} 36 | */ 37 | @Override 38 | public ValidationRegistry get(SchemaField field) { 39 | return new CatalogValidationRegistry(field).build(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/validation/impl/TraversablePath.java: -------------------------------------------------------------------------------- 1 | /* 2 | * ADOBE CONFIDENTIAL 3 | * __________________ 4 | * Copyright 2018 Adobe Systems Incorporated 5 | * All Rights Reserved. 6 | * 7 | * NOTICE: All information contained herein is, and remains 8 | * the property of Adobe Systems Incorporated and its suppliers, 9 | * if any. The intellectual and technical concepts contained 10 | * herein are proprietary to Adobe Systems Incorporated and its 11 | * suppliers and are protected by all applicable intellectual property laws, 12 | * including trade secret and copyright laws. 13 | * 14 | * Dissemination of this information or reproduction of this material 15 | * is strictly forbidden unless prior written permission is obtained 16 | * from Adobe Systems Incorporated. 17 | * 18 | * 19 | */ 20 | package com.adobe.platform.ecosystem.examples.data.validation.impl; 21 | 22 | import org.apache.commons.lang.StringUtils; 23 | 24 | import java.util.ArrayList; 25 | 26 | /** 27 | * A representation of path 28 | * containing names of nodes 29 | * visited during the traversal. 30 | *

31 | * eg: Using below representation 32 | * to store information about all 33 | * possible distinct paths from root 34 | * to leaf(s) while traversing 35 | * XDM schema. 36 | * 37 | * @author vedhera on 11/29/2018. 38 | */ 39 | public class TraversablePath { 40 | private ArrayList path; 41 | 42 | private TraversablePath() { 43 | this.path = new ArrayList<>(); 44 | } 45 | 46 | public static TraversablePath path() { 47 | return new TraversablePath(); 48 | } 49 | 50 | public TraversablePath withNode(String node) { 51 | this.path.add(node); 52 | return this; 53 | } 54 | 55 | public boolean isNullOrEmpty() { 56 | return this.path == null || 57 | this.path.size() == 0; 58 | } 59 | 60 | public String getRootNode() { 61 | return this.path.get(0); 62 | } 63 | 64 | public TraversablePath setRootNode(String root) { 65 | this.path.add(0, root); 66 | return this; 67 | } 68 | 69 | public String buildFieldName() { 70 | return StringUtils.join(this.path, '.'); 71 | } 72 | 73 | @SuppressWarnings("unchecked") 74 | public static TraversablePath clone(TraversablePath path) { 75 | return new TraversablePath() 76 | .setPath((ArrayList) path.getPath().clone()); 77 | } 78 | 79 | ArrayList getPath() { 80 | return path; 81 | } 82 | 83 | TraversablePath setPath(ArrayList path) { 84 | this.path = path; 85 | return this; 86 | } 87 | 88 | @Override 89 | public boolean equals(Object o) { 90 | if (this == o) { 91 | return true; 92 | } 93 | 94 | if (!(o instanceof TraversablePath)) { 95 | return false; 96 | } 97 | 98 | TraversablePath that = (TraversablePath) o; 99 | 100 | return getPath().equals(that.getPath()); 101 | } 102 | 103 | @Override 104 | public int hashCode() { 105 | return getPath().hashCode(); 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/wiring/DataWiringParam.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.wiring; 18 | 19 | import com.adobe.platform.ecosystem.examples.catalog.model.DataSet; 20 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 21 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKUtil; 22 | 23 | /** 24 | * Entity to encapsulate parameters 25 | * required by underlying reader/ 26 | * writer factories. 27 | */ 28 | public class DataWiringParam { 29 | private String imsOrg; 30 | 31 | private DataSet dataSet; 32 | 33 | public DataWiringParam( String imsOrg, DataSet dataSet) { 34 | this.imsOrg = imsOrg; 35 | this.dataSet = dataSet; 36 | } 37 | 38 | public String getImsOrg() { 39 | return imsOrg; 40 | } 41 | 42 | public String getAuthToken() throws ConnectorSDKException{ 43 | return ConnectorSDKUtil.getInstance().getAccessToken(); 44 | } 45 | 46 | public DataSet getDataSet() { 47 | return dataSet; 48 | } 49 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/wiring/Wiring.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.wiring; 18 | 19 | import com.adobe.platform.ecosystem.examples.data.read.DataReaderFactory; 20 | import com.adobe.platform.ecosystem.examples.data.write.DataWriterFactory; 21 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 22 | 23 | /** 24 | * Created by vedhera on 8/25/2017. 25 | */ 26 | 27 | /** 28 | * Data wirings. 29 | */ 30 | public interface Wiring { 31 | public DataReaderFactory dataReaderFactory(); 32 | 33 | public DataWriterFactory dataWriterFactory(); 34 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/write/DataFormatterFactory.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.write; 18 | 19 | import com.adobe.platform.ecosystem.examples.data.FileFormat; 20 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 21 | 22 | /** 23 | * Created by vardgupt on 10/24/2017. 24 | */ 25 | 26 | /** 27 | * Interface for defining formatters 28 | * capable of writing different formats 29 | * to target dataSets. 30 | */ 31 | public interface DataFormatterFactory { 32 | /** 33 | * Interface to return formatter 34 | * based on file type. 35 | * @param fileFormat 36 | * @return Concrete implementation of {@link Formatter} based on 37 | * fileFormat. 38 | * @throws ConnectorSDKException 39 | */ 40 | Formatter getFormatter(FileFormat fileFormat) throws ConnectorSDKException; 41 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/write/DataWriterFactory.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.write; 18 | 19 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 20 | /** 21 | * Created by vedhera on 8/25/2017. 22 | */ 23 | 24 | public interface DataWriterFactory { 25 | 26 | /** This method provides writer Instance which has the capability to write on Platform 27 | * @return 28 | * @throws ConnectorSDKException 29 | */ 30 | Writer getWriter() throws ConnectorSDKException; 31 | 32 | /** This method currently supports to getWriter instance with an ability to use flush strategy where a user can control how many records should be there in single file 33 | * @param writeAttributes 34 | * @return 35 | * @throws ConnectorSDKException 36 | */ 37 | Writer getWriter(WriteAttributes writeAttributes) throws ConnectorSDKException; 38 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/write/FlushHandler.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.write; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | import java.util.stream.Collectors; 22 | import java.util.stream.Stream; 23 | 24 | 25 | 26 | /** 27 | * @author vardgupt 28 | 29 | * FlushHandler class contains attributes for flush strategy like 30 | * size of a record, maximum no. of records which should be put in buffer before flush is called. 31 | */ 32 | public class FlushHandler { 33 | 34 | private long batchSize; 35 | private final long fileMaxSize = 256*1024*1024l; 36 | private long sizeOfRecord; 37 | private long rowsProcessed; 38 | private List dataTable; 39 | 40 | public FlushHandler(long sizeOfRecord){ 41 | //Keep size as 3 times the first record 42 | this.sizeOfRecord = sizeOfRecord*3; 43 | this.batchSize = fileMaxSize/this.sizeOfRecord; 44 | } 45 | 46 | public long getBatchSize() { 47 | return batchSize; 48 | } 49 | 50 | public long getSizeOfRecord() { 51 | return sizeOfRecord; 52 | } 53 | 54 | public long getRowsProcessed() { 55 | return rowsProcessed; 56 | } 57 | 58 | public void incrementProcessedRows(long rowsProcessed) { 59 | this.rowsProcessed+= rowsProcessed; 60 | } 61 | 62 | public void addRecords(List recordsTobeAdded) { 63 | if(this.dataTable==null){ 64 | this.dataTable = new ArrayList<>(); 65 | } 66 | dataTable.addAll(recordsTobeAdded); 67 | } 68 | 69 | public List getDataTable() { 70 | return this.dataTable; 71 | } 72 | 73 | public void reset(){ 74 | this.rowsProcessed = 0; 75 | this.dataTable.clear(); 76 | } 77 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/write/Formatter.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.write; 18 | 19 | import java.util.List; 20 | 21 | import com.adobe.platform.ecosystem.examples.catalog.model.SDKField; 22 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 23 | import org.json.simple.JSONObject; 24 | 25 | /** 26 | * Created by vardgupt on 10/24/2017. 27 | */ 28 | 29 | 30 | public interface Formatter { 31 | public static final String delimiter = "_"; 32 | 33 | /** 34 | * Returns buffer for input data 35 | * which could be written to downstream 36 | * solutions. 37 | * @param sdkFields 38 | * @param dataTable 39 | * @return 40 | * @throws ConnectorSDKException 41 | */ 42 | byte[] getBuffer(List sdkFields, List> dataTable) throws ConnectorSDKException; 43 | 44 | /** 45 | * Returns buffer for input data 46 | * which could be written to downstream 47 | * solutions. 48 | * @param dataTable rows of input data 49 | * @return buffer to be written. 50 | * @throws ConnectorSDKException 51 | */ 52 | byte[] getBuffer(List dataTable) throws ConnectorSDKException; 53 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/write/PlatformDataWriterFactory.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.write; 18 | 19 | import com.adobe.platform.ecosystem.examples.data.FileFormat; 20 | import org.apache.http.client.HttpClient; 21 | import com.adobe.platform.ecosystem.examples.catalog.api.CatalogService; 22 | import com.adobe.platform.ecosystem.examples.data.ingestion.api.DataIngestionService; 23 | import com.adobe.platform.ecosystem.examples.data.wiring.DataWiringParam; 24 | import com.adobe.platform.ecosystem.examples.data.write.writer.DataIngestionAPIWriter; 25 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 26 | 27 | /** 28 | * Created by vedhera on 8/25/2017. 29 | */ 30 | 31 | public class PlatformDataWriterFactory implements DataWriterFactory { 32 | 33 | private final DataWiringParam param; 34 | private final CatalogService cs; 35 | private final DataIngestionService dis; 36 | private final HttpClient httpClient; 37 | private final PlatformDataFormatterFactory formatterFactory; 38 | 39 | public PlatformDataWriterFactory(DataWiringParam param, 40 | CatalogService cs, 41 | DataIngestionService dis, 42 | HttpClient httpClient, 43 | PlatformDataFormatterFactory formatterFactory) { 44 | this.param = param; 45 | this.cs = cs; 46 | this.dis = dis; 47 | this.httpClient = httpClient; 48 | this.formatterFactory = formatterFactory; 49 | } 50 | 51 | /** 52 | * {@inheritDoc} 53 | */ 54 | @Override 55 | public Writer getWriter() throws ConnectorSDKException { 56 | return this.getWriter(new WriteAttributes.WriteAttributesBuilder().build()); 57 | } 58 | 59 | @Override 60 | public Writer getWriter(WriteAttributes writeAttributes) throws ConnectorSDKException { 61 | FileFormat format = param.getDataSet().getFileDescription().getFormat(); 62 | formatterFactory.setWriteAttributes(writeAttributes); 63 | Formatter formatter = formatterFactory.getFormatter(format); 64 | return new DataIngestionAPIWriter(dis, param, format, formatter, writeAttributes, cs); 65 | } 66 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/write/WriteAttributes.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.write; 18 | 19 | /** 20 | * @author vardgupt 21 | * 22 | */ 23 | public class WriteAttributes { 24 | 25 | private Boolean isFlushStrategyRequired; 26 | 27 | private long sizeOfRecord; 28 | 29 | private FlushHandler flushHandler; 30 | 31 | private boolean isEOF; 32 | 33 | private boolean fullSchemaRequired; 34 | 35 | private WriteAttributes(WriteAttributesBuilder writeAttributesBuilder) { 36 | this.setFlushStrategyRequired(writeAttributesBuilder.getIsFlushStrategyRequired()); 37 | this.setSizeOfRecord(writeAttributesBuilder.getSizeOfRecord()); 38 | this.setFullSchemaRequired(writeAttributesBuilder.isFullSchemaRequired()); 39 | if(sizeOfRecord>0 && isFlushStrategyRequired){ 40 | setFlushHandler(new FlushHandler(sizeOfRecord)); 41 | } 42 | } 43 | public Boolean isFlushStrategyRequired() { 44 | return isFlushStrategyRequired; 45 | } 46 | public void setFlushStrategyRequired(Boolean isFlushStrategyRequired) { 47 | this.isFlushStrategyRequired = isFlushStrategyRequired; 48 | } 49 | 50 | public void setSizeOfRecord(long sizeOfRecord) { 51 | this.sizeOfRecord = sizeOfRecord; 52 | } 53 | 54 | public FlushHandler getFlushHandler() { 55 | return flushHandler; 56 | } 57 | public void setFlushHandler(FlushHandler flushHandler) { 58 | this.flushHandler = flushHandler; 59 | } 60 | 61 | public boolean isEOF() { 62 | return isEOF; 63 | } 64 | public void setEOF(boolean isEOF) { 65 | this.isEOF = isEOF; 66 | } 67 | 68 | public boolean isFullSchemaRequired() { 69 | return fullSchemaRequired; 70 | } 71 | 72 | public WriteAttributes setFullSchemaRequired(boolean fullSchemaRequired) { 73 | this.fullSchemaRequired = fullSchemaRequired; 74 | return this; 75 | } 76 | 77 | public static class WriteAttributesBuilder{ 78 | 79 | private Boolean isFlushStrategyRequired = false; 80 | 81 | private long sizeOfRecord; 82 | 83 | //Setting default value to false. 84 | private boolean isFullSchemaRequired = false; 85 | 86 | public Boolean getIsFlushStrategyRequired() { 87 | return isFlushStrategyRequired; 88 | } 89 | 90 | public WriteAttributesBuilder withFlushStrategy(Boolean isFlushStrategyRequired) { 91 | this.isFlushStrategyRequired = isFlushStrategyRequired; 92 | return this; 93 | } 94 | 95 | public long getSizeOfRecord() { 96 | return sizeOfRecord; 97 | } 98 | 99 | public WriteAttributesBuilder withSizeOfRecord(long sizeOfRecord) { 100 | this.sizeOfRecord = sizeOfRecord; 101 | return this; 102 | } 103 | 104 | public boolean isFullSchemaRequired() { 105 | return isFullSchemaRequired; 106 | } 107 | 108 | public WriteAttributes build(){ 109 | return new WriteAttributes(this); 110 | } 111 | } 112 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/write/Writer.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.write; 18 | 19 | import java.util.List; 20 | 21 | import com.adobe.platform.ecosystem.examples.catalog.model.SDKField; 22 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 23 | /** 24 | * Created by vardgupt on 10/17/2017. 25 | */ 26 | 27 | public interface Writer { 28 | String delimiter = "_"; 29 | 30 | /** 31 | * 32 | * @param sdkFields 33 | * @param dataTable 34 | * @return 35 | * @throws ConnectorSDKException 36 | */ 37 | int write(List sdkFields, List> dataTable) throws ConnectorSDKException; 38 | 39 | /** 40 | * 41 | * @param isSuccess 42 | * @return 43 | * @throws ConnectorSDKException 44 | */ 45 | int markBatchCompletion(Boolean isSuccess) throws ConnectorSDKException; 46 | 47 | /** 48 | * 49 | * @param isSuccess 50 | * @return 51 | * @throws ConnectorSDKException 52 | */ 53 | int markBatchCompletion(Boolean isSuccess, Boolean shouldPollForBatchStatus) throws ConnectorSDKException; 54 | 55 | /** 56 | * Interface for exposing write capabilities for 57 | * procedural connector. 58 | * @param dataTable 59 | * @return 60 | * @throws ConnectorSDKException 61 | */ 62 | int write(List dataTable) throws ConnectorSDKException; 63 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/write/converter/PipelineToJSONConverter.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.write.converter; 18 | 19 | import org.json.simple.JSONObject; 20 | import org.json.simple.parser.JSONParser; 21 | 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | import java.util.logging.Level; 25 | import java.util.logging.Logger; 26 | 27 | /** 28 | * Responsible for transforming data 29 | * coming from pipeline to the standardised 30 | * SDKModel. 31 | * Going forward this can be abstracted through 32 | * a factory which can work for several pipeline 33 | * input types. Currently it supports a Json object as 34 | * string and converts it to JSON object.. 35 | * 36 | * @author vedhera 37 | */ 38 | public class PipelineToJSONConverter { 39 | private static final JSONParser parser = new org.json.simple.parser.JSONParser(); 40 | 41 | private static final Logger logger = Logger.getLogger(PipelineToJSONConverter.class.getName()); 42 | 43 | public static List getFields(List pipelineData, String rootId) { 44 | List records = new ArrayList<>(); 45 | if(pipelineData == null) { 46 | logger.log(Level.INFO,"Input data is empty for transformation."); 47 | return records; 48 | } 49 | for (Object data : pipelineData) { 50 | try { 51 | String stringData = (String) data; 52 | records.add((JSONObject) ((JSONObject) parser.parse(stringData)).get(rootId)); 53 | } catch (Exception ex) { 54 | logger.severe("Error while parsing data: " + data + " to JSON object."); 55 | } 56 | } 57 | return records; 58 | } 59 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/write/field/converter/parquet/ParquetFieldConverter.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.write.field.converter.parquet; 18 | 19 | import com.adobe.platform.ecosystem.examples.parquet.model.ParquetIOField; 20 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 21 | 22 | import java.util.List; 23 | 24 | /** 25 | * Interface to convert a given data type to 26 | * {@link ParquetIOField} to be able to 27 | * communicate correctly with parquet sdk. 28 | * @param 29 | * @author vedhera 2/8/2018. 30 | */ 31 | public interface ParquetFieldConverter { 32 | /** 33 | * API to convert a given reference 34 | * object to List of {@link ParquetIOField}. 35 | * 36 | * @param data Data from which {@link ParquetIOField} will be constructed. 37 | * @return List of parquet-IO fields. 38 | */ 39 | List convert(T data) throws ConnectorSDKException; 40 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/write/field/converter/parquet/catalog/CachedCatalogSchemaParquetFieldConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2020 Adobe. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.adobe.platform.ecosystem.examples.data.write.field.converter.parquet.catalog; 17 | 18 | 19 | import com.adobe.platform.ecosystem.examples.catalog.model.SchemaField; 20 | import com.adobe.platform.ecosystem.examples.data.write.field.converter.parquet.ParquetFieldConverter; 21 | import com.adobe.platform.ecosystem.examples.parquet.model.ParquetIOField; 22 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 23 | 24 | import java.util.List; 25 | import java.util.Optional; 26 | 27 | /** 28 | * Cached implementation to convert 29 | * SchemaFields to ParquetIOFields. 30 | * 31 | * @author vedhera. 32 | */ 33 | public class CachedCatalogSchemaParquetFieldConverter implements ParquetFieldConverter> { 34 | private final ParquetFieldConverter> converter; 35 | private Optional> parquetFields = Optional.empty(); 36 | 37 | 38 | public CachedCatalogSchemaParquetFieldConverter(ParquetFieldConverter> converter) { 39 | this.converter = converter; 40 | } 41 | 42 | @Override 43 | public List convert(List data) throws ConnectorSDKException { 44 | if (!parquetFields.isPresent()) { 45 | parquetFields = Optional.of(converter.convert(data)); 46 | } 47 | return parquetFields.get(); 48 | } 49 | 50 | CachedCatalogSchemaParquetFieldConverter setParquetIOFields(List fields) { 51 | this.parquetFields = Optional.of(fields); 52 | return this; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/data/write/writer/extractor/Extractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017-2018 Adobe. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.adobe.platform.ecosystem.examples.data.write.writer.extractor; 17 | 18 | import org.json.simple.JSONObject; 19 | 20 | import java.util.List; 21 | 22 | /** 23 | * Interface to identify if an extract 24 | * can be made from a given {@link JSONObject} 25 | * and be split into multiple JSONObject. 26 | * 27 | * Any client for this interface should invoke 28 | * following API in order: 29 | * 1. {@link Extractor#isExtractRequired(Object)} 30 | * 2. {@link Extractor#isValid(Object)} 31 | * 3. {@link Extractor#extract(Object)} 32 | * 33 | * @param 34 | * @author vedhera 07/23/2018. 35 | */ 36 | public interface Extractor { 37 | 38 | /** 39 | * API to check if at all multiple 40 | * objects of type T 41 | * can be extracted from input. 42 | * 43 | * @param input 44 | * @return boolean value. 45 | */ 46 | boolean isExtractRequired(T input); 47 | 48 | /** 49 | * API to validate if input 50 | * can be used to extract multiple 51 | * values. 52 | * 53 | * @param input 54 | * @return boolean value 55 | */ 56 | boolean isValid(T input); 57 | 58 | /** 59 | * API to extract values and return 60 | * {@link List} of objects of type 61 | * T. 62 | * 63 | * @param input 64 | * @return List of objects of same 65 | * type T 66 | */ 67 | List extract(T input); 68 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/filter/Filter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Adobe. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.adobe.platform.ecosystem.examples.filter; 17 | 18 | import java.util.List; 19 | 20 | /** 21 | * Interface defined for filtering 22 | * types of type {@code T} 23 | * @param Input type 24 | * 25 | * @author vedhera on 7/27/2018. 26 | */ 27 | public interface Filter{ 28 | 29 | /** 30 | * API to check if the 31 | * filter can be applied 32 | * on an input based on 33 | * {@code id}. 34 | * 35 | * @param id 36 | * @return {@code true} if filter 37 | * can be applied. 38 | */ 39 | boolean canApply(String id); 40 | 41 | /** 42 | * API to perform filter operation. 43 | * 44 | * @param input 45 | * @return filtered output. 46 | */ 47 | List filter(List input); 48 | } 49 | -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/filter/FilterFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Adobe. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.adobe.platform.ecosystem.examples.filter; 17 | 18 | 19 | import com.adobe.platform.ecosystem.examples.catalog.model.SchemaField; 20 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKUtil; 21 | 22 | import java.util.ArrayList; 23 | import java.util.Arrays; 24 | import java.util.List; 25 | import java.util.stream.Collectors; 26 | 27 | import static com.adobe.platform.ecosystem.examples.constants.SDKConstants.CATALOG_FIELD_FILTER_DELIMITER; 28 | import static com.adobe.platform.ecosystem.examples.constants.SDKConstants.CATALOG_FIELD_FILTER_PROPERTY; 29 | 30 | 31 | /** 32 | * Factory for providing different 33 | * filters. 34 | * 35 | * @author vedhera on 7/27/2018. 36 | */ 37 | public class FilterFactory { 38 | 39 | /** 40 | * Provider for filter for 41 | * Catalog schema. 42 | * 43 | * @return {@link Filter} for {@link com.adobe.platform.ecosystem.examples.catalog.model.SchemaField} 44 | */ 45 | public static List> provideSchemaFieldFilter() { 46 | final String schemaFilter = ConnectorSDKUtil.getSystemProperty(CATALOG_FIELD_FILTER_PROPERTY); 47 | 48 | if (schemaFilter == null || schemaFilter.isEmpty()) { 49 | return new ArrayList<>(); 50 | } 51 | return Arrays.stream(schemaFilter.split(CATALOG_FIELD_FILTER_DELIMITER)) 52 | .map(filterId -> { 53 | // Each filterId will be of type 'context/profile.identities.primary' 54 | final String id = filterId.split("[.]")[0]; 55 | final String filterPath = filterId.replaceFirst(id + ".", ""); 56 | return new SchemaFieldFilter(id, filterPath); 57 | }).collect(Collectors.toList()); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/filter/SchemaFieldFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 Adobe. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.adobe.platform.ecosystem.examples.filter; 17 | 18 | 19 | import com.adobe.platform.ecosystem.examples.catalog.model.SchemaField; 20 | 21 | import java.util.Arrays; 22 | import java.util.Iterator; 23 | import java.util.LinkedList; 24 | import java.util.List; 25 | 26 | /** 27 | * Concrete implementation for 28 | * providing filter for Catalog's 29 | * {@link SchemaField}. 30 | * 31 | * @author vedhera 7/27/2018. 32 | */ 33 | public class SchemaFieldFilter implements Filter { 34 | 35 | private final String filterId; 36 | private final String branchToFilter; 37 | 38 | public SchemaFieldFilter(String filterId, String branchToFilter) { 39 | this.filterId = filterId; 40 | this.branchToFilter = branchToFilter; 41 | } 42 | 43 | /** 44 | * {@inheritDoc} 45 | */ 46 | @Override 47 | public boolean canApply(String id) { 48 | return filterId.equalsIgnoreCase(id); 49 | } 50 | 51 | /** 52 | * {@inheritDoc} 53 | */ 54 | @Override 55 | public List filter(List input) { 56 | LinkedList linkedList = new LinkedList<>(); 57 | Arrays.stream(branchToFilter.split("[.]")) 58 | .forEach( token -> linkedList.addLast(token)); 59 | 60 | return filterRecurse(input, linkedList.iterator()); 61 | } 62 | 63 | private List filterRecurse(List input, Iterator iterator) { 64 | final String pathToken = iterator.next(); 65 | SchemaField nodeToDelete = null; 66 | for(SchemaField schemaField : input) { 67 | if (schemaField.getName().equals(pathToken)) { 68 | if (iterator.hasNext()) { 69 | filterRecurse( 70 | schemaField.getSubFields(), 71 | iterator 72 | ); 73 | } else { // Last node. 74 | nodeToDelete = schemaField; 75 | } 76 | 77 | } else { 78 | // do nothing. 79 | } 80 | } 81 | input.remove(nodeToDelete); 82 | return input; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/schemaregistry/SchemaRegistryFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * ADOBE CONFIDENTIAL 3 | * __________________ 4 | * Copyright 2019 Adobe Systems Incorporated 5 | * All Rights Reserved. 6 | * 7 | * NOTICE: All information contained herein is, and remains 8 | * the property of Adobe Systems Incorporated and its suppliers, 9 | * if any. The intellectual and technical concepts contained 10 | * herein are proprietary to Adobe Systems Incorporated and its 11 | * suppliers and are protected by all applicable intellectual property laws, 12 | * including trade secret and copyright laws. 13 | * 14 | * Dissemination of this information or reproduction of this material 15 | * is strictly forbidden unless prior written permission is obtained 16 | * from Adobe Systems Incorporated. 17 | * 18 | * 19 | */ 20 | package com.adobe.platform.ecosystem.examples.schemaregistry; 21 | 22 | import com.adobe.platform.ecosystem.examples.schemaregistry.api.SchemaRegistryService; 23 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 24 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKUtil; 25 | import com.adobe.platform.ecosystem.examples.util.ResourceName; 26 | import org.apache.http.client.HttpClient; 27 | 28 | /** 29 | * Factory for obtaining SchemaRegistryService 30 | * 31 | * @author shesriva on 23/01/2019. 32 | * */ 33 | public class SchemaRegistryFactory { 34 | public static SchemaRegistryService getSchemaRegistryService(HttpClient httpClient) throws ConnectorSDKException { 35 | ConnectorSDKUtil adobeResourceUtil = ConnectorSDKUtil.getInstance(); 36 | String schemaRegistryEndPoint = adobeResourceUtil.getEndPoint(ResourceName.SCHEMA_REGISTRY); 37 | return new SchemaRegistryServiceImpl(schemaRegistryEndPoint, httpClient); 38 | } 39 | 40 | public static SchemaRegistryService getSchemaRegistryService() throws ConnectorSDKException{ 41 | return getSchemaRegistryService(null); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/schemaregistry/api/SchemaRegistryService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * ADOBE CONFIDENTIAL 3 | * __________________ 4 | * Copyright 2019 Adobe Systems Incorporated 5 | * All Rights Reserved. 6 | * 7 | * NOTICE: All information contained herein is, and remains 8 | * the property of Adobe Systems Incorporated and its suppliers, 9 | * if any. The intellectual and technical concepts contained 10 | * herein are proprietary to Adobe Systems Incorporated and its 11 | * suppliers and are protected by all applicable intellectual property laws, 12 | * including trade secret and copyright laws. 13 | * 14 | * Dissemination of this information or reproduction of this material 15 | * is strictly forbidden unless prior written permission is obtained 16 | * from Adobe Systems Incorporated. 17 | * 18 | * 19 | */ 20 | package com.adobe.platform.ecosystem.examples.schemaregistry.api; 21 | 22 | import com.adobe.platform.ecosystem.examples.catalog.model.SchemaField; 23 | import com.adobe.platform.ecosystem.examples.catalog.model.SchemaRef; 24 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 25 | 26 | import java.util.List; 27 | 28 | /** 29 | * XDMRegistryService interface. 30 | * 31 | * @author shesriva on 23/01/2019. 32 | * */ 33 | public interface SchemaRegistryService { 34 | 35 | /** 36 | * Fetch Schema from XDM Registry 37 | * 38 | * @param imsOrg imsOrg in context. 39 | * @param authToken authToken 40 | * @param schemaRef schemaRef 41 | * @param useFlatNamesForLeafNodes flag for flattening names at leaf nodes. 42 | * @return List of SchemaFields. 43 | * @throws ConnectorSDKException 44 | */ 45 | List getSchemaFields(String imsOrg, String authToken, SchemaRef schemaRef, boolean useFlatNamesForLeafNodes) throws ConnectorSDKException; 46 | } 47 | -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/util/ConnectorSDKException.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.util; 18 | 19 | /** 20 | * Created by nidhi on 15/9/17. 21 | */ 22 | public class ConnectorSDKException extends Exception { 23 | 24 | public ConnectorSDKException(String message, Throwable cause){ 25 | super(message, cause); 26 | } 27 | 28 | public ConnectorSDKException(String message){ 29 | super(message); 30 | } 31 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/util/JsonUtil.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.util; 18 | 19 | /** 20 | * Created by vedhera on 8/25/2017. 21 | */ 22 | 23 | import org.json.simple.JSONArray; 24 | import org.json.simple.JSONObject; 25 | 26 | /** 27 | * Util class to extract keys 28 | * from different json objects. 29 | */ 30 | public class JsonUtil { 31 | 32 | public static String getString(JSONObject jsonObject, String key) { 33 | if(jsonObject.containsKey(key)) { 34 | return (String) jsonObject.get(key); 35 | } else { 36 | return ""; 37 | } 38 | } 39 | 40 | public static Boolean getBoolean(JSONObject jsonObject, String key) { 41 | if(jsonObject.containsKey(key)) { 42 | return (Boolean) jsonObject.get(key); 43 | } else { 44 | return false; 45 | } 46 | } 47 | 48 | public static long getLong(JSONObject jsonObject, String key) { 49 | if(jsonObject.containsKey(key)) { 50 | return (long) jsonObject.get(key); 51 | } else { 52 | return 0; 53 | } 54 | } 55 | 56 | public static int getInteger(JSONObject jsonObject, String key) { 57 | if(jsonObject.containsKey(key)) { 58 | return (int) jsonObject.get(key); 59 | } else { 60 | return -1; 61 | } 62 | } 63 | 64 | public static JSONArray getJsonArray(JSONObject jsonObject, String key) { 65 | if(jsonObject.containsKey(key)) { 66 | return (JSONArray) jsonObject.get(key); 67 | } else { 68 | return new JSONArray(); 69 | } 70 | } 71 | 72 | public static JSONObject getJsonObject(JSONObject jsonObject, String key) { 73 | if(jsonObject.containsKey(key)) { 74 | return (JSONObject) jsonObject.get(key); 75 | } else { 76 | return new JSONObject(); 77 | } 78 | } 79 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/util/ResourceName.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.util; 18 | 19 | /** 20 | * Created by nidhi on 5/10/17. 21 | */ 22 | public enum ResourceName { 23 | ADOBE_IO("adobeio", "adobeioEndPoint"), 24 | CATALOG("catalog", "catalogEndPoint"), 25 | SCHEMA_REGISTRY("schemaRegistry", "schemaRegistryEndPoint"), 26 | IMS("ims", "imsEndPoint"), 27 | LOOKUP("lookup", "lookupEndPoint"), 28 | DATA_ACCESS("dataAccess", "dataAccessEndpoint"), 29 | DATA_INGESTION("dataIngestion", "dataIngestionEndpoint"); 30 | 31 | private String resource; 32 | private String endPointKey; 33 | 34 | private ResourceName(String resource, String endPointKey){ 35 | this.resource = resource; 36 | this.endPointKey = endPointKey; 37 | } 38 | 39 | public String getResourceName(){ 40 | return resource; 41 | } 42 | 43 | public String getEndPointKey(){ 44 | return endPointKey; 45 | } 46 | } -------------------------------------------------------------------------------- /examples/src/main/java/com/adobe/platform/ecosystem/examples/util/ResourceReader.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.util; 18 | 19 | import java.io.IOException; 20 | import java.io.InputStream; 21 | import java.util.Properties; 22 | import java.util.logging.Logger; 23 | 24 | /** 25 | * Created by vedhera on 8/25/2017. 26 | */ 27 | 28 | /** 29 | * Utility to read properties. 30 | */ 31 | public class ResourceReader { 32 | private final static String PROPERTY_FILE = "config.properties"; 33 | 34 | public static final Logger logger = Logger.getLogger(ResourceReader.class.getName()); 35 | 36 | public String getProperty(String key) { 37 | return readPropertiesFromResource().getProperty(key); 38 | } 39 | 40 | public Properties readPropertiesFromResource() { 41 | return readConfigPropertiesFromResource(null); 42 | } 43 | 44 | public Properties readConfigPropertiesFromResource(String env) { 45 | String propFileName = ResourceReader.PROPERTY_FILE; 46 | propFileName = (env == null ? "dev/" + propFileName : env + "/" + propFileName); 47 | propFileName = "/" + propFileName; 48 | return loadProperties(propFileName); 49 | } 50 | 51 | public Properties readDefaultConfigFromResource() { 52 | return loadProperties("/" + ResourceReader.PROPERTY_FILE); 53 | } 54 | 55 | private Properties loadProperties(String propertyFilePath) { 56 | Properties prop = new Properties(); 57 | InputStream inputStream; 58 | inputStream = getClass().getResourceAsStream(propertyFilePath); 59 | try { 60 | prop.load(inputStream); 61 | } catch (IOException e) { 62 | logger.severe("Error while readPropertiesFromResource : " + e.getMessage()); 63 | } 64 | return prop; 65 | } 66 | 67 | public Properties getSystemProperties() { 68 | return System.getProperties(); 69 | } 70 | } -------------------------------------------------------------------------------- /examples/src/main/resources/config.properties: -------------------------------------------------------------------------------- 1 | lookupEndPoint=http://lookupservice-dev-5647e8dd25--lookupservice.gw.ethos01-dev-va7.ethos.adobe.net/lookup/dataSets//keys?imsOrg=table1&start=-1&limit=-1&keysOnly=false 2 | imsEndPoint=https://ims-na1-stg1.adobelogin.com 3 | catalogEndPoint=https://catalog-dev-3ab32a5a32--catalog.gw.ethos01-dev-va7.ethos.adobe.net 4 | dataAccessEndpoint=http://dataaccess-dev-1257d2bc84--data-access-api.gw.ethos01-dev-va7.ethos.adobe.net 5 | defaultDataSetFileReadCount=10 6 | defaultJWTTokenExpiryInDays=259200 7 | connectionPoolLimit=50 -------------------------------------------------------------------------------- /examples/src/main/resources/dev/config.properties: -------------------------------------------------------------------------------- 1 | adobeioEndPoint=https://platform-int.adobe.io 2 | lookupEndPoint=https://lookupservice-dev-va7.dev.cloud.adobe.io/lookup/dataSets//keys?imsOrg=table1&start=-1&limit=-1&keysOnly=false 3 | imsEndPoint=https://ims-na1-stg1.adobelogin.com 4 | catalogEndPoint=/data/foundation/catalog 5 | schemaRegistryEndPoint=/data/foundation/schemaregistry/ 6 | dataAccessEndpoint=/data/foundation/export 7 | dataIngestionEndpoint=/data/foundation/import 8 | defaultDataSetFileReadCount=10 9 | connectionPoolLimit=50 -------------------------------------------------------------------------------- /examples/src/main/resources/log4j2.properties: -------------------------------------------------------------------------------- 1 | 2 | appender.out.type = Console 3 | appender.out.name = out 4 | appender.out.layout.type = PatternLayout 5 | appender.out.layout.pattern = [%30.30t] %-30.30c{1} %-5p %m%n 6 | rootLogger.level = INFO 7 | rootLogger.appenderRef.out.ref = out 8 | -------------------------------------------------------------------------------- /examples/src/main/resources/pre-prod/config.properties: -------------------------------------------------------------------------------- 1 | adobeioEndPoint=https://platform-int.adobe.io 2 | lookupEndPoint=https://lookupservice-dev-va7.dev.cloud.adobe.io/lookup/dataSets//keys?imsOrg=table1&start=-1&limit=-1&keysOnly=false 3 | imsEndPoint=https://ims-na1-stg1.adobelogin.com 4 | catalogEndPoint=/data/foundation/catalog 5 | schemaRegistryEndPoint=/data/foundation/schemaregistry/ 6 | dataAccessEndpoint=/data/foundation/export 7 | dataIngestionEndpoint=/data/foundation/import 8 | defaultDataSetFileReadCount=10 9 | connectionPoolLimit=50 10 | -------------------------------------------------------------------------------- /examples/src/main/resources/prod/config.properties: -------------------------------------------------------------------------------- 1 | adobeioEndPoint=https://platform.adobe.io 2 | imsEndPoint=https://ims-na1.adobelogin.com 3 | catalogEndPoint=/data/foundation/catalog 4 | schemaRegistryEndPoint=/data/foundation/schemaregistry/ 5 | dataAccessEndpoint=/data/foundation/export 6 | dataIngestionEndpoint=/data/foundation/import 7 | defaultDataSetFileReadCount=10 8 | connectionPoolLimit=50 9 | -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/catalog/impl/CatalogFactoryTest.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.catalog.impl; 18 | 19 | import com.adobe.platform.ecosystem.examples.catalog.api.CatalogService; 20 | import com.adobe.platform.ecosystem.ut.BaseTest; 21 | import org.junit.Test; 22 | 23 | import static org.junit.Assert.assertTrue; 24 | 25 | public class CatalogFactoryTest extends BaseTest{ 26 | 27 | @Test 28 | public void testCatalogFactory() throws Exception{ 29 | setUp(); 30 | assertTrue(CatalogFactory.getCatalogService()!=null); 31 | CatalogService cat = CatalogFactory.getCatalogService(null); 32 | assertTrue(cat != null); 33 | } 34 | 35 | } -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/catalog/model/ConnectionTest.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.catalog.model; 18 | 19 | import com.adobe.platform.ecosystem.ut.BaseTest; 20 | import org.json.simple.parser.ParseException; 21 | import org.junit.Before; 22 | import org.junit.Test; 23 | 24 | import static org.junit.Assert.assertTrue; 25 | 26 | /** 27 | * created by prigarg on 10/10/17 28 | */ 29 | public class ConnectionTest extends BaseTest{ 30 | 31 | Connection connection = null; 32 | 33 | @Before 34 | public void setupConnection() throws ParseException { 35 | connection = getConnectionFromStringWithConnectionId(connectionSample, "conId"); 36 | } 37 | 38 | @Test 39 | public void testGetIsDuleEnabled() throws ParseException { 40 | assertTrue(connection.getIsDuleEnabled() == true); 41 | } 42 | 43 | } -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/catalog/model/DataSetViewTest.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.catalog.model; 18 | 19 | import com.adobe.platform.ecosystem.ut.BaseTest; 20 | import org.json.simple.parser.ParseException; 21 | import org.junit.Before; 22 | import org.junit.Test; 23 | 24 | import static org.junit.Assert.assertTrue; 25 | 26 | /** 27 | * created by prigarg on 10/10/17 28 | */ 29 | public class DataSetViewTest extends BaseTest{ 30 | DataSetView dsv = null; 31 | 32 | @Before 33 | public void setupDataSetView() throws ParseException { 34 | dsv = getDataSetViewFromStringWithId(dataSetViewSample, "dsvId"); 35 | } 36 | 37 | @Test 38 | public void testGetIsDuleEnabled() throws ParseException { 39 | assertTrue(dsv.getIsDuleEnabled() == true); 40 | } 41 | 42 | } -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/data/access/factory/DataAccessServiceFactoryTest.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.access.factory; 18 | 19 | import com.adobe.platform.ecosystem.examples.data.access.api.DataAccessService; 20 | import com.adobe.platform.ecosystem.examples.data.access.impl.DataAccessServiceImpl; 21 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 22 | import com.adobe.platform.ecosystem.ut.BaseTest; 23 | import org.junit.Test; 24 | 25 | import static org.junit.Assert.assertTrue; 26 | 27 | /** 28 | * Created by vedhera on 10/10/2017. 29 | */ 30 | public class DataAccessServiceFactoryTest extends BaseTest{ 31 | @Test 32 | public void testCatalogFactory() throws ConnectorSDKException{ 33 | DataAccessServiceFactory dasf = new DataAccessServiceFactory(); 34 | assertTrue(dasf.getDataAccessService(null) != null); 35 | DataAccessService das = DataAccessServiceFactory.getDataAccessService(null); 36 | assertTrue(das != null); 37 | assertTrue(das instanceof DataAccessServiceImpl); 38 | } 39 | } -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/data/access/model/DataAccessFileEntityTest.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.access.model; 18 | 19 | import com.adobe.platform.ecosystem.ut.BaseTest; 20 | import org.json.simple.JSONArray; 21 | import org.json.simple.JSONObject; 22 | import org.json.simple.parser.JSONParser; 23 | import org.json.simple.parser.ParseException; 24 | import org.junit.Before; 25 | import org.junit.Test; 26 | 27 | import static org.junit.Assert.assertTrue; 28 | 29 | /** 30 | * Created by vedhera on 01/09/2018. 31 | */ 32 | public class DataAccessFileEntityTest extends BaseTest { 33 | DataAccessFileEntity dataAccessFileEntity = null; 34 | 35 | @Before 36 | public void setupProcessingEntities() throws ParseException { 37 | dataAccessFileEntity = getDASFileEntityFromString(dataAccessServiceFileEntityResponse); 38 | } 39 | 40 | @Test 41 | public void testGetDataSetFileId() { 42 | assertTrue(dataAccessFileEntity.getDataSetFileId().equals("dataSetFileId1")); 43 | } 44 | 45 | @Test 46 | public void testGetDataSetViewId() { 47 | assertTrue(dataAccessFileEntity.getDataSetViewId().equals("dataSetViewId1")); 48 | } 49 | 50 | @Test 51 | public void testGetDASFileHref() { 52 | assertTrue(dataAccessFileEntity.getDataAccessServiceHref().equals("https://platform-int.adobe.io:443/data/foundation/export/files/dataSetFileId1")); 53 | } 54 | 55 | private DataAccessFileEntity getDASFileEntityFromString(String processingEntitiesJsonString) throws ParseException { 56 | JSONParser jsonParser = new JSONParser(); 57 | JSONObject jObject = (JSONObject)(jsonParser.parse(processingEntitiesJsonString)); 58 | JSONArray jArray = (JSONArray) jObject.get("data"); 59 | return new DataAccessFileEntity((JSONObject) jArray.get(0)); 60 | } 61 | } -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/data/access/model/DataSetFileProcessingEntityTest.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.access.model; 18 | 19 | import com.adobe.platform.ecosystem.ut.BaseTest; 20 | import org.json.simple.JSONArray; 21 | import org.json.simple.JSONObject; 22 | import org.json.simple.parser.JSONParser; 23 | import org.json.simple.parser.ParseException; 24 | import org.junit.Before; 25 | import org.junit.Test; 26 | 27 | import static org.junit.Assert.assertTrue; 28 | 29 | /** 30 | * Created by vedhera on 10/10/2017. 31 | */ 32 | 33 | public class DataSetFileProcessingEntityTest extends BaseTest { 34 | DataSetFileProcessingEntity dsfpe = null; 35 | 36 | @Before 37 | public void setupProcessingEntities() throws ParseException { 38 | dsfpe = getDataSetFileProcessingEntityFromString(dataAccessFilesResponse); 39 | } 40 | 41 | @Test 42 | public void testGetHref() throws ParseException { 43 | assertTrue(dsfpe.getHref().equals("href1")); 44 | } 45 | 46 | @Test 47 | public void testGetLength() throws ParseException { 48 | assertTrue(dsfpe.getLength() == 204); 49 | } 50 | 51 | @Test 52 | public void testGetName() throws ParseException { 53 | assertTrue(dsfpe.getName().equals("sql.csv")); 54 | } 55 | 56 | private DataSetFileProcessingEntity getDataSetFileProcessingEntityFromString(String processingEntitiesJsonString) throws ParseException { 57 | JSONParser jsonParser = new JSONParser(); 58 | JSONObject jObject = (JSONObject)(jsonParser.parse(processingEntitiesJsonString)); 59 | JSONArray jArray = (JSONArray) jObject.get("data"); 60 | return new DataSetFileProcessingEntity((JSONObject) jArray.get(0)); 61 | } 62 | } -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/data/functions/FieldConverterFunctionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017-2018 Adobe. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.adobe.platform.ecosystem.examples.data.functions; 17 | 18 | import com.adobe.platform.ecosystem.examples.parquet.model.ParquetIODataType; 19 | import com.adobe.platform.ecosystem.examples.catalog.model.DataType; 20 | import org.junit.Test; 21 | 22 | import java.util.function.Function; 23 | 24 | /** 25 | * @author vedhera om 2/20/2018 26 | */ 27 | public class FieldConverterFunctionTest { 28 | private final Function function = FieldConverterFunction.catalogToParquetFieldFunction; 29 | 30 | @Test 31 | public void testStringConversion() { 32 | assert (function.apply(DataType.StringType) == ParquetIODataType.STRING); 33 | } 34 | 35 | @Test 36 | public void testBooleanConversion() { 37 | assert (function.apply(DataType.BooleanType) == ParquetIODataType.BOOLEAN); 38 | } 39 | 40 | @Test 41 | public void testByteConversion() { 42 | assert (function.apply(DataType.ByteType) == ParquetIODataType.BYTE); 43 | } 44 | @Test 45 | public void testShortConversion() { 46 | assert (function.apply(DataType.ShortType) == ParquetIODataType.SHORT); 47 | } 48 | 49 | @Test 50 | public void testIntegerConversion() { 51 | assert (function.apply(DataType.IntegerType) == ParquetIODataType.INTEGER); 52 | } 53 | 54 | @Test 55 | public void testLongConversion() { 56 | assert (function.apply(DataType.LongType) == ParquetIODataType.LONG); 57 | } 58 | 59 | @Test 60 | public void testFloatConversion() { 61 | assert (function.apply(DataType.FloatType) == ParquetIODataType.FLOAT); 62 | } 63 | 64 | @Test 65 | public void testDateConversion() { 66 | assert (function.apply(DataType.DateType) == ParquetIODataType.DATE); 67 | } 68 | 69 | @Test 70 | public void testDateTimeConversion() { 71 | assert (function.apply(DataType.DateTimeType) == ParquetIODataType.TIMESTAMP); 72 | } 73 | 74 | @Test 75 | public void testDoubleConversion() { 76 | assert (function.apply(DataType.DoubleType) == ParquetIODataType.DOUBLE); 77 | } 78 | 79 | @Test 80 | public void testBinaryConversion() { 81 | assert (function.apply(DataType.BinaryType) == ParquetIODataType.BINARY); 82 | } 83 | 84 | @Test 85 | public void testObjectConversion() { 86 | assert (function.apply(DataType.Field_ObjectType) == ParquetIODataType.GROUP); 87 | } 88 | 89 | @Test 90 | public void testArrayConversion() { 91 | assert (function.apply(DataType.Field_ArrayType) == ParquetIODataType.LIST); 92 | } 93 | } -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/data/ingestion/impl/DataIngestionServiceTest.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.ingestion.impl; 18 | 19 | import static org.junit.Assert.assertFalse; 20 | import static org.junit.Assert.assertTrue; 21 | 22 | import org.json.simple.JSONObject; 23 | import org.junit.Before; 24 | import org.junit.Test; 25 | 26 | import com.adobe.platform.ecosystem.examples.data.FileFormat; 27 | 28 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 29 | import com.adobe.platform.ecosystem.ut.BaseTest; 30 | 31 | /** 32 | * Created by vardgupt on 10/10/2017. 33 | */ 34 | 35 | public class DataIngestionServiceTest extends BaseTest { 36 | 37 | private DataIngestionServiceImpl dis = new DataIngestionServiceImpl("testEndpoint",httpClient); 38 | 39 | private JSONObject payload; 40 | 41 | public DataIngestionServiceTest() throws ConnectorSDKException { 42 | } 43 | 44 | @Before 45 | public void setupDataWiring() throws Exception { 46 | setUp(); 47 | payload = new JSONObject(); 48 | payload.put("dataSetId", "testDataSetId"); 49 | } 50 | 51 | @Test 52 | public void testConstructorWithOneArguement() { 53 | DataIngestionServiceImpl ds = null; 54 | try { 55 | ds = new DataIngestionServiceImpl("testEndpoint"); 56 | } catch (ConnectorSDKException e) { 57 | assertTrue(false); 58 | } 59 | assertTrue(ds != null); 60 | } 61 | 62 | @Test 63 | public void testConstructorWithTwoArguement() { 64 | assertTrue(dis != null); 65 | } 66 | 67 | @Test(expected = ConnectorSDKException.class) 68 | public void testGetBatchId() throws Exception { 69 | setupTestForHttpOutput(batchSample); 70 | dis.getBatchId("testIMSOrg","testAccessToken", payload); 71 | } 72 | 73 | @Test 74 | public void testWriteToBatch() throws ConnectorSDKException { 75 | String bufferString = "testData"; 76 | try{ 77 | setupTestForHttpOutput(batchSample); 78 | dis.writeToBatch("testBatchId", "testDataSetId", "testIMSOrg", "testAccessToken", FileFormat.PARQUET, bufferString.getBytes()); 79 | } 80 | catch(Exception e){ 81 | assertTrue(true); 82 | } 83 | } 84 | 85 | @Test 86 | public void testSignalBatchCompletion() throws ConnectorSDKException { 87 | try{ 88 | setupTestForHttpOutput(batchSample); 89 | dis.signalBatchCompletion("testBatchId", "testIMSOrg", "testAccessToken"); 90 | } 91 | catch(Exception e){ 92 | assertFalse(true); 93 | } 94 | } 95 | 96 | } -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/data/validation/impl/CatalogValidationRegistryFactoryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * ADOBE CONFIDENTIAL 3 | * __________________ 4 | * Copyright 2018 Adobe Systems Incorporated 5 | * All Rights Reserved. 6 | * 7 | * NOTICE: All information contained herein is, and remains 8 | * the property of Adobe Systems Incorporated and its suppliers, 9 | * if any. The intellectual and technical concepts contained 10 | * herein are proprietary to Adobe Systems Incorporated and its 11 | * suppliers and are protected by all applicable intellectual property laws, 12 | * including trade secret and copyright laws. 13 | * 14 | * Dissemination of this information or reproduction of this material 15 | * is strictly forbidden unless prior written permission is obtained 16 | * from Adobe Systems Incorporated. 17 | * 18 | * 19 | */ 20 | package com.adobe.platform.ecosystem.examples.data.validation.impl; 21 | 22 | import com.adobe.platform.ecosystem.examples.catalog.model.DataType; 23 | import com.adobe.platform.ecosystem.examples.catalog.model.SchemaField; 24 | import com.adobe.platform.ecosystem.examples.data.validation.api.ValidationRegistry; 25 | import org.junit.Before; 26 | import org.junit.Test; 27 | import org.mockito.Mock; 28 | 29 | import java.util.ArrayList; 30 | 31 | import static org.junit.Assert.assertNotNull; 32 | import static org.junit.Assert.assertTrue; 33 | import static org.mockito.Mockito.when; 34 | import static org.mockito.MockitoAnnotations.initMocks; 35 | 36 | /** 37 | * @author vedhera on 11/26/2018 38 | */ 39 | public class CatalogValidationRegistryFactoryTest { 40 | 41 | @Mock 42 | SchemaField schemaField; 43 | 44 | private CatalogValidationRegistryFactory factory; 45 | 46 | @Before 47 | public void setup() { 48 | initMocks(this); 49 | 50 | when(schemaField.getType()).thenReturn(DataType.StringType); 51 | when(schemaField.getName()).thenReturn("primitiveField"); 52 | when(schemaField.getRules()).thenReturn(new ArrayList<>()); 53 | 54 | factory = new CatalogValidationRegistryFactory(); 55 | } 56 | 57 | @Test 58 | public void testRegistryFactory() { 59 | ValidationRegistry validationRegistry = factory.get(schemaField); 60 | assertNotNull(validationRegistry); 61 | assertTrue(validationRegistry instanceof CatalogValidationRegistry); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/data/validation/impl/rules/IntegerValidationRuleTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * ADOBE CONFIDENTIAL 3 | * __________________ 4 | * Copyright 2018 Adobe Systems Incorporated 5 | * All Rights Reserved. 6 | * 7 | * NOTICE: All information contained herein is, and remains 8 | * the property of Adobe Systems Incorporated and its suppliers, 9 | * if any. The intellectual and technical concepts contained 10 | * herein are proprietary to Adobe Systems Incorporated and its 11 | * suppliers and are protected by all applicable intellectual property laws, 12 | * including trade secret and copyright laws. 13 | * 14 | * Dissemination of this information or reproduction of this material 15 | * is strictly forbidden unless prior written permission is obtained 16 | * from Adobe Systems Incorporated. 17 | * 18 | * 19 | */ 20 | package com.adobe.platform.ecosystem.examples.data.validation.impl.rules; 21 | 22 | import com.adobe.platform.ecosystem.examples.data.validation.exception.ValidationException; 23 | import org.junit.Before; 24 | import org.junit.Test; 25 | 26 | import static org.junit.Assert.assertEquals; 27 | 28 | /** 29 | * @author vedhera on 11/26/2018. 30 | */ 31 | public class IntegerValidationRuleTest { 32 | private SchemaValidationRule integerValidationRule; 33 | 34 | @Before 35 | public void setup() { 36 | integerValidationRule = IntegerValidationRule.integerValidationRuleBuilder() 37 | .withMinimum(4) 38 | .withMaximum(8) 39 | .build(); 40 | } 41 | 42 | @Test 43 | public void testSetup() { 44 | IntegerValidationRule rule = (IntegerValidationRule) integerValidationRule; 45 | assertEquals(rule.getMinimum().get(), new Integer(4)); 46 | assertEquals(rule.getMaximum().get(), new Integer(8)); 47 | } 48 | 49 | @Test(expected = ValidationException.class) 50 | public void testMinLength() throws ValidationException { 51 | integerValidationRule.apply(0); 52 | } 53 | 54 | @Test(expected = ValidationException.class) 55 | public void testMaxLength() throws ValidationException { 56 | integerValidationRule.apply(10); 57 | } 58 | 59 | @Test 60 | public void testAllRulesPass() throws ValidationException { 61 | integerValidationRule.apply(6); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/data/validation/impl/rules/LongValidationRuleTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * ADOBE CONFIDENTIAL 3 | * __________________ 4 | * Copyright 2018 Adobe Systems Incorporated 5 | * All Rights Reserved. 6 | * 7 | * NOTICE: All information contained herein is, and remains 8 | * the property of Adobe Systems Incorporated and its suppliers, 9 | * if any. The intellectual and technical concepts contained 10 | * herein are proprietary to Adobe Systems Incorporated and its 11 | * suppliers and are protected by all applicable intellectual property laws, 12 | * including trade secret and copyright laws. 13 | * 14 | * Dissemination of this information or reproduction of this material 15 | * is strictly forbidden unless prior written permission is obtained 16 | * from Adobe Systems Incorporated. 17 | * 18 | * 19 | */ 20 | package com.adobe.platform.ecosystem.examples.data.validation.impl.rules; 21 | 22 | import com.adobe.platform.ecosystem.examples.data.validation.exception.ValidationException; 23 | import org.junit.Before; 24 | import org.junit.Test; 25 | 26 | import static org.junit.Assert.assertEquals; 27 | 28 | /** 29 | * @author vedhera on 11/26/2018. 30 | */ 31 | public class LongValidationRuleTest { 32 | private SchemaValidationRule longValidationRule; 33 | 34 | @Before 35 | public void setup() { 36 | longValidationRule = LongValidationRule.longValidationRuleBuilder() 37 | .withMinimum(4) 38 | .withMaximum(8) 39 | .build(); 40 | } 41 | 42 | @Test 43 | public void testSetup() { 44 | LongValidationRule rule = (LongValidationRule) longValidationRule; 45 | assertEquals(rule.getMinimum().get(), new Long(4)); 46 | assertEquals(rule.getMaximum().get(), new Long(8)); 47 | } 48 | 49 | @Test(expected = ValidationException.class) 50 | public void testMinLength() throws ValidationException { 51 | longValidationRule.apply(0l); 52 | } 53 | 54 | @Test(expected = ValidationException.class) 55 | public void testMaxLength() throws ValidationException { 56 | longValidationRule.apply(10l); 57 | } 58 | 59 | @Test 60 | public void testAllRulesPass() throws ValidationException { 61 | longValidationRule.apply(6l); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/data/validation/impl/rules/StringValidationRuleTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * ADOBE CONFIDENTIAL 3 | * __________________ 4 | * Copyright 2018 Adobe Systems Incorporated 5 | * All Rights Reserved. 6 | * 7 | * NOTICE: All information contained herein is, and remains 8 | * the property of Adobe Systems Incorporated and its suppliers, 9 | * if any. The intellectual and technical concepts contained 10 | * herein are proprietary to Adobe Systems Incorporated and its 11 | * suppliers and are protected by all applicable intellectual property laws, 12 | * including trade secret and copyright laws. 13 | * 14 | * Dissemination of this information or reproduction of this material 15 | * is strictly forbidden unless prior written permission is obtained 16 | * from Adobe Systems Incorporated. 17 | * 18 | * 19 | */ 20 | package com.adobe.platform.ecosystem.examples.data.validation.impl.rules; 21 | 22 | import com.adobe.platform.ecosystem.examples.data.validation.exception.ValidationException; 23 | import org.junit.Before; 24 | import org.junit.Test; 25 | 26 | import java.util.Arrays; 27 | import java.util.Optional; 28 | 29 | import static org.junit.Assert.assertEquals; 30 | 31 | /** 32 | * @author vedhera on 11/26/2018. 33 | */ 34 | public class StringValidationRuleTest { 35 | 36 | private SchemaValidationRule stringValidationRule; 37 | 38 | @Before 39 | public void setup() { 40 | stringValidationRule = StringValidationRule.stringValidationRuleBuilder() 41 | .withMinLength(4) 42 | .withMaxLength(8) 43 | .withEnumList(Arrays.asList("value1", "value2")) 44 | .build(); 45 | } 46 | 47 | @Test 48 | public void testSetup() { 49 | StringValidationRule rule = (StringValidationRule) stringValidationRule; 50 | assertEquals(rule.getMinLength().get(), new Integer(4)); 51 | assertEquals(rule.getMaxLength().get(), new Integer(8)); 52 | assertEquals(rule.getEnumList().get().size(), 2); 53 | assertEquals(rule.getFormat(), Optional.empty()); 54 | assertEquals(rule.getPattern(), Optional.empty()); 55 | } 56 | 57 | @Test(expected = ValidationException.class) 58 | public void testMinLength() throws ValidationException { 59 | stringValidationRule.apply("ab"); 60 | } 61 | 62 | @Test(expected = ValidationException.class) 63 | public void testMaxLength() throws ValidationException { 64 | stringValidationRule.apply("ababababa"); 65 | } 66 | 67 | @Test(expected = ValidationException.class) 68 | public void testEnumValidation() throws ValidationException { 69 | stringValidationRule.apply("value3"); 70 | } 71 | 72 | @Test 73 | public void testAllRulesPass() throws ValidationException { 74 | stringValidationRule.apply("value1"); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/data/wiring/DataWiringTest.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.wiring; 18 | 19 | import com.adobe.platform.ecosystem.ut.BaseTest; 20 | import org.junit.Before; 21 | import org.junit.Test; 22 | 23 | import static org.junit.Assert.assertTrue; 24 | 25 | public class DataWiringTest extends BaseTest{ 26 | 27 | DataWiring dataWiring = null; 28 | 29 | @Before 30 | public void setupDataWiring() throws Exception { 31 | setUp(); 32 | setupTestForHttpOutput(credSample); 33 | setupDataset(); 34 | dataWiring = new DataWiring("testImsOrg", dataset,httpClient); 35 | } 36 | 37 | @Test 38 | public void testConstructorWithHttpClient() { 39 | assertTrue(dataWiring!=null); 40 | } 41 | 42 | @Test 43 | public void testGetDataReaderFactory() { 44 | assertTrue(dataWiring.dataReaderFactory()!=null); 45 | } 46 | 47 | @Test 48 | public void testGetDataWriterFactory() { 49 | assertTrue(dataWiring.dataWriterFactory()!=null); 50 | } 51 | 52 | } -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/data/write/PlatformDataFormatterFactoryTest.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.write; 18 | 19 | import static org.junit.Assert.assertTrue; 20 | import static org.mockito.MockitoAnnotations.initMocks; 21 | 22 | import com.adobe.platform.ecosystem.examples.data.FileFormat; 23 | import com.adobe.platform.ecosystem.examples.data.validation.api.ValidationRegistryFactory; 24 | import org.junit.Before; 25 | import org.junit.Test; 26 | 27 | import com.adobe.platform.ecosystem.examples.parquet.write.ParquetIOWriter; 28 | import com.adobe.platform.ecosystem.examples.parquet.write.ParquetIOWriterImpl; 29 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 30 | import com.adobe.platform.ecosystem.ut.BaseTest; 31 | import org.mockito.Mock; 32 | 33 | public class PlatformDataFormatterFactoryTest extends BaseTest { 34 | 35 | @Mock 36 | ValidationRegistryFactory registryFactory; 37 | 38 | @Before 39 | public void setup() { 40 | initMocks(this); 41 | } 42 | 43 | @Test 44 | public void testGetFormatter() { 45 | ParquetIOWriter writer = new ParquetIOWriterImpl(); 46 | PlatformDataFormatterFactory platFormatter = new PlatformDataFormatterFactory(writer, param, registryFactory); 47 | platFormatter.setWriteAttributes(new WriteAttributes.WriteAttributesBuilder().build()); 48 | assertTrue(platFormatter.getFormatter(FileFormat.CSV) != null); 49 | assertTrue(platFormatter.getFormatter(FileFormat.PARQUET) != null); 50 | assertTrue(platFormatter.getFormatter(FileFormat.JSON) != null); 51 | } 52 | } -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/data/write/PlatformDataWriterFactoryTest.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.write; 18 | 19 | import com.adobe.platform.ecosystem.examples.data.validation.api.ValidationRegistryFactory; 20 | import com.adobe.platform.ecosystem.examples.parquet.write.ParquetIOWriter; 21 | import com.adobe.platform.ecosystem.examples.parquet.write.ParquetIOWriterImpl; 22 | import com.adobe.platform.ecosystem.examples.catalog.model.FileDescription; 23 | import com.adobe.platform.ecosystem.examples.data.FileFormat; 24 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 25 | import com.adobe.platform.ecosystem.ut.BaseTest; 26 | import org.junit.Before; 27 | import org.junit.Test; 28 | import org.mockito.Mock; 29 | import org.mockito.Mockito; 30 | 31 | import static org.junit.Assert.assertTrue; 32 | import static org.mockito.MockitoAnnotations.initMocks; 33 | 34 | public class PlatformDataWriterFactoryTest extends BaseTest { 35 | 36 | @Mock 37 | ValidationRegistryFactory registryFactory; 38 | 39 | @Before 40 | public void setUp() { 41 | initMocks(this); 42 | FileDescription mockFileDescription = Mockito.mock(FileDescription.class); 43 | Mockito.when(dataset.getFileDescription()).thenReturn(mockFileDescription); 44 | Mockito.when(mockFileDescription.getFormat()).thenReturn(FileFormat.PARQUET); 45 | } 46 | 47 | @Test 48 | public void testGetWriter() { 49 | ParquetIOWriter writer = new ParquetIOWriterImpl(); 50 | PlatformDataFormatterFactory formatterFactory = new PlatformDataFormatterFactory(writer, param, registryFactory); 51 | PlatformDataWriterFactory platWriter = new PlatformDataWriterFactory(param, catService, dis, httpClient, formatterFactory); 52 | try { 53 | Mockito.when(dis.getBatchId(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn("sfdsgsgsdgtw"); 54 | assertTrue(platWriter.getWriter() != null); 55 | } catch (ConnectorSDKException e) { 56 | e.printStackTrace(); 57 | } 58 | } 59 | } -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/data/write/converter/PipelineToJSONConverterTest.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.data.write.converter; 18 | 19 | import org.json.simple.JSONObject; 20 | import org.junit.Before; 21 | import org.junit.Test; 22 | 23 | import java.util.ArrayList; 24 | import java.util.List; 25 | import java.util.Set; 26 | 27 | /** 28 | * @author vedhera on 2/20/2018 29 | */ 30 | public class PipelineToJSONConverterTest { 31 | private List pipelineData; 32 | 33 | private final String ROOT_ID = "rootId"; 34 | 35 | @Before 36 | public void setup() { 37 | JSONObject row1 = new JSONObject(); 38 | JSONObject innerData1 = new JSONObject(); 39 | innerData1.put("key1_1", "value1_1"); 40 | innerData1.put("key1_2", "value1_2"); 41 | row1.put(ROOT_ID, innerData1); 42 | 43 | JSONObject row2 = new JSONObject(); 44 | JSONObject innerData2 = new JSONObject(); 45 | innerData2.put("key2_1", "value2_1"); 46 | innerData2.put("key2_2", "value2_2"); 47 | row2.put(ROOT_ID, innerData2); 48 | 49 | pipelineData = new ArrayList<>(); 50 | pipelineData.add(row1.toString()); 51 | pipelineData.add(row2.toString()); 52 | } 53 | 54 | @Test 55 | public void TestConversionForEmptyPipelineData() { 56 | List jsonData = PipelineToJSONConverter.getFields(null, ROOT_ID); 57 | assert (jsonData.size() == 0); 58 | } 59 | 60 | @Test 61 | public void TestConversionForCorruptPipelineData() { 62 | List badData = new ArrayList<>(); 63 | badData.add("{ bad JSon String"); 64 | List jsonData = PipelineToJSONConverter.getFields(badData, ROOT_ID); 65 | assert (jsonData.size() == 0); 66 | } 67 | 68 | @Test 69 | public void TestPipelineDataToJsonConversion() { 70 | List jsonData = PipelineToJSONConverter.getFields(pipelineData, ROOT_ID); 71 | assert (jsonData.size() == 2); 72 | JSONObject row1 = jsonData.get(0); 73 | Set firstKeySet = row1.keySet(); 74 | assert (firstKeySet.contains("key1_1")); 75 | assert (firstKeySet.contains("key1_2")); 76 | 77 | JSONObject row2 = jsonData.get(1); 78 | Set secondKeySet = row2.keySet(); 79 | assert (secondKeySet.contains("key2_1")); 80 | assert (secondKeySet.contains("key2_2")); 81 | } 82 | } -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/data/write/mapper/MapperUtilTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017-2018 Adobe. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.adobe.platform.ecosystem.examples.data.write.mapper; 17 | 18 | import com.adobe.platform.ecosystem.examples.catalog.model.SDKField; 19 | import org.json.simple.JSONObject; 20 | import org.junit.Test; 21 | 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | 25 | import static junit.framework.Assert.assertEquals; 26 | import static junit.framework.Assert.assertNotNull; 27 | import static junit.framework.Assert.assertTrue; 28 | 29 | /** 30 | * @author vedhera on 7/19/2018. 31 | */ 32 | public class MapperUtilTest { 33 | 34 | 35 | @Test 36 | public void getJsonObjectTest() { 37 | 38 | List sdkFields = new ArrayList(); 39 | SDKField field01 = new SDKField("a.b.c", "string"); 40 | SDKField field02 = new SDKField("a.b.d", "string"); 41 | SDKField field05 = new SDKField("a.e.c.d", "string"); 42 | SDKField field03 = new SDKField("e.f", "string"); 43 | SDKField field04 = new SDKField("e.g", "string"); 44 | 45 | sdkFields.add(field01); 46 | sdkFields.add(field02); 47 | sdkFields.add(field05); 48 | sdkFields.add(field03); 49 | sdkFields.add(field04); 50 | 51 | List> dataTable = new ArrayList>(); 52 | 53 | ArrayList record01 = new ArrayList(); 54 | 55 | record01.add("val01"); 56 | record01.add("val02"); 57 | record01.add("val05"); 58 | record01.add("val03"); 59 | record01.add("val04"); 60 | dataTable.add(record01); 61 | 62 | List convertedData = MapperUtil.convert(sdkFields, dataTable); 63 | 64 | assertNotNull(convertedData); 65 | assertEquals(convertedData.size(), 1); 66 | assertEquals(convertedData.get(0).keySet().size(), 2); 67 | assertTrue(convertedData.get(0).get("a") instanceof JSONObject); 68 | assertTrue(convertedData.get(0).get("e") instanceof JSONObject); 69 | 70 | JSONObject hierarchy1 = (JSONObject) convertedData.get(0).get("a"); 71 | JSONObject hierarchy2 = (JSONObject) convertedData.get(0).get("e"); 72 | 73 | assertEquals(hierarchy1.keySet().size(), 2); 74 | assertEquals(hierarchy2.keySet().size(), 2); 75 | 76 | assertTrue(hierarchy1.get("b") instanceof JSONObject); 77 | assertTrue(hierarchy1.get("e") instanceof JSONObject); 78 | 79 | JSONObject hierarchy1_1 = (JSONObject) hierarchy1.get("b"); 80 | JSONObject hierarchy1_2 = (JSONObject) hierarchy1.get("e"); 81 | assertEquals(hierarchy1_1.keySet().size(), 2); 82 | assertTrue(hierarchy1_1.get("c") instanceof String); 83 | assertTrue(hierarchy1_1.get("d") instanceof String); 84 | assertEquals(hierarchy1_1.get("c"), "val01"); 85 | assertEquals(hierarchy1_1.get("d"), "val02"); 86 | 87 | assertEquals(hierarchy1_2.keySet().size(), 1); 88 | 89 | 90 | assertTrue(hierarchy2.get("f") instanceof String); 91 | assertEquals(hierarchy2.get("f"), "val03"); 92 | assertTrue(hierarchy2.get("g") instanceof String); 93 | assertEquals(hierarchy2.get("g"), "val04"); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/schemaregistry/SchemaRegistryFactoryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * ADOBE CONFIDENTIAL 3 | * __________________ 4 | * Copyright 2019 Adobe Systems Incorporated 5 | * All Rights Reserved. 6 | * 7 | * NOTICE: All information contained herein is, and remains 8 | * the property of Adobe Systems Incorporated and its suppliers, 9 | * if any. The intellectual and technical concepts contained 10 | * herein are proprietary to Adobe Systems Incorporated and its 11 | * suppliers and are protected by all applicable intellectual property laws, 12 | * including trade secret and copyright laws. 13 | * 14 | * Dissemination of this information or reproduction of this material 15 | * is strictly forbidden unless prior written permission is obtained 16 | * from Adobe Systems Incorporated. 17 | * 18 | * 19 | */ 20 | package com.adobe.platform.ecosystem.examples.schemaregistry; 21 | 22 | import com.adobe.platform.ecosystem.examples.schemaregistry.api.SchemaRegistryService; 23 | import com.adobe.platform.ecosystem.ut.BaseTest; 24 | import org.junit.Test; 25 | 26 | import static org.junit.Assert.assertTrue; 27 | 28 | public class SchemaRegistryFactoryTest extends BaseTest { 29 | 30 | @Test 31 | public void testSchemaRegistryFactory() throws Exception { 32 | setUp(); 33 | assertTrue(SchemaRegistryFactory.getSchemaRegistryService() != null); 34 | SchemaRegistryService schemaRegistryService = SchemaRegistryFactory.getSchemaRegistryService(null); 35 | assertTrue(schemaRegistryService != null); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/schemaregistry/SchemaRegistryServiceTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * ADOBE CONFIDENTIAL 3 | * __________________ 4 | * Copyright 2019 Adobe Systems Incorporated 5 | * All Rights Reserved. 6 | * 7 | * NOTICE: All information contained herein is, and remains 8 | * the property of Adobe Systems Incorporated and its suppliers, 9 | * if any. The intellectual and technical concepts contained 10 | * herein are proprietary to Adobe Systems Incorporated and its 11 | * suppliers and are protected by all applicable intellectual property laws, 12 | * including trade secret and copyright laws. 13 | * 14 | * Dissemination of this information or reproduction of this material 15 | * is strictly forbidden unless prior written permission is obtained 16 | * from Adobe Systems Incorporated. 17 | * 18 | * 19 | */ 20 | package com.adobe.platform.ecosystem.examples.schemaregistry; 21 | 22 | import com.adobe.platform.ecosystem.examples.catalog.model.SchemaField; 23 | import com.adobe.platform.ecosystem.examples.catalog.model.SchemaRef; 24 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKException; 25 | import com.adobe.platform.ecosystem.examples.util.ConnectorSDKUtil; 26 | import com.adobe.platform.ecosystem.examples.util.ResourceName; 27 | import com.adobe.platform.ecosystem.ut.BaseTest; 28 | import org.json.simple.JSONObject; 29 | import org.junit.Before; 30 | import org.junit.Test; 31 | 32 | import java.io.IOException; 33 | import java.util.List; 34 | 35 | import static org.junit.Assert.assertEquals; 36 | import static org.junit.Assert.assertTrue; 37 | 38 | public class SchemaRegistryServiceTest extends BaseTest { 39 | SchemaRegistryServiceImpl schemaRegistryService; 40 | 41 | @Before 42 | public void setupTest() throws Exception { 43 | setUp(); 44 | schemaRegistryService = new SchemaRegistryServiceImpl(ConnectorSDKUtil.getInstance().getEndPoint(ResourceName.SCHEMA_REGISTRY), httpClient); 45 | } 46 | 47 | @Test 48 | public void testConstructor() throws Exception { 49 | assertTrue(schemaRegistryService != null); 50 | 51 | schemaRegistryService = new SchemaRegistryServiceImpl(null); 52 | assertTrue(schemaRegistryService != null); 53 | 54 | schemaRegistryService = new SchemaRegistryServiceImpl(ConnectorSDKUtil.getInstance().getEndPoint(ResourceName.SCHEMA_REGISTRY)); 55 | assertTrue(schemaRegistryService != null); 56 | } 57 | 58 | @Test 59 | public void testGetSchemaFields() throws IOException, ConnectorSDKException { 60 | setupTestForHttpOutput(schemaSample); 61 | JSONObject jsonObject = new JSONObject(); 62 | jsonObject.put("id" , "https://ns.adobe.com/test/Id"); 63 | jsonObject.put("contentType" , "application/vnd.adobe.xed+json; version=1.0"); 64 | SchemaRef schemaRef = new SchemaRef(jsonObject); 65 | List schemaFields = schemaRegistryService.getSchemaFields("testOrg", "testToken", schemaRef, true); 66 | assertTrue(schemaFields != null); 67 | assertEquals(schemaFields.size(),4); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/util/ConnectorSDKUtilTest.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.util; 18 | 19 | import static org.junit.Assert.*; 20 | 21 | import java.util.HashMap; 22 | import java.util.Map; 23 | import java.util.Properties; 24 | 25 | import org.json.simple.JSONArray; 26 | import org.json.simple.JSONObject; 27 | import org.junit.Before; 28 | import org.junit.Test; 29 | 30 | public class ConnectorSDKUtilTest { 31 | 32 | 33 | @Test 34 | public void testInitialize() { 35 | Map inputProps = new HashMap(); 36 | try { 37 | ConnectorSDKUtil.initialize(inputProps); 38 | } catch (ConnectorSDKException e) { 39 | assert(e.getMessage().contains("ConnectorSDKUtil is not initialized")); 40 | } 41 | assert(ConnectorSDKUtil.httpClient != null); 42 | } 43 | 44 | @Test 45 | public void testInitLogger() { 46 | try { 47 | ConnectorSDKUtil.initializeLogger(); 48 | } catch (Exception e) { 49 | fail("Exception thrown while initiating logger!"); 50 | } 51 | } 52 | 53 | } -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/util/HTTPClientUtilTest.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.util; 18 | 19 | import static org.junit.Assert.*; 20 | 21 | import java.io.IOException; 22 | import java.util.HashMap; 23 | import java.util.Map; 24 | import java.util.Properties; 25 | 26 | import org.apache.http.HttpRequest; 27 | import org.apache.http.client.ClientProtocolException; 28 | import org.apache.http.client.methods.HttpGet; 29 | import org.apache.http.client.methods.HttpUriRequest; 30 | import org.json.simple.JSONArray; 31 | import org.json.simple.JSONObject; 32 | import org.junit.Before; 33 | import org.junit.Test; 34 | 35 | public class HTTPClientUtilTest { 36 | 37 | 38 | @Test 39 | public void testRetryHandlerHostNotFound() { 40 | HttpUriRequest req = new HttpGet("http://abcxyz7289289839.com"); 41 | try { 42 | HttpClientUtil.getHttpClient().execute(req); 43 | fail("Error was not thrown"); 44 | } catch (Exception e) { 45 | //test passes here 46 | } 47 | } 48 | 49 | @Test 50 | public void testRetryHandlerSSLErrorRetry() { 51 | HttpUriRequest req = new HttpGet("http://adobe.com:443"); 52 | try { 53 | HttpClientUtil.getHttpClient().execute(req); 54 | fail("Error was not thrown"); 55 | } catch (Exception e) { 56 | e.printStackTrace(); 57 | //test passes here 58 | } 59 | } 60 | 61 | } -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/util/JsonUtilTest.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.util; 18 | 19 | import static org.junit.Assert.*; 20 | 21 | import org.json.simple.JSONArray; 22 | import org.json.simple.JSONObject; 23 | import org.junit.Before; 24 | import org.junit.Test; 25 | 26 | public class JsonUtilTest { 27 | 28 | JSONObject jsonObj = new JSONObject(); 29 | JSONArray jsonarray = new JSONArray(); 30 | 31 | @Before 32 | public void before() { 33 | jsonObj.put("intkey", 1); 34 | jsonObj.put("boolkey", true); 35 | 36 | jsonarray.add(50); 37 | jsonObj.put("arraykey", jsonarray); 38 | } 39 | 40 | @Test 41 | public void testConstructor() { 42 | JsonUtil jsonUtil = new JsonUtil(); 43 | assertTrue(jsonUtil != null); 44 | } 45 | 46 | @Test 47 | public void testGetInteger() { 48 | assertTrue(JsonUtil.getInteger(jsonObj, "intkey") == 1); 49 | assertTrue(JsonUtil.getInteger(jsonObj, "intkey1") == -1); 50 | } 51 | 52 | @Test 53 | public void testGetBoolean() { 54 | assertTrue(JsonUtil.getBoolean(jsonObj, "boolkey")); 55 | assertTrue(!JsonUtil.getBoolean(jsonObj, "boolkey1")); 56 | } 57 | 58 | @Test 59 | public void testJsonArray() { 60 | assertTrue(JsonUtil.getJsonArray(jsonObj, "arraykey").size() > 0); 61 | assertTrue(JsonUtil.getJsonArray(jsonObj, "arraykey1").size() == 0); 62 | } 63 | } -------------------------------------------------------------------------------- /examples/src/test/java/com/adobe/platform/ecosystem/examples/util/ResourceReaderTest.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.util; 18 | 19 | import static org.junit.Assert.*; 20 | 21 | import java.util.Properties; 22 | 23 | import org.junit.Test; 24 | 25 | public class ResourceReaderTest { 26 | 27 | Properties props = new ResourceReader().readPropertiesFromResource(); 28 | 29 | @Test 30 | public void testReadConfig() { 31 | assertTrue(props.size() > 0); 32 | } 33 | 34 | @Test 35 | public void testReadSpecificConfig() { 36 | assertTrue(new ResourceReader().getProperty("lookupEndPoint")!=null); 37 | assertTrue(new ResourceReader().getProperty("NonExistentLookupEndpoint")==null); 38 | } 39 | 40 | @Test 41 | public void testConstructor() { 42 | ResourceReader resReader = new ResourceReader(); 43 | assertTrue(resReader != null); 44 | } 45 | 46 | @Test 47 | public void testReadDefaultConfig() { 48 | Properties defaultProps = new ResourceReader().readDefaultConfigFromResource(); 49 | assertTrue(defaultProps.size() > 0); 50 | assertTrue(defaultProps.getProperty("defaultJWTTokenExpiryInDays") != null); 51 | } 52 | } -------------------------------------------------------------------------------- /examples/src/test/resources/pdfTest.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/experience-platform-etl-reference/e7040e9fcbc002dc4ee75b75f4e81324f238e0b0/examples/src/test/resources/pdfTest.parquet -------------------------------------------------------------------------------- /examples/src/test/resources/primitive.json: -------------------------------------------------------------------------------- 1 | { 2 | "properties" : { 3 | "firstName": { 4 | "title": "First name", 5 | "type": "string", 6 | "description": "The first segment of the name in the writing order most commonly accepted in the language of the name. In many cultures this is the preferred personal or given name.\n\nThe `firstName` and `lastName` properties have been introduced to maintain compatibility with existing systems that model names in a simplified, non-semantic, and non-internationalizable way. Using `xdm:fullName` is always preferable.", 7 | "meta:xdmType": "string", 8 | "meta:xdmField": "xdm:firstName" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /examples/src/test/resources/primitiveXdmArray.json: -------------------------------------------------------------------------------- 1 | { 2 | "properties" : { 3 | "contentTypes": { 4 | "type": "array", 5 | "description": "The content types that this channel can deliver.", 6 | "meta:xdmType": "array", 7 | "items": { 8 | "type": "string", 9 | "format": "uri", 10 | "description": "The `@type` of an XDM-defined content type that is supported by this channel.", 11 | "meta:xdmType": "string" 12 | }, 13 | "meta:xdmField": "xdm:contentTypes" 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /examples/src/test/resources/sample.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/experience-platform-etl-reference/e7040e9fcbc002dc4ee75b75f4e81324f238e0b0/examples/src/test/resources/sample.parquet -------------------------------------------------------------------------------- /examples/src/test/resources/secret.key: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/experience-platform-etl-reference/e7040e9fcbc002dc4ee75b75f4e81324f238e0b0/examples/src/test/resources/secret.key -------------------------------------------------------------------------------- /examples/src/test/resources/secret_invalid.key: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/experience-platform-etl-reference/e7040e9fcbc002dc4ee75b75f4e81324f238e0b0/examples/src/test/resources/secret_invalid.key -------------------------------------------------------------------------------- /parquetio/src/main/java/com/adobe/platform/ecosystem/examples/parquet/entity/Node.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.parquet.entity; 18 | 19 | import java.util.ArrayList; 20 | import java.util.LinkedHashMap; 21 | 22 | /** 23 | * Created by vardgupt on 10/10/2017. 24 | */ 25 | 26 | /** 27 | * Entity to hold information 28 | * about one node of the schema 29 | * tree. eg: The columns after flatenning 30 | * are a_b_c and a_e_d. Here 'a' will 31 | * represent one node with level 0 and 32 | * children as 2 ('b' and 'e') respectively. 33 | */ 34 | public class Node implements Cloneable { 35 | 36 | String key; 37 | LinkedHashMap> children = null; 38 | int depth; 39 | String dataType; 40 | 41 | public Node(String key, String dataType, int depth, LinkedHashMap> children){ 42 | this.key = key; 43 | this.depth = depth; 44 | this.children = children; 45 | this.dataType = dataType; 46 | } 47 | 48 | public String getKey() { 49 | return key; 50 | } 51 | 52 | 53 | public LinkedHashMap> getChildren() { 54 | return children; 55 | } 56 | 57 | public void setChildren(LinkedHashMap> children) { 58 | this.children = children; 59 | } 60 | 61 | public int getDepth() { 62 | return depth; 63 | } 64 | 65 | 66 | public String getDataType() { 67 | return dataType; 68 | } 69 | 70 | public Object clone()throws CloneNotSupportedException{ 71 | return super.clone(); 72 | } 73 | } -------------------------------------------------------------------------------- /parquetio/src/main/java/com/adobe/platform/ecosystem/examples/parquet/exception/ErrorCode.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.parquet.exception; 18 | 19 | /** 20 | * Base interface for defining 21 | * error codes. 22 | * 23 | * @author vedhera 18/01/2018 24 | */ 25 | public interface ErrorCode { 26 | /** 27 | * Interface for returning specifc code. 28 | * @return returns message associated with error code. 29 | */ 30 | String getErrorMessage(); 31 | } -------------------------------------------------------------------------------- /parquetio/src/main/java/com/adobe/platform/ecosystem/examples/parquet/exception/ParquetIOErrorCode.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.parquet.exception; 18 | 19 | /** 20 | * Error codes specific to ParquetIO library. 21 | * 22 | * @author vedhera 18/01/2018 23 | */ 24 | public enum ParquetIOErrorCode implements ErrorCode { 25 | PAQUETIO_READER_NOT_INITIALISED(101,"Reader not initialized for read. Initialize first."), 26 | PARQUETIO_IO_EXCEPTION(102,"IO Exception while reading group."), 27 | PARQUETIO_FILE_DELLETION_EXCEPTION(103,"Exception while deleting file on local system."), 28 | PARQUETIO_INPUT_CONFIGURATION_NULL(104,"Input configuration is null. Kindly initialize first."), 29 | PARQUETIO_READER_CLOSE_EXCEPTION(105,"Error while closing hadoop parquet reader."), 30 | PARQUETIO_READER_METADATA_NULL_EXCEPTION(106,"Metadata read from parquet file is null. Kindly check input file."); 31 | 32 | private final int number; 33 | 34 | private final String description; 35 | 36 | ParquetIOErrorCode(int number, String description) { 37 | this.number = number; 38 | this.description = description; 39 | } 40 | 41 | /** 42 | * {@inheritDoc} 43 | */ 44 | @Override 45 | public String getErrorMessage() { 46 | return "[Code: " 47 | + this.number 48 | + " message: " 49 | + this.description 50 | + "]"; 51 | } 52 | } -------------------------------------------------------------------------------- /parquetio/src/main/java/com/adobe/platform/ecosystem/examples/parquet/exception/ParquetIOException.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.parquet.exception; 18 | 19 | import java.io.PrintWriter; 20 | import java.util.HashMap; 21 | import java.util.Map; 22 | 23 | /** 24 | * Custom exception class for 25 | * ETL-Parquet IO library. 26 | * 27 | * @author vedhera 18/01/2018 28 | */ 29 | public class ParquetIOException extends Exception { 30 | private ErrorCode errorCode; 31 | 32 | private final Map properties = new HashMap<>(); 33 | 34 | public ParquetIOException(ErrorCode errorCode) { 35 | super(errorCode.getErrorMessage()); 36 | this.errorCode = errorCode; 37 | } 38 | 39 | public ParquetIOException(ErrorCode errorCode, Throwable cause) { 40 | super(cause); 41 | this.errorCode = errorCode; 42 | } 43 | 44 | @SuppressWarnings("unchecked") 45 | public T get(String name) { 46 | return (T)properties.get(name); 47 | } 48 | 49 | public ParquetIOException set(String name, Object value) { 50 | properties.put(name, value); 51 | return this; 52 | } 53 | 54 | private String formatErrorMessage() { 55 | StringBuilder builder = new StringBuilder(); 56 | builder.append(this.errorCode.getErrorMessage()); 57 | builder.append("\n"); 58 | 59 | for (String key : properties.keySet()) { 60 | builder.append("\t" + key + "=[" + properties.get(key) + "]"); 61 | } 62 | return builder.toString(); 63 | } 64 | 65 | @Override 66 | public void printStackTrace() { 67 | PrintWriter s = new PrintWriter(System.err); 68 | synchronized (s) { 69 | s.println(formatErrorMessage()); 70 | } 71 | s.flush(); 72 | super.printStackTrace(); 73 | } 74 | } -------------------------------------------------------------------------------- /parquetio/src/main/java/com/adobe/platform/ecosystem/examples/parquet/model/ParquetIOField.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.parquet.model; 18 | 19 | import java.util.List; 20 | 21 | /** 22 | * POJO to represent metadata which 23 | * can be later transformed to parquet 24 | * {@link org.apache.parquet.schema.MessageType} schema. 25 | * 26 | * @author vedhera on 2/8/2018 27 | */ 28 | public class ParquetIOField { 29 | private String name; 30 | 31 | private ParquetIODataType type; 32 | 33 | private ParquetIORepetitionType repetitionType; 34 | 35 | private List subFields; 36 | 37 | public ParquetIOField(String name, ParquetIODataType type, ParquetIORepetitionType repetitionType, List subFields) { 38 | this.name = name; 39 | this.type = type; 40 | this.repetitionType = repetitionType; 41 | this.subFields = subFields; 42 | } 43 | 44 | public String getName() { 45 | return name; 46 | } 47 | 48 | public ParquetIODataType getType() { 49 | return type; 50 | } 51 | 52 | public boolean isPrimitive() { 53 | return this.type != ParquetIODataType.GROUP 54 | && this.type != ParquetIODataType.LIST 55 | && this.type != ParquetIODataType.Map; 56 | } 57 | 58 | public boolean isMapType() { 59 | return this.type == ParquetIODataType.Map; 60 | } 61 | 62 | public boolean isListType() { 63 | return this.type == ParquetIODataType.LIST; 64 | } 65 | 66 | public boolean isRepetetive() { 67 | return this.repetitionType == ParquetIORepetitionType.REPEATED; 68 | } 69 | 70 | public boolean isRequired() { 71 | return this.repetitionType == ParquetIORepetitionType.REQUIRED; 72 | } 73 | 74 | public boolean isOptional() { 75 | return this.repetitionType == ParquetIORepetitionType.OPTIONAL; 76 | } 77 | 78 | public ParquetIORepetitionType getRepetitionType() { 79 | return repetitionType; 80 | } 81 | 82 | public List getSubFields() { 83 | return subFields; 84 | } 85 | } -------------------------------------------------------------------------------- /parquetio/src/main/java/com/adobe/platform/ecosystem/examples/parquet/model/ParquetIORepetitionType.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.parquet.model; 18 | 19 | import org.apache.parquet.schema.Type; 20 | 21 | /** 22 | * Enum to represent repetition 23 | * of any parquet field/group. 24 | * 25 | * @author vedhera on 2/9/2018. 26 | */ 27 | public enum ParquetIORepetitionType { 28 | OPTIONAL("optional"), 29 | REPEATED("repeated"), 30 | REQUIRED("required"); 31 | 32 | private final String repString; 33 | 34 | ParquetIORepetitionType(String repString) { 35 | this.repString = repString; 36 | } 37 | 38 | public static ParquetIORepetitionType fromRepetition(Type.Repetition repetition) { 39 | switch (repetition) { 40 | case OPTIONAL: 41 | return OPTIONAL; 42 | case REPEATED: 43 | return REPEATED; 44 | case REQUIRED: 45 | return REQUIRED; 46 | default: 47 | return OPTIONAL; 48 | } 49 | } 50 | } -------------------------------------------------------------------------------- /parquetio/src/main/java/com/adobe/platform/ecosystem/examples/parquet/read/ParquetIOReader.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.parquet.read; 18 | 19 | import com.adobe.platform.ecosystem.examples.parquet.exception.ParquetIOException; 20 | import com.adobe.platform.ecosystem.examples.parquet.model.ParquetIOField; 21 | import org.apache.hadoop.fs.Path; 22 | import org.json.simple.JSONObject; 23 | 24 | import java.io.File; 25 | import java.io.IOException; 26 | import java.util.Iterator; 27 | import java.util.List; 28 | 29 | /** 30 | * Created by vedhera on 24/10/2017. 31 | */ 32 | 33 | /** 34 | * Interface to expose API's 35 | * to read records from parquet file. 36 | */ 37 | public interface ParquetIOReader { 38 | /** 39 | * abstract method to process data and 40 | * return list of JsonObject with {@code rows} 41 | * or less records. 42 | * @param rows 43 | * @return 44 | */ 45 | List processData(int rows) throws ParquetIOException; 46 | 47 | /** 48 | * abstract method to check 49 | * whether records can be read 50 | * from an exisiting parquet file. 51 | * @return boolean 52 | */ 53 | boolean hasBufferData(); 54 | 55 | /** 56 | * abstract method to init 57 | * parquet reader for a file 58 | * on disk. 59 | * @param file 60 | */ 61 | void initFileForRead(File file) throws ParquetIOException; 62 | 63 | @Deprecated 64 | /** 65 | * abstract method to init 66 | * parquet reader for a 67 | * {@link org.apache.hadoop.fs.Path}. 68 | * 69 | * @param path path pointing to the file. 70 | * Can be both a remote or a 71 | * local path. 72 | */ 73 | ParquetIOReader initFileForRead(Path path) throws ParquetIOException; 74 | 75 | /** 76 | * abstract method to init 77 | * parquet reader for a 78 | * {@link org.apache.hadoop.fs.Path}. 79 | * 80 | */ 81 | ParquetIOReader initFileForRead() throws ParquetIOException; 82 | 83 | /** 84 | * abstract method to provide iterator 85 | * for reading records. 86 | * @return Iterator 87 | */ 88 | Iterator getIterator() throws ParquetIOException; 89 | 90 | /** 91 | * API to close the reader 92 | * which in turn closes on 93 | * the hadoop parquet reader 94 | */ 95 | void readerClose() throws ParquetIOException; 96 | 97 | /** 98 | * API to extract schema from the 99 | * parquet file. 100 | * 101 | * @return equivalent schema 102 | * for parquet file. 103 | * @throws ParquetIOException 104 | */ 105 | List getSchema() throws ParquetIOException; 106 | } -------------------------------------------------------------------------------- /parquetio/src/main/java/com/adobe/platform/ecosystem/examples/parquet/read/ParquetIOReaderIterator.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.parquet.read; 18 | 19 | import com.adobe.platform.ecosystem.examples.parquet.exception.ParquetIOErrorCode; 20 | import com.adobe.platform.ecosystem.examples.parquet.exception.ParquetIOException; 21 | import org.apache.parquet.example.data.Group; 22 | import org.apache.parquet.hadoop.ParquetReader; 23 | import org.json.simple.JSONObject; 24 | 25 | import java.io.IOException; 26 | import java.util.Iterator; 27 | import java.util.NoSuchElementException; 28 | import java.util.logging.Logger; 29 | 30 | /** 31 | * Created by vedhera on 30/11/2017. 32 | */ 33 | 34 | /** 35 | * Iterable class to read records iteratively from 36 | * Parquet record reader. 37 | * @param 38 | */ 39 | public class ParquetIOReaderIterator implements Iterable { 40 | private T readingGroup; 41 | private final ParquetReader reader; 42 | private final boolean doFlatten; 43 | private final static Logger LOGGER = Logger.getLogger(ParquetIOReaderIterator.class.getName()); 44 | 45 | ParquetIOReaderIterator(ParquetReader reader, boolean doFlatten) throws ParquetIOException { 46 | if(reader == null) { 47 | throw new ParquetIOException(ParquetIOErrorCode.PAQUETIO_READER_NOT_INITIALISED); 48 | } 49 | this.reader = reader; 50 | this.doFlatten = doFlatten; 51 | try { 52 | this.readingGroup = reader.read(); 53 | } catch (IOException ex) { 54 | throw new ParquetIOException(ParquetIOErrorCode.PARQUETIO_IO_EXCEPTION); 55 | } 56 | } 57 | 58 | @Override 59 | public Iterator iterator() { 60 | return new Iterator() { 61 | @Override 62 | public boolean hasNext() { 63 | return readingGroup != null; 64 | } 65 | 66 | @Override 67 | public JSONObject next() { 68 | T tempGroup; 69 | JSONObject value; 70 | try { 71 | tempGroup = readingGroup; 72 | readingGroup = reader.read(); 73 | value = ReaderUtil.getJSONDataFromGroup(tempGroup,doFlatten); 74 | } catch (IOException e) { 75 | LOGGER.severe("Error occurred while reading next record from iterator: " + e); 76 | throw new NoSuchElementException("Error occurred while reading next record from iterator: " + e); 77 | } 78 | return value; 79 | } 80 | }; 81 | } 82 | } -------------------------------------------------------------------------------- /parquetio/src/main/java/com/adobe/platform/ecosystem/examples/parquet/read/configuration/ParquetReaderConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017-2018 Adobe. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.adobe.platform.ecosystem.examples.parquet.read.configuration; 17 | 18 | import org.apache.hadoop.conf.Configuration; 19 | import org.apache.hadoop.fs.Path; 20 | 21 | import java.util.function.Consumer; 22 | 23 | /** 24 | * POJO to encapsulate members required 25 | * for initializing {@link com.adobe.platform.ecosystem.examples.parquet.read.ParquetIOReader} 26 | * 27 | * @author vedhera on 5/14/2018. 28 | */ 29 | public class ParquetReaderConfiguration { 30 | 31 | private final Path path; 32 | 33 | private final Configuration configuration; 34 | 35 | private final boolean shouldFlattenData; 36 | 37 | private ParquetReaderConfiguration(Path path, Configuration configuration, boolean shouldFlattenData) { 38 | this.path = path; 39 | this.configuration = configuration; 40 | this.shouldFlattenData = shouldFlattenData; 41 | } 42 | 43 | public static Builder builder() { 44 | return new Builder(); 45 | } 46 | 47 | public Path getPath() { 48 | return path; 49 | } 50 | 51 | public Configuration getConfiguration() { 52 | return configuration; 53 | } 54 | 55 | public boolean getShouldFlattenData() { 56 | return shouldFlattenData; 57 | } 58 | 59 | /** 60 | * Fluent builder of type 61 | * {@link ParquetReaderConfiguration} 62 | */ 63 | public static class Builder { 64 | public Path path; 65 | 66 | public Configuration configuration; 67 | 68 | public boolean shouldFlattenData; 69 | 70 | public Builder with(Consumer builderConsumer) { 71 | builderConsumer.accept(this); 72 | return this; 73 | } 74 | 75 | public ParquetReaderConfiguration build() { 76 | return new ParquetReaderConfiguration(path, configuration, shouldFlattenData); 77 | } 78 | 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /parquetio/src/main/java/com/adobe/platform/ecosystem/examples/parquet/utility/ParquetIOUtil.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.parquet.utility; 18 | 19 | import org.apache.parquet.io.api.Binary; 20 | 21 | import java.io.File; 22 | import java.io.IOException; 23 | import java.util.Random; 24 | 25 | public class ParquetIOUtil { 26 | 27 | private final static long JULIAN_DAY_OF_EPOCH = 2440588; 28 | 29 | public static String generateRandomText() { 30 | String SALTCHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"; 31 | StringBuilder salt = new StringBuilder(); 32 | Random rnd = new Random(); 33 | while (salt.length() < 18) { // length of the random string. 34 | int index = (int) (rnd.nextFloat() * SALTCHARS.length()); 35 | salt.append(SALTCHARS.charAt(index)); 36 | } 37 | String saltStr = salt.toString(); 38 | return saltStr; 39 | } 40 | 41 | public static long dateFromInt96( Binary value ) { 42 | byte[] readBuffer = value.getBytes(); 43 | if ( readBuffer.length != 12 ) { 44 | throw new RuntimeException( "Invalid byte array length for INT96" ); 45 | } 46 | 47 | long timeOfDayNanos = 48 | ( ( (long) readBuffer[7] << 56 ) + ( (long) ( readBuffer[6] & 255 ) << 48 ) 49 | + ( (long) ( readBuffer[5] & 255 ) << 40 ) + ( (long) ( readBuffer[4] & 255 ) << 32 ) 50 | + ( (long) ( readBuffer[3] & 255 ) << 24 ) + ( ( readBuffer[2] & 255 ) << 16 ) 51 | + ( ( readBuffer[1] & 255 ) << 8 ) + ( ( readBuffer[0] & 255 ) << 0 ) ); 52 | 53 | int julianDay = 54 | ( (int) ( readBuffer[11] & 255 ) << 24 ) + ( ( readBuffer[10] & 255 ) << 16 ) 55 | + ( ( readBuffer[9] & 255 ) << 8 ) + ( ( readBuffer[8] & 255 ) << 0 ); 56 | 57 | return ( julianDay - JULIAN_DAY_OF_EPOCH ) * 24L * 60L * 60L * 1000L + timeOfDayNanos / 1000000; 58 | } 59 | 60 | public static File getLocalFilePath(String fileName) { 61 | String suffix = fileName.endsWith(".parquet") ? "" : ".parquet"; 62 | File tempFile = null; 63 | try { 64 | tempFile = File.createTempFile(fileName, suffix); 65 | if (!tempFile.delete()) { 66 | throw new RuntimeException("Unable to delete the local temp file for logs"); 67 | } 68 | } catch (IOException e) { 69 | throw new RuntimeException("Caught IO exception for creating temp file. " + e); 70 | } 71 | return tempFile; 72 | } 73 | } -------------------------------------------------------------------------------- /parquetio/src/main/java/com/adobe/platform/ecosystem/examples/parquet/wiring/api/ParquetIO.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.parquet.wiring.api; 18 | 19 | import com.adobe.platform.ecosystem.examples.parquet.exception.ParquetIOException; 20 | import com.adobe.platform.ecosystem.examples.parquet.read.ParquetIOReader; 21 | import com.adobe.platform.ecosystem.examples.parquet.read.configuration.ParquetReaderConfiguration; 22 | import com.adobe.platform.ecosystem.examples.parquet.write.ParquetIOWriter; 23 | 24 | /** 25 | * Created by vardgupt on 10/10/2017. 26 | */ 27 | 28 | /** 29 | * Interfaces for parquet 30 | * reader/writers. 31 | */ 32 | public interface ParquetIO { 33 | /** 34 | * Interface to provide reader for 35 | * performing parquet read operations. 36 | * 37 | * @param doFlatten to flatten the data after reading. 38 | * @return {@code ParquetIOReader} instance 39 | */ 40 | ParquetIOReader getParquetIOReader(boolean doFlatten); 41 | 42 | /** 43 | * Interface to provide reader for 44 | * performing parquet read operations. 45 | * 46 | * @param readerConfiguration 47 | * @return {@code ParquetIOReader} instance 48 | */ 49 | ParquetIOReader getParquetIOReader(ParquetReaderConfiguration readerConfiguration) throws ParquetIOException; 50 | 51 | /** 52 | * Interface to provide writer for 53 | * performing parquet writer operations. 54 | * 55 | * @return 56 | */ 57 | ParquetIOWriter getParquetIOWriter(); 58 | } -------------------------------------------------------------------------------- /parquetio/src/main/java/com/adobe/platform/ecosystem/examples/parquet/wiring/impl/ParquetIOImpl.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.parquet.wiring.impl; 18 | 19 | import com.adobe.platform.ecosystem.examples.parquet.exception.ParquetIOErrorCode; 20 | import com.adobe.platform.ecosystem.examples.parquet.exception.ParquetIOException; 21 | import com.adobe.platform.ecosystem.examples.parquet.read.ParquetIOReader; 22 | import com.adobe.platform.ecosystem.examples.parquet.read.ParquetIOReaderImpl; 23 | import com.adobe.platform.ecosystem.examples.parquet.read.configuration.ParquetReaderConfiguration; 24 | import com.adobe.platform.ecosystem.examples.parquet.wiring.api.ParquetIO; 25 | import com.adobe.platform.ecosystem.examples.parquet.write.ParquetIOWriter; 26 | import com.adobe.platform.ecosystem.examples.parquet.write.ParquetIOWriterImpl; 27 | import org.apache.hadoop.conf.Configuration; 28 | 29 | 30 | /** 31 | * Created by vedhera on 10/10/2017. 32 | */ 33 | public class ParquetIOImpl implements ParquetIO { 34 | 35 | /** 36 | * {@inheritDoc} 37 | */ 38 | @Override 39 | public ParquetIOReader getParquetIOReader(boolean doFlatten) { 40 | Configuration configuration = new Configuration(); 41 | return new ParquetIOReaderImpl.ParquetIOReaderBuilder() 42 | .with(readerBuilder -> { 43 | readerBuilder.conf = configuration; 44 | readerBuilder.doFlatten = doFlatten; 45 | }) 46 | .build(); 47 | } 48 | 49 | /** 50 | * {@inheritDoc} 51 | */ 52 | @Override 53 | public ParquetIOReader getParquetIOReader(ParquetReaderConfiguration readerConfiguration) throws ParquetIOException { 54 | if(readerConfiguration == null) { 55 | throw new ParquetIOException(ParquetIOErrorCode.PARQUETIO_INPUT_CONFIGURATION_NULL); 56 | } 57 | 58 | ParquetIOReader reader = new ParquetIOReaderImpl.ParquetIOReaderBuilder() 59 | .with(readerBuilder -> { 60 | readerBuilder.conf = readerConfiguration.getConfiguration(); 61 | readerBuilder.doFlatten = readerConfiguration.getShouldFlattenData(); 62 | readerBuilder.path = readerConfiguration.getPath(); 63 | }) 64 | .build(); 65 | reader.initFileForRead(); 66 | return reader; 67 | } 68 | 69 | /** 70 | * {@inheritDoc} 71 | */ 72 | @Override 73 | public ParquetIOWriter getParquetIOWriter() { 74 | return new ParquetIOWriterImpl(); 75 | } 76 | } -------------------------------------------------------------------------------- /parquetio/src/main/java/com/adobe/platform/ecosystem/examples/parquet/write/ParquetIOWriter.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.parquet.write; 18 | 19 | 20 | /** 21 | * Created by vardgupt on 10/10/2017. 22 | */ 23 | 24 | import com.adobe.platform.ecosystem.examples.parquet.exception.ParquetIOException; 25 | import com.adobe.platform.ecosystem.examples.parquet.model.ParquetIOField; 26 | import org.apache.parquet.example.data.simple.SimpleGroup; 27 | import org.apache.parquet.schema.MessageType; 28 | 29 | import java.io.File; 30 | import java.io.IOException; 31 | import java.util.List; 32 | import java.util.Map; 33 | 34 | /** 35 | * Interface to expose API for writing 36 | * to parquet format. 37 | */ 38 | public interface ParquetIOWriter { 39 | /** 40 | * Interface to construct parquet schema from 41 | * parquet-IO fields. 42 | * @param fields 43 | * @return Parquet's message type. 44 | */ 45 | MessageType getSchema(List fields); 46 | 47 | /** 48 | * 49 | * @param columnToTypeMap 50 | * @param delimiter 51 | * @return 52 | */ 53 | MessageType getSchema(Map columnToTypeMap, String delimiter); 54 | 55 | 56 | /** 57 | * @param schema 58 | * @param fileName 59 | * @param noOfRecords 60 | * @return 61 | * @throws IOException 62 | */ 63 | File writeSampleParquetFile(MessageType schema, String fileName, int noOfRecords) throws ParquetIOException; 64 | 65 | 66 | /** 67 | * @param schema 68 | * @param fileName 69 | * @return 70 | * @throws IOException 71 | */ 72 | File writeParquetFile(MessageType schema, String fileName,List records) throws ParquetIOException; 73 | } -------------------------------------------------------------------------------- /parquetio/src/test/java/com/adobe/platform/ecosystem/examples/parquet/read/ParquetIOReaderIteratorTest.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.parquet.read; 18 | 19 | import org.junit.Test; 20 | 21 | /** 22 | * Created by vedhera on 30/11/2017. 23 | */ 24 | 25 | public class ParquetIOReaderIteratorTest { 26 | 27 | @Test(expected = Exception.class) 28 | public void testConstructor() throws Exception { 29 | new ParquetIOReaderIterator(null, false); 30 | } 31 | } -------------------------------------------------------------------------------- /parquetio/src/test/java/com/adobe/platform/ecosystem/examples/parquet/ut/BaseTest.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.parquet.ut; 18 | 19 | import java.util.LinkedHashMap; 20 | 21 | import com.adobe.platform.ecosystem.examples.parquet.wiring.api.ParquetIO; 22 | import com.adobe.platform.ecosystem.examples.parquet.wiring.impl.ParquetIOImpl; 23 | import com.adobe.platform.ecosystem.examples.parquet.write.ParquetIOWriter; 24 | import com.adobe.platform.ecosystem.examples.parquet.write.ParquetIOWriterImpl; 25 | 26 | 27 | public class BaseTest { 28 | public ParquetIOWriter writer = new ParquetIOWriterImpl(); 29 | public String rootNodeName = "root"; 30 | public String defaultDataType = "group"; 31 | public int noOfRecords = 10; 32 | public String sampleParquetFileName = "unit_test"; 33 | public String delimiter = "_"; 34 | 35 | 36 | public LinkedHashMap setupHierarchicalMap(){ 37 | LinkedHashMap map = new LinkedHashMap(); 38 | map.put("A_B", "binary"); 39 | map.put("A_C", "binary"); 40 | map.put("A_D", "binary"); 41 | map.put("A_E", "binary"); 42 | map.put("A_F", "binary"); 43 | map.put("G", "binary"); 44 | 45 | return map; 46 | } 47 | 48 | public LinkedHashMap setupFlatMap(){ 49 | LinkedHashMap map = new LinkedHashMap(); 50 | map.put("A", "binary"); 51 | map.put("B", "binary"); 52 | map.put("C", "binary"); 53 | map.put("D", "binary"); 54 | map.put("E", "binary"); 55 | map.put("F", "binary"); 56 | 57 | return map; 58 | } 59 | 60 | public String[] getFlattenArray(){ 61 | String[] flattenedArray = {"A_B","A_C","A_X_Y"}; 62 | return flattenedArray; 63 | } 64 | } -------------------------------------------------------------------------------- /parquetio/src/test/java/com/adobe/platform/ecosystem/examples/parquet/wiring/ParquetIOImplTest.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2017-2018 Adobe. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.adobe.platform.ecosystem.examples.parquet.wiring; 18 | 19 | import com.adobe.platform.ecosystem.examples.parquet.read.ParquetIOReader; 20 | import com.adobe.platform.ecosystem.examples.parquet.wiring.impl.ParquetIOImpl; 21 | import com.adobe.platform.ecosystem.examples.parquet.write.ParquetIOWriter; 22 | import org.junit.Test; 23 | 24 | import static org.junit.Assert.assertTrue; 25 | 26 | /** 27 | * Created by vedhera on 25/10/2017. 28 | */ 29 | public class ParquetIOImplTest { 30 | private ParquetIOImpl pio = new ParquetIOImpl(); 31 | 32 | @Test 33 | public void getReaderTest(){ 34 | assertTrue(pio.getParquetIOReader(false) != null); 35 | assertTrue(pio.getParquetIOReader(false) instanceof ParquetIOReader); 36 | } 37 | 38 | @Test 39 | public void getWriterTest(){ 40 | assertTrue(pio.getParquetIOWriter() != null); 41 | assertTrue(pio.getParquetIOWriter() instanceof ParquetIOWriter); 42 | } 43 | } -------------------------------------------------------------------------------- /parquetio/src/test/resources/allDataTypes.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/experience-platform-etl-reference/e7040e9fcbc002dc4ee75b75f4e81324f238e0b0/parquetio/src/test/resources/allDataTypes.parquet -------------------------------------------------------------------------------- /parquetio/src/test/resources/int96file.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/experience-platform-etl-reference/e7040e9fcbc002dc4ee75b75f4e81324f238e0b0/parquetio/src/test/resources/int96file.parquet -------------------------------------------------------------------------------- /parquetio/src/test/resources/test_snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/experience-platform-etl-reference/e7040e9fcbc002dc4ee75b75f4e81324f238e0b0/parquetio/src/test/resources/test_snappy.parquet -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.adobe.platform 8 | ecosystem-refs 9 | 1.0-SNAPSHOT 10 | pom 11 | 12 | 13 | parquetio 14 | examples 15 | 16 | 17 | 18 | --------------------------------------------------------------------------------