├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── build.sbt ├── project └── plugins.sbt └── src ├── main ├── resources │ └── log4j.properties └── scala │ └── com │ └── datawizards │ └── dmg │ ├── DataModelGenerator.scala │ ├── annotations │ ├── es │ │ └── package.scala │ ├── hive │ │ └── package.scala │ └── package.scala │ ├── dialects │ ├── AvroSchemaDialect.scala │ ├── AvroSchemaRegistryDialect.scala │ ├── DatabaseDialect.scala │ ├── DecoratorDialect.scala │ ├── Dialect.scala │ ├── ElasticsearchDialect.scala │ ├── H2Dialect.scala │ ├── HiveDialect.scala │ ├── JavaDialect.scala │ ├── MetaDataWithDialectExtractor.scala │ ├── MySQLDialect.scala │ └── RedshiftDialect.scala │ ├── examples │ ├── ArrayTypeExample.scala │ ├── AvroSchemaExample.scala │ ├── CopyAvroSchemaToHDFS.scala │ ├── CreateElasticsearchIndex.scala │ ├── CreateElasticsearchTemplate.scala │ ├── CreateHiveTableBatchExample.scala │ ├── CreateHiveTableExample.scala │ ├── CustomTableColumnNameExample.scala │ ├── ElasticsearchExample.scala │ ├── H2Example.scala │ ├── HiveExample.scala │ ├── MultipleCustomNames.scala │ ├── MySQLExample.scala │ ├── RedshiftExample.scala │ ├── RegisterAvroSchema.scala │ ├── StructTypesExample.scala │ ├── TableColumnCommentsExample.scala │ └── TestModel.scala │ ├── metadata │ ├── MetaDataExtractor.scala │ └── package.scala │ ├── repository │ ├── AvroSchemaRegistryRepository.scala │ └── AvroSchemaRegistryRepositoryImpl.scala │ └── service │ ├── AvroSchemaRegistryService.scala │ ├── AvroSchemaRegistryServiceImpl.scala │ ├── ConsoleCommandExecutor.scala │ ├── ElasticsearchService.scala │ ├── ElasticsearchServiceImpl.scala │ ├── HDFSService.scala │ ├── HDFSServiceImpl.scala │ ├── HiveService.scala │ ├── HiveServiceImpl.scala │ └── TemplateHandler.scala └── test └── scala └── com └── datawizards └── dmg ├── DataModelGeneratorBaseTest.scala ├── ExtractClassMetaDataForDialectTest.scala ├── TestModel.scala ├── customizations ├── MultipleCustomNamesTest.scala ├── NullableTest.scala ├── PlaceholdersTest.scala └── UnderscoreConversionTest.scala ├── dialects ├── GenerateAvroSchemaRegistryTest.scala ├── GenerateAvroSchemaTest.scala ├── GenerateElasticsearchMappingTest.scala ├── GenerateH2ModelTest.scala ├── GenerateHiveModelTest.scala ├── GenerateJavaClassTest.scala ├── GenerateMySQLModelTest.scala └── GenerateRedshiftModelTest.scala ├── metadata ├── CV.scala ├── Company.scala ├── MetaDataExtractorTest.scala ├── Person.scala ├── PersonFull.scala ├── PersonPartitioned.scala ├── WorkExperience.scala └── anotherAnnotation.scala └── service ├── AvroSchemaGenerateTest.scala ├── CreateElasticsearchIndexTest.scala ├── HiveTableBatchCreateTest.scala └── RegisterAvroSchemaTest.scala /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.log 3 | 4 | target 5 | .idea 6 | temp -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | 3 | scala: 4 | 2.11.8 5 | 6 | jdk: 7 | - oraclejdk8 8 | 9 | script: 10 | - sbt clean coverage test coverageReport 11 | 12 | after_success: 13 | - bash <(curl -s https://codecov.io/bash) -t 33d595f5-40e3-48e3-b7c9-0b6bac2ef0c9 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | name := "data-model-generator" 2 | 3 | organization := "com.github.piotr-kalanski" 4 | 5 | version := "0.7.7" 6 | 7 | scalaVersion := "2.11.8" 8 | 9 | licenses := Seq("Apache License, Version 2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0.txt")) 10 | 11 | homepage := Some(url("https://github.com/piotr-kalanski/data-model-generator")) 12 | 13 | scmInfo := Some( 14 | ScmInfo( 15 | url("https://github.com/piotr-kalanski/data-model-generator"), 16 | "scm:git:ssh://github.com/piotr-kalanski/data-model-generator.git" 17 | ) 18 | ) 19 | 20 | developers := List( 21 | Developer( 22 | id = "kalan", 23 | name = "Piotr Kalanski", 24 | email = "piotr.kalanski@gmail.com", 25 | url = url("https://github.com/piotr-kalanski") 26 | ) 27 | ) 28 | 29 | libraryDependencies ++= Seq( 30 | "org.scala-lang" % "scala-reflect" % "2.11.8", 31 | "log4j" % "log4j" % "1.2.17", 32 | "com.github.piotr-kalanski" %% "es-client" % "0.2.1", 33 | "org.scalatest" %% "scalatest" % "2.2.6" % "test", 34 | "junit" % "junit" % "4.10" % "test" 35 | ) 36 | 37 | coverageExcludedPackages := "com.datawizards.dmg.examples.*" 38 | 39 | publishMavenStyle := true 40 | 41 | publishTo := { 42 | val nexus = "https://oss.sonatype.org/" 43 | if (isSnapshot.value) 44 | Some("snapshots" at nexus + "content/repositories/snapshots") 45 | else 46 | Some("releases" at nexus + "service/local/staging/deploy/maven2") 47 | } 48 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | logLevel := Level.Warn 2 | 3 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.0") 4 | 5 | addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "1.1") 6 | -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, file, stdout_error 3 | log4j.org.apache.spark=INFO, file, stdout_error 4 | log4j.logger.com.datawizards=INFO, file, stdout_info 5 | 6 | # Direct log messages to a log file 7 | log4j.appender.file=org.apache.log4j.RollingFileAppender 8 | log4j.appender.file.File=data-model-generator.log 9 | log4j.appender.file.MaxFileSize=10MB 10 | log4j.appender.file.MaxBackupIndex=10 11 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 12 | log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n 13 | 14 | # Direct log messages to stdout 15 | log4j.appender.stdout_error=org.apache.log4j.ConsoleAppender 16 | log4j.appender.stdout_error.Target=System.out 17 | log4j.appender.stdout_error.layout=org.apache.log4j.PatternLayout 18 | log4j.appender.stdout_error.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n 19 | log4j.appender.stdout_error.Threshold=ERROR 20 | 21 | # Direct log messages to stdout 22 | log4j.appender.stdout_info=org.apache.log4j.ConsoleAppender 23 | log4j.appender.stdout_info.Target=System.out 24 | log4j.appender.stdout_info.layout=org.apache.log4j.PatternLayout 25 | log4j.appender.stdout_info.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %20c{1}:%-8L %m%n -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/DataModelGenerator.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg 2 | 3 | import com.datawizards.dmg.dialects.{Dialect, MetaDataWithDialectExtractor} 4 | import com.datawizards.dmg.metadata._ 5 | import org.apache.log4j.Logger 6 | 7 | import scala.reflect.ClassTag 8 | import scala.reflect.runtime.universe.TypeTag 9 | 10 | object DataModelGenerator { 11 | 12 | private val log = Logger.getLogger(getClass.getName) 13 | 14 | /** 15 | * Generate data model for provided class 16 | * 17 | * @param dialect DB dialect e.g. H2, Hive, Redshift, Avro schema, Elasticsearch mapping 18 | * @tparam T type for which generate data model 19 | */ 20 | def generate[T: ClassTag: TypeTag](dialect: Dialect): String = 21 | generate(dialect, getClassMetaData[T](dialect)) 22 | 23 | /** 24 | * Generate data model for provided class 25 | * 26 | * @param dialect DB dialect e.g. H2, Hive, Redshift, Avro schema, Elasticsearch mapping 27 | * @param classTypeMetaData extracted class metadata 28 | * @tparam T type for which generate data model 29 | */ 30 | def generate[T: ClassTag: TypeTag](dialect: Dialect, classTypeMetaData: ClassTypeMetaData): String = { 31 | val ct = implicitly[ClassTag[T]].runtimeClass 32 | log.info(s"Generating model for class: [${ct.getName}], dialect: [$dialect]") 33 | generateDataModel(dialect, classTypeMetaData) 34 | } 35 | 36 | private def getClassMetaData[T: ClassTag: TypeTag](dialect: Dialect): ClassTypeMetaData = 37 | MetaDataWithDialectExtractor.extractClassMetaDataForDialect[T](Some(dialect)) 38 | 39 | private def generateDataModel(dialect: Dialect, classTypeMetaData: ClassTypeMetaData): String = { 40 | dialect.generateDataModel(classTypeMetaData, generateFieldsExpressions(dialect, classTypeMetaData)) 41 | } 42 | 43 | private def generateFieldsExpressions(dialect: Dialect, classTypeMetaData: ClassTypeMetaData): Iterable[String] = { 44 | classTypeMetaData 45 | .fields 46 | .withFilter(f => dialect.generateColumn(f)) 47 | .map(f => dialect.generateClassFieldExpression(f)) 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/annotations/es/package.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.annotations 2 | 3 | import scala.annotation.StaticAnnotation 4 | 5 | /** 6 | * Elasticsearch mapping dedicated annotations 7 | */ 8 | package object es { 9 | 10 | /** 11 | * The index option controls whether field values are indexed. 12 | * [[https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-index.html]] 13 | * 14 | */ 15 | final class esIndex(value: String) extends StaticAnnotation 16 | 17 | /** 18 | * Date format 19 | * [[https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html]] 20 | * 21 | */ 22 | final class esFormat(value: String) extends StaticAnnotation 23 | 24 | /** 25 | * Index settings 26 | * [[https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html]] 27 | * 28 | */ 29 | final class esSetting(key: String, value: Any) extends StaticAnnotation 30 | 31 | /** 32 | * Elasticsearch template 33 | * [[https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html]] 34 | * 35 | */ 36 | final class esTemplate(value: String) extends StaticAnnotation 37 | } 38 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/annotations/hive/package.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.annotations 2 | 3 | import scala.annotation.StaticAnnotation 4 | 5 | /** 6 | * HIVE DDL dedicated annotations 7 | */ 8 | package object hive { 9 | 10 | /** 11 | * Generated Hive DLL should be EXTERNAL table 12 | * 13 | * @param location location of data 14 | */ 15 | final class hiveExternalTable(val location: String) extends StaticAnnotation 16 | 17 | /** 18 | * STORED AS X clause 19 | * 20 | * @param format format of data e.g. PARQUET 21 | */ 22 | final class hiveStoredAs(val format: String) extends StaticAnnotation 23 | 24 | /** 25 | * ROW FORMAT SERDE X clause 26 | * 27 | * @param format row format of data e.g. org.apache.hadoop.hive.serde2.avro.AvroSerDe, org.apache.hive.hcatalog.data.JsonSerDe 28 | */ 29 | final class hiveRowFormatSerde(val format: String) extends StaticAnnotation 30 | 31 | /** 32 | * Hive TBLPROPERTIES section 33 | * 34 | * @param key key of table property 35 | * @param value value of table property 36 | */ 37 | final class hiveTableProperty(val key: String, val value: String) extends StaticAnnotation 38 | 39 | /** 40 | * Marking column as Hive partition 41 | * 42 | * @param order order of partitioning, without providing order value partitioning order is the same as order of class fields 43 | */ 44 | final class hivePartitionColumn(val order: Int=0) extends StaticAnnotation 45 | } 46 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/annotations/package.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg 2 | 3 | import com.datawizards.dmg.dialects.Dialect 4 | 5 | import scala.annotation.StaticAnnotation 6 | 7 | package object annotations { 8 | 9 | /** 10 | * @param name custom table name 11 | * @param dialect if provided then custom name only for this dialect 12 | */ 13 | final class table(val name: String, val dialect: Dialect) extends StaticAnnotation { 14 | def this(name: String) = this(name, null) 15 | } 16 | 17 | /** 18 | * @param name custom column name 19 | * @param dialect if provided then custom name only for this dialect 20 | */ 21 | final class column(val name: String, val dialect: Dialect) extends StaticAnnotation { 22 | def this(name: String) = this(name, null) 23 | } 24 | 25 | /** 26 | * @param value documentation comment 27 | */ 28 | final class comment(val value: String, val dialect: Dialect) extends StaticAnnotation { 29 | def this(value: String) = this(value, null) 30 | } 31 | 32 | /** 33 | * @param value column length 34 | */ 35 | final class length(val value: Int, val dialect: Dialect) extends StaticAnnotation { 36 | def this(value: Int) = this(value, null) 37 | } 38 | 39 | /** 40 | * Not null field 41 | */ 42 | final class notNull(val dialect: Dialect) extends StaticAnnotation { 43 | def this() = this(null) 44 | } 45 | 46 | /** 47 | * Convert table name and column names to underscore. 48 | * 49 | * Example: personName -> person_name 50 | * 51 | * @param dialect Dialect for which make conversion 52 | */ 53 | final class underscore(val dialect: Dialect) extends StaticAnnotation { 54 | def this() = this(null) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/dialects/AvroSchemaDialect.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | import com.datawizards.dmg.metadata._ 4 | 5 | object AvroSchemaDialect extends Dialect { 6 | 7 | override def intType: String = "int" 8 | 9 | override def stringType: String = "string" 10 | 11 | override def longType: String = "long" 12 | 13 | override def doubleType: String = "double" 14 | 15 | override def floatType: String = "float" 16 | 17 | override def shortType: String = "bytes" 18 | 19 | override def booleanType: String = "boolean" 20 | 21 | override def byteType: String = "bytes" 22 | 23 | override def dateType: String = "int" 24 | 25 | override def timestampType: String = "long" 26 | 27 | override def binaryType: String = generateArrayTypeExpression(generatePrimitiveTypeExpression(ByteType)) 28 | 29 | override def bigDecimalType: String = "double" 30 | 31 | override def bigIntegerType: String = "double" 32 | 33 | override def generatePrimitiveTypeExpression(p: PrimitiveTypeMetaData): String = 34 | s""""${mapPrimitiveDataType(p)}"""" 35 | 36 | override def generateArrayTypeExpression(elementTypeExpression: String): String = 37 | s""""array", "items": $elementTypeExpression""" 38 | 39 | override def generateClassTypeExpression(classTypeMetaData: ClassTypeMetaData, fieldNamesWithExpressions: Iterable[(String, String)]): String = 40 | s""""record", "fields": [${fieldNamesWithExpressions.map{case (k,v) => s"""{"name": "$k", "type": $v}"""}.mkString(", ")}]""" 41 | 42 | 43 | override def generateMapTypeExpression(keyExpression: String, valueExpression: String): String = 44 | s""""map", "values": $valueExpression""" 45 | 46 | override def toString: String = "AvroSchemaDialect" 47 | 48 | override def generateDataModel(classTypeMetaData: ClassTypeMetaData, fieldsExpressions: Iterable[String]): String = { 49 | val fieldsExpression = fieldsExpressions.mkString(",\n ") 50 | val tableDoc = 51 | if (comment(classTypeMetaData).isEmpty) "" 52 | else 53 | s""" 54 | | "doc": "${comment(classTypeMetaData).get}",""".stripMargin 55 | 56 | s"""{ 57 | | "namespace": "${classTypeMetaData.packageName}", 58 | | "type": "record", 59 | | "name": "${classTypeMetaData.typeName}",$tableDoc 60 | | "fields": [ 61 | | $fieldsExpression 62 | | ] 63 | |}""".stripMargin 64 | } 65 | 66 | override def generateClassFieldExpression(f: ClassFieldMetaData, typeExpression: String, level: Int): String = 67 | s"""{"name": "${f.fieldName}", """ + 68 | s""""type": ${getAvroTypeName(f)}""" + 69 | (f.fieldType match { 70 | case a:CollectionTypeMetaData => s""", "items": ${getArrayItemsType(a.elementType)}""" 71 | case s:ClassTypeMetaData => s""", "fields": [${s.fields.map(f => s"""{"name": "${f.fieldName}", "type": ${generateTypeExpression(f.fieldType)}}""").mkString(", ")}]""" 72 | case m:MapTypeMetaData => s""", "values": ${generateTypeExpression(m.valueType)}""" 73 | case _ => "" 74 | }) + 75 | s"""${if(comment(f).isEmpty) "" else s""", "doc": "${comment(f).get}""""}}""" 76 | 77 | private def getArrayItemsType(typeMetaData: TypeMetaData): String = typeMetaData match { 78 | case p:PrimitiveTypeMetaData => s"""${generateTypeExpression(p)}""" 79 | case c:CollectionTypeMetaData => s"""{"type": "array", "items": ${getArrayItemsType(c.elementType)}}""" 80 | case c:ClassTypeMetaData => s"""{"type": "record", "fields": [${c.fields.map(f => s"""{"name": "${f.fieldName}", "type": ${generateTypeExpression(f.fieldType)}}""").mkString(", ")}]}""" 81 | case _ => throw new Exception("Not supported type: " + typeMetaData) 82 | } 83 | 84 | private def getAvroTypeName(f: ClassFieldMetaData): String = f.fieldType match { 85 | case p:PrimitiveTypeMetaData => 86 | if(p == BinaryType) binaryType 87 | else if(notNull(f)) s""""${mapPrimitiveDataType(p)}"""" 88 | else s"""["null", "${mapPrimitiveDataType(p)}"]""" 89 | case c:CollectionTypeMetaData => "\"array\"" 90 | case c:ClassTypeMetaData => "\"record\"" 91 | case m:MapTypeMetaData => "\"map\"" 92 | case _ => throw new Exception("Not supported type: " + f.fieldType) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/dialects/AvroSchemaRegistryDialect.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | object AvroSchemaRegistryDialect extends DecoratorDialect(AvroSchemaDialect) { 4 | protected def decorate(dataModel: String): String = 5 | s"""{"schema": 6 | |"${dataModel.replace("\"", "\\\"")}" 7 | |}""".stripMargin 8 | } 9 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/dialects/DatabaseDialect.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | import com.datawizards.dmg.metadata._ 4 | 5 | trait DatabaseDialect extends Dialect { 6 | 7 | override def generateDataModel(classTypeMetaData: ClassTypeMetaData, fieldsExpressions: Iterable[String]): String = 8 | createTableExpression(classTypeMetaData) + 9 | generateColumnsExpression(classTypeMetaData, fieldsExpressions) + 10 | additionalTableProperties(classTypeMetaData) + ";\n" + 11 | additionalTableExpressions(classTypeMetaData) 12 | 13 | protected def createTableExpression(classTypeMetaData: ClassTypeMetaData): String = 14 | s"CREATE TABLE ${classTypeMetaData.typeName}" 15 | 16 | protected def generateColumnsExpression(classTypeMetaData: ClassTypeMetaData, fieldsExpressions: Iterable[String]): String = 17 | "(\n " + fieldsExpressions.mkString(",\n ") + "\n)" 18 | 19 | override def generateClassFieldExpression(f: ClassFieldMetaData, typeExpression: String, level: Int): String = 20 | generateFieldName(f.fieldName) + " " + typeExpression + 21 | (if(fieldLength(f).isEmpty) "" else s"(${fieldLength(f).get})") + 22 | fieldAdditionalExpressions(f) 23 | 24 | protected def fieldAdditionalExpressions(f: ClassFieldMetaData): String 25 | 26 | protected def additionalTableProperties(classTypeMetaData: ClassTypeMetaData): String 27 | 28 | protected def additionalTableExpressions(classTypeMetaData: ClassTypeMetaData): String 29 | 30 | private def generateFieldName(columnName: String): String = 31 | if(reservedKeywords.contains(columnName.toUpperCase)) 32 | escapeColumnName(columnName) 33 | else if(specialCharacters.exists(columnName.contains(_))) 34 | escapeColumnName(columnName) 35 | else 36 | columnName 37 | 38 | protected val specialCharacters = Seq( 39 | " ", "\\", "!", "@", "#", "%", "^", "&", "*", "(", ")", "-", "+", "=", "[", "]", "{", "}", ";", ":", "'", "\"", ",", ".", "<", ">", "/", "?", "|", "~" 40 | ) 41 | 42 | protected def reservedKeywords: Seq[String] 43 | 44 | protected def escapeColumnName(columnName: String): String 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/dialects/DecoratorDialect.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | import com.datawizards.dmg.metadata._ 4 | 5 | abstract class DecoratorDialect(dialect: Dialect) extends Dialect { 6 | 7 | override def intType: String = dialect.intType 8 | 9 | override def stringType: String = dialect.stringType 10 | 11 | override def longType: String = dialect.longType 12 | 13 | override def doubleType: String = dialect.doubleType 14 | 15 | override def floatType: String = dialect.floatType 16 | 17 | override def shortType: String = dialect.shortType 18 | 19 | override def booleanType: String = dialect.booleanType 20 | 21 | override def byteType: String = dialect.byteType 22 | 23 | override def dateType: String = dialect.dateType 24 | 25 | override def timestampType: String = dialect.timestampType 26 | 27 | override def binaryType: String = dialect.binaryType 28 | 29 | override def bigDecimalType: String = dialect.bigDecimalType 30 | 31 | override def bigIntegerType: String = dialect.bigIntegerType 32 | 33 | override def generateClassFieldExpression(f: ClassFieldMetaData, typeExpression: String, level: Int): String = 34 | dialect.generateClassFieldExpression(f, typeExpression, level) 35 | 36 | override def generateDataModel(classTypeMetaData: ClassTypeMetaData, fieldsExpressions: Iterable[String]): String = 37 | decorate(dialect.generateDataModel(classTypeMetaData, fieldsExpressions)) 38 | 39 | protected def decorate(dataModel: String): String 40 | 41 | override def generateArrayTypeExpression(elementTypeExpression: String): String = 42 | dialect.generateArrayTypeExpression(elementTypeExpression) 43 | 44 | override def generateMapTypeExpression(keyExpression: String, valueExpression: String): String = 45 | dialect.generateMapTypeExpression(keyExpression, valueExpression) 46 | 47 | override def generateClassTypeExpression(classTypeMetaData: ClassTypeMetaData, fieldNamesWithExpressions: Iterable[(String, String)]): String = 48 | dialect.generateClassTypeExpression(classTypeMetaData, fieldNamesWithExpressions) 49 | } 50 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/dialects/Dialect.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | import com.datawizards.dmg.metadata._ 4 | import org.apache.log4j.Logger 5 | 6 | trait Dialect { 7 | protected val log:Logger = Logger.getLogger(getClass.getName) 8 | 9 | private val Length = "com.datawizards.dmg.annotations.length" 10 | private val Comment = "com.datawizards.dmg.annotations.comment" 11 | private val NotNull = "com.datawizards.dmg.annotations.notNull" 12 | 13 | def generateDataModel(classTypeMetaData: ClassTypeMetaData, fieldsExpressions: Iterable[String]): String 14 | 15 | def mapPrimitiveDataType(primitiveType: PrimitiveTypeMetaData): String = primitiveType match { 16 | case IntegerType => intType 17 | case StringType => stringType 18 | case LongType => longType 19 | case DoubleType => doubleType 20 | case FloatType => floatType 21 | case ShortType => shortType 22 | case BooleanType => booleanType 23 | case ByteType => byteType 24 | case DateType => dateType 25 | case TimestampType => timestampType 26 | case BinaryType => binaryType 27 | case BigDecimalType => bigDecimalType 28 | case BigIntegerType => bigIntegerType 29 | case _ => throw new Exception("Not supported type: " + primitiveType) 30 | } 31 | 32 | def intType: String 33 | def stringType: String 34 | def longType: String 35 | def doubleType: String 36 | def floatType: String 37 | def shortType: String 38 | def booleanType: String 39 | def byteType: String 40 | def dateType: String 41 | def timestampType: String 42 | def binaryType: String 43 | def bigDecimalType: String 44 | def bigIntegerType: String 45 | 46 | def fieldLength(f: ClassFieldMetaData): Option[String] = MetaDataWithDialectExtractor.getAnnotationForDialect(Length, Some(this), f).map(_.getValue) 47 | 48 | def comment(a: HasAnnotations): Option[String] = MetaDataWithDialectExtractor.getAnnotationForDialect(Comment, Some(this), a).map(_.getValue) 49 | 50 | def notNull(f: ClassFieldMetaData): Boolean = MetaDataWithDialectExtractor.getAnnotationForDialect(NotNull, Some(this), f).nonEmpty 51 | 52 | def generateClassFieldExpression(f: ClassFieldMetaData): String = 53 | generateClassFieldExpression(f, 0) 54 | 55 | def generateClassFieldExpression(f: ClassFieldMetaData, level: Int): String = { 56 | val typeExpression = generateTypeExpression(f.fieldType, level) 57 | generateClassFieldExpression(f, typeExpression, level) 58 | } 59 | 60 | def generateClassFieldExpression(f: ClassFieldMetaData, typeExpression: String): String = 61 | generateClassFieldExpression(f, typeExpression, 0) 62 | 63 | def generateClassFieldExpression(f: ClassFieldMetaData, typeExpression: String, level: Int): String 64 | 65 | def generateTypeExpression(typeMetaData: TypeMetaData): String = 66 | generateTypeExpression(typeMetaData, 0) 67 | 68 | def generateTypeExpression(typeMetaData: TypeMetaData, level: Int): String = typeMetaData match { 69 | case p:PrimitiveTypeMetaData => generatePrimitiveTypeExpression(p) 70 | case c:CollectionTypeMetaData => generateArrayTypeExpression(generateTypeExpression(c.elementType, level)) 71 | case m:MapTypeMetaData => generateMapTypeExpression(generateTypeExpression(m.keyType, level), generateTypeExpression(m.valueType, level)) 72 | case c:ClassTypeMetaData => generateClassTypeExpression(c, c.fields.map(f => (f.fieldName, generateClassFieldExpression(f, level+1)))) 73 | } 74 | 75 | def generatePrimitiveTypeExpression(p:PrimitiveTypeMetaData): String = 76 | mapPrimitiveDataType(p) 77 | 78 | def generateArrayTypeExpression(elementTypeExpression: String): String 79 | 80 | def generateMapTypeExpression(keyExpression: String, valueExpression: String): String 81 | 82 | def generateClassTypeExpression(classTypeMetaData: ClassTypeMetaData, fieldNamesWithExpressions: Iterable[(String, String)]): String 83 | 84 | def generateColumn(f: ClassFieldMetaData): Boolean = true 85 | } 86 | 87 | 88 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/dialects/ElasticsearchDialect.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | import com.datawizards.dmg.metadata._ 4 | 5 | object ElasticsearchDialect extends Dialect { 6 | 7 | override def intType: String = "integer" 8 | 9 | override def stringType: String = "string" 10 | 11 | override def longType: String = "long" 12 | 13 | override def doubleType: String = "double" 14 | 15 | override def floatType: String = "float" 16 | 17 | override def shortType: String = "short" 18 | 19 | override def booleanType: String = "boolean" 20 | 21 | override def byteType: String = "byte" 22 | 23 | override def dateType: String = "date" 24 | 25 | override def timestampType: String = "date" 26 | 27 | override def binaryType: String = byteType 28 | 29 | override def bigDecimalType: String = "double" 30 | 31 | override def bigIntegerType: String = "double" 32 | 33 | override def toString: String = "ElasticsearchDialect" 34 | 35 | override def generateDataModel(classTypeMetaData: ClassTypeMetaData, fieldsExpressions: Iterable[String]): String = 36 | "{\n" + 37 | templateExpression(classTypeMetaData) + 38 | settingsExpression(classTypeMetaData) + 39 | mappingsExpression(classTypeMetaData, fieldsExpressions) + 40 | "\n}" 41 | 42 | private def templateExpression(classTypeMetaData: ClassTypeMetaData): String = { 43 | val template = classTypeMetaData.getAnnotationValue(EsTemplate) 44 | if(template.isEmpty) "" 45 | else 46 | s""" 47 | | "template" : "${template.get}",""".stripMargin 48 | } 49 | 50 | private def settingsExpression(classTypeMetaData: ClassTypeMetaData): String = 51 | { 52 | val settings = classTypeMetaData.annotations.filter(_.name == EsSetting) 53 | if(settings.isEmpty) 54 | "" 55 | else 56 | "\n \"settings\" : {\n " + 57 | settings 58 | .map(a => { 59 | val key = a.attributes.find(_.name == "key").get.value 60 | val value = a.attributes.find(_.name == "value").get.value 61 | s""""$key" : ${addApostrophesIfRequired(value)}""" 62 | }) 63 | .mkString(",\n ") + 64 | "\n }," 65 | } 66 | 67 | private def mappingsExpression(classTypeMetaData: ClassTypeMetaData, fieldsExpressions: Iterable[String]): String = 68 | s""" 69 | | "mappings" : { 70 | | "${classTypeMetaData.typeName}" : { 71 | | "properties" : { 72 | | ${fieldsExpressions.mkString(",\n ")} 73 | | } 74 | | } 75 | | }""".stripMargin 76 | 77 | private val EsTemplate: String = "com.datawizards.dmg.annotations.es.esTemplate" 78 | private val EsSetting: String = "com.datawizards.dmg.annotations.es.esSetting" 79 | 80 | private def addApostrophesIfRequired(value: String): String = 81 | if(value.exists(_.isLetter)) "\"" + value + "\"" 82 | else value 83 | 84 | override def generateClassFieldExpression(f: ClassFieldMetaData, typeExpression: String, level: Int): String = 85 | s""""${f.fieldName}" : """ + (f.fieldType match { 86 | case _:ClassTypeMetaData => typeExpression 87 | case c:CollectionTypeMetaData => c.elementType match { 88 | case _:ClassTypeMetaData => typeExpression 89 | case _ => s"""{"type" : $typeExpression}""" 90 | } 91 | case _ => s"""{"type" : $typeExpression${fieldParametersExpressions(f)}}""" 92 | }) 93 | 94 | override def generatePrimitiveTypeExpression(p: PrimitiveTypeMetaData): String = 95 | s""""${mapPrimitiveDataType(p)}"""" 96 | 97 | override def generateArrayTypeExpression(elementTypeExpression: String): String = 98 | elementTypeExpression 99 | 100 | override def generateClassTypeExpression(classTypeMetaData: ClassTypeMetaData, fieldNamesWithExpressions: Iterable[(String, String)]): String = 101 | s"""{"properties" : {${fieldNamesWithExpressions.map{case (k,v) => v}.mkString(", ")}}}""" 102 | 103 | override def generateMapTypeExpression(keyExpression: String, valueExpression: String): String = { 104 | log.warn("Elasticsearch doesn't support Map type. Column converted to string") 105 | "\"string\"" 106 | } 107 | 108 | private def fieldParametersExpressions(f: ClassFieldMetaData): String = 109 | indexParameterExpression(f) + 110 | formatParameterExpression(f) 111 | 112 | private def indexParameterExpression(f: ClassFieldMetaData): String = 113 | generateFieldParameterExpression(f, "com.datawizards.dmg.annotations.es.esIndex", "index") 114 | 115 | private def formatParameterExpression(f: ClassFieldMetaData): String = 116 | generateFieldParameterExpression(f, "com.datawizards.dmg.annotations.es.esFormat", "format") 117 | 118 | private def generateFieldParameterExpression(f: ClassFieldMetaData, annotationName: String, parameterName: String): String = { 119 | val annotation = f.getAnnotationValue(annotationName) 120 | if(annotation.isDefined) s""", "$parameterName" : "${annotation.get}"""" 121 | else "" 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/dialects/H2Dialect.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | import com.datawizards.dmg.metadata._ 3 | 4 | object H2Dialect extends DatabaseDialect { 5 | override def intType: String = "INT" 6 | 7 | override def stringType: String = "VARCHAR" 8 | 9 | override def longType: String = "BIGINT" 10 | 11 | override def doubleType: String = "DOUBLE" 12 | 13 | override def floatType: String = "REAL" 14 | 15 | override def shortType: String = "SMALLINT" 16 | 17 | override def booleanType: String = "BOOLEAN" 18 | 19 | override def byteType: String = "TINYINT" 20 | 21 | override def dateType: String = "DATE" 22 | 23 | override def timestampType: String = "TIMESTAMP" 24 | 25 | override def binaryType: String = "BINARY" 26 | 27 | override def bigDecimalType: String = "DECIMAL(38,18)" 28 | 29 | override def bigIntegerType: String = "BIGINT" 30 | 31 | override def generateArrayTypeExpression(elementTypeExpression: String): String = "ARRAY" 32 | 33 | override def generateClassTypeExpression(classTypeMetaData: ClassTypeMetaData, fieldNamesWithExpressions: Iterable[(String, String)]): String = "OTHER" 34 | 35 | override def generateMapTypeExpression(keyExpression: String, valueExpression: String): String = { 36 | log.warn("H2 doesn't support Map type. Column converted to VARCHAR") 37 | "VARCHAR" 38 | } 39 | 40 | override def toString: String = "H2Dialect" 41 | 42 | override protected def fieldAdditionalExpressions(f: ClassFieldMetaData): String = 43 | notNullExpression(f) + 44 | commentExpression(f) 45 | 46 | override protected def additionalTableProperties(classTypeMetaData: ClassTypeMetaData): String = "" 47 | 48 | override protected def additionalTableExpressions(classTypeMetaData: ClassTypeMetaData): String = 49 | if(comment(classTypeMetaData).isDefined) 50 | s""" 51 | |COMMENT ON TABLE ${classTypeMetaData.typeName} IS '${comment(classTypeMetaData).get}';""".stripMargin 52 | else "" 53 | 54 | private def notNullExpression(f: ClassFieldMetaData): String = 55 | if(notNull(f)) " NOT NULL" else "" 56 | 57 | private def commentExpression(f: ClassFieldMetaData): String = 58 | if(comment(f).isEmpty) "" else s" COMMENT '${comment(f).get}'" 59 | 60 | override protected def reservedKeywords = Seq( 61 | "ALL", 62 | "ALTER", 63 | "AND", 64 | "AS", 65 | "BETWEEN", 66 | "CASE", 67 | "COLUMN", 68 | "CREATE", 69 | "DATABASE", 70 | "DATE", 71 | "DELETE", 72 | "DISTINCT", 73 | "DROP", 74 | "ELSE", 75 | "END", 76 | "EXISTS", 77 | "FALSE", 78 | "FETCH", 79 | "FULL", 80 | "GRANT", 81 | "GROUP", 82 | "HAVING", 83 | "INNER", 84 | "INSERT", 85 | "INTO", 86 | "JOIN", 87 | "LEFT", 88 | "NOT", 89 | "NULL", 90 | "OR", 91 | "ORDER", 92 | "OUTER", 93 | "SELECT", 94 | "TABLE", 95 | "TRUE", 96 | "UNION", 97 | "UPDATE", 98 | "USER", 99 | "USING", 100 | "VALUES", 101 | "WHEN", 102 | "WHERE" 103 | ) 104 | 105 | override protected def escapeColumnName(columnName: String) = 106 | s""""$columnName"""" 107 | } 108 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/dialects/HiveDialect.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | import com.datawizards.dmg.metadata._ 4 | 5 | import scala.util.Try 6 | 7 | object HiveDialect extends DatabaseDialect { 8 | override def intType: String = "INT" 9 | 10 | override def stringType: String = "STRING" 11 | 12 | override def longType: String = "BIGINT" 13 | 14 | override def doubleType: String = "DOUBLE" 15 | 16 | override def floatType: String = "FLOAT" 17 | 18 | override def shortType: String = "SMALLINT" 19 | 20 | override def booleanType: String = "BOOLEAN" 21 | 22 | override def byteType: String = "TINYINT" 23 | 24 | override def dateType: String = "DATE" 25 | 26 | override def timestampType: String = "TIMESTAMP" 27 | 28 | override def binaryType: String = "BINARY" 29 | 30 | override def bigDecimalType: String = "DECIMAL(38,18)" 31 | 32 | override def bigIntegerType: String = "BIGINT" 33 | 34 | override def generateArrayTypeExpression(elementTypeExpression: String): String = 35 | s"ARRAY<$elementTypeExpression>" 36 | 37 | override def generateClassTypeExpression(classTypeMetaData: ClassTypeMetaData, fieldNamesWithExpressions: Iterable[(String, String)]): String = 38 | s"STRUCT<${classTypeMetaData.fields.map(f => s"${f.fieldName} : ${generateTypeExpression(f.fieldType)}").mkString(", ")}>" 39 | 40 | override def generateMapTypeExpression(keyExpression: String, valueExpression: String): String = 41 | s"MAP<$keyExpression, $valueExpression>" 42 | 43 | override def toString: String = "HiveDialect" 44 | 45 | override protected def generateColumnsExpression(classTypeMetaData: ClassTypeMetaData, fieldsExpressions: Iterable[String]): String = 46 | if(isAvroSchemaURLProvided(classTypeMetaData)) "" 47 | else super.generateColumnsExpression(classTypeMetaData, fieldsExpressions) 48 | 49 | override protected def fieldAdditionalExpressions(f: ClassFieldMetaData): String = 50 | if(comment(f).isEmpty) "" else s" COMMENT '${comment(f).get}'" 51 | 52 | override protected def additionalTableProperties(classTypeMetaData: ClassTypeMetaData): String = 53 | commentExpression(classTypeMetaData) + 54 | partitionedByExpression(classTypeMetaData) + 55 | rowFormatSerdeExpression(classTypeMetaData) + 56 | storedAsExpression(classTypeMetaData) + 57 | locationExpression(classTypeMetaData) + 58 | tablePropertiesExpression(classTypeMetaData) 59 | 60 | override protected def additionalTableExpressions(classTypeMetaData: ClassTypeMetaData): String = { 61 | if(hiveExternalTableLocation(classTypeMetaData).isDefined && partitionedByExpression(classTypeMetaData).nonEmpty) 62 | s"MSCK REPAIR TABLE ${classTypeMetaData.typeName};\n" 63 | else 64 | "" 65 | } 66 | 67 | override protected def createTableExpression(classTypeMetaData: ClassTypeMetaData): String = 68 | s"CREATE ${if(hiveExternalTableLocation(classTypeMetaData).isDefined) "EXTERNAL " else ""}TABLE ${classTypeMetaData.typeName}" 69 | 70 | override def generateColumn(f: ClassFieldMetaData): Boolean = 71 | !isPartitionField(f) 72 | 73 | private val HivePartitionColumn: String = "com.datawizards.dmg.annotations.hive.hivePartitionColumn" 74 | private val HiveRowFormatSerde: String = "com.datawizards.dmg.annotations.hive.hiveRowFormatSerde" 75 | private val HiveTableProperty: String = "com.datawizards.dmg.annotations.hive.hiveTableProperty" 76 | private val HiveStoredAs: String = "com.datawizards.dmg.annotations.hive.hiveStoredAs" 77 | private val HiveExternalTable: String = "com.datawizards.dmg.annotations.hive.hiveExternalTable" 78 | 79 | private def isPartitionField(field: ClassFieldMetaData): Boolean = 80 | field 81 | .annotations 82 | .exists(_.name == HivePartitionColumn) 83 | 84 | private def isAvroSchemaURLProvided(classTypeMetaData: ClassTypeMetaData): Boolean = 85 | classTypeMetaData 86 | .annotations 87 | .filter(_.name == HiveTableProperty) 88 | .exists(_.attributes.exists(_.value == "avro.schema.url")) 89 | 90 | private def commentExpression(classTypeMetaData: ClassTypeMetaData): String = 91 | if(comment(classTypeMetaData).isDefined) 92 | s""" 93 | |COMMENT '${comment(classTypeMetaData).get}'""".stripMargin 94 | else "" 95 | 96 | private def partitionedByExpression(classTypeMetaData: ClassTypeMetaData): String = 97 | { 98 | val partitionFields = getPartitionFields(classTypeMetaData) 99 | if(partitionFields.isEmpty) 100 | "" 101 | else { 102 | var fieldOrder = 0 103 | "\nPARTITIONED BY(" + 104 | partitionFields 105 | .map(f => s"${f.fieldName} ${generateTypeExpression(f.fieldType)}") 106 | .mkString(", ") + 107 | ")" 108 | } 109 | } 110 | 111 | private def getPartitionFields(classTypeMetaData: ClassTypeMetaData): Seq[ClassFieldMetaData] = { 112 | val partitionFields = classTypeMetaData.fields.filter(_.annotations.exists(_.name == HivePartitionColumn)) 113 | var fieldOrder = 0 114 | partitionFields 115 | .map(f => { 116 | val partitionColumn = f.annotations.find(_.name == HivePartitionColumn).get 117 | val order = Try(partitionColumn.attributes.find(_.name == "order").get.value.toInt).getOrElse(0) 118 | fieldOrder += 1 119 | (f, order, fieldOrder) 120 | }) 121 | .toSeq 122 | .sortWith{case (e1,e2) => e1._2 < e2._2 || (e1._2 == e2._2 && e1._3 < e2._3)} 123 | .map(_._1) 124 | } 125 | 126 | private def rowFormatSerdeExpression(classTypeMetaData: ClassTypeMetaData): String = 127 | { 128 | val rowFormatSerde = classTypeMetaData.getAnnotationValue(HiveRowFormatSerde) 129 | if(rowFormatSerde.isDefined) 130 | s""" 131 | |ROW FORMAT SERDE '${rowFormatSerde.get}'""".stripMargin 132 | else "" 133 | } 134 | 135 | private def storedAsExpression(classTypeMetaData: ClassTypeMetaData): String = 136 | { 137 | val storedAs = classTypeMetaData.getAnnotationValue(HiveStoredAs) 138 | if(storedAs.isDefined) 139 | s""" 140 | |STORED AS ${storedAs.get.replace("\\'","'")}""".stripMargin 141 | else "" 142 | } 143 | 144 | private def locationExpression(classTypeMetaData: ClassTypeMetaData): String = 145 | { 146 | val externalTableLocation = hiveExternalTableLocation(classTypeMetaData) 147 | if(externalTableLocation.isDefined) 148 | s""" 149 | |LOCATION '${externalTableLocation.get}'""".stripMargin 150 | else "" 151 | } 152 | 153 | private def tablePropertiesExpression(classTypeMetaData: ClassTypeMetaData): String = 154 | { 155 | val tableProperties = classTypeMetaData.annotations.filter(_.name == HiveTableProperty) 156 | if(tableProperties.isEmpty) 157 | "" 158 | else 159 | "\nTBLPROPERTIES(\n " + 160 | tableProperties 161 | .map(a => { 162 | val key = a.attributes.find(_.name == "key").get.value 163 | val value = a.attributes.find(_.name == "value").get.value 164 | s"'$key' = '$value'" 165 | }) 166 | .mkString(",\n ") + 167 | "\n)" 168 | } 169 | 170 | private def hiveExternalTableLocation(classTypeMetaData: ClassTypeMetaData): Option[String] = 171 | classTypeMetaData.getAnnotationValue(HiveExternalTable) 172 | 173 | override protected def reservedKeywords = Seq( 174 | "ALL", 175 | "ALTER", 176 | "AND", 177 | "AS", 178 | "BETWEEN", 179 | "CASE", 180 | "COLUMN", 181 | "CREATE", 182 | "DATABASE", 183 | "DATE", 184 | "DELETE", 185 | "DISTINCT", 186 | "DROP", 187 | "ELSE", 188 | "END", 189 | "EXISTS", 190 | "FALSE", 191 | "FETCH", 192 | "FULL", 193 | "FUNCTION", 194 | "GRANT", 195 | "GROUP", 196 | "HAVING", 197 | "INNER", 198 | "INSERT", 199 | "INTO", 200 | "JOIN", 201 | "LEFT", 202 | "NOT", 203 | "NULL", 204 | "OR", 205 | "ORDER", 206 | "OUTER", 207 | "SELECT", 208 | "TABLE", 209 | "TRUE", 210 | "UNION", 211 | "UPDATE", 212 | "USER", 213 | "USING", 214 | "VALUES", 215 | "WHEN", 216 | "WHERE" 217 | ) 218 | 219 | override protected def escapeColumnName(columnName: String) = 220 | s"`$columnName`" 221 | } 222 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/dialects/JavaDialect.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | import com.datawizards.dmg.metadata._ 4 | 5 | object JavaDialect extends Dialect { 6 | 7 | override def intType: String = "Integer" 8 | 9 | override def stringType: String = "String" 10 | 11 | override def longType: String = "Long" 12 | 13 | override def doubleType: String = "Double" 14 | 15 | override def floatType: String = "Float" 16 | 17 | override def shortType: String = "Short" 18 | 19 | override def booleanType: String = "Boolean" 20 | 21 | override def byteType: String = "Byte" 22 | 23 | override def dateType: String = "java.util.Date" 24 | 25 | override def timestampType: String = "java.sql.Timestamp" 26 | 27 | override def binaryType: String = generateArrayTypeExpression(byteType) 28 | 29 | override def bigDecimalType: String = "java.math.BigDecimal" 30 | 31 | override def bigIntegerType: String = "java.math.BigInteger" 32 | 33 | override def generateArrayTypeExpression(elementTypeExpression: String): String = 34 | s"java.util.List<$elementTypeExpression>" 35 | 36 | override def generateMapTypeExpression(keyExpression: String, valueExpression: String): String = 37 | s"""java.util.Map<$keyExpression, $valueExpression>""" 38 | 39 | override def generateClassTypeExpression(classTypeMetaData: ClassTypeMetaData, fieldNamesWithExpressions: Iterable[(String, String)]): String = 40 | classTypeMetaData.packageName + "." + classTypeMetaData.typeName 41 | 42 | override def toString: String = "JavaDialect" 43 | 44 | override def generateDataModel(classTypeMetaData: ClassTypeMetaData, fieldsExpressions: Iterable[String]): String = 45 | s"""public class ${classTypeMetaData.typeName} { 46 | |${generatePrivateFieldsExpression(fieldsExpressions)} 47 | |${generateDefaultConstructor(classTypeMetaData)} 48 | |${generateConstructor(classTypeMetaData)} 49 | |${generateGettersAndSetters(classTypeMetaData)} 50 | |}""".stripMargin 51 | 52 | override def generateClassFieldExpression(f: ClassFieldMetaData, typeExpression: String, level: Int): String = 53 | s" private $typeExpression ${f.fieldName};" 54 | 55 | private def generatePrivateFieldsExpression(fieldsExpressions: Iterable[String]): String = 56 | fieldsExpressions.mkString("\n") 57 | 58 | private def generateDefaultConstructor(classTypeMetaData: ClassTypeMetaData): String = 59 | s"\n public ${classTypeMetaData.typeName}() {}" 60 | 61 | private def generateConstructor(classTypeMetaData: ClassTypeMetaData): String = 62 | s""" 63 | | public ${classTypeMetaData.typeName}(${classTypeMetaData.fields.map(f => s"${generateTypeExpression(f.fieldType)} ${f.fieldName}").mkString(", ")}) { 64 | | ${classTypeMetaData.fields.map(f => s"this.${f.fieldName} = ${f.fieldName};").mkString("\n ")} 65 | | }""".stripMargin 66 | 67 | private def generateGettersAndSetters(classTypeMetaData: ClassTypeMetaData): String = 68 | classTypeMetaData 69 | .fields 70 | .map(f => 71 | s""" 72 | | public ${generateTypeExpression(f.fieldType)} get${f.fieldName.capitalize}() { 73 | | return ${f.fieldName}; 74 | | } 75 | | 76 | | public void set${f.fieldName.capitalize}(${generateTypeExpression(f.fieldType)} ${f.fieldName}) { 77 | | this.${f.fieldName} = ${f.fieldName}; 78 | | }""".stripMargin) 79 | .mkString("\n") 80 | } 81 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/dialects/MetaDataWithDialectExtractor.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | import com.datawizards.dmg.metadata.MetaDataExtractor.extractClassMetaData 4 | import com.datawizards.dmg.metadata._ 5 | 6 | import scala.reflect.runtime.universe.TypeTag 7 | 8 | object MetaDataWithDialectExtractor { 9 | /** 10 | * Extract class metadata and change types and fields taking into account dialect as context. 11 | * @param dialect rename class and field names according to dialect. If dialect is null, then 12 | */ 13 | def extractClassMetaDataForDialect[T: TypeTag](dialect: Option[Dialect]): ClassTypeMetaData = 14 | changeName(dialect, extractClassMetaData[T]()) 15 | 16 | private def changeName(dialect: Option[Dialect], c: ClassTypeMetaData): ClassTypeMetaData = 17 | c.copy( 18 | typeName = getClassName(dialect, c), 19 | fields = c.fields.map(f => changeName(dialect, f, c)) 20 | ) 21 | 22 | private def changeName(dialect: Option[Dialect], classFieldMetaData: ClassFieldMetaData, classTypeMetaData: ClassTypeMetaData): ClassFieldMetaData = 23 | classFieldMetaData.copy( 24 | fieldName = getFieldName(dialect, classFieldMetaData, classTypeMetaData), 25 | fieldType = changeName(dialect, classFieldMetaData.fieldType) 26 | ) 27 | 28 | private def changeName(dialect: Option[Dialect], typeMetaData: TypeMetaData): TypeMetaData = typeMetaData match { 29 | case p:PrimitiveTypeMetaData => p 30 | case c:CollectionTypeMetaData => c 31 | case m:MapTypeMetaData => m 32 | case c:ClassTypeMetaData => changeName(dialect, c) 33 | } 34 | 35 | private val Table = "com.datawizards.dmg.annotations.table" 36 | private val Column = "com.datawizards.dmg.annotations.column" 37 | private val Underscore = "com.datawizards.dmg.annotations.underscore" 38 | 39 | 40 | private def getClassName(dialect: Option[Dialect], classMetaData: ClassTypeMetaData): String = { 41 | val dialectSpecificTableAnnotation = getAnnotationForDialect(Table, dialect, classMetaData) 42 | 43 | dialectSpecificTableAnnotation.map(a => a.attributes.filter(_.name == "name").head.value ) 44 | .getOrElse(convertToUnderscoreIfRequired(classMetaData.typeName, dialect, classMetaData)) 45 | } 46 | 47 | private def getFieldName(dialect: Option[Dialect], classFieldMetaData: ClassFieldMetaData, classTypeMetaData: ClassTypeMetaData): String = { 48 | val columnAnnotations = getAnnotationForDialect(Column, dialect, classFieldMetaData) 49 | 50 | columnAnnotations.map(a => a.attributes.filter(_.name == "name").head.value) 51 | .getOrElse(convertToUnderscoreIfRequired(classFieldMetaData.fieldName, dialect, classTypeMetaData)) 52 | } 53 | 54 | private def convertToUnderscoreIfRequired(name: String, dialect: Option[Dialect], classTypeMetaData: ClassTypeMetaData): String = { 55 | val underscoreAnnotation: Option[AnnotationMetaData] = getAnnotationForDialect(Underscore, dialect, classTypeMetaData) 56 | underscoreAnnotation.map(a => name.replaceAll("(.)(\\p{Upper})","$1_$2").toLowerCase).getOrElse(name) 57 | } 58 | 59 | def getAnnotationForDialect(annotationName: String, dialect: Option[Dialect], hasAnnotations: HasAnnotations): Option[AnnotationMetaData] = { 60 | if(dialect.isDefined){ 61 | val dialectName = dialect.get.toString 62 | val annotationsFiltered: Iterable[AnnotationMetaData] = hasAnnotations.annotations 63 | .filter(a => a.name == annotationName) 64 | 65 | val annotation: Option[AnnotationMetaData] = annotationsFiltered 66 | .find(a => a.attributes.find(_.name == "dialect").map(p => p.value.endsWith(dialectName)).getOrElse(false) ) 67 | .orElse(annotationsFiltered.find(a => !a.attributes.exists(_.name == "dialect") ).headOption) 68 | annotation 69 | } else { 70 | val annotation: Option[AnnotationMetaData] = hasAnnotations.annotations 71 | .filter(a => a.name == annotationName) 72 | .find(a => !a.attributes.exists(_.name == "dialect") ) 73 | annotation 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/dialects/MySQLDialect.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | import com.datawizards.dmg.metadata._ 4 | 5 | object MySQLDialect extends DatabaseDialect { 6 | override def intType: String = "INT" 7 | 8 | override def stringType: String = "VARCHAR" 9 | 10 | override def longType: String = "BIGINT" 11 | 12 | override def doubleType: String = "DOUBLE" 13 | 14 | override def floatType: String = "FLOAT" 15 | 16 | override def shortType: String = "SMALLINT" 17 | 18 | override def booleanType: String = "BOOLEAN" 19 | 20 | override def byteType: String = "SMALLINT" 21 | 22 | override def dateType: String = "DATE" 23 | 24 | override def timestampType: String = "TIMESTAMP" 25 | 26 | override def binaryType: String = "BINARY" 27 | 28 | override def bigDecimalType: String = "DECIMAL(38,18)" 29 | 30 | override def bigIntegerType: String = "BIGINT" 31 | 32 | override def generateArrayTypeExpression(elementTypeExpression: String): String = { 33 | log.warn("MySQL doesn't support ARRAY type. Column converted to JSON.") 34 | "JSON" 35 | } 36 | 37 | override def generateClassTypeExpression(classTypeMetaData: ClassTypeMetaData, fieldNamesWithExpressions: Iterable[(String, String)]): String = { 38 | log.warn("MySQL doesn't support Struct type. Column converted to JSON") 39 | "JSON" 40 | } 41 | 42 | override def generateMapTypeExpression(keyExpression: String, valueExpression: String): String = { 43 | log.warn("MySQL doesn't support Map type. Column converted to JSON") 44 | "JSON" 45 | } 46 | 47 | override def toString: String = "MySQLDialect" 48 | 49 | override protected def fieldAdditionalExpressions(f: ClassFieldMetaData): String = 50 | notNullExpression(f) + 51 | commentExpression(f) 52 | 53 | override protected def additionalTableProperties(classTypeMetaData: ClassTypeMetaData): String = 54 | if(comment(classTypeMetaData).isDefined) 55 | s"\nCOMMENT = '${comment(classTypeMetaData).get}'" 56 | else "" 57 | 58 | override protected def additionalTableExpressions(classTypeMetaData: ClassTypeMetaData): String = "" 59 | 60 | private def notNullExpression(f: ClassFieldMetaData): String = 61 | if(notNull(f)) " NOT NULL" else "" 62 | 63 | private def commentExpression(f: ClassFieldMetaData): String = 64 | if(comment(f).isEmpty) "" else s" COMMENT '${comment(f).get}'" 65 | 66 | override protected def reservedKeywords = Seq( 67 | "ALL", 68 | "ALTER", 69 | "AND", 70 | "AS", 71 | "BETWEEN", 72 | "CASE", 73 | "COLUMN", 74 | "CREATE", 75 | "DATABASE", 76 | "DATE", 77 | "DELETE", 78 | "DISTINCT", 79 | "DROP", 80 | "ELSE", 81 | "END", 82 | "EXISTS", 83 | "FALSE", 84 | "FETCH", 85 | "FULL", 86 | "GRANT", 87 | "GROUP", 88 | "HAVING", 89 | "INNER", 90 | "INSERT", 91 | "INTO", 92 | "JOIN", 93 | "LEFT", 94 | "NOT", 95 | "NULL", 96 | "OR", 97 | "ORDER", 98 | "OUTER", 99 | "SELECT", 100 | "TABLE", 101 | "TRUE", 102 | "UNION", 103 | "UPDATE", 104 | "USER", 105 | "USING", 106 | "VALUES", 107 | "WHEN", 108 | "WHERE" 109 | ) 110 | 111 | override protected def escapeColumnName(columnName: String) = 112 | s""""$columnName"""" 113 | } 114 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/dialects/RedshiftDialect.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | import com.datawizards.dmg.metadata._ 4 | 5 | object RedshiftDialect extends DatabaseDialect { 6 | override def intType: String = "INTEGER" 7 | 8 | override def stringType: String = "VARCHAR" 9 | 10 | override def longType: String = "BIGINT" 11 | 12 | override def doubleType: String = "DOUBLE PRECISION" 13 | 14 | override def floatType: String = "REAL" 15 | 16 | override def shortType: String = "SMALLINT" 17 | 18 | override def booleanType: String = "BOOLEAN" 19 | 20 | override def byteType: String = "SMALLINT" 21 | 22 | override def dateType: String = "DATE" 23 | 24 | override def timestampType: String = "TIMESTAMP" 25 | 26 | override def binaryType: String = generateArrayTypeExpression(byteType) 27 | 28 | override def bigDecimalType: String = "DECIMAL(38,18)" 29 | 30 | override def bigIntegerType: String = "BIGINT" 31 | 32 | override def generateArrayTypeExpression(elementTypeExpression: String): String = { 33 | log.warn("Redshift doesn't support ARRAY type. Column converted to VARCHAR.") 34 | "VARCHAR" 35 | } 36 | 37 | override def generateClassTypeExpression(classTypeMetaData: ClassTypeMetaData, fieldNamesWithExpressions: Iterable[(String, String)]): String = { 38 | log.warn("Redshift doesn't support Struct type. Column converted to VARCHAR.") 39 | "VARCHAR" 40 | } 41 | 42 | override def generateMapTypeExpression(keyExpression: String, valueExpression: String): String = { 43 | log.warn("Redshift doesn't support Map type. Column converted to VARCHAR") 44 | "VARCHAR" 45 | } 46 | 47 | override def toString: String = "RedshiftDialect" 48 | 49 | override protected def fieldAdditionalExpressions(f: ClassFieldMetaData): String = 50 | notNullExpression(f) 51 | 52 | override protected def additionalTableProperties(classTypeMetaData: ClassTypeMetaData): String = "" 53 | 54 | override protected def additionalTableExpressions(classTypeMetaData: ClassTypeMetaData): String = 55 | ( 56 | if(comment(classTypeMetaData).isDefined) 57 | s"\nCOMMENT ON TABLE ${classTypeMetaData.typeName} IS '${comment(classTypeMetaData).get}';" 58 | else "" 59 | ) + classTypeMetaData.fields.map(f => 60 | if(comment(f).isDefined) 61 | s"\nCOMMENT ON COLUMN ${classTypeMetaData.typeName}.${f.fieldName} IS '${comment(f).get}';" 62 | else "" 63 | ).mkString("") 64 | 65 | private def notNullExpression(f: ClassFieldMetaData): String = 66 | if(notNull(f)) " NOT NULL" else "" 67 | 68 | override protected def reservedKeywords = Seq( 69 | "ALL", 70 | "ALTER", 71 | "AND", 72 | "AS", 73 | "BETWEEN", 74 | "CASE", 75 | "COLUMN", 76 | "CREATE", 77 | "DATABASE", 78 | "DATE", 79 | "DELETE", 80 | "DISTINCT", 81 | "DROP", 82 | "ELSE", 83 | "END", 84 | "EXISTS", 85 | "FALSE", 86 | "FETCH", 87 | "FULL", 88 | "GRANT", 89 | "GROUP", 90 | "HAVING", 91 | "INNER", 92 | "INSERT", 93 | "INTO", 94 | "JOIN", 95 | "LEFT", 96 | "NOT", 97 | "NULL", 98 | "OR", 99 | "ORDER", 100 | "OUTER", 101 | "SELECT", 102 | "TABLE", 103 | "TRUE", 104 | "UNION", 105 | "UPDATE", 106 | "USER", 107 | "USING", 108 | "VALUES", 109 | "WHEN", 110 | "WHERE" 111 | ) 112 | 113 | override protected def escapeColumnName(columnName: String) = 114 | s""""$columnName"""" 115 | } 116 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/examples/ArrayTypeExample.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.examples 2 | 3 | import com.datawizards.dmg.examples.TestModel.CV 4 | import com.datawizards.dmg.{DataModelGenerator, dialects} 5 | 6 | object ArrayTypeExample extends App { 7 | println(DataModelGenerator.generate[CV](dialects.H2Dialect)) 8 | println(DataModelGenerator.generate[CV](dialects.HiveDialect)) 9 | println(DataModelGenerator.generate[CV](dialects.ElasticsearchDialect)) 10 | println(DataModelGenerator.generate[CV](dialects.AvroSchemaDialect)) 11 | println(DataModelGenerator.generate[CV](dialects.AvroSchemaRegistryDialect)) 12 | } 13 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/examples/AvroSchemaExample.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.examples 2 | 3 | import com.datawizards.dmg.examples.TestModel.{ClassWithAllSimpleTypes, Person} 4 | import com.datawizards.dmg.{DataModelGenerator, dialects} 5 | 6 | object AvroSchemaExample extends App { 7 | println(DataModelGenerator.generate[Person](dialects.AvroSchemaDialect)) 8 | println(DataModelGenerator.generate[ClassWithAllSimpleTypes](dialects.AvroSchemaDialect)) 9 | 10 | println(DataModelGenerator.generate[Person](dialects.AvroSchemaRegistryDialect)) 11 | println(DataModelGenerator.generate[ClassWithAllSimpleTypes](dialects.AvroSchemaRegistryDialect)) 12 | } 13 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/examples/CopyAvroSchemaToHDFS.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.examples 2 | 3 | import com.datawizards.dmg.examples.TestModel.Person 4 | import com.datawizards.dmg.service.AvroSchemaRegistryServiceImpl 5 | 6 | object CopyAvroSchemaToHDFS extends App { 7 | val service = new AvroSchemaRegistryServiceImpl("http://localhost:8081") 8 | service.copyAvroSchemaToHdfs[Person]("/metadata/schemas/person") 9 | } -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/examples/CreateElasticsearchIndex.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.examples 2 | 3 | import com.datawizards.dmg.examples.TestModel.Person 4 | import com.datawizards.dmg.service.ElasticsearchServiceImpl 5 | 6 | object CreateElasticsearchIndex extends App { 7 | val service = new ElasticsearchServiceImpl("http://localhost:9200") 8 | service.createIndex[Person]("person") 9 | 10 | println("Index:") 11 | println(service.getIndexSettings("person")) 12 | } 13 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/examples/CreateElasticsearchTemplate.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.examples 2 | 3 | import com.datawizards.dmg.examples.TestModel.PersonWithMultipleEsAnnotations 4 | import com.datawizards.dmg.service.ElasticsearchServiceImpl 5 | 6 | object CreateElasticsearchTemplate extends App { 7 | val service = new ElasticsearchServiceImpl("http://localhost:9200") 8 | service.updateTemplate[PersonWithMultipleEsAnnotations]("people") 9 | 10 | println("Template:") 11 | println(service.getTemplate("people")) 12 | } 13 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/examples/CreateHiveTableBatchExample.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.examples 2 | 3 | import com.datawizards.dmg.examples.TestModel.{Person, PersonWithCustomName} 4 | import com.datawizards.dmg.service.HiveServiceImpl 5 | 6 | object CreateHiveTableBatchExample extends App { 7 | HiveServiceImpl.batchCreateTable() 8 | .createTable[Person] 9 | .createTable[PersonWithCustomName] 10 | .execute() 11 | } 12 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/examples/CreateHiveTableExample.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.examples 2 | 3 | import com.datawizards.dmg.examples.TestModel.Person 4 | import com.datawizards.dmg.service.HiveServiceImpl 5 | 6 | object CreateHiveTableExample extends App { 7 | HiveServiceImpl.createHiveTable[Person]() 8 | HiveServiceImpl.createHiveTableIfNotExists[Person]() 9 | } 10 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/examples/CustomTableColumnNameExample.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.examples 2 | 3 | import com.datawizards.dmg.examples.TestModel.PersonWithCustomName 4 | import com.datawizards.dmg.{DataModelGenerator, dialects} 5 | 6 | object CustomTableColumnNameExample extends App { 7 | println(DataModelGenerator.generate[PersonWithCustomName](dialects.H2Dialect)) 8 | } 9 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/examples/ElasticsearchExample.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.examples 2 | 3 | import com.datawizards.dmg.examples.TestModel._ 4 | import com.datawizards.dmg.{DataModelGenerator, dialects} 5 | 6 | object ElasticsearchExample extends App { 7 | println(DataModelGenerator.generate[Person](dialects.ElasticsearchDialect)) 8 | println(DataModelGenerator.generate[ClassWithAllSimpleTypes](dialects.ElasticsearchDialect)) 9 | println(DataModelGenerator.generate[PersonWithMultipleEsAnnotations](dialects.ElasticsearchDialect)) 10 | } 11 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/examples/H2Example.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.examples 2 | 3 | import com.datawizards.dmg.{DataModelGenerator, dialects} 4 | import com.datawizards.dmg.examples.TestModel.{ClassWithAllSimpleTypes, Person} 5 | 6 | object H2Example extends App { 7 | println(DataModelGenerator.generate[Person](dialects.H2Dialect)) 8 | println(DataModelGenerator.generate[ClassWithAllSimpleTypes](dialects.H2Dialect)) 9 | } 10 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/examples/HiveExample.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.examples 2 | 3 | import com.datawizards.dmg.examples.TestModel._ 4 | import com.datawizards.dmg.{DataModelGenerator, dialects} 5 | 6 | object HiveExample extends App { 7 | println(DataModelGenerator.generate[ParquetTableWithManyAnnotations](dialects.HiveDialect)) 8 | println(DataModelGenerator.generate[AvroTableWithManyAnnotations](dialects.HiveDialect)) 9 | } 10 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/examples/MultipleCustomNames.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.examples 2 | 3 | import com.datawizards.dmg.{DataModelGenerator, dialects} 4 | import com.datawizards.dmg.examples.TestModel.PersonWithMultipleCustomNames 5 | 6 | object MultipleCustomNames extends App { 7 | println(DataModelGenerator.generate[PersonWithMultipleCustomNames](dialects.H2Dialect)) 8 | println(DataModelGenerator.generate[PersonWithMultipleCustomNames](dialects.HiveDialect)) 9 | println(DataModelGenerator.generate[PersonWithMultipleCustomNames](dialects.RedshiftDialect)) 10 | println(DataModelGenerator.generate[PersonWithMultipleCustomNames](dialects.AvroSchemaDialect)) 11 | println(DataModelGenerator.generate[PersonWithMultipleCustomNames](dialects.ElasticsearchDialect)) 12 | } 13 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/examples/MySQLExample.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.examples 2 | 3 | import com.datawizards.dmg.examples.TestModel._ 4 | import com.datawizards.dmg.{DataModelGenerator, dialects} 5 | 6 | object MySQLExample extends App { 7 | println(DataModelGenerator.generate[Person](dialects.MySQLDialect)) 8 | println(DataModelGenerator.generate[ClassWithAllSimpleTypes](dialects.MySQLDialect)) 9 | println(DataModelGenerator.generate[PersonWithComments](dialects.MySQLDialect)) 10 | println(DataModelGenerator.generate[CV](dialects.MySQLDialect)) 11 | } 12 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/examples/RedshiftExample.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.examples 2 | 3 | import com.datawizards.dmg.examples.TestModel._ 4 | import com.datawizards.dmg.{DataModelGenerator, dialects} 5 | 6 | object RedshiftExample extends App { 7 | println(DataModelGenerator.generate[Person](dialects.RedshiftDialect)) 8 | println(DataModelGenerator.generate[ClassWithAllSimpleTypes](dialects.RedshiftDialect)) 9 | println(DataModelGenerator.generate[PersonWithComments](dialects.RedshiftDialect)) 10 | println(DataModelGenerator.generate[CV](dialects.RedshiftDialect)) 11 | } 12 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/examples/RegisterAvroSchema.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.examples 2 | 3 | import com.datawizards.dmg.examples.TestModel.Person 4 | import com.datawizards.dmg.service.AvroSchemaRegistryServiceImpl 5 | 6 | object RegisterAvroSchema extends App { 7 | val service = new AvroSchemaRegistryServiceImpl("http://localhost:8081") 8 | service.registerSchema[Person]("person") 9 | 10 | println("Subjects:") 11 | println(service.subjects()) 12 | 13 | println("Registered schema:") 14 | println(service.fetchSchema("person")) 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/examples/StructTypesExample.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.examples 2 | 3 | import com.datawizards.dmg.examples.TestModel.Book 4 | import com.datawizards.dmg.{DataModelGenerator, dialects} 5 | 6 | object StructTypesExample extends App { 7 | println(DataModelGenerator.generate[Book](dialects.H2Dialect)) 8 | println(DataModelGenerator.generate[Book](dialects.HiveDialect)) 9 | println(DataModelGenerator.generate[Book](dialects.RedshiftDialect)) 10 | println(DataModelGenerator.generate[Book](dialects.AvroSchemaDialect)) 11 | println(DataModelGenerator.generate[Book](dialects.ElasticsearchDialect)) 12 | } 13 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/examples/TableColumnCommentsExample.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.examples 2 | 3 | import com.datawizards.dmg.examples.TestModel.PersonWithComments 4 | import com.datawizards.dmg.{DataModelGenerator, dialects} 5 | 6 | object TableColumnCommentsExample extends App { 7 | println(DataModelGenerator.generate[PersonWithComments](dialects.H2Dialect)) 8 | println(DataModelGenerator.generate[PersonWithComments](dialects.AvroSchemaDialect)) 9 | println(DataModelGenerator.generate[PersonWithComments](dialects.HiveDialect)) 10 | } 11 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/examples/TestModel.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.examples 2 | 3 | import java.sql.{Date, Timestamp} 4 | 5 | import com.datawizards.dmg.annotations._ 6 | import com.datawizards.dmg.annotations.es.{esFormat, esIndex, esSetting, esTemplate} 7 | import com.datawizards.dmg.annotations.hive._ 8 | import com.datawizards.dmg.dialects 9 | 10 | object TestModel { 11 | case class Person(name: String, age: Int) 12 | case class ClassWithAllSimpleTypes( 13 | strVal: String, 14 | intVal: Int, 15 | longVal: Long, 16 | doubleVal: Double, 17 | floatVal: Float, 18 | shortVal: Short, 19 | booleanVal: Boolean, 20 | byteVal: Byte, 21 | dateVal: Date, 22 | timestampVal: Timestamp 23 | ) 24 | case class PersonWithCustomName( 25 | @column(name="personName") name: String, 26 | @column(name="personAge") age: Int, 27 | gender: String 28 | ) 29 | @comment("People data") 30 | case class PersonWithComments( 31 | @comment("Person name") name: String, 32 | age: Int 33 | ) 34 | @table("people") 35 | @table("PEOPLE", dialects.H2Dialect) 36 | @table("person", dialects.ElasticsearchDialect) 37 | case class PersonWithMultipleCustomNames( 38 | @column("NAME", dialects.H2Dialect) 39 | @column("personNameEs", dialects.ElasticsearchDialect) 40 | name: String, 41 | @column("personAge") 42 | @column("AGE", dialects.H2Dialect) 43 | @column("personAgeEs", dialects.ElasticsearchDialect) 44 | age: Int 45 | ) 46 | case class CV(skills: Seq[String]) 47 | case class Book(title: String, year: Int, owner: Person, authors: Seq[Person]) 48 | 49 | @table("CUSTOM_TABLE_NAME") 50 | @comment("Table comment") 51 | @hiveStoredAs(format="PARQUET") 52 | @hiveExternalTable(location="hdfs:///data/table") 53 | @hiveTableProperty("key1", "value1") 54 | @hiveTableProperty("key2", "value2") 55 | @hiveTableProperty("key3", "value3") 56 | case class ParquetTableWithManyAnnotations( 57 | @column("eventTime") 58 | @comment("Event time") 59 | time: Timestamp, 60 | @comment("Event name") 61 | event: String, 62 | @comment("User id") 63 | user: String, 64 | @hivePartitionColumn(order=3) 65 | day: Int, 66 | @hivePartitionColumn(order=1) 67 | year: Int, 68 | @hivePartitionColumn(order=2) 69 | month: Int 70 | ) 71 | 72 | @table("CUSTOM_TABLE_NAME") 73 | @comment("Table comment") 74 | @hiveRowFormatSerde(format="org.apache.hadoop.hive.serde2.avro.AvroSerDe") 75 | @hiveStoredAs("INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'") 76 | @hiveExternalTable(location="hdfs:///data/table") 77 | @hiveTableProperty("avro.schema.url", "hdfs:///metadata/table.avro") 78 | @hiveTableProperty("key1", "value1") 79 | @hiveTableProperty("key2", "value2") 80 | @hiveTableProperty("key3", "value3") 81 | case class AvroTableWithManyAnnotations( 82 | @column("eventTime") 83 | @comment("Event time") 84 | time: Timestamp, 85 | @comment("Event name") 86 | event: String, 87 | @comment("User id") 88 | user: String, 89 | @hivePartitionColumn(order=3) 90 | day: Int, 91 | @hivePartitionColumn(order=1) 92 | year: Int, 93 | @hivePartitionColumn(order=2) 94 | month: Int 95 | ) 96 | 97 | @table("people") 98 | @esTemplate("people*") 99 | @esSetting("number_of_shards", 1) 100 | @esSetting("number_of_replicas", 3) 101 | case class PersonWithMultipleEsAnnotations( 102 | @esIndex("not_analyzed") 103 | @column("personName") 104 | name: String, 105 | @column("personBirthday") 106 | @esFormat("yyyy-MM-dd") 107 | birthday: Date 108 | ) 109 | } 110 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/metadata/MetaDataExtractor.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.metadata 2 | 3 | import scala.reflect.NameTransformer 4 | import scala.reflect.runtime.universe._ 5 | 6 | /** 7 | * Class responsible for extracting metadata about case class: fields, annotations with attributes. 8 | * It is used to simplify getting such information, because current Scala API is very hard to use. 9 | */ 10 | object MetaDataExtractor { 11 | 12 | private def mirror = 13 | runtimeMirror(Thread.currentThread().getContextClassLoader) 14 | 15 | /** 16 | * Extract class metadata. 17 | * This ignores all mappings to dialect-specific names. 18 | */ 19 | def extractClassMetaData[T: TypeTag](): ClassTypeMetaData = { 20 | val tpe = localTypeOf[T] 21 | tpe match { 22 | case t if definedByConstructorParams(t) => extractClassMetaData(t) 23 | case other => 24 | throw new UnsupportedOperationException(s"MetaData for type $other is not supported") 25 | } 26 | } 27 | 28 | def extractTypeMetaData[T : TypeTag](): TypeMetaData = 29 | extractTypeMetaData(localTypeOf[T]) 30 | 31 | private def extractTypeMetaData(tpe: `Type`): TypeMetaData = tpe match { 32 | case t if t <:< localTypeOf[Option[_]] => 33 | val TypeRef(_, _, Seq(optType)) = t 34 | extractTypeMetaData(optType) 35 | case t if t <:< localTypeOf[Array[Byte]] => BinaryType 36 | case t if t <:< localTypeOf[Array[_]] => 37 | val TypeRef(_, _, Seq(elementType)) = t 38 | CollectionTypeMetaData(extractTypeMetaData(elementType)) 39 | case t if t <:< localTypeOf[Seq[_]] => 40 | val TypeRef(_, _, Seq(elementType)) = t 41 | CollectionTypeMetaData(extractTypeMetaData(elementType)) 42 | case t if t <:< localTypeOf[Map[_, _]] => 43 | val TypeRef(_, _, Seq(keyType, valueType)) = t 44 | MapTypeMetaData(extractTypeMetaData(keyType), extractTypeMetaData(valueType)) 45 | case t if t <:< localTypeOf[Iterable[_]] => 46 | val TypeRef(_, _, Seq(elementType)) = t 47 | CollectionTypeMetaData(extractTypeMetaData(elementType)) 48 | case t if t <:< localTypeOf[String] => StringType 49 | case t if t <:< localTypeOf[java.sql.Timestamp] => TimestampType 50 | case t if t <:< localTypeOf[java.util.Date] => DateType 51 | case t if t <:< localTypeOf[java.sql.Date] => DateType 52 | case t if t <:< localTypeOf[BigDecimal] => BigDecimalType 53 | case t if t <:< localTypeOf[java.math.BigDecimal] => BigDecimalType 54 | case t if t <:< localTypeOf[java.math.BigInteger] => BigIntegerType 55 | case t if t <:< localTypeOf[scala.math.BigInt] => BigIntegerType 56 | case t if t <:< localTypeOf[java.lang.Integer] => IntegerType 57 | case t if t <:< localTypeOf[java.lang.Long] => LongType 58 | case t if t <:< localTypeOf[java.lang.Double] => DoubleType 59 | case t if t <:< localTypeOf[java.lang.Float] => FloatType 60 | case t if t <:< localTypeOf[java.lang.Short] => ShortType 61 | case t if t <:< localTypeOf[java.lang.Byte] => ByteType 62 | case t if t <:< localTypeOf[java.lang.Boolean] => BooleanType 63 | case t if t <:< definitions.IntTpe => IntegerType 64 | case t if t <:< definitions.LongTpe => LongType 65 | case t if t <:< definitions.DoubleTpe => DoubleType 66 | case t if t <:< definitions.FloatTpe => FloatType 67 | case t if t <:< definitions.ShortTpe => ShortType 68 | case t if t <:< definitions.ByteTpe => ByteType 69 | case t if t <:< definitions.BooleanTpe => BooleanType 70 | case t if definedByConstructorParams(t) => extractClassMetaData(t) 71 | case other => 72 | throw new UnsupportedOperationException(s"MetaData for type $other is not supported") 73 | } 74 | 75 | private def extractClassMetaData(tpe: `Type`): ClassTypeMetaData = { 76 | val cls = mirror.runtimeClass(tpe) 77 | ClassTypeMetaData( 78 | packageName = cls.getPackage.getName, 79 | originalTypeName = cls.getSimpleName, 80 | typeName = cls.getSimpleName, 81 | annotations = extractAnnotations(tpe.typeSymbol), 82 | fields = extractClassFields(tpe) 83 | ) 84 | } 85 | 86 | private def localTypeOf[T: TypeTag]: Type = MetaDataExtractor.synchronized { 87 | val tag = implicitly[TypeTag[T]] 88 | tag.in(mirror).tpe.dealias 89 | } 90 | 91 | /** 92 | * Whether the fields of the given type is defined entirely by its constructor parameters. 93 | */ 94 | private def definedByConstructorParams(tpe: Type): Boolean = 95 | tpe <:< localTypeOf[Product] 96 | 97 | private def extractClassFields(tpe: Type): Iterable[ClassFieldMetaData] = 98 | tpe 99 | .typeSymbol 100 | .asClass 101 | .primaryConstructor 102 | .typeSignature 103 | .paramLists 104 | .head 105 | .map(f => ClassFieldMetaData( 106 | originalFieldName = NameTransformer.decode(f.name.toString), 107 | fieldName = NameTransformer.decode(f.name.toString), 108 | fieldType = extractTypeMetaData(f.typeSignature), 109 | annotations = extractAnnotations(f) 110 | )) 111 | 112 | private def extractAnnotations(symbol: Symbol): Seq[AnnotationMetaData] = 113 | symbol 114 | .annotations 115 | .map(a => 116 | AnnotationMetaData( 117 | name = a.tree.tpe.toString, 118 | attributes = { 119 | val params = a.tree.tpe.decls.find(_.name.toString == "").get.asMethod.paramLists.flatten.map(f => f.name.toString) 120 | val values = a.tree.children.tail 121 | (params zip values) 122 | .map(pv => AnnotationAttributeMetaData( 123 | name = pv._1, 124 | value = pv._2.toString.stripPrefix("\"").stripSuffix("\"") 125 | )) 126 | } 127 | ) 128 | ) 129 | } 130 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/metadata/package.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg 2 | 3 | package object metadata { 4 | case class AnnotationAttributeMetaData(name: String, value: String) 5 | 6 | trait HasValueAttribute { 7 | val attributes: Seq[AnnotationAttributeMetaData] 8 | 9 | def getValue: String = { 10 | attributes.head.value 11 | } 12 | } 13 | 14 | case class AnnotationMetaData(name: String, attributes: Seq[AnnotationAttributeMetaData]) extends HasValueAttribute 15 | 16 | trait HasAnnotations { 17 | val annotations: Iterable[AnnotationMetaData] 18 | 19 | def getAnnotationValue(annotationName: String): Option[String] = { 20 | annotations.find(_.name == annotationName).map(_.getValue) 21 | } 22 | 23 | def annotationExists(annotationName: String): Boolean = { 24 | val annotation = annotations.find(_.name == annotationName) 25 | annotation.isDefined 26 | } 27 | 28 | def getAnnotationsByName(annotationName: String): Iterable[AnnotationMetaData] = 29 | annotations.filter(_.name == annotationName) 30 | } 31 | 32 | sealed trait TypeMetaData { 33 | val packageName: String 34 | val typeName: String 35 | } 36 | 37 | sealed abstract class PrimitiveTypeMetaData(val packageName: String, val typeName: String) extends TypeMetaData 38 | 39 | case object IntegerType extends PrimitiveTypeMetaData("scala", "Int") 40 | case object StringType extends PrimitiveTypeMetaData("scala", "String") 41 | case object LongType extends PrimitiveTypeMetaData("scala", "Long") 42 | case object DoubleType extends PrimitiveTypeMetaData("scala", "Double") 43 | case object FloatType extends PrimitiveTypeMetaData("scala", "Float") 44 | case object ShortType extends PrimitiveTypeMetaData("scala", "Short") 45 | case object ByteType extends PrimitiveTypeMetaData("scala", "Byte") 46 | case object BooleanType extends PrimitiveTypeMetaData("scala", "Boolean") 47 | case object BinaryType extends PrimitiveTypeMetaData("scala", "Binary") 48 | case object TimestampType extends PrimitiveTypeMetaData("java.sql", "Timestamp") 49 | case object DateType extends PrimitiveTypeMetaData("java.util", "Date") 50 | case object BigDecimalType extends PrimitiveTypeMetaData("scala", "BigDecimal") 51 | case object BigIntegerType extends PrimitiveTypeMetaData("scala.math", "BigInt") 52 | 53 | case class CollectionTypeMetaData( 54 | elementType: TypeMetaData 55 | ) extends TypeMetaData { 56 | val packageName: String = "scala" 57 | val typeName: String = "Seq" 58 | } 59 | 60 | case class MapTypeMetaData( 61 | keyType: TypeMetaData, 62 | valueType: TypeMetaData 63 | ) extends TypeMetaData { 64 | val packageName: String = "scala" 65 | val typeName: String = "Map" 66 | } 67 | 68 | case class ClassFieldMetaData( 69 | /** 70 | * Original Scala class field name 71 | */ 72 | originalFieldName: String, 73 | 74 | /** 75 | * Field name taking into account annotations 76 | */ 77 | fieldName: String, 78 | fieldType: TypeMetaData, 79 | annotations: Iterable[AnnotationMetaData] 80 | ) extends HasAnnotations 81 | 82 | case class ClassTypeMetaData( 83 | packageName: String, 84 | 85 | /** 86 | * Original Scala class name 87 | */ 88 | originalTypeName: String, 89 | 90 | /** 91 | * Type name taking into account annotations 92 | */ 93 | typeName: String, 94 | annotations: Iterable[AnnotationMetaData], 95 | fields: Iterable[ClassFieldMetaData] 96 | ) extends TypeMetaData with HasAnnotations 97 | 98 | } 99 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/repository/AvroSchemaRegistryRepository.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.repository 2 | 3 | trait AvroSchemaRegistryRepository { 4 | def registerSchema(subject: String, schema: String): Unit 5 | def subjects(): Iterable[String] 6 | def fetchSchema(subject: String, version: String): String 7 | } 8 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/repository/AvroSchemaRegistryRepositoryImpl.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.repository 2 | 3 | import scalaj.http._ 4 | import org.json4s._ 5 | import org.json4s.native.JsonMethods._ 6 | 7 | class AvroSchemaRegistryRepositoryImpl(url: String) extends AvroSchemaRegistryRepository { 8 | override def registerSchema(subject: String, schema: String): Unit = { 9 | val endpoint = url + "/subjects/" + subject + "/versions" 10 | val request = Http(endpoint) 11 | .header("Content-Type", "application/vnd.schemaregistry.v1+json") 12 | .postData(schema.replace("\r", "").replace("\n", "")) 13 | val response: HttpResponse[String] = request.asString 14 | if(response.code != 200) { 15 | throw new Exception(response.body) 16 | } 17 | } 18 | 19 | override def subjects(): Iterable[String] = { 20 | implicit val formats = DefaultFormats 21 | val endpoint = url + "/subjects" 22 | val response = Http(endpoint).asString 23 | val body = response.body 24 | val json = parse(body) 25 | json.extract[Array[String]] 26 | } 27 | 28 | override def fetchSchema(subject: String, version: String): String = { 29 | implicit val formats = DefaultFormats 30 | val endpoint = url + "/subjects/" + subject + "/versions/" + version 31 | val response = Http(endpoint).asString 32 | val body = response.body 33 | val json = parse(body) 34 | (json \ "schema").extract[String] 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/service/AvroSchemaRegistryService.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.service 2 | 3 | import com.datawizards.dmg.{DataModelGenerator, dialects} 4 | import com.datawizards.dmg.repository.AvroSchemaRegistryRepository 5 | 6 | import scala.reflect.ClassTag 7 | import scala.reflect.runtime.universe.TypeTag 8 | import org.apache.log4j.Logger 9 | 10 | trait AvroSchemaRegistryService { 11 | private val log = Logger.getLogger(getClass.getName) 12 | protected val repository: AvroSchemaRegistryRepository 13 | protected val hdfsService: HDFSService 14 | 15 | def generateSchema[T: ClassTag: TypeTag](variables: Map[String, String] = Map.empty): String = { 16 | TemplateHandler.inflate(DataModelGenerator.generate[T](dialects.AvroSchemaDialect), variables) 17 | } 18 | 19 | def generateSchemaForAvroSchemaRegistry[T: ClassTag: TypeTag](variables: Map[String, String] = Map.empty): String = { 20 | TemplateHandler.inflate(DataModelGenerator.generate[T](dialects.AvroSchemaRegistryDialect), variables) 21 | } 22 | 23 | def registerSchema[T: ClassTag: TypeTag](subject: String): Unit = { 24 | val schema = generateSchemaForAvroSchemaRegistry[T]() 25 | registerSchema(schema, subject) 26 | } 27 | 28 | def registerSchema(schema: String, subject: String): Unit = { 29 | repository.registerSchema(subject, schema) 30 | log.info(s"Registered schema [$subject] at avro schema registry.") 31 | } 32 | 33 | def subjects(): Iterable[String] = 34 | repository.subjects() 35 | 36 | def fetchSchema(subject: String, version: String): String = 37 | repository.fetchSchema(subject, version) 38 | 39 | def fetchSchema(subject: String): String = 40 | fetchSchema(subject, "latest") 41 | 42 | def copyAvroSchemaToHdfs(schema: String, hdfsPath: String): Unit = { 43 | val file = "/tmp/dmg_" + java.util.UUID.randomUUID().toString 44 | hdfsService.copyLocalFileToHDFS(file, hdfsPath) 45 | } 46 | 47 | def copyAvroSchemaToHdfs[T: ClassTag: TypeTag](hdfsPath: String): Unit = { 48 | val schema = generateSchema[T]() 49 | copyAvroSchemaToHdfs(schema, hdfsPath) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/service/AvroSchemaRegistryServiceImpl.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.service 2 | 3 | import com.datawizards.dmg.repository.{AvroSchemaRegistryRepository, AvroSchemaRegistryRepositoryImpl} 4 | 5 | class AvroSchemaRegistryServiceImpl(url: String) extends AvroSchemaRegistryService { 6 | override protected val repository: AvroSchemaRegistryRepository = new AvroSchemaRegistryRepositoryImpl(url) 7 | override protected val hdfsService: HDFSService = HDFSServiceImpl 8 | } 9 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/service/ConsoleCommandExecutor.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.service 2 | 3 | import org.apache.log4j.Logger 4 | import sys.process._ 5 | 6 | object ConsoleCommandExecutor { 7 | private val log = Logger.getLogger(getClass.getName) 8 | 9 | def execute(command: String): Unit = { 10 | log.info(s"Executing command: [$command]") 11 | val returnCode: Int = command.!<(ProcessLogger(outLine => log.info(outLine))) 12 | log.info(s"Command finished. Return code ${returnCode}") 13 | if(returnCode != 0){ 14 | throw new RuntimeException(s"Failed executing command: ${command}") 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/service/ElasticsearchService.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.service 2 | 3 | import com.datawizards.dmg.{DataModelGenerator, dialects} 4 | import com.datawizards.esclient.repository.ElasticsearchRepository 5 | import org.apache.log4j.Logger 6 | 7 | import scala.reflect.ClassTag 8 | import scala.reflect.runtime.universe.TypeTag 9 | 10 | trait ElasticsearchService { 11 | private val log = Logger.getLogger(getClass.getName) 12 | protected val repository: ElasticsearchRepository 13 | 14 | def updateTemplate[T: ClassTag: TypeTag](templateName: String, variables: Map[String, String] = Map.empty): Unit = { 15 | val template = TemplateHandler.inflate(DataModelGenerator.generate[T](dialects.ElasticsearchDialect), variables) 16 | repository.deleteTemplate(templateName) 17 | repository.updateTemplate(templateName, template) 18 | log.info(s"Updated template [$templateName] at Elasticsearch.") 19 | } 20 | 21 | def updateTemplateIfNotExists[T: ClassTag: TypeTag](templateName: String, variables: Map[String, String] = Map.empty): Unit = { 22 | val template = TemplateHandler.inflate(DataModelGenerator.generate[T](dialects.ElasticsearchDialect), variables) 23 | if(!repository.templateExists(templateName)) { 24 | repository.updateTemplate(templateName, template) 25 | log.info(s"Updated template [$templateName] at Elasticsearch.") 26 | } 27 | } 28 | 29 | def getTemplate(templateName: String): String = 30 | repository.getTemplate(templateName) 31 | 32 | def createIndexIfNotExists[T: ClassTag: TypeTag](indexName: String, variables: Map[String, String] = Map.empty): Unit = { 33 | val mapping = TemplateHandler.inflate(DataModelGenerator.generate[T](dialects.ElasticsearchDialect), variables) 34 | if(!repository.indexExists(indexName)) { 35 | repository.createIndex(indexName, mapping) 36 | log.info(s"Created index [$indexName] at Elasticsearch.") 37 | } 38 | } 39 | 40 | def createIndex[T: ClassTag: TypeTag](indexName: String, variables: Map[String, String] = Map.empty): Unit = { 41 | val mapping = TemplateHandler.inflate(DataModelGenerator.generate[T](dialects.ElasticsearchDialect), variables) 42 | repository.deleteIndex(indexName) 43 | repository.createIndex(indexName, mapping) 44 | log.info(s"Created index [$indexName] at Elasticsearch.") 45 | } 46 | 47 | def getIndexSettings(indexName: String): String = 48 | repository.getIndexSettings(indexName) 49 | } 50 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/service/ElasticsearchServiceImpl.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.service 2 | 3 | import com.datawizards.esclient.repository.{ElasticsearchRepository, ElasticsearchRepositoryImpl} 4 | 5 | class ElasticsearchServiceImpl(url: String) extends ElasticsearchService { 6 | override protected val repository: ElasticsearchRepository = new ElasticsearchRepositoryImpl(url) 7 | } 8 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/service/HDFSService.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.service 2 | 3 | trait HDFSService { 4 | def copyLocalFileToHDFS(localFilePath: String, hdfsPath: String): Unit 5 | } 6 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/service/HDFSServiceImpl.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.service 2 | 3 | object HDFSServiceImpl extends HDFSService { 4 | override def copyLocalFileToHDFS(localFilePath: String, hdfsPath: String): Unit = { 5 | val command = s"hdfs dfs -copyFromLocal $localFilePath $hdfsPath" 6 | ConsoleCommandExecutor.execute(command) 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/service/HiveService.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.service 2 | 3 | import scala.reflect.ClassTag 4 | import scala.reflect.runtime.universe.TypeTag 5 | 6 | trait HiveService { 7 | 8 | /** 9 | * Generates data model for provided type and creates new Hive table. 10 | * Before creating table it drops table with the same name. 11 | */ 12 | def createHiveTable[T: ClassTag: TypeTag](variables: Map[String, String] = Map.empty): Unit 13 | 14 | /** 15 | * Generates data model for provided type and creates new Hive table. 16 | */ 17 | def createHiveTableIfNotExists[T: ClassTag: TypeTag](variables: Map[String, String] = Map.empty): Unit 18 | } 19 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/service/HiveServiceImpl.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.service 2 | 3 | import com.datawizards.dmg.dialects.MetaDataWithDialectExtractor 4 | import com.datawizards.dmg.{DataModelGenerator, dialects} 5 | import org.apache.log4j.Logger 6 | 7 | import scala.reflect.ClassTag 8 | import scala.reflect.runtime.universe.TypeTag 9 | 10 | object HiveServiceImpl extends HiveService { 11 | private val log = Logger.getLogger(getClass.getName) 12 | 13 | class BatchCreateTable { 14 | val sqlScriptBuilder = new StringBuilder 15 | def createTable[T: ClassTag: TypeTag]: BatchCreateTable = { 16 | sqlScriptBuilder ++= buildCreateTableTemplate[T]() 17 | sqlScriptBuilder ++= "\n\n" 18 | this 19 | } 20 | 21 | def getScript: String = sqlScriptBuilder.toString() 22 | 23 | def execute(variables: Map[String, String] = Map.empty): Unit = { 24 | executeHiveScript(TemplateHandler.inflate(getScript, variables)) 25 | } 26 | } 27 | 28 | def batchCreateTable(): BatchCreateTable = new BatchCreateTable() 29 | 30 | override def createHiveTable[T: ClassTag: TypeTag](variables: Map[String, String] = Map.empty): Unit = { 31 | executeHiveScript(TemplateHandler.inflate(buildCreateTableTemplate[T](), variables)) 32 | } 33 | 34 | override def createHiveTableIfNotExists[T: ClassTag: TypeTag](variables: Map[String, String] = Map.empty): Unit = { 35 | val createTableExpression = DataModelGenerator.generate[T](dialects.HiveDialect) 36 | executeHiveScript(TemplateHandler.inflate(createTableExpression, variables)) 37 | } 38 | 39 | private def buildCreateTableTemplate[T: ClassTag: TypeTag](): String = { 40 | val classTypeMetaData = MetaDataWithDialectExtractor.extractClassMetaDataForDialect[T](Some(dialects.HiveDialect)) 41 | val tableName = classTypeMetaData.typeName 42 | val createTableExpression = DataModelGenerator.generate[T](dialects.HiveDialect, classTypeMetaData) 43 | val dropTableExpression = s"DROP TABLE IF EXISTS $tableName;\n" 44 | dropTableExpression + createTableExpression 45 | } 46 | 47 | private def executeHiveScript(sql: String): Unit = { 48 | val file = "/tmp/dmg_" + java.util.UUID.randomUUID().toString 49 | val pw = new java.io.PrintWriter(file) 50 | pw.write(sql) 51 | pw.close() 52 | val command = s"hive -f $file" 53 | ConsoleCommandExecutor.execute(command) 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/main/scala/com/datawizards/dmg/service/TemplateHandler.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.service 2 | 3 | object TemplateHandler { 4 | def inflate(template: String, variables: Map[String, String]): String = { 5 | val re = "\\$\\{.*?\\}".r 6 | var result = template 7 | for(k <- re.findAllIn(template)) 8 | result = result.replace(k, variables(k.substring(2, k.length-1))) 9 | result 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/DataModelGeneratorBaseTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg 2 | 3 | import org.scalatest.{FunSuite, Matchers} 4 | 5 | trait DataModelGeneratorBaseTest extends FunSuite with Matchers { 6 | def assertResultIgnoringNewLines(expected: String)(result: String): Unit = 7 | stripNewLines(result) should equal(stripNewLines(expected)) 8 | 9 | private def stripNewLines(v: String): String = v.replace("\n", "").replace("\r", "") 10 | } 11 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/ExtractClassMetaDataForDialectTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg 2 | 3 | import com.datawizards.dmg.dialects.MetaDataWithDialectExtractor 4 | import com.datawizards.dmg.metadata.PersonFull 5 | import org.scalatest.{FunSuite, Matchers} 6 | 7 | class ExtractClassMetaDataForDialectTest extends FunSuite with Matchers { 8 | 9 | test("PersonFull - Elasticsearch") { 10 | val a = MetaDataWithDialectExtractor.extractClassMetaDataForDialect[PersonFull](Some(com.datawizards.dmg.dialects.ElasticsearchDialect)) 11 | 12 | println(a) 13 | 14 | a.typeName should equal("person_full_es") 15 | a.fields.size should equal (3) 16 | a.fields.toList(0).fieldName should equal("personName") 17 | a.fields.toList(1).fieldName should equal("personAge") 18 | a.fields.toList(2).fieldName should equal("personTitle") 19 | } 20 | 21 | test("PersonFull - MySQL") { 22 | val a = MetaDataWithDialectExtractor.extractClassMetaDataForDialect[PersonFull](Some(com.datawizards.dmg.dialects.MySQLDialect)) 23 | 24 | 25 | println(a) 26 | 27 | a.typeName should equal("person_full") 28 | a.fields.toList(0).fieldName should equal("person_name") 29 | a.fields.toList(1).fieldName should equal("person_age") 30 | a.fields.toList(2).fieldName should equal("title") 31 | } 32 | 33 | 34 | test("PersonFull - None dialect") { 35 | val a = MetaDataWithDialectExtractor.extractClassMetaDataForDialect[PersonFull](None) 36 | 37 | println(a) 38 | 39 | a.typeName should equal("person_full") 40 | a.fields.toList(0).fieldName should equal("person_name") 41 | a.fields.toList(1).fieldName should equal("person_age") 42 | a.fields.toList(2).fieldName should equal("title") 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/TestModel.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg 2 | 3 | import java.sql.Timestamp 4 | import java.sql.Date 5 | 6 | import com.datawizards.dmg.annotations._ 7 | import com.datawizards.dmg.annotations.hive._ 8 | import com.datawizards.dmg.annotations.es._ 9 | 10 | object TestModel { 11 | case class Person(name: String, age: Int) 12 | case class ClassWithAllSimpleTypes( 13 | strVal: String, 14 | intVal: Int, 15 | longVal: Long, 16 | doubleVal: Double, 17 | floatVal: Float, 18 | shortVal: Short, 19 | booleanVal: Boolean, 20 | byteVal: Byte, 21 | dateVal: Date, 22 | timestampVal: Timestamp 23 | ) 24 | 25 | @table("PEOPLE") 26 | case class PersonWithCustomName( 27 | @column("personName") name: String, 28 | age: Int 29 | ) 30 | 31 | @comment("People data") 32 | case class PersonWithComments( 33 | @comment("Person name") name: String, 34 | age: Int 35 | ) 36 | 37 | case class PersonWithCustomLength( 38 | @length(1000) name: String, 39 | age: Int 40 | ) 41 | 42 | @table("people") 43 | @table("PEOPLE", dialects.H2Dialect) 44 | @table("person", dialects.ElasticsearchDialect) 45 | case class PersonWithMultipleCustomNames( 46 | @column("NAME", dialects.H2Dialect) 47 | @column("personNameEs", dialects.ElasticsearchDialect) 48 | name: String, 49 | @column("personAge") 50 | @column("AGE", dialects.H2Dialect) 51 | @column("personAgeEs", dialects.ElasticsearchDialect) 52 | age: Int 53 | ) 54 | 55 | case class CV(skills: Seq[String], grades: Seq[Int]) 56 | case class NestedArray(nested: Seq[Seq[String]], nested3: Seq[Seq[Seq[Int]]]) 57 | case class Book(title: String, year: Int, owner: Person, authors: Seq[Person]) 58 | 59 | @hiveExternalTable(location="hdfs:///data/people") 60 | case class PersonExternalTable(name: String, age: Int) 61 | 62 | @hiveStoredAs(format="PARQUET") 63 | case class PersonStoredAsParquet(name: String, age: Int) 64 | 65 | @hiveStoredAs(format="INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'") 66 | case class PersonStoredAsAvro(name: String, age: Int) 67 | 68 | @hiveRowFormatSerde(format="org.apache.hadoop.hive.serde2.avro.AvroSerDe") 69 | case class PersonRowFormatSerde(name: String, age: Int) 70 | 71 | @hiveTableProperty("key1", "value1") 72 | @hiveTableProperty("key2", "value2") 73 | @hiveTableProperty("key3", "value3") 74 | case class PersonMultipleTableProperties(name: String, age: Int) 75 | 76 | @hiveTableProperty("avro.schema.url", "hdfs:///metadata/person.avro") 77 | case class PersonAvroSchemaURL(name: String, age: Int) 78 | 79 | case class PersonWithNull( 80 | @notNull 81 | name: String, 82 | age: Int 83 | ) 84 | 85 | case class ClicksPartitioned( 86 | time: Timestamp, 87 | event: String, 88 | user: String, 89 | @hivePartitionColumn 90 | year: Int, 91 | @hivePartitionColumn 92 | month: Int, 93 | @hivePartitionColumn 94 | day: Int 95 | ) 96 | 97 | case class ClicksPartitionedWithOrder( 98 | time: Timestamp, 99 | event: String, 100 | user: String, 101 | @hivePartitionColumn(order=3) 102 | day: Int, 103 | @hivePartitionColumn(order=1) 104 | year: Int, 105 | @hivePartitionColumn(order=2) 106 | month: Int 107 | ) 108 | 109 | @table("CUSTOM_TABLE_NAME") 110 | @comment("Table comment") 111 | @hiveStoredAs(format="PARQUET") 112 | @hiveExternalTable(location="hdfs:///data/table") 113 | @hiveTableProperty("key1", "value1") 114 | @hiveTableProperty("key2", "value2") 115 | @hiveTableProperty("key3", "value3") 116 | case class ParquetTableWithManyAnnotations( 117 | @column("eventTime") 118 | @comment("Event time") 119 | time: Timestamp, 120 | @comment("Event name") 121 | event: String, 122 | @comment("User id") 123 | user: String, 124 | @hivePartitionColumn(order=3) 125 | day: Int, 126 | @hivePartitionColumn(order=1) 127 | year: Int, 128 | @hivePartitionColumn(order=2) 129 | month: Int 130 | ) 131 | 132 | @table("CUSTOM_TABLE_NAME") 133 | @comment("Table comment") 134 | @hiveRowFormatSerde(format="org.apache.hadoop.hive.serde2.avro.AvroSerDe") 135 | @hiveStoredAs("INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'") 136 | @hiveExternalTable(location="hdfs:///data/table") 137 | @hiveTableProperty("avro.schema.url", "hdfs:///metadata/table.avro") 138 | @hiveTableProperty("key1", "value1") 139 | @hiveTableProperty("key2", "value2") 140 | @hiveTableProperty("key3", "value3") 141 | case class AvroTableWithManyAnnotations( 142 | @column("eventTime") 143 | @comment("Event time") 144 | time: Timestamp, 145 | @comment("Event name") 146 | event: String, 147 | @comment("User id") 148 | user: String, 149 | @hivePartitionColumn(order=3) 150 | day: Int, 151 | @hivePartitionColumn(order=1) 152 | year: Int, 153 | @hivePartitionColumn(order=2) 154 | month: Int 155 | ) 156 | 157 | case class PersonEsIndexSettings( 158 | @esIndex("not_analyzed") name: String, 159 | age: Int 160 | ) 161 | 162 | case class PersonWithDateFormat( 163 | name: String, 164 | @esFormat("yyyy-MM-dd") birthday: Date 165 | ) 166 | 167 | @esSetting("number_of_shards", 1) 168 | @esSetting("number_of_replicas", 3) 169 | @esSetting("blocks.read_only", true) 170 | @esSetting("codec", "best_compression") 171 | case class PersonWithIndexSettings(name: String, age: Int) 172 | 173 | @esTemplate("people*") 174 | case class PersonWithEsTemplate(name: String, age: Int) 175 | 176 | @table("people") 177 | @esTemplate("people*") 178 | @esSetting("number_of_shards", 1) 179 | @esSetting("number_of_replicas", 3) 180 | case class PersonWithMultipleEsAnnotations( 181 | @esIndex("not_analyzed") 182 | @column("personName") 183 | name: String, 184 | @column("personBirthday") 185 | @esFormat("yyyy-MM-dd") 186 | birthday: Date 187 | ) 188 | 189 | @underscore(dialect = dialects.H2Dialect) 190 | case class PersonWithUnderscore( 191 | personName: String, 192 | personAge: Int 193 | ) 194 | 195 | @underscore(dialect = dialects.H2Dialect) 196 | @underscore(dialect = dialects.HiveDialect) 197 | @underscore(dialect = dialects.RedshiftDialect) 198 | @table("PEOPLE", dialect = dialects.HiveDialect) 199 | case class PersonWithUnderscoreWithMultipleNames( 200 | @column("name", dialect = dialects.HiveDialect) 201 | @column("name", dialect = dialects.RedshiftDialect) 202 | personName: String, 203 | @column("age", dialect = dialects.RedshiftDialect) 204 | personAge: Int 205 | ) 206 | 207 | @table("${table_name}") 208 | case class PersonWithPlaceholderVariables( 209 | @comment("Person ${name_comment}") name: String, 210 | age: Int 211 | ) 212 | 213 | case class ClassWithMap(map: Map[Int, Boolean]) 214 | case class ClassWithDash(`add-id`: String) 215 | case class ClassWithReservedKeywords(select: String, where: String) 216 | case class ClassWithArrayByte(arr: Array[Byte]) 217 | case class ClassWithBigInteger(n1: BigInt) 218 | case class ClassWithBigDecimal(n1: BigDecimal) 219 | 220 | 221 | @comment(value="mysql comment", dialect=dialects.MySQLDialect) 222 | @comment(value="hive comment", dialect=dialects.HiveDialect) 223 | @underscore(dialect=dialects.HiveDialect) 224 | case class ClassWithMultipleDialects( 225 | @comment(value="mysql comment 2", dialect=dialects.MySQLDialect) 226 | @comment(value="hive comment 2", dialect=dialects.HiveDialect) 227 | @length(value=200, dialect=dialects.MySQLDialect) 228 | someColumn: String, 229 | 230 | @comment(value="mysql comment 3", dialect=dialects.MySQLDialect) 231 | @comment(value="general comment 3") 232 | @notNull(dialect=dialects.MySQLDialect) 233 | anotherColumn: Integer 234 | ) 235 | 236 | @underscore 237 | case class DefaultUnderscore( 238 | @length(123) 239 | @notNull 240 | @comment("asdf") 241 | someColumn: String 242 | ) 243 | } 244 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/customizations/MultipleCustomNamesTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.customizations 2 | 3 | import com.datawizards.dmg.TestModel._ 4 | import com.datawizards.dmg.dialects._ 5 | import com.datawizards.dmg.{DataModelGenerator, DataModelGeneratorBaseTest} 6 | import org.junit.runner.RunWith 7 | import org.scalatest.junit.JUnitRunner 8 | 9 | @RunWith(classOf[JUnitRunner]) 10 | class MultipleCustomNamesTest extends DataModelGeneratorBaseTest { 11 | 12 | test("H2") { 13 | val expected = 14 | """CREATE TABLE PEOPLE( 15 | | NAME VARCHAR, 16 | | AGE INT 17 | |);""".stripMargin 18 | 19 | assertResultIgnoringNewLines(expected) { 20 | DataModelGenerator.generate[PersonWithMultipleCustomNames](H2Dialect) 21 | } 22 | } 23 | 24 | test("Hive") { 25 | val expected = 26 | """CREATE TABLE people( 27 | | name STRING, 28 | | personAge INT 29 | |);""".stripMargin 30 | 31 | assertResultIgnoringNewLines(expected) { 32 | DataModelGenerator.generate[PersonWithMultipleCustomNames](HiveDialect) 33 | } 34 | } 35 | 36 | test("Hive 2") { 37 | val expected ="""CREATE TABLE class_with_multiple_dialects( 38 | | some_column STRING COMMENT 'hive comment 2', 39 | | another_column INT COMMENT 'general comment 3' 40 | |)COMMENT 'hive comment';""" 41 | .stripMargin 42 | 43 | assertResultIgnoringNewLines(expected) { 44 | DataModelGenerator.generate[ClassWithMultipleDialects](HiveDialect) 45 | } 46 | } 47 | 48 | test("MySQL 2") { 49 | val expected = 50 | """CREATE TABLE ClassWithMultipleDialects( 51 | | someColumn VARCHAR(200) COMMENT 'mysql comment 2', 52 | | anotherColumn INT NOT NULL COMMENT 'mysql comment 3' 53 | |)COMMENT = 'mysql comment';""" 54 | .stripMargin 55 | 56 | assertResultIgnoringNewLines(expected) { 57 | DataModelGenerator.generate[ClassWithMultipleDialects](MySQLDialect) 58 | } 59 | } 60 | 61 | test("DefaultUnderscore MySQL 2") { 62 | val expected = 63 | """CREATE TABLE default_underscore( 64 | | some_column VARCHAR(123) NOT NULL COMMENT 'asdf' 65 | |);""" 66 | .stripMargin 67 | 68 | assertResultIgnoringNewLines(expected) { 69 | DataModelGenerator.generate[DefaultUnderscore](MySQLDialect) 70 | } 71 | } 72 | 73 | test("Redshift") { 74 | val expected = 75 | """CREATE TABLE people( 76 | | name VARCHAR, 77 | | personAge INTEGER 78 | |);""".stripMargin 79 | 80 | assertResultIgnoringNewLines(expected) { 81 | DataModelGenerator.generate[PersonWithMultipleCustomNames](RedshiftDialect) 82 | } 83 | } 84 | 85 | test("Avro schema") { 86 | val expected = 87 | """{ 88 | | "namespace": "com.datawizards.dmg", 89 | | "type": "record", 90 | | "name": "people", 91 | | "fields": [ 92 | | {"name": "name", "type": ["null", "string"]}, 93 | | {"name": "personAge", "type": ["null", "int"]} 94 | | ] 95 | |}""".stripMargin 96 | 97 | assertResultIgnoringNewLines(expected) { 98 | DataModelGenerator.generate[PersonWithMultipleCustomNames](AvroSchemaDialect) 99 | } 100 | } 101 | 102 | test("Elasticsearch") { 103 | val expected = 104 | """{ 105 | | "mappings" : { 106 | | "person" : { 107 | | "properties" : { 108 | | "personNameEs" : {"type" : "string"}, 109 | | "personAgeEs" : {"type" : "integer"} 110 | | } 111 | | } 112 | | } 113 | |}""".stripMargin 114 | 115 | assertResultIgnoringNewLines(expected) { 116 | DataModelGenerator.generate[PersonWithMultipleCustomNames](ElasticsearchDialect) 117 | } 118 | } 119 | 120 | } 121 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/customizations/NullableTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.customizations 2 | 3 | import com.datawizards.dmg.TestModel._ 4 | import com.datawizards.dmg.dialects._ 5 | import com.datawizards.dmg.{DataModelGenerator, DataModelGeneratorBaseTest} 6 | import org.junit.runner.RunWith 7 | import org.scalatest.junit.JUnitRunner 8 | 9 | @RunWith(classOf[JUnitRunner]) 10 | class NullableTest extends DataModelGeneratorBaseTest { 11 | 12 | test("H2") { 13 | val expected = 14 | """CREATE TABLE PersonWithNull( 15 | | name VARCHAR NOT NULL, 16 | | age INT 17 | |);""".stripMargin 18 | 19 | assertResultIgnoringNewLines(expected) { 20 | DataModelGenerator.generate[PersonWithNull](H2Dialect) 21 | } 22 | } 23 | 24 | test("Hive") { 25 | val expected = 26 | """CREATE TABLE PersonWithNull( 27 | | name STRING, 28 | | age INT 29 | |);""".stripMargin 30 | 31 | assertResultIgnoringNewLines(expected) { 32 | DataModelGenerator.generate[PersonWithNull](HiveDialect) 33 | } 34 | } 35 | 36 | test("Redshift") { 37 | val expected = 38 | """CREATE TABLE PersonWithNull( 39 | | name VARCHAR NOT NULL, 40 | | age INTEGER 41 | |);""".stripMargin 42 | 43 | assertResultIgnoringNewLines(expected) { 44 | DataModelGenerator.generate[PersonWithNull](RedshiftDialect) 45 | } 46 | } 47 | 48 | test("MySQL") { 49 | val expected = 50 | """CREATE TABLE PersonWithNull( 51 | | name VARCHAR NOT NULL, 52 | | age INT 53 | |);""".stripMargin 54 | 55 | assertResultIgnoringNewLines(expected) { 56 | DataModelGenerator.generate[PersonWithNull](MySQLDialect) 57 | } 58 | } 59 | 60 | test("Avro schema") { 61 | val expected = 62 | """{ 63 | | "namespace": "com.datawizards.dmg", 64 | | "type": "record", 65 | | "name": "PersonWithNull", 66 | | "fields": [ 67 | | {"name": "name", "type": "string"}, 68 | | {"name": "age", "type": ["null", "int"]} 69 | | ] 70 | |}""".stripMargin 71 | 72 | assertResultIgnoringNewLines(expected) { 73 | DataModelGenerator.generate[PersonWithNull](AvroSchemaDialect) 74 | } 75 | } 76 | 77 | test("Elasticsearch") { 78 | val expected = 79 | """{ 80 | | "mappings" : { 81 | | "PersonWithNull" : { 82 | | "properties" : { 83 | | "name" : {"type" : "string"}, 84 | | "age" : {"type" : "integer"} 85 | | } 86 | | } 87 | | } 88 | |}""".stripMargin 89 | 90 | assertResultIgnoringNewLines(expected) { 91 | DataModelGenerator.generate[PersonWithNull](ElasticsearchDialect) 92 | } 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/customizations/PlaceholdersTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.customizations 2 | 3 | import com.datawizards.dmg.TestModel.PersonWithPlaceholderVariables 4 | import com.datawizards.dmg.dialects.H2Dialect 5 | import com.datawizards.dmg.service.TemplateHandler 6 | import com.datawizards.dmg.{DataModelGenerator, DataModelGeneratorBaseTest} 7 | import org.junit.runner.RunWith 8 | import org.scalatest.junit.JUnitRunner 9 | 10 | @RunWith(classOf[JUnitRunner]) 11 | class PlaceholdersTest extends DataModelGeneratorBaseTest { 12 | 13 | test("Placeholder variables") { 14 | val expected = 15 | """CREATE TABLE SCHEMA.PEOPLE( 16 | | name VARCHAR COMMENT 'Person first and last name', 17 | | age INT 18 | |);""".stripMargin 19 | 20 | assertResultIgnoringNewLines(expected) { 21 | TemplateHandler.inflate(DataModelGenerator.generate[PersonWithPlaceholderVariables](H2Dialect), Map("table_name" -> "SCHEMA.PEOPLE", "name_comment" -> "first and last name")) 22 | } 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/customizations/UnderscoreConversionTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.customizations 2 | 3 | import com.datawizards.dmg.TestModel._ 4 | import com.datawizards.dmg.dialects.{H2Dialect, HiveDialect, MySQLDialect, RedshiftDialect} 5 | import com.datawizards.dmg.{DataModelGenerator, DataModelGeneratorBaseTest} 6 | import org.junit.runner.RunWith 7 | import org.scalatest.junit.JUnitRunner 8 | 9 | @RunWith(classOf[JUnitRunner]) 10 | class UnderscoreConversionTest extends DataModelGeneratorBaseTest { 11 | 12 | test("Underscore conversion - H2") { 13 | val expected = 14 | """CREATE TABLE person_with_underscore_with_multiple_names( 15 | | person_name VARCHAR, 16 | | person_age INT 17 | |);""".stripMargin 18 | 19 | assertResultIgnoringNewLines(expected) { 20 | DataModelGenerator.generate[PersonWithUnderscoreWithMultipleNames](H2Dialect) 21 | } 22 | } 23 | 24 | test("Underscore conversion - Hive") { 25 | val expected = 26 | """CREATE TABLE PEOPLE( 27 | | name STRING, 28 | | person_age INT 29 | |);""".stripMargin 30 | 31 | assertResultIgnoringNewLines(expected) { 32 | DataModelGenerator.generate[PersonWithUnderscoreWithMultipleNames](HiveDialect) 33 | } 34 | } 35 | 36 | test("Underscore conversion - Redshift") { 37 | val expected = 38 | """CREATE TABLE person_with_underscore_with_multiple_names( 39 | | name VARCHAR, 40 | | age INTEGER 41 | |);""".stripMargin 42 | 43 | assertResultIgnoringNewLines(expected) { 44 | DataModelGenerator.generate[PersonWithUnderscoreWithMultipleNames](RedshiftDialect) 45 | } 46 | } 47 | 48 | test("Underscore conversion - MySQL") { 49 | val expected = 50 | """CREATE TABLE PersonWithUnderscoreWithMultipleNames( 51 | | personName VARCHAR, 52 | | personAge INT 53 | |);""".stripMargin 54 | 55 | assertResultIgnoringNewLines(expected) { 56 | DataModelGenerator.generate[PersonWithUnderscoreWithMultipleNames](MySQLDialect) 57 | } 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/dialects/GenerateAvroSchemaRegistryTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | import com.datawizards.dmg.{DataModelGenerator, DataModelGeneratorBaseTest} 4 | import com.datawizards.dmg.TestModel.{ClassWithAllSimpleTypes, Person} 5 | import org.junit.runner.RunWith 6 | import org.scalatest.junit.JUnitRunner 7 | 8 | @RunWith(classOf[JUnitRunner]) 9 | class GenerateAvroSchemaRegistryTest extends DataModelGeneratorBaseTest { 10 | 11 | test("Simple model") { 12 | val expected = 13 | """{"schema": 14 | |"{ 15 | | \"namespace\": \"com.datawizards.dmg\", 16 | | \"type\": \"record\", 17 | | \"name\": \"Person\", 18 | | \"fields\": [ 19 | | {\"name\": \"name\", \"type\": [\"null\", \"string\"]}, 20 | | {\"name\": \"age\", \"type\": [\"null\", \"int\"]} 21 | | ] 22 | |}" 23 | |}""".stripMargin 24 | 25 | assertResultIgnoringNewLines(expected) { 26 | DataModelGenerator.generate[Person](AvroSchemaRegistryDialect) 27 | } 28 | } 29 | 30 | test("ClassWithAllSimpleTypes") { 31 | val expected = 32 | """{"schema": 33 | |"{ 34 | | \"namespace\": \"com.datawizards.dmg\", 35 | | \"type\": \"record\", 36 | | \"name\": \"ClassWithAllSimpleTypes\", 37 | | \"fields\": [ 38 | | {\"name\": \"strVal\", \"type\": [\"null\", \"string\"]}, 39 | | {\"name\": \"intVal\", \"type\": [\"null\", \"int\"]}, 40 | | {\"name\": \"longVal\", \"type\": [\"null\", \"long\"]}, 41 | | {\"name\": \"doubleVal\", \"type\": [\"null\", \"double\"]}, 42 | | {\"name\": \"floatVal\", \"type\": [\"null\", \"float\"]}, 43 | | {\"name\": \"shortVal\", \"type\": [\"null\", \"bytes\"]}, 44 | | {\"name\": \"booleanVal\", \"type\": [\"null\", \"boolean\"]}, 45 | | {\"name\": \"byteVal\", \"type\": [\"null\", \"bytes\"]}, 46 | | {\"name\": \"dateVal\", \"type\": [\"null\", \"int\"]}, 47 | | {\"name\": \"timestampVal\", \"type\": [\"null\", \"long\"]} 48 | | ] 49 | |}" 50 | |}""".stripMargin 51 | 52 | assertResultIgnoringNewLines(expected) { 53 | DataModelGenerator.generate[ClassWithAllSimpleTypes](AvroSchemaRegistryDialect) 54 | } 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/dialects/GenerateAvroSchemaTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | import com.datawizards.dmg.{DataModelGenerator, DataModelGeneratorBaseTest} 4 | import com.datawizards.dmg.TestModel._ 5 | import org.junit.runner.RunWith 6 | import org.scalatest.junit.JUnitRunner 7 | 8 | @RunWith(classOf[JUnitRunner]) 9 | class GenerateAvroSchemaTest extends DataModelGeneratorBaseTest { 10 | 11 | test("Simple model") { 12 | val expected = 13 | """{ 14 | | "namespace": "com.datawizards.dmg", 15 | | "type": "record", 16 | | "name": "Person", 17 | | "fields": [ 18 | | {"name": "name", "type": ["null", "string"]}, 19 | | {"name": "age", "type": ["null", "int"]} 20 | | ] 21 | |}""".stripMargin 22 | 23 | assertResultIgnoringNewLines(expected) { 24 | DataModelGenerator.generate[Person](AvroSchemaDialect) 25 | } 26 | } 27 | 28 | test("ClassWithAllSimpleTypes") { 29 | val expected = 30 | """{ 31 | | "namespace": "com.datawizards.dmg", 32 | | "type": "record", 33 | | "name": "ClassWithAllSimpleTypes", 34 | | "fields": [ 35 | | {"name": "strVal", "type": ["null", "string"]}, 36 | | {"name": "intVal", "type": ["null", "int"]}, 37 | | {"name": "longVal", "type": ["null", "long"]}, 38 | | {"name": "doubleVal", "type": ["null", "double"]}, 39 | | {"name": "floatVal", "type": ["null", "float"]}, 40 | | {"name": "shortVal", "type": ["null", "bytes"]}, 41 | | {"name": "booleanVal", "type": ["null", "boolean"]}, 42 | | {"name": "byteVal", "type": ["null", "bytes"]}, 43 | | {"name": "dateVal", "type": ["null", "int"]}, 44 | | {"name": "timestampVal", "type": ["null", "long"]} 45 | | ] 46 | |}""".stripMargin 47 | 48 | assertResultIgnoringNewLines(expected) { 49 | DataModelGenerator.generate[ClassWithAllSimpleTypes](AvroSchemaDialect) 50 | } 51 | } 52 | 53 | test("Table and column comment") { 54 | val expected = 55 | """{ 56 | | "namespace": "com.datawizards.dmg", 57 | | "type": "record", 58 | | "name": "PersonWithComments", 59 | | "doc": "People data", 60 | | "fields": [ 61 | | {"name": "name", "type": ["null", "string"], "doc": "Person name"}, 62 | | {"name": "age", "type": ["null", "int"]} 63 | | ] 64 | |}""".stripMargin 65 | 66 | assertResultIgnoringNewLines(expected) { 67 | DataModelGenerator.generate[PersonWithComments](AvroSchemaDialect) 68 | } 69 | } 70 | 71 | test("Array type") { 72 | val expected = 73 | """{ 74 | | "namespace": "com.datawizards.dmg", 75 | | "type": "record", 76 | | "name": "CV", 77 | | "fields": [ 78 | | {"name": "skills", "type": "array", "items": "string"}, 79 | | {"name": "grades", "type": "array", "items": "int"} 80 | | ] 81 | |}""".stripMargin 82 | 83 | assertResultIgnoringNewLines(expected) { 84 | DataModelGenerator.generate[CV](AvroSchemaDialect) 85 | } 86 | } 87 | 88 | test("Nested array type") { 89 | val expected = 90 | """{ 91 | | "namespace": "com.datawizards.dmg", 92 | | "type": "record", 93 | | "name": "NestedArray", 94 | | "fields": [ 95 | | {"name": "nested", "type": "array", "items": {"type": "array", "items": "string"}}, 96 | | {"name": "nested3", "type": "array", "items": {"type": "array", "items": {"type": "array", "items": "int"}}} 97 | | ] 98 | |}""".stripMargin 99 | 100 | assertResultIgnoringNewLines(expected) { 101 | DataModelGenerator.generate[NestedArray](AvroSchemaDialect) 102 | } 103 | } 104 | 105 | test("Struct types") { 106 | val expected = 107 | """{ 108 | | "namespace": "com.datawizards.dmg", 109 | | "type": "record", 110 | | "name": "Book", 111 | | "fields": [ 112 | | {"name": "title", "type": ["null", "string"]}, 113 | | {"name": "year", "type": ["null", "int"]}, 114 | | {"name": "owner", "type": "record", "fields": [{"name": "name", "type": "string"}, {"name": "age", "type": "int"}]}, 115 | | {"name": "authors", "type": "array", "items": {"type": "record", "fields": [{"name": "name", "type": "string"}, {"name": "age", "type": "int"}]}} 116 | | ] 117 | |}""".stripMargin 118 | 119 | assertResultIgnoringNewLines(expected) { 120 | DataModelGenerator.generate[Book](AvroSchemaDialect) 121 | } 122 | } 123 | 124 | test("Map type") { 125 | val expected = 126 | """{ 127 | | "namespace": "com.datawizards.dmg", 128 | | "type": "record", 129 | | "name": "ClassWithMap", 130 | | "fields": [ 131 | | {"name": "map", "type": "map", "values": "boolean"} 132 | | ] 133 | |}""".stripMargin 134 | 135 | assertResultIgnoringNewLines(expected) { 136 | DataModelGenerator.generate[ClassWithMap](AvroSchemaDialect) 137 | } 138 | } 139 | 140 | test("ClassWithArrayByte") { 141 | val expected = 142 | """{ 143 | | "namespace": "com.datawizards.dmg", 144 | | "type": "record", 145 | | "name": "ClassWithArrayByte", 146 | | "fields": [ 147 | | {"name": "arr", "type": "array", "items": "bytes"} 148 | | ] 149 | |}""".stripMargin 150 | 151 | assertResultIgnoringNewLines(expected) { 152 | DataModelGenerator.generate[ClassWithArrayByte](AvroSchemaDialect) 153 | } 154 | } 155 | 156 | test("ClassWithBigInteger") { 157 | val expected = 158 | """{ 159 | | "namespace": "com.datawizards.dmg", 160 | | "type": "record", 161 | | "name": "ClassWithBigInteger", 162 | | "fields": [ 163 | | {"name": "n1", "type": ["null", "double"]} 164 | | ] 165 | |}""".stripMargin 166 | 167 | assertResultIgnoringNewLines(expected) { 168 | DataModelGenerator.generate[ClassWithBigInteger](AvroSchemaDialect) 169 | } 170 | } 171 | 172 | test("ClassWithBigDecimal") { 173 | val expected = 174 | """{ 175 | | "namespace": "com.datawizards.dmg", 176 | | "type": "record", 177 | | "name": "ClassWithBigDecimal", 178 | | "fields": [ 179 | | {"name": "n1", "type": ["null", "double"]} 180 | | ] 181 | |}""".stripMargin 182 | 183 | assertResultIgnoringNewLines(expected) { 184 | DataModelGenerator.generate[ClassWithBigDecimal](AvroSchemaDialect) 185 | } 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/dialects/GenerateElasticsearchMappingTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | import com.datawizards.dmg.{DataModelGenerator, DataModelGeneratorBaseTest} 4 | import com.datawizards.dmg.TestModel._ 5 | import org.junit.runner.RunWith 6 | import org.scalatest.junit.JUnitRunner 7 | 8 | @RunWith(classOf[JUnitRunner]) 9 | class GenerateElasticsearchMappingTest extends DataModelGeneratorBaseTest { 10 | 11 | test("Simple model") { 12 | val expected = 13 | """{ 14 | | "mappings" : { 15 | | "Person" : { 16 | | "properties" : { 17 | | "name" : {"type" : "string"}, 18 | | "age" : {"type" : "integer"} 19 | | } 20 | | } 21 | | } 22 | |}""".stripMargin 23 | 24 | assertResultIgnoringNewLines(expected) { 25 | DataModelGenerator.generate[Person](ElasticsearchDialect) 26 | } 27 | } 28 | 29 | test("ClassWithAllSimpleTypes") { 30 | val expected = 31 | """{ 32 | | "mappings" : { 33 | | "ClassWithAllSimpleTypes" : { 34 | | "properties" : { 35 | | "strVal" : {"type" : "string"}, 36 | | "intVal" : {"type" : "integer"}, 37 | | "longVal" : {"type" : "long"}, 38 | | "doubleVal" : {"type" : "double"}, 39 | | "floatVal" : {"type" : "float"}, 40 | | "shortVal" : {"type" : "short"}, 41 | | "booleanVal" : {"type" : "boolean"}, 42 | | "byteVal" : {"type" : "byte"}, 43 | | "dateVal" : {"type" : "date"}, 44 | | "timestampVal" : {"type" : "date"} 45 | | } 46 | | } 47 | | } 48 | |}""".stripMargin 49 | 50 | assertResultIgnoringNewLines(expected) { 51 | DataModelGenerator.generate[ClassWithAllSimpleTypes](ElasticsearchDialect) 52 | } 53 | } 54 | 55 | test("Array type") { 56 | val expected = 57 | """{ 58 | | "mappings" : { 59 | | "CV" : { 60 | | "properties" : { 61 | | "skills" : {"type" : "string"}, 62 | | "grades" : {"type" : "integer"} 63 | | } 64 | | } 65 | | } 66 | |}""".stripMargin 67 | 68 | assertResultIgnoringNewLines(expected) { 69 | DataModelGenerator.generate[CV](ElasticsearchDialect) 70 | } 71 | } 72 | 73 | test("Nested array type") { 74 | val expected = 75 | """{ 76 | | "mappings" : { 77 | | "NestedArray" : { 78 | | "properties" : { 79 | | "nested" : {"type" : "string"}, 80 | | "nested3" : {"type" : "integer"} 81 | | } 82 | | } 83 | | } 84 | |}""".stripMargin 85 | 86 | assertResultIgnoringNewLines(expected) { 87 | DataModelGenerator.generate[NestedArray](ElasticsearchDialect) 88 | } 89 | } 90 | 91 | test("Struct types") { 92 | val expected = 93 | """{ 94 | | "mappings" : { 95 | | "Book" : { 96 | | "properties" : { 97 | | "title" : {"type" : "string"}, 98 | | "year" : {"type" : "integer"}, 99 | | "owner" : {"properties" : {"name" : {"type" : "string"}, "age" : {"type" : "integer"}}}, 100 | | "authors" : {"properties" : {"name" : {"type" : "string"}, "age" : {"type" : "integer"}}} 101 | | } 102 | | } 103 | | } 104 | |}""".stripMargin 105 | 106 | assertResultIgnoringNewLines(expected) { 107 | DataModelGenerator.generate[Book](ElasticsearchDialect) 108 | } 109 | } 110 | 111 | test("index option") { 112 | val expected = 113 | """{ 114 | | "mappings" : { 115 | | "PersonEsIndexSettings" : { 116 | | "properties" : { 117 | | "name" : {"type" : "string", "index" : "not_analyzed"}, 118 | | "age" : {"type" : "integer"} 119 | | } 120 | | } 121 | | } 122 | |}""".stripMargin 123 | 124 | assertResultIgnoringNewLines(expected) { 125 | DataModelGenerator.generate[PersonEsIndexSettings](ElasticsearchDialect) 126 | } 127 | } 128 | 129 | test("format option") { 130 | val expected = 131 | """{ 132 | | "mappings" : { 133 | | "PersonWithDateFormat" : { 134 | | "properties" : { 135 | | "name" : {"type" : "string"}, 136 | | "birthday" : {"type" : "date", "format" : "yyyy-MM-dd"} 137 | | } 138 | | } 139 | | } 140 | |}""".stripMargin 141 | 142 | assertResultIgnoringNewLines(expected) { 143 | DataModelGenerator.generate[PersonWithDateFormat](ElasticsearchDialect) 144 | } 145 | } 146 | 147 | test("index settings") { 148 | val expected = 149 | """{ 150 | | "settings" : { 151 | | "number_of_shards" : 1, 152 | | "number_of_replicas" : 3, 153 | | "blocks.read_only" : "true", 154 | | "codec" : "best_compression" 155 | | }, 156 | | "mappings" : { 157 | | "PersonWithIndexSettings" : { 158 | | "properties" : { 159 | | "name" : {"type" : "string"}, 160 | | "age" : {"type" : "integer"} 161 | | } 162 | | } 163 | | } 164 | |}""".stripMargin 165 | 166 | assertResultIgnoringNewLines(expected) { 167 | DataModelGenerator.generate[PersonWithIndexSettings](ElasticsearchDialect) 168 | } 169 | } 170 | 171 | test("Template") { 172 | val expected = 173 | """{ 174 | | "template" : "people*", 175 | | "mappings" : { 176 | | "PersonWithEsTemplate" : { 177 | | "properties" : { 178 | | "name" : {"type" : "string"}, 179 | | "age" : {"type" : "integer"} 180 | | } 181 | | } 182 | | } 183 | |}""".stripMargin 184 | 185 | assertResultIgnoringNewLines(expected) { 186 | DataModelGenerator.generate[PersonWithEsTemplate](ElasticsearchDialect) 187 | } 188 | } 189 | 190 | test("Multiple options") { 191 | val expected = 192 | """{ 193 | | "template" : "people*", 194 | | "settings" : { 195 | | "number_of_shards" : 1, 196 | | "number_of_replicas" : 3 197 | | }, 198 | | "mappings" : { 199 | | "people" : { 200 | | "properties" : { 201 | | "personName" : {"type" : "string", "index" : "not_analyzed"}, 202 | | "personBirthday" : {"type" : "date", "format" : "yyyy-MM-dd"} 203 | | } 204 | | } 205 | | } 206 | |}""".stripMargin 207 | 208 | assertResultIgnoringNewLines(expected) { 209 | DataModelGenerator.generate[PersonWithMultipleEsAnnotations](ElasticsearchDialect) 210 | } 211 | } 212 | 213 | test("Map type") { 214 | val expected = 215 | """{ 216 | | "mappings" : { 217 | | "ClassWithMap" : { 218 | | "properties" : { 219 | | "map" : {"type" : "string"} 220 | | } 221 | | } 222 | | } 223 | |}""".stripMargin 224 | 225 | assertResultIgnoringNewLines(expected) { 226 | DataModelGenerator.generate[ClassWithMap](ElasticsearchDialect) 227 | } 228 | } 229 | 230 | test("ClassWithArrayByte") { 231 | val expected = 232 | """{ 233 | | "mappings" : { 234 | | "ClassWithArrayByte" : { 235 | | "properties" : { 236 | | "arr" : {"type" : "byte"} 237 | | } 238 | | } 239 | | } 240 | |}""".stripMargin 241 | 242 | assertResultIgnoringNewLines(expected) { 243 | DataModelGenerator.generate[ClassWithArrayByte](ElasticsearchDialect) 244 | } 245 | } 246 | 247 | test("ClassWithBigInteger") { 248 | val expected = 249 | """{ 250 | | "mappings" : { 251 | | "ClassWithBigInteger" : { 252 | | "properties" : { 253 | | "n1" : {"type" : "double"} 254 | | } 255 | | } 256 | | } 257 | |}""".stripMargin 258 | 259 | assertResultIgnoringNewLines(expected) { 260 | DataModelGenerator.generate[ClassWithBigInteger](ElasticsearchDialect) 261 | } 262 | } 263 | 264 | test("ClassWithBigDecimal") { 265 | val expected = 266 | """{ 267 | | "mappings" : { 268 | | "ClassWithBigDecimal" : { 269 | | "properties" : { 270 | | "n1" : {"type" : "double"} 271 | | } 272 | | } 273 | | } 274 | |}""".stripMargin 275 | 276 | assertResultIgnoringNewLines(expected) { 277 | DataModelGenerator.generate[ClassWithBigDecimal](ElasticsearchDialect) 278 | } 279 | } 280 | } 281 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/dialects/GenerateH2ModelTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | import com.datawizards.dmg.{DataModelGenerator, DataModelGeneratorBaseTest} 4 | import com.datawizards.dmg.TestModel._ 5 | import org.junit.runner.RunWith 6 | import org.scalatest.junit.JUnitRunner 7 | 8 | @RunWith(classOf[JUnitRunner]) 9 | class GenerateH2ModelTest extends DataModelGeneratorBaseTest { 10 | 11 | test("Simple model") { 12 | val expected = 13 | """CREATE TABLE Person( 14 | | name VARCHAR, 15 | | age INT 16 | |);""".stripMargin 17 | 18 | assertResultIgnoringNewLines(expected) { 19 | DataModelGenerator.generate[Person](H2Dialect) 20 | } 21 | } 22 | 23 | test("ClassWithAllSimpleTypes") { 24 | val expected = 25 | """CREATE TABLE ClassWithAllSimpleTypes( 26 | | strVal VARCHAR, 27 | | intVal INT, 28 | | longVal BIGINT, 29 | | doubleVal DOUBLE, 30 | | floatVal REAL, 31 | | shortVal SMALLINT, 32 | | booleanVal BOOLEAN, 33 | | byteVal TINYINT, 34 | | dateVal DATE, 35 | | timestampVal TIMESTAMP 36 | |);""".stripMargin 37 | 38 | assertResultIgnoringNewLines(expected) { 39 | DataModelGenerator.generate[ClassWithAllSimpleTypes](H2Dialect) 40 | } 41 | } 42 | 43 | test("Custom column and table name") { 44 | val expected = 45 | """CREATE TABLE PEOPLE( 46 | | personName VARCHAR, 47 | | age INT 48 | |);""".stripMargin 49 | 50 | assertResultIgnoringNewLines(expected) { 51 | DataModelGenerator.generate[PersonWithCustomName](H2Dialect) 52 | } 53 | } 54 | 55 | test("Table and column comment") { 56 | val expected = 57 | """CREATE TABLE PersonWithComments( 58 | | name VARCHAR COMMENT 'Person name', 59 | | age INT 60 | |); 61 | |COMMENT ON TABLE PersonWithComments IS 'People data';""".stripMargin 62 | 63 | assertResultIgnoringNewLines(expected) { 64 | DataModelGenerator.generate[PersonWithComments](H2Dialect) 65 | } 66 | } 67 | 68 | test("Column length") { 69 | val expected = 70 | """CREATE TABLE PersonWithCustomLength( 71 | | name VARCHAR(1000), 72 | | age INT 73 | |);""".stripMargin 74 | 75 | assertResultIgnoringNewLines(expected) { 76 | DataModelGenerator.generate[PersonWithCustomLength](H2Dialect) 77 | } 78 | } 79 | 80 | test("Array type") { 81 | val expected = 82 | """CREATE TABLE CV( 83 | | skills ARRAY, 84 | | grades ARRAY 85 | |);""".stripMargin 86 | 87 | assertResultIgnoringNewLines(expected) { 88 | DataModelGenerator.generate[CV](H2Dialect) 89 | } 90 | } 91 | 92 | test("Nested array type") { 93 | val expected = 94 | """CREATE TABLE NestedArray( 95 | | nested ARRAY, 96 | | nested3 ARRAY 97 | |);""".stripMargin 98 | 99 | assertResultIgnoringNewLines(expected) { 100 | DataModelGenerator.generate[NestedArray](H2Dialect) 101 | } 102 | } 103 | 104 | test("Struct types") { 105 | val expected = 106 | """CREATE TABLE Book( 107 | | title VARCHAR, 108 | | year INT, 109 | | owner OTHER, 110 | | authors ARRAY 111 | |);""".stripMargin 112 | 113 | assertResultIgnoringNewLines(expected) { 114 | DataModelGenerator.generate[Book](H2Dialect) 115 | } 116 | } 117 | 118 | test("Underscore conversion") { 119 | val expected = 120 | """CREATE TABLE person_with_underscore( 121 | | person_name VARCHAR, 122 | | person_age INT 123 | |);""".stripMargin 124 | 125 | assertResultIgnoringNewLines(expected) { 126 | DataModelGenerator.generate[PersonWithUnderscore](H2Dialect) 127 | } 128 | } 129 | 130 | test("Map type") { 131 | val expected = 132 | """CREATE TABLE ClassWithMap( 133 | | map VARCHAR 134 | |);""".stripMargin 135 | 136 | assertResultIgnoringNewLines(expected) { 137 | DataModelGenerator.generate[ClassWithMap](H2Dialect) 138 | } 139 | } 140 | 141 | test("ClassWithDash") { 142 | val expected = 143 | """CREATE TABLE ClassWithDash( 144 | | "add-id" VARCHAR 145 | |);""".stripMargin 146 | 147 | assertResultIgnoringNewLines(expected) { 148 | DataModelGenerator.generate[ClassWithDash](H2Dialect) 149 | } 150 | } 151 | 152 | test("reserverd keywords") { 153 | val expected = 154 | """CREATE TABLE ClassWithReservedKeywords( 155 | | "select" VARCHAR, 156 | | "where" VARCHAR 157 | |);""".stripMargin 158 | 159 | assertResultIgnoringNewLines(expected) { 160 | DataModelGenerator.generate[ClassWithReservedKeywords](H2Dialect) 161 | } 162 | } 163 | 164 | test("ClassWithArrayByte") { 165 | val expected = 166 | """CREATE TABLE ClassWithArrayByte( 167 | | arr BINARY 168 | |);""".stripMargin 169 | 170 | assertResultIgnoringNewLines(expected) { 171 | DataModelGenerator.generate[ClassWithArrayByte](H2Dialect) 172 | } 173 | } 174 | 175 | test("ClassWithBigInteger") { 176 | val expected = 177 | """CREATE TABLE ClassWithBigInteger( 178 | | n1 BIGINT 179 | |);""".stripMargin 180 | 181 | assertResultIgnoringNewLines(expected) { 182 | DataModelGenerator.generate[ClassWithBigInteger](H2Dialect) 183 | } 184 | } 185 | 186 | test("ClassWithBigDecimal") { 187 | val expected = 188 | """CREATE TABLE ClassWithBigDecimal( 189 | | n1 DECIMAL(38,18) 190 | |);""".stripMargin 191 | 192 | assertResultIgnoringNewLines(expected) { 193 | DataModelGenerator.generate[ClassWithBigDecimal](H2Dialect) 194 | } 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/dialects/GenerateHiveModelTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | import com.datawizards.dmg.{DataModelGenerator, DataModelGeneratorBaseTest} 4 | import com.datawizards.dmg.TestModel._ 5 | import org.junit.runner.RunWith 6 | import org.scalatest.junit.JUnitRunner 7 | 8 | @RunWith(classOf[JUnitRunner]) 9 | class GenerateHiveModelTest extends DataModelGeneratorBaseTest { 10 | 11 | test("Simple model") { 12 | val expected = 13 | """CREATE TABLE Person( 14 | | name STRING, 15 | | age INT 16 | |);""".stripMargin 17 | 18 | assertResultIgnoringNewLines(expected) { 19 | DataModelGenerator.generate[Person](HiveDialect) 20 | } 21 | } 22 | 23 | test("ClassWithAllSimpleTypes") { 24 | val expected = 25 | """CREATE TABLE ClassWithAllSimpleTypes( 26 | | strVal STRING, 27 | | intVal INT, 28 | | longVal BIGINT, 29 | | doubleVal DOUBLE, 30 | | floatVal FLOAT, 31 | | shortVal SMALLINT, 32 | | booleanVal BOOLEAN, 33 | | byteVal TINYINT, 34 | | dateVal DATE, 35 | | timestampVal TIMESTAMP 36 | |);""".stripMargin 37 | 38 | assertResultIgnoringNewLines(expected) { 39 | DataModelGenerator.generate[ClassWithAllSimpleTypes](HiveDialect) 40 | } 41 | } 42 | 43 | test("Table and column comment") { 44 | val expected = 45 | """CREATE TABLE PersonWithComments( 46 | | name STRING COMMENT 'Person name', 47 | | age INT 48 | |) 49 | |COMMENT 'People data';""".stripMargin 50 | 51 | assertResultIgnoringNewLines(expected) { 52 | DataModelGenerator.generate[PersonWithComments](HiveDialect) 53 | } 54 | } 55 | 56 | test("Column length") { 57 | val expected = 58 | """CREATE TABLE PersonWithCustomLength( 59 | | name STRING(1000), 60 | | age INT 61 | |);""".stripMargin 62 | 63 | assertResultIgnoringNewLines(expected) { 64 | DataModelGenerator.generate[PersonWithCustomLength](HiveDialect) 65 | } 66 | } 67 | 68 | test("Array type") { 69 | val expected = 70 | """CREATE TABLE CV( 71 | | skills ARRAY, 72 | | grades ARRAY 73 | |);""".stripMargin 74 | 75 | assertResultIgnoringNewLines(expected) { 76 | DataModelGenerator.generate[CV](HiveDialect) 77 | } 78 | } 79 | 80 | test("Nested array type") { 81 | val expected = 82 | """CREATE TABLE NestedArray( 83 | | nested ARRAY>, 84 | | nested3 ARRAY>> 85 | |);""".stripMargin 86 | 87 | assertResultIgnoringNewLines(expected) { 88 | DataModelGenerator.generate[NestedArray](HiveDialect) 89 | } 90 | } 91 | 92 | test("Struct types") { 93 | val expected = 94 | """CREATE TABLE Book( 95 | | title STRING, 96 | | year INT, 97 | | owner STRUCT, 98 | | authors ARRAY> 99 | |);""".stripMargin 100 | 101 | assertResultIgnoringNewLines(expected) { 102 | DataModelGenerator.generate[Book](HiveDialect) 103 | } 104 | } 105 | 106 | test("External table") { 107 | val expected = 108 | """CREATE EXTERNAL TABLE PersonExternalTable( 109 | | name STRING, 110 | | age INT 111 | |) 112 | |LOCATION 'hdfs:///data/people';""".stripMargin 113 | 114 | assertResultIgnoringNewLines(expected) { 115 | DataModelGenerator.generate[PersonExternalTable](HiveDialect) 116 | } 117 | } 118 | 119 | test("STORED AS PARQUET") { 120 | val expected = 121 | """CREATE TABLE PersonStoredAsParquet( 122 | | name STRING, 123 | | age INT 124 | |) 125 | |STORED AS PARQUET;""".stripMargin 126 | 127 | assertResultIgnoringNewLines(expected) { 128 | DataModelGenerator.generate[PersonStoredAsParquet](HiveDialect) 129 | } 130 | } 131 | 132 | test("STORED AS avro") { 133 | val expected = 134 | """CREATE TABLE PersonStoredAsAvro( 135 | | name STRING, 136 | | age INT 137 | |) 138 | |STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat';""".stripMargin 139 | 140 | assertResultIgnoringNewLines(expected) { 141 | DataModelGenerator.generate[PersonStoredAsAvro](HiveDialect) 142 | } 143 | } 144 | 145 | test("ROW FORMAT SERDE") { 146 | val expected = 147 | """CREATE TABLE PersonRowFormatSerde( 148 | | name STRING, 149 | | age INT 150 | |) 151 | |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe';""".stripMargin 152 | 153 | assertResultIgnoringNewLines(expected) { 154 | DataModelGenerator.generate[PersonRowFormatSerde](HiveDialect) 155 | } 156 | } 157 | 158 | test("Multiple table properties") { 159 | val expected = 160 | """CREATE TABLE PersonMultipleTableProperties( 161 | | name STRING, 162 | | age INT 163 | |) 164 | |TBLPROPERTIES( 165 | | 'key1' = 'value1', 166 | | 'key2' = 'value2', 167 | | 'key3' = 'value3' 168 | |);""".stripMargin 169 | 170 | assertResultIgnoringNewLines(expected) { 171 | DataModelGenerator.generate[PersonMultipleTableProperties](HiveDialect) 172 | } 173 | } 174 | 175 | test("Table properties - avro schema url") { 176 | val expected = 177 | """CREATE TABLE PersonAvroSchemaURL 178 | |TBLPROPERTIES( 179 | | 'avro.schema.url' = 'hdfs:///metadata/person.avro' 180 | |);""".stripMargin 181 | 182 | assertResultIgnoringNewLines(expected) { 183 | DataModelGenerator.generate[PersonAvroSchemaURL](HiveDialect) 184 | } 185 | } 186 | 187 | test("Paritioned by") { 188 | val expected = 189 | """CREATE TABLE ClicksPartitioned( 190 | | time TIMESTAMP, 191 | | event STRING, 192 | | `user` STRING 193 | |) 194 | |PARTITIONED BY(year INT, month INT, day INT);""".stripMargin 195 | 196 | assertResultIgnoringNewLines(expected) { 197 | DataModelGenerator.generate[ClicksPartitioned](HiveDialect) 198 | } 199 | } 200 | 201 | test("Paritioned by - order") { 202 | val expected = 203 | """CREATE TABLE ClicksPartitionedWithOrder( 204 | | time TIMESTAMP, 205 | | event STRING, 206 | | `user` STRING 207 | |) 208 | |PARTITIONED BY(year INT, month INT, day INT);""".stripMargin 209 | 210 | assertResultIgnoringNewLines(expected) { 211 | DataModelGenerator.generate[ClicksPartitionedWithOrder](HiveDialect) 212 | } 213 | } 214 | 215 | test("ParquetTableWithManyAnnotations") { 216 | val expected = 217 | """CREATE EXTERNAL TABLE CUSTOM_TABLE_NAME( 218 | | eventTime TIMESTAMP COMMENT 'Event time', 219 | | event STRING COMMENT 'Event name', 220 | | `user` STRING COMMENT 'User id' 221 | |) 222 | |COMMENT 'Table comment' 223 | |PARTITIONED BY(year INT, month INT, day INT) 224 | |STORED AS PARQUET 225 | |LOCATION 'hdfs:///data/table' 226 | |TBLPROPERTIES( 227 | | 'key1' = 'value1', 228 | | 'key2' = 'value2', 229 | | 'key3' = 'value3' 230 | |); 231 | |MSCK REPAIR TABLE CUSTOM_TABLE_NAME;""".stripMargin 232 | 233 | println(expected) 234 | println(DataModelGenerator.generate[ParquetTableWithManyAnnotations](HiveDialect)) 235 | 236 | assertResultIgnoringNewLines(expected) { 237 | DataModelGenerator.generate[ParquetTableWithManyAnnotations](HiveDialect) 238 | } 239 | } 240 | 241 | test("AvroTableWithManyAnnotations") { 242 | val expected = 243 | """CREATE EXTERNAL TABLE CUSTOM_TABLE_NAME 244 | |COMMENT 'Table comment' 245 | |PARTITIONED BY(year INT, month INT, day INT) 246 | |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' 247 | |STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' 248 | |LOCATION 'hdfs:///data/table' 249 | |TBLPROPERTIES( 250 | | 'avro.schema.url' = 'hdfs:///metadata/table.avro', 251 | | 'key1' = 'value1', 252 | | 'key2' = 'value2', 253 | | 'key3' = 'value3' 254 | |); 255 | |MSCK REPAIR TABLE CUSTOM_TABLE_NAME;""".stripMargin 256 | 257 | assertResultIgnoringNewLines(expected) { 258 | DataModelGenerator.generate[AvroTableWithManyAnnotations](HiveDialect) 259 | } 260 | } 261 | 262 | test("Map type") { 263 | val expected = 264 | """CREATE TABLE ClassWithMap( 265 | | map MAP 266 | |);""".stripMargin 267 | 268 | assertResultIgnoringNewLines(expected) { 269 | DataModelGenerator.generate[ClassWithMap](HiveDialect) 270 | } 271 | } 272 | 273 | test("ClassWithDash") { 274 | val expected = 275 | """CREATE TABLE ClassWithDash( 276 | | `add-id` STRING 277 | |);""".stripMargin 278 | 279 | assertResultIgnoringNewLines(expected) { 280 | DataModelGenerator.generate[ClassWithDash](HiveDialect) 281 | } 282 | } 283 | 284 | test("reserverd keywords") { 285 | val expected = 286 | """CREATE TABLE ClassWithReservedKeywords( 287 | | `select` STRING, 288 | | `where` STRING 289 | |);""".stripMargin 290 | 291 | assertResultIgnoringNewLines(expected) { 292 | DataModelGenerator.generate[ClassWithReservedKeywords](HiveDialect) 293 | } 294 | } 295 | 296 | test("ClassWithArrayByte") { 297 | val expected = 298 | """CREATE TABLE ClassWithArrayByte( 299 | | arr BINARY 300 | |);""".stripMargin 301 | 302 | assertResultIgnoringNewLines(expected) { 303 | DataModelGenerator.generate[ClassWithArrayByte](HiveDialect) 304 | } 305 | } 306 | 307 | test("ClassWithBigInteger") { 308 | val expected = 309 | """CREATE TABLE ClassWithBigInteger( 310 | | n1 BIGINT 311 | |);""".stripMargin 312 | 313 | assertResultIgnoringNewLines(expected) { 314 | DataModelGenerator.generate[ClassWithBigInteger](HiveDialect) 315 | } 316 | } 317 | 318 | test("ClassWithBigDecimal") { 319 | val expected = 320 | """CREATE TABLE ClassWithBigDecimal( 321 | | n1 DECIMAL(38,18) 322 | |);""".stripMargin 323 | 324 | assertResultIgnoringNewLines(expected) { 325 | DataModelGenerator.generate[ClassWithBigDecimal](HiveDialect) 326 | } 327 | } 328 | 329 | } 330 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/dialects/GenerateJavaClassTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | import com.datawizards.dmg.TestModel._ 4 | import com.datawizards.dmg.{DataModelGenerator, DataModelGeneratorBaseTest} 5 | import org.junit.runner.RunWith 6 | import org.scalatest.junit.JUnitRunner 7 | 8 | @RunWith(classOf[JUnitRunner]) 9 | class GenerateJavaClassTest extends DataModelGeneratorBaseTest { 10 | 11 | test("Simple model") { 12 | val expected = 13 | """public class Person { 14 | | private String name; 15 | | private Integer age; 16 | | 17 | | public Person() {} 18 | | 19 | | public Person(String name, Integer age) { 20 | | this.name = name; 21 | | this.age = age; 22 | | } 23 | | 24 | | public String getName() { 25 | | return name; 26 | | } 27 | | 28 | | public void setName(String name) { 29 | | this.name = name; 30 | | } 31 | | 32 | | public Integer getAge() { 33 | | return age; 34 | | } 35 | | 36 | | public void setAge(Integer age) { 37 | | this.age = age; 38 | | } 39 | |}""".stripMargin 40 | 41 | assertResultIgnoringNewLines(expected) { 42 | DataModelGenerator.generate[Person](JavaDialect) 43 | } 44 | } 45 | 46 | test("ClassWithAllSimpleTypes") { 47 | val expected = 48 | """public class ClassWithAllSimpleTypes { 49 | | private String strVal; 50 | | private Integer intVal; 51 | | private Long longVal; 52 | | private Double doubleVal; 53 | | private Float floatVal; 54 | | private Short shortVal; 55 | | private Boolean booleanVal; 56 | | private Byte byteVal; 57 | | private java.util.Date dateVal; 58 | | private java.sql.Timestamp timestampVal; 59 | | 60 | | public ClassWithAllSimpleTypes() {} 61 | | 62 | | public ClassWithAllSimpleTypes(String strVal, Integer intVal, Long longVal, Double doubleVal, Float floatVal, Short shortVal, Boolean booleanVal, Byte byteVal, java.util.Date dateVal, java.sql.Timestamp timestampVal) { 63 | | this.strVal = strVal; 64 | | this.intVal = intVal; 65 | | this.longVal = longVal; 66 | | this.doubleVal = doubleVal; 67 | | this.floatVal = floatVal; 68 | | this.shortVal = shortVal; 69 | | this.booleanVal = booleanVal; 70 | | this.byteVal = byteVal; 71 | | this.dateVal = dateVal; 72 | | this.timestampVal = timestampVal; 73 | | } 74 | | 75 | | public String getStrVal() { 76 | | return strVal; 77 | | } 78 | | 79 | | public void setStrVal(String strVal) { 80 | | this.strVal = strVal; 81 | | } 82 | | 83 | | public Integer getIntVal() { 84 | | return intVal; 85 | | } 86 | | 87 | | public void setIntVal(Integer intVal) { 88 | | this.intVal = intVal; 89 | | } 90 | | 91 | | public Long getLongVal() { 92 | | return longVal; 93 | | } 94 | | 95 | | public void setLongVal(Long longVal) { 96 | | this.longVal = longVal; 97 | | } 98 | | 99 | | public Double getDoubleVal() { 100 | | return doubleVal; 101 | | } 102 | | 103 | | public void setDoubleVal(Double doubleVal) { 104 | | this.doubleVal = doubleVal; 105 | | } 106 | | 107 | | public Float getFloatVal() { 108 | | return floatVal; 109 | | } 110 | | 111 | | public void setFloatVal(Float floatVal) { 112 | | this.floatVal = floatVal; 113 | | } 114 | | 115 | | public Short getShortVal() { 116 | | return shortVal; 117 | | } 118 | | 119 | | public void setShortVal(Short shortVal) { 120 | | this.shortVal = shortVal; 121 | | } 122 | | 123 | | public Boolean getBooleanVal() { 124 | | return booleanVal; 125 | | } 126 | | 127 | | public void setBooleanVal(Boolean booleanVal) { 128 | | this.booleanVal = booleanVal; 129 | | } 130 | | 131 | | public Byte getByteVal() { 132 | | return byteVal; 133 | | } 134 | | 135 | | public void setByteVal(Byte byteVal) { 136 | | this.byteVal = byteVal; 137 | | } 138 | | 139 | | public java.util.Date getDateVal() { 140 | | return dateVal; 141 | | } 142 | | 143 | | public void setDateVal(java.util.Date dateVal) { 144 | | this.dateVal = dateVal; 145 | | } 146 | | 147 | | public java.sql.Timestamp getTimestampVal() { 148 | | return timestampVal; 149 | | } 150 | | 151 | | public void setTimestampVal(java.sql.Timestamp timestampVal) { 152 | | this.timestampVal = timestampVal; 153 | | } 154 | |}""".stripMargin 155 | 156 | assertResultIgnoringNewLines(expected) { 157 | DataModelGenerator.generate[ClassWithAllSimpleTypes](JavaDialect) 158 | } 159 | } 160 | 161 | test("Array type") { 162 | val expected = 163 | """public class CV { 164 | | private java.util.List skills; 165 | | private java.util.List grades; 166 | | 167 | | public CV() {} 168 | | 169 | | public CV(java.util.List skills, java.util.List grades) { 170 | | this.skills = skills; 171 | | this.grades = grades; 172 | | } 173 | | 174 | | public java.util.List getSkills() { 175 | | return skills; 176 | | } 177 | | 178 | | public void setSkills(java.util.List skills) { 179 | | this.skills = skills; 180 | | } 181 | | 182 | | public java.util.List getGrades() { 183 | | return grades; 184 | | } 185 | | 186 | | public void setGrades(java.util.List grades) { 187 | | this.grades = grades; 188 | | } 189 | |}""".stripMargin 190 | 191 | assertResultIgnoringNewLines(expected) { 192 | DataModelGenerator.generate[CV](JavaDialect) 193 | } 194 | } 195 | 196 | test("Nested array type") { 197 | val expected = 198 | """public class NestedArray { 199 | | private java.util.List> nested; 200 | | private java.util.List>> nested3; 201 | | 202 | | public NestedArray() {} 203 | | 204 | | public NestedArray(java.util.List> nested, java.util.List>> nested3) { 205 | | this.nested = nested; 206 | | this.nested3 = nested3; 207 | | } 208 | | 209 | | public java.util.List> getNested() { 210 | | return nested; 211 | | } 212 | | 213 | | public void setNested(java.util.List> nested) { 214 | | this.nested = nested; 215 | | } 216 | | 217 | | public java.util.List>> getNested3() { 218 | | return nested3; 219 | | } 220 | | 221 | | public void setNested3(java.util.List>> nested3) { 222 | | this.nested3 = nested3; 223 | | } 224 | |}""".stripMargin 225 | 226 | assertResultIgnoringNewLines(expected) { 227 | DataModelGenerator.generate[NestedArray](JavaDialect) 228 | } 229 | } 230 | 231 | test("Struct types") { 232 | val expected = 233 | """public class Book { 234 | | private String title; 235 | | private Integer year; 236 | | private com.datawizards.dmg.Person owner; 237 | | private java.util.List authors; 238 | | 239 | | public Book() {} 240 | | 241 | | public Book(String title, Integer year, com.datawizards.dmg.Person owner, java.util.List authors) { 242 | | this.title = title; 243 | | this.year = year; 244 | | this.owner = owner; 245 | | this.authors = authors; 246 | | } 247 | | 248 | | public String getTitle() { 249 | | return title; 250 | | } 251 | | 252 | | public void setTitle(String title) { 253 | | this.title = title; 254 | | } 255 | | 256 | | public Integer getYear() { 257 | | return year; 258 | | } 259 | | 260 | | public void setYear(Integer year) { 261 | | this.year = year; 262 | | } 263 | | 264 | | public com.datawizards.dmg.Person getOwner() { 265 | | return owner; 266 | | } 267 | | 268 | | public void setOwner(com.datawizards.dmg.Person owner) { 269 | | this.owner = owner; 270 | | } 271 | | 272 | | public java.util.List getAuthors() { 273 | | return authors; 274 | | } 275 | | 276 | | public void setAuthors(java.util.List authors) { 277 | | this.authors = authors; 278 | | } 279 | |}""".stripMargin 280 | 281 | assertResultIgnoringNewLines(expected) { 282 | DataModelGenerator.generate[Book](JavaDialect) 283 | } 284 | } 285 | 286 | test("Map type") { 287 | val expected = 288 | """public class ClassWithMap { 289 | | private java.util.Map map; 290 | | 291 | | public ClassWithMap() {} 292 | | 293 | | public ClassWithMap(java.util.Map map) { 294 | | this.map = map; 295 | | } 296 | | 297 | | public java.util.Map getMap() { 298 | | return map; 299 | | } 300 | | 301 | | public void setMap(java.util.Map map) { 302 | | this.map = map; 303 | | } 304 | |}""".stripMargin 305 | 306 | assertResultIgnoringNewLines(expected) { 307 | DataModelGenerator.generate[ClassWithMap](JavaDialect) 308 | } 309 | } 310 | 311 | test("ClassWithArrayByte") { 312 | val expected = 313 | """public class ClassWithArrayByte { 314 | | private java.util.List arr; 315 | | 316 | | public ClassWithArrayByte() {} 317 | | 318 | | public ClassWithArrayByte(java.util.List arr) { 319 | | this.arr = arr; 320 | | } 321 | | 322 | | public java.util.List getArr() { 323 | | return arr; 324 | | } 325 | | 326 | | public void setArr(java.util.List arr) { 327 | | this.arr = arr; 328 | | } 329 | |}""".stripMargin 330 | 331 | assertResultIgnoringNewLines(expected) { 332 | DataModelGenerator.generate[ClassWithArrayByte](JavaDialect) 333 | } 334 | } 335 | 336 | test("ClassWithBigInteger") { 337 | val expected = 338 | """public class ClassWithBigInteger { 339 | | private java.math.BigInteger n1; 340 | | 341 | | public ClassWithBigInteger() {} 342 | | 343 | | public ClassWithBigInteger(java.math.BigInteger n1) { 344 | | this.n1 = n1; 345 | | } 346 | | 347 | | public java.math.BigInteger getN1() { 348 | | return n1; 349 | | } 350 | | 351 | | public void setN1(java.math.BigInteger n1) { 352 | | this.n1 = n1; 353 | | } 354 | |}""".stripMargin 355 | 356 | assertResultIgnoringNewLines(expected) { 357 | DataModelGenerator.generate[ClassWithBigInteger](JavaDialect) 358 | } 359 | } 360 | 361 | test("ClassWithBigDecimal") { 362 | val expected = 363 | """public class ClassWithBigDecimal { 364 | | private java.math.BigDecimal n1; 365 | | 366 | | public ClassWithBigDecimal() {} 367 | | 368 | | public ClassWithBigDecimal(java.math.BigDecimal n1) { 369 | | this.n1 = n1; 370 | | } 371 | | 372 | | public java.math.BigDecimal getN1() { 373 | | return n1; 374 | | } 375 | | 376 | | public void setN1(java.math.BigDecimal n1) { 377 | | this.n1 = n1; 378 | | } 379 | |}""".stripMargin 380 | 381 | assertResultIgnoringNewLines(expected) { 382 | DataModelGenerator.generate[ClassWithBigDecimal](JavaDialect) 383 | } 384 | } 385 | 386 | } -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/dialects/GenerateMySQLModelTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | import com.datawizards.dmg.TestModel._ 4 | import com.datawizards.dmg.{DataModelGenerator, DataModelGeneratorBaseTest} 5 | import org.junit.runner.RunWith 6 | import org.scalatest.junit.JUnitRunner 7 | 8 | @RunWith(classOf[JUnitRunner]) 9 | class GenerateMySQLModelTest extends DataModelGeneratorBaseTest { 10 | 11 | test("Simple model") { 12 | val expected = 13 | """CREATE TABLE Person( 14 | | name VARCHAR, 15 | | age INT 16 | |);""".stripMargin 17 | 18 | assertResultIgnoringNewLines(expected) { 19 | DataModelGenerator.generate[Person](MySQLDialect) 20 | } 21 | } 22 | 23 | test("ClassWithAllSimpleTypes") { 24 | val expected = 25 | """CREATE TABLE ClassWithAllSimpleTypes( 26 | | strVal VARCHAR, 27 | | intVal INT, 28 | | longVal BIGINT, 29 | | doubleVal DOUBLE, 30 | | floatVal FLOAT, 31 | | shortVal SMALLINT, 32 | | booleanVal BOOLEAN, 33 | | byteVal SMALLINT, 34 | | dateVal DATE, 35 | | timestampVal TIMESTAMP 36 | |);""".stripMargin 37 | 38 | assertResultIgnoringNewLines(expected) { 39 | DataModelGenerator.generate[ClassWithAllSimpleTypes](MySQLDialect) 40 | } 41 | } 42 | 43 | test("Table and column comment") { 44 | val expected = 45 | """CREATE TABLE PersonWithComments( 46 | | name VARCHAR COMMENT 'Person name', 47 | | age INT 48 | |) 49 | |COMMENT = 'People data';""".stripMargin 50 | 51 | assertResultIgnoringNewLines(expected) { 52 | DataModelGenerator.generate[PersonWithComments](MySQLDialect) 53 | } 54 | } 55 | 56 | test("Column length") { 57 | val expected = 58 | """CREATE TABLE PersonWithCustomLength( 59 | | name VARCHAR(1000), 60 | | age INT 61 | |);""".stripMargin 62 | 63 | assertResultIgnoringNewLines(expected) { 64 | DataModelGenerator.generate[PersonWithCustomLength](MySQLDialect) 65 | } 66 | } 67 | 68 | test("Array type") { 69 | val expected = 70 | """CREATE TABLE CV( 71 | | skills JSON, 72 | | grades JSON 73 | |);""".stripMargin 74 | 75 | assertResultIgnoringNewLines(expected) { 76 | DataModelGenerator.generate[CV](MySQLDialect) 77 | } 78 | } 79 | 80 | test("Nested array type") { 81 | val expected = 82 | """CREATE TABLE NestedArray( 83 | | nested JSON, 84 | | nested3 JSON 85 | |);""".stripMargin 86 | 87 | assertResultIgnoringNewLines(expected) { 88 | DataModelGenerator.generate[NestedArray](MySQLDialect) 89 | } 90 | } 91 | 92 | test("Struct types") { 93 | val expected = 94 | """CREATE TABLE Book( 95 | | title VARCHAR, 96 | | year INT, 97 | | owner JSON, 98 | | authors JSON 99 | |);""".stripMargin 100 | 101 | assertResultIgnoringNewLines(expected) { 102 | DataModelGenerator.generate[Book](MySQLDialect) 103 | } 104 | } 105 | 106 | test("Map type") { 107 | val expected = 108 | """CREATE TABLE ClassWithMap( 109 | | map JSON 110 | |);""".stripMargin 111 | 112 | assertResultIgnoringNewLines(expected) { 113 | DataModelGenerator.generate[ClassWithMap](MySQLDialect) 114 | } 115 | } 116 | 117 | test("ClassWithDash") { 118 | val expected = 119 | """CREATE TABLE ClassWithDash( 120 | | "add-id" VARCHAR 121 | |);""".stripMargin 122 | 123 | assertResultIgnoringNewLines(expected) { 124 | DataModelGenerator.generate[ClassWithDash](MySQLDialect) 125 | } 126 | } 127 | 128 | test("reserverd keywords") { 129 | val expected = 130 | """CREATE TABLE ClassWithReservedKeywords( 131 | | "select" VARCHAR, 132 | | "where" VARCHAR 133 | |);""".stripMargin 134 | 135 | assertResultIgnoringNewLines(expected) { 136 | DataModelGenerator.generate[ClassWithReservedKeywords](MySQLDialect) 137 | } 138 | } 139 | 140 | test("ClassWithArrayByte") { 141 | val expected = 142 | """CREATE TABLE ClassWithArrayByte( 143 | | arr BINARY 144 | |);""".stripMargin 145 | 146 | assertResultIgnoringNewLines(expected) { 147 | DataModelGenerator.generate[ClassWithArrayByte](MySQLDialect) 148 | } 149 | } 150 | 151 | test("ClassWithBigInteger") { 152 | val expected = 153 | """CREATE TABLE ClassWithBigInteger( 154 | | n1 BIGINT 155 | |);""".stripMargin 156 | 157 | assertResultIgnoringNewLines(expected) { 158 | DataModelGenerator.generate[ClassWithBigInteger](MySQLDialect) 159 | } 160 | } 161 | 162 | test("ClassWithBigDecimal") { 163 | val expected = 164 | """CREATE TABLE ClassWithBigDecimal( 165 | | n1 DECIMAL(38,18) 166 | |);""".stripMargin 167 | 168 | assertResultIgnoringNewLines(expected) { 169 | DataModelGenerator.generate[ClassWithBigDecimal](MySQLDialect) 170 | } 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/dialects/GenerateRedshiftModelTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.dialects 2 | 3 | import com.datawizards.dmg.TestModel._ 4 | import com.datawizards.dmg.{DataModelGenerator, DataModelGeneratorBaseTest} 5 | import org.junit.runner.RunWith 6 | import org.scalatest.junit.JUnitRunner 7 | 8 | @RunWith(classOf[JUnitRunner]) 9 | class GenerateRedshiftModelTest extends DataModelGeneratorBaseTest { 10 | 11 | test("Simple model") { 12 | val expected = 13 | """CREATE TABLE Person( 14 | | name VARCHAR, 15 | | age INTEGER 16 | |);""".stripMargin 17 | 18 | assertResultIgnoringNewLines(expected) { 19 | DataModelGenerator.generate[Person](RedshiftDialect) 20 | } 21 | } 22 | 23 | test("ClassWithAllSimpleTypes") { 24 | val expected = 25 | """CREATE TABLE ClassWithAllSimpleTypes( 26 | | strVal VARCHAR, 27 | | intVal INTEGER, 28 | | longVal BIGINT, 29 | | doubleVal DOUBLE PRECISION, 30 | | floatVal REAL, 31 | | shortVal SMALLINT, 32 | | booleanVal BOOLEAN, 33 | | byteVal SMALLINT, 34 | | dateVal DATE, 35 | | timestampVal TIMESTAMP 36 | |);""".stripMargin 37 | 38 | assertResultIgnoringNewLines(expected) { 39 | DataModelGenerator.generate[ClassWithAllSimpleTypes](RedshiftDialect) 40 | } 41 | } 42 | 43 | test("Table and column comment") { 44 | val expected = 45 | """CREATE TABLE PersonWithComments( 46 | | name VARCHAR, 47 | | age INTEGER 48 | |); 49 | |COMMENT ON TABLE PersonWithComments IS 'People data'; 50 | |COMMENT ON COLUMN PersonWithComments.name IS 'Person name';""".stripMargin 51 | 52 | assertResultIgnoringNewLines(expected) { 53 | DataModelGenerator.generate[PersonWithComments](RedshiftDialect) 54 | } 55 | } 56 | 57 | test("Column length") { 58 | val expected = 59 | """CREATE TABLE PersonWithCustomLength( 60 | | name VARCHAR(1000), 61 | | age INTEGER 62 | |);""".stripMargin 63 | 64 | assertResultIgnoringNewLines(expected) { 65 | DataModelGenerator.generate[PersonWithCustomLength](RedshiftDialect) 66 | } 67 | } 68 | 69 | test("Array type") { 70 | val expected = 71 | """CREATE TABLE CV( 72 | | skills VARCHAR, 73 | | grades VARCHAR 74 | |);""".stripMargin 75 | 76 | assertResultIgnoringNewLines(expected) { 77 | DataModelGenerator.generate[CV](RedshiftDialect) 78 | } 79 | } 80 | 81 | test("Nested array type") { 82 | val expected = 83 | """CREATE TABLE NestedArray( 84 | | nested VARCHAR, 85 | | nested3 VARCHAR 86 | |);""".stripMargin 87 | 88 | assertResultIgnoringNewLines(expected) { 89 | DataModelGenerator.generate[NestedArray](RedshiftDialect) 90 | } 91 | } 92 | 93 | test("Struct types") { 94 | val expected = 95 | """CREATE TABLE Book( 96 | | title VARCHAR, 97 | | year INTEGER, 98 | | owner VARCHAR, 99 | | authors VARCHAR 100 | |);""".stripMargin 101 | 102 | assertResultIgnoringNewLines(expected) { 103 | DataModelGenerator.generate[Book](RedshiftDialect) 104 | } 105 | } 106 | 107 | test("Map type") { 108 | val expected = 109 | """CREATE TABLE ClassWithMap( 110 | | map VARCHAR 111 | |);""".stripMargin 112 | 113 | assertResultIgnoringNewLines(expected) { 114 | DataModelGenerator.generate[ClassWithMap](RedshiftDialect) 115 | } 116 | } 117 | 118 | test("ClassWithDash") { 119 | val expected = 120 | """CREATE TABLE ClassWithDash( 121 | | "add-id" VARCHAR 122 | |);""".stripMargin 123 | 124 | assertResultIgnoringNewLines(expected) { 125 | DataModelGenerator.generate[ClassWithDash](RedshiftDialect) 126 | } 127 | } 128 | 129 | test("reserverd keywords") { 130 | val expected = 131 | """CREATE TABLE ClassWithReservedKeywords( 132 | | "select" VARCHAR, 133 | | "where" VARCHAR 134 | |);""".stripMargin 135 | 136 | assertResultIgnoringNewLines(expected) { 137 | DataModelGenerator.generate[ClassWithReservedKeywords](RedshiftDialect) 138 | } 139 | } 140 | 141 | test("ClassWithArrayByte") { 142 | val expected = 143 | """CREATE TABLE ClassWithArrayByte( 144 | | arr VARCHAR 145 | |);""".stripMargin 146 | 147 | assertResultIgnoringNewLines(expected) { 148 | DataModelGenerator.generate[ClassWithArrayByte](RedshiftDialect) 149 | } 150 | } 151 | 152 | test("ClassWithBigInteger") { 153 | val expected = 154 | """CREATE TABLE ClassWithBigInteger( 155 | | n1 BIGINT 156 | |);""".stripMargin 157 | 158 | assertResultIgnoringNewLines(expected) { 159 | DataModelGenerator.generate[ClassWithBigInteger](RedshiftDialect) 160 | } 161 | } 162 | 163 | test("ClassWithBigDecimal") { 164 | val expected = 165 | """CREATE TABLE ClassWithBigDecimal( 166 | | n1 DECIMAL(38,18) 167 | |);""".stripMargin 168 | 169 | assertResultIgnoringNewLines(expected) { 170 | DataModelGenerator.generate[ClassWithBigDecimal](RedshiftDialect) 171 | } 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/metadata/CV.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.metadata 2 | 3 | import com.datawizards.dmg.annotations.column 4 | 5 | case class CV( 6 | person: Person, 7 | @column(name="workExperience") 8 | experience: Iterable[WorkExperience], 9 | `Wierd chars - / +`: String 10 | ) 11 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/metadata/Company.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.metadata 2 | 3 | import com.datawizards.dmg.annotations.column 4 | 5 | case class Company( 6 | @column(name="companyName") 7 | name: String, 8 | address: String, 9 | industry: String 10 | ) 11 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/metadata/MetaDataExtractorTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.metadata 2 | 3 | import org.junit.runner.RunWith 4 | import org.scalatest.FunSuite 5 | import org.scalatest.junit.JUnitRunner 6 | 7 | @RunWith(classOf[JUnitRunner]) 8 | class MetaDataExtractorTest extends FunSuite { 9 | 10 | def personTypeMetaData = ClassTypeMetaData( 11 | packageName = "com.datawizards.dmg.metadata" 12 | , originalTypeName = "Person" 13 | , typeName = "Person" 14 | , annotations = Seq( 15 | AnnotationMetaData( 16 | name = "com.datawizards.dmg.annotations.table" 17 | , attributes = Seq( 18 | AnnotationAttributeMetaData( 19 | name = "name" 20 | , value = "PEOPLE" 21 | ) 22 | ) 23 | ) 24 | ) 25 | , fields = Seq( 26 | ClassFieldMetaData( 27 | originalFieldName = "name" 28 | , fieldName = "name" 29 | , fieldType = StringType 30 | , annotations = Seq( 31 | AnnotationMetaData( 32 | name = "com.datawizards.dmg.annotations.column" 33 | , attributes = Seq( 34 | AnnotationAttributeMetaData( 35 | name = "name" 36 | , value = "personName_es" 37 | ), 38 | AnnotationAttributeMetaData( 39 | name = "dialect" 40 | , value = "com.datawizards.dmg.dialects.ElasticsearchDialect" 41 | ) 42 | ) 43 | ), 44 | AnnotationMetaData( 45 | name = "com.datawizards.dmg.annotations.column" 46 | , attributes = Seq( 47 | AnnotationAttributeMetaData( 48 | name = "name" 49 | , value = "person_name" 50 | ) 51 | ) 52 | ), 53 | AnnotationMetaData( 54 | name = "com.datawizards.dmg.metadata.anotherAnnotation" 55 | , attributes = Seq( 56 | AnnotationAttributeMetaData( 57 | name = "value" 58 | , value = "name2" 59 | ) 60 | ) 61 | ) 62 | ) 63 | ), 64 | ClassFieldMetaData( 65 | originalFieldName = "age" 66 | , fieldName = "age" 67 | , fieldType = IntegerType 68 | , annotations = Seq( 69 | AnnotationMetaData( 70 | name = "com.datawizards.dmg.annotations.column" 71 | , attributes = Seq( 72 | AnnotationAttributeMetaData( 73 | name = "name" 74 | , value = "age_es" 75 | ), 76 | AnnotationAttributeMetaData( 77 | name = "dialect" 78 | , value = "com.datawizards.dmg.dialects.ElasticsearchDialect" 79 | ) 80 | ) 81 | ) 82 | ) 83 | ), 84 | ClassFieldMetaData( 85 | originalFieldName = "title" 86 | , fieldName = "title" 87 | , fieldType = StringType 88 | , annotations = Seq() 89 | ) 90 | ) 91 | ) 92 | 93 | test("Extract Person MetaData") { 94 | val expected = personTypeMetaData 95 | val result = MetaDataExtractor.extractTypeMetaData[Person]() 96 | 97 | assertResult(expected)(result) 98 | } 99 | 100 | test("Extract CV MetaData") { 101 | val expected = ClassTypeMetaData( 102 | packageName = "com.datawizards.dmg.metadata" 103 | , originalTypeName = "CV" 104 | , typeName = "CV" 105 | , annotations = Seq.empty 106 | , fields = Seq( 107 | ClassFieldMetaData( 108 | originalFieldName = "person" 109 | , fieldName = "person" 110 | , fieldType = personTypeMetaData 111 | , annotations = Seq.empty 112 | ), 113 | ClassFieldMetaData( 114 | originalFieldName = "experience" 115 | , fieldName = "experience" 116 | , fieldType = CollectionTypeMetaData( 117 | elementType = ClassTypeMetaData( 118 | packageName = "com.datawizards.dmg.metadata" 119 | , originalTypeName = "WorkExperience" 120 | , typeName = "WorkExperience" 121 | , annotations = Seq.empty 122 | , fields = Seq( 123 | ClassFieldMetaData( 124 | originalFieldName = "start" 125 | , fieldName = "start" 126 | , fieldType = DateType 127 | , annotations = Seq.empty 128 | ), 129 | ClassFieldMetaData( 130 | originalFieldName = "end" 131 | , fieldName = "end" 132 | , fieldType = DateType 133 | , annotations = Seq.empty 134 | ), 135 | ClassFieldMetaData( 136 | originalFieldName = "jobTitle" 137 | , fieldName = "jobTitle" 138 | , fieldType = StringType 139 | , annotations = Seq.empty 140 | ), 141 | ClassFieldMetaData( 142 | originalFieldName = "company" 143 | , fieldName = "company" 144 | , fieldType = ClassTypeMetaData( 145 | packageName = "com.datawizards.dmg.metadata" 146 | , originalTypeName = "Company" 147 | , typeName = "Company" 148 | , annotations = Seq.empty 149 | , fields = Seq( 150 | ClassFieldMetaData( 151 | originalFieldName = "name" 152 | , fieldName = "name" 153 | , fieldType = StringType 154 | , annotations = Seq( 155 | AnnotationMetaData( 156 | name = "com.datawizards.dmg.annotations.column" 157 | , attributes = Seq( 158 | AnnotationAttributeMetaData( 159 | name = "name" 160 | , value = "companyName" 161 | ) 162 | ) 163 | ) 164 | ) 165 | ), 166 | ClassFieldMetaData( 167 | originalFieldName = "address" 168 | , fieldName = "address" 169 | , fieldType = StringType 170 | , annotations = Seq.empty 171 | ), 172 | ClassFieldMetaData( 173 | originalFieldName = "industry" 174 | , fieldName = "industry" 175 | , fieldType = StringType 176 | , annotations = Seq.empty 177 | ) 178 | ) 179 | ) 180 | , annotations = Seq.empty 181 | ) 182 | ) 183 | ) 184 | ) 185 | , annotations = Seq( 186 | AnnotationMetaData( 187 | name = "com.datawizards.dmg.annotations.column" 188 | , attributes = Seq( 189 | AnnotationAttributeMetaData( 190 | name = "name" 191 | , value = "workExperience" 192 | ) 193 | ) 194 | ) 195 | ) 196 | ), 197 | ClassFieldMetaData( 198 | originalFieldName = "Wierd chars - / +" 199 | , fieldName = "Wierd chars - / +" 200 | , fieldType = StringType 201 | , annotations = Seq.empty 202 | ) 203 | ) 204 | ) 205 | 206 | val result = MetaDataExtractor.extractTypeMetaData[CV]() 207 | 208 | assertResult(expected)(result) 209 | } 210 | } 211 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/metadata/Person.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.metadata 2 | 3 | import com.datawizards.dmg.annotations._ 4 | import com.datawizards.dmg.dialects._ 5 | 6 | @table("PEOPLE") 7 | case class Person( 8 | @column(name="personName_es", dialect = ElasticsearchDialect) 9 | @column(name="person_name") 10 | @anotherAnnotation("name2") 11 | name: String, 12 | @column(name="age_es", dialect = ElasticsearchDialect) 13 | age: Int, 14 | title: String 15 | ) 16 | 17 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/metadata/PersonFull.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.metadata 2 | 3 | import com.datawizards.dmg.annotations.{column, table} 4 | import com.datawizards.dmg.dialects._ 5 | 6 | @table("person_full") 7 | @table("person_full_es", ElasticsearchDialect) 8 | case class PersonFull( 9 | @column("person_name") 10 | @column("personName", ElasticsearchDialect) 11 | name: Option[String] = None 12 | , 13 | @column("person_age") 14 | @column("personAge", ElasticsearchDialect) 15 | age: Option[Int] = None 16 | , 17 | @column("personTitle", ElasticsearchDialect) 18 | title: Option[String] = None 19 | ) 20 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/metadata/PersonPartitioned.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.metadata 2 | 3 | import com.datawizards.dmg.annotations._ 4 | import com.datawizards.dmg.annotations.hive.{hiveExternalTable, hivePartitionColumn} 5 | import com.datawizards.dmg.dialects._ 6 | 7 | @table("person_partitioned_hehehe") 8 | @hiveExternalTable("s3://some/path") 9 | case class PersonPartitioned( 10 | @column(name="personName_es", dialect = ElasticsearchDialect) 11 | @column(name="person_name") 12 | @anotherAnnotation("name2") 13 | name: String, 14 | @column(name="age_es", dialect = ElasticsearchDialect) 15 | age: Int, 16 | title: String, 17 | @hivePartitionColumn 18 | birthYear: Int, 19 | @hivePartitionColumn 20 | birthMonth: Int, 21 | @hivePartitionColumn 22 | birthDay: Int 23 | ) 24 | 25 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/metadata/WorkExperience.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.metadata 2 | 3 | import java.util.Date 4 | 5 | case class WorkExperience( 6 | start: Date, 7 | end: Date, 8 | jobTitle: String, 9 | company: Company 10 | ) 11 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/metadata/anotherAnnotation.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.metadata 2 | 3 | import scala.annotation.StaticAnnotation 4 | 5 | final class anotherAnnotation(val value: String) extends StaticAnnotation 6 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/service/AvroSchemaGenerateTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.service 2 | 3 | import com.datawizards.dmg.DataModelGeneratorBaseTest 4 | import com.datawizards.dmg.TestModel.PersonWithPlaceholderVariables 5 | import com.datawizards.dmg.repository.AvroSchemaRegistryRepository 6 | import org.junit.runner.RunWith 7 | import org.scalatest.{FunSuite, Matchers} 8 | import org.scalatest.junit.JUnitRunner 9 | 10 | import scala.collection.mutable 11 | 12 | @RunWith(classOf[JUnitRunner]) 13 | class AvroSchemaGenerateTest extends DataModelGeneratorBaseTest { 14 | val service: AvroSchemaRegistryService = new AvroSchemaRegistryService { 15 | override protected val repository: AvroSchemaRegistryRepository = new AvroSchemaRegistryRepository { 16 | 17 | private val schemas = mutable.Map[String, String]() 18 | 19 | override def fetchSchema(subject: String, version: String): String = 20 | schemas(subject) 21 | 22 | override def subjects(): Iterable[String] = 23 | schemas.keys 24 | 25 | override def registerSchema(subject: String, schema: String): Unit = { 26 | schemas += subject -> schema 27 | } 28 | } 29 | override protected val hdfsService:HDFSService = HDFSServiceImpl 30 | } 31 | 32 | test("Register and fetch avro schema") { 33 | val expected = 34 | """{ 35 | | "namespace": "com.datawizards.dmg", 36 | | "type": "record", 37 | | "name": "tttt", 38 | | "fields": [ 39 | | {"name": "name", "type": ["null", "string"], "doc": "Person some comment"}, 40 | | {"name": "age", "type": ["null", "int"]} 41 | | ] 42 | |}""".stripMargin 43 | 44 | assertResultIgnoringNewLines(expected){ 45 | service.generateSchema[PersonWithPlaceholderVariables](Map("table_name" -> "tttt", "name_comment" -> "some comment")) 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/service/CreateElasticsearchIndexTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.service 2 | 3 | import com.datawizards.dmg.DataModelGenerator 4 | import com.datawizards.dmg.TestModel._ 5 | import com.datawizards.dmg.dialects._ 6 | import com.datawizards.esclient.dto.SearchResult 7 | import com.datawizards.esclient.repository.ElasticsearchRepository 8 | import org.junit.runner.RunWith 9 | import org.scalatest.FunSuite 10 | import org.scalatest.junit.JUnitRunner 11 | 12 | import scala.collection.mutable 13 | import scala.reflect.ClassTag 14 | import scala.reflect.runtime.universe 15 | 16 | @RunWith(classOf[JUnitRunner]) 17 | class CreateElasticsearchIndexTest extends FunSuite { 18 | 19 | val service: ElasticsearchService = new ElasticsearchService { 20 | override protected val repository: ElasticsearchRepository = new ElasticsearchRepository { 21 | 22 | private val templates = mutable.Map[String, String]() 23 | private val indexes = mutable.Map[String, String]() 24 | 25 | override def updateTemplate(templateName: String, mapping: String): Unit = 26 | templates(templateName) = mapping 27 | 28 | override def getTemplate(templateName: String): String = 29 | templates(templateName) 30 | 31 | override def createIndex(indexName: String, mapping: String): Unit = 32 | indexes(indexName) = mapping 33 | 34 | override def getIndexSettings(indexName: String): String = 35 | indexes(indexName) 36 | 37 | override def deleteTemplate(templateName: String): Unit = 38 | templates.remove(templateName) 39 | 40 | override def templateExists(templateName: String): Boolean = 41 | templates.contains(templateName) 42 | 43 | override def deleteIndex(indexName: String): Unit = 44 | indexes.remove(indexName) 45 | 46 | override def indexExists(indexName: String): Boolean = 47 | indexes.contains(indexName) 48 | 49 | override def status(): Boolean = true 50 | 51 | override def index[T <: AnyRef](indexName: String, typeName: String, documentId: String, document: T): Unit = 52 | { /* do nothing */ } 53 | 54 | override def index(indexName: String, typeName: String, documentId: String, document: String): Unit = 55 | { /* do nothing */ } 56 | 57 | override def read[T](indexName: String, typeName: String, documentId: String)(implicit ct: ClassTag[T], tt: universe.TypeTag[T]): T = 58 | "".asInstanceOf[T] 59 | 60 | override def append[T <: AnyRef](indexName: String, typeName: String, document: T): Unit = 61 | { /* do nothing */ } 62 | 63 | override def append(indexName: String, typeName: String, document: String): Unit = 64 | { /* do nothing */ } 65 | 66 | override def search[T](indexName: String)(implicit evidence$3: ClassTag[T], evidence$4: universe.TypeTag[T]): SearchResult[T] = 67 | SearchResult(0, Traversable.empty) 68 | } 69 | } 70 | 71 | test("Create and get Elasticsearch template") { 72 | val name = "person" 73 | service.updateTemplate[Person](name) 74 | 75 | assertResult(DataModelGenerator.generate[Person](ElasticsearchDialect)) { 76 | service.getTemplate(name) 77 | } 78 | } 79 | 80 | test("Create and get Elasticsearch index") { 81 | val name = "person" 82 | service.createIndex[Person](name) 83 | 84 | assertResult(DataModelGenerator.generate[Person](ElasticsearchDialect)) { 85 | service.getIndexSettings(name) 86 | } 87 | } 88 | 89 | test("Try to create index if exists") { 90 | val name = "person" 91 | service.createIndex[Person](name) 92 | service.createIndexIfNotExists[Book](name) 93 | 94 | assertResult(DataModelGenerator.generate[Person](ElasticsearchDialect)) { 95 | service.getIndexSettings(name) 96 | } 97 | } 98 | 99 | test("Create index twice") { 100 | val name = "person" 101 | service.createIndex[Book](name) 102 | service.createIndex[Person](name) 103 | 104 | assertResult(DataModelGenerator.generate[Person](ElasticsearchDialect)) { 105 | service.getIndexSettings(name) 106 | } 107 | } 108 | 109 | test("Try to create template if exists") { 110 | val name = "person" 111 | service.updateTemplate[Person](name) 112 | service.updateTemplateIfNotExists[Book](name) 113 | 114 | assertResult(DataModelGenerator.generate[Person](ElasticsearchDialect)) { 115 | service.getTemplate(name) 116 | } 117 | } 118 | 119 | test("Create template twice") { 120 | val name = "person" 121 | service.updateTemplate[Book](name) 122 | service.updateTemplate[Person](name) 123 | 124 | assertResult(DataModelGenerator.generate[Person](ElasticsearchDialect)) { 125 | service.getTemplate(name) 126 | } 127 | } 128 | 129 | } 130 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/service/HiveTableBatchCreateTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.service 2 | 3 | import com.datawizards.dmg.DataModelGeneratorBaseTest 4 | import com.datawizards.dmg.TestModel.PersonWithPlaceholderVariables 5 | import com.datawizards.dmg.examples.TestModel.{Person, PersonWithCustomName} 6 | import com.datawizards.dmg.metadata.PersonPartitioned 7 | import com.datawizards.dmg.repository.AvroSchemaRegistryRepository 8 | import org.junit.runner.RunWith 9 | import org.scalatest.junit.JUnitRunner 10 | 11 | import scala.collection.mutable 12 | 13 | @RunWith(classOf[JUnitRunner]) 14 | class HiveTableBatchCreateTest extends DataModelGeneratorBaseTest { 15 | test("Batch create hive table") { 16 | val expected = 17 | """DROP TABLE IF EXISTS Person; 18 | |CREATE TABLE Person( 19 | | name STRING, 20 | | age INT 21 | |); 22 | | 23 | | 24 | |DROP TABLE IF EXISTS person_partitioned_hehehe; 25 | |CREATE EXTERNAL TABLE person_partitioned_hehehe( 26 | | person_name STRING, 27 | | age INT, 28 | | title STRING 29 | |) 30 | |PARTITIONED BY(birthYear INT, birthMonth INT, birthDay INT) 31 | |LOCATION 's3://some/path'; 32 | |MSCK REPAIR TABLE person_partitioned_hehehe; 33 | | 34 | | 35 | |DROP TABLE IF EXISTS PersonWithCustomName; 36 | |CREATE TABLE PersonWithCustomName( 37 | | personName STRING, 38 | | personAge INT, 39 | | gender STRING 40 | |); 41 | | 42 | | 43 | |""".stripMargin 44 | 45 | val script = HiveServiceImpl.batchCreateTable() 46 | .createTable[Person] 47 | .createTable[PersonPartitioned] 48 | .createTable[PersonWithCustomName] 49 | .getScript 50 | 51 | assertResultIgnoringNewLines(expected){script} 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/test/scala/com/datawizards/dmg/service/RegisterAvroSchemaTest.scala: -------------------------------------------------------------------------------- 1 | package com.datawizards.dmg.service 2 | 3 | import com.datawizards.dmg.DataModelGenerator 4 | import com.datawizards.dmg.TestModel.Person 5 | import com.datawizards.dmg.dialects.AvroSchemaRegistryDialect 6 | import com.datawizards.dmg.repository.AvroSchemaRegistryRepository 7 | import org.junit.runner.RunWith 8 | import org.scalatest.FunSuite 9 | import org.scalatest.junit.JUnitRunner 10 | 11 | import scala.collection.mutable 12 | 13 | @RunWith(classOf[JUnitRunner]) 14 | class RegisterAvroSchemaTest extends FunSuite { 15 | 16 | val service: AvroSchemaRegistryService = new AvroSchemaRegistryService { 17 | override protected val repository: AvroSchemaRegistryRepository = new AvroSchemaRegistryRepository { 18 | 19 | private val schemas = mutable.Map[String, String]() 20 | 21 | override def fetchSchema(subject: String, version: String): String = 22 | schemas(subject) 23 | 24 | override def subjects(): Iterable[String] = 25 | schemas.keys 26 | 27 | override def registerSchema(subject: String, schema: String): Unit = { 28 | schemas += subject -> schema 29 | } 30 | } 31 | override protected val hdfsService:HDFSService = HDFSServiceImpl 32 | } 33 | 34 | test("Register and fetch avro schema") { 35 | val subject = "person" 36 | service.registerSchema[Person](subject) 37 | service.subjects().exists(s => s == subject) 38 | assertResult(DataModelGenerator.generate[Person](AvroSchemaRegistryDialect)) { 39 | service.fetchSchema(subject) 40 | } 41 | } 42 | 43 | } 44 | --------------------------------------------------------------------------------